store history using sqlite, add suggestion with open

src/browserwindow.cpp

@@ -2,6 +2,7 @@
 
 #include "adblock.h"
 #include "commandline.h"
+#include "completionview.h"
 #include "hints.h"
 #include "statusbar.h"
 #include "urlutil.h"
@@ -11,8 +12,10 @@
 #include <QEvent>
 #include <QKeyEvent>
 #include <QMouseEvent>
+#include <QRegularExpression>
 #include <QVBoxLayout>
 #include <QWebEnginePage>
+#include <QWebEngineProfile>
 #include <QWebEngineScript>
 #include <QWebEngineView>
 #include <QWidget>
@@ -21,6 +24,14 @@ namespace {
 
 constexpr int kScrollStep = 60;
 constexpr double kZoomStep = 0.1;
+constexpr int kCompletionLimit = 100;
+
+// ":open <query>" or ":o <query>"; captures the prefix and the query.
+const QRegularExpression &openCommand()
+{
+    static const QRegularExpression re(QStringLiteral("^(:(?:open|o) +)(.*)$"));
+    return re;
+}
 
 } // namespace
 
@@ -31,6 +42,7 @@ BrowserWindow::BrowserWindow(QWebEngineProfile *profile, AdBlocker *adBlocker,
     , m_view(new QWebEngineView(profile, this))
     , m_status(new StatusBar(this))
     , m_commandLine(new CommandLine(this))
+    , m_completion(new CompletionView(this))
 {
     setupUi();
     connectView();
@@ -42,6 +54,8 @@ BrowserWindow::BrowserWindow(QWebEngineProfile *profile, AdBlocker *adBlocker,
     connect(m_commandLine, &CommandLine::cancelled, this, [this] {
         setMode(Mode::Normal);
     });
+    connect(m_commandLine, &QLineEdit::textEdited, this, &BrowserWindow::updateCompletion);
+    connect(m_commandLine, &CommandLine::completionCycled, this, &BrowserWindow::cycleCompletion);
     connect(m_adBlocker, &AdBlocker::message, m_status, &StatusBar::showMessage);
 
     qApp->installEventFilter(this);
@@ -61,6 +75,7 @@ void BrowserWindow::setupUi()
     layout->setContentsMargins(0, 0, 0, 0);
     layout->setSpacing(0);
     layout->addWidget(m_view, 1);
+    layout->addWidget(m_completion);
     layout->addWidget(m_status);
     layout->addWidget(m_commandLine);
     setCentralWidget(central);
@@ -80,11 +95,17 @@ void BrowserWindow::connectView()
     connect(m_view, &QWebEngineView::titleChanged, this, [this](const QString &title) {
         setWindowTitle(title.isEmpty() ? QStringLiteral("att")
                                        : QStringLiteral("%1 - att").arg(title));
+        m_history.updateTitle(m_view->url(), title);
     });
     connect(m_view, &QWebEngineView::loadFinished, this, [this](bool ok) {
         m_status->setProgress(100);
-        if (!ok)
+        if (!ok) {
             m_status->showMessage(QStringLiteral("Failed to load page"), true);
+            return;
+        }
+        // Here rather than on urlChanged, which also fires for every redirect
+        // hop and in-page #anchor jump.
+        m_history.addVisit(m_view->url(), m_view->title());
     });
 }
 
@@ -96,6 +117,9 @@ void BrowserWindow::setMode(Mode mode)
     m_mode = mode;
     m_pendingKeys.clear();
 
+    if (mode != Mode::Command)
+        m_completion->setEntries({});
+
     switch (mode) {
     case Mode::Normal:
         m_status->setMode(QString());
@@ -109,6 +133,7 @@ void BrowserWindow::setMode(Mode mode)
         break;
     case Mode::Command:
         m_status->hide();
+        updateCompletion();
         break;
     case Mode::Hint:
         m_status->setMode(QStringLiteral("hint"));
@@ -363,6 +388,14 @@ void BrowserWindow::runCommand(const QString &command)
         m_view->forward();
     } else if (name == QLatin1String("stop")) {
         m_view->stop();
+    } else if (name == QLatin1String("history-clear")) {
+        if (m_history.clear()) {
+            // Also forget Chromium's own record, which colours visited links.
+            m_view->page()->profile()->clearAllVisitedLinks();
+            m_status->showMessage(QStringLiteral("History cleared"));
+        } else {
+            m_status->showMessage(QStringLiteral("Could not clear history"), true);
+        }
     } else if (name == QLatin1String("adblock-update")) {
         m_adBlocker->update();
     } else if (name == QLatin1String("adblock-toggle")) {
@@ -374,6 +407,30 @@ void BrowserWindow::runCommand(const QString &command)
     }
 }
 
+void BrowserWindow::updateCompletion()
+{
+    const QRegularExpressionMatch match = openCommand().match(m_commandLine->text());
+    if (!match.hasMatch()) {
+        m_completion->setEntries({});
+        return;
+    }
+    const QStringList words = match.captured(2).split(QLatin1Char(' '), Qt::SkipEmptyParts);
+    m_completion->setEntries(m_history.search(words, kCompletionLimit));
+}
+
+void BrowserWindow::cycleCompletion(int step)
+{
+    if (!m_completion->isVisible())
+        return;
+    const QString url = m_completion->cycle(step);
+    if (url.isEmpty())
+        return;
+    // Keep whichever of ":o " / ":open " the user typed.
+    const QRegularExpressionMatch match = openCommand().match(m_commandLine->text());
+    const QString prefix = match.hasMatch() ? match.captured(1) : QStringLiteral(":open ");
+    m_commandLine->setCommandText(prefix + url);
+}
+
 void BrowserWindow::openUrl(const QString &input)
 {
     const QUrl url = urlutil::fromUserInput(input);

src/browserwindow.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "history.h"
+
 #include <QMainWindow>
 #include <QString>
 #include <QUrl>
@@ -7,6 +9,7 @@
 
 class AdBlocker;
 class CommandLine;
+class CompletionView;
 class StatusBar;
 class QWebEngineProfile;
 class QWebEngineView;
@@ -45,6 +48,8 @@ private:
     void handleHintResult(const QVariantMap &result);
     void clickAt(const QPointF &pos);
     void runCommand(const QString &command);
+    void updateCompletion();
+    void cycleCompletion(int step);
     void openUrl(const QString &input);
     void scrollBy(int dx, int dy);
     void scrollPages(double factor);
@@ -55,6 +60,8 @@ private:
     QWebEngineView *m_view;
     StatusBar *m_status;
     CommandLine *m_commandLine;
+    CompletionView *m_completion;
+    History m_history;
 
     Mode m_mode = Mode::Normal;
     QString m_pendingKeys;

src/commandline.cpp

@@ -1,5 +1,6 @@
 #include "commandline.h"
 
+#include <QEvent>
 #include <QKeyEvent>
 #include <QSignalBlocker>
 
@@ -27,6 +28,26 @@ void CommandLine::openWith(const QString &text)
     end(false);
 }
 
+void CommandLine::setCommandText(const QString &text)
+{
+    setTextSilently(text);
+    end(false);
+}
+
+bool CommandLine::event(QEvent *event)
+{
+    // QWidget::event turns Tab into a focus change before keyPressEvent would
+    // ever see it.
+    if (event->type() == QEvent::KeyPress) {
+        const int key = static_cast<QKeyEvent *>(event)->key();
+        if (key == Qt::Key_Tab || key == Qt::Key_Backtab) {
+            emit completionCycled(key == Qt::Key_Tab ? 1 : -1);
+            return true;
+        }
+    }
+    return QLineEdit::event(event);
+}
+
 void CommandLine::keyPressEvent(QKeyEvent *event)
 {
     switch (event->key()) {
@@ -96,6 +117,8 @@ void CommandLine::recallHistory(int direction)
     setTextSilently(pos == m_history.size() ? QStringLiteral(":")
                                             : QLatin1Char(':') + m_history.at(pos));
     end(false);
+    // Recalling is an edit as far as completion is concerned.
+    emit textEdited(text());
 }
 
 void CommandLine::cancel()

src/commandline.h

@@ -14,13 +14,19 @@ public:
 
     // text should include the leading ":", e.g. ":open ".
     void openWith(const QString &text);
+    // Replaces the text without emitting textEdited, e.g. when a completion
+    // is picked.
+    void setCommandText(const QString &text);
 
 signals:
     // Emitted with the command only, i.e. without the leading ":".
     void commandEntered(const QString &command);
     void cancelled();
+    // Tab (1) / Shift-Tab (-1).
+    void completionCycled(int step);
 
 protected:
+    bool event(QEvent *event) override;
     void keyPressEvent(QKeyEvent *event) override;
     void focusOutEvent(QFocusEvent *event) override;
 

src/completionview.cpp

@@ -0,0 +1,66 @@
+#include "completionview.h"
+
+#include <QHeaderView>
+
+namespace {
+
+constexpr int kMaxVisibleRows = 12;
+
+} // namespace
+
+CompletionView::CompletionView(QWidget *parent)
+    : QTreeWidget(parent)
+{
+    setObjectName(QStringLiteral("completion"));
+    setFocusPolicy(Qt::NoFocus);
+    setColumnCount(2);
+    setHeaderHidden(true);
+    setRootIsDecorated(false);
+    setUniformRowHeights(true);
+    setFrameShape(QFrame::NoFrame);
+    setSelectionMode(QAbstractItemView::SingleSelection);
+    setHorizontalScrollBarPolicy(Qt::ScrollBarAlwaysOff);
+    setTextElideMode(Qt::ElideRight);
+    header()->setStretchLastSection(false);
+    header()->setSectionResizeMode(QHeaderView::Stretch);
+    hide();
+}
+
+void CompletionView::setEntries(const QList<History::Entry> &entries)
+{
+    clear();
+    if (entries.isEmpty()) {
+        hide();
+        return;
+    }
+
+    QList<QTreeWidgetItem *> items;
+    items.reserve(entries.size());
+    for (const History::Entry &entry : entries) {
+        auto *item = new QTreeWidgetItem({entry.url, entry.title});
+        item->setToolTip(0, entry.url);
+        items.append(item);
+    }
+    addTopLevelItems(items);
+
+    const int rows = qMin(int(entries.size()), kMaxVisibleRows);
+    setFixedHeight(rows * sizeHintForRow(0) + 2 * frameWidth());
+    show();
+}
+
+QString CompletionView::cycle(int step)
+{
+    const int count = topLevelItemCount();
+    if (count == 0)
+        return {};
+
+    QTreeWidgetItem *current = currentItem();
+    int row = current && current->isSelected() ? indexOfTopLevelItem(current) + step
+                                               : (step > 0 ? 0 : count - 1);
+    row = ((row % count) + count) % count;
+
+    QTreeWidgetItem *item = topLevelItem(row);
+    setCurrentItem(item);
+    scrollToItem(item);
+    return item->text(0);
+}

src/completionview.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "history.h"
+
+#include <QTreeWidget>
+
+// The list above the command line offering history entries for ":open", with
+// URL and title columns, like qutebrowser's completion widget. It never takes
+// focus: keys stay with the CommandLine, which drives it via cycle().
+class CompletionView : public QTreeWidget
+{
+    Q_OBJECT
+
+public:
+    explicit CompletionView(QWidget *parent = nullptr);
+
+    // Shows the entries with nothing selected, or hides the view if empty.
+    void setEntries(const QList<History::Entry> &entries);
+
+    // Moves the selection step rows (wrapping) and returns the selected URL.
+    QString cycle(int step);
+};

src/history.cpp

@@ -0,0 +1,173 @@
+#include "history.h"
+
+#include <QDateTime>
+#include <QDir>
+#include <QSqlError>
+#include <QSqlQuery>
+#include <QStandardPaths>
+#include <QVariant>
+
+namespace {
+
+constexpr auto kConnection = "history";
+
+// LIKE treats % and _ as wildcards; escape them so they match literally.
+QString likePattern(const QString &word)
+{
+    QString escaped = word;
+    escaped.replace(QLatin1Char('\\'), QLatin1String("\\\\"));
+    escaped.replace(QLatin1Char('%'), QLatin1String("\\%"));
+    escaped.replace(QLatin1Char('_'), QLatin1String("\\_"));
+    return QLatin1Char('%') + escaped + QLatin1Char('%');
+}
+
+QString storedUrl(const QUrl &url)
+{
+    // The same readable form the status bar shows: it is what completion
+    // displays and what the user types to match against. Pretty-decoding keeps
+    // escapes that matter, like %2F, so the URL still round-trips.
+    //
+    // Without the #fragment: anchors on one page are one entry, and a title
+    // set after an in-page jump still finds the row its load created.
+    return url.adjusted(QUrl::RemoveFragment).toDisplayString();
+}
+
+} // namespace
+
+History::History()
+{
+    const QString dir = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation);
+    QDir().mkpath(dir);
+
+    m_db = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), QString::fromLatin1(kConnection));
+    m_db.setDatabaseName(dir + QStringLiteral("/history.sqlite"));
+    if (!m_db.open()) {
+        qWarning("history: cannot open database: %s", qPrintable(m_db.lastError().text()));
+        return;
+    }
+
+    m_open = exec(QStringLiteral("PRAGMA journal_mode=WAL"))
+             && exec(QStringLiteral("PRAGMA synchronous=NORMAL"))
+             && exec(QStringLiteral("CREATE TABLE IF NOT EXISTS visits ("
+                                    " url TEXT NOT NULL,"
+                                    " title TEXT NOT NULL,"
+                                    " at INTEGER NOT NULL)"))
+             && exec(QStringLiteral("CREATE TABLE IF NOT EXISTS urls ("
+                                    " url TEXT PRIMARY KEY,"
+                                    " title TEXT NOT NULL,"
+                                    " last_visit INTEGER NOT NULL,"
+                                    " visits INTEGER NOT NULL DEFAULT 1)"))
+             && exec(QStringLiteral("CREATE INDEX IF NOT EXISTS urls_last_visit"
+                                    " ON urls(last_visit DESC)"));
+}
+
+History::~History()
+{
+    m_db.close();
+    m_db = QSqlDatabase();
+    QSqlDatabase::removeDatabase(QString::fromLatin1(kConnection));
+}
+
+bool History::isRecordable(const QUrl &url)
+{
+    const QString scheme = url.scheme();
+    return scheme == QLatin1String("https") || scheme == QLatin1String("http")
+           || scheme == QLatin1String("file");
+}
+
+void History::addVisit(const QUrl &url, const QString &title)
+{
+    if (!m_open || !isRecordable(url))
+        return;
+
+    const QString u = storedUrl(url);
+    const qint64 now = QDateTime::currentSecsSinceEpoch();
+
+    m_db.transaction();
+
+    QSqlQuery visit(m_db);
+    visit.prepare(QStringLiteral("INSERT INTO visits (url, title, at) VALUES (?, ?, ?)"));
+    visit.addBindValue(u);
+    visit.addBindValue(title);
+    visit.addBindValue(now);
+
+    QSqlQuery upsert(m_db);
+    upsert.prepare(QStringLiteral(
+        "INSERT INTO urls (url, title, last_visit) VALUES (?, ?, ?)"
+        " ON CONFLICT(url) DO UPDATE SET"
+        "  title = CASE WHEN excluded.title != '' THEN excluded.title ELSE title END,"
+        "  last_visit = excluded.last_visit,"
+        "  visits = visits + 1"));
+    upsert.addBindValue(u);
+    upsert.addBindValue(title);
+    upsert.addBindValue(now);
+
+    if (visit.exec() && upsert.exec()) {
+        m_db.commit();
+    } else {
+        qWarning("history: cannot record visit: %s",
+                 qPrintable((visit.lastError().isValid() ? visit : upsert).lastError().text()));
+        m_db.rollback();
+    }
+}
+
+void History::updateTitle(const QUrl &url, const QString &title)
+{
+    if (!m_open || title.isEmpty() || !isRecordable(url))
+        return;
+
+    QSqlQuery query(m_db);
+    query.prepare(QStringLiteral("UPDATE urls SET title = ? WHERE url = ?"));
+    query.addBindValue(title);
+    query.addBindValue(storedUrl(url));
+    query.exec();
+}
+
+QList<History::Entry> History::search(const QStringList &words, int limit) const
+{
+    QList<Entry> entries;
+    if (!m_open)
+        return entries;
+
+    QString sql = QStringLiteral("SELECT url, title FROM urls");
+    for (qsizetype i = 0; i < words.size(); ++i) {
+        sql += i == 0 ? QStringLiteral(" WHERE ") : QStringLiteral(" AND ");
+        sql += QStringLiteral("(url LIKE ? ESCAPE '\\' OR title LIKE ? ESCAPE '\\')");
+    }
+    sql += QStringLiteral(" ORDER BY last_visit DESC LIMIT ?");
+
+    QSqlQuery query(m_db);
+    query.setForwardOnly(true);
+    query.prepare(sql);
+    for (const QString &word : words) {
+        const QString pattern = likePattern(word);
+        query.addBindValue(pattern);
+        query.addBindValue(pattern);
+    }
+    query.addBindValue(limit);
+
+    if (!query.exec()) {
+        qWarning("history: search failed: %s", qPrintable(query.lastError().text()));
+        return entries;
+    }
+    while (query.next())
+        entries.append({query.value(0).toString(), query.value(1).toString()});
+    return entries;
+}
+
+bool History::clear()
+{
+    if (!m_open)
+        return false;
+    return exec(QStringLiteral("DELETE FROM visits")) && exec(QStringLiteral("DELETE FROM urls"))
+           && exec(QStringLiteral("VACUUM"));
+}
+
+bool History::exec(const QString &sql)
+{
+    QSqlQuery query(m_db);
+    if (query.exec(sql))
+        return true;
+    qWarning("history: %s: %s", qPrintable(sql), qPrintable(query.lastError().text()));
+    return false;
+}

src/history.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <QList>
+#include <QSqlDatabase>
+#include <QString>
+#include <QStringList>
+#include <QUrl>
+
+// Browsing history in history.sqlite under AppDataLocation. Two tables:
+// "visits" is an append-only log of every page load; "urls" holds one row per
+// URL with its latest title and visit time, and is what completion searches.
+//
+// Queries run synchronously on the GUI thread. That is fine for SQLite at
+// browser-history sizes, with WAL so a write never waits on a full fsync.
+class History
+{
+public:
+    struct Entry {
+        QString url;
+        QString title;
+    };
+
+    History();
+    ~History();
+
+    History(const History &) = delete;
+    History &operator=(const History &) = delete;
+
+    bool isOpen() const { return m_open; }
+
+    void addVisit(const QUrl &url, const QString &title);
+    // Pages often set their title after the load finished.
+    void updateTitle(const QUrl &url, const QString &title);
+
+    // Most recently visited first. Every word must appear in the URL or the
+    // title, case-insensitively.
+    QList<Entry> search(const QStringList &words, int limit) const;
+
+    bool clear();
+
+    // Whether a URL belongs in history at all: no internal or data pages.
+    static bool isRecordable(const QUrl &url);
+
+private:
+    bool exec(const QString &sql);
+
+    QSqlDatabase m_db;
+    bool m_open = false;
+};

src/main.cpp

@@ -27,6 +27,14 @@ constexpr auto kStyleSheet = R"(
     QLabel#statusProgress {
         color: #8a8a8a;
     }
+    QTreeWidget#completion {
+        background-color: #262626;
+        color: #d8d8d8;
+        font-family: monospace;
+        border-top: 1px solid #3a3a3a;
+        selection-background-color: #e8c000;
+        selection-color: #000000;
+    }
     QLineEdit#commandLine {
         padding: 1px 4px;
         selection-background-color: #445566;