refactor(chatlog): Add a class to manage history through the IChatLog interface

CMakeLists.txt

@@ -351,6 +351,8 @@ set(${PROJECT_NAME}_SOURCES
   src/model/ichatlog.h
   src/model/sessionchatlog.h
   src/model/sessionchatlog.cpp
+  src/model/chathistory.h
+  src/model/chathistory.cpp
   src/net/bootstrapnodeupdater.cpp
   src/net/bootstrapnodeupdater.h
   src/net/avatarbroadcaster.cpp

src/model/chathistory.cpp

@@ -0,0 +1,460 @@
+/*
+    Copyright © 2019 by The qTox Project Contributors
+
+    This file is part of qTox, a Qt-based graphical interface for Tox.
+
+    qTox is libre software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    qTox is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with qTox.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "chathistory.h"
+#include "src/persistence/settings.h"
+#include "src/widget/form/chatform.h"
+
+namespace {
+/**
+ * @brief Determines if the given idx needs to be loaded from history
+ * @param[in] idx index to check
+ * @param[in] sessionChatLog SessionChatLog containing currently loaded items
+ * @return True if load is needed
+ */
+bool needsLoadFromHistory(ChatLogIdx idx, const SessionChatLog& sessionChatLog)
+{
+    return idx < sessionChatLog.getFirstIdx();
+}
+
+/**
+ * @brief Gets the initial chat log index for a sessionChatLog with 0 items loaded from history.
+ * Needed to keep history indexes in sync with chat log indexes
+ * @param[in] history
+ * @param[in] f
+ * @return Initial chat log index
+ */
+ChatLogIdx getInitialChatLogIdx(History* history, Friend& f)
+{
+    if (!history) {
+        return ChatLogIdx(0);
+    }
+
+    return ChatLogIdx(history->getNumMessagesForFriend(f.getPublicKey()));
+}
+
+/**
+ * @brief Finds the first item in sessionChatLog that contains a message
+ * @param[in] sessionChatLog
+ * @return index of first message
+ */
+ChatLogIdx findFirstMessage(const SessionChatLog& sessionChatLog)
+{
+    auto it = sessionChatLog.getFirstIdx();
+    while (it < sessionChatLog.getNextIdx()) {
+        if (sessionChatLog.at(it).getContentType() == ChatLogItem::ContentType::message) {
+            return it;
+        }
+        it++;
+    }
+    return ChatLogIdx(-1);
+}
+
+/**
+ * @brief Handles presence of aciton prefix in content
+ * @param[in/out] content
+ * @return True if was an action
+ */
+bool handleActionPrefix(QString& content)
+{
+    // Unfortunately due to legacy reasons we have to continue
+    // inserting and parsing for ACTION_PREFIX in our messages even
+    // though we have the ability to something more intelligent now
+    // that we aren't owned by chatform logic
+    auto isAction = content.startsWith(ChatForm::ACTION_PREFIX, Qt::CaseInsensitive);
+    if (isAction) {
+        content.remove(0, ChatForm::ACTION_PREFIX.size());
+    }
+
+    return isAction;
+}
+} // namespace
+
+ChatHistory::ChatHistory(Friend& f_, History* history_, const ICoreIdHandler& coreIdHandler,
+                         const Settings& settings_, IMessageDispatcher& messageDispatcher)
+    : f(f_)
+    , history(history_)
+    , sessionChatLog(getInitialChatLogIdx(history, f), coreIdHandler)
+    , settings(settings_)
+    , coreIdHandler(coreIdHandler)
+{
+    connect(&messageDispatcher, &IMessageDispatcher::messageSent, this, &ChatHistory::onMessageSent);
+    connect(&messageDispatcher, &IMessageDispatcher::messageComplete, this,
+            &ChatHistory::onMessageComplete);
+
+    if (canUseHistory()) {
+        // Defer messageSent callback until we finish firing off all our unsent messages.
+        // If it was connected all our unsent messages would be re-added ot history again
+        dispatchUnsentMessages(messageDispatcher);
+    }
+
+    // Now that we've fired off our unsent messages we can connect the message
+    connect(&messageDispatcher, &IMessageDispatcher::messageReceived, this,
+            &ChatHistory::onMessageReceived);
+
+    // NOTE: this has to be done _after_ sending all sent messages since initial
+    // state of the message has to be marked according to our dispatch state
+    constexpr auto defaultNumMessagesToLoad = 100;
+    auto firstChatLogIdx = sessionChatLog.getFirstIdx().get() < defaultNumMessagesToLoad
+                               ? ChatLogIdx(0)
+                               : sessionChatLog.getFirstIdx() - defaultNumMessagesToLoad;
+
+    if (canUseHistory()) {
+        loadHistoryIntoSessionChatLog(firstChatLogIdx);
+    }
+
+    // We don't manage any of the item updates ourselves, we just forward along
+    // the underlying sessionChatLog's updates
+    connect(&sessionChatLog, &IChatLog::itemUpdated, this, &IChatLog::itemUpdated);
+}
+
+const ChatLogItem& ChatHistory::at(ChatLogIdx idx) const
+{
+    if (canUseHistory()) {
+        ensureIdxInSessionChatLog(idx);
+    }
+
+    return sessionChatLog.at(idx);
+}
+
+SearchResult ChatHistory::searchForward(SearchPos startIdx, const QString& phrase,
+                                        const ParameterSearch& parameter) const
+{
+    if (startIdx.logIdx >= getNextIdx()) {
+        SearchResult res;
+        res.found = false;
+        return res;
+    }
+
+    if (canUseHistory()) {
+        ensureIdxInSessionChatLog(startIdx.logIdx);
+    }
+
+    return sessionChatLog.searchForward(startIdx, phrase, parameter);
+}
+
+SearchResult ChatHistory::searchBackward(SearchPos startIdx, const QString& phrase,
+                                         const ParameterSearch& parameter) const
+{
+    auto res = sessionChatLog.searchBackward(startIdx, phrase, parameter);
+
+    if (res.found || !canUseHistory()) {
+        return res;
+    }
+
+    auto earliestMessage = findFirstMessage(sessionChatLog);
+
+    auto earliestMessageDate =
+        (earliestMessage == ChatLogIdx(-1))
+            ? QDateTime::currentDateTime()
+            : sessionChatLog.at(earliestMessage).getContentAsMessage().message.timestamp;
+
+    // Roundabout way of getting the first idx but I don't want to have to
+    // deal with re-implementing so we'll just piece what we want together...
+    //
+    // If the double disk access is real bad we can optimize this by adding
+    // another function to history
+    auto dateWherePhraseFound =
+        history->getDateWhereFindPhrase(f.getPublicKey().toString(), earliestMessageDate, phrase,
+                                        parameter);
+
+    auto loadIdx = history->getNumMessagesForFriendBeforeDate(f.getPublicKey(), dateWherePhraseFound);
+    loadHistoryIntoSessionChatLog(ChatLogIdx(loadIdx));
+
+    // Reset search pos to the message we just loaded to avoid a double search
+    startIdx.logIdx = ChatLogIdx(loadIdx);
+    startIdx.numMatches = 0;
+    return sessionChatLog.searchBackward(startIdx, phrase, parameter);
+}
+
+ChatLogIdx ChatHistory::getFirstIdx() const
+{
+    if (canUseHistory()) {
+        return ChatLogIdx(0);
+    } else {
+        return sessionChatLog.getFirstIdx();
+    }
+}
+
+ChatLogIdx ChatHistory::getNextIdx() const
+{
+    return sessionChatLog.getNextIdx();
+}
+
+std::vector<IChatLog::DateChatLogIdxPair> ChatHistory::getDateIdxs(const QDate& startDate,
+                                                                   size_t maxDates) const
+{
+    if (canUseHistory()) {
+        auto counts = history->getNumMessagesForFriendBeforeDateBoundaries(f.getPublicKey(),
+                                                                           startDate, maxDates);
+
+        std::vector<IChatLog::DateChatLogIdxPair> ret;
+        std::transform(counts.begin(), counts.end(), std::back_inserter(ret),
+                       [&](const History::DateIdx& historyDateIdx) {
+                           DateChatLogIdxPair pair;
+                           pair.date = historyDateIdx.date;
+                           pair.idx.get() = historyDateIdx.numMessagesIn;
+                           return pair;
+                       });
+
+        // Do not re-search in the session chat log. If we have history the query to the history should have been sufficient
+        return ret;
+    } else {
+        return sessionChatLog.getDateIdxs(startDate, maxDates);
+    }
+}
+
+void ChatHistory::onFileUpdated(const ToxPk& sender, const ToxFile& file)
+{
+    if (canUseHistory()) {
+        switch (file.status) {
+        case ToxFile::INITIALIZING: {
+            // Note: There is some implcit coupling between history and the current
+            // chat log. Both rely on generating a new id based on the state of
+            // initializing. If this is changed in the session chat log we'll end up
+            // with a different order when loading from history
+            history->addNewFileMessage(f.getPublicKey().toString(), file.resumeFileId, file.fileName,
+                                       file.filePath, file.filesize, sender.toString(),
+                                       QDateTime::currentDateTime(), f.getDisplayedName());
+            break;
+        }
+        case ToxFile::CANCELED:
+        case ToxFile::FINISHED:
+        case ToxFile::BROKEN: {
+            const bool isSuccess = file.status == ToxFile::FINISHED;
+            history->setFileFinished(file.resumeFileId, isSuccess, file.filePath,
+                                     file.hashGenerator->result());
+            break;
+        }
+        case ToxFile::PAUSED:
+        case ToxFile::TRANSMITTING:
+        default:
+            break;
+        }
+    }
+
+    sessionChatLog.onFileUpdated(sender, file);
+}
+
+void ChatHistory::onFileTransferRemotePausedUnpaused(const ToxPk& sender, const ToxFile& file,
+                                                     bool paused)
+{
+    sessionChatLog.onFileTransferRemotePausedUnpaused(sender, file, paused);
+}
+
+void ChatHistory::onFileTransferBrokenUnbroken(const ToxPk& sender, const ToxFile& file, bool broken)
+{
+    sessionChatLog.onFileTransferBrokenUnbroken(sender, file, broken);
+}
+
+void ChatHistory::onMessageReceived(const ToxPk& sender, const Message& message)
+{
+    if (canUseHistory()) {
+        auto friendPk = f.getPublicKey().toString();
+        auto displayName = f.getDisplayedName();
+        auto content = message.content;
+        if (message.isAction) {
+            content = ChatForm::ACTION_PREFIX + content;
+        }
+
+        history->addNewMessage(friendPk, content, friendPk, message.timestamp, true, displayName);
+    }
+
+    sessionChatLog.onMessageReceived(sender, message);
+}
+
+void ChatHistory::onMessageSent(DispatchedMessageId id, const Message& message)
+{
+    if (canUseHistory()) {
+        auto selfPk = coreIdHandler.getSelfPublicKey().toString();
+        auto friendPk = f.getPublicKey().toString();
+
+        auto content = message.content;
+        if (message.isAction) {
+            content = ChatForm::ACTION_PREFIX + content;
+        }
+
+        auto username = coreIdHandler.getUsername();
+
+        auto onInsertion = [this, id](RowId historyId) { handleDispatchedMessage(id, historyId); };
+
+        history->addNewMessage(friendPk, content, selfPk, message.timestamp, false, username,
+                               onInsertion);
+    }
+
+    sessionChatLog.onMessageSent(id, message);
+}
+
+void ChatHistory::onMessageComplete(DispatchedMessageId id)
+{
+    if (canUseHistory()) {
+        completeMessage(id);
+    }
+
+    sessionChatLog.onMessageComplete(id);
+}
+
+/**
+ * @brief Forces the given index and all future indexes to be in the chatlog
+ * @param[in] idx
+ * @note Marked const since this doesn't change _external_ state of the class. We
+     still have all the same items at all the same indexes, we've just stuckem
+     in ram
+ */
+void ChatHistory::ensureIdxInSessionChatLog(ChatLogIdx idx) const
+{
+    if (needsLoadFromHistory(idx, sessionChatLog)) {
+        loadHistoryIntoSessionChatLog(idx);
+    }
+}
+/**
+ * @brief Unconditionally loads the given index and all future messages that
+ * are not in the session chat log into the session chat log
+ * @param[in] idx
+ * @note Marked const since this doesn't change _external_ state of the class. We
+   still have all the same items at all the same indexes, we've just stuckem
+   in ram
+ * @note no end idx as we always load from start -> latest. In the future we
+ * could have a less contiguous history
+ */
+void ChatHistory::loadHistoryIntoSessionChatLog(ChatLogIdx start) const
+{
+    if (!needsLoadFromHistory(start, sessionChatLog)) {
+        return;
+    }
+
+    auto end = sessionChatLog.getFirstIdx();
+
+    // We know that both history and us have a start index of 0 so the type
+    // conversion should be safe
+    assert(getFirstIdx() == ChatLogIdx(0));
+    auto messages = history->getMessagesForFriend(f.getPublicKey(), start.get(), end.get());
+
+    assert(messages.size() == end.get() - start.get());
+    ChatLogIdx nextIdx = start;
+
+    for (const auto& message : messages) {
+        // Note that message.id is _not_ a valid conversion here since it is a
+        // global id not a per-chat id like the ChatLogIdx
+        auto currentIdx = nextIdx++;
+        auto sender = ToxId(message.sender).getPublicKey();
+        switch (message.content.getType()) {
+        case HistMessageContentType::file: {
+            const auto date = message.timestamp;
+            const auto file = message.content.asFile();
+            const auto chatLogFile = ChatLogFile{date, file};
+            sessionChatLog.insertFileAtIdx(currentIdx, sender, message.dispName, chatLogFile);
+            break;
+        }
+        case HistMessageContentType::message: {
+            auto messageContent = message.content.asMessage();
+
+            auto isAction = handleActionPrefix(messageContent);
+
+            // It's okay to skip the message processor here. The processor is
+            // meant to convert between boundaries of our internal
+            // representation. We already had to go through the processor before
+            // we hit IMessageDispatcher's signals which history listens for.
+            // Items added to history have already been sent so we know they already
+            // reflect what was sent/received.
+            auto processedMessage = Message{isAction, messageContent, message.timestamp};
+
+            auto dispatchedMessageIt =
+                std::find_if(dispatchedMessageRowIdMap.begin(), dispatchedMessageRowIdMap.end(),
+                             [&](RowId dispatchedId) { return dispatchedId == message.id; });
+
+            bool isComplete = dispatchedMessageIt == dispatchedMessageRowIdMap.end();
+
+            if (isComplete) {
+                auto chatLogMessage = ChatLogMessage{true, processedMessage};
+                sessionChatLog.insertMessageAtIdx(currentIdx, sender, message.dispName, chatLogMessage);
+            } else {
+                // If the message is incomplete we have to pretend we sent it to ensure
+                // sessionChatLog state is correct
+                sessionChatLog.onMessageSent(dispatchedMessageIt.key(), processedMessage);
+            }
+            break;
+        }
+        }
+    }
+
+    assert(nextIdx == end);
+}
+
+/**
+ * @brief Sends any unsent messages in history to the underlying message dispatcher
+ * @param[in] messageDispatcher
+ */
+void ChatHistory::dispatchUnsentMessages(IMessageDispatcher& messageDispatcher)
+{
+    auto unsentMessages = history->getUnsentMessagesForFriend(f.getPublicKey());
+    for (auto& message : unsentMessages) {
+        // We should only store messages as unsent, if this changes in the
+        // future we need to extend this logic
+        assert(message.content.getType() == HistMessageContentType::message);
+
+        auto messageContent = message.content.asMessage();
+        auto isAction = handleActionPrefix(messageContent);
+
+        // NOTE: timestamp will be generated in messageDispatcher but we haven't
+        // hooked up our history callback so it will not be shown in our chatlog
+        // with the new timestamp. This is intentional as everywhere else we use
+        // attempted send time (which is whenever the it was initially inserted
+        // into history
+        auto dispatchIds = messageDispatcher.sendMessage(isAction, messageContent);
+
+        // We should only send a single message, but in the odd case where we end
+        // up having to split more than when we added the message to history we'll
+        // just associate the last dispatched id with the history message
+        handleDispatchedMessage(dispatchIds.second, message.id);
+
+        // We don't add the messages to the underlying chatlog since
+        // 1. We don't even know the ChatLogIdx of this message
+        // 2. We only want to display the latest N messages on boot by default,
+        //    even if there are more than N messages that haven't been sent
+    }
+}
+
+void ChatHistory::handleDispatchedMessage(DispatchedMessageId dispatchId, RowId historyId)
+{
+    auto completedMessageIt = completedMessages.find(dispatchId);
+    if (completedMessageIt == completedMessages.end()) {
+        dispatchedMessageRowIdMap.insert(dispatchId, historyId);
+    } else {
+        history->markAsSent(historyId);
+        completedMessages.erase(completedMessageIt);
+    }
+}
+
+void ChatHistory::completeMessage(DispatchedMessageId id)
+{
+    auto dispatchedMessageIt = dispatchedMessageRowIdMap.find(id);
+
+    if (dispatchedMessageIt == dispatchedMessageRowIdMap.end()) {
+        completedMessages.insert(id);
+    } else {
+        history->markAsSent(*dispatchedMessageIt);
+        dispatchedMessageRowIdMap.erase(dispatchedMessageIt);
+    }
+}
+
+bool ChatHistory::canUseHistory() const
+{
+    return history && settings.getEnableLogging();
+}

src/model/chathistory.h

@@ -0,0 +1,79 @@
+/*
+    Copyright © 2019 by The qTox Project Contributors
+
+    This file is part of qTox, a Qt-based graphical interface for Tox.
+
+    qTox is libre software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    qTox is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with qTox.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef CHAT_HISTORY_H
+#define CHAT_HISTORY_H
+
+#include "ichatlog.h"
+#include "sessionchatlog.h"
+#include "src/persistence/history.h"
+
+#include <QSet>
+
+class Settings;
+
+class ChatHistory : public IChatLog
+{
+    Q_OBJECT
+public:
+    ChatHistory(Friend& f_, History* history_, const ICoreIdHandler& coreIdHandler,
+                const Settings& settings, IMessageDispatcher& messageDispatcher);
+    const ChatLogItem& at(ChatLogIdx idx) const override;
+    SearchResult searchForward(SearchPos startIdx, const QString& phrase,
+                               const ParameterSearch& parameter) const override;
+    SearchResult searchBackward(SearchPos startIdx, const QString& phrase,
+                                const ParameterSearch& parameter) const override;
+    ChatLogIdx getFirstIdx() const override;
+    ChatLogIdx getNextIdx() const override;
+    std::vector<DateChatLogIdxPair> getDateIdxs(const QDate& startDate, size_t maxDates) const override;
+
+public slots:
+    void onFileUpdated(const ToxPk& sender, const ToxFile& file);
+    void onFileTransferRemotePausedUnpaused(const ToxPk& sender, const ToxFile& file, bool paused);
+    void onFileTransferBrokenUnbroken(const ToxPk& sender, const ToxFile& file, bool broken);
+
+private slots:
+    void onMessageReceived(const ToxPk& sender, const Message& message);
+    void onMessageSent(DispatchedMessageId id, const Message& message);
+    void onMessageComplete(DispatchedMessageId id);
+
+private:
+    void ensureIdxInSessionChatLog(ChatLogIdx idx) const;
+    void loadHistoryIntoSessionChatLog(ChatLogIdx start) const;
+    void dispatchUnsentMessages(IMessageDispatcher& messageDispatcher);
+    void handleDispatchedMessage(DispatchedMessageId dispatchId, RowId historyId);
+    void completeMessage(DispatchedMessageId id);
+    bool canUseHistory() const;
+
+    Friend& f;
+    History* history;
+    mutable SessionChatLog sessionChatLog;
+    const Settings& settings;
+    const ICoreIdHandler& coreIdHandler;
+
+    // If a message completes before it's inserted into history it will end up
+    // in this set
+    QSet<DispatchedMessageId> completedMessages;
+
+    // If a message is inserted into history before it gets a completion
+    // callback it will end up in this map
+    QMap<DispatchedMessageId, RowId> dispatchedMessageRowIdMap;
+};
+
+#endif /*CHAT_HISTORY_H*/

src/persistence/history.cpp

@@ -520,6 +520,7 @@ void History::setFileFinished(const QString& fileId, bool success, const QString
 
     fileInfos.remove(fileId);
 }
+
 /**
  * @brief Fetches chat messages from the database.
  * @param friendPk Friend publick key to fetch.
@@ -592,6 +593,122 @@ QList<History::DateMessages> History::getChatHistoryCounts(const ToxPk& friendPk
     return counts;
 }
 
+size_t History::getNumMessagesForFriend(const ToxPk& friendPk)
+{
+    return getNumMessagesForFriendBeforeDate(friendPk,
+                                             // Maximum possible time
+                                             QDateTime::fromMSecsSinceEpoch(
+                                                 std::numeric_limits<int64_t>::max()));
+}
+
+size_t History::getNumMessagesForFriendBeforeDate(const ToxPk& friendPk, const QDateTime& date)
+{
+    QString queryText = QString("SELECT COUNT(history.id) "
+                                "FROM history "
+                                "JOIN peers chat ON chat_id = chat.id "
+                                "WHERE chat.public_key='%1'"
+                                "AND timestamp < %2;")
+                            .arg(friendPk.toString())
+                            .arg(date.toMSecsSinceEpoch());
+
+    size_t numMessages = 0;
+    auto rowCallback = [&numMessages](const QVector<QVariant>& row) {
+        numMessages = row[0].toLongLong();
+    };
+
+    db->execNow({queryText, rowCallback});
+
+    return numMessages;
+}
+
+QList<History::HistMessage> History::getMessagesForFriend(const ToxPk& friendPk, size_t firstIdx,
+                                                          size_t lastIdx)
+{
+    QList<HistMessage> messages;
+
+    // Don't forget to update the rowCallback if you change the selected columns!
+    QString queryText =
+        QString("SELECT history.id, faux_offline_pending.id, timestamp, "
+                "chat.public_key, aliases.display_name, sender.public_key, "
+                "message, file_transfers.file_restart_id, "
+                "file_transfers.file_path, file_transfers.file_name, "
+                "file_transfers.file_size, file_transfers.direction, "
+                "file_transfers.file_state FROM history "
+                "LEFT JOIN faux_offline_pending ON history.id = faux_offline_pending.id "
+                "JOIN peers chat ON history.chat_id = chat.id "
+                "JOIN aliases ON sender_alias = aliases.id "
+                "JOIN peers sender ON aliases.owner = sender.id "
+                "LEFT JOIN file_transfers ON history.file_id = file_transfers.id "
+                "WHERE chat.public_key='%1' "
+                "LIMIT %2 OFFSET %3;")
+            .arg(friendPk.toString())
+            .arg(lastIdx - firstIdx)
+            .arg(firstIdx);
+
+    auto rowCallback = [&messages](const QVector<QVariant>& row) {
+        // dispName and message could have null bytes, QString::fromUtf8
+        // truncates on null bytes so we strip them
+        auto id = RowId{row[0].toLongLong()};
+        auto isOfflineMessage = row[1].isNull();
+        auto timestamp = QDateTime::fromMSecsSinceEpoch(row[2].toLongLong());
+        auto friend_key = row[3].toString();
+        auto display_name = QString::fromUtf8(row[4].toByteArray().replace('\0', ""));
+        auto sender_key = row[5].toString();
+        if (row[7].isNull()) {
+            messages += {id,           isOfflineMessage, timestamp,        friend_key,
+                         display_name, sender_key,       row[6].toString()};
+        } else {
+            ToxFile file;
+            file.fileKind = TOX_FILE_KIND_DATA;
+            file.resumeFileId = row[7].toString().toUtf8();
+            file.filePath = row[8].toString();
+            file.fileName = row[9].toString();
+            file.filesize = row[10].toLongLong();
+            file.direction = static_cast<ToxFile::FileDirection>(row[11].toLongLong());
+            file.status = static_cast<ToxFile::FileStatus>(row[12].toInt());
+            messages +=
+                {id, isOfflineMessage, timestamp, friend_key, display_name, sender_key, file};
+        }
+    };
+
+    db->execNow({queryText, rowCallback});
+
+    return messages;
+}
+
+QList<History::HistMessage> History::getUnsentMessagesForFriend(const ToxPk& friendPk)
+{
+    auto queryText =
+        QString("SELECT history.id, faux_offline_pending.id, timestamp, chat.public_key, "
+                "aliases.display_name, sender.public_key, message "
+                "FROM history "
+                "JOIN faux_offline_pending ON history.id = faux_offline_pending.id "
+                "JOIN peers chat on chat.public_key = '%1' "
+                "JOIN aliases on sender_alias = aliases.id "
+                "JOIN peers sender on aliases.owner = sender.id;")
+            .arg(friendPk.toString());
+
+    QList<History::HistMessage> ret;
+    auto rowCallback = [&ret](const QVector<QVariant>& row) {
+        // dispName and message could have null bytes, QString::fromUtf8
+        // truncates on null bytes so we strip them
+        auto id = RowId{row[0].toLongLong()};
+        auto isOfflineMessage = row[1].isNull();
+        auto timestamp = QDateTime::fromMSecsSinceEpoch(row[2].toLongLong());
+        auto friend_key = row[3].toString();
+        auto display_name = QString::fromUtf8(row[4].toByteArray().replace('\0', ""));
+        auto sender_key = row[5].toString();
+        if (row[6].isNull()) {
+            ret += {id,           isOfflineMessage, timestamp,        friend_key,
+                    display_name, sender_key,       row[6].toString()};
+        }
+    };
+
+    db->execNow({queryText, rowCallback});
+
+    return ret;
+}
+
 /**
  * @brief Search phrase in chat messages
  * @param friendPk Friend public key
@@ -681,6 +798,65 @@ QDateTime History::getDateWhereFindPhrase(const QString& friendPk, const QDateTi
     return result;
 }
 
+/**
+ * @brief Gets date boundaries in conversation with friendPk. History doesn't model conversation indexes,
+ * but we can count messages between us and friendPk to effectively give us an index. This function
+ * returns how many messages have happened between us <-> friendPk each time the date changes
+ * @param[in] friendPk ToxPk of conversation to retrieve
+ * @param[in] from Start date to look from
+ * @param[in] maxNum Maximum number of date boundaries to retrieve
+ * @note This API may seem a little strange, why not use QDate from and QDate to? The intent is to
+ * have an API that can be used to get the first item after a date (for search) and to get a list
+ * of date changes (for loadHistory). We could write two separate queries but the query is fairly
+ * intricate compared to our other ones so reducing duplication of it is preferable.
+ */
+QList<History::DateIdx> History::getNumMessagesForFriendBeforeDateBoundaries(const ToxPk& friendPk,
+                                                                             const QDate& from,
+                                                                             size_t maxNum)
+{
+    auto friendPkString = friendPk.toString();
+
+    // No guarantee that this is the most efficient way to do this...
+    // We want to count messages that happened for a friend before a
+    // certain date. We do this by re-joining our table a second time
+    // but this time with the only filter being that our id is less than
+    // the ID of the corresponding row in the table that is grouped by day
+    auto countMessagesForFriend =
+        QString("SELECT COUNT(*) - 1 " // Count - 1 corresponds to 0 indexed message id for friend
+                "FROM history countHistory "            // Import unfiltered table as countHistory
+                "JOIN peers chat ON chat_id = chat.id " // link chat_id to chat.id
+                "WHERE chat.public_key = '%1'"          // filter this conversation
+                "AND countHistory.id <= history.id") // and filter that our unfiltered table history id only has elements up to history.id
+            .arg(friendPkString);
+
+    auto limitString = (maxNum) ? QString("LIMIT %1").arg(maxNum) : QString("");
+
+    auto queryString = QString("SELECT (%1), (timestamp / 1000 / 60 / 60 / 24) AS day "
+                               "FROM history "
+                               "JOIN peers chat ON chat_id = chat.id "
+                               "WHERE chat.public_key = '%2' "
+                               "AND timestamp >= %3 "
+                               "GROUP by day "
+                               "%4;")
+                           .arg(countMessagesForFriend)
+                           .arg(friendPkString)
+                           .arg(QDateTime(from).toMSecsSinceEpoch())
+                           .arg(limitString);
+
+    QList<DateIdx> dateIdxs;
+    auto rowCallback = [&dateIdxs](const QVector<QVariant>& row) {
+        DateIdx dateIdx;
+        dateIdx.numMessagesIn = row[0].toLongLong();
+        dateIdx.date =
+            QDateTime::fromMSecsSinceEpoch(row[1].toLongLong() * 24 * 60 * 60 * 1000).date();
+        dateIdxs.append(dateIdx);
+    };
+
+    db->execNow({queryString, rowCallback});
+
+    return dateIdxs;
+}
+
 /**
  * @brief get start date of correspondence
  * @param friendPk Friend public key

src/persistence/history.h

@@ -149,6 +149,12 @@ public:
         uint count;
     };
 
+    struct DateIdx
+    {
+        QDate date;
+        size_t numMessagesIn;
+    };
+
 public:
     explicit History(std::shared_ptr<RawDatabase> db);
     ~History();
@@ -173,8 +179,14 @@ public:
                                               const QDateTime& to);
     QList<HistMessage> getChatHistoryDefaultNum(const QString& friendPk);
     QList<DateMessages> getChatHistoryCounts(const ToxPk& friendPk, const QDate& from, const QDate& to);
+    size_t getNumMessagesForFriend(const ToxPk& friendPk);
+    size_t getNumMessagesForFriendBeforeDate(const ToxPk& friendPk, const QDateTime& date);
+    QList<HistMessage> getMessagesForFriend(const ToxPk& friendPk, size_t firstIdx, size_t lastIdx);
+    QList<HistMessage> getUnsentMessagesForFriend(const ToxPk& friendPk);
     QDateTime getDateWhereFindPhrase(const QString& friendPk, const QDateTime& from, QString phrase,
                                      const ParameterSearch& parameter);
+    QList<DateIdx> getNumMessagesForFriendBeforeDateBoundaries(const ToxPk& friendPk,
+                                                               const QDate& from, size_t maxNum);
     QDateTime getStartDateChatHistory(const QString& friendPk);
 
     void markAsSent(RowId messageId);