1
0
mirror of https://github.com/qTox/qTox.git synced 2024-03-22 14:00:36 +08:00

feat(history): load set number of messages from history

Fix #3124
Fix #3004

Instead of loading a set 7 days of history. Better performance when there are lots of messages, and better context when friends haven't talked in over a week.

Removed historyBaselineDate, introduced in deb8440c6a to fix duplicate messages, but duplicate messages were very likely fixed by https://github.com/qTox/qTox/pull/4607.

Also refactored history loading.
This commit is contained in:
Anthony Bilinski 2018-04-23 16:51:26 -07:00
parent dfd2de836e
commit ca32e77d74
No known key found for this signature in database
GPG Key ID: 2AA8E0DA1B31FB3C
9 changed files with 204 additions and 126 deletions

View File

@ -380,10 +380,10 @@ void ChatLog::insertChatlineOnTop(ChatLine::Ptr l)
if (!l.get())
return;
insertChatlineOnTop(QList<ChatLine::Ptr>() << l);
insertChatlinesOnTop(QList<ChatLine::Ptr>() << l);
}
void ChatLog::insertChatlineOnTop(const QList<ChatLine::Ptr>& newLines)
void ChatLog::insertChatlinesOnTop(const QList<ChatLine::Ptr>& newLines)
{
if (newLines.isEmpty())
return;

View File

@ -43,7 +43,7 @@ public:
void insertChatlineAtBottom(ChatLine::Ptr l);
void insertChatlineOnTop(ChatLine::Ptr l);
void insertChatlineOnTop(const QList<ChatLine::Ptr>& newLines);
void insertChatlinesOnTop(const QList<ChatLine::Ptr>& newLines);
void clearSelection();
void clear();
void copySelectedText(bool toSelectionBuffer = false) const;

View File

@ -34,6 +34,8 @@
* Caches mappings to speed up message saving.
*/
static constexpr int NUM_MESSAGES_DEFAULT = 100; // arbitrary number of messages loaded when not loading by date
/**
* @brief Prepares the database to work with the history.
* @param db This database will be prepared for use with the history.
@ -249,46 +251,29 @@ void History::addNewMessage(const QString& friendPk, const QString& message, con
* @param to End of period to fetch.
* @return List of messages.
*/
QList<History::HistMessage> History::getChatHistory(const QString& friendPk, const QDateTime& from,
QList<History::HistMessage> History::getChatHistoryFromDate(const QString& friendPk, const QDateTime& from,
const QDateTime& to)
{
if (!isValid()) {
return {};
}
QList<HistMessage> messages;
auto rowCallback = [&messages](const QVector<QVariant>& row) {
// dispName and message could have null bytes, QString::fromUtf8
// truncates on null bytes so we strip them
messages += {row[0].toLongLong(),
row[1].isNull(),
QDateTime::fromMSecsSinceEpoch(row[2].toLongLong()),
row[3].toString(),
QString::fromUtf8(row[4].toByteArray().replace('\0', "")),
row[5].toString(),
QString::fromUtf8(row[6].toByteArray().replace('\0', ""))};
};
// Don't forget to update the rowCallback if you change the selected columns!
QString queryText =
QString("SELECT history.id, faux_offline_pending.id, timestamp, "
"chat.public_key, aliases.display_name, sender.public_key, "
"message FROM history "
"LEFT JOIN faux_offline_pending ON history.id = faux_offline_pending.id "
"JOIN peers chat ON chat_id = chat.id "
"JOIN aliases ON sender_alias = aliases.id "
"JOIN peers sender ON aliases.owner = sender.id "
"WHERE timestamp BETWEEN %1 AND %2 AND chat.public_key='%3';")
.arg(from.toMSecsSinceEpoch())
.arg(to.toMSecsSinceEpoch())
.arg(friendPk);
db->execNow({queryText, rowCallback});
return messages;
return getChatHistory(friendPk, from, to, 0);
}
/**
* @brief Fetches the latest set amount of messages from the database.
* @param friendPk Friend public key to fetch.
* @return List of messages.
*/
QList<History::HistMessage> History::getChatHistoryDefaultNum(const QString& friendPk)
{
if (!isValid()) {
return {};
}
return getChatHistory(friendPk, QDateTime::fromMSecsSinceEpoch(0), QDateTime::currentDateTime(), NUM_MESSAGES_DEFAULT);
}
/**
* @brief Fetches chat messages counts for each day from the database.
* @param friendPk Friend public key to fetch.
@ -375,3 +360,54 @@ void History::markAsSent(qint64 messageId)
db->execLater(QString("DELETE FROM faux_offline_pending WHERE id=%1;").arg(messageId));
}
/**
* @brief Fetches chat messages from the database.
* @param friendPk Friend publick key to fetch.
* @param from Start of period to fetch.
* @param to End of period to fetch.
* @param numMessages max number of messages to fetch.
* @return List of messages.
*/
QList<History::HistMessage> History::getChatHistory(const QString& friendPk, const QDateTime& from,
const QDateTime& to, int numMessages)
{
QList<HistMessage> messages;
auto rowCallback = [&messages](const QVector<QVariant>& row) {
// dispName and message could have null bytes, QString::fromUtf8
// truncates on null bytes so we strip them
messages += {row[0].toLongLong(),
row[1].isNull(),
QDateTime::fromMSecsSinceEpoch(row[2].toLongLong()),
row[3].toString(),
QString::fromUtf8(row[4].toByteArray().replace('\0', "")),
row[5].toString(),
QString::fromUtf8(row[6].toByteArray().replace('\0', ""))};
};
// Don't forget to update the rowCallback if you change the selected columns!
QString queryText =
QString("SELECT history.id, faux_offline_pending.id, timestamp, "
"chat.public_key, aliases.display_name, sender.public_key, "
"message FROM history "
"LEFT JOIN faux_offline_pending ON history.id = faux_offline_pending.id "
"JOIN peers chat ON chat_id = chat.id "
"JOIN aliases ON sender_alias = aliases.id "
"JOIN peers sender ON aliases.owner = sender.id "
"WHERE timestamp BETWEEN %1 AND %2 AND chat.public_key='%3'")
.arg(from.toMSecsSinceEpoch())
.arg(to.toMSecsSinceEpoch())
.arg(friendPk);
if (numMessages) {
queryText = "SELECT * FROM (" + queryText +
QString(" ORDER BY history.id DESC limit %1) AS T1 ORDER BY T1.id ASC;").arg(numMessages);
} else {
queryText = queryText + ";";
}
db->execNow({queryText, rowCallback});
return messages;
}

View File

@ -78,9 +78,9 @@ public:
const QDateTime& time, bool isSent, QString dispName,
const std::function<void(int64_t)>& insertIdCallback = {});
QList<HistMessage> getChatHistory(const QString& friendPk, const QDateTime& from,
QList<HistMessage> getChatHistoryFromDate(const QString& friendPk, const QDateTime& from,
const QDateTime& to);
QList<HistMessage> getChatHistoryDefaultNum(const QString& friendPk);
QList<DateMessages> getChatHistoryCounts(const ToxPk& friendPk, const QDate& from, const QDate& to);
QDateTime getDateWhereFindPhrase(const QString& friendPk, const QDateTime& from, QString phrase);
@ -93,6 +93,8 @@ protected:
QString dispName, std::function<void(int64_t)> insertIdCallback = {});
private:
QList<HistMessage> getChatHistory(const QString& friendPk, const QDateTime& from,
const QDateTime& to, int numMessages);
std::shared_ptr<RawDatabase> db;
QHash<QString, int64_t> peers;
};

View File

@ -63,9 +63,9 @@
* @brief stopNotification Tell others to stop notification of a call.
*/
static const int CHAT_WIDGET_MIN_HEIGHT = 50;
static const int SCREENSHOT_GRABBER_OPENING_DELAY = 500;
static const int TYPING_NOTIFICATION_DURATION = 3000;
static constexpr int CHAT_WIDGET_MIN_HEIGHT = 50;
static constexpr int SCREENSHOT_GRABBER_OPENING_DELAY = 500;
static constexpr int TYPING_NOTIFICATION_DURATION = 3000;
const QString ChatForm::ACTION_PREFIX = QStringLiteral("/me ");
@ -205,7 +205,7 @@ ChatForm::ChatForm(Friend* chatFriend, History* history)
updateCallButtons();
if (Nexus::getProfile()->isHistoryEnabled()) {
loadHistory(QDateTime::currentDateTime().addDays(-7), true);
loadHistoryDefaultNum(true);
}
setAcceptDrops(true);
@ -503,14 +503,14 @@ void ChatForm::onSearchUp(const QString& phrase)
if (startLine == 0) {
QString pk = f->getPublicKey().toString();
QDateTime newBaseData = history->getDateWhereFindPhrase(pk, earliestMessage, phrase);
QDateTime newBaseDate = history->getDateWhereFindPhrase(pk, earliestMessage, phrase);
if (!newBaseData.isValid()) {
if (!newBaseDate.isValid()) {
return;
}
searchAfterLoadHistory = true;
loadHistory(newBaseData);
loadHistoryByDateRange(newBaseDate);
return;
}
@ -519,15 +519,15 @@ void ChatForm::onSearchUp(const QString& phrase)
if (!isSearch) {
QString pk = f->getPublicKey().toString();
QDateTime newBaseData = history->getDateWhereFindPhrase(pk, earliestMessage, phrase);
QDateTime newBaseDate = history->getDateWhereFindPhrase(pk, earliestMessage, phrase);
if (!newBaseData.isValid()) {
if (!newBaseDate.isValid()) {
return;
}
searchPoint.setX(numLines);
searchAfterLoadHistory = true;
loadHistory(newBaseData);
loadHistoryByDateRange(newBaseDate);
}
}
@ -693,13 +693,6 @@ void ChatForm::clearChatArea(bool notInForm)
offlineEngine->removeAllReceipts();
}
void ChatForm::onLoadChatHistory()
{
if (sender() == f) {
loadHistory(QDateTime::currentDateTime().addDays(-7), true);
}
}
QString getMsgAuthorDispName(const ToxPk& authorPk, const QString& dispName)
{
QString authorStr;
@ -716,10 +709,19 @@ QString getMsgAuthorDispName(const ToxPk& authorPk, const QString& dispName)
return authorStr;
}
// TODO: Split on smaller methods (style)
void ChatForm::loadHistory(const QDateTime& since, bool processUndelivered)
void ChatForm::loadHistoryDefaultNum(bool processUndelivered)
{
QDateTime now = historyBaselineDate.addMSecs(-1);
QString pk = f->getPublicKey().toString();
QList<History::HistMessage> msgs = history->getChatHistoryDefaultNum(pk);
if (!msgs.isEmpty()) {
earliestMessage = msgs.back().timestamp;
}
handleLoadedMessages(msgs, processUndelivered);
}
void ChatForm::loadHistoryByDateRange(const QDateTime& since, bool processUndelivered)
{
QDateTime now = QDateTime::currentDateTime();
if (since > now) {
return;
}
@ -736,72 +738,96 @@ void ChatForm::loadHistory(const QDateTime& since, bool processUndelivered)
}
QString pk = f->getPublicKey().toString();
QList<History::HistMessage> msgs = history->getChatHistory(pk, since, now);
earliestMessage = since;
QList<History::HistMessage> msgs = history->getChatHistoryFromDate(pk, since, now);
handleLoadedMessages(msgs, processUndelivered);
}
void ChatForm::handleLoadedMessages(QList<History::HistMessage> newHistMsgs, bool processUndelivered)
{
ToxPk prevIdBackup = previousId;
previousId = ToxPk{};
QList<ChatLine::Ptr> historyMessages;
QList<ChatLine::Ptr> chatLines;
Core* core = Core::getInstance();
QDate lastDate(1, 0, 0);
for (const auto& it : msgs) {
// Show the date every new day
QDateTime msgDateTime = it.timestamp.toLocalTime();
QDate msgDate = msgDateTime.date();
if (msgDate > lastDate) {
lastDate = msgDate;
QString dateText = msgDate.toString(Settings::getInstance().getDateFormat());
auto msg = ChatMessage::createChatInfoMessage(dateText, ChatMessage::INFO, QDateTime());
historyMessages.append(msg);
for (const auto& histMessage : newHistMsgs) {
MessageMetadata const metadata = getMessageMetadata(histMessage);
lastDate = addDateLineIfNeeded(chatLines, lastDate, histMessage, metadata);
auto msg = chatMessageFromHistMessage(histMessage, metadata);
if (processUndelivered) {
sendLoadedMessage(msg, metadata);
}
// Show each messages
const Core* core = Core::getInstance();
ToxPk authorPk(ToxId(it.sender).getPublicKey());
QString authorStr = getMsgAuthorDispName(authorPk, it.dispName);
bool isSelf = authorPk == core->getSelfId().getPublicKey();
bool isAction = it.message.startsWith(ACTION_PREFIX, Qt::CaseInsensitive);
bool needSending = !it.isSent && isSelf;
QString messageText = isAction ? it.message.mid(ACTION_PREFIX.length()) : it.message;
ChatMessage::MessageType type = isAction ? ChatMessage::ACTION : ChatMessage::NORMAL;
QDateTime dateTime = needSending ? QDateTime() : msgDateTime;
auto msg = ChatMessage::createChatMessage(authorStr, messageText, type, isSelf, dateTime);
if (!isAction && needsToHideName(authorPk, msgDateTime)) {
msg->hideSender();
}
previousId = authorPk;
prevMsgDateTime = msgDateTime;
if (needSending && processUndelivered) {
Core* core = Core::getInstance();
uint32_t friendId = f->getId();
QString stringMsg = msg->toString();
int receipt = isAction ? core->sendAction(friendId, stringMsg)
: core->sendMessage(friendId, stringMsg);
getOfflineMsgEngine()->registerReceipt(receipt, it.id, msg);
}
historyMessages.append(msg);
chatLines.append(msg);
previousId = metadata.authorPk;
prevMsgDateTime = metadata.msgDateTime;
}
previousId = prevIdBackup;
earliestMessage = since;
QScrollBar* verticalBar = chatWidget->verticalScrollBar();
int savedSliderPos = verticalBar->maximum() - verticalBar->value();
chatWidget->insertChatlineOnTop(historyMessages);
savedSliderPos = verticalBar->maximum() - savedSliderPos;
verticalBar->setValue(savedSliderPos);
if (searchAfterLoadHistory && historyMessages.isEmpty()) {
insertChatlines(chatLines);
if (searchAfterLoadHistory && chatLines.isEmpty()) {
onContinueSearch();
}
}
void ChatForm::insertChatlines(QList<ChatLine::Ptr> chatLines)
{
QScrollBar* verticalBar = chatWidget->verticalScrollBar();
int savedSliderPos = verticalBar->maximum() - verticalBar->value();
chatWidget->insertChatlinesOnTop(chatLines);
savedSliderPos = verticalBar->maximum() - savedSliderPos;
verticalBar->setValue(savedSliderPos);
}
QDate ChatForm::addDateLineIfNeeded(QList<ChatLine::Ptr> msgs, QDate const& lastDate, History::HistMessage const& newMessage, MessageMetadata const& metadata)
{
// Show the date every new day
QDate newDate = metadata.msgDateTime.date();
if (newDate > lastDate) {
QString dateText = newDate.toString(Settings::getInstance().getDateFormat());
auto msg = ChatMessage::createChatInfoMessage(dateText, ChatMessage::INFO, QDateTime());
msgs.append(msg);
return newDate;
}
return lastDate;
}
ChatForm::MessageMetadata ChatForm::getMessageMetadata(History::HistMessage const& histMessage)
{
const ToxPk authorPk = ToxId(histMessage.sender).getPublicKey();
const QDateTime msgDateTime = histMessage.timestamp.toLocalTime();
const bool isSelf = Core::getInstance()->getSelfId().getPublicKey() == authorPk;
const bool needSending = !histMessage.isSent && isSelf;
const bool isAction = histMessage.message.startsWith(ACTION_PREFIX, Qt::CaseInsensitive);
const qint64 id = histMessage.id;
return {isSelf, needSending, isAction, id, authorPk, msgDateTime};
}
ChatMessage::Ptr ChatForm::chatMessageFromHistMessage(History::HistMessage const& histMessage, MessageMetadata const& metadata)
{
ToxPk authorPk(ToxId(histMessage.sender).getPublicKey());
QString authorStr = getMsgAuthorDispName(authorPk, histMessage.dispName);
QString messageText = metadata.isAction ? histMessage.message.mid(ACTION_PREFIX.length()) : histMessage.message;
ChatMessage::MessageType type = metadata.isAction ? ChatMessage::ACTION : ChatMessage::NORMAL;
QDateTime dateTime = metadata.needSending ? QDateTime() : metadata.msgDateTime;
auto msg = ChatMessage::createChatMessage(authorStr, messageText, type, metadata.isSelf, dateTime);
if (!metadata.isAction && needsToHideName(authorPk, metadata.msgDateTime)) {
msg->hideSender();
}
return msg;
}
void ChatForm::sendLoadedMessage(ChatMessage::Ptr chatMsg, MessageMetadata const& metadata)
{
if (!metadata.needSending) {
return;
}
Core* core = Core::getInstance();
uint32_t friendId = f->getId();
QString stringMsg = chatMsg->toString();
int receipt = metadata.isAction ? core->sendAction(friendId, stringMsg)
: core->sendMessage(friendId, stringMsg);
getOfflineMsgEngine()->registerReceipt(receipt, metadata.id, chatMsg);
}
void ChatForm::onScreenshotClicked()
{
doScreenshot();
@ -853,7 +879,7 @@ void ChatForm::onLoadHistory()
LoadHistoryDialog dlg(f->getPublicKey());
if (dlg.exec()) {
QDateTime fromTime = dlg.getFromDate();
loadHistory(fromTime);
loadHistoryByDateRange(fromTime);
}
}
@ -1022,7 +1048,7 @@ void ChatForm::onExportChat()
QString pk = f->getPublicKey().toString();
QDateTime epochStart = QDateTime::fromMSecsSinceEpoch(0);
QDateTime now = QDateTime::currentDateTime();
QList<History::HistMessage> msgs = history->getChatHistory(pk, epochStart, now);
QList<History::HistMessage> msgs = history->getChatHistoryFromDate(pk, epochStart, now);
QString path = QFileDialog::getSaveFileName(0, tr("Save chat log"), QString{}, QString{}, 0,
QFileDialog::DontUseNativeDialog);

View File

@ -27,6 +27,7 @@
#include "genericchatform.h"
#include "src/core/core.h"
#include "src/persistence/history.h"
#include "src/widget/tool/screenshotgrabber.h"
class CallConfirmWidget;
@ -45,7 +46,8 @@ public:
ChatForm(Friend* chatFriend, History* history);
~ChatForm();
void setStatusMessage(const QString& newMessage);
void loadHistory(const QDateTime& since, bool processUndelivered = false);
void loadHistoryByDateRange(const QDateTime& since, bool processUndelivered = false);
void loadHistoryDefaultNum(bool processUndelivered = false);
void dischargeReceipt(int receipt);
void setFriendTyping(bool isTyping);
@ -83,7 +85,6 @@ private slots:
void onAttachClicked() override;
void onScreenshotClicked() override;
void onLoadChatHistory();
void onTextEditChanged();
void onCallTriggered();
void onVideoCallTriggered();
@ -107,6 +108,27 @@ private slots:
void onExportChat();
private:
struct MessageMetadata {
const bool isSelf;
const bool needSending;
const bool isAction;
const qint64 id;
const ToxPk authorPk;
const QDateTime msgDateTime;
MessageMetadata(bool isSelf, bool needSending, bool isAction, qint64 id, ToxPk authorPk, QDateTime msgDateTime) :
needSending{needSending},
isSelf{isSelf},
isAction{isAction},
id{id},
authorPk{authorPk},
msgDateTime{msgDateTime} {}
};
void handleLoadedMessages(QList<History::HistMessage> newHistMsgs, bool processUndelivered);
QDate addDateLineIfNeeded(QList<ChatLine::Ptr> msgs, QDate const& lastDate, History::HistMessage const& newMessage, MessageMetadata const& metadata);
MessageMetadata getMessageMetadata(History::HistMessage const& histMessage);
ChatMessage::Ptr chatMessageFromHistMessage(History::HistMessage const& histMessage, MessageMetadata const& metadata);
void sendLoadedMessage(ChatMessage::Ptr chatMsg, MessageMetadata const& metadata);
void insertChatlines(QList<ChatLine::Ptr> chatLines);
void updateMuteMicButton();
void updateMuteVolButton();
void retranslateUi();

View File

@ -50,11 +50,6 @@
* @class GenericChatForm
* @brief Parent class for all chatforms. It's provide the minimum required UI
* elements and methods to work with chat messages.
*
* TODO: reword
* @var GenericChatForm::historyBaselineDate
* @brief Used by HistoryKeeper to load messages from t to historyBaselineDate
* (excluded)
*/
#define SET_STYLESHEET(x) (x)->setStyleSheet(Style::getStylesheet(":/ui/" #x "/" #x ".css"))
@ -670,7 +665,6 @@ void GenericChatForm::clearChatArea(bool notinform)
addSystemInfoMessage(tr("Cleared"), ChatMessage::INFO, QDateTime::currentDateTime());
earliestMessage = QDateTime(); // null
historyBaselineDate = QDateTime::currentDateTime();
emit chatAreaCleared();
}

View File

@ -151,7 +151,6 @@ protected:
QDateTime prevMsgDateTime;
QDateTime earliestMessage;
QDateTime historyBaselineDate = QDateTime::currentDateTime();
QMenu menu;

View File

@ -933,9 +933,8 @@ void Widget::setStatusMessage(const QString& statusMessage)
void Widget::reloadHistory()
{
QDateTime weekAgo = QDateTime::currentDateTime().addDays(-7);
for (auto f : FriendList::getAllFriends()) {
chatForms[f->getId()]->loadHistory(weekAgo, true);
chatForms[f->getId()]->loadHistoryDefaultNum(true);
}
}