feat(messages): Multipacket message support

* Introduced ToxExt and CoreExt abstraction
    * Along with interfaces for mocking and unit testing
* Add "supportedExtensions" concept to Friend
* Dispatch messages to CoreExt instead of Core when friend supports
extended messages
    * Only split messages for core when extended messages are unavailable
* Offline message engine/History not altered. Currently only valid for
an existing session after extension negotiation has completed

CMakeLists.txt

@@ -218,6 +218,8 @@ set(${PROJECT_NAME}_SOURCES
   src/chatlog/textformatter.h
   src/core/coreav.cpp
   src/core/coreav.h
+  src/core/coreext.cpp
+  src/core/coreext.h
   src/core/core.cpp
   src/core/corefile.cpp
   src/core/corefile.h

cmake/Dependencies.cmake

@@ -33,6 +33,9 @@ find_package(Qt5Test          REQUIRED)
 find_package(Qt5Widgets       REQUIRED)
 find_package(Qt5Xml           REQUIRED)
 
+find_package(ToxExt                         REQUIRED)
+find_package(ToxExtensionMessages           REQUIRED)
+
 function(add_dependency)
   set(ALL_LIBRARIES ${ALL_LIBRARIES} ${ARGN} PARENT_SCOPE)
 endfunction()
@@ -47,6 +50,10 @@ add_dependency(
   Qt5::Widgets
   Qt5::Xml)
 
+add_dependency(
+  ToxExt::ToxExt
+  ToxExtensionMessages::ToxExtensionMessages)
+
 include(CMakeParseArguments)
 
 function(search_dependency pkg)

src/core/core.cpp

@@ -21,6 +21,8 @@
 #include "core.h"
 #include "coreav.h"
 #include "corefile.h"
+
+#include "src/core/coreext.h"
 #include "src/core/dhtserver.h"
 #include "src/core/icoresettings.h"
 #include "src/core/toxlogger.h"
@@ -515,6 +517,7 @@ void Core::registerCallbacks(Tox* tox)
     tox_callback_conference_peer_list_changed(tox, onGroupPeerListChange);
     tox_callback_conference_peer_name(tox, onGroupPeerNameChange);
     tox_callback_conference_title(tox, onGroupTitleChange);
+    tox_callback_friend_lossless_packet(tox, onLosslessPacket);
 }
 
 /**
@@ -639,6 +642,9 @@ ToxCorePtr Core::makeToxCore(const QByteArray& savedata, const ICoreSettings* co
         return {};
     }
 
+    core->ext = CoreExt::makeCoreExt(core->tox.get());
+    connect(core.get(), &Core::friendStatusChanged, core->ext.get(), &CoreExt::onFriendStatusChanged);
+
     registerCallbacks(core->tox.get());
 
     // connect the thread with the Core
@@ -714,6 +720,16 @@ QMutex &Core::getCoreLoopLock() const
     return coreLoopLock;
 }
 
+const CoreExt* Core::getExt() const
+{
+    return ext.get();
+}
+
+CoreExt* Core::getExt()
+{
+    return ext.get();
+}
+
 /* Using the now commented out statements in checkConnection(), I watched how
  * many ticks disconnects-after-initial-connect lasted. Out of roughly 15 trials,
  * 5 disconnected; 4 were DCd for less than 20 ticks, while the 5th was ~50 ticks.
@@ -734,6 +750,7 @@ void Core::process()
 
     static int tolerance = CORE_DISCONNECT_TOLERANCE;
     tox_iterate(tox.get(), this);
+    ext->process();
 
 #ifdef DEBUG
     // we want to see the debug messages immediately
@@ -988,6 +1005,16 @@ void Core::onGroupTitleChange(Tox*, uint32_t groupId, uint32_t peerId, const uin
     emit core->groupTitleChanged(groupId, author, ToxString(cTitle, length).getQString());
 }
 
+/**
+ * @brief Handling of custom lossless packets received by toxcore. Currently only used to forward toxext packets to CoreExt
+ */
+void Core::onLosslessPacket(Tox*, uint32_t friendId,
+                            const uint8_t* data, size_t length, void* vCore)
+{
+    Core* core = static_cast<Core*>(vCore);
+    core->ext->onLosslessPacket(friendId, data, length);
+}
+
 void Core::onReadReceiptCallback(Tox*, uint32_t friendId, uint32_t receipt, void* core)
 {
     emit static_cast<Core*>(core)->receiptRecieved(friendId, ReceiptNum{receipt});

src/core/core.h

@@ -44,6 +44,7 @@
 
 class CoreAV;
 class CoreFile;
+class CoreExt;
 class IAudioControl;
 class ICoreSettings;
 class GroupInvite;
@@ -79,6 +80,8 @@ public:
     Tox* getTox() const;
     QMutex& getCoreLoopLock() const;
 
+    const CoreExt* getExt() const;
+    CoreExt* getExt();
     ~Core();
 
     static const QString TOX_EXT;
@@ -215,6 +218,9 @@ private:
                                       size_t length, void* core);
     static void onGroupTitleChange(Tox* tox, uint32_t groupId, uint32_t peerId,
                                    const uint8_t* cTitle, size_t length, void* vCore);
+
+    static void onLosslessPacket(Tox* tox, uint32_t friendId,
+                                   const uint8_t* data, size_t length, void* core);
     static void onReadReceiptCallback(Tox* tox, uint32_t friendId, uint32_t receipt, void* core);
 
     void sendGroupMessageWithType(int groupId, const QString& message, Tox_Message_Type type);
@@ -249,6 +255,7 @@ private:
 
     std::unique_ptr<CoreFile> file;
     CoreAV* av = nullptr;
+    std::unique_ptr<CoreExt> ext;
     QTimer* toxTimer = nullptr;
     // recursive, since we might call our own functions
     mutable QMutex coreLoopLock{QMutex::Recursive};

src/core/coreext.cpp

@@ -0,0 +1,159 @@
+/*
+    Copyright © 2019-2020 by The qTox Project Contributors
+
+    This file is part of qTox, a Qt-based graphical interface for Tox.
+
+    qTox is libre software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    qTox is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with qTox.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "coreext.h"
+#include "toxstring.h"
+
+#include <QDateTime>
+#include <QTimeZone>
+#include <QtCore>
+
+#include <memory>
+#include <cassert>
+
+extern "C" {
+#include <toxext/toxext.h>
+#include <tox_extension_messages.h>
+}
+
+std::unique_ptr<CoreExt> CoreExt::makeCoreExt(Tox* core) {
+    auto toxExtPtr = toxext_init(core);
+    if (!toxExtPtr) {
+        return nullptr;
+    }
+
+    auto toxExt = ExtensionPtr<ToxExt>(toxExtPtr, toxext_free);
+    return std::unique_ptr<CoreExt>(new CoreExt(std::move(toxExt)));
+}
+
+CoreExt::CoreExt(ExtensionPtr<ToxExt> toxExt_)
+    : toxExt(std::move(toxExt_))
+    , toxExtMessages(nullptr, nullptr)
+{
+    toxExtMessages = ExtensionPtr<ToxExtensionMessages>(
+        tox_extension_messages_register(
+            toxExt.get(),
+            CoreExt::onExtendedMessageReceived,
+            CoreExt::onExtendedMessageReceipt,
+            CoreExt::onExtendedMessageNegotiation,
+            this,
+            TOX_EXTENSION_MESSAGES_DEFAULT_MAX_RECEIVING_MESSAGE_SIZE),
+        tox_extension_messages_free);
+}
+
+void CoreExt::process()
+{
+    toxext_iterate(toxExt.get());
+}
+
+void CoreExt::onLosslessPacket(uint32_t friendId, const uint8_t* data, size_t length)
+{
+    if (is_toxext_packet(data, length)) {
+        toxext_handle_lossless_custom_packet(toxExt.get(), friendId, data, length);
+    }
+}
+
+CoreExt::Packet::Packet(
+    ToxExtPacketList* packetList,
+    ToxExtensionMessages* toxExtMessages,
+    uint32_t friendId,
+    PacketPassKey)
+    : toxExtMessages(toxExtMessages)
+    , packetList(packetList)
+    , friendId(friendId)
+{}
+
+std::unique_ptr<ICoreExtPacket> CoreExt::getPacket(uint32_t friendId)
+{
+    return std::unique_ptr<Packet>(new Packet(
+        toxext_packet_list_create(toxExt.get(), friendId),
+        toxExtMessages.get(),
+        friendId,
+        PacketPassKey{}));
+}
+
+uint64_t CoreExt::Packet::addExtendedMessage(QString message)
+{
+    if (hasBeenSent) {
+        assert(false);
+        qWarning() << "Invalid use of CoreExt::Packet";
+        // Hope that UINT64_MAX will never collide with an actual receipt num
+        // that we care about
+        return UINT64_MAX;
+    }
+
+    ToxString toxString(message);
+    Tox_Extension_Messages_Error err;
+
+    return tox_extension_messages_append(
+        toxExtMessages,
+        packetList,
+        toxString.data(),
+        toxString.size(),
+        friendId,
+        &err);
+}
+
+bool CoreExt::Packet::send()
+{
+    auto ret = toxext_send(packetList);
+    if (ret != TOXEXT_SUCCESS) {
+        qWarning() << "Failed to send packet";
+    }
+    // Indicate we've sent the packet even on failure since our packetlist will
+    // be invalid no matter what
+    hasBeenSent = true;
+    return ret == TOXEXT_SUCCESS;
+}
+
+void CoreExt::onFriendStatusChanged(uint32_t friendId, Status::Status status)
+{
+    const auto prevStatusIt = currentStatuses.find(friendId);
+    const auto prevStatus = prevStatusIt == currentStatuses.end()
+        ? Status::Status::Offline : prevStatusIt->second;
+
+    currentStatuses[friendId] = status;
+
+    // Does not depend on prevStatus since prevStatus could be newly
+    // constructed. In this case we should still ensure the rest of the system
+    // knows there is no extension support
+    if (status == Status::Status::Offline) {
+        emit extendedMessageSupport(friendId, false);
+    } else if (prevStatus == Status::Status::Offline) {
+        tox_extension_messages_negotiate(toxExtMessages.get(), friendId);
+    }
+}
+
+void CoreExt::onExtendedMessageReceived(uint32_t friendId, const uint8_t* data, size_t size, void* userData)
+{
+    QString msg = ToxString(data, size).getQString();
+    emit static_cast<CoreExt*>(userData)->extendedMessageReceived(friendId, msg);
+}
+
+void CoreExt::onExtendedMessageReceipt(uint32_t friendId, uint64_t receiptId, void* userData)
+{
+    emit static_cast<CoreExt*>(userData)->extendedReceiptReceived(friendId, receiptId);
+}
+
+void CoreExt::onExtendedMessageNegotiation(uint32_t friendId, bool compatible, uint64_t maxMessageSize, void* userData)
+{
+    auto coreExt = static_cast<CoreExt*>(userData);
+    emit coreExt->extendedMessageSupport(friendId, compatible);
+}
+

src/core/coreext.h

@@ -0,0 +1,145 @@
+/*
+    Copyright © 2019-2020 by The qTox Project Contributors
+
+    This file is part of qTox, a Qt-based graphical interface for Tox.
+
+    qTox is libre software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    qTox is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with qTox.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "src/model/status.h"
+#include "icoreextpacket.h"
+
+#include <QObject>
+#include <QMap>
+
+#include <bitset>
+#include <memory>
+#include <unordered_map>
+
+struct Tox;
+struct ToxExt;
+struct ToxExtensionMessages;
+struct ToxExtPacketList;
+
+/**
+ * Bridge between the toxext library and the rest of qTox.
+ */
+class CoreExt : public QObject, public ICoreExtPacketAllocator
+{
+    Q_OBJECT
+private:
+    // PassKey idiom to prevent others from making PacketBuilders
+    struct PacketPassKey {};
+public:
+
+    /**
+     * @brief Creates a CoreExt instance. Using a pointer here makes our
+     *  registrations with extensions significantly easier to manage
+     *
+     * @param[in] pointer to core tox instance
+     * @return CoreExt on success, nullptr on failure
+     */
+    static std::unique_ptr<CoreExt> makeCoreExt(Tox* core);
+
+    // We do registration with our own pointer, need to ensure we're in a stable location
+    CoreExt(CoreExt const& other) = delete;
+    CoreExt(CoreExt&& other) = delete;
+    CoreExt& operator=(CoreExt const& other) = delete;
+    CoreExt& operator=(CoreExt&& other) = delete;
+
+    /**
+     * @brief Periodic service function
+     */
+    void process();
+
+    /**
+     * @brief Handles extension related lossless packets
+     * @param[in] friendId Core id of friend
+     * @param[in] data Packet data
+     * @param[in] length Length of packet data
+     */
+    void onLosslessPacket(uint32_t friendId, const uint8_t* data, size_t length);
+
+    /**
+     * See documentation of ICoreExtPacket
+     */
+    class Packet : public ICoreExtPacket
+    {
+    public:
+        /**
+         * @brief Internal constructor for a packet.
+         */
+        Packet(
+            ToxExtPacketList* packetList,
+            ToxExtensionMessages* toxExtMessages,
+            uint32_t friendId,
+            PacketPassKey);
+
+        // Delete copy constructor, we shouldn't be able to copy
+        Packet(Packet const& other) = delete;
+
+        Packet(Packet&& other)
+        {
+            toxExtMessages = other.toxExtMessages;
+            packetList = other.packetList;
+            friendId = other.friendId;
+            hasBeenSent = other.hasBeenSent;
+            other.toxExtMessages = nullptr;
+            other.packetList = nullptr;
+            other.friendId = 0;
+            other.hasBeenSent = false;
+        }
+
+        uint64_t addExtendedMessage(QString message) override;
+
+        bool send() override;
+    private:
+        bool hasBeenSent = false;
+        // Note: non-owning pointer
+        ToxExtensionMessages* toxExtMessages;
+        // Note: packetList is freed on send() call
+        ToxExtPacketList* packetList;
+        uint32_t friendId;
+    };
+
+    std::unique_ptr<ICoreExtPacket> getPacket(uint32_t friendId) override;
+
+signals:
+    void extendedMessageReceived(uint32_t friendId, const QString& message);
+    void extendedReceiptReceived(uint32_t friendId, uint64_t receiptId);
+    void extendedMessageSupport(uint32_t friendId, bool supported);
+
+public slots:
+    void onFriendStatusChanged(uint32_t friendId, Status::Status status);
+
+private:
+
+    static void onExtendedMessageReceived(uint32_t friendId, const uint8_t* data, size_t size, void* userData);
+    static void onExtendedMessageReceipt(uint32_t friendId, uint64_t receiptId, void* userData);
+    static void onExtendedMessageNegotiation(uint32_t friendId, bool compatible, uint64_t maxMessageSize, void* userData);
+
+    // A little extra cost to hide the deleters, but this lets us fwd declare
+    // and prevent any extension headers from leaking out to the rest of the
+    // system
+    template <class T>
+    using ExtensionPtr = std::unique_ptr<T, void(*)(T*)>;
+
+    CoreExt(ExtensionPtr<ToxExt> toxExt);
+
+    std::unordered_map<uint32_t, Status::Status> currentStatuses;
+    ExtensionPtr<ToxExt> toxExt;
+    ExtensionPtr<ToxExtensionMessages> toxExtMessages;
+};

src/core/extension.h

@@ -0,0 +1,32 @@
+/*
+    Copyright © 2019 by The qTox Project Contributors
+
+    This file is part of qTox, a Qt-based graphical interface for Tox.
+
+    qTox is libre software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    qTox is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with qTox.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <bitset>
+
+// Do not use enum class because we use these as indexes frequently (see ExtensionSet)
+struct ExtensionType
+{
+    enum {
+        messages,
+        max
+    };
+};
+using ExtensionSet = std::bitset<ExtensionType::max>;

src/core/icoreextpacket.h

@@ -0,0 +1,68 @@
+/*
+    Copyright © 2019 by The qTox Project Contributors
+
+    This file is part of qTox, a Qt-based graphical interface for Tox.
+
+    qTox is libre software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    qTox is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with qTox.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <QDateTime>
+
+#include <cstdint>
+#include <memory>
+
+/**
+ * Abstraction around the toxext packet. The toxext flow is to allow several extensions
+ * to tack onto the same packet before sending it to avoid needing the toxext overhead
+ * for every single extension. This abstraction models a toxext packet list.
+ *
+ * Intent is to retrieve a ICoreExtPacket from an ICoreExtPacketAllocator, append all
+ * relevant extension data, and then finally send the packet. After sending the packet
+ * is no longer guaranteed to be valid.
+ */
+class ICoreExtPacket
+{
+public:
+    virtual ~ICoreExtPacket() = default;
+
+    /**
+     * @brief Adds message to packet
+     * @return Extended message receipt, UINT64_MAX on failure
+     * @note Any other extensions related to this message have to be added
+     * _before_ the message itself
+     */
+    virtual uint64_t addExtendedMessage(QString message) = 0;
+
+    /**
+     * @brief Consumes the packet constructed with PacketBuilder packet and
+     * sends it to toxext
+     */
+    virtual bool send() = 0;
+};
+
+/**
+ * Provider of toxext packets
+ */
+class ICoreExtPacketAllocator
+{
+public:
+    virtual ~ICoreExtPacketAllocator() = default;
+
+    /**
+     * @brief Gets a new packet builder for friend with core friend id friendId
+     */
+    virtual std::unique_ptr<ICoreExtPacket> getPacket(uint32_t friendId) = 0;
+};

src/model/friend.cpp

@@ -173,3 +173,14 @@ bool Friend::useHistory() const
 {
     return true;
 }
+
+void Friend::setExtendedMessageSupport(bool supported)
+{
+    supportedExtensions[ExtensionType::messages] = supported;
+    emit extensionSupportChanged(supportedExtensions);
+}
+
+ExtensionSet Friend::getSupportedExtensions() const
+{
+    return supportedExtensions;
+}

src/model/friend.h

@@ -21,6 +21,7 @@
 
 #include "contact.h"
 #include "src/core/core.h"
+#include "src/core/extension.h"
 #include "src/core/toxid.h"
 #include "src/core/contactid.h"
 #include "src/model/status.h"
@@ -50,20 +51,24 @@ public:
     uint32_t getId() const override;
     const ContactId& getPersistentId() const override;
 
+    void finishNegotiation();
     void setStatus(Status::Status s);
     Status::Status getStatus() const;
     bool useHistory() const final;
 
+    void setExtendedMessageSupport(bool supported);
+    ExtensionSet getSupportedExtensions() const;
+
 signals:
     void nameChanged(const ToxPk& friendId, const QString& name);
     void aliasChanged(const ToxPk& friendId, QString alias);
     void statusChanged(const ToxPk& friendId, Status::Status status);
     void onlineOfflineChanged(const ToxPk& friendId, bool isOnline);
     void statusMessageChanged(const ToxPk& friendId, const QString& message);
+    void extensionSupportChanged(ExtensionSet extensions);
     void loadChatHistory();
 
 public slots:
-
 private:
     QString userName;
     QString userAlias;
@@ -72,4 +77,5 @@ private:
     uint32_t friendId;
     bool hasNewEvents;
     Status::Status friendStatus;
+    ExtensionSet supportedExtensions;
 };

src/model/friendmessagedispatcher.cpp

@@ -21,7 +21,6 @@
 #include "src/persistence/settings.h"
 #include "src/model/status.h"
 
-
 namespace {
 
 /**
@@ -43,12 +42,15 @@ bool sendMessageToCore(ICoreFriendMessageSender& messageSender, const Friend& f,
 }
 } // namespace
 
+
 FriendMessageDispatcher::FriendMessageDispatcher(Friend& f_, MessageProcessor processor_,
-                                                 ICoreFriendMessageSender& messageSender_)
+                                                 ICoreFriendMessageSender& messageSender_,
+                                                 ICoreExtPacketAllocator& coreExtPacketAllocator_)
     : f(f_)
     , messageSender(messageSender_)
     , offlineMsgEngine(&f_, &messageSender_)
     , processor(std::move(processor_))
+    , coreExtPacketAllocator(coreExtPacketAllocator_)
 {
     connect(&f, &Friend::onlineOfflineChanged, this, &FriendMessageDispatcher::onFriendOnlineOfflineChanged);
 }
@@ -61,7 +63,9 @@ FriendMessageDispatcher::sendMessage(bool isAction, const QString& content)
 {
     const auto firstId = nextMessageId;
     auto lastId = nextMessageId;
-    for (const auto& message : processor.processOutgoingMessage(isAction, content)) {
+    auto supportedExtensions = f.getSupportedExtensions();
+    const bool needsSplit = !supportedExtensions[ExtensionType::messages];
+    for (const auto& message : processor.processOutgoingMessage(isAction, content, needsSplit)) {
         auto messageId = nextMessageId++;
         lastId = messageId;
         auto onOfflineMsgComplete = [this, messageId] { emit this->messageComplete(messageId); };
@@ -70,9 +74,23 @@ FriendMessageDispatcher::sendMessage(bool isAction, const QString& content)
 
         bool messageSent = false;
 
+        // NOTE: This branch is getting a little hairy but will be cleaned up in the following commit
         if (Status::isOnline(f.getStatus())) {
+
+            // Action messages go over the regular mesage channel so we cannot use extensions with them
+            if (supportedExtensions[ExtensionType::messages] && !isAction) {
+                auto packet = coreExtPacketAllocator.getPacket(f.getId());
+
+                if (supportedExtensions[ExtensionType::messages]) {
+                    // NOTE: Dirty hack to get extensions working that will be fixed in the following commit
+                    receipt.get() = packet->addExtendedMessage(message.content);
+                }
+
+                messageSent = packet->send();
+            } else {
                 messageSent = sendMessageToCore(messageSender, f, message, receipt);
             }
+        }
 
         if (!messageSent) {
             offlineMsgEngine.addUnsentMessage(message, onOfflineMsgComplete);
@@ -92,7 +110,7 @@ FriendMessageDispatcher::sendMessage(bool isAction, const QString& content)
  */
 void FriendMessageDispatcher::onMessageReceived(bool isAction, const QString& content)
 {
-    emit this->messageReceived(f.getPublicKey(), processor.processIncomingMessage(isAction, content));
+    emit this->messageReceived(f.getPublicKey(), processor.processIncomingCoreMessage(isAction, content));
 }
 
 /**
@@ -104,6 +122,18 @@ void FriendMessageDispatcher::onReceiptReceived(ReceiptNum receipt)
     offlineMsgEngine.onReceiptReceived(receipt);
 }
 
+void FriendMessageDispatcher::onExtMessageReceived(const QString& content)
+{
+    auto message = processor.processIncomingExtMessage(content);
+    emit this->messageReceived(f.getPublicKey(), message);
+}
+
+void FriendMessageDispatcher::onExtReceiptReceived(uint64_t receiptId)
+{
+    // NOTE: Reusing ReceiptNum is a dirty hack that will be cleaned up in the following commit
+    offlineMsgEngine.onReceiptReceived(ReceiptNum(receiptId));
+}
+
 /**
  * @brief Handles status change for friend
  * @note Parameters just to fit slot api

src/model/friendmessagedispatcher.h

@@ -35,18 +35,22 @@ class FriendMessageDispatcher : public IMessageDispatcher
     Q_OBJECT
 public:
     FriendMessageDispatcher(Friend& f, MessageProcessor processor,
-                            ICoreFriendMessageSender& messageSender);
+                            ICoreFriendMessageSender& messageSender,
+                            ICoreExtPacketAllocator& coreExt);
 
     std::pair<DispatchedMessageId, DispatchedMessageId> sendMessage(bool isAction,
                                                                     const QString& content) override;
     void onMessageReceived(bool isAction, const QString& content);
     void onReceiptReceived(ReceiptNum receipt);
+    void onExtMessageReceived(const QString& message);
+    void onExtReceiptReceived(uint64_t receiptId);
     void clearOutgoingMessages();
 private slots:
     void onFriendOnlineOfflineChanged(const ToxPk& key, bool isOnline);
 
 private:
     Friend& f;
+    ICoreExtPacketAllocator& coreExtPacketAllocator;
     DispatchedMessageId nextMessageId = DispatchedMessageId(0);
 
     ICoreFriendMessageSender& messageSender;

src/model/groupmessagedispatcher.cpp

@@ -41,7 +41,7 @@ GroupMessageDispatcher::sendMessage(bool isAction, QString const& content)
     const auto firstMessageId = nextMessageId;
     auto lastMessageId = firstMessageId;
 
-    for (auto const& message : processor.processOutgoingMessage(isAction, content)) {
+    for (auto const& message : processor.processOutgoingMessage(isAction, content, true /*needsSplit*/)) {
         auto messageId = nextMessageId++;
         lastMessageId = messageId;
         if (group.getPeersCount() != 1) {
@@ -84,5 +84,5 @@ void GroupMessageDispatcher::onMessageReceived(const ToxPk& sender, bool isActio
         return;
     }
 
-    emit messageReceived(sender, processor.processIncomingMessage(isAction, content));
+    emit messageReceived(sender, processor.processIncomingCoreMessage(isAction, content));
 }

src/model/message.cpp

@@ -21,6 +21,8 @@
 #include "friend.h"
 #include "src/core/core.h"
 
+#include <cassert>
+
 void MessageProcessor::SharedParams::onUserNameSet(const QString& username)
 {
     QString sanename = username;
@@ -49,11 +51,14 @@ MessageProcessor::MessageProcessor(const MessageProcessor::SharedParams& sharedP
 /**
  * @brief Converts an outgoing message into one (or many) sanitized Message(s)
  */
-std::vector<Message> MessageProcessor::processOutgoingMessage(bool isAction, QString const& content)
+std::vector<Message> MessageProcessor::processOutgoingMessage(bool isAction, QString const& content, bool needsSplit)
 {
     std::vector<Message> ret;
 
-    QStringList splitMsgs = Core::splitMessage(content);
+    const auto splitMsgs = needsSplit
+        ? Core::splitMessage(content)
+        : QStringList({content});
+
     ret.reserve(splitMsgs.size());
 
     QDateTime timestamp = QDateTime::currentDateTime();
@@ -69,11 +74,10 @@ std::vector<Message> MessageProcessor::processOutgoingMessage(bool isAction, QSt
     return ret;
 }
 
-
 /**
  * @brief Converts an incoming message into a sanitized Message
  */
-Message MessageProcessor::processIncomingMessage(bool isAction, QString const& message)
+Message MessageProcessor::processIncomingCoreMessage(bool isAction, QString const& message)
 {
     QDateTime timestamp = QDateTime::currentDateTime();
     auto ret = Message{};
@@ -109,3 +113,17 @@ Message MessageProcessor::processIncomingMessage(bool isAction, QString const& m
 
     return ret;
 }
+
+Message MessageProcessor::processIncomingExtMessage(const QString& content)
+{
+    // Note: detectingMentions not implemented here since mentions are only
+    // currently useful in group messages which do not support extensions. If we
+    // were to support mentions we would probably want to do something more
+    // intelligent anyways
+    assert(detectingMentions == false);
+    auto message = Message();
+    message.timestamp = QDateTime::currentDateTime();
+    message.content = content;
+
+    return message;
+}

src/model/message.h

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include "src/core/coreext.h"
+
 #include <QDateTime>
 #include <QRegularExpression>
 #include <QString>
@@ -26,6 +28,7 @@
 #include <vector>
 
 class Friend;
+class CoreExt;
 
 // NOTE: This could be extended in the future to handle all text processing (see
 // ChatMessage::createChatMessage)
@@ -89,9 +92,9 @@ public:
 
     MessageProcessor(const SharedParams& sharedParams);
 
-    std::vector<Message> processOutgoingMessage(bool isAction, QString const& content);
+    std::vector<Message> processOutgoingMessage(bool isAction, const QString& content, bool needsSplit);
-
+    Message processIncomingCoreMessage(bool isAction, const QString& content);
-    Message processIncomingMessage(bool isAction, QString const& message);
+    Message processIncomingExtMessage(const QString& content);
 
     /**
      * @brief Enables mention detection in the processor

src/nexus.cpp

@@ -108,6 +108,8 @@ void Nexus::start()
     qRegisterMetaType<GroupInvite>("GroupInvite");
     qRegisterMetaType<ReceiptNum>("ReceiptNum");
     qRegisterMetaType<RowId>("RowId");
+    qRegisterMetaType<uint64_t>("uint64_t");
+    qRegisterMetaType<ExtensionSet>("ExtensionSet");
 
     qApp->setQuitOnLastWindowClosed(false);
 

src/widget/widget.cpp

@@ -695,6 +695,13 @@ void Widget::onCoreChanged(Core& core)
     connect(&core, &Core::friendTypingChanged, this, &Widget::onFriendTypingChanged);
     connect(&core, &Core::groupSentFailed, this, &Widget::onGroupSendFailed);
     connect(&core, &Core::usernameSet, this, &Widget::refreshPeerListsLocal);
+
+    auto coreExt = core.getExt();
+
+    connect(coreExt, &CoreExt::extendedMessageReceived, this, &Widget::onFriendExtMessageReceived);
+    connect(coreExt, &CoreExt::extendedReceiptReceived, this, &Widget::onExtReceiptReceived);
+    connect(coreExt, &CoreExt::extendedMessageSupport, this, &Widget::onExtendedMessageSupport);
+
     connect(this, &Widget::statusSet, &core, &Core::setStatus);
     connect(this, &Widget::friendRequested, &core, &Core::requestFriendship);
     connect(this, &Widget::friendRequestAccepted, &core, &Core::acceptFriendRequest);
@@ -1146,7 +1153,7 @@ void Widget::addFriend(uint32_t friendId, const ToxPk& friendPk)
 
     auto messageProcessor = MessageProcessor(sharedMessageProcessorParams);
     auto friendMessageDispatcher =
-        std::make_shared<FriendMessageDispatcher>(*newfriend, std::move(messageProcessor), *core);
+        std::make_shared<FriendMessageDispatcher>(*newfriend, std::move(messageProcessor), *core, *core->getExt());
 
     // Note: We do not have to connect the message dispatcher signals since
     // ChatHistory hooks them up in a very specific order
@@ -1401,6 +1408,39 @@ void Widget::onReceiptReceived(int friendId, ReceiptNum receipt)
     friendMessageDispatchers[f->getPublicKey()]->onReceiptReceived(receipt);
 }
 
+void Widget::onExtendedMessageSupport(uint32_t friendNumber, bool compatible)
+{
+    const auto& friendKey = FriendList::id2Key(friendNumber);
+    Friend* f = FriendList::findFriend(friendKey);
+    if (!f) {
+        return;
+    }
+
+    f->setExtendedMessageSupport(compatible);
+}
+
+void Widget::onFriendExtMessageReceived(uint32_t friendNumber, const QString& message)
+{
+    const auto& friendKey = FriendList::id2Key(friendNumber);
+    Friend* f = FriendList::findFriend(friendKey);
+    if (!f) {
+        return;
+    }
+
+    friendMessageDispatchers[f->getPublicKey()]->onExtMessageReceived(message);
+}
+
+void Widget::onExtReceiptReceived(uint32_t friendNumber, uint64_t receiptId)
+{
+    const auto& friendKey = FriendList::id2Key(friendNumber);
+    Friend* f = FriendList::findFriend(friendKey);
+    if (!f) {
+        return;
+    }
+
+    friendMessageDispatchers[f->getPublicKey()]->onExtReceiptReceived(receiptId);
+}
+
 void Widget::addFriendDialog(const Friend* frnd, ContentDialog* dialog)
 {
     const ToxPk& friendPk = frnd->getPublicKey();

src/widget/widget.h

@@ -172,6 +172,9 @@ public slots:
     void onFriendAliasChanged(const ToxPk& friendId, const QString& alias);
     void onFriendMessageReceived(uint32_t friendnumber, const QString& message, bool isAction);
     void onReceiptReceived(int friendId, ReceiptNum receipt);
+    void onExtendedMessageSupport(uint32_t friendNumber, bool supported);
+    void onFriendExtMessageReceived(uint32_t friendNumber, const QString& message);
+    void onExtReceiptReceived(uint32_t friendNumber, uint64_t receiptId);
     void onFriendRequestReceived(const ToxPk& friendPk, const QString& message);
     void onFileReceiveRequested(const ToxFile& file);
     void onEmptyGroupCreated(uint32_t groupnumber, const GroupId& groupId, const QString& title);
@@ -344,6 +347,7 @@ private:
     QMap<ToxPk, std::shared_ptr<ChatHistory>> friendChatLogs;
     QMap<ToxPk, std::shared_ptr<FriendChatroom>> friendChatrooms;
     QMap<ToxPk, ChatForm*> chatForms;
+    std::map<ToxPk, std::unique_ptr<QTimer>> negotiateTimers;
 
     QMap<GroupId, GroupWidget*> groupWidgets;
     QMap<GroupId, std::shared_ptr<GroupMessageDispatcher>> groupMessageDispatchers;

test/model/friendmessagedispatcher_test.cpp

@@ -28,6 +28,45 @@
 #include <deque>
 
 
+class MockCoreExtPacket : public ICoreExtPacket
+{
+public:
+
+    MockCoreExtPacket(uint64_t& numSentMessages, uint64_t& currentReceiptId)
+        : numSentMessages(numSentMessages)
+        , currentReceiptId(currentReceiptId)
+    {}
+
+    uint64_t addExtendedMessage(QString message) override
+    {
+        this->message = message;
+        return currentReceiptId++;
+    }
+
+    bool send() override
+    {
+        numSentMessages++;
+        return true;
+    }
+
+    uint64_t& numSentMessages;
+    uint64_t& currentReceiptId;
+    QDateTime senderTimestamp;
+    QString message;
+};
+
+class MockCoreExtPacketAllocator : public ICoreExtPacketAllocator
+{
+public:
+    std::unique_ptr<ICoreExtPacket> getPacket(uint32_t friendId) override
+    {
+        return std::unique_ptr<MockCoreExtPacket>(new MockCoreExtPacket(numSentMessages, currentReceiptId));
+    }
+
+    uint64_t numSentMessages;
+    uint64_t currentReceiptId;
+};
+
 class MockFriendMessageSender : public ICoreFriendMessageSender
 {
 public:
@@ -69,6 +108,8 @@ private slots:
     void testMessageSending();
     void testOfflineMessages();
     void testFailedMessage();
+    void testNegotiationFailure();
+    void testNegotiationSuccess();
 
     void onMessageSent(DispatchedMessageId id, Message message)
     {
@@ -93,6 +134,7 @@ private:
     // All unique_ptrs to make construction/init() easier to manage
     std::unique_ptr<Friend> f;
     std::unique_ptr<MockFriendMessageSender> messageSender;
+    std::unique_ptr<MockCoreExtPacketAllocator> coreExtPacketAllocator;
     std::unique_ptr<MessageProcessor::SharedParams> sharedProcessorParams;
     std::unique_ptr<MessageProcessor> messageProcessor;
     std::unique_ptr<FriendMessageDispatcher> friendMessageDispatcher;
@@ -110,11 +152,12 @@ void TestFriendMessageDispatcher::init()
     f = std::unique_ptr<Friend>(new Friend(0, ToxPk()));
     f->setStatus(Status::Status::Online);
     messageSender = std::unique_ptr<MockFriendMessageSender>(new MockFriendMessageSender());
+    coreExtPacketAllocator = std::unique_ptr<MockCoreExtPacketAllocator>(new MockCoreExtPacketAllocator());
     sharedProcessorParams =
         std::unique_ptr<MessageProcessor::SharedParams>(new MessageProcessor::SharedParams());
     messageProcessor = std::unique_ptr<MessageProcessor>(new MessageProcessor(*sharedProcessorParams));
     friendMessageDispatcher = std::unique_ptr<FriendMessageDispatcher>(
-        new FriendMessageDispatcher(*f, *messageProcessor, *messageSender));
+        new FriendMessageDispatcher(*f, *messageProcessor, *messageSender, *coreExtPacketAllocator));
 
     connect(friendMessageDispatcher.get(), &FriendMessageDispatcher::messageSent, this,
             &TestFriendMessageDispatcher::onMessageSent);
@@ -227,5 +270,26 @@ void TestFriendMessageDispatcher::testFailedMessage()
     QVERIFY(messageSender->numSentMessages == 1);
 }
 
+void TestFriendMessageDispatcher::testNegotiationFailure()
+{
+    friendMessageDispatcher->sendMessage(false, "test");
+
+    QVERIFY(messageSender->numSentMessages == 1);
+    QVERIFY(coreExtPacketAllocator->numSentMessages == 0);
+}
+
+void TestFriendMessageDispatcher::testNegotiationSuccess()
+{
+    f->setExtendedMessageSupport(true);
+
+    friendMessageDispatcher->sendMessage(false, "test");
+
+    QVERIFY(coreExtPacketAllocator->numSentMessages == 1);
+
+    friendMessageDispatcher->sendMessage(false, "test");
+    QVERIFY(coreExtPacketAllocator->numSentMessages == 2);
+    QVERIFY(messageSender->numSentMessages == 0);
+}
+
 QTEST_GUILESS_MAIN(TestFriendMessageDispatcher)
 #include "friendmessagedispatcher_test.moc"

test/model/messageprocessor_test.cpp

@@ -66,55 +66,55 @@ void TestMessageProcessor::testSelfMention()
     for (const auto& str : {testUserName, testToxPk}) {
 
         // Using my name or public key should match
-        auto processedMessage = messageProcessor.processIncomingMessage(false, str % " hi");
+        auto processedMessage = messageProcessor.processIncomingCoreMessage(false, str % " hi");
         QVERIFY(messageHasSelfMention(processedMessage));
 
         // Action messages should match too
-        processedMessage = messageProcessor.processIncomingMessage(true, str % " hi");
+        processedMessage = messageProcessor.processIncomingCoreMessage(true, str % " hi");
         QVERIFY(messageHasSelfMention(processedMessage));
 
         // Too much text shouldn't match
-        processedMessage = messageProcessor.processIncomingMessage(false, str % "2");
+        processedMessage = messageProcessor.processIncomingCoreMessage(false, str % "2");
         QVERIFY(!messageHasSelfMention(processedMessage));
 
         // Unless it's a colon
-        processedMessage = messageProcessor.processIncomingMessage(false, str % ": test");
+        processedMessage = messageProcessor.processIncomingCoreMessage(false, str % ": test");
         QVERIFY(messageHasSelfMention(processedMessage));
 
         // remove last character
         QString chopped = str;
         chopped.chop(1);
         // Too little text shouldn't match
-        processedMessage = messageProcessor.processIncomingMessage(false, chopped);
+        processedMessage = messageProcessor.processIncomingCoreMessage(false, chopped);
         QVERIFY(!messageHasSelfMention(processedMessage));
 
         // make lower case
         QString lower = QString(str).toLower();
 
         // The regex should be case insensitive
-        processedMessage = messageProcessor.processIncomingMessage(false, lower % " hi");
+        processedMessage = messageProcessor.processIncomingCoreMessage(false, lower % " hi");
         QVERIFY(messageHasSelfMention(processedMessage));
     }
 
     // New user name changes should be detected
     sharedParams.onUserNameSet("NewUserName");
-    auto processedMessage = messageProcessor.processIncomingMessage(false, "NewUserName: hi");
+    auto processedMessage = messageProcessor.processIncomingCoreMessage(false, "NewUserName: hi");
     QVERIFY(messageHasSelfMention(processedMessage));
 
     // Special characters should be removed
     sharedParams.onUserNameSet("New\nUserName");
-    processedMessage = messageProcessor.processIncomingMessage(false, "NewUserName: hi");
+    processedMessage = messageProcessor.processIncomingCoreMessage(false, "NewUserName: hi");
     QVERIFY(messageHasSelfMention(processedMessage));
 
     // Regression tests for: https://github.com/qTox/qTox/issues/2119
     {
         // Empty usernames should not match
         sharedParams.onUserNameSet("");
-        processedMessage = messageProcessor.processIncomingMessage(false, "");
+        processedMessage = messageProcessor.processIncomingCoreMessage(false, "");
         QVERIFY(!messageHasSelfMention(processedMessage));
 
         // Empty usernames matched on everything, ensure this is not the case
-        processedMessage = messageProcessor.processIncomingMessage(false, "a");
+        processedMessage = messageProcessor.processIncomingCoreMessage(false, "a");
         QVERIFY(!messageHasSelfMention(processedMessage));
     }
 }
@@ -133,7 +133,7 @@ void TestMessageProcessor::testOutgoingMessage()
         testStr += "a";
     }
 
-    auto messages = messageProcessor.processOutgoingMessage(false, testStr);
+    auto messages = messageProcessor.processOutgoingMessage(false, testStr, true /*needsSplit*/);
 
     // The message processor should split our messages
     QVERIFY(messages.size() == 2);
@@ -147,7 +147,7 @@ void TestMessageProcessor::testIncomingMessage()
     // Nothing too special happening on the incoming side if we aren't looking for self mentions
     auto sharedParams = MessageProcessor::SharedParams();
     auto messageProcessor = MessageProcessor(sharedParams);
-    auto message = messageProcessor.processIncomingMessage(false, "test");
+    auto message = messageProcessor.processIncomingCoreMessage(false, "test");
 
     QVERIFY(message.isAction == false);
     QVERIFY(message.content == "test");