Multithread CoreAV

This is a little less insane, gives an overall 2x speedup on debug builds video calls

src/core/core.cpp

@@ -330,6 +330,7 @@ void Core::start()
         GUI::setEnabled(true);
 
     process(); // starts its own timer
+    av->start();
 }
 
 /* Using the now commented out statements in checkConnection(), I watched how
@@ -344,11 +345,13 @@ void Core::start()
 void Core::process()
 {
     if (!isReady())
+    {
+        av->stop();
         return;
+    }
 
     static int tolerance = CORE_DISCONNECT_TOLERANCE;
     tox_iterate(tox);
-    av->process();
 
 #ifdef DEBUG
     //we want to see the debug messages immediately
@@ -365,8 +368,7 @@ void Core::process()
         tolerance = 3*CORE_DISCONNECT_TOLERANCE;
     }
 
-    unsigned sleeptime = qMin(tox_iteration_interval(tox), toxav_iteration_interval(av->getToxAv()));
-    sleeptime = qMin(sleeptime, CoreFile::corefileIterationInterval());
+    unsigned sleeptime = qMin(tox_iteration_interval(tox), CoreFile::corefileIterationInterval());
     toxTimer->start(sleeptime);
 }
 
@@ -727,9 +729,7 @@ void Core::removeGroup(int groupId, bool fake)
         return;
 
     tox_del_groupchat(tox, groupId);
-
-    if (!av->groupCalls[groupId].inactive)
-        av->leaveGroupCall(groupId);
+    av->leaveGroupCall(groupId);
 }
 
 QString Core::getUsername() const
@@ -1207,6 +1207,7 @@ void Core::setNospam(uint32_t nospam)
 void Core::killTimers(bool onlyStop)
 {
     assert(QThread::currentThread() == coreThread);
+    av->stop();
     toxTimer->stop();
     if (!onlyStop)
     {

src/core/coreav.cpp

@@ -24,7 +24,9 @@
 #include "src/persistence/settings.h"
 #include "src/video/videoframe.h"
 #include "src/video/corevideosource.h"
-#include <assert.h>
+#include <cassert>
+#include <QThread>
+#include <QTimer>
 #include <QDebug>
 
 #ifdef QTOX_FILTER_AUDIO
@@ -37,7 +39,14 @@ IndexedList<ToxGroupCall> CoreAV::groupCalls;
 using namespace std;
 
 CoreAV::CoreAV(Tox *tox)
+    : coreavThread{new QThread}, iterateTimer{new QTimer{this}}
 {
+    coreavThread->setObjectName("qTox CoreAV");
+    moveToThread(coreavThread.get());
+
+    iterateTimer->setSingleShot(true);
+    connect(iterateTimer.get(), &QTimer::timeout, this, &CoreAV::process);
+
     toxav = toxav_new(tox, nullptr);
 
     toxav_callback_call(toxav, CoreAV::callCallback, this);
@@ -46,6 +55,8 @@ CoreAV::CoreAV(Tox *tox)
     toxav_callback_video_bit_rate_status(toxav, CoreAV::videoBitrateCallback, this);
     toxav_callback_audio_receive_frame(toxav, CoreAV::audioFrameCallback, this);
     toxav_callback_video_receive_frame(toxav, CoreAV::videoFrameCallback, this);
+
+    coreavThread->start();
 }
 
 CoreAV::~CoreAV()
@@ -60,9 +71,26 @@ const ToxAV *CoreAV::getToxAv() const
     return toxav;
 }
 
+void CoreAV::start()
+{
+    // Timers can only be touched from their own thread
+    if (QThread::currentThread() != coreavThread.get())
+        return (void)QMetaObject::invokeMethod(this, "start", Qt::BlockingQueuedConnection);
+    iterateTimer->start();
+}
+
+void CoreAV::stop()
+{
+    // Timers can only be touched from their own thread
+    if (QThread::currentThread() != coreavThread.get())
+        return (void)QMetaObject::invokeMethod(this, "stop", Qt::BlockingQueuedConnection);
+    iterateTimer->stop();
+}
+
 void CoreAV::process()
 {
     toxav_iterate(toxav);
+    iterateTimer->start(toxav_iteration_interval(toxav));
 }
 
 bool CoreAV::anyActiveCalls()
@@ -169,6 +197,8 @@ bool CoreAV::sendCallAudio(uint32_t callId)
 
 void CoreAV::sendCallVideo(uint32_t callId, shared_ptr<VideoFrame> vframe)
 {
+    // We might be running in the FFmpeg thread and holding the CameraSource lock
+    // So be careful not to deadlock with anything while toxav locks in toxav_video_send_frame
     if (!calls.contains(callId))
         return;
 
@@ -360,7 +390,7 @@ void CoreAV::stateCallback(ToxAV *, uint32_t friendNum, uint32_t state, void *_s
 
     if (state & TOXAV_FRIEND_CALL_STATE_ERROR)
     {
-        qWarning() << "Call with friend"<<friendNum<<"died of unnatural causes";
+        qWarning() << "Call with friend"<<friendNum<<"died of unnatural causes!";
         calls.remove(friendNum);
         emit self->avCallFailed(friendNum);
     }

src/core/coreav.h

@@ -40,6 +40,7 @@ class AudioFilterer;
 #endif
 
 class QTimer;
+class QThread;
 class CoreVideoSource;
 class CameraSource;
 class VideoSource;
@@ -57,8 +58,6 @@ public:
 
     const ToxAV* getToxAv() const;
 
-    void process();
-
     bool anyActiveCalls(); ///< true is any calls are currently active (note: a call about to start is not yet active)
     void prepareCall(uint32_t friendId, ToxAV *toxav, bool videoEnabled);
     void cleanupCall(uint32_t friendId);
@@ -88,6 +87,10 @@ public:
     void micMuteToggle(uint32_t friendId);
     void volMuteToggle(uint32_t friendId);
 
+public slots:
+    void start();
+    void stop();
+
 signals:
     void avInvite(uint32_t friendId, bool video);
     void avStart(uint32_t friendId, bool video);
@@ -99,6 +102,7 @@ signals:
     void videoFrameReceived(vpx_image* frame);
 
 private:
+    void process();
     static void callCallback(ToxAV *toxAV, uint32_t friendNum, bool audio, bool video, void* self);
     static void stateCallback(ToxAV *toxAV, uint32_t friendNum, uint32_t state, void* self);
     static void audioBitrateCallback(ToxAV *toxAV, uint32_t friendNum, bool stable, uint32_t rate, void* self);
@@ -108,6 +112,9 @@ private:
     static void videoFrameCallback(ToxAV *toxAV, uint32_t friendNum, uint16_t w, uint16_t h,
                                    const uint8_t *y, const uint8_t *u, const uint8_t *v,
                                    int32_t ystride, int32_t ustride, int32_t vstride, void* self);
+    /// Intercepts a function call and moves it to another thread
+    /// Useful to move callbacks from the toxcore thread to our thread
+    template <class... Args> void asyncTransplantThunk(void(*fun)(Args...), Args... args);
 
 private:
     static constexpr uint32_t AUDIO_DEFAULT_BITRATE = 64; ///< In kb/s. More than enough for Opus.
@@ -115,11 +122,12 @@ private:
 
 private:
     ToxAV* toxav;
+    std::unique_ptr<QThread> coreavThread;
+    std::unique_ptr<QTimer> iterateTimer;
     static IndexedList<ToxFriendCall> calls;
     static IndexedList<ToxGroupCall> groupCalls; // Maps group IDs to ToxGroupCalls
 
     friend class Audio;
-    friend class Core;
 };
 
 #endif // COREAV_H

src/core/toxcall.cpp

@@ -5,6 +5,7 @@
 #include "src/video/camerasource.h"
 #include "src/video/corevideosource.h"
 #include <QTimer>
+#include <QtConcurrent/QtConcurrent>
 
 #ifdef QTOX_FILTER_AUDIO
 #include "src/audio/audiofilterer.h"
@@ -126,7 +127,10 @@ ToxFriendCall::~ToxFriendCall()
 {
     if (videoEnabled)
     {
-        CameraSource::getInstance().unsubscribe();
+        // This destructor could be running in a toxav callback while holding toxav locks.
+        // If the CameraSource thread calls toxav *_send_frame, we might deadlock the toxav and CameraSource locks,
+        // so we unsuscribe asynchronously, it's fine if the webcam takes a couple milliseconds more to poweroff.
+        QtConcurrent::run([](){CameraSource::getInstance().unsubscribe();});
         if (videoSource)
         {
             videoSource->setDeleteOnClose(true);