extend ToxAV API to support different threads for audio and video

This allows for prioritizing audio over video transcoding and improves
performance when multiple CPU cores are available.

toxav/toxav.api.h

@@ -43,6 +43,10 @@ extern "C" {
  * Only ${toxAV.iterate} is thread-safe, all other functions must run from the
  * tox thread.
  *
+ * Important exceptions are the `*_iterate` and `*_iterate_interval`
+ * functions. You have to choose either the single thread or the multi thread
+ * functions and read their documentation.
+ *
  * A common way to run ToxAV (multiple or single instance) is to have a thread,
  * separate from tox instance thread, running a simple ${toxAV.iterate} loop,
  * sleeping for ${toxAV.iteration_interval} * milliseconds on each iteration.
@@ -56,6 +60,16 @@ extern "C" {
  * error code.
  */
 
+/** \subsection multi-threading Separate audio and video threads
+ *
+ * ToxAV supports either a single thread for audio and video or decoding and
+ * encoding them in separate threads. You have to choose one mode and can not
+ * mix function calls to those different modes.
+ *
+ * For best results use the multi-threaded mode and run the audio thread with
+ * higher priority than the video thread. This prioritizes audio over video.
+ */
+
 /**
  * External Tox type.
  */
@@ -118,7 +132,7 @@ tox::this *tox { get(); }
 
 /*******************************************************************************
  *
- * :: A/V event loop
+ * :: A/V event loop, single thread
  *
  ******************************************************************************/
 
@@ -126,6 +140,7 @@ tox::this *tox { get(); }
 /**
  * Returns the interval in milliseconds when the next toxav_iterate call should
  * be. If no call is active at the moment, this function returns 200.
+ * This function MUST be called from the same thread as toxav_iterate.
  */
 const uint32_t iteration_interval();
 
@@ -136,6 +151,43 @@ const uint32_t iteration_interval();
  */
 void iterate();
 
+/*******************************************************************************
+ *
+ * :: A/V event loop, multiple threads
+ *
+ ******************************************************************************/
+
+/**
+ * Returns the interval in milliseconds when the next toxav_audio_iterate call
+ * should be. If no call is active at the moment, this function returns 200.
+ * This function MUST be called from the same thread as toxav_audio_iterate.
+ */
+const uint32_t audio_iteration_interval();
+
+/**
+ * Main loop for the session. This function needs to be called in intervals of
+ * toxav_audio_iteration_interval() milliseconds. It is best called in a
+ * separate thread from tox_iterate and toxav_video_iterate. The thread calling
+ * this function should have higher priority than the one calling
+ * toxav_video_iterate to prioritize audio over video.
+ */
+void audio_iterate();
+
+/**
+ * Returns the interval in milliseconds when the next toxav_video_iterate call
+ * should be. If no call is active at the moment, this function returns 200.
+ * This function MUST be called from the same thread as toxav_video_iterate.
+ */
+const uint32_t video_iteration_interval();
+
+/**
+ * Main loop for the session. This function needs to be called in intervals of
+ * toxav_video_iteration_interval() milliseconds. It is best called in a
+ * separate thread from tox_iterate and toxav_audio_iterate. The thread calling
+ * this function should have lower priority than the one calling
+ * toxav_audio_iterate to prioritize audio over video.
+ */
+void video_iterate();
 
 /*******************************************************************************
  *

toxav/toxav.c

@@ -67,6 +67,16 @@ typedef struct ToxAVCall_s {
     struct ToxAVCall_s *next;
 } ToxAVCall;
 
+
+/** Decode time statistics */
+typedef struct DecodeTimeStats_s {
+    int32_t count; /** Measure count */
+    int32_t total; /** Last cycle total */
+    int32_t average; /** Average decoding time in ms */
+
+    uint32_t interval; /** Calculated iteration interval */
+} DecodeTimeStats;
+
 struct ToxAV {
     Tox *tox;
     Messenger *m;
@@ -97,12 +107,9 @@ struct ToxAV {
     toxav_video_bit_rate_cb *vbcb;
     void *vbcb_user_data;
 
-    /** Decode time measures */
-    int32_t dmssc; /** Measure count */
-    int32_t dmsst; /** Last cycle total */
-    int32_t dmssa; /** Average decoding time in ms */
-
-    uint32_t interval; /** Calculated interval */
+    /* keep track of decode times for audio and video */
+    DecodeTimeStats audio_stats;
+    DecodeTimeStats video_stats;
     Mono_Time *toxav_mono_time; /** ToxAV's own mono_time instance */
 };
 
@@ -123,6 +130,19 @@ static ToxAVCall *call_remove(ToxAVCall *call);
 static bool call_prepare_transmission(ToxAVCall *call);
 static void call_kill_transmission(ToxAVCall *call);
 
+/**
+ * @brief initialize d with default values
+ * @param d struct to be initialized, must not be nullptr
+ */
+static void init_decode_time_stats(DecodeTimeStats *d)
+{
+    assert(d != nullptr);
+    d->count = 0;
+    d->total = 0;
+    d->average = 0;
+    d->interval = IDLE_ITERATION_INTERVAL_MS;
+}
+
 ToxAV *toxav_new(Tox *tox, Toxav_Err_New *error)
 {
     Toxav_Err_New rc = TOXAV_ERR_NEW_OK;
@@ -169,7 +189,8 @@ ToxAV *toxav_new(Tox *tox, Toxav_Err_New *error)
         goto RETURN;
     }
 
-    av->interval = IDLE_ITERATION_INTERVAL_MS;
+    init_decode_time_stats(&av->audio_stats);
+    init_decode_time_stats(&av->video_stats);
     av->msi->av = av;
 
     msi_register_callback(av->msi, callback_invite, MSI_ON_INVITE);
@@ -228,11 +249,48 @@ Tox *toxav_get_tox(const ToxAV *av)
 {
     return av->tox;
 }
+
+uint32_t toxav_audio_iteration_interval(const ToxAV *av)
+{
+    return av->calls ? av->audio_stats.interval : IDLE_ITERATION_INTERVAL_MS;
+}
+
+uint32_t toxav_video_iteration_interval(const ToxAV *av)
+{
+    return av->calls ? av->video_stats.interval : IDLE_ITERATION_INTERVAL_MS;
+}
+
 uint32_t toxav_iteration_interval(const ToxAV *av)
 {
-    return av->calls ? av->interval : IDLE_ITERATION_INTERVAL_MS;
+    return min_u32(toxav_audio_iteration_interval(av),
+                   toxav_video_iteration_interval(av));
 }
-void toxav_iterate(ToxAV *av)
+
+/**
+ * @brief calc_interval Calculates the needed iteration interval based on previous decode times
+ * @param av ToxAV struct to work on
+ * @param stats Statistics to update
+ * @param frame_time the duration of the current frame in ms
+ * @param start_time the timestamp when decoding of this frame started
+ */
+static void calc_interval(ToxAV *av, DecodeTimeStats *stats, int32_t frame_time, uint64_t start_time)
+{
+    stats->interval = frame_time < stats->average ? 0 : (frame_time - stats->average);
+    stats->total += current_time_monotonic(av->m->mono_time) - start_time;
+
+    if (++stats->count == 3) {
+        stats->average = stats->total / 3 + 5; /* NOTE: Magic Offset for precision */
+        stats->count = 0;
+        stats->total = 0;
+    }
+}
+
+/**
+ * @brief common iterator function for audio and video calls
+ * @param av pointer to ToxAV structure of current instance
+ * @param audio if true, iterate audio, video else
+ */
+static void iterate_common(ToxAV *av, bool audio)
 {
     pthread_mutex_lock(av->mutex);
 
@@ -242,21 +300,26 @@ void toxav_iterate(ToxAV *av)
     }
 
     uint64_t start = current_time_monotonic(av->toxav_mono_time);
-    // time until this audio or video frame is over
+    // time until the first audio or video frame is over
     int32_t frame_time = IDLE_ITERATION_INTERVAL_MS;
 
     for (ToxAVCall *i = av->calls[av->calls_head]; i; i = i->next) {
-        if (i->active) {
-            pthread_mutex_lock(i->toxav_call_mutex);
-            pthread_mutex_unlock(av->mutex);
+        if (!i->active) {
+            continue;
+        }
 
+        pthread_mutex_lock(i->toxav_call_mutex);
+        pthread_mutex_unlock(av->mutex);
+
+        if (audio) {
             ac_iterate(i->audio);
-            vc_iterate(i->video);
 
             if (i->msi_call->self_capabilities & MSI_CAP_R_AUDIO &&
                     i->msi_call->peer_capabilities & MSI_CAP_S_AUDIO) {
                 frame_time = min_s32(i->audio->lp_frame_duration, frame_time);
             }
+        } else {
+            vc_iterate(i->video);
 
             if (i->msi_call->self_capabilities & MSI_CAP_R_VIDEO &&
                     i->msi_call->peer_capabilities & MSI_CAP_S_VIDEO) {
@@ -264,30 +327,39 @@ void toxav_iterate(ToxAV *av)
                 frame_time = min_u32(i->video->lcfd, frame_time);
                 pthread_mutex_unlock(i->video->queue_mutex);
             }
+        }
 
-            uint32_t fid = i->friend_number;
+        uint32_t fid = i->friend_number;
 
-            pthread_mutex_unlock(i->toxav_call_mutex);
-            pthread_mutex_lock(av->mutex);
+        pthread_mutex_unlock(i->toxav_call_mutex);
+        pthread_mutex_lock(av->mutex);
 
-            /* In case this call is popped from container stop iteration */
-            if (call_get(av, fid) != i) {
-                break;
-            }
+        /* In case this call is popped from container stop iteration */
+        if (call_get(av, fid) != i) {
+            break;
         }
     }
 
-    av->interval = frame_time < av->dmssa ? 0 : (frame_time - av->dmssa);
-    av->dmsst += current_time_monotonic(av->toxav_mono_time) - start;
-
-    if (++av->dmssc == 3) {
-        av->dmssa = av->dmsst / 3 + 5; /* NOTE Magic Offset 5 for precision */
-        av->dmssc = 0;
-        av->dmsst = 0;
-    }
-
+    DecodeTimeStats *stats = audio ? &av->audio_stats : &av->video_stats;
+    calc_interval(av, stats, frame_time, start);
     pthread_mutex_unlock(av->mutex);
 }
+void toxav_audio_iterate(ToxAV *av)
+{
+    iterate_common(av, true);
+}
+
+void toxav_video_iterate(ToxAV *av)
+{
+    iterate_common(av, false);
+}
+
+void toxav_iterate(ToxAV *av)
+{
+    toxav_audio_iterate(av);
+    toxav_video_iterate(av);
+}
+
 bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate,
                 Toxav_Err_Call *error)
 {

toxav/toxav.h

@@ -39,6 +39,10 @@ extern "C" {
  * Only toxav_iterate is thread-safe, all other functions must run from the
  * tox thread.
  *
+ * Important exceptions are the `*_iterate` and `*_iterate_interval`
+ * functions. You have to choose either the single thread or the multi thread
+ * functions and read their documentation.
+ *
  * A common way to run ToxAV (multiple or single instance) is to have a thread,
  * separate from tox instance thread, running a simple toxav_iterate loop,
  * sleeping for toxav_iteration_interval * milliseconds on each iteration.
@@ -51,6 +55,15 @@ extern "C" {
  * fail if mutexes are held by tox thread in which case they will set SYNC
  * error code.
  */
+/** \subsection multi-threading Separate audio and video threads
+ *
+ * ToxAV supports either a single thread for audio and video or decoding and
+ * encoding them in separate threads. You have to choose one mode and can not
+ * mix function calls to those different modes.
+ *
+ * For best results use the multi-threaded mode and run the audio thread with
+ * higher priority than the video thread. This prioritizes audio over video.
+ */
 /**
  * External Tox type.
  */
@@ -132,7 +145,7 @@ Tox *toxav_get_tox(const ToxAV *av);
 
 /*******************************************************************************
  *
- * :: A/V event loop
+ * :: A/V event loop, single thread
  *
  ******************************************************************************/
 
@@ -141,6 +154,7 @@ Tox *toxav_get_tox(const ToxAV *av);
 /**
  * Returns the interval in milliseconds when the next toxav_iterate call should
  * be. If no call is active at the moment, this function returns 200.
+ * This function MUST be called from the same thread as toxav_iterate.
  */
 uint32_t toxav_iteration_interval(const ToxAV *av);
 
@@ -152,6 +166,47 @@ uint32_t toxav_iteration_interval(const ToxAV *av);
 void toxav_iterate(ToxAV *av);
 
 
+/*******************************************************************************
+ *
+ * :: A/V event loop, multiple threads
+ *
+ ******************************************************************************/
+
+
+
+/**
+ * Returns the interval in milliseconds when the next toxav_audio_iterate call
+ * should be. If no call is active at the moment, this function returns 200.
+ * This function MUST be called from the same thread as toxav_audio_iterate.
+ */
+uint32_t toxav_audio_iteration_interval(const ToxAV *av);
+
+/**
+ * Main loop for the session. This function needs to be called in intervals of
+ * toxav_audio_iteration_interval() milliseconds. It is best called in a
+ * separate thread from tox_iterate and toxav_video_iterate. The thread calling
+ * this function should have higher priority than the one calling
+ * toxav_video_iterate to prioritize audio over video.
+ */
+void toxav_audio_iterate(ToxAV *av);
+
+/**
+ * Returns the interval in milliseconds when the next toxav_video_iterate call
+ * should be. If no call is active at the moment, this function returns 200.
+ * This function MUST be called from the same thread as toxav_video_iterate.
+ */
+uint32_t toxav_video_iteration_interval(const ToxAV *av);
+
+/**
+ * Main loop for the session. This function needs to be called in intervals of
+ * toxav_video_iteration_interval() milliseconds. It is best called in a
+ * separate thread from tox_iterate and toxav_audio_iterate. The thread calling
+ * this function should have lower priority than the one calling
+ * toxav_audio_iterate to prioritize audio over video.
+ */
+void toxav_video_iterate(ToxAV *av);
+
+
 /*******************************************************************************
  *
  * :: Call setup