diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index d9adb4fe..0434a3c6 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -10,8 +10,6 @@ libtoxav_la_SOURCES = ../toxav/rtp.h \ ../toxav/msi.c \ ../toxav/group.h \ ../toxav/group.c \ - ../toxav/codec.h \ - ../toxav/codec.c \ ../toxav/audio.h \ ../toxav/audio.c \ ../toxav/video.h \ diff --git a/toxav/audio.c b/toxav/audio.c index f3e969e9..dc85452a 100644 --- a/toxav/audio.c +++ b/toxav/audio.c @@ -83,6 +83,7 @@ ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *c * do error correction with opus */ ac->last_packet_frame_duration = 120; ac->last_packet_sampling_rate = 48000; + ac->last_packet_channel_count = 1; ac->av = av; ac->friend_id = friend_id; @@ -119,7 +120,7 @@ void ac_do(ACSession* ac) return; /* Enough space for the maximum frame size (120 ms 48 KHz audio) */ - int16_t tmp[5760]; + int16_t tmp[5760 * 2]; RTPMessage *msg; int rc = 0; @@ -130,9 +131,8 @@ void ac_do(ACSession* ac) if (rc == 2) { LOGGER_DEBUG("OPUS correction"); - rc = opus_decode(ac->decoder, NULL, 0, tmp, - (ac->last_packet_sampling_rate * ac->last_packet_frame_duration / 1000) / - ac->last_packet_channel_count, 1); + int fs = (ac->last_packet_sampling_rate * ac->last_packet_frame_duration) / 1000; + rc = opus_decode(ac->decoder, NULL, 0, tmp, fs, 1); } else { /* Get values from packet and decode. */ /* NOTE: This didn't work very well @@ -152,10 +152,9 @@ void ac_do(ACSession* ac) ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); - /* - * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, - * it didn't work quite well. - */ + /** NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, + * it didn't work quite well. + */ if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) { LOGGER_WARNING("Failed to reconfigure decoder!"); rtp_free_msg(NULL, msg); @@ -169,7 +168,7 @@ void ac_do(ACSession* ac) if (rc < 0) { LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (ac->acb.first) { - ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate * ac->last_packet_channel_count; + ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate; ac->acb.first(ac->av, ac->friend_id, tmp, rc * ac->last_packet_channel_count, ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second); @@ -179,6 +178,37 @@ void ac_do(ACSession* ac) } pthread_mutex_unlock(ac->queue_mutex); } +int ac_queue_message(void* acp, struct RTPMessage_s *msg) +{ + if (!acp || !msg) + return -1; + + if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyAudio % 128) { + LOGGER_WARNING("Got dummy!"); + rtp_free_msg(NULL, msg); + return 0; + } + + if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeAudio % 128) { + LOGGER_WARNING("Invalid payload type!"); + rtp_free_msg(NULL, msg); + return -1; + } + + ACSession* ac = acp; + + pthread_mutex_lock(ac->queue_mutex); + int rc = jbuf_write(ac->j_buf, msg); + pthread_mutex_unlock(ac->queue_mutex); + + if (rc == -1) { + LOGGER_WARNING("Could not queue the message!"); + rtp_free_msg(NULL, msg); + return -1; + } + + return 0; +} int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) { if (!ac) @@ -210,21 +240,7 @@ int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); return 0; } -/* called from rtp */ -void ac_queue_message(void* acp, RTPMessage *msg) -{ - if (!acp || !msg) - return; - - ACSession* ac = acp; - pthread_mutex_lock(ac->queue_mutex); - int ret = jbuf_write(ac->j_buf, msg); - pthread_mutex_unlock(ac->queue_mutex); - - if (ret == -1) - rtp_free_msg(NULL, msg); -} /* JITTER BUFFER WORK */ diff --git a/toxav/audio.h b/toxav/audio.h index 62a28cdf..2cb0d8f6 100644 --- a/toxav/audio.h +++ b/toxav/audio.h @@ -29,6 +29,8 @@ #include "../toxcore/util.h" +struct RTPMessage_s; + typedef struct ACSession_s { /* encoding */ OpusEncoder *encoder; @@ -56,5 +58,6 @@ typedef struct ACSession_s { ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data); void ac_kill(ACSession* ac); void ac_do(ACSession* ac); +int ac_queue_message(void *acp, struct RTPMessage_s *msg); int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); #endif /* AUDIO_H */ \ No newline at end of file diff --git a/toxav/av_test.c b/toxav/av_test.c index 994f9cf6..dce63184 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -77,8 +77,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 1 -#define TEST_TRANSFER_V 0 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -329,6 +329,9 @@ void* iterate_toxav (void * data) toxav_iterate(data_cast->BobAV); int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); + printf("\rIteration interval: %d ", rc); + fflush(stdout); + #if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1 cvWaitKey(rc); #else diff --git a/toxav/codec.c b/toxav/codec.c deleted file mode 100644 index 57e43c67..00000000 --- a/toxav/codec.c +++ /dev/null @@ -1,686 +0,0 @@ -/** codec.c - * - * Copyright (C) 2013-2015 Tox project All Rights Reserved. - * - * This file is part of Tox. - * - * Tox is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Tox is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Tox. If not, see . - * - */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif /* HAVE_CONFIG_H */ - -#include "../toxcore/logger.h" -#include "../toxcore/util.h" - -#include -#include -#include -#include -#include -#include - -#include "msi.h" -#include "rtp.h" -#include "codec.h" - -#define DEFAULT_JBUF 3 - -/* Good quality encode. */ -#define MAX_DECODE_TIME_US 0 - -#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ -#define VIDEOFRAME_HEADER_SIZE 0x2 - -/* FIXME: Might not be enough? NOTE: I think it is enough */ -#define VIDEO_DECODE_BUFFER_SIZE 20 - -#define ARRAY(TYPE__) struct { uint16_t size; TYPE__ data[]; } - -typedef ARRAY(uint8_t) Payload; - -/* JITTER BUFFER WORK */ -typedef struct JitterBuffer_s { - RTPMessage **queue; - uint32_t size; - uint32_t capacity; - uint16_t bottom; - uint16_t top; -} JitterBuffer; - -static JitterBuffer *jbuf_new(uint32_t capacity) -{ - unsigned int size = 1; - - while (size <= (capacity * 4)) { - size *= 2; - } - - JitterBuffer *q; - - if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL; - - if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { - free(q); - return NULL; - } - - q->size = size; - q->capacity = capacity; - return q; -} - -static void jbuf_clear(JitterBuffer *q) -{ - for (; q->bottom != q->top; ++q->bottom) { - if (q->queue[q->bottom % q->size]) { - rtp_free_msg(NULL, q->queue[q->bottom % q->size]); - q->queue[q->bottom % q->size] = NULL; - } - } -} - -static void jbuf_free(JitterBuffer *q) -{ - if (!q) return; - - jbuf_clear(q); - free(q->queue); - free(q); -} - -static int jbuf_write(JitterBuffer *q, RTPMessage *m) -{ - uint16_t sequnum = m->header->sequnum; - - unsigned int num = sequnum % q->size; - - if ((uint32_t)(sequnum - q->bottom) > q->size) { - LOGGER_DEBUG("Clearing filled jitter buffer: %p", q); - - jbuf_clear(q); - q->bottom = sequnum - q->capacity; - q->queue[num] = m; - q->top = sequnum + 1; - return 0; - } - - if (q->queue[num]) - return -1; - - q->queue[num] = m; - - if ((sequnum - q->bottom) >= (q->top - q->bottom)) - q->top = sequnum + 1; - - return 0; -} - -/* success is set to 0 when there is nothing to dequeue, - * 1 when there's a good packet, - * 2 when there's a lost packet */ -static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success) -{ - if (q->top == q->bottom) { - *success = 0; - return NULL; - } - - unsigned int num = q->bottom % q->size; - - if (q->queue[num]) { - RTPMessage *ret = q->queue[num]; - q->queue[num] = NULL; - ++q->bottom; - *success = 1; - return ret; - } - - if ((uint32_t)(q->top - q->bottom) > q->capacity) { - ++q->bottom; - *success = 2; - return NULL; - } - - *success = 0; - return NULL; -} - -static int convert_bw_to_sampling_rate(int bw) -{ - switch(bw) - { - case OPUS_BANDWIDTH_NARROWBAND: return 8000; - case OPUS_BANDWIDTH_MEDIUMBAND: return 12000; - case OPUS_BANDWIDTH_WIDEBAND: return 16000; - case OPUS_BANDWIDTH_SUPERWIDEBAND: return 24000; - case OPUS_BANDWIDTH_FULLBAND: return 48000; - default: return -1; - } -} - -OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count) -{ - int status = OPUS_OK; - OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); - return NULL; - } - - status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate)); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); - goto FAILURE; - } - - status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); - goto FAILURE; - } - - return rc; - -FAILURE: - opus_encoder_destroy(rc); - return NULL; -} - -bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) -{ - assert(dest); - - vpx_codec_enc_cfg_t cfg; - int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); - return false; - } - - rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, - VPX_ENCODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); - return false; - } - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = 800; - cfg.g_h = 600; - cfg.g_pass = VPX_RC_ONE_PASS; - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; - - rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - vpx_codec_destroy(dest); - } - - return true; -} - -bool reconfigure_audio_decoder(CSession* cs, int32_t sampling_rate, int8_t channels) -{ - if (sampling_rate != cs->last_decoding_sampling_rate || channels != cs->last_decoding_channel_count) { - if (current_time_monotonic() - cs->last_decoder_reconfiguration < 500) - return false; - - int status; - OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status ); - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status)); - return false; - } - - cs->last_decoding_sampling_rate = sampling_rate; - cs->last_decoding_channel_count = channels; - cs->last_decoder_reconfiguration = current_time_monotonic(); - - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = new_dec; - - LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels); - } - - return true; -} - -/* PUBLIC */ -void cs_do(CSession *cs) -{ - /* Codec session should always be protected by call mutex so no need to check for cs validity - */ - - if (!cs) - return; - - Payload *p; - int rc; - - int success = 0; - - LOGGED_LOCK(cs->queue_mutex); - - /********************* AUDIO *********************/ - if (cs->audio_decoder) { - RTPMessage *msg; - - /* The maximum for 120 ms 48 KHz audio */ - int16_t tmp[5760]; - - while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { - LOGGED_UNLOCK(cs->queue_mutex); - - if (success == 2) { - LOGGER_DEBUG("OPUS correction"); - rc = opus_decode(cs->audio_decoder, NULL, 0, tmp, - (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) / - cs->last_packet_channel_count, 1); - } else { - /* Get values from packet and decode. */ - /* NOTE: This didn't work very well - rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); - if (rc != -1) { - cs->last_packet_sampling_rate = rc; - } else { - LOGGER_WARNING("Failed to load packet values!"); - rtp_free_msg(NULL, msg); - continue; - }*/ - - - /* Pick up sampling rate from packet */ - memcpy(&cs->last_packet_sampling_rate, msg->data, 4); - cs->last_packet_sampling_rate = ntohl(cs->last_packet_sampling_rate); - - cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); - - /* - * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, - * it didn't work quite well. - */ - if (!reconfigure_audio_decoder(cs, cs->last_packet_sampling_rate, cs->last_packet_channel_count)) { - LOGGER_WARNING("Failed to reconfigure decoder!"); - rtp_free_msg(NULL, msg); - continue; - } - - rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); - rtp_free_msg(NULL, msg); - } - - if (rc < 0) { - LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); - } else if (cs->acb.first) { - cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count; - - cs->acb.first(cs->av, cs->friend_id, tmp, rc * cs->last_packet_channel_count, - cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); - - } - - LOGGED_LOCK(cs->queue_mutex); - } - } - - /********************* VIDEO *********************/ - if (cs->vbuf_raw && !rb_empty(cs->vbuf_raw)) { - /* Decode video */ - rb_read(cs->vbuf_raw, (void**)&p); - - /* Leave space for (possibly) other thread to queue more data after we read it here */ - LOGGED_UNLOCK(cs->queue_mutex); - - rc = vpx_codec_decode(cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); - free(p); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); - } else { - vpx_codec_iter_t iter = NULL; - vpx_image_t *dest = vpx_codec_get_frame(cs->v_decoder, &iter); - - /* Play decoded images */ - for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { - if (cs->vcb.first) - cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h, - (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2], - dest->stride[0], dest->stride[1], dest->stride[2], cs->vcb.second); - - vpx_img_free(dest); - } - } - - return; - } - - LOGGED_UNLOCK(cs->queue_mutex); -} -CSession *cs_new(uint32_t peer_video_frame_piece_size) -{ - CSession *cs = calloc(sizeof(CSession), 1); - - if (!cs) { - LOGGER_WARNING("Allocation failed! Application might misbehave!"); - return NULL; - } - - if (create_recursive_mutex(cs->queue_mutex) != 0) { - LOGGER_WARNING("Failed to create recursive mutex!"); - free(cs); - return NULL; - } - - /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ - /* Create decoders and set up their values - */ - - /* - * AUDIO - */ - - int status; - cs->audio_decoder = opus_decoder_create(48000, 2, &status ); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); - goto FAILURE; - } - - cs->last_decoding_channel_count = 2; - cs->last_decoding_sampling_rate = 48000; - cs->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */ - - /* These need to be set in order to properly - * do error correction with opus */ - cs->last_packet_frame_duration = 120; - cs->last_packet_sampling_rate = 48000; - - if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { - LOGGER_WARNING("Jitter buffer creaton failed!"); - opus_decoder_destroy(cs->audio_decoder); - goto FAILURE; - } - - /* - * VIDEO - */ - int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, - NULL, 0, VPX_DECODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); - goto AUDIO_DECODER_CLEANUP; - } - - if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) { - vpx_codec_destroy(cs->v_decoder); - goto AUDIO_DECODER_CLEANUP; - } - - if ( !(cs->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) ) { - free(cs->frame_buf); - vpx_codec_destroy(cs->v_decoder); - goto AUDIO_DECODER_CLEANUP; - } - - if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) - goto FAILURE; - - cs->linfts = current_time_monotonic(); - cs->lcfd = 60; - /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ - - /* Initialize encoders with default values */ - cs->audio_encoder = create_audio_encoder(48000, 48000, 2); - if (cs->audio_encoder == NULL) - goto VIDEO_DECODER_CLEANUP; - - cs->last_encoding_bitrate = 48000; - cs->last_encoding_sampling_rate = 48000; - cs->last_encoding_channel_count = 2; - - if (!create_video_encoder(cs->v_encoder, 500000)) { - opus_encoder_destroy(cs->audio_encoder); - goto VIDEO_DECODER_CLEANUP; - } - - cs->peer_video_frame_piece_size = peer_video_frame_piece_size; - - return cs; - -VIDEO_DECODER_CLEANUP: - rb_free(cs->vbuf_raw); - free(cs->frame_buf); - vpx_codec_destroy(cs->v_decoder); -AUDIO_DECODER_CLEANUP: - opus_decoder_destroy(cs->audio_decoder); - jbuf_free(cs->j_buf); -FAILURE: - pthread_mutex_destroy(cs->queue_mutex); - free(cs); - return NULL; -} -void cs_kill(CSession *cs) -{ - if (!cs) - return; - - /* NOTE: queue_message() will not be called since - * the callback is unregistered before cs_kill is called. - */ - - opus_encoder_destroy(cs->audio_encoder); - opus_decoder_destroy(cs->audio_decoder); - jbuf_free(cs->j_buf); - vpx_codec_destroy(cs->v_encoder); - vpx_codec_destroy(cs->v_decoder); - rb_free(cs->vbuf_raw); - free(cs->frame_buf); - free(cs->split_video_frame); - - pthread_mutex_destroy(cs->queue_mutex); - - LOGGER_DEBUG("Terminated codec state: %p", cs); - free(cs); -} -void cs_init_video_splitter_cycle(CSession* cs) -{ - cs->split_video_frame[0] = cs->frameid_out++; - cs->split_video_frame[1] = 0; -} -int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length) -{ - cs->processing_video_frame = payload; - cs->processing_video_frame_size = length; - - return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; -} -const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size) -{ - if (!cs || !size) return NULL; - - if (cs->processing_video_frame_size > VIDEOFRAME_PIECE_SIZE) { - memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, - cs->processing_video_frame, - VIDEOFRAME_PIECE_SIZE); - - cs->processing_video_frame += VIDEOFRAME_PIECE_SIZE; - cs->processing_video_frame_size -= VIDEOFRAME_PIECE_SIZE; - - *size = VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE; - } else { - memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, - cs->processing_video_frame, - cs->processing_video_frame_size); - - *size = cs->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE; - } - - cs->split_video_frame[1]++; - - return cs->split_video_frame; -} -int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height) -{ - vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; - if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) - return 0; /* Nothing changed */ - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = width; - cfg.g_h = height; - - int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - return 0; -} -int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels) -{ - /* Values are checked in toxav.c */ - - if (cs->last_encoding_sampling_rate != sampling_rate || cs->last_encoding_channel_count != channels) { - OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels); - if (new_encoder == NULL) - return -1; - - opus_encoder_destroy(cs->audio_encoder); - cs->audio_encoder = new_encoder; - } else if (cs->last_encoding_bitrate == bitrate) - return 0; /* Nothing changed */ - else { - int status = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); - - if ( status != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); - return -1; - } - } - - cs->last_encoding_bitrate = bitrate; - cs->last_encoding_sampling_rate = sampling_rate; - cs->last_encoding_channel_count = channels; - - LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); - return 0; -} -/* Called from RTP */ -void queue_message(RTPSession *session, RTPMessage *msg) -{ - CSession *cs = session->cs; - - if (!cs) - return; - - /* Audio */ - if (session->payload_type == rtp_TypeAudio % 128) { - LOGGED_LOCK(cs->queue_mutex); - int ret = jbuf_write(cs->j_buf, msg); - LOGGED_UNLOCK(cs->queue_mutex); - - if (ret == -1) { - rtp_free_msg(NULL, msg); - } - } - /* Video */ - else { - uint8_t *packet = msg->data; - uint32_t packet_size = msg->length; - - if (packet_size < VIDEOFRAME_HEADER_SIZE) - goto end; - - uint8_t diff = packet[0] - cs->frameid_in; - - if (diff != 0) { - if (diff < 225) { /* New frame */ - /* Flush last frames' data and get ready for this frame */ - Payload *p = malloc(sizeof(Payload) + cs->frame_size); - - if (p) { - LOGGED_LOCK(cs->queue_mutex); - - if (rb_full(cs->vbuf_raw)) { - LOGGER_DEBUG("Dropped video frame"); - Payload *tp; - rb_read(cs->vbuf_raw, (void**)&tp); - free(tp); - } else { - p->size = cs->frame_size; - memcpy(p->data, cs->frame_buf, cs->frame_size); - } - - /* Calculate time took for peer to send us this frame */ - uint32_t t_lcfd = current_time_monotonic() - cs->linfts; - cs->lcfd = t_lcfd > 100 ? cs->lcfd : t_lcfd; - cs->linfts = current_time_monotonic(); - - rb_write(cs->vbuf_raw, p); - LOGGED_UNLOCK(cs->queue_mutex); - } else { - LOGGER_WARNING("Allocation failed! Program might misbehave!"); - goto end; - } - - cs->frameid_in = packet[0]; - memset(cs->frame_buf, 0, cs->frame_size); - cs->frame_size = 0; - - } else { /* Old frame; drop */ - LOGGER_DEBUG("Old packet: %u", packet[0]); - goto end; - } - } - - uint8_t piece_number = packet[1]; - - uint32_t length_before_piece = ((piece_number - 1) * cs->peer_video_frame_piece_size); - uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE); - - if (framebuf_new_length > MAX_VIDEOFRAME_SIZE) { - goto end; - } - - /* Otherwise it's part of the frame so just process */ - /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */ - - memcpy(cs->frame_buf + length_before_piece, - packet + VIDEOFRAME_HEADER_SIZE, - packet_size - VIDEOFRAME_HEADER_SIZE); - - if (framebuf_new_length > cs->frame_size) - cs->frame_size = framebuf_new_length; - -end: - rtp_free_msg(NULL, msg); - } -} diff --git a/toxav/codec.h b/toxav/codec.h deleted file mode 100644 index 497016eb..00000000 --- a/toxav/codec.h +++ /dev/null @@ -1,125 +0,0 @@ -/** codec.h - * - * Copyright (C) 2013-2015 Tox project All Rights Reserved. - * - * This file is part of Tox. - * - * Tox is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Tox is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Tox. If not, see . - * - */ - -#ifndef CODEC_H -#define CODEC_H - -#include "toxav.h" -#include "rtp.h" - -#include "../toxcore/util.h" - -#include -#include -#include - -#include -#include -#include -#include -#include -#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx()) -#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx()) - -/* Audio encoding/decoding */ -#include - -typedef struct CSession_s { - - /* VIDEO - * - * - */ - - /* video encoding */ - vpx_codec_ctx_t v_encoder[1]; - uint32_t frame_counter; - - /* video decoding */ - vpx_codec_ctx_t v_decoder[1]; - void *vbuf_raw; /* Un-decoded data */ - - /* Data handling */ - uint8_t *frame_buf; /* buffer for split video payloads */ - uint32_t frame_size; /* largest address written to in frame_buf for current input frame */ - uint8_t frameid_in, frameid_out; /* id of input and output video frame */ - uint64_t linfts; /* Last received frame time stamp */ - uint32_t lcfd; /* Last calculated frame duration for incoming video payload */ - - /* Limits */ - uint32_t peer_video_frame_piece_size; - - /* Splitting */ - uint8_t *split_video_frame; - const uint8_t *processing_video_frame; - uint16_t processing_video_frame_size; - - - - /* AUDIO - * - * - */ - - /* audio encoding */ - OpusEncoder *audio_encoder; - int32_t last_encoding_sampling_rate; - int32_t last_encoding_channel_count; - int32_t last_encoding_bitrate; - - /* audio decoding */ - OpusDecoder *audio_decoder; - int32_t last_packet_channel_count; - int32_t last_packet_sampling_rate; - int32_t last_packet_frame_duration; - int32_t last_decoding_sampling_rate; - int32_t last_decoding_channel_count; - uint64_t last_decoder_reconfiguration; - struct JitterBuffer_s *j_buf; - - - /* OTHER - * - * - */ - ToxAV *av; - int32_t friend_id; - - PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ - PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ - - pthread_mutex_t queue_mutex[1]; -} CSession; - - -void cs_do(CSession *cs); -/* Make sure to be called BEFORE corresponding rtp_new */ -CSession *cs_new(uint32_t peer_mvfpsz); -/* Make sure to be called AFTER corresponding rtp_kill */ -void cs_kill(CSession *cs); - -void cs_init_video_splitter_cycle(CSession *cs); -int cs_update_video_splitter_cycle(CSession* cs, const uint8_t* payload, uint16_t length); -const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size); - -int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height); -int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels); -#endif /* CODEC_H */ diff --git a/toxav/msi.c b/toxav/msi.c index b7926e07..0bd04c56 100644 --- a/toxav/msi.c +++ b/toxav/msi.c @@ -99,9 +99,9 @@ void handle_msi_packet ( Messenger *m, uint32_t friend_id, const uint8_t *data, */ void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) { - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); session->callbacks[id] = callback; - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); } MSISession *msi_new ( Messenger *messenger ) { @@ -141,7 +141,7 @@ int msi_kill ( MSISession *session ) } m_callback_msi_packet((struct Messenger *) session->messenger, NULL, NULL); - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if (session->calls) { MSIMessage msg; @@ -154,7 +154,7 @@ int msi_kill ( MSISession *session ) } } - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); pthread_mutex_destroy(session->mutex); LOGGER_DEBUG("Terminated session: %p", session); @@ -165,17 +165,17 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ { LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if (get_call(session, friend_id) != NULL) { LOGGER_ERROR("Already in a call"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } (*call) = new_call ( session, friend_id ); if ( *call == NULL ) { - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } @@ -195,7 +195,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_ (*call)->state = msi_CallRequesting; LOGGER_DEBUG("Invite sent"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } int msi_hangup ( MSICall* call ) @@ -203,7 +203,7 @@ int msi_hangup ( MSICall* call ) LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); MSISession* session = call->session; - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); MSIMessage msg; msg_init(&msg, requ_pop); @@ -211,7 +211,7 @@ int msi_hangup ( MSICall* call ) send_message ( session->messenger, call->friend_id, &msg ); kill_call(call); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } int msi_answer ( MSICall* call, uint8_t capabilities ) @@ -219,13 +219,13 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); MSISession* session = call->session; - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if ( call->state != msi_CallRequested ) { /* Though sending in invalid state will not cause anything wierd * Its better to not do it like a maniac */ LOGGER_ERROR("Call is in invalid state!"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } @@ -243,7 +243,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities ) send_message ( session->messenger, call->friend_id, &msg ); call->state = msi_CallActive; - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } @@ -252,7 +252,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id); MSISession* session = call->session; - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); if ( call->state != msi_CallActive ) { /* Sending capabilities change can cause error on other side if @@ -263,7 +263,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) * like new. TODO: explain this better */ LOGGER_ERROR("Call is in invalid state!"); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return -1; } @@ -277,7 +277,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities ) send_message ( call->session->messenger, call->friend_id, &msg ); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return 0; } @@ -349,7 +349,7 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length ) case IDVFPSZ: CHECK_SIZE(it, size_constraint, 2); SET_UINT16(it, dest->vfpsz); - dest->vfpsz = ntohs(dest->vfpsz); + dest->vfpsz.value = ntohs(dest->vfpsz.value); if (dest->vfpsz.value > 1200) { LOGGER_ERROR("Invalid vfpsz param"); @@ -425,7 +425,7 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg ) } if (msg->vfpsz.exists) { - uint16_t nb_vfpsz = htons(msg->vfpsz); + uint16_t nb_vfpsz = htons(msg->vfpsz.value); it = msg_parse_header_out(IDVFPSZ, it, &nb_vfpsz, sizeof(nb_vfpsz), &size); } @@ -588,17 +588,17 @@ void on_peer_status(Messenger* m, uint32_t friend_id, uint8_t status, void* data case 0: { /* Friend is now offline */ LOGGER_DEBUG("Friend %d is now offline", friend_id); - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); MSICall* call = get_call(session, friend_id); if (call == NULL) { - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return; } invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */ kill_call(call); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); } break; @@ -766,20 +766,20 @@ void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data, LOGGER_DEBUG("Successfully parsed message"); } - LOGGED_LOCK(session->mutex); + pthread_mutex_lock(session->mutex); MSICall *call = get_call(session, friend_id); if (call == NULL) { if (msg.request.value != requ_push) { send_error(m, friend_id, msi_EStrayMessage); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return; } call = new_call(session, friend_id); if (call == NULL) { send_error(m, friend_id, msi_ESystem); - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); return; } } @@ -789,5 +789,5 @@ void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data, else handle_pop(call, &msg); /* always kills the call */ - LOGGED_UNLOCK(session->mutex); + pthread_mutex_unlock(session->mutex); } diff --git a/toxav/rtp.c b/toxav/rtp.c index 9ef41b35..9657da67 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -68,12 +68,6 @@ typedef struct RTCPSession_s { } RTCPSession; - -/* These are defined externally */ -void ac_queue_message(void *acp, RTPMessage *msg); -void vc_queue_message(void *vcp, RTPMessage *msg); - - RTPHeader *parse_header_in ( const uint8_t *payload, int length ); RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ); RTPMessage *msg_parse ( const uint8_t *data, int length ); @@ -100,7 +94,7 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num ) retu->tstate = rtp_StateNormal; retu->m = messenger; - retu->dest = friend_num; + retu->friend_id = friend_num; if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) { LOGGER_WARNING("Alloc failed! Program might misbehave!"); @@ -155,7 +149,7 @@ void rtp_do(RTPSession *session) return; if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { - send_rtcp_report(session->rtcp_session, session->m, session->dest); + send_rtcp_report(session->rtcp_session, session->m, session->friend_id); } if (rb_full(session->rtcp_session->pl_stats)) { @@ -202,15 +196,15 @@ int rtp_start_receiving(RTPSession* session) if (session == NULL) return -1; - if (m_callback_rtp_packet(session->m, session->dest, session->prefix, + if (m_callback_rtp_packet(session->m, session->friend_id, session->prefix, handle_rtp_packet, session) == -1) { LOGGER_WARNING("Failed to register rtp receive handler"); return -1; } - if (m_callback_rtp_packet(session->m, session->dest, session->rtcp_session->prefix, + if (m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix, handle_rtcp_packet, session->rtcp_session) == -1) { LOGGER_WARNING("Failed to register rtcp receive handler"); - m_callback_rtp_packet(session->m, session->dest, session->prefix, NULL, NULL); + m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL); return -1; } @@ -221,8 +215,8 @@ int rtp_stop_receiving(RTPSession* session) if (session == NULL) return -1; - m_callback_rtp_packet(session->m, session->dest, session->prefix, NULL, NULL); - m_callback_rtp_packet(session->m, session->dest, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ + m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL); + m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ return 0; } @@ -253,7 +247,7 @@ int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length ) memcpy ( it, data, length ); - if ( -1 == send_custom_lossy_packet(session->m, session->dest, parsed, parsed_len) ) { + if ( -1 == send_custom_lossy_packet(session->m, session->friend_id, parsed, parsed_len) ) { LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); return -1; } @@ -546,7 +540,6 @@ void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber) } int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { - /* TODO on message callback */ RTPSession *session = object; RTPMessage *msg; @@ -578,20 +571,12 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data session->rtcp_session->last_received_packets ++; - /* Check if this session can handle the packet */ - if (session->payload_type != session->prefix % 128) { - LOGGER_WARNING("Friend %d sent invalid payload type!", session->dest); - rtp_free_msg(msg); - return -1; + if (session->mcb) + return session->mcb (session->cs, msg); + else { + rtp_free_msg(session, msg); + return 0; } - - /* Handle */ - if (session->payload_type == rtp_TypeAudio % 128) - ac_queue_message(session->cs, msg); - else /* It can only be video */ - vc_queue_message(session->cs, msg); - - return 0; } int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) { diff --git a/toxav/rtp.h b/toxav/rtp.h index a28ae7bc..3056b54e 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -42,11 +42,13 @@ #define MAX_RTP_SIZE 65535 /** - * Payload type identifier. Also used as rtp callback prefix. + * Payload type identifier. Also used as rtp callback prefix. (Not dummies) */ enum { rtp_TypeAudio = 192, - rtp_TypeVideo + rtp_TypeVideo, + rtp_TypeDummyAudio, + rtp_TypeDummyVideo, }; typedef enum { @@ -79,7 +81,7 @@ typedef struct { /** * Standard rtp message. */ -typedef struct { +typedef struct RTPMessage_s { RTPHeader *header; RTPExtHeader *ext_header; @@ -91,34 +93,36 @@ typedef struct { * RTP control session. */ typedef struct { - uint8_t version; - uint8_t padding; - uint8_t extension; - uint8_t cc; - uint8_t marker; - uint8_t payload_type; - uint16_t sequnum; /* Sending sequence number */ - uint16_t rsequnum; /* Receiving sequence number */ - uint32_t rtimestamp; - uint32_t ssrc; - uint32_t *csrc; + uint8_t version; + uint8_t padding; + uint8_t extension; + uint8_t cc; + uint8_t marker; + uint8_t payload_type; + uint16_t sequnum; /* Sending sequence number */ + uint16_t rsequnum; /* Receiving sequence number */ + uint32_t rtimestamp; + uint32_t ssrc; + uint32_t *csrc; /* If some additional data must be sent via message * apply it here. Only by allocating this member you will be * automatically placing it within a message. */ - RTPExtHeader *ext_header; + RTPExtHeader *ext_header; /* Msg prefix for core to know when recving */ - uint8_t prefix; - - int dest; + uint8_t prefix; + Messenger *m; + int friend_id; + RTPTransmissionState tstate; struct RTCPSession_s *rtcp_session; - struct CSession_s *cs; - Messenger *m; + + + void *cs; + int (*mcb) (void*, RTPMessage* msg); - RTPTransmissionState tstate; } RTPSession; /** diff --git a/toxav/toxav.c b/toxav/toxav.c index 6f712af9..5cb614d4 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -23,7 +23,8 @@ #include "config.h" #endif /* HAVE_CONFIG_H */ -#include "msi.h" /* Includes codec.h, rtp.h and toxav.h */ +#include "msi.h" +#include "rtp.h" #include "../toxcore/Messenger.h" #include "../toxcore/logger.h" @@ -35,20 +36,17 @@ #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) -enum { - audio_index, - video_index, -}; typedef struct ToxAVCall_s { ToxAV* av; - RTPSession *rtps[2]; /* Audio is first and video is second */ - CSession *cs; - pthread_mutex_t mutex_audio_sending[1]; - pthread_mutex_t mutex_video_sending[1]; - /* Only audio or video can be decoded at the time */ - pthread_mutex_t mutex_decoding[1]; + pthread_mutex_t mutex_audio[1]; + PAIR(RTPSession *, ACSession *) audio; + + pthread_mutex_t mutex_video[1]; + PAIR(RTPSession *, VCSession *) video; + + pthread_mutex_t mutex[1]; bool active; MSICall* msi_call; @@ -57,8 +55,8 @@ typedef struct ToxAVCall_s { uint32_t audio_bit_rate; /* Sending audio bitrate */ uint32_t video_bit_rate; /* Sending video bitrate */ - uint8_t last_self_capabilities; - uint8_t last_peer_capabilities; + /** Required for monitoring */ + uint8_t previous_self_capabilities; /** Quality control */ uint64_t time_audio_good; @@ -181,7 +179,7 @@ void toxav_kill(ToxAV* av) { if (av == NULL) return; - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); msi_kill(av->msi); @@ -194,7 +192,7 @@ void toxav_kill(ToxAV* av) } } - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); pthread_mutex_destroy(av->mutex); free(av); } @@ -212,9 +210,9 @@ uint32_t toxav_iteration_interval(const ToxAV* av) void toxav_iterate(ToxAV* av) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); if (av->calls == NULL) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return; } @@ -224,30 +222,36 @@ void toxav_iterate(ToxAV* av) ToxAVCall* i = av->calls[av->calls_head]; for (; i; i = i->next) { if (i->active) { - LOGGED_LOCK(i->mutex_decoding); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_lock(i->mutex); + pthread_mutex_unlock(av->mutex); + + rtp_do(i->audio.first); + ac_do(i->audio.second); + + rtp_do(i->video.first); + vc_do(i->video.second); - cs_do(i->cs); - rtp_do(i->rtps[audio_index]); - rtp_do(i->rtps[video_index]); qc_do(i); - if (i->last_self_capabilities & msi_CapRAudio) /* Receiving audio */ - rc = MIN(i->cs->last_packet_frame_duration, rc); - if (i->last_self_capabilities & msi_CapRVideo) /* Receiving video */ - rc = MIN(i->cs->lcfd, rc); /* TODO handle on/off */ + if (i->msi_call->self_capabilities & msi_CapRAudio && + i->msi_call->peer_capabilities & msi_CapSAudio) + rc = MIN(i->audio.second->last_packet_frame_duration, rc); + + if (i->msi_call->self_capabilities & msi_CapRVideo && + i->msi_call->peer_capabilities & msi_CapSVideo) + rc = MIN(i->video.second->lcfd, rc); uint32_t fid = i->friend_id; - LOGGED_UNLOCK(i->mutex_decoding); - LOGGED_LOCK(av->mutex); + pthread_mutex_unlock(i->mutex); + pthread_mutex_lock(av->mutex); /* In case this call is popped from container stop iteration */ if (call_get(av, fid) != i) break; } } - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); av->dmsst += current_time_monotonic() - start; @@ -261,46 +265,46 @@ void toxav_iterate(ToxAV* av) bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); ToxAVCall* call = call_new(av, friend_number, error); if (call == NULL) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return false; } call->audio_bit_rate = audio_bit_rate; call->video_bit_rate = video_bit_rate; - call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; + call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo; - call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_invite(av->msi, &call->msi_call, friend_number, call->last_self_capabilities) != 0) { + if (msi_invite(av->msi, &call->msi_call, friend_number, call->previous_self_capabilities) != 0) { call_remove(call); if (error) *error = TOXAV_ERR_CALL_MALLOC; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return false; } call->msi_call->av_call = call; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); return true; } void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->ccb.first = function; av->ccb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; if (m_friend_exists(av->m, friend_number) == 0) { @@ -329,17 +333,17 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui call->audio_bit_rate = audio_bit_rate; call->video_bit_rate = video_bit_rate; - call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; + call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo; - call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; - call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; + call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; + call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; - if (msi_answer(call->msi_call, call->last_self_capabilities) != 0) + if (msi_answer(call->msi_call, call->previous_self_capabilities) != 0) rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */ END: - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); if (error) *error = rc; @@ -349,15 +353,15 @@ END: void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->scb.first = function; av->scb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; if (m_friend_exists(av->m, friend_number) == 0) { @@ -381,18 +385,18 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co /* Only act if paused and had media transfer active before */ if (call->msi_call->self_capabilities == 0 && - call->last_self_capabilities ) { + call->previous_self_capabilities ) { if (msi_change_capabilities(call->msi_call, - call->last_self_capabilities) == -1) { + call->previous_self_capabilities) == -1) { /* The only reason for this function to fail is invalid state * ( not active ) */ rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; goto END; } - rtp_start_receiving(call->rtps[audio_index]); - rtp_start_receiving(call->rtps[video_index]); + rtp_start_receiving(call->audio.first); + rtp_start_receiving(call->video.first); } } break; @@ -404,7 +408,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co /* Only act if not already paused */ if (call->msi_call->self_capabilities) { - call->last_self_capabilities = call->msi_call->self_capabilities; + call->previous_self_capabilities = call->msi_call->self_capabilities; if (msi_change_capabilities(call->msi_call, 0) == -1 ) { /* The only reason for this function to fail is invalid state @@ -413,8 +417,8 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_stop_receiving(call->rtps[audio_index]); - rtp_stop_receiving(call->rtps[video_index]); + rtp_stop_receiving(call->audio.first); + rtp_stop_receiving(call->video.first); } } break; @@ -442,7 +446,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_stop_receiving(call->rtps[audio_index]); + rtp_stop_receiving(call->audio.first); } else { /* This call was already muted so notify the friend that he can * start sending audio again @@ -455,7 +459,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_start_receiving(call->rtps[audio_index]); + rtp_start_receiving(call->audio.first); } } break; @@ -474,7 +478,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_stop_receiving(call->rtps[video_index]); + rtp_stop_receiving(call->video.first); } else { /* This call was already muted so notify the friend that he can * start sending video again @@ -487,13 +491,13 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co goto END; } - rtp_start_receiving(call->rtps[video_index]); + rtp_start_receiving(call->video.first); } } break; } END: - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); if (error) *error = rc; @@ -516,19 +520,19 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_ goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; goto END; } /* Decoding mutex is locked because of quality control */ - LOGGED_LOCK(call->mutex_decoding); + pthread_mutex_lock(call->mutex); call->audio_bit_rate = audio_bit_rate; - LOGGED_UNLOCK(call->mutex_decoding); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); END: if (error) @@ -552,19 +556,19 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_ goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; goto END; } /* Decoding mutex is locked because of quality control */ - LOGGED_LOCK(call->mutex_decoding); + pthread_mutex_lock(call->mutex); call->video_bit_rate = video_bit_rate; - LOGGED_UNLOCK(call->mutex_decoding); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(call->mutex); + pthread_mutex_unlock(av->mutex); END: if (error) @@ -575,10 +579,10 @@ END: void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->rvcb.first = function; av->rvcb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) @@ -591,25 +595,25 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - LOGGED_LOCK(call->mutex_video_sending); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_lock(call->mutex_video); + pthread_mutex_unlock(av->mutex); if ( y == NULL || u == NULL || v == NULL ) { - LOGGED_UNLOCK(call->mutex_video_sending); + pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) { - LOGGED_UNLOCK(call->mutex_video_sending); + if ( vc_reconfigure_encoder(call->video.second, call->video_bit_rate, width, height) != 0 ) { + pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -626,29 +630,29 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); - int vrc = vpx_codec_encode(call->cs->v_encoder, &img, - call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); + int vrc = vpx_codec_encode(call->video.second->v_encoder, &img, + call->video.second->frame_counter, 1, 0, MAX_ENCODE_TIME_US); vpx_img_free(&img); if ( vrc != VPX_CODEC_OK) { - LOGGED_UNLOCK(call->mutex_video_sending); + pthread_mutex_unlock(call->mutex_video); LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } } - ++call->cs->frame_counter; + ++call->video.second->frame_counter; { /* Split and send */ vpx_codec_iter_t iter = NULL; const vpx_codec_cx_pkt_t *pkt; - cs_init_video_splitter_cycle(call->cs); + vc_init_video_splitter_cycle(call->video.second); - while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { + while ( (pkt = vpx_codec_get_cx_data(call->video.second->v_encoder, &iter)) ) { if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - int parts = cs_update_video_splitter_cycle(call->cs, pkt->data.frame.buf, + int parts = vc_update_video_splitter_cycle(call->video.second, pkt->data.frame.buf, pkt->data.frame.sz); if (parts < 0) /* Should never happen though */ @@ -659,10 +663,10 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u int i; for (i = 0; i < parts; i++) { - iter = cs_iterate_split_video_frame(call->cs, &part_size); + iter = vc_iterate_split_video_frame(call->video.second, &part_size); - if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) { - LOGGED_UNLOCK(call->mutex_video_sending); + if (rtp_send_msg(call->video.first, iter, part_size) < 0) { + pthread_mutex_unlock(call->mutex_video); LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); goto END; } @@ -671,7 +675,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u } } - LOGGED_UNLOCK(call->mutex_video_sending); + pthread_mutex_unlock(call->mutex_video); END: if (error) @@ -682,10 +686,10 @@ END: void toxav_callback_audio_frame_request(ToxAV* av, toxav_audio_frame_request_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->racb.first = function; av->racb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) @@ -698,32 +702,32 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; } - LOGGED_LOCK(call->mutex_audio_sending); - LOGGED_UNLOCK(av->mutex); + pthread_mutex_lock(call->mutex_audio); + pthread_mutex_unlock(av->mutex); if ( pcm == NULL ) { - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } if ( channels > 2 ) { - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } { /* Encode and send */ - if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { - LOGGED_UNLOCK(call->mutex_audio_sending); + if (ac_reconfigure_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -732,12 +736,12 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc sampling_rate = htonl(sampling_rate); memcpy(dest, &sampling_rate, sizeof(sampling_rate)); - int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, + int vrc = opus_encode(call->audio.second->encoder, pcm, sample_count, dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); if (vrc < 0) { LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -745,13 +749,13 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc // LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels, // ntohl(sampling_rate)); - if (rtp_send_msg(call->rtps[audio_index], dest, vrc + sizeof(sampling_rate)) != 0) { + if (rtp_send_msg(call->audio.first, dest, vrc + sizeof(sampling_rate)) != 0) { LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } } - LOGGED_UNLOCK(call->mutex_audio_sending); + pthread_mutex_unlock(call->mutex_audio); END: if (error) @@ -762,18 +766,18 @@ END: void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->vcb.first = function; av->vcb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) { - LOGGED_LOCK(av->mutex); + pthread_mutex_lock(av->mutex); av->acb.first = function; av->acb.second = user_data; - LOGGED_UNLOCK(av->mutex); + pthread_mutex_unlock(av->mutex); } @@ -785,12 +789,12 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* int callback_invite(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL); if (av_call == NULL) { LOGGER_WARNING("Failed to initialize call..."); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return -1; } @@ -801,72 +805,72 @@ int callback_invite(void* toxav_inst, MSICall* call) toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_start(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); ToxAVCall* av_call = call_get(toxav, call->friend_id); if (av_call == NULL) { /* Should this ever happen? */ - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return -1; } if (!call_prepare_transmission(av_call)) { callback_error(toxav_inst, call); call_remove(av_call); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return -1; } invoke_call_state(toxav, call->friend_id, call->peer_capabilities); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_end(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_END); call_kill_transmission(call->av_call); call_remove(call->av_call); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_error(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR); call_kill_transmission(call->av_call); call_remove(call->av_call); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } int callback_capabilites(void* toxav_inst, MSICall* call) { ToxAV* toxav = toxav_inst; - LOGGED_LOCK(toxav->mutex); + pthread_mutex_lock(toxav->mutex); invoke_call_state(toxav, call->friend_id, call->peer_capabilities); - LOGGED_UNLOCK(toxav->mutex); + pthread_mutex_unlock(toxav->mutex); return 0; } @@ -982,10 +986,8 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number) void qc_do(ToxAVCall* call) { - /* Please NOTE: The quality control is rather basic, - * advanced algorithms will be applied in the future - */ - switch(call->rtps[audio_index]->tstate) { + /* + switch(call->audio.first->tstate) { case rtp_StateBad: LOGGER_DEBUG("Suggesting lower bitrate for audio..."); call->time_audio_good = 0; @@ -1007,9 +1009,9 @@ void qc_do(ToxAVCall* call) case rtp_StateNormal: call->time_audio_good = 0; break; - } - - switch(call->rtps[video_index]->tstate) { + }*/ + /* + switch(call->video.first->tstate) { case rtp_StateBad: LOGGER_DEBUG("Suggesting lower bitrate for video..."); call->time_video_good = 0; @@ -1030,8 +1032,7 @@ void qc_do(ToxAVCall* call) case rtp_StateNormal: call->time_video_good = 0; break; - } - + }*/ } void call_remove(ToxAVCall* call) @@ -1086,61 +1087,50 @@ bool call_prepare_transmission(ToxAVCall* call) return true; } - if (create_recursive_mutex(call->mutex_audio_sending) != 0) + if (create_recursive_mutex(call->mutex_audio) != 0) return false; - if (create_recursive_mutex(call->mutex_video_sending) != 0) { + if (create_recursive_mutex(call->mutex_video) != 0) { goto AUDIO_SENDING_MUTEX_CLEANUP; } - if (create_recursive_mutex(call->mutex_decoding) != 0) { + if (create_recursive_mutex(call->mutex) != 0) { goto VIDEO_SENDING_MUTEX_CLEANUP; } - /* Creates both audio and video encoders and decoders with some default values. - * Make sure to reconfigure encoders dynamically when sending data - */ - call->cs = cs_new(call->msi_call->peer_vfpsz); - if ( !call->cs ) { - LOGGER_ERROR("Error while starting Codec State!\n"); - goto FAILURE; - } - - call->cs->av = av; - call->cs->friend_id = call->friend_id; - - memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); - memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); - - { /* Prepare audio RTP */ - call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); + { /* Prepare audio */ + call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_id); + call->audio.second = ac_new(av, call->friend_id, av->acb.first, av->acb.second); - if ( !call->rtps[audio_index] ) { - LOGGER_ERROR("Error while starting audio RTP session!\n"); + if ( !call->audio.first || !call->audio.second ) { + LOGGER_ERROR("Error while starting audio!\n"); goto FAILURE; } - call->rtps[audio_index]->cs = call->cs; + call->audio.first->cs = call->audio.second; + call->audio.first->mcb = ac_queue_message; - if (rtp_start_receiving(call->rtps[audio_index]) != 0) { + if (rtp_start_receiving(call->audio.first) != 0) { LOGGER_WARNING("Failed to enable audio receiving!"); goto FAILURE; } } - { /* Prepare video RTP */ - call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); + { /* Prepare video */ + call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_id); + call->video.second = vc_new(av, call->friend_id, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz); - if ( !call->rtps[video_index] ) { - LOGGER_ERROR("Error while starting video RTP session!\n"); + if ( !call->video.first || !call->video.second ) { + LOGGER_ERROR("Error while starting video!\n"); goto FAILURE; } - call->rtps[video_index]->cs = call->cs; + call->video.first->cs = call->video.second; + call->video.first->mcb = vc_queue_message; - if (rtp_start_receiving(call->rtps[video_index]) != 0) { - LOGGER_WARNING("Failed to enable audio receiving!"); + if (rtp_start_receiving(call->video.first) != 0) { + LOGGER_WARNING("Failed to enable video receiving!"); goto FAILURE; } } @@ -1149,17 +1139,19 @@ bool call_prepare_transmission(ToxAVCall* call) return true; FAILURE: - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; - cs_kill(call->cs); - call->cs = NULL; - pthread_mutex_destroy(call->mutex_decoding); + rtp_kill(call->audio.first); + ac_kill(call->audio.second); + call->audio.first = NULL; + call->audio.second = NULL; + rtp_kill(call->video.first); + vc_kill(call->video.second); + call->video.first = NULL; + call->video.second = NULL; + pthread_mutex_destroy(call->mutex); VIDEO_SENDING_MUTEX_CLEANUP: - pthread_mutex_destroy(call->mutex_video_sending); + pthread_mutex_destroy(call->mutex_video); AUDIO_SENDING_MUTEX_CLEANUP: - pthread_mutex_destroy(call->mutex_audio_sending); + pthread_mutex_destroy(call->mutex_audio); return false; } @@ -1170,23 +1162,24 @@ void call_kill_transmission(ToxAVCall* call) call->active = 0; - LOGGED_LOCK(call->mutex_audio_sending); - LOGGED_UNLOCK(call->mutex_audio_sending); - LOGGED_LOCK(call->mutex_video_sending); - LOGGED_UNLOCK(call->mutex_video_sending); - LOGGED_LOCK(call->mutex_decoding); - LOGGED_UNLOCK(call->mutex_decoding); + pthread_mutex_lock(call->mutex_audio); + pthread_mutex_unlock(call->mutex_audio); + pthread_mutex_lock(call->mutex_video); + pthread_mutex_unlock(call->mutex_video); + pthread_mutex_lock(call->mutex); + pthread_mutex_unlock(call->mutex); + rtp_kill(call->audio.first); + ac_kill(call->audio.second); + call->audio.first = NULL; + call->audio.second = NULL; - rtp_kill(call->rtps[audio_index]); - call->rtps[audio_index] = NULL; - rtp_kill(call->rtps[video_index]); - call->rtps[video_index] = NULL; + rtp_kill(call->video.first); + vc_kill(call->video.second); + call->video.first = NULL; + call->video.second = NULL; - cs_kill(call->cs); - call->cs = NULL; - - pthread_mutex_destroy(call->mutex_audio_sending); - pthread_mutex_destroy(call->mutex_video_sending); - pthread_mutex_destroy(call->mutex_decoding); + pthread_mutex_destroy(call->mutex_audio); + pthread_mutex_destroy(call->mutex_video); + pthread_mutex_destroy(call->mutex); } diff --git a/toxav/video.c b/toxav/video.c index d51cfd4a..039fc2a0 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -24,6 +24,7 @@ #include "video.h" #include "msi.h" +#include "rtp.h" #include "../toxcore/logger.h" #include "../toxcore/network.h" @@ -78,7 +79,9 @@ VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* c vc->linfts = current_time_monotonic(); vc->lcfd = 60; - + vc->vcb.first = cb; + vc->vcb.second = cb_data; + vc->friend_id = friend_id; vc->peer_video_frame_piece_size = mvfpsz; return vc; @@ -187,35 +190,25 @@ const uint8_t* vc_iterate_split_video_frame(VCSession* vc, uint16_t* size) return vc->split_video_frame; } -int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) -{ - if (!vc) - return; - - vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc; - if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) - return 0; /* Nothing changed */ - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = width; - cfg.g_h = height; - - int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg); - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - return 0; -} -/* Called from RTP */ -void vc_queue_message(void* vcp, RTPMessage *msg) +int vc_queue_message(void* vcp, struct RTPMessage_s *msg) { /* This function does the reconstruction of video packets. * See more info about video splitting in docs */ if (!vcp || !msg) - return; + return -1; + + if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyVideo % 128) { + LOGGER_WARNING("Got dummy!"); + rtp_free_msg(NULL, msg); + return 0; + } + + if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeVideo % 128) { + LOGGER_WARNING("Invalid payload type!"); + rtp_free_msg(NULL, msg); + return -1; + } VCSession* vc = vcp; @@ -233,7 +226,7 @@ void vc_queue_message(void* vcp, RTPMessage *msg) Payload *p = malloc(sizeof(Payload) + vc->frame_size); if (p) { - LOGGED_LOCK(vc->queue_mutex); + pthread_mutex_lock(vc->queue_mutex); if (rb_full(vc->vbuf_raw)) { LOGGER_DEBUG("Dropped video frame"); @@ -251,7 +244,7 @@ void vc_queue_message(void* vcp, RTPMessage *msg) vc->linfts = current_time_monotonic(); rb_write(vc->vbuf_raw, p); - LOGGED_UNLOCK(vc->queue_mutex); + pthread_mutex_unlock(vc->queue_mutex); } else { LOGGER_WARNING("Allocation failed! Program might misbehave!"); goto end; @@ -288,7 +281,30 @@ void vc_queue_message(void* vcp, RTPMessage *msg) end: rtp_free_msg(NULL, msg); + return 0; } +int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) +{ + if (!vc) + return; + + vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc; + if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + return 0; /* Nothing changed */ + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = width; + cfg.g_h = height; + + int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg); + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + return -1; + } + + return 0; +} + bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) diff --git a/toxav/video.h b/toxav/video.h index c1678ad2..ed264f36 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -36,6 +36,8 @@ #include "../toxcore/util.h" +struct RTPMessage_s; + typedef struct VCSession_s { /* encoding */ @@ -76,6 +78,7 @@ void vc_do(VCSession* vc); void vc_init_video_splitter_cycle(VCSession* vc); int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length); const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size); +int vc_queue_message(void *vcp, struct RTPMessage_s *msg); int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); #endif /* VIDEO_H */ \ No newline at end of file