From fdaad0b7c008b65646ca5cbd62ac0d305617ecfe Mon Sep 17 00:00:00 2001 From: mannol Date: Sun, 29 Mar 2015 01:10:34 +0100 Subject: [PATCH] A little CS cleanup --- toxav/av_test.c | 108 +++++------ toxav/codec.c | 479 +++++++++++++++++++----------------------------- toxav/codec.h | 81 ++------ toxav/rtp.h | 2 +- toxav/toxav.c | 160 +++++++++------- toxav/toxav.h | 10 +- 6 files changed, 363 insertions(+), 477 deletions(-) diff --git a/toxav/av_test.c b/toxav/av_test.c index 5ca53eb9..f0ad6a01 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -134,14 +134,14 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, alDeleteBuffers(processed - 1, bufids + 1); bufid = bufids[0]; } -// else if(queued < 16) + else if(queued < 16) alGenBuffers(1, &bufid); -// else -// return; + else + return; alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, - pcm, sample_count * channels * 2, sampling_rate); + pcm, sample_count * 2, sampling_rate); alSourceQueueBuffers(adout, 1, &bufid); int32_t state; @@ -285,6 +285,36 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) return rc; } +ALCdevice* open_audio_device(const char* audio_out_dev_name) +{ + ALCdevice* rc; + rc = alcOpenDevice(audio_out_dev_name); + if ( !rc ) { + printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError()); + exit(1); + } + + ALCcontext* out_ctx = alcCreateContext(rc, NULL); + alcMakeContextCurrent(out_ctx); + + uint32_t buffers[10]; + alGenBuffers(10, buffers); + alGenSources((uint32_t)1, &adout); + alSourcei(adout, AL_LOOPING, AL_FALSE); + + int16_t zeros[10000]; + memset(zeros, 0, sizeof(zeros)); + + int i; + for ( i = 0; i < 10; ++i ) + alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, sizeof(zeros), 48000); + + alSourceQueueBuffers(adout, 10, buffers); + alSourcePlay(adout); + + return rc; +} + int print_audio_devices() { const char *device; @@ -373,49 +403,18 @@ int main (int argc, char** argv) } } - ALCdevice* audio_out_device; - - { /* Open output device */ - const char* audio_out_dev_name = NULL; - - int i = 0; - for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; - audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i) - if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1)) - break; - - audio_out_device = alcOpenDevice(audio_out_dev_name); - if ( !audio_out_device ) { - printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError()); - exit(1); - } - - ALCcontext* out_ctx = alcCreateContext(audio_out_device, NULL); - alcMakeContextCurrent(out_ctx); - - uint32_t buffers[5]; - alGenBuffers(5, buffers); - alGenSources((uint32_t)1, &adout); - alSourcei(adout, AL_LOOPING, AL_FALSE); - - uint16_t zeros[10000]; - memset(zeros, 0, 10000); - - for ( i = 0; i < 5; ++i ) - alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, 10000, 48000); - - alSourceQueueBuffers(adout, 5, buffers); - alSourcePlay(adout); - - printf("Using audio device: %s\n", audio_out_dev_name); - } + const char* audio_out_dev_name = NULL; + + int i = 0; + for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; + audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i) + if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1)) + break; + printf("Using audio device: %s\n", audio_out_dev_name); printf("Using audio file: %s\n", af_name); printf("Using video file: %s\n", vf_name); - - - /* START TOX NETWORK */ @@ -697,11 +696,11 @@ int main (int argc, char** argv) printf("Failed to open the file.\n"); exit(1); } + ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name); - /* Run for 5 seconds */ uint32_t frame_duration = 10; - int16_t PCM[10000]; + int16_t PCM[5760]; time_t start_time = time(NULL); time_t expected_time = af_info.frames / af_info.samplerate + 2; @@ -711,18 +710,21 @@ int main (int argc, char** argv) int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { - TOXAV_ERR_SEND_FRAME rc; - if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { - printf("Error sending frame of size %ld: %d\n", count, rc); - exit(1); - } + t_toxav_receive_audio_frame_cb(BobAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); +// TOXAV_ERR_SEND_FRAME rc; +// if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { +// printf("Error sending frame of size %ld: %d\n", count, rc); +// exit(1); +// } } - - iterate_tox(bootstrap, AliceAV, BobAV); + c_sleep(frame_duration); +// iterate_tox(bootstrap, AliceAV, BobAV); } + printf("Played file in: %lu\n", time(NULL) - start_time); + alcCloseDevice(audio_out_device); sf_close(af_handle); { /* Hangup */ @@ -774,7 +776,5 @@ int main (int argc, char** argv) tox_kill(bootstrap); printf("\nTest successful!\n"); - - alcCloseDevice(audio_out_device); return 0; } diff --git a/toxav/codec.c b/toxav/codec.c index b9cbbc06..b0f2ed31 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -238,10 +238,80 @@ static int convert_bw_to_sampling_rate(int bw) } } +OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count) +{ + int status = OPUS_OK; + OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); + return NULL; + } + + status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + goto FAILURE; + } + + return rc; + +FAILURE: + opus_encoder_destroy(rc); + return NULL; +} + +bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) +{ + assert(dest); + + vpx_codec_enc_cfg_t cfg; + int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); + return false; + } + + rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, + VPX_ENCODER_ABI_VERSION); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); + return false; + } + + cfg.rc_target_bitrate = bitrate; + cfg.g_w = 800; + cfg.g_h = 600; + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + cfg.g_lag_in_frames = 0; + cfg.kf_min_dist = 0; + cfg.kf_max_dist = 48; + cfg.kf_mode = VPX_KF_AUTO; + + rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + vpx_codec_destroy(dest); + } + + return true; +} /* PUBLIC */ -void cs_do(CSSession *cs) +void cs_do(CSession *cs) { /* Codec session should always be protected by call mutex so no need to check for cs validity */ @@ -261,7 +331,7 @@ void cs_do(CSSession *cs) RTPMessage *msg; /* The maximum for 120 ms 48 KHz audio */ - int16_t tmp[20000]; + int16_t tmp[5760]; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { pthread_mutex_unlock(cs->queue_mutex); @@ -276,7 +346,7 @@ void cs_do(CSSession *cs) rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; - cs->last_packet_channels = opus_packet_get_nb_channels(msg->data); + cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data); cs->last_packet_frame_duration = ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) @@ -287,7 +357,7 @@ void cs_do(CSSession *cs) continue; } - rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, sizeof(tmp), 0); + rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, 5760, 0); rtp_free_msg(NULL, msg); } @@ -295,21 +365,11 @@ void cs_do(CSSession *cs) LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { /* Play */ - LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc, - cs->last_packet_channels, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); - /* According to https://tools.ietf.org/html/rfc6716#section-2.1.2 - * Every encoder can encode both mono and stereo data so we must - * determine which format is selected. - */ - - if (cs->last_packet_channels == 2) { - /* The packet is encoded with stereo encoder */ - } - - cs->acb.first(cs->agent, cs->friend_id, tmp, rc, - cs->last_packet_channels, cs->last_packet_sampling_rate, cs->acb.second); + cs->acb.first(cs->av, cs->friend_id, tmp, rc, + cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second); } pthread_mutex_lock(cs->queue_mutex); @@ -336,7 +396,7 @@ void cs_do(CSSession *cs) /* Play decoded images */ for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { if (cs->vcb.first) - cs->vcb.first(cs->agent, cs->friend_id, dest->d_w, dest->d_h, + cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h, (const uint8_t**)dest->planes, dest->stride, cs->vcb.second); vpx_img_free(dest); @@ -349,29 +409,103 @@ void cs_do(CSSession *cs) pthread_mutex_unlock(cs->queue_mutex); } -CSSession *cs_new(uint32_t peer_video_frame_piece_size) +CSession *cs_new(uint32_t peer_video_frame_piece_size) { - CSSession *cs = calloc(sizeof(CSSession), 1); + CSession *cs = calloc(sizeof(CSession), 1); if (!cs) { LOGGER_WARNING("Allocation failed! Application might misbehave!"); return NULL; } - if (create_recursive_mutex(cs->queue_mutex) != 0) { LOGGER_WARNING("Failed to create recursive mutex!"); free(cs); return NULL; } + /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ + /* Create decoders and set up their values + */ + + /* + * AUDIO + */ + + int status; + cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be mono */ + + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); + goto FAILURE; + } + + /* These need to be set in order to properly + * do error correction with opus */ + cs->last_packet_frame_duration = 120; + cs->last_packet_sampling_rate = 48000; + + if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { + LOGGER_WARNING("Jitter buffer creaton failed!"); + opus_decoder_destroy(cs->audio_decoder); + goto FAILURE; + } + + /* + * VIDEO + */ + int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, + NULL, 0, VPX_DECODER_ABI_VERSION); + + if ( rc != VPX_CODEC_OK) { + LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); + goto AUDIO_DECODER_CLEANUP; + } + + if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) { + vpx_codec_destroy(cs->v_decoder); + goto AUDIO_DECODER_CLEANUP; + } + + if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) { + free(cs->frame_buf); + vpx_codec_destroy(cs->v_decoder); + goto AUDIO_DECODER_CLEANUP; + } + /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ + + /* Initialize encoders with default values */ + cs->audio_encoder = create_audio_encoder(48000, 48000, 2); + if (cs->audio_encoder == NULL) + goto VIDEO_DECODER_CLEANUP; + + cs->last_encoding_bitrate = 48000; + cs->last_encoding_sampling_rate = 48000; + cs->last_encoding_channel_count = 2; + + if (!create_video_encoder(cs->v_encoder, 500000)) { + opus_encoder_destroy(cs->audio_encoder); + goto VIDEO_DECODER_CLEANUP; + } cs->peer_video_frame_piece_size = peer_video_frame_piece_size; return cs; + +VIDEO_DECODER_CLEANUP: + buffer_free(cs->vbuf_raw); + free(cs->frame_buf); + vpx_codec_destroy(cs->v_decoder); +AUDIO_DECODER_CLEANUP: + opus_decoder_destroy(cs->audio_decoder); + jbuf_free(cs->j_buf); +FAILURE: + pthread_mutex_destroy(cs->queue_mutex); + free(cs); + return NULL; } -void cs_kill(CSSession *cs) +void cs_kill(CSession *cs) { if (!cs) return; @@ -380,10 +514,13 @@ void cs_kill(CSSession *cs) * the callback is unregistered before cs_kill is called. */ - cs_disable_audio_sending(cs); - cs_disable_audio_receiving(cs); - cs_disable_video_sending(cs); - cs_disable_video_receiving(cs); + vpx_codec_destroy(cs->v_encoder); + vpx_codec_destroy(cs->v_decoder); + opus_encoder_destroy(cs->audio_encoder); + opus_decoder_destroy(cs->audio_decoder); + buffer_free(cs->vbuf_raw); + jbuf_free(cs->j_buf); + free(cs->frame_buf); pthread_mutex_destroy(cs->queue_mutex); @@ -391,15 +528,13 @@ void cs_kill(CSSession *cs) free(cs); } - - -void cs_init_video_splitter_cycle(CSSession* cs) +void cs_init_video_splitter_cycle(CSession* cs) { cs->split_video_frame[0] = cs->frameid_out++; cs->split_video_frame[1] = 0; } -int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16_t length) +int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length) { cs->processing_video_frame = payload; cs->processing_video_frame_size = length; @@ -407,7 +542,7 @@ int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16 return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; } -const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size) +const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size) { if (!cs || !size) return NULL; @@ -433,295 +568,61 @@ const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size) return cs->split_video_frame; } - - -int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height) +int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height) { - if (!cs->v_encoding) - return -1; - - /* TODO FIXME reference is safe? */ vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; + if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height) + return 0; /* Nothing changed */ - if (cfg.g_w == width && cfg.g_h == height) - return 0; -/* - if (width * height > cs->max_width * cs->max_height) { - vpx_codec_ctx_t v_encoder = cs->v_encoder; - - if (init_video_encoder(cs, width, height, cs->video_bitrate) == -1) { - cs->v_encoder = v_encoder; - return cs_ErrorSettingVideoResolution; - } - - vpx_codec_destroy(&v_encoder); - return 0; - }*/ - - LOGGER_DEBUG("New video resolution: %u %u", width, height); + cfg.rc_target_bitrate = bitrate; cfg.g_w = width; cfg.g_h = height; - int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return cs_ErrorSettingVideoResolution; - } - - return 0; -} - -int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate) -{ - if (!cs->v_encoding) - return -1; - - /* TODO FIXME reference is safe? */ - vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; - if (cfg.rc_target_bitrate == bitrate) - return 0; - - LOGGER_DEBUG("New video bitrate: %u", bitrate); - cfg.rc_target_bitrate = bitrate; int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); if ( rc != VPX_CODEC_OK) { LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - return cs_ErrorSettingVideoBitrate; + return -1; } return 0; } - - -int cs_enable_video_sending(CSSession* cs, uint32_t bitrate) +int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels) { - if (cs->v_encoding) - return 0; + /* Values are checked in toxav.c */ - vpx_codec_enc_cfg_t cfg; - int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); - - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - rc = vpx_codec_enc_init_ver(cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, - VPX_ENCODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - /* So that we can use cs_disable_video_sending to clean up */ - cs->v_encoding = true; - - if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) - goto FAILURE; - - cfg.rc_target_bitrate = bitrate; - cfg.g_w = 800; - cfg.g_h = 600; - cfg.g_pass = VPX_RC_ONE_PASS; - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; - - - rc = vpx_codec_control(cs->v_encoder, VP8E_SET_CPUUSED, 8); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); - goto FAILURE; - } - - return 0; - -FAILURE: - cs_disable_video_sending(cs); - return -1; -} - -int cs_enable_video_receiving(CSSession* cs) -{ - if (cs->v_decoding) - return 0; - - int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, - NULL, 0, VPX_DECODER_ABI_VERSION); - - if ( rc != VPX_CODEC_OK) { - LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); - return -1; - } - - /* So that we can use cs_disable_video_sending to clean up */ - cs->v_decoding = true; - - if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) - goto FAILURE; - - if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) - goto FAILURE; - - return 0; - -FAILURE: - cs_disable_video_receiving(cs); - return -1; -} - - - -void cs_disable_video_sending(CSSession* cs) -{ - if (cs->v_encoding) { - cs->v_encoding = false; + if (cs->last_encoding_sampling_rate != sampling_rate || cs->last_encoding_channel_count != channels) { + OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels); + if (new_encoder == NULL) + return -1; - free(cs->split_video_frame); - cs->split_video_frame = NULL; - - vpx_codec_destroy(cs->v_encoder); - } -} - -void cs_disable_video_receiving(CSSession* cs) -{ - if (cs->v_decoding) { - cs->v_decoding = false; - - buffer_free(cs->vbuf_raw); - cs->vbuf_raw = NULL; - free(cs->frame_buf); - cs->frame_buf = NULL; - - vpx_codec_destroy(cs->v_decoder); - } -} - - - -int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) -{ - if (cs->audio_encoder == NULL) - return -1; - - int rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(rate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - LOGGER_DEBUG("Set new encoder bitrate to: %d", rate); - return 0; -} - - - -void cs_disable_audio_sending(CSSession* cs) -{ - if ( cs->audio_encoder ) { opus_encoder_destroy(cs->audio_encoder); - cs->audio_encoder = NULL; - } -} - -void cs_disable_audio_receiving(CSSession* cs) -{ - if ( cs->audio_decoder ) { - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = NULL; - jbuf_free(cs->j_buf); - cs->j_buf = NULL; + cs->audio_encoder = new_encoder; + } else if (cs->last_encoding_bitrate == bitrate) + return 0; /* Nothing changed */ + else { + int status = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); - /* These need to be set in order to properly - * do error correction with opus */ - cs->last_packet_frame_duration = 120; - cs->last_packet_sampling_rate = 48000; + if ( status != OPUS_OK ) { + LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); + return -1; + } } -} - - -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate) -{ - if (cs->audio_encoder) - return 0; - - int rc = OPUS_OK; - cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); - return -1; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - goto FAILURE; - } - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); - goto FAILURE; - } - - return 0; - -FAILURE: - cs_disable_audio_sending(cs); - return -1; -} - -int cs_enable_audio_receiving(CSSession* cs) -{ - if (cs->audio_decoder) - return 0; - - int rc; - cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); - return -1; - } - - - if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) { - LOGGER_WARNING("Jitter buffer creaton failed!"); - opus_decoder_destroy(cs->audio_decoder); - cs->audio_decoder = NULL; - return -1; - } - - - /* These need to be set in order to properly - * do error correction with opus */ - cs->last_packet_frame_duration = 120; - cs->last_packet_sampling_rate = 48000; - + cs->last_encoding_bitrate = bitrate; + cs->last_encoding_sampling_rate = sampling_rate; + cs->last_encoding_channel_count = channels; return 0; } - /* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) { /* This function is unregistered during call termination befor destroying * Codec session so no need to check for validity of cs TODO properly check video cycle */ - CSSession *cs = session->cs; + CSession *cs = session->cs; if (!cs) return; diff --git a/toxav/codec.h b/toxav/codec.h index 526a80d5..b13203f1 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -42,32 +42,7 @@ #define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; } -typedef void (*CSAudioCallback) (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data); -typedef void (*CSVideoCallback) (void *agent, int32_t call_idx, const vpx_image_t *img, void *data); - -/** - * Codec capabilities - */ -typedef enum { - cs_AudioEncoding = 1 << 0, - cs_AudioDecoding = 1 << 1, - cs_VideoEncoding = 1 << 2, - cs_VideoDecoding = 1 << 3 -} CSCapabilities; - -/** - * Codec errors. - */ -typedef enum { - cs_ErrorSettingVideoResolution = -30, - cs_ErrorSettingVideoBitrate = -31, - cs_ErrorSplittingVideoPayload = -32, -} CSError; - -/** - * Codec session - controling codec - */ -typedef struct CSSession_s { +typedef struct CSession_s { /* VIDEO * @@ -76,12 +51,10 @@ typedef struct CSSession_s { /* video encoding */ vpx_codec_ctx_t v_encoder[1]; - bool v_encoding; uint32_t frame_counter; /* video decoding */ vpx_codec_ctx_t v_decoder[1]; - bool v_decoding; void *vbuf_raw; /* Un-decoded data */ /* Data handling */ @@ -107,10 +80,13 @@ typedef struct CSSession_s { /* audio encoding */ OpusEncoder *audio_encoder; + int32_t last_encoding_sampling_rate; + int32_t last_encoding_channel_count; + int32_t last_encoding_bitrate; /* audio decoding */ OpusDecoder *audio_decoder; - int32_t last_packet_channels; + int32_t last_packet_channel_count; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; struct JitterBuffer_s *j_buf; @@ -120,55 +96,28 @@ typedef struct CSSession_s { * * */ - void *agent; /* Pointer to ToxAV TODO make this pointer to ToxAV*/ + ToxAV *av; int32_t friend_id; PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ pthread_mutex_t queue_mutex[1]; -} CSSession; +} CSession; -/** - * Generic - */ -void cs_do(CSSession *cs); - +void cs_do(CSession *cs); /* Make sure to be called BEFORE corresponding rtp_new */ -CSSession *cs_new(uint32_t peer_mvfpsz); +CSession *cs_new(uint32_t peer_mvfpsz); /* Make sure to be called AFTER corresponding rtp_kill */ -void cs_kill(CSSession *cs); - - -/** - * VIDEO HANDLING - */ -void cs_init_video_splitter_cycle(CSSession *cs); -int cs_update_video_splitter_cycle(CSSession* cs, const uint8_t* payload, uint16_t length); -const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size); - -int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height); -int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate); - -int cs_enable_video_sending(CSSession* cs, uint32_t bitrate); -int cs_enable_video_receiving(CSSession* cs); - -void cs_disable_video_sending(CSSession* cs); -void cs_disable_video_receiving(CSSession* cs); - -/** - * AUDIO HANDLING - */ -int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate); - -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate); -int cs_enable_audio_receiving(CSSession* cs); - -void cs_disable_audio_sending(CSSession* cs); -void cs_disable_audio_receiving(CSSession* cs); +void cs_kill(CSession *cs); +void cs_init_video_splitter_cycle(CSession *cs); +int cs_update_video_splitter_cycle(CSession* cs, const uint8_t* payload, uint16_t length); +const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size); +int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height); +int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels); /* Internal. Called from rtp_handle_message */ diff --git a/toxav/rtp.h b/toxav/rtp.h index 2950941b..3bfbb7af 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -101,7 +101,7 @@ typedef struct { int dest; - struct CSSession_s *cs; + struct CSession_s *cs; Messenger *m; } RTPSession; diff --git a/toxav/toxav.c b/toxav/toxav.c index 84a0c43a..62fed33f 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -43,7 +43,7 @@ enum { typedef struct ToxAVCall_s { ToxAV* av; RTPSession *rtps[2]; /* Audio is first and video is second */ - CSSession *cs; + CSession *cs; pthread_mutex_t mutex_audio_sending[1]; pthread_mutex_t mutex_video_sending[1]; @@ -54,8 +54,8 @@ typedef struct ToxAVCall_s { MSICall* msi_call; uint32_t friend_id; - uint32_t s_audio_b; /* Sending audio bitrate */ - uint32_t s_video_b; /* Sending video bitrate */ + uint32_t audio_bit_rate; /* Sending audio bitrate */ + uint32_t video_bit_rate; /* Sending video bitrate */ uint8_t last_self_capabilities; uint8_t last_peer_capabilities; @@ -244,8 +244,8 @@ bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint return false; } - call->s_audio_b = audio_bit_rate; - call->s_video_b = video_bit_rate; + call->audio_bit_rate = audio_bit_rate; + call->video_bit_rate = video_bit_rate; call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; @@ -302,8 +302,8 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui goto END; } - call->s_audio_b = audio_bit_rate; - call->s_video_b = video_bit_rate; + call->audio_bit_rate = audio_bit_rate; + call->video_bit_rate = video_bit_rate; call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; @@ -479,12 +479,70 @@ END: bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) { - /* TODO */ + TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; + ToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; + goto END; + } + + if (audio_bitrate_invalid(audio_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_INVALID; + goto END; + } + + pthread_mutex_lock(av->mutex); + call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; + goto END; + } + + /* NOTE: no need to lock*/ + call->audio_bit_rate = audio_bit_rate; + pthread_mutex_unlock(av->mutex); + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_BIT_RATE_OK; } bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) { - /* TODO */ + TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK; + ToxAVCall* call; + + if (m_friend_exists(av->m, friend_number) == 0) { + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND; + goto END; + } + + if (video_bitrate_invalid(video_bit_rate)) { + rc = TOXAV_ERR_BIT_RATE_INVALID; + goto END; + } + + pthread_mutex_lock(av->mutex); + call = call_get(av, friend_number); + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { + pthread_mutex_unlock(av->mutex); + rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; + goto END; + } + + /* NOTE: no need to lock*/ + call->video_bit_rate = video_bit_rate; + pthread_mutex_unlock(av->mutex); + +END: + if (error) + *error = rc; + + return rc == TOXAV_ERR_BIT_RATE_OK; } void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) @@ -507,7 +565,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); - if (call == NULL || !call->active) { + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; @@ -516,20 +574,13 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u pthread_mutex_lock(call->mutex_video_sending); pthread_mutex_unlock(av->mutex); - if (call->msi_call->state != msi_CallActive) { - /* TODO */ - pthread_mutex_unlock(call->mutex_video_sending); - rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; - goto END; - } - if ( y == NULL || u == NULL || v == NULL ) { pthread_mutex_unlock(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) { + if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) { pthread_mutex_unlock(call->mutex_video_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; @@ -550,7 +601,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u int vrc = vpx_codec_encode(call->cs->v_encoder, &img, call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); - vpx_img_free(&img); /* FIXME don't free? */ + vpx_img_free(&img); if ( vrc != VPX_CODEC_OK) { pthread_mutex_unlock(call->mutex_video_sending); LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); @@ -621,7 +672,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc pthread_mutex_lock(av->mutex); call = call_get(av, friend_number); - if (call == NULL || !call->active) { + if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { pthread_mutex_unlock(av->mutex); rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; goto END; @@ -630,33 +681,34 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc pthread_mutex_lock(call->mutex_audio_sending); pthread_mutex_unlock(av->mutex); - if (call->msi_call->state != msi_CallActive) { - /* TODO */ - pthread_mutex_unlock(call->mutex_audio_sending); - rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED; - goto END; - } - if ( pcm == NULL ) { pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_NULL; goto END; } - if ( channels != 1 && channels != 2 ) { + if ( channels > 2 ) { pthread_mutex_unlock(call->mutex_audio_sending); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } { /* Encode and send */ + if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { + pthread_mutex_unlock(call->mutex_audio_sending); + rc = TOXAV_ERR_SEND_FRAME_INVALID; + goto END; + } + + + LOGGER_DEBUG("Sending audio frame size: %d; channels: %d; srate: %d", sample_count, channels, sampling_rate); uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); if (vrc < 0) { LOGGER_WARNING("Failed to encode frame"); - rc = TOXAV_ERR_SEND_FRAME_INVALID; pthread_mutex_unlock(call->mutex_audio_sending); + rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } @@ -914,19 +966,19 @@ bool call_prepare_transmission(ToxAVCall* call) } if (pthread_mutex_init(call->mutex_audio_sending, NULL) != 0) - goto MUTEX_INIT_ERROR; + return false; if (pthread_mutex_init(call->mutex_video_sending, NULL) != 0) { - pthread_mutex_destroy(call->mutex_audio_sending); - goto MUTEX_INIT_ERROR; + goto AUDIO_SENDING_MUTEX_CLEANUP; } if (pthread_mutex_init(call->mutex_decoding, NULL) != 0) { - pthread_mutex_destroy(call->mutex_audio_sending); - pthread_mutex_destroy(call->mutex_video_sending); - goto MUTEX_INIT_ERROR; + goto VIDEO_SENDING_MUTEX_CLEANUP; } + /* Creates both audio and video encoders and decoders with some default values. + * Make sure to reconfigure encoders dynamically when sending data + */ call->cs = cs_new(call->msi_call->peer_vfpsz); if ( !call->cs ) { @@ -934,13 +986,13 @@ bool call_prepare_transmission(ToxAVCall* call) goto FAILURE; } - call->cs->agent = av; + call->cs->av = av; call->cs->friend_id = call->friend_id; memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); - { /* Prepare audio */ + { /* Prepare audio RTP */ call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); if ( !call->rtps[audio_index] ) { @@ -950,24 +1002,13 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[audio_index]->cs = call->cs; - /* Only enable sending if bitrate is defined */ - if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000) != 0) { - LOGGER_WARNING("Failed to enable audio sending!"); - goto FAILURE; - } - - if (cs_enable_audio_receiving(call->cs) != 0) { - LOGGER_WARNING("Failed to enable audio receiving!"); - goto FAILURE; - } - if (rtp_start_receiving(call->rtps[audio_index]) != 0) { LOGGER_WARNING("Failed to enable audio receiving!"); goto FAILURE; } } - { /* Prepare video */ + { /* Prepare video RTP */ call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); if ( !call->rtps[video_index] ) { @@ -977,17 +1018,6 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[video_index]->cs = call->cs; - /* Only enable sending if bitrate is defined */ - if (call->s_video_b > 0 && cs_enable_video_sending(call->cs, call->s_video_b) != 0) { - LOGGER_WARNING("Failed to enable video sending!"); - goto FAILURE; - } - - if (cs_enable_video_receiving(call->cs) != 0) { - LOGGER_WARNING("Failed to enable video receiving!"); - goto FAILURE; - } - if (rtp_start_receiving(call->rtps[video_index]) != 0) { LOGGER_WARNING("Failed to enable audio receiving!"); goto FAILURE; @@ -1004,13 +1034,11 @@ FAILURE: call->rtps[video_index] = NULL; cs_kill(call->cs); call->cs = NULL; - pthread_mutex_destroy(call->mutex_audio_sending); - pthread_mutex_destroy(call->mutex_video_sending); pthread_mutex_destroy(call->mutex_decoding); - return false; - -MUTEX_INIT_ERROR: - LOGGER_ERROR("Mutex initialization failed!\n"); +VIDEO_SENDING_MUTEX_CLEANUP: + pthread_mutex_destroy(call->mutex_video_sending); +AUDIO_SENDING_MUTEX_CLEANUP: + pthread_mutex_destroy(call->mutex_audio_sending); return false; } diff --git a/toxav/toxav.h b/toxav/toxav.h index 48bb6b8c..ae95c61b 100644 --- a/toxav/toxav.h +++ b/toxav/toxav.h @@ -324,7 +324,15 @@ typedef enum TOXAV_ERR_BIT_RATE { /** * The bit rate passed was not one of the supported values. */ - TOXAV_ERR_BIT_RATE_INVALID + TOXAV_ERR_BIT_RATE_INVALID, + /** + * The friend_number passed did not designate a valid friend. + */ + TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND, + /** + * This client is currently not in a call with the friend. + */ + TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL } TOXAV_ERR_BIT_RATE; /** * Set the audio bit rate to be used in subsequent audio frames.