StarMirror/toxcore
1
0
Fork 0
mirror of https://github.com/irungentoo/toxcore.git synced 2024-03-22 13:30:51 +08:00
Code Issues Projects Releases Wiki Activity

A little CS cleanup

Browse Source
This commit is contained in:
mannol 2015-03-29 01:10:34 +01:00
parent e65efc8936
commit fdaad0b7c0
6 changed files with 363 additions and 477 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
toxav/av_test.c
View File

@ -134,14 +134,14 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number,
alDeleteBuffers(processed - 1, bufids + 1);
bufid = bufids[0];
}
// else if(queued < 16)
else if(queued < 16)
alGenBuffers(1, &bufid);
// else
// return;
else
return;
alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16,
pcm, sample_count * channels * 2, sampling_rate);
pcm, sample_count * 2, sampling_rate);
alSourceQueueBuffers(adout, 1, &bufid);
int32_t state;
@ -285,6 +285,36 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img)
return rc;
}
ALCdevice* open_audio_device(const char* audio_out_dev_name)
{
ALCdevice* rc;
rc = alcOpenDevice(audio_out_dev_name);
if ( !rc ) {
printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError());
exit(1);
}
ALCcontext* out_ctx = alcCreateContext(rc, NULL);
alcMakeContextCurrent(out_ctx);
uint32_t buffers[10];
alGenBuffers(10, buffers);
alGenSources((uint32_t)1, &adout);
alSourcei(adout, AL_LOOPING, AL_FALSE);
int16_t zeros[10000];
memset(zeros, 0, sizeof(zeros));
int i;
for ( i = 0; i < 10; ++i )
alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, sizeof(zeros), 48000);
alSourceQueueBuffers(adout, 10, buffers);
alSourcePlay(adout);
return rc;
}
int print_audio_devices()
{
const char *device;
@ -373,49 +403,18 @@ int main (int argc, char** argv)
}
}
ALCdevice* audio_out_device;
{ /* Open output device */
const char* audio_out_dev_name = NULL;
int i = 0;
for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx;
audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i)
if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1))
break;
audio_out_device = alcOpenDevice(audio_out_dev_name);
if ( !audio_out_device ) {
printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError());
exit(1);
}
ALCcontext* out_ctx = alcCreateContext(audio_out_device, NULL);
alcMakeContextCurrent(out_ctx);
uint32_t buffers[5];
alGenBuffers(5, buffers);
alGenSources((uint32_t)1, &adout);
alSourcei(adout, AL_LOOPING, AL_FALSE);
uint16_t zeros[10000];
memset(zeros, 0, 10000);
for ( i = 0; i < 5; ++i )
alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, 10000, 48000);
alSourceQueueBuffers(adout, 5, buffers);
alSourcePlay(adout);
printf("Using audio device: %s\n", audio_out_dev_name);
}
const char* audio_out_dev_name = NULL;
int i = 0;
for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx;
audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i)
if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1))
break;
printf("Using audio device: %s\n", audio_out_dev_name);
printf("Using audio file: %s\n", af_name);
printf("Using video file: %s\n", vf_name);
/* START TOX NETWORK */
@ -697,11 +696,11 @@ int main (int argc, char** argv)
printf("Failed to open the file.\n");
exit(1);
}
ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name);
/* Run for 5 seconds */
uint32_t frame_duration = 10;
int16_t PCM[10000];
int16_t PCM[5760];
time_t start_time = time(NULL);
time_t expected_time = af_info.frames / af_info.samplerate + 2;
@ -711,18 +710,21 @@ int main (int argc, char** argv)
int64_t count = sf_read_short(af_handle, PCM, frame_size);
if (count > 0) {
TOXAV_ERR_SEND_FRAME rc;
if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) {
printf("Error sending frame of size %ld: %d\n", count, rc);
exit(1);
}
t_toxav_receive_audio_frame_cb(BobAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL);
// TOXAV_ERR_SEND_FRAME rc;
// if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) {
// printf("Error sending frame of size %ld: %d\n", count, rc);
// exit(1);
// }
}
iterate_tox(bootstrap, AliceAV, BobAV);
c_sleep(frame_duration);
// iterate_tox(bootstrap, AliceAV, BobAV);
}
printf("Played file in: %lu\n", time(NULL) - start_time);
alcCloseDevice(audio_out_device);
sf_close(af_handle);
{ /* Hangup */
@ -774,7 +776,5 @@ int main (int argc, char** argv)
tox_kill(bootstrap);
printf("\nTest successful!\n");
alcCloseDevice(audio_out_device);
return 0;
}

479
toxav/codec.c
View File

@ -238,10 +238,80 @@ static int convert_bw_to_sampling_rate(int bw)
}
}
OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count)
{
int status = OPUS_OK;
OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status);
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status));
return NULL;
}
status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate));
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10));
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
return rc;
FAILURE:
opus_encoder_destroy(rc);
return NULL;
}
bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate)
{
assert(dest);
vpx_codec_enc_cfg_t cfg;
int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
return false;
}
rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0,
VPX_ENCODER_ABI_VERSION);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
return false;
}
cfg.rc_target_bitrate = bitrate;
cfg.g_w = 800;
cfg.g_h = 600;
cfg.g_pass = VPX_RC_ONE_PASS;
cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
cfg.g_lag_in_frames = 0;
cfg.kf_min_dist = 0;
cfg.kf_max_dist = 48;
cfg.kf_mode = VPX_KF_AUTO;
rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
vpx_codec_destroy(dest);
}
return true;
}
/* PUBLIC */
void cs_do(CSSession *cs)
void cs_do(CSession *cs)
{
/* Codec session should always be protected by call mutex so no need to check for cs validity
*/
@ -261,7 +331,7 @@ void cs_do(CSSession *cs)
RTPMessage *msg;
/* The maximum for 120 ms 48 KHz audio */
int16_t tmp[20000];
int16_t tmp[5760];
while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) {
pthread_mutex_unlock(cs->queue_mutex);
@ -276,7 +346,7 @@ void cs_do(CSSession *cs)
rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data));
if (rc != -1) {
cs->last_packet_sampling_rate = rc;
cs->last_packet_channels = opus_packet_get_nb_channels(msg->data);
cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data);
cs->last_packet_frame_duration =
( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 )
@ -287,7 +357,7 @@ void cs_do(CSSession *cs)
continue;
}
rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, sizeof(tmp), 0);
rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, 5760, 0);
rtp_free_msg(NULL, msg);
}
@ -295,21 +365,11 @@ void cs_do(CSSession *cs)
LOGGER_WARNING("Decoding error: %s", opus_strerror(rc));
} else if (cs->acb.first) {
/* Play */
LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc,
cs->last_packet_channels, cs->last_packet_sampling_rate, cs->last_packet_frame_duration);
cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->last_packet_frame_duration);
/* According to https://tools.ietf.org/html/rfc6716#section-2.1.2
* Every encoder can encode both mono and stereo data so we must
* determine which format is selected.
*/
if (cs->last_packet_channels == 2) {
/* The packet is encoded with stereo encoder */
}
cs->acb.first(cs->agent, cs->friend_id, tmp, rc,
cs->last_packet_channels, cs->last_packet_sampling_rate, cs->acb.second);
cs->acb.first(cs->av, cs->friend_id, tmp, rc,
cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second);
}
pthread_mutex_lock(cs->queue_mutex);
@ -336,7 +396,7 @@ void cs_do(CSSession *cs)
/* Play decoded images */
for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) {
if (cs->vcb.first)
cs->vcb.first(cs->agent, cs->friend_id, dest->d_w, dest->d_h,
cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h,
(const uint8_t**)dest->planes, dest->stride, cs->vcb.second);
vpx_img_free(dest);
@ -349,29 +409,103 @@ void cs_do(CSSession *cs)
pthread_mutex_unlock(cs->queue_mutex);
}
CSSession *cs_new(uint32_t peer_video_frame_piece_size)
CSession *cs_new(uint32_t peer_video_frame_piece_size)
{
CSSession *cs = calloc(sizeof(CSSession), 1);
CSession *cs = calloc(sizeof(CSession), 1);
if (!cs) {
LOGGER_WARNING("Allocation failed! Application might misbehave!");
return NULL;
}
if (create_recursive_mutex(cs->queue_mutex) != 0) {
LOGGER_WARNING("Failed to create recursive mutex!");
free(cs);
return NULL;
}
/*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
/* Create decoders and set up their values
*/
/*
* AUDIO
*/
int status;
cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be mono */
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status));
goto FAILURE;
}
/* These need to be set in order to properly
* do error correction with opus */
cs->last_packet_frame_duration = 120;
cs->last_packet_sampling_rate = 48000;
if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) {
LOGGER_WARNING("Jitter buffer creaton failed!");
opus_decoder_destroy(cs->audio_decoder);
goto FAILURE;
}
/*
* VIDEO
*/
int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE,
NULL, 0, VPX_DECODER_ABI_VERSION);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
goto AUDIO_DECODER_CLEANUP;
}
if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) {
vpx_codec_destroy(cs->v_decoder);
goto AUDIO_DECODER_CLEANUP;
}
if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) {
free(cs->frame_buf);
vpx_codec_destroy(cs->v_decoder);
goto AUDIO_DECODER_CLEANUP;
}
/*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
/* Initialize encoders with default values */
cs->audio_encoder = create_audio_encoder(48000, 48000, 2);
if (cs->audio_encoder == NULL)
goto VIDEO_DECODER_CLEANUP;
cs->last_encoding_bitrate = 48000;
cs->last_encoding_sampling_rate = 48000;
cs->last_encoding_channel_count = 2;
if (!create_video_encoder(cs->v_encoder, 500000)) {
opus_encoder_destroy(cs->audio_encoder);
goto VIDEO_DECODER_CLEANUP;
}
cs->peer_video_frame_piece_size = peer_video_frame_piece_size;
return cs;
VIDEO_DECODER_CLEANUP:
buffer_free(cs->vbuf_raw);
free(cs->frame_buf);
vpx_codec_destroy(cs->v_decoder);
AUDIO_DECODER_CLEANUP:
opus_decoder_destroy(cs->audio_decoder);
jbuf_free(cs->j_buf);
FAILURE:
pthread_mutex_destroy(cs->queue_mutex);
free(cs);
return NULL;
}
void cs_kill(CSSession *cs)
void cs_kill(CSession *cs)
{
if (!cs)
return;
@ -380,10 +514,13 @@ void cs_kill(CSSession *cs)
* the callback is unregistered before cs_kill is called.
*/
cs_disable_audio_sending(cs);
cs_disable_audio_receiving(cs);
cs_disable_video_sending(cs);
cs_disable_video_receiving(cs);
vpx_codec_destroy(cs->v_encoder);
vpx_codec_destroy(cs->v_decoder);
opus_encoder_destroy(cs->audio_encoder);
opus_decoder_destroy(cs->audio_decoder);
buffer_free(cs->vbuf_raw);
jbuf_free(cs->j_buf);
free(cs->frame_buf);
pthread_mutex_destroy(cs->queue_mutex);
@ -391,15 +528,13 @@ void cs_kill(CSSession *cs)
free(cs);
}
void cs_init_video_splitter_cycle(CSSession* cs)
void cs_init_video_splitter_cycle(CSession* cs)
{
cs->split_video_frame[0] = cs->frameid_out++;
cs->split_video_frame[1] = 0;
}
int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16_t length)
int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length)
{
cs->processing_video_frame = payload;
cs->processing_video_frame_size = length;
@ -407,7 +542,7 @@ int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16
return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1;
}
const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size)
const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size)
{
if (!cs || !size) return NULL;
@ -433,295 +568,61 @@ const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size)
return cs->split_video_frame;
}
int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height)
int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height)
{
if (!cs->v_encoding)
return -1;
/* TODO FIXME reference is safe? */
vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc;
if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height)
return 0; /* Nothing changed */
if (cfg.g_w == width && cfg.g_h == height)
return 0;
/*
if (width * height > cs->max_width * cs->max_height) {
vpx_codec_ctx_t v_encoder = cs->v_encoder;
if (init_video_encoder(cs, width, height, cs->video_bitrate) == -1) {
cs->v_encoder = v_encoder;
return cs_ErrorSettingVideoResolution;
}
vpx_codec_destroy(&v_encoder);
return 0;
}*/
LOGGER_DEBUG("New video resolution: %u %u", width, height);
cfg.rc_target_bitrate = bitrate;
cfg.g_w = width;
cfg.g_h = height;
int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
return cs_ErrorSettingVideoResolution;
}
return 0;
}
int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate)
{
if (!cs->v_encoding)
return -1;
/* TODO FIXME reference is safe? */
vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc;
if (cfg.rc_target_bitrate == bitrate)
return 0;
LOGGER_DEBUG("New video bitrate: %u", bitrate);
cfg.rc_target_bitrate = bitrate;
int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
return cs_ErrorSettingVideoBitrate;
return -1;
}
return 0;
}
int cs_enable_video_sending(CSSession* cs, uint32_t bitrate)
int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels)
{
if (cs->v_encoding)
return 0;
/* Values are checked in toxav.c */
vpx_codec_enc_cfg_t cfg;
int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
return -1;
}
rc = vpx_codec_enc_init_ver(cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0,
VPX_ENCODER_ABI_VERSION);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
return -1;
}
/* So that we can use cs_disable_video_sending to clean up */
cs->v_encoding = true;
if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) )
goto FAILURE;
cfg.rc_target_bitrate = bitrate;
cfg.g_w = 800;
cfg.g_h = 600;
cfg.g_pass = VPX_RC_ONE_PASS;
cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
cfg.g_lag_in_frames = 0;
cfg.kf_min_dist = 0;
cfg.kf_max_dist = 48;
cfg.kf_mode = VPX_KF_AUTO;
rc = vpx_codec_control(cs->v_encoder, VP8E_SET_CPUUSED, 8);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
goto FAILURE;
}
return 0;
FAILURE:
cs_disable_video_sending(cs);
return -1;
}
int cs_enable_video_receiving(CSSession* cs)
{
if (cs->v_decoding)
return 0;
int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE,
NULL, 0, VPX_DECODER_ABI_VERSION);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
return -1;
}
/* So that we can use cs_disable_video_sending to clean up */
cs->v_decoding = true;
if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) )
goto FAILURE;
if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) )
goto FAILURE;
return 0;
FAILURE:
cs_disable_video_receiving(cs);
return -1;
}
void cs_disable_video_sending(CSSession* cs)
{
if (cs->v_encoding) {
cs->v_encoding = false;
if (cs->last_encoding_sampling_rate != sampling_rate || cs->last_encoding_channel_count != channels) {
OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels);
if (new_encoder == NULL)
return -1;
free(cs->split_video_frame);
cs->split_video_frame = NULL;
vpx_codec_destroy(cs->v_encoder);
}
}
void cs_disable_video_receiving(CSSession* cs)
{
if (cs->v_decoding) {
cs->v_decoding = false;
buffer_free(cs->vbuf_raw);
cs->vbuf_raw = NULL;
free(cs->frame_buf);
cs->frame_buf = NULL;
vpx_codec_destroy(cs->v_decoder);
}
}
int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate)
{
if (cs->audio_encoder == NULL)
return -1;
int rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(rate));
if ( rc != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
return -1;
}
LOGGER_DEBUG("Set new encoder bitrate to: %d", rate);
return 0;
}
void cs_disable_audio_sending(CSSession* cs)
{
if ( cs->audio_encoder ) {
opus_encoder_destroy(cs->audio_encoder);
cs->audio_encoder = NULL;
}
}
void cs_disable_audio_receiving(CSSession* cs)
{
if ( cs->audio_decoder ) {
opus_decoder_destroy(cs->audio_decoder);
cs->audio_decoder = NULL;
jbuf_free(cs->j_buf);
cs->j_buf = NULL;
cs->audio_encoder = new_encoder;
} else if (cs->last_encoding_bitrate == bitrate)
return 0; /* Nothing changed */
else {
int status = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate));
/* These need to be set in order to properly
* do error correction with opus */
cs->last_packet_frame_duration = 120;
cs->last_packet_sampling_rate = 48000;
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
return -1;
}
}
}
int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate)
{
if (cs->audio_encoder)
return 0;
int rc = OPUS_OK;
cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc);
if ( rc != OPUS_OK ) {
LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc));
return -1;
}
rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate));
if ( rc != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
goto FAILURE;
}
rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10));
if ( rc != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
goto FAILURE;
}
return 0;
FAILURE:
cs_disable_audio_sending(cs);
return -1;
}
int cs_enable_audio_receiving(CSSession* cs)
{
if (cs->audio_decoder)
return 0;
int rc;
cs->audio_decoder = opus_decoder_create(48000, 2, &rc );
if ( rc != OPUS_OK ) {
LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc));
return -1;
}
if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) {
LOGGER_WARNING("Jitter buffer creaton failed!");
opus_decoder_destroy(cs->audio_decoder);
cs->audio_decoder = NULL;
return -1;
}
/* These need to be set in order to properly
* do error correction with opus */
cs->last_packet_frame_duration = 120;
cs->last_packet_sampling_rate = 48000;
cs->last_encoding_bitrate = bitrate;
cs->last_encoding_sampling_rate = sampling_rate;
cs->last_encoding_channel_count = channels;
return 0;
}
/* Called from RTP */
void queue_message(RTPSession *session, RTPMessage *msg)
{
/* This function is unregistered during call termination befor destroying
* Codec session so no need to check for validity of cs TODO properly check video cycle
*/
CSSession *cs = session->cs;
CSession *cs = session->cs;
if (!cs)
return;

81
toxav/codec.h
View File

@ -42,32 +42,7 @@
#define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; }
typedef void (*CSAudioCallback) (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data);
typedef void (*CSVideoCallback) (void *agent, int32_t call_idx, const vpx_image_t *img, void *data);
/**
* Codec capabilities
*/
typedef enum {
cs_AudioEncoding = 1 << 0,
cs_AudioDecoding = 1 << 1,
cs_VideoEncoding = 1 << 2,
cs_VideoDecoding = 1 << 3
} CSCapabilities;
/**
* Codec errors.
*/
typedef enum {
cs_ErrorSettingVideoResolution = -30,
cs_ErrorSettingVideoBitrate = -31,
cs_ErrorSplittingVideoPayload = -32,
} CSError;
/**
* Codec session - controling codec
*/
typedef struct CSSession_s {
typedef struct CSession_s {
/* VIDEO
*
@ -76,12 +51,10 @@ typedef struct CSSession_s {
/* video encoding */
vpx_codec_ctx_t v_encoder[1];
bool v_encoding;
uint32_t frame_counter;
/* video decoding */
vpx_codec_ctx_t v_decoder[1];
bool v_decoding;
void *vbuf_raw; /* Un-decoded data */
/* Data handling */
@ -107,10 +80,13 @@ typedef struct CSSession_s {
/* audio encoding */
OpusEncoder *audio_encoder;
int32_t last_encoding_sampling_rate;
int32_t last_encoding_channel_count;
int32_t last_encoding_bitrate;
/* audio decoding */
OpusDecoder *audio_decoder;
int32_t last_packet_channels;
int32_t last_packet_channel_count;
int32_t last_packet_sampling_rate;
int32_t last_packet_frame_duration;
struct JitterBuffer_s *j_buf;
@ -120,55 +96,28 @@ typedef struct CSSession_s {
*
*
*/
void *agent; /* Pointer to ToxAV TODO make this pointer to ToxAV*/
ToxAV *av;
int32_t friend_id;
PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */
PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */
pthread_mutex_t queue_mutex[1];
} CSSession;
} CSession;
/**
* Generic
*/
void cs_do(CSSession *cs);
void cs_do(CSession *cs);
/* Make sure to be called BEFORE corresponding rtp_new */
CSSession *cs_new(uint32_t peer_mvfpsz);
CSession *cs_new(uint32_t peer_mvfpsz);
/* Make sure to be called AFTER corresponding rtp_kill */
void cs_kill(CSSession *cs);
/**
* VIDEO HANDLING
*/
void cs_init_video_splitter_cycle(CSSession *cs);
int cs_update_video_splitter_cycle(CSSession* cs, const uint8_t* payload, uint16_t length);
const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size);
int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height);
int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate);
int cs_enable_video_sending(CSSession* cs, uint32_t bitrate);
int cs_enable_video_receiving(CSSession* cs);
void cs_disable_video_sending(CSSession* cs);
void cs_disable_video_receiving(CSSession* cs);
/**
* AUDIO HANDLING
*/
int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate);
int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate);
int cs_enable_audio_receiving(CSSession* cs);
void cs_disable_audio_sending(CSSession* cs);
void cs_disable_audio_receiving(CSSession* cs);
void cs_kill(CSession *cs);
void cs_init_video_splitter_cycle(CSession *cs);
int cs_update_video_splitter_cycle(CSession* cs, const uint8_t* payload, uint16_t length);
const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size);
int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height);
int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels);
/* Internal. Called from rtp_handle_message */

2
toxav/rtp.h
View File

@ -101,7 +101,7 @@ typedef struct {
int dest;
struct CSSession_s *cs;
struct CSession_s *cs;
Messenger *m;
} RTPSession;

160
toxav/toxav.c
View File

@ -43,7 +43,7 @@ enum {
typedef struct ToxAVCall_s {
ToxAV* av;
RTPSession *rtps[2]; /* Audio is first and video is second */
CSSession *cs;
CSession *cs;
pthread_mutex_t mutex_audio_sending[1];
pthread_mutex_t mutex_video_sending[1];
@ -54,8 +54,8 @@ typedef struct ToxAVCall_s {
MSICall* msi_call;
uint32_t friend_id;
uint32_t s_audio_b; /* Sending audio bitrate */
uint32_t s_video_b; /* Sending video bitrate */
uint32_t audio_bit_rate; /* Sending audio bitrate */
uint32_t video_bit_rate; /* Sending video bitrate */
uint8_t last_self_capabilities;
uint8_t last_peer_capabilities;
@ -244,8 +244,8 @@ bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint
return false;
}
call->s_audio_b = audio_bit_rate;
call->s_video_b = video_bit_rate;
call->audio_bit_rate = audio_bit_rate;
call->video_bit_rate = video_bit_rate;
call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo;
@ -302,8 +302,8 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui
goto END;
}
call->s_audio_b = audio_bit_rate;
call->s_video_b = video_bit_rate;
call->audio_bit_rate = audio_bit_rate;
call->video_bit_rate = video_bit_rate;
call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo;
@ -479,12 +479,70 @@ END:
bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error)
{
/* TODO */
TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK;
ToxAVCall* call;
if (m_friend_exists(av->m, friend_number) == 0) {
rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND;
goto END;
}
if (audio_bitrate_invalid(audio_bit_rate)) {
rc = TOXAV_ERR_BIT_RATE_INVALID;
goto END;
}
pthread_mutex_lock(av->mutex);
call = call_get(av, friend_number);
if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
pthread_mutex_unlock(av->mutex);
rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL;
goto END;
}
/* NOTE: no need to lock*/
call->audio_bit_rate = audio_bit_rate;
pthread_mutex_unlock(av->mutex);
END:
if (error)
*error = rc;
return rc == TOXAV_ERR_BIT_RATE_OK;
}
bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error)
{
/* TODO */
TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK;
ToxAVCall* call;
if (m_friend_exists(av->m, friend_number) == 0) {
rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND;
goto END;
}
if (video_bitrate_invalid(video_bit_rate)) {
rc = TOXAV_ERR_BIT_RATE_INVALID;
goto END;
}
pthread_mutex_lock(av->mutex);
call = call_get(av, friend_number);
if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
pthread_mutex_unlock(av->mutex);
rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL;
goto END;
}
/* NOTE: no need to lock*/
call->video_bit_rate = video_bit_rate;
pthread_mutex_unlock(av->mutex);
END:
if (error)
*error = rc;
return rc == TOXAV_ERR_BIT_RATE_OK;
}
void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data)
@ -507,7 +565,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
pthread_mutex_lock(av->mutex);
call = call_get(av, friend_number);
if (call == NULL || !call->active) {
if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
pthread_mutex_unlock(av->mutex);
rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
goto END;
@ -516,20 +574,13 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
pthread_mutex_lock(call->mutex_video_sending);
pthread_mutex_unlock(av->mutex);
if (call->msi_call->state != msi_CallActive) {
/* TODO */
pthread_mutex_unlock(call->mutex_video_sending);
rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED;
goto END;
}
if ( y == NULL || u == NULL || v == NULL ) {
pthread_mutex_unlock(call->mutex_video_sending);
rc = TOXAV_ERR_SEND_FRAME_NULL;
goto END;
}
if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) {
if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) {
pthread_mutex_unlock(call->mutex_video_sending);
rc = TOXAV_ERR_SEND_FRAME_INVALID;
goto END;
@ -550,7 +601,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
int vrc = vpx_codec_encode(call->cs->v_encoder, &img,
call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US);
vpx_img_free(&img); /* FIXME don't free? */
vpx_img_free(&img);
if ( vrc != VPX_CODEC_OK) {
pthread_mutex_unlock(call->mutex_video_sending);
LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc));
@ -621,7 +672,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc
pthread_mutex_lock(av->mutex);
call = call_get(av, friend_number);
if (call == NULL || !call->active) {
if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
pthread_mutex_unlock(av->mutex);
rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
goto END;
@ -630,33 +681,34 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc
pthread_mutex_lock(call->mutex_audio_sending);
pthread_mutex_unlock(av->mutex);
if (call->msi_call->state != msi_CallActive) {
/* TODO */
pthread_mutex_unlock(call->mutex_audio_sending);
rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED;
goto END;
}
if ( pcm == NULL ) {
pthread_mutex_unlock(call->mutex_audio_sending);
rc = TOXAV_ERR_SEND_FRAME_NULL;
goto END;
}
if ( channels != 1 && channels != 2 ) {
if ( channels > 2 ) {
pthread_mutex_unlock(call->mutex_audio_sending);
rc = TOXAV_ERR_SEND_FRAME_INVALID;
goto END;
}
{ /* Encode and send */
if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) {
pthread_mutex_unlock(call->mutex_audio_sending);
rc = TOXAV_ERR_SEND_FRAME_INVALID;
goto END;
}
LOGGER_DEBUG("Sending audio frame size: %d; channels: %d; srate: %d", sample_count, channels, sampling_rate);
uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */];
int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest));
if (vrc < 0) {
LOGGER_WARNING("Failed to encode frame");
rc = TOXAV_ERR_SEND_FRAME_INVALID;
pthread_mutex_unlock(call->mutex_audio_sending);
rc = TOXAV_ERR_SEND_FRAME_INVALID;
goto END;
}
@ -914,19 +966,19 @@ bool call_prepare_transmission(ToxAVCall* call)
}
if (pthread_mutex_init(call->mutex_audio_sending, NULL) != 0)
goto MUTEX_INIT_ERROR;
return false;
if (pthread_mutex_init(call->mutex_video_sending, NULL) != 0) {
pthread_mutex_destroy(call->mutex_audio_sending);
goto MUTEX_INIT_ERROR;
goto AUDIO_SENDING_MUTEX_CLEANUP;
}
if (pthread_mutex_init(call->mutex_decoding, NULL) != 0) {
pthread_mutex_destroy(call->mutex_audio_sending);
pthread_mutex_destroy(call->mutex_video_sending);
goto MUTEX_INIT_ERROR;
goto VIDEO_SENDING_MUTEX_CLEANUP;
}
/* Creates both audio and video encoders and decoders with some default values.
* Make sure to reconfigure encoders dynamically when sending data
*/
call->cs = cs_new(call->msi_call->peer_vfpsz);
if ( !call->cs ) {
@ -934,13 +986,13 @@ bool call_prepare_transmission(ToxAVCall* call)
goto FAILURE;
}
call->cs->agent = av;
call->cs->av = av;
call->cs->friend_id = call->friend_id;
memcpy(&call->cs->acb, &av->acb, sizeof(av->acb));
memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb));
{ /* Prepare audio */
{ /* Prepare audio RTP */
call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id);
if ( !call->rtps[audio_index] ) {
@ -950,24 +1002,13 @@ bool call_prepare_transmission(ToxAVCall* call)
call->rtps[audio_index]->cs = call->cs;
/* Only enable sending if bitrate is defined */
if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000) != 0) {
LOGGER_WARNING("Failed to enable audio sending!");
goto FAILURE;
}
if (cs_enable_audio_receiving(call->cs) != 0) {
LOGGER_WARNING("Failed to enable audio receiving!");
goto FAILURE;
}
if (rtp_start_receiving(call->rtps[audio_index]) != 0) {
LOGGER_WARNING("Failed to enable audio receiving!");
goto FAILURE;
}
}
{ /* Prepare video */
{ /* Prepare video RTP */
call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id);
if ( !call->rtps[video_index] ) {
@ -977,17 +1018,6 @@ bool call_prepare_transmission(ToxAVCall* call)
call->rtps[video_index]->cs = call->cs;
/* Only enable sending if bitrate is defined */
if (call->s_video_b > 0 && cs_enable_video_sending(call->cs, call->s_video_b) != 0) {
LOGGER_WARNING("Failed to enable video sending!");
goto FAILURE;
}
if (cs_enable_video_receiving(call->cs) != 0) {
LOGGER_WARNING("Failed to enable video receiving!");
goto FAILURE;
}
if (rtp_start_receiving(call->rtps[video_index]) != 0) {
LOGGER_WARNING("Failed to enable audio receiving!");
goto FAILURE;
@ -1004,13 +1034,11 @@ FAILURE:
call->rtps[video_index] = NULL;
cs_kill(call->cs);
call->cs = NULL;
pthread_mutex_destroy(call->mutex_audio_sending);
pthread_mutex_destroy(call->mutex_video_sending);
pthread_mutex_destroy(call->mutex_decoding);
return false;
MUTEX_INIT_ERROR:
LOGGER_ERROR("Mutex initialization failed!\n");
VIDEO_SENDING_MUTEX_CLEANUP:
pthread_mutex_destroy(call->mutex_video_sending);
AUDIO_SENDING_MUTEX_CLEANUP:
pthread_mutex_destroy(call->mutex_audio_sending);
return false;
}

10
toxav/toxav.h
View File

@ -324,7 +324,15 @@ typedef enum TOXAV_ERR_BIT_RATE {
/**
* The bit rate passed was not one of the supported values.
*/
TOXAV_ERR_BIT_RATE_INVALID
TOXAV_ERR_BIT_RATE_INVALID,
/**
* The friend_number passed did not designate a valid friend.
*/
TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND,
/**
* This client is currently not in a call with the friend.
*/
TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL
} TOXAV_ERR_BIT_RATE;
/**
* Set the audio bit rate to be used in subsequent audio frames.