StarMirror/toxcore
1
0
Fork 0
mirror of https://github.com/irungentoo/toxcore.git synced 2024-03-22 13:30:51 +08:00
Code Issues Projects Releases Wiki Activity

Final touchups

Browse Source
This commit is contained in:
mannol 2015-04-21 02:31:12 +02:00
parent cbb8fdd4ea
commit 3fd0ee5f08
13 changed files with 923 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
toxav/Makefile.inc
View File

@ -12,6 +12,10 @@ libtoxav_la_SOURCES = ../toxav/rtp.h \
../toxav/group.c \
../toxav/codec.h \
../toxav/codec.c \
../toxav/audio.h \
../toxav/audio.c \
../toxav/video.h \
../toxav/video.c \
../toxav/toxav.h \
../toxav/toxav.c

383
toxav/audio.c Normal file
View File

@ -0,0 +1,383 @@
/** audio.c
*
* Copyright (C) 2013-2015 Tox project All Rights Reserved.
*
* This file is part of Tox.
*
* Tox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Tox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Tox. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdlib.h>
#include "audio.h"
#include "rtp.h"
#include "../toxcore/logger.h"
static struct JitterBuffer *jbuf_new(uint32_t capacity);
static void jbuf_clear(struct JitterBuffer *q);
static void jbuf_free(struct JitterBuffer *q);
static int jbuf_write(struct JitterBuffer *q, RTPMessage *m);
static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success);
OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count);
bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels);
ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data)
{
ACSession *ac = calloc(sizeof(ACSession), 1);
if (!ac) {
LOGGER_WARNING("Allocation failed! Application might misbehave!");
return NULL;
}
if (create_recursive_mutex(ac->queue_mutex) != 0) {
LOGGER_WARNING("Failed to create recursive mutex!");
free(ac);
return NULL;
}
int status;
ac->decoder = opus_decoder_create(48000, 2, &status );
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status));
goto BASE_CLEANUP;
}
if ( !(ac->j_buf = jbuf_new(3)) ) {
LOGGER_WARNING("Jitter buffer creaton failed!");
opus_decoder_destroy(ac->decoder);
goto BASE_CLEANUP;
}
/* Initialize encoders with default values */
ac->encoder = create_audio_encoder(48000, 48000, 2);
if (ac->encoder == NULL)
goto DECODER_CLEANUP;
ac->last_encoding_bitrate = 48000;
ac->last_encoding_sampling_rate = 48000;
ac->last_encoding_channel_count = 2;
ac->last_decoding_channel_count = 2;
ac->last_decoding_sampling_rate = 48000;
ac->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */
/* These need to be set in order to properly
* do error correction with opus */
ac->last_packet_frame_duration = 120;
ac->last_packet_sampling_rate = 48000;
ac->av = av;
ac->friend_id = friend_id;
ac->acb.first = cb;
ac->acb.second = cb_data;
return ac;
DECODER_CLEANUP:
opus_decoder_destroy(ac->decoder);
jbuf_free(ac->j_buf);
BASE_CLEANUP:
pthread_mutex_destroy(ac->queue_mutex);
free(ac);
return NULL;
}
void ac_kill(ACSession* ac)
{
if (!ac)
return;
opus_encoder_destroy(ac->encoder);
opus_decoder_destroy(ac->decoder);
jbuf_free(ac->j_buf);
pthread_mutex_destroy(ac->queue_mutex);
LOGGER_DEBUG("Terminated audio handler: %p", ac);
free(ac);
}
void ac_do(ACSession* ac)
{
if (!ac)
return;
/* Enough space for the maximum frame size (120 ms 48 KHz audio) */
int16_t tmp[5760];
RTPMessage *msg;
int rc = 0;
pthread_mutex_lock(ac->queue_mutex);
while ((msg = jbuf_read(ac->j_buf, &rc)) || rc == 2) {
pthread_mutex_unlock(ac->queue_mutex);
if (rc == 2) {
LOGGER_DEBUG("OPUS correction");
rc = opus_decode(ac->decoder, NULL, 0, tmp,
(ac->last_packet_sampling_rate * ac->last_packet_frame_duration / 1000) /
ac->last_packet_channel_count, 1);
} else {
/* Get values from packet and decode. */
/* NOTE: This didn't work very well
rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data));
if (rc != -1) {
cs->last_packet_sampling_rate = rc;
} else {
LOGGER_WARNING("Failed to load packet values!");
rtp_free_msg(NULL, msg);
continue;
}*/
/* Pick up sampling rate from packet */
memcpy(&ac->last_packet_sampling_rate, msg->data, 4);
ac->last_packet_sampling_rate = ntohl(ac->last_packet_sampling_rate);
ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4);
/*
* NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa,
* it didn't work quite well.
*/
if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) {
LOGGER_WARNING("Failed to reconfigure decoder!");
rtp_free_msg(NULL, msg);
continue;
}
rc = opus_decode(ac->decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0);
rtp_free_msg(NULL, msg);
}
if (rc < 0) {
LOGGER_WARNING("Decoding error: %s", opus_strerror(rc));
} else if (ac->acb.first) {
ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate * ac->last_packet_channel_count;
ac->acb.first(ac->av, ac->friend_id, tmp, rc * ac->last_packet_channel_count,
ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second);
}
return;
}
pthread_mutex_unlock(ac->queue_mutex);
}
int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels)
{
if (!ac)
return;
/* Values are checked in toxav.c */
if (ac->last_encoding_sampling_rate != sampling_rate || ac->last_encoding_channel_count != channels) {
OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels);
if (new_encoder == NULL)
return -1;
opus_encoder_destroy(ac->encoder);
ac->encoder = new_encoder;
} else if (ac->last_encoding_bitrate == bitrate)
return 0; /* Nothing changed */
else {
int status = opus_encoder_ctl(ac->encoder, OPUS_SET_BITRATE(bitrate));
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
return -1;
}
}
ac->last_encoding_bitrate = bitrate;
ac->last_encoding_sampling_rate = sampling_rate;
ac->last_encoding_channel_count = channels;
LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels);
return 0;
}
/* called from rtp */
void ac_queue_message(void* acp, RTPMessage *msg)
{
if (!acp || !msg)
return;
ACSession* ac = acp;
pthread_mutex_lock(ac->queue_mutex);
int ret = jbuf_write(ac->j_buf, msg);
pthread_mutex_unlock(ac->queue_mutex);
if (ret == -1)
rtp_free_msg(NULL, msg);
}
/* JITTER BUFFER WORK */
struct JitterBuffer {
RTPMessage **queue;
uint32_t size;
uint32_t capacity;
uint16_t bottom;
uint16_t top;
};
static struct JitterBuffer *jbuf_new(uint32_t capacity)
{
unsigned int size = 1;
while (size <= (capacity * 4)) {
size *= 2;
}
struct JitterBuffer *q;
if ( !(q = calloc(sizeof(struct JitterBuffer), 1)) ) return NULL;
if (!(q->queue = calloc(sizeof(RTPMessage *), size))) {
free(q);
return NULL;
}
q->size = size;
q->capacity = capacity;
return q;
}
static void jbuf_clear(struct JitterBuffer *q)
{
for (; q->bottom != q->top; ++q->bottom) {
if (q->queue[q->bottom % q->size]) {
rtp_free_msg(NULL, q->queue[q->bottom % q->size]);
q->queue[q->bottom % q->size] = NULL;
}
}
}
static void jbuf_free(struct JitterBuffer *q)
{
if (!q) return;
jbuf_clear(q);
free(q->queue);
free(q);
}
static int jbuf_write(struct JitterBuffer *q, RTPMessage *m)
{
uint16_t sequnum = m->header->sequnum;
unsigned int num = sequnum % q->size;
if ((uint32_t)(sequnum - q->bottom) > q->size) {
LOGGER_DEBUG("Clearing filled jitter buffer: %p", q);
jbuf_clear(q);
q->bottom = sequnum - q->capacity;
q->queue[num] = m;
q->top = sequnum + 1;
return 0;
}
if (q->queue[num])
return -1;
q->queue[num] = m;
if ((sequnum - q->bottom) >= (q->top - q->bottom))
q->top = sequnum + 1;
return 0;
}
static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success)
{
if (q->top == q->bottom) {
*success = 0;
return NULL;
}
unsigned int num = q->bottom % q->size;
if (q->queue[num]) {
RTPMessage *ret = q->queue[num];
q->queue[num] = NULL;
++q->bottom;
*success = 1;
return ret;
}
if ((uint32_t)(q->top - q->bottom) > q->capacity) {
++q->bottom;
*success = 2;
return NULL;
}
*success = 0;
return NULL;
}
OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count)
{
int status = OPUS_OK;
OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status);
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status));
return NULL;
}
status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate));
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10));
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
goto FAILURE;
}
return rc;
FAILURE:
opus_encoder_destroy(rc);
return NULL;
}
bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels)
{
if (sampling_rate != ac->last_decoding_sampling_rate || channels != ac->last_decoding_channel_count) {
if (current_time_monotonic() - ac->last_decoder_reconfiguration < 500)
return false;
int status;
OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status );
if ( status != OPUS_OK ) {
LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status));
return false;
}
ac->last_decoding_sampling_rate = sampling_rate;
ac->last_decoding_channel_count = channels;
ac->last_decoder_reconfiguration = current_time_monotonic();
opus_decoder_destroy(ac->decoder);
ac->decoder = new_dec;
LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels);
}
return true;
}

60
toxav/audio.h Normal file
View File

@ -0,0 +1,60 @@
/** audio.h
*
* Copyright (C) 2013-2015 Tox project All Rights Reserved.
*
* This file is part of Tox.
*
* Tox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Tox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Tox. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef AUDIO_H
#define AUDIO_H
#include <opus.h>
#include <pthread.h>
#include "toxav.h"
#include "../toxcore/util.h"
typedef struct ACSession_s {
/* encoding */
OpusEncoder *encoder;
int32_t last_encoding_sampling_rate;
int32_t last_encoding_channel_count;
int32_t last_encoding_bitrate;
/* decoding */
OpusDecoder *decoder;
int32_t last_packet_channel_count;
int32_t last_packet_sampling_rate;
int32_t last_packet_frame_duration;
int32_t last_decoding_sampling_rate;
int32_t last_decoding_channel_count;
uint64_t last_decoder_reconfiguration;
void *j_buf;
pthread_mutex_t queue_mutex[1];
ToxAV* av;
uint32_t friend_id;
PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */
} ACSession;
ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data);
void ac_kill(ACSession* ac);
void ac_do(ACSession* ac);
int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels);
#endif /* AUDIO_H */

19
toxav/av_test.c
View File

@ -77,8 +77,8 @@
#define TEST_REJECT 0
#define TEST_CANCEL 0
#define TEST_MUTE_UNMUTE 0
#define TEST_TRANSFER_A 0
#define TEST_TRANSFER_V 1
#define TEST_TRANSFER_A 1
#define TEST_TRANSFER_V 0
typedef struct {
@ -320,21 +320,28 @@ int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV)
void* iterate_toxav (void * data)
{
struct toxav_thread_data* data_cast = data;
// cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE);
#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1
cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE);
#endif
while (data_cast->sig == 0) {
toxav_iterate(data_cast->AliceAV);
toxav_iterate(data_cast->BobAV);
int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV));
// cvWaitKey(rc);
#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1
cvWaitKey(rc);
#else
c_sleep(rc);
#endif
}
data_cast->sig = 1;
// cvDestroyWindow(vdout);
#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1
cvDestroyWindow(vdout);
#endif
pthread_exit(NULL);
}

10
toxav/codec.c
View File

@ -328,7 +328,6 @@ void cs_do(CSession *cs)
LOGGER_WARNING("Failed to reconfigure decoder!");
rtp_free_msg(NULL, msg);
continue;
// goto DONE;
}
rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0);
@ -347,7 +346,6 @@ void cs_do(CSession *cs)
LOGGED_LOCK(cs->queue_mutex);
}
// DONE:;
}
/********************* VIDEO *********************/
@ -497,12 +495,12 @@ void cs_kill(CSession *cs)
* the callback is unregistered before cs_kill is called.
*/
vpx_codec_destroy(cs->v_encoder);
vpx_codec_destroy(cs->v_decoder);
opus_encoder_destroy(cs->audio_encoder);
opus_decoder_destroy(cs->audio_decoder);
rb_free(cs->vbuf_raw);
jbuf_free(cs->j_buf);
vpx_codec_destroy(cs->v_encoder);
vpx_codec_destroy(cs->v_decoder);
rb_free(cs->vbuf_raw);
free(cs->frame_buf);
free(cs->split_video_frame);
@ -600,7 +598,7 @@ void queue_message(RTPSession *session, RTPMessage *msg)
{
CSession *cs = session->cs;
if (!cs)
if (!cs)
return;
/* Audio */

3
toxav/codec.h
View File

@ -42,9 +42,6 @@
/* Audio encoding/decoding */
#include <opus.h>
#define PACKED_AUDIO_SIZE(x) (x + 5)
#define UNPACKED_AUDIO_SIZE(x) (x - 5)
typedef struct CSession_s {
/* VIDEO

21
toxav/msi.c
View File

@ -188,7 +188,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_
msg.capabilities.value = capabilities;
msg.vfpsz.exists = true;
msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE);
msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE;
send_message ( (*call)->session->messenger, (*call)->friend_id, &msg );
@ -238,7 +238,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities )
msg.capabilities.value = capabilities;
msg.vfpsz.exists = true;
msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE);
msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE;
send_message ( session->messenger, call->friend_id, &msg );
@ -349,6 +349,12 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length )
case IDVFPSZ:
CHECK_SIZE(it, size_constraint, 2);
SET_UINT16(it, dest->vfpsz);
dest->vfpsz = ntohs(dest->vfpsz);
if (dest->vfpsz.value > 1200) {
LOGGER_ERROR("Invalid vfpsz param");
return -1;
}
break;
default:
@ -419,8 +425,9 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg )
}
if (msg->vfpsz.exists) {
it = msg_parse_header_out(IDVFPSZ, it, &msg->vfpsz.value,
sizeof(msg->vfpsz.value), &size);
uint16_t nb_vfpsz = htons(msg->vfpsz);
it = msg_parse_header_out(IDVFPSZ, it, &nb_vfpsz,
sizeof(nb_vfpsz), &size);
}
if ( it == parsed ) {
@ -620,7 +627,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg )
goto FAILURE;
}
call->peer_vfpsz = ntohs(msg->vfpsz.value);
call->peer_vfpsz = msg->vfpsz.value;
}
@ -645,7 +652,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg )
* is not terminated on our side. We can assume that
* in this case we can automatically answer the re-call.
*/
if (call->peer_vfpsz != ntohs(msg->vfpsz.value)) {
if (call->peer_vfpsz != msg->vfpsz.value) {
LOGGER_WARNING("Friend sent invalid parameters for re-call");
call->error = msi_EInvalidParam;
invoke_callback(call, msi_OnError);
@ -661,7 +668,7 @@ void handle_push ( MSICall *call, const MSIMessage *msg )
msg.capabilities.value = call->self_capabilities;
msg.vfpsz.exists = true;
msg.vfpsz.value = htons(VIDEOFRAME_PIECE_SIZE);
msg.vfpsz.value = VIDEOFRAME_PIECE_SIZE;
send_message ( call->session->messenger, call->friend_id, &msg );

3
toxav/msi.h
View File

@ -25,7 +25,8 @@
#include <inttypes.h>
#include <pthread.h>
#include "codec.h"
#include "audio.h"
#include "video.h"
#include "../toxcore/Messenger.h"
/** Preconfigured value for video splitting */

27
toxav/rtp.c
View File

@ -69,9 +69,11 @@ typedef struct RTCPSession_s {
/* These are defined externally */
void ac_queue_message(void *acp, RTPMessage *msg);
void vc_queue_message(void *vcp, RTPMessage *msg);
/* queue_message() is defined in codec.c */
void queue_message(RTPSession *session, RTPMessage *msg);
RTPHeader *parse_header_in ( const uint8_t *payload, int length );
RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length );
RTPMessage *msg_parse ( const uint8_t *data, int length );
@ -395,6 +397,10 @@ RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length )
}
RTPMessage *msg_parse ( const uint8_t *data, int length )
{
/* TODO: data dynamic, [0]
* TODO: dummy payload type
* TODO: parse header before allocating message
*/
RTPMessage *retu = calloc(1, sizeof (RTPMessage));
retu->header = parse_header_in ( data, length ); /* It allocates memory and all */
@ -540,6 +546,7 @@ void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber)
}
int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object )
{
/* TODO on message callback */
RTPSession *session = object;
RTPMessage *msg;
@ -570,8 +577,20 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data
}
session->rtcp_session->last_received_packets ++;
queue_message(session, msg);
/* Check if this session can handle the packet */
if (session->payload_type != session->prefix % 128) {
LOGGER_WARNING("Friend %d sent invalid payload type!", session->dest);
rtp_free_msg(msg);
return -1;
}
/* Handle */
if (session->payload_type == rtp_TypeAudio % 128)
ac_queue_message(session->cs, msg);
else /* It can only be video */
vc_queue_message(session->cs, msg);
return 0;
}
int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object )

11
toxav/toxav.c
View File

@ -1170,11 +1170,6 @@ void call_kill_transmission(ToxAVCall* call)
call->active = 0;
rtp_kill(call->rtps[audio_index]);
call->rtps[audio_index] = NULL;
rtp_kill(call->rtps[video_index]);
call->rtps[video_index] = NULL;
LOGGED_LOCK(call->mutex_audio_sending);
LOGGED_UNLOCK(call->mutex_audio_sending);
LOGGED_LOCK(call->mutex_video_sending);
@ -1182,6 +1177,12 @@ void call_kill_transmission(ToxAVCall* call)
LOGGED_LOCK(call->mutex_decoding);
LOGGED_UNLOCK(call->mutex_decoding);
rtp_kill(call->rtps[audio_index]);
call->rtps[audio_index] = NULL;
rtp_kill(call->rtps[video_index]);
call->rtps[video_index] = NULL;
cs_kill(call->cs);
call->cs = NULL;

2
toxav/toxav.h
View File

@ -499,7 +499,7 @@ bool toxav_send_audio_frame(ToxAV *av, uint32_t friend_number,
* Y = width * height, U = (width/2) * (height/2) and V = (width/2) * (height/2).
* @param ystride
* @param ustride
* @param vstride Strides data. Indexing is the same as in 'planes' param.
* @param vstride Strides data.
*/
typedef void toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number,
uint16_t width, uint16_t height,

332
toxav/video.c Normal file
View File

@ -0,0 +1,332 @@
/** video.c
*
* Copyright (C) 2013-2015 Tox project All Rights Reserved.
*
* This file is part of Tox.
*
* Tox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Tox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Tox. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdlib.h>
#include <assert.h>
#include "video.h"
#include "msi.h"
#include "../toxcore/logger.h"
#include "../toxcore/network.h"
/* Good quality encode. */
#define MAX_DECODE_TIME_US 0
#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */
#define VIDEOFRAME_HEADER_SIZE 0x2
/* FIXME: Might not be enough? NOTE: I think it is enough */
#define VIDEO_DECODE_BUFFER_SIZE 20
typedef struct { uint16_t size; uint8_t data[]; } Payload;
bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate);
VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* cb, void* cb_data, uint32_t mvfpsz)
{
VCSession *vc = calloc(sizeof(VCSession), 1);
if (!vc) {
LOGGER_WARNING("Allocation failed! Application might misbehave!");
return NULL;
}
if (create_recursive_mutex(vc->queue_mutex) != 0) {
LOGGER_WARNING("Failed to create recursive mutex!");
free(vc);
return NULL;
}
if ( !(vc->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) )
goto BASE_CLEANUP;
if ( !(vc->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) )
goto BASE_CLEANUP;
if ( !(vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) )
goto BASE_CLEANUP;
int rc = vpx_codec_dec_init_ver(vc->v_decoder, VIDEO_CODEC_DECODER_INTERFACE,
NULL, 0, VPX_DECODER_ABI_VERSION);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
goto BASE_CLEANUP;
}
if (!create_video_encoder(vc->v_encoder, 500000)) {
vpx_codec_destroy(vc->v_decoder);
goto BASE_CLEANUP;
}
vc->linfts = current_time_monotonic();
vc->lcfd = 60;
vc->peer_video_frame_piece_size = mvfpsz;
return vc;
BASE_CLEANUP:
pthread_mutex_destroy(vc->queue_mutex);
rb_free(vc->vbuf_raw);
free(vc->split_video_frame);
free(vc->frame_buf);
free(vc);
return NULL;
}
void vc_kill(VCSession* vc)
{
if (!vc)
return;
vpx_codec_destroy(vc->v_encoder);
vpx_codec_destroy(vc->v_decoder);
rb_free(vc->vbuf_raw);
free(vc->split_video_frame);
free(vc->frame_buf);
pthread_mutex_destroy(vc->queue_mutex);
LOGGER_DEBUG("Terminated video handler: %p", vc);
free(vc);
}
void vc_do(VCSession* vc)
{
if (!vc)
return;
Payload *p;
int rc;
pthread_mutex_lock(vc->queue_mutex);
if (rb_read(vc->vbuf_raw, (void**)&p)) {
pthread_mutex_unlock(vc->queue_mutex);
rc = vpx_codec_decode(vc->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US);
free(p);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc));
} else {
vpx_codec_iter_t iter = NULL;
vpx_image_t *dest = vpx_codec_get_frame(vc->v_decoder, &iter);
/* Play decoded images */
for (; dest; dest = vpx_codec_get_frame(vc->v_decoder, &iter)) {
if (vc->vcb.first)
vc->vcb.first(vc->av, vc->friend_id, dest->d_w, dest->d_h,
(const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2],
dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb.second);
vpx_img_free(dest);
}
}
return;
}
pthread_mutex_unlock(vc->queue_mutex);
}
void vc_init_video_splitter_cycle(VCSession* vc)
{
if (!vc)
return;
vc->split_video_frame[0] = vc->frameid_out++;
vc->split_video_frame[1] = 0;
}
int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length)
{
if (!vc)
return;
vc->processing_video_frame = payload;
vc->processing_video_frame_size = length;
return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1;
}
const uint8_t* vc_iterate_split_video_frame(VCSession* vc, uint16_t* size)
{
if (!vc || !size)
return NULL;
if (vc->processing_video_frame_size > VIDEOFRAME_PIECE_SIZE) {
memcpy(vc->split_video_frame + VIDEOFRAME_HEADER_SIZE,
vc->processing_video_frame,
VIDEOFRAME_PIECE_SIZE);
vc->processing_video_frame += VIDEOFRAME_PIECE_SIZE;
vc->processing_video_frame_size -= VIDEOFRAME_PIECE_SIZE;
*size = VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE;
} else {
memcpy(vc->split_video_frame + VIDEOFRAME_HEADER_SIZE,
vc->processing_video_frame,
vc->processing_video_frame_size);
*size = vc->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE;
}
vc->split_video_frame[1]++;
return vc->split_video_frame;
}
int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height)
{
if (!vc)
return;
vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc;
if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height)
return 0; /* Nothing changed */
cfg.rc_target_bitrate = bitrate;
cfg.g_w = width;
cfg.g_h = height;
int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
return -1;
}
return 0;
}
/* Called from RTP */
void vc_queue_message(void* vcp, RTPMessage *msg)
{
/* This function does the reconstruction of video packets.
* See more info about video splitting in docs
*/
if (!vcp || !msg)
return;
VCSession* vc = vcp;
uint8_t *packet = msg->data;
uint32_t packet_size = msg->length;
if (packet_size < VIDEOFRAME_HEADER_SIZE)
goto end;
uint8_t diff = packet[0] - vc->frameid_in;
if (diff != 0) {
if (diff < 225) { /* New frame */
/* Flush last frames' data and get ready for this frame */
Payload *p = malloc(sizeof(Payload) + vc->frame_size);
if (p) {
LOGGED_LOCK(vc->queue_mutex);
if (rb_full(vc->vbuf_raw)) {
LOGGER_DEBUG("Dropped video frame");
Payload *tp;
rb_read(vc->vbuf_raw, (void**)&tp);
free(tp);
} else {
p->size = vc->frame_size;
memcpy(p->data, vc->frame_buf, vc->frame_size);
}
/* Calculate time took for peer to send us this frame */
uint32_t t_lcfd = current_time_monotonic() - vc->linfts;
vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd;
vc->linfts = current_time_monotonic();
rb_write(vc->vbuf_raw, p);
LOGGED_UNLOCK(vc->queue_mutex);
} else {
LOGGER_WARNING("Allocation failed! Program might misbehave!");
goto end;
}
vc->frameid_in = packet[0];
memset(vc->frame_buf, 0, vc->frame_size);
vc->frame_size = 0;
} else { /* Old frame; drop */
LOGGER_DEBUG("Old packet: %u", packet[0]);
goto end;
}
}
uint8_t piece_number = packet[1];
uint32_t length_before_piece = ((piece_number - 1) * vc->peer_video_frame_piece_size);
uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE);
if (framebuf_new_length > MAX_VIDEOFRAME_SIZE)
goto end;
/* Otherwise it's part of the frame so just process */
/* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */
memcpy(vc->frame_buf + length_before_piece,
packet + VIDEOFRAME_HEADER_SIZE,
packet_size - VIDEOFRAME_HEADER_SIZE);
if (framebuf_new_length > vc->frame_size)
vc->frame_size = framebuf_new_length;
end:
rtp_free_msg(NULL, msg);
}
bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate)
{
assert(dest);
vpx_codec_enc_cfg_t cfg;
int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
if (rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
return false;
}
rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0,
VPX_ENCODER_ABI_VERSION);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
return false;
}
cfg.rc_target_bitrate = bitrate;
cfg.g_w = 800;
cfg.g_h = 600;
cfg.g_pass = VPX_RC_ONE_PASS;
cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
cfg.g_lag_in_frames = 0;
cfg.kf_min_dist = 0;
cfg.kf_max_dist = 48;
cfg.kf_mode = VPX_KF_AUTO;
rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8);
if ( rc != VPX_CODEC_OK) {
LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
vpx_codec_destroy(dest);
}
return true;
}

81
toxav/video.h Normal file
View File

@ -0,0 +1,81 @@
/** video.h
*
* Copyright (C) 2013-2015 Tox project All Rights Reserved.
*
* This file is part of Tox.
*
* Tox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Tox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Tox. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifndef VIDEO_H
#define VIDEO_H
#include <vpx/vpx_decoder.h>
#include <vpx/vpx_encoder.h>
#include <vpx/vp8dx.h>
#include <vpx/vp8cx.h>
#include <vpx/vpx_image.h>
#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx())
#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx())
#include <pthread.h>
#include "toxav.h"
#include "../toxcore/util.h"
typedef struct VCSession_s {
/* encoding */
vpx_codec_ctx_t v_encoder[1];
uint32_t frame_counter;
/* decoding */
vpx_codec_ctx_t v_decoder[1];
void *vbuf_raw; /* Un-decoded data */
/* Data handling */
uint8_t *frame_buf; /* buffer for split video payloads */
uint32_t frame_size; /* largest address written to in frame_buf for current input frame */
uint8_t frameid_in, frameid_out; /* id of input and output video frame */
uint64_t linfts; /* Last received frame time stamp */
uint32_t lcfd; /* Last calculated frame duration for incoming video payload */
/* Limits */
uint32_t peer_video_frame_piece_size;
/* Splitting */
uint8_t *split_video_frame;
const uint8_t *processing_video_frame;
uint16_t processing_video_frame_size;
ToxAV *av;
int32_t friend_id;
PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */
pthread_mutex_t queue_mutex[1];
} VCSession;
VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb *cb, void *cb_data, uint32_t mvfpsz);
void vc_kill(VCSession* vc);
void vc_do(VCSession* vc);
void vc_init_video_splitter_cycle(VCSession* vc);
int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length);
const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size);
int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height);
#endif /* VIDEO_H */