StarMirror/toxcore
1
0
Fork 0
mirror of https://github.com/irungentoo/toxcore.git synced 2024-03-22 13:30:51 +08:00
Code Issues Projects Releases Wiki Activity

tox A/V: encode/decode and display/playback

Browse Source
This commit is contained in:
Martijn 2013-10-13 16:14:27 +02:00 committed by BtbN
parent f2497b6589
commit 1b971de651
3 changed files with 1636 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

823
toxmsi/AV_codec.c Normal file
View File

@ -0,0 +1,823 @@
/* AV_codec.c
// *
* Audio and video codec intitialisation, encoding/decoding and playback
*
* Copyright (C) 2013 Tox project All Rights Reserved.
*
* This file is part of Tox.
*
* Tox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Tox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Tox. If not, see <http://www.gnu.org/licenses/>.
*
*/
/*----------------------------------------------------------------------------------*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif /* HAVE_CONFIG_H */
#include <stdio.h>
#include <math.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libavdevice/avdevice.h>
#include <libavutil/opt.h>
#include <AL/al.h>
#include <AL/alc.h>
#include <SDL.h>
#include <SDL_thread.h>
#include <pthread.h>
#include <opus/opus.h>
#include "toxmsi.h"
#include "toxmsi_message.h"
#include "toxrtp_message.h"
#include "toxrtp/tests/test_helper.h"
#include "phone.h"
#include "AV_codec.h"
int display_received_frame(codec_state *cs, AVFrame *r_video_frame)
{
AVPicture pict;
SDL_LockYUVOverlay(cs->video_picture.bmp);
pict.data[0] = cs->video_picture.bmp->pixels[0];
pict.data[1] = cs->video_picture.bmp->pixels[2];
pict.data[2] = cs->video_picture.bmp->pixels[1];
pict.linesize[0] = cs->video_picture.bmp->pitches[0];
pict.linesize[1] = cs->video_picture.bmp->pitches[2];
pict.linesize[2] = cs->video_picture.bmp->pitches[1];
/* Convert the image into YUV format that SDL uses */
sws_scale(cs->sws_SDL_r_ctx, (uint8_t const * const *)r_video_frame->data, r_video_frame->linesize, 0,
cs->video_decoder_ctx->height, pict.data, pict.linesize );
SDL_UnlockYUVOverlay(cs->video_picture.bmp);
SDL_Rect rect;
rect.x = 0;
rect.y = 0;
rect.w = cs->video_decoder_ctx->width;
rect.h = cs->video_decoder_ctx->height;
SDL_DisplayYUVOverlay(cs->video_picture.bmp, &rect);
return 1;
}
struct jitter_buffer {
rtp_msg_t **queue;
uint16_t capacity;
uint16_t size;
uint16_t front;
uint16_t rear;
uint8_t queue_ready;
uint16_t current_id;
uint32_t current_ts;
uint8_t id_set;
};
struct jitter_buffer *create_queue(int capacity)
{
struct jitter_buffer *q;
q = (struct jitter_buffer *)calloc(sizeof(struct jitter_buffer),1);
q->queue = (rtp_msg_t **)calloc((sizeof(rtp_msg_t) * capacity),1);
int i = 0;
for (i = 0; i < capacity; ++i) {
q->queue[i] = NULL;
}
q->size = 0;
q->capacity = capacity;
q->front = 0;
q->rear = -1;
q->queue_ready = 0;
q->current_id = 0;
q->current_ts = 0;
q->id_set = 0;
return q;
}
/* returns 1 if 'a' has a higher sequence number than 'b' */
uint8_t sequence_number_older(uint16_t sn_a, uint16_t sn_b, uint32_t ts_a, uint32_t ts_b)
{
/* should be stable enough */
return (sn_a > sn_b || ts_a > ts_b);
}
/* success is 0 when there is nothing to dequeue, 1 when there's a good packet, 2 when there's a lost packet */
rtp_msg_t *dequeue(struct jitter_buffer *q, int *success)
{
if (q->size == 0 || q->queue_ready == 0) {
q->queue_ready = 0;
*success = 0;
return NULL;
}
int front = q->front;
if (q->id_set == 0) {
q->current_id = q->queue[front]->_header->_sequence_number;
q->current_ts = q->queue[front]->_header->_timestamp;
q->id_set = 1;
} else {
int next_id = q->queue[front]->_header->_sequence_number;
int next_ts = q->queue[front]->_header->_timestamp;
/* if this packet is indeed the expected packet */
if (next_id == (q->current_id + 1) % _MAX_SEQU_NUM) {
q->current_id = next_id;
q->current_ts = next_ts;
} else {
if (sequence_number_older(next_id, q->current_id, next_ts, q->current_ts)) {
printf("nextid: %d current: %d\n", next_id, q->current_id);
q->current_id = (q->current_id + 1) % _MAX_SEQU_NUM;
*success = 2; /* tell the decoder the packet is lost */
return NULL;
} else {
/* packet too old */
printf("packet too old\n");
*success = 0;
return NULL;
}
}
}
q->size--;
q->front++;
if (q->front == q->capacity)
q->front = 0;
*success = 1;
q->current_id = q->queue[front]->_header->_sequence_number;
q->current_ts = q->queue[front]->_header->_timestamp;
return q->queue[front];
}
int empty_queue(struct jitter_buffer *q)
{
while (q->size > 0) {
q->size--;
/* FIXME: */
/* rtp_free_msg(cs->_rtp_video, q->queue[q->front]); */
q->front++;
if (q->front == q->capacity)
q->front = 0;
}
q->id_set = 0;
q->queue_ready = 0;
return 0;
}
int queue(struct jitter_buffer *q, rtp_msg_t *pk)
{
if (q->size == q->capacity) {
printf("buffer full, emptying buffer...\n");
empty_queue(q);
return 0;
}
if (q->size > 8)
q->queue_ready = 1;
++q->size;
++q->rear;
if (q->rear == q->capacity)
q->rear = 0;
q->queue[q->rear] = pk;
int a;
int b;
int j;
a = q->rear;
for (j = 0; j < q->size - 1; ++j) {
b = a - 1;
if (b < 0)
b += q->capacity;
if (sequence_number_older(q->queue[b]->_header->_sequence_number, q->queue[a]->_header->_sequence_number,
q->queue[b]->_header->_timestamp, q->queue[a]->_header->_timestamp)) {
rtp_msg_t *temp;
temp = q->queue[a];
q->queue[a] = q->queue[b];
q->queue[b] = temp;
printf("had to swap\n");
} else {
break;
}
a -= 1;
if (a < 0)
a += q->capacity;
}
if (pk)
return 1;
return 0;
}
int init_receive_audio(codec_state *cs)
{
int err = OPUS_OK;
cs->audio_decoder = opus_decoder_create(48000, 1, &err);
opus_decoder_init(cs->audio_decoder, 48000, 1);
printf("init audio decoder successful\n");
return 1;
}
int init_receive_video(codec_state *cs)
{
cs->video_decoder = avcodec_find_decoder(VIDEO_CODEC);
if (!cs->video_decoder) {
printf("init video_decoder failed\n");
return 0;
}
cs->video_decoder_ctx = avcodec_alloc_context3(cs->video_decoder);
if (!cs->video_decoder_ctx) {
printf("init video_decoder_ctx failed\n");
return 0;
}
if (avcodec_open2(cs->video_decoder_ctx, cs->video_decoder, NULL) < 0) {
printf("opening video decoder failed\n");
return 0;
}
printf("init video decoder successful\n");
return 1;
}
int init_send_video(codec_state *cs)
{
cs->video_input_format = av_find_input_format(VIDEO_DRIVER);
if (avformat_open_input(&cs->video_format_ctx, DEFAULT_WEBCAM, cs->video_input_format, NULL) != 0) {
printf("opening video_input_format failed\n");
return 0;
}
avformat_find_stream_info(cs->video_format_ctx, NULL);
av_dump_format(cs->video_format_ctx, 0, DEFAULT_WEBCAM, 0);
int i;
for (i = 0; i < cs->video_format_ctx->nb_streams; ++i) {
if (cs->video_format_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
cs->video_stream = i;
break;
}
}
cs->webcam_decoder_ctx = cs->video_format_ctx->streams[cs->video_stream]->codec;
cs->webcam_decoder = avcodec_find_decoder(cs->webcam_decoder_ctx->codec_id);
if (cs->webcam_decoder == NULL) {
printf("Unsupported codec\n");
return 0;
}
if (cs->webcam_decoder_ctx == NULL) {
printf("init webcam_decoder_ctx failed\n");
return 0;
}
if (avcodec_open2(cs->webcam_decoder_ctx, cs->webcam_decoder, NULL) < 0) {
printf("opening webcam decoder failed\n");
return 0;
}
cs->video_encoder = avcodec_find_encoder(VIDEO_CODEC);
if (!cs->video_encoder) {
printf("init video_encoder failed\n");
return 0;
}
cs->video_encoder_ctx = avcodec_alloc_context3(cs->video_encoder);
if (!cs->video_encoder_ctx) {
printf("init video_encoder_ctx failed\n");
return 0;
}
cs->video_encoder_ctx->bit_rate = VIDEO_BITRATE;
cs->video_encoder_ctx->rc_min_rate = cs->video_encoder_ctx->rc_max_rate = cs->video_encoder_ctx->bit_rate;
av_opt_set_double(cs->video_encoder_ctx->priv_data, "max-intra-rate", 90, 0);
av_opt_set(cs->video_encoder_ctx->priv_data, "quality", "realtime", 0);
cs->video_encoder_ctx->thread_count = 4;
cs->video_encoder_ctx->rc_buffer_aggressivity = 0.95;
cs->video_encoder_ctx->rc_buffer_size = VIDEO_BITRATE * 6;
cs->video_encoder_ctx->profile = 3;
cs->video_encoder_ctx->qmax = 54;
cs->video_encoder_ctx->qmin = 4;
AVRational myrational = {1, 25};
cs->video_encoder_ctx->time_base = myrational;
cs->video_encoder_ctx->gop_size = 99999;
cs->video_encoder_ctx->pix_fmt = PIX_FMT_YUV420P;
cs->video_encoder_ctx->width = cs->webcam_decoder_ctx->width;
cs->video_encoder_ctx->height = cs->webcam_decoder_ctx->height;
if (avcodec_open2(cs->video_encoder_ctx, cs->video_encoder, NULL) < 0) {
printf("opening video encoder failed\n");
return 0;
}
printf("init video encoder successful\n");
return 1;
}
int init_send_audio(codec_state *cs)
{
cs->support_send_audio = 0;
const ALchar *pDeviceList = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER);
int i = 0;
const ALchar *device_names[20];
if (pDeviceList) {
printf("\nAvailable Capture Devices are:\n");
while (*pDeviceList) {
device_names[i] = pDeviceList;
printf("%d) %s\n", i, device_names[i]);
pDeviceList += strlen(pDeviceList) + 1;
++i;
}
}
printf("enter capture device number: \n");
char dev[2];
fgets(dev, sizeof(dev), stdin);
cs->audio_capture_device = alcCaptureOpenDevice(device_names[dev[0] - 48], AUDIO_SAMPLE_RATE, AL_FORMAT_MONO16,
AUDIO_FRAME_SIZE * 4);
if (alcGetError(cs->audio_capture_device) != AL_NO_ERROR) {
printf("could not start capture device! %d\n", alcGetError(cs->audio_capture_device));
return 0;
}
int err = OPUS_OK;
cs->audio_bitrate = AUDIO_BITRATE;
cs->audio_encoder = opus_encoder_create(AUDIO_SAMPLE_RATE, 1, OPUS_APPLICATION_VOIP, &err);
err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_bitrate));
err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10));
err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE));
opus_encoder_init(cs->audio_encoder, AUDIO_SAMPLE_RATE, 1, OPUS_APPLICATION_VOIP);
int nfo;
err = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_LOOKAHEAD(&nfo));
/* printf("Encoder lookahead delay : %d\n", nfo); */
printf("init audio encoder successful\n");
return 1;
}
int init_encoder(codec_state *cs)
{
avdevice_register_all();
avcodec_register_all();
avdevice_register_all();
av_register_all();
pthread_mutex_init(&cs->rtp_msg_mutex_lock, NULL);
pthread_mutex_init(&cs->avcodec_mutex_lock, NULL);
cs->support_send_video = init_send_video(cs);
cs->support_send_audio = init_send_audio(cs);
cs->send_audio = 1;
cs->send_video = 1;
return 1;
}
int init_decoder(codec_state *cs)
{
avdevice_register_all();
avcodec_register_all();
avdevice_register_all();
av_register_all();
cs->receive_video = 0;
cs->receive_audio = 0;
cs->support_receive_video = init_receive_video(cs);
cs->support_receive_audio = init_receive_audio(cs);
cs->receive_audio = 1;
cs->receive_video = 1;
return 1;
}
int video_encoder_refresh(codec_state *cs, int bps)
{
if (cs->video_encoder_ctx)
avcodec_close(cs->video_encoder_ctx);
cs->video_encoder = avcodec_find_encoder(VIDEO_CODEC);
if (!cs->video_encoder) {
printf("init video_encoder failed\n");
return -1;
}
cs->video_encoder_ctx = avcodec_alloc_context3(cs->video_encoder);
if (!cs->video_encoder_ctx) {
printf("init video_encoder_ctx failed\n");
return -1;
}
cs->video_encoder_ctx->bit_rate = bps;
cs->video_encoder_ctx->rc_min_rate = cs->video_encoder_ctx->rc_max_rate = cs->video_encoder_ctx->bit_rate;
av_opt_set_double(cs->video_encoder_ctx->priv_data, "max-intra-rate", 90, 0);
av_opt_set(cs->video_encoder_ctx->priv_data, "quality", "realtime", 0);
cs->video_encoder_ctx->thread_count = 4;
cs->video_encoder_ctx->rc_buffer_aggressivity = 0.95;
cs->video_encoder_ctx->rc_buffer_size = bps * 6;
cs->video_encoder_ctx->profile = 0;
cs->video_encoder_ctx->qmax = 54;
cs->video_encoder_ctx->qmin = 4;
AVRational myrational = {1, 25};
cs->video_encoder_ctx->time_base = myrational;
cs->video_encoder_ctx->gop_size = 99999;
cs->video_encoder_ctx->pix_fmt = PIX_FMT_YUV420P;
cs->video_encoder_ctx->width = cs->webcam_decoder_ctx->width;
cs->video_encoder_ctx->height = cs->webcam_decoder_ctx->height;
if (avcodec_open2(cs->video_encoder_ctx, cs->video_encoder, NULL) < 0) {
printf("opening video encoder failed\n");
return -1;
}
return 0;
}
void *encode_video_thread(void *arg)
{
codec_state *cs = (codec_state *)arg;
AVPacket pkt1, *packet = &pkt1;
int p = 0;
int err;
int got_packet;
rtp_msg_t *s_video_msg;
int video_frame_finished;
AVFrame *s_video_frame;
AVFrame *webcam_frame;
s_video_frame = avcodec_alloc_frame();
webcam_frame = avcodec_alloc_frame();
AVPacket enc_video_packet;
uint8_t *buffer;
int numBytes;
/* Determine required buffer size and allocate buffer */
numBytes = avpicture_get_size(PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height);
buffer = (uint8_t *)av_calloc(numBytes * sizeof(uint8_t),1);
avpicture_fill((AVPicture *)s_video_frame, buffer, PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width,
cs->webcam_decoder_ctx->height);
cs->sws_ctx = sws_getContext(cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height,
cs->webcam_decoder_ctx->pix_fmt, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height, PIX_FMT_YUV420P,
SWS_BILINEAR, NULL, NULL, NULL);
while (!cs->quit && cs->send_video) {
if (av_read_frame(cs->video_format_ctx, packet) < 0) {
printf("error reading frame\n");
if (cs->video_format_ctx->pb->error != 0)
break;
continue;
}
if (packet->stream_index == cs->video_stream) {
if (avcodec_decode_video2(cs->webcam_decoder_ctx, webcam_frame, &video_frame_finished, packet) < 0) {
printf("couldn't decode\n");
continue;
}
av_free_packet(packet);
sws_scale(cs->sws_ctx, (uint8_t const * const *)webcam_frame->data, webcam_frame->linesize, 0,
cs->webcam_decoder_ctx->height, s_video_frame->data, s_video_frame->linesize);
/* create a new I-frame every 60 frames */
++p;
if (p == 60) {
s_video_frame->pict_type = AV_PICTURE_TYPE_BI ;
} else if (p == 61) {
s_video_frame->pict_type = AV_PICTURE_TYPE_I ;
p = 0;
} else {
s_video_frame->pict_type = AV_PICTURE_TYPE_P ;
}
if (video_frame_finished) {
err = avcodec_encode_video2(cs->video_encoder_ctx, &enc_video_packet, s_video_frame, &got_packet);
if (err < 0) {
printf("could not encode video frame\n");
continue;
}
if (!got_packet) {
continue;
}
pthread_mutex_lock(&cs->rtp_msg_mutex_lock);
THREADLOCK()
if (!enc_video_packet.data) fprintf(stderr, "video packet data is NULL\n");
s_video_msg = rtp_msg_new ( cs->_rtp_video, enc_video_packet.data, enc_video_packet.size ) ;
if (!s_video_msg) {
printf("invalid message\n");
}
rtp_send_msg ( cs->_rtp_video, s_video_msg, cs->_networking );
THREADUNLOCK()
pthread_mutex_unlock(&cs->rtp_msg_mutex_lock);
av_free_packet(&enc_video_packet);
}
} else {
av_free_packet(packet);
}
}
/* clean up codecs */
pthread_mutex_lock(&cs->avcodec_mutex_lock);
av_free(buffer);
av_free(webcam_frame);
av_free(s_video_frame);
sws_freeContext(cs->sws_ctx);
avcodec_close(cs->webcam_decoder_ctx);
avcodec_close(cs->video_encoder_ctx);
pthread_mutex_unlock(&cs->avcodec_mutex_lock);
pthread_exit ( NULL );
}
void *encode_audio_thread(void *arg)
{
codec_state *cs = (codec_state *)arg;
rtp_msg_t *s_audio_msg;
unsigned char encoded_data[4096];
int encoded_size = 0;
int16_t frame[4096];
int frame_size = AUDIO_FRAME_SIZE;
ALint sample = 0;
alcCaptureStart(cs->audio_capture_device);
while (!cs->quit && cs->send_audio) {
alcGetIntegerv(cs->audio_capture_device, ALC_CAPTURE_SAMPLES, (ALCsizei)sizeof(ALint), &sample);
if (sample >= frame_size) {
alcCaptureSamples(cs->audio_capture_device, frame, frame_size);
encoded_size = opus_encode(cs->audio_encoder, frame, frame_size, encoded_data, 480);
if (encoded_size <= 0) {
printf("Could not encode audio packet\n");
} else {
pthread_mutex_lock(&cs->rtp_msg_mutex_lock);
THREADLOCK()
rtp_set_payload_type(cs->_rtp_audio, 96);
s_audio_msg = rtp_msg_new (cs->_rtp_audio, encoded_data, encoded_size) ;
rtp_send_msg ( cs->_rtp_audio, s_audio_msg, cs->_networking );
pthread_mutex_unlock(&cs->rtp_msg_mutex_lock);
THREADUNLOCK()
}
} else {
usleep(1000);
}
}
/* clean up codecs */
pthread_mutex_lock(&cs->avcodec_mutex_lock);
alcCaptureStop(cs->audio_capture_device);
alcCaptureCloseDevice(cs->audio_capture_device);
pthread_mutex_unlock(&cs->avcodec_mutex_lock);
pthread_exit ( NULL );
}
int video_decoder_refresh(codec_state *cs, int width, int height)
{
printf("need to refresh\n");
screen = SDL_SetVideoMode(width, height, 0, 0);
if (cs->video_picture.bmp)
SDL_FreeYUVOverlay(cs->video_picture.bmp);
cs->video_picture.bmp = SDL_CreateYUVOverlay(width, height, SDL_YV12_OVERLAY, screen);
cs->sws_SDL_r_ctx = sws_getContext(width, height, cs->video_decoder_ctx->pix_fmt, width, height, PIX_FMT_YUV420P,
SWS_BILINEAR, NULL, NULL, NULL);
return 1;
}
void *decode_video_thread(void *arg)
{
codec_state *cs = (codec_state *)arg;
cs->video_stream = 0;
rtp_msg_t *r_msg;
int dec_frame_finished;
AVFrame *r_video_frame;
r_video_frame = avcodec_alloc_frame();
AVPacket dec_video_packet;
av_new_packet (&dec_video_packet, 65536);
int width = 0;
int height = 0;
while (!cs->quit && cs->receive_video) {
r_msg = rtp_recv_msg ( cs->_rtp_video );
if (r_msg) {
memcpy(dec_video_packet.data, r_msg->_data, r_msg->_length);
dec_video_packet.size = r_msg->_length;
avcodec_decode_video2(cs->video_decoder_ctx, r_video_frame, &dec_frame_finished, &dec_video_packet);
if (dec_frame_finished) {
if (cs->video_decoder_ctx->width != width || cs->video_decoder_ctx->height != height) {
width = cs->video_decoder_ctx->width;
height = cs->video_decoder_ctx->height;
printf("w: %d h%d \n", width, height);
video_decoder_refresh(cs, width, height);
}
display_received_frame(cs, r_video_frame);
} else {
/* TODO: request the sender to create a new i-frame immediatly */
printf("bad video packet\n");
}
rtp_free_msg(cs->_rtp_video, r_msg);
}
usleep(1000);
}
printf("vend\n");
/* clean up codecs */
pthread_mutex_lock(&cs->avcodec_mutex_lock);
av_free(r_video_frame);
avcodec_close(cs->video_decoder_ctx);
pthread_mutex_unlock(&cs->avcodec_mutex_lock);
pthread_exit ( NULL );
}
void *decode_audio_thread(void *arg)
{
codec_state *cs = (codec_state *)arg;
rtp_msg_t *r_msg;
int frame_size = AUDIO_FRAME_SIZE;
int data_size;
ALCdevice *dev;
ALCcontext *ctx;
ALuint source, *buffers;
dev = alcOpenDevice(NULL);
ctx = alcCreateContext(dev, NULL);
alcMakeContextCurrent(ctx);
int openal_buffers = 5;
buffers = calloc(sizeof(ALuint) * openal_buffers,1);
alGenBuffers(openal_buffers, buffers);
alGenSources((ALuint)1, &source);
alSourcei(source, AL_LOOPING, AL_FALSE);
ALuint buffer;
ALint val;
ALenum error;
uint16_t zeros[frame_size];
int i;
for (i = 0; i < frame_size; i++) {
zeros[i] = 0;
}
for (i = 0; i < openal_buffers; ++i) {
alBufferData(buffers[i], AL_FORMAT_MONO16, zeros, frame_size, 48000);
}
alSourceQueueBuffers(source, openal_buffers, buffers);
alSourcePlay(source);
if (alGetError() != AL_NO_ERROR) {
fprintf(stderr, "Error starting audio\n");
cs->quit = 1;
}
struct jitter_buffer *j_buf = NULL;
j_buf = create_queue(20);
int success = 0;
int dec_frame_len;
opus_int16 PCM[frame_size];
while (!cs->quit && cs->receive_audio) {
THREADLOCK()
r_msg = rtp_recv_msg ( cs->_rtp_audio );
if (r_msg) {
/* push the packet into the queue */
queue(j_buf, r_msg);
}
/* grab a packet from the queue */
success = 0;
alGetSourcei(source, AL_BUFFERS_PROCESSED, &val);
if (val > 0)
r_msg = dequeue(j_buf, &success);
if (success > 0) {
/* good packet */
if (success == 1) {
dec_frame_len = opus_decode(cs->audio_decoder, r_msg->_data, r_msg->_length, PCM, frame_size, 0);
rtp_free_msg(cs->_rtp_audio, r_msg);
}
/* lost packet */
if (success == 2) {
printf("lost packet\n");
dec_frame_len = opus_decode(cs->audio_decoder, NULL, 0, PCM, frame_size, 1);
}
if (dec_frame_len > 0) {
alGetSourcei(source, AL_BUFFERS_PROCESSED, &val);
if (val <= 0)
continue;
alSourceUnqueueBuffers(source, 1, &buffer);
data_size = av_samples_get_buffer_size(NULL, 1, dec_frame_len, AV_SAMPLE_FMT_S16, 1);
alBufferData(buffer, AL_FORMAT_MONO16, PCM, data_size, 48000);
int error = alGetError();
if (error != AL_NO_ERROR) {
fprintf(stderr, "Error setting buffer %d\n", error);
break;
}
alSourceQueueBuffers(source, 1, &buffer);
if (alGetError() != AL_NO_ERROR) {
fprintf(stderr, "error: could not buffer audio\n");
break;
}
alGetSourcei(source, AL_SOURCE_STATE, &val);
if (val != AL_PLAYING)
alSourcePlay(source);
}
}
THREADUNLOCK()
usleep(1000);
}
/* clean up codecs */
pthread_mutex_lock(&cs->avcodec_mutex_lock);
/* clean up openal */
alDeleteSources(1, &source);
alDeleteBuffers(openal_buffers, buffers);
alcMakeContextCurrent(NULL);
alcDestroyContext(ctx);
alcCloseDevice(dev);
pthread_mutex_unlock(&cs->avcodec_mutex_lock);
pthread_exit ( NULL );
}

168
toxmsi/AV_codec.h Normal file
View File

@ -0,0 +1,168 @@
/* AV_codec.h
*
* Audio and video codec intitialisation, encoding/decoding and playback
*
* Copyright (C) 2013 Tox project All Rights Reserved.
*
* This file is part of Tox.
*
* Tox is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Tox is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Tox. If not, see <http://www.gnu.org/licenses/>.
*
*/
/*----------------------------------------------------------------------------------*/
#ifndef _AVCODEC_H_
#define _AVCODEC_H_
#include <stdio.h>
#include <math.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libavdevice/avdevice.h>
#include <libavutil/opt.h>
#include <pthread.h>
#include <AL/al.h>
#include <AL/alc.h>
#include "toxrtp.h"
#include "tox.h"
#include <SDL.h>
#include <opus/opus.h>
/* ffmpeg VP8 codec ID */
#define VIDEO_CODEC AV_CODEC_ID_VP8
/* ffmpeg Opus codec ID */
#define AUDIO_CODEC AV_CODEC_ID_OPUS
/* default video bitrate in bytes/s */
#define VIDEO_BITRATE 10*1000
/* default audio bitrate in bytes/s */
#define AUDIO_BITRATE 64000
/* audio frame duration in miliseconds */
#define AUDIO_FRAME_DURATION 20
/* audio sample rate recommended to be 48kHz for Opus */
#define AUDIO_SAMPLE_RATE 48000
/* the amount of samples in one audio frame */
#define AUDIO_FRAME_SIZE AUDIO_SAMPLE_RATE*AUDIO_FRAME_DURATION/1000
/* the quit event for SDL */
#define FF_QUIT_EVENT (SDL_USEREVENT + 2)
#ifdef __linux__
#define VIDEO_DRIVER "video4linux2"
#define DEFAULT_WEBCAM "/dev/video0"
#endif
#ifdef WIN32
#define VIDEO_DRIVER "vfwcap"
#define DEFAULT_WEBCAM "0"
#endif
SDL_Surface *screen;
typedef struct {
SDL_Overlay *bmp;
int width, height;
} VideoPicture;
typedef struct {
uint8_t send_audio;
uint8_t receive_audio;
uint8_t send_video;
uint8_t receive_video;
uint8_t support_send_audio;
uint8_t support_send_video;
uint8_t support_receive_audio;
uint8_t support_receive_video;
/* video encoding */
AVInputFormat *video_input_format;
AVFormatContext *video_format_ctx;
uint8_t video_stream;
AVCodecContext *webcam_decoder_ctx;
AVCodec *webcam_decoder;
AVCodecContext *video_encoder_ctx;
AVCodec *video_encoder;
/* video decoding */
AVCodecContext *video_decoder_ctx;
AVCodec *video_decoder;
/* audio encoding */
ALCdevice *audio_capture_device;
OpusEncoder *audio_encoder;
int audio_bitrate;
/* audio decoding */
OpusDecoder *audio_decoder;
uint8_t req_video_refresh;
/* context for converting image format to something SDL can use*/
struct SwsContext *sws_SDL_r_ctx;
/* context for converting webcam image format to something the video encoder can use */
struct SwsContext *sws_ctx;
/* rendered video picture, ready for display */
VideoPicture video_picture;
rtp_session_t *_rtp_video;
rtp_session_t *_rtp_audio;
int socket;
Networking_Core *_networking;
pthread_t encode_audio_thread;
pthread_t encode_video_thread;
pthread_t decode_audio_thread;
pthread_t decode_video_thread;
pthread_mutex_t rtp_msg_mutex_lock;
pthread_mutex_t avcodec_mutex_lock;
uint8_t quit;
SDL_Event SDL_event;
msi_session_t *_msi;
uint32_t _frame_rate;
uint16_t _send_port, _recv_port;
int _tox_sock;
//pthread_id _medialoop_id;
} codec_state;
int display_received_frame(codec_state *cs, AVFrame *r_video_frame);
int init_receive_audio(codec_state *cs);
int init_decoder(codec_state *cs);
int init_send_video(codec_state *cs);
int init_send_audio(codec_state *cs);
int init_encoder(codec_state *cs);
int video_encoder_refresh(codec_state *cs, int bps);
void *encode_video_thread(void *arg);
void *encode_audio_thread(void *arg);
int video_decoder_refresh(codec_state *cs, int width, int height);
int handle_rtp_video_packet(codec_state *cs, rtp_msg_t *r_msg);
void *decode_video_thread(void *arg);
void *decode_audio_thread(void *arg);
#endif

645
toxmsi/phone.c Normal file
View File

@ -0,0 +1,645 @@
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif /* HAVE_CONFIG_H */
#define _BSD_SOURCE
#define _GNU_SOURCE
#define _CT_PHONE
#ifdef _CT_PHONE
#include "phone.h"
#include <stdarg.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <termios.h>
#include <pthread.h>
#include "AV_codec.h"
void INFO (const char* _format, ...)
{
printf("\r[!] ");
va_list _arg;
va_start (_arg, _format);
vfprintf (stdout, _format, _arg);
va_end (_arg);
printf("\n\r >> ");
fflush(stdout);
}
int rtp_handlepacket ( void* _object, tox_IP_Port ip_port, uint8_t* data, uint32_t length )
{
phone_t* _phone = _object;
rtp_msg_t* _msg;
uint8_t _payload_id;
if ( _phone->_msi->_call && _phone->_msi->_call->_state == call_active ){
_msg = rtp_msg_parse ( NULL, data + 1, length - 1 ); /* ignore marker byte */
if ( !_msg )
return 0;
_payload_id = rtp_header_get_setting_payload_type(_msg->_header);
if ( _payload_id == _PAYLOAD_OPUS && _phone->_rtp_audio )
rtp_store_msg(_phone->_rtp_audio, _msg);
else if ( _payload_id == _PAYLOAD_VP8 && _phone->_rtp_video )
rtp_store_msg(_phone->_rtp_video, _msg);
else rtp_free_msg( NULL, _msg);
}
return SUCCESS;
}
int msi_handlepacket ( void* _object, tox_IP_Port ip_port, uint8_t* data, uint32_t length )
{
msi_session_t* _session = _object;
msi_msg_t* _msg;
_msg = msi_parse_msg ( data + 1 ); /* ignore marker byte */
if ( _msg ) {
/* my current solution for "hole punching" */
_session->_friend_id = ip_port;
} else {
return FAILURE;
}
/* place message in a session */
msi_store_msg(_session, _msg);
return SUCCESS;
}
void* phone_receivepacket ( void* _phone_p )
{
phone_t* _phone = _phone_p;
networking_registerhandler(_phone->_networking, MSI_PACKET, msi_handlepacket, _phone->_msi);
networking_registerhandler(_phone->_networking, RTP_PACKET, rtp_handlepacket, _phone);
/* Now start main networking loop */
while ( _phone->_networking ) { /* so not thread safe */
networking_poll(_phone->_networking);
usleep(10000);
}
pthread_exit ( NULL );
}
/* Media transport callback */
typedef struct hmtc_args_s {
rtp_session_t** _rtp_audio;
rtp_session_t** _rtp_video;
call_type* _local_type_call;
call_state* _this_call;
void *_core_handler;
} hmtc_args_t;
void* phone_handle_media_transport_poll ( void* _hmtc_args_p )
{
rtp_msg_t* _audio_msg, * _video_msg;
hmtc_args_t* _hmtc_args = _hmtc_args_p;
rtp_session_t* _rtp_audio = *_hmtc_args->_rtp_audio;
rtp_session_t* _rtp_video = *_hmtc_args->_rtp_video;
call_type* _type = _hmtc_args->_local_type_call;
void* _core_handler = _hmtc_args->_core_handler;
call_state* _this_call = _hmtc_args->_this_call;
while ( *_this_call == call_active ) {
// THREADLOCK()
_audio_msg = rtp_recv_msg ( _rtp_audio );
_video_msg = rtp_recv_msg ( _rtp_video );
if ( _audio_msg ) {
/* Do whatever with msg */
puts("audio");
/* Do whatever with msg
puts(_audio_msg->_data);*/
rtp_free_msg ( _rtp_audio, _audio_msg );
}
if ( _video_msg ) {
/* Do whatever with msg */
puts("video");
/* Do whatever with msg
puts(_video_msg->_data); */
rtp_free_msg ( _rtp_video, _video_msg );
_video_msg = NULL;
}
/* -------------------- */
_audio_msg = rtp_msg_new ( _rtp_audio, (const uint8_t*)"audio\0", 6 ) ;
rtp_send_msg ( _rtp_audio, _audio_msg, _core_handler );
_audio_msg = NULL;
if ( *_type == type_video ){ /* if local call send video */
_video_msg = rtp_msg_new ( _rtp_video, (const uint8_t*)"video\0", 6 ) ;
rtp_send_msg ( _rtp_video, _video_msg, _core_handler );
_video_msg = NULL;
}
//THREADUNLOCK()
usleep ( 10000 );
/* -------------------- */
}
//THREADLOCK()
if ( _audio_msg ){
rtp_free_msg(_rtp_audio, _audio_msg);
}
if ( _video_msg ) {
rtp_free_msg(_rtp_video, _video_msg);
}
rtp_release_session_recv(_rtp_video);
rtp_release_session_recv(_rtp_audio);
rtp_terminate_session(_rtp_audio);
rtp_terminate_session(_rtp_video);
*_hmtc_args->_rtp_audio = NULL;
*_hmtc_args->_rtp_video = NULL;
free(_hmtc_args_p);
//THREADUNLOCK()
INFO("Media thread finished!");
pthread_exit ( NULL );
}
pthread_t phone_startmedia_loop ( phone_t* _phone )
{
if ( !_phone ){
return 0;
}
int _status;
uint8_t _prefix = RTP_PACKET;
pthread_t _rtp_tid;
int _rtp_thread_running = 1;
_phone->_rtp_audio = rtp_init_session ( -1, 1 );
rtp_set_prefix ( _phone->_rtp_audio, &_prefix, 1 );
rtp_add_receiver ( _phone->_rtp_audio, &_phone->_msi->_friend_id );
rtp_set_payload_type(_phone->_rtp_audio, _PAYLOAD_OPUS);
_phone->_rtp_video = rtp_init_session ( -1, 1 );
rtp_set_prefix ( _phone->_rtp_video, &_prefix, 1 );
rtp_add_receiver ( _phone->_rtp_video, &_phone->_msi->_friend_id );
rtp_set_payload_type(_phone->_rtp_video, _PAYLOAD_VP8);
hmtc_args_t* rtp_targs = calloc(sizeof(hmtc_args_t),1);
rtp_targs->_rtp_audio = &_phone->_rtp_audio;
rtp_targs->_rtp_video = &_phone->_rtp_video;
rtp_targs->_local_type_call = &_phone->_msi->_call->_type_local;
rtp_targs->_this_call = &_phone->_msi->_call->_state;
rtp_targs->_core_handler = _phone->_networking;
codec_state *cs;
cs=_phone->cs;
//_status = pthread_create ( &_rtp_tid, NULL, phone_handle_media_transport_poll, rtp_targs );
cs->_rtp_audio=_phone->_rtp_audio;
cs->_rtp_video=_phone->_rtp_video;
cs->_networking=_phone->_networking;
cs->socket=_phone->_tox_sock;
cs->quit = 0;
printf("support: %d %d\n",cs->support_send_audio,cs->support_send_video);
if(cs->support_send_audio&&cs->support_send_video) /* quick fix */
pthread_create(&_phone->cs->encode_audio_thread, NULL, encode_audio_thread, _phone->cs);
if(cs->support_receive_audio)
pthread_create(&_phone->cs->decode_audio_thread, NULL, decode_audio_thread, _phone->cs);
if(cs->support_send_video)
pthread_create(&_phone->cs->encode_video_thread, NULL, encode_video_thread, _phone->cs);
if(cs->support_receive_video)
pthread_create(&_phone->cs->decode_video_thread, NULL, decode_video_thread, _phone->cs);
//
return 1;
}
/* Some example callbacks */
MCBTYPE callback_recv_invite ( MCBARGS )
{
const char* _call_type;
msi_session_t* _msi = _arg;
/* Get the last one */
call_type _type = _msi->_call->_type_peer[_msi->_call->_participants - 1];
switch ( _type ){
case type_audio:
_call_type = "audio";
break;
case type_video:
_call_type = "video";
break;
}
INFO( "Incoming %s call!", _call_type );
}
MCBTYPE callback_recv_trying ( MCBARGS )
{
INFO ( "Trying...");
}
MCBTYPE callback_recv_ringing ( MCBARGS )
{
INFO ( "Ringing!" );
}
MCBTYPE callback_recv_starting ( MCBARGS )
{
msi_session_t* _session = _arg;
if ( !phone_startmedia_loop(_session->_agent_handler) ){
INFO("Starting call failed!");
} else {
INFO ("Call started! ( press h to hangup )");
}
}
MCBTYPE callback_recv_ending ( MCBARGS )
{
msi_session_t* _session = _arg;
phone_t * _phone = _session->_agent_handler;
_phone->cs->quit=1;
if(_phone->cs->encode_video_thread)
pthread_join(_phone->cs->encode_video_thread,NULL);
if(_phone->cs->encode_audio_thread)
pthread_join(_phone->cs->encode_audio_thread,NULL);
if(_phone->cs->decode_audio_thread)
pthread_join(_phone->cs->decode_audio_thread,NULL);
if(_phone->cs->decode_video_thread)
pthread_join(_phone->cs->decode_video_thread,NULL);
SDL_Quit();
printf("all A/V threads successfully shut down\n");
INFO ( "Call ended!" );
}
MCBTYPE callback_recv_error ( MCBARGS )
{
msi_session_t* _session = _arg;
INFO( "Error: %s", _session->_last_error_str );
}
MCBTYPE callback_call_started ( MCBARGS )
{
msi_session_t* _session = _arg;
if ( !phone_startmedia_loop(_session->_agent_handler) ){
INFO("Starting call failed!");
} else {
INFO ("Call started! ( press h to hangup )");
}
}
MCBTYPE callback_call_canceled ( MCBARGS )
{
INFO ( "Call canceled!" );
}
MCBTYPE callback_call_rejected ( MCBARGS )
{
INFO ( "Call rejected!\n" );
}
MCBTYPE callback_call_ended ( MCBARGS )
{
msi_session_t* _session = _arg;
phone_t * _phone = _session->_agent_handler;
_phone->cs->quit=1;
if(_phone->cs->encode_video_thread)
pthread_join(_phone->cs->encode_video_thread,NULL);
if(_phone->cs->encode_audio_thread)
pthread_join(_phone->cs->encode_audio_thread,NULL);
if(_phone->cs->decode_audio_thread)
pthread_join(_phone->cs->decode_audio_thread,NULL);
if(_phone->cs->decode_video_thread)
pthread_join(_phone->cs->decode_video_thread,NULL);
SDL_Quit();
printf("all A/V threads successfully shut down\n");
INFO ( "Call ended!" );
}
MCBTYPE callback_requ_timeout ( MCBARGS )
{
INFO( "No answer! " );
}
phone_t* initPhone(uint16_t _listen_port, uint16_t _send_port)
{
phone_t* _retu = calloc(sizeof(phone_t),1);
_retu->cs = av_calloc(sizeof(codec_state),1);
/* Initialize our mutex */
pthread_mutex_init ( &_mutex, NULL );
IP_Port _local;
ip_init(&_local.ip, 0);
_local.ip.ip4.uint32 = htonl ( INADDR_ANY );
/* Bind local receive port to any address */
_retu->_networking = new_networking ( _local.ip, _listen_port );
if ( !_retu->_networking ) {
fprintf ( stderr, "new_networking() failed!\n" );
return NULL;
}
_retu->_send_port = _send_port;
_retu->_recv_port = _listen_port;
_retu->_tox_sock = _retu->_networking->sock;
_retu->_rtp_audio = NULL;
_retu->_rtp_video = NULL;
/* Initialize msi */
_retu->_msi = msi_init_session ( _retu->_networking, (const uint8_t*)_USERAGENT );
if ( !_retu->_msi ) {
fprintf ( stderr, "msi_init_session() failed\n" );
return NULL;
}
/* Initiate codecs */
init_encoder(_retu->cs);
init_decoder(_retu->cs);
_retu->_msi->_agent_handler = _retu;
/* Initiate callbacks */
msi_register_callback_send ( sendpacket ); /* Using core's send */
msi_register_callback_call_started ( callback_call_started );
msi_register_callback_call_canceled ( callback_call_canceled );
msi_register_callback_call_rejected ( callback_call_rejected );
msi_register_callback_call_ended ( callback_call_ended );
msi_register_callback_recv_invite ( callback_recv_invite );
msi_register_callback_recv_ringing ( callback_recv_ringing );
msi_register_callback_recv_starting ( callback_recv_starting );
msi_register_callback_recv_ending ( callback_recv_ending );
msi_register_callback_recv_error(callback_recv_error);
msi_register_callback_requ_timeout ( callback_requ_timeout );
/* ------------------ */
/* Now start msi main loop. It's a must!
* Define a frequency in ms; 10 ms is just fine
*/
msi_start_main_loop ( _retu->_msi, 10 );
return _retu;
}
pthread_t phone_startmain_loop(phone_t* _phone)
{
int _status;
/* Start receive thread */
pthread_t _recv_thread, _phone_loop_thread;
_status = pthread_create ( &_recv_thread, NULL, phone_receivepacket, _phone );
if ( _status < 0 ) {
printf ( "Error while starting handle call: %d, %s\n", errno, strerror ( errno ) );
return 0;
}
_status = pthread_detach ( _recv_thread );
if ( _status < 0 ) {
printf ( "Error while starting handle call: %d, %s\n", errno, strerror ( errno ) );
return 0;
}
_status = pthread_create ( &_phone_loop_thread, NULL, phone_poll, _phone );
if ( _status < 0 ) {
printf ( "Error while starting main phone loop: %d, %s\n", errno, strerror ( errno ) );
return 0;
}
_status = pthread_join ( _phone_loop_thread, NULL );
if ( _status < 0 ) {
printf ( "Error while starting main phone loop: %d, %s\n", errno, strerror ( errno ) );
return 0;
}
return _phone_loop_thread;
}
void* phone_poll ( void* _p_phone )
{
phone_t* _phone = _p_phone;
int _status = SUCCESS;
char _line[100];
size_t _len;
char _dest[17]; /* For parsing destination ip */
memset(_dest, '\0', 17);
INFO("Welcome to tox_phone version: " _USERAGENT "\n"
"Usage: \n"
"c [a/v] (type) [0.0.0.0] (dest ip) (calls dest ip)\n"
"h (if call is active hang up)\n"
"a [a/v] (answer incoming call: a - audio / v - audio + video (audio is default))\n"
"r (reject incoming call)\n"
"q (quit)\n"
"================================================================================"
);
while ( 1 )
{
fgets(_line, sizeof(_line), stdin);
int i;
for (i = 0; i < 100; i++) {
if (_line[i] == '\n') {
_line[i] = '\0';
}
}
_len = strlen(_line);
if ( !_len ){
printf(" >> "); fflush(stdout);
continue;
}
if ( _len > 1 && _line[1] != ' ' && _line[1] != '\n' ){
INFO("Invalid input!");
continue;
}
switch (_line[0]){
case 'c':
{
if ( _phone->_msi->_call ){
INFO("Already in a call");
break;
}
call_type _ctype;
if ( _len < 11 ){
INFO("Invalid input; usage: c a/v 0.0.0.0");
break;
}
else if ( _line[2] == 'a' || _line[2] != 'v' ){ /* default and audio */
_ctype = type_audio;
}
else { /* video */
_ctype = type_video;
}
strcpy(_dest, _line + 4 );
_status = t_setipport(_dest, _phone->_send_port, &(_phone->_msi->_friend_id));
if ( _status < 0 ){
INFO("Could not resolve address!");
} else {
/* Set timeout */
msi_invite ( _phone->_msi, _ctype, 30 * 1000 );
INFO("Calling!");
}
t_memset((uint8_t*)_dest, '\0', 17);
} break;
case 'h':
{
if ( !_phone->_msi->_call ){
break;
}
msi_hangup(_phone->_msi);
INFO("Hung up...");
} break;
case 'a':
{
if ( _phone->_msi->_call && _phone->_msi->_call->_state != call_starting ) {
break;
}
if ( _len > 1 && _line[2] == 'v' )
msi_answer(_phone->_msi, type_video);
else
msi_answer(_phone->_msi, type_audio);
} break;
case 'r':
{
if ( _phone->_msi->_call && _phone->_msi->_call->_state != call_starting ){
break;
}
msi_reject(_phone->_msi);
INFO("Call Rejected...");
} break;
case 'q':
{
INFO("Quitting!");
pthread_exit(NULL);
}
default:
{
INFO("Invalid command!");
} break;
}
usleep(1000);
}
pthread_exit(NULL);
}
int quitPhone(phone_t* _phone)
{
if ( _phone->_msi->_call ){
msi_hangup(_phone->_msi); /* Hangup the phone first */
}
msi_terminate_session(_phone->_msi);
pthread_mutex_destroy ( &_mutex );
printf("\r[i] Quit!\n");
return SUCCESS;
}
/* ---------------------- */
int print_help ( const char* _name )
{
printf ( "Usage: %s -m (mode) -r/s ( for setting the ports on test version )\n", _name );
return FAILURE;
}
int main ( int argc, char* argv [] )
{
arg_t* _args = parse_args ( argc, argv );
const char* _mode = find_arg_duble ( _args, "-m" );
uint16_t _listen_port;
uint16_t _send_port;
if ( !_mode )
return print_help ( argv[0] );
if ( _mode[0] == 'r' ) {
_send_port = 31000;
_listen_port = 31001;
} else if ( _mode[0] == 's' ) {
_send_port = 31001;
_listen_port = 31000;
} else return print_help ( argv[0] );
phone_t* _phone = initPhone(_listen_port, _send_port);
if ( _phone ){
phone_startmain_loop(_phone);
quitPhone(_phone);
}
return SUCCESS;
}
#endif /* _CT_PHONE */