diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 8e2be25e..a73ddc91 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -37,14 +37,14 @@ libtoxav_la_LIBADD = libtoxcore.la \ -noinst_PROGRAMS += av_test +#noinst_PROGRAMS += av_test -av_test_SOURCES = ../toxav/av_test.c +#av_test_SOURCES = ../toxav/av_test.c -av_test_CFLAGS = $(LIBSODIUM_CFLAGS) \ +#av_test_CFLAGS = $(LIBSODIUM_CFLAGS) \ $(NACL_CFLAGS) -av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ +#av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ $(NACL_LDFLAGS) \ libtoxav.la \ libtoxcore.la \ diff --git a/toxav/av_test.c b/toxav/av_test.c index f0ad6a01..b7260d7d 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -1,4 +1,4 @@ -#include "toxav.h" +#include "../toxav/toxav.h" #include "../toxcore/tox.h" /* For playing audio data */ @@ -11,9 +11,7 @@ /* For reading and displaying video data */ #include #include - -/* For converting images TODO remove */ -#include +#include #include @@ -29,6 +27,23 @@ #define c_sleep(x) usleep(1000*x) +#define CLIP(X) ( (X) > 255 ? 255 : (X) < 0 ? 0 : X) + +// RGB -> YUV +#define RGB2Y(R, G, B) CLIP(( ( 66 * (R) + 129 * (G) + 25 * (B) + 128) >> 8) + 16) +#define RGB2U(R, G, B) CLIP(( ( -38 * (R) - 74 * (G) + 112 * (B) + 128) >> 8) + 128) +#define RGB2V(R, G, B) CLIP(( ( 112 * (R) - 94 * (G) - 18 * (B) + 128) >> 8) + 128) + +// YUV -> RGB +#define C(Y) ( (Y) - 16 ) +#define D(U) ( (U) - 128 ) +#define E(V) ( (V) - 128 ) + +#define YUV2R(Y, U, V) CLIP(( 298 * C(Y) + 409 * E(V) + 128) >> 8) +#define YUV2G(Y, U, V) CLIP(( 298 * C(Y) - 100 * D(U) - 208 * E(V) + 128) >> 8) +#define YUV2B(Y, U, V) CLIP(( 298 * C(Y) + 516 * D(U) + 128) >> 8) + + /* Enable/disable tests */ #define TEST_REGULAR_AV 0 #define TEST_REGULAR_A 0 @@ -36,8 +51,8 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 1 -#define TEST_TRANSFER_V 0 +#define TEST_TRANSFER_A 0 +#define TEST_TRANSFER_V 1 typedef struct { @@ -83,38 +98,32 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, uint8_t const *planes[], int32_t const stride[], void *user_data) { - IplImage output_img; - const int bpl = stride[VPX_PLANE_Y]; - const int cxbpl = stride[VPX_PLANE_V]; + uint16_t *img_data = malloc(height * width * 6); - output_img.imageData = malloc(width * height * 3); - output_img.imageSize = width * height * 3; - output_img.width = width; - output_img.height = height; - - /* FIXME: possible bug? */ - const uint8_t* yData = planes[VPX_PLANE_Y]; - const uint8_t* uData = planes[VPX_PLANE_V]; - const uint8_t* vData = planes[VPX_PLANE_U]; - - // convert from planar to packed - int y = 0; - for (; y < height; ++y) - { - int x = 0; - for (; x < width; ++x) - { - uint8_t Y = planes[VPX_PLANE_Y][x + y * bpl]; - uint8_t U = planes[VPX_PLANE_V][x/(1 << 1) + y/(1 << 1)*cxbpl]; - uint8_t V = planes[VPX_PLANE_U][x/(1 << 1) + y/(1 << 1)*cxbpl]; - output_img.imageData[width * 3 * y + x * 3 + 0] = Y; - output_img.imageData[width * 3 * y + x * 3 + 1] = U; - output_img.imageData[width * 3 * y + x * 3 + 2] = V; + unsigned long int i, j; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; ++j) { + uint8_t *point = (void*)img_data + 3 * ((i * width) + j); + int y = planes[0][(i * stride[0]) + j]; + int u = planes[1][((i / 2) * stride[1]) + (j / 2)]; + int v = planes[2][((i / 2) * stride[2]) + (j / 2)]; + + point[0] = YUV2R(y, u, v); + point[1] = YUV2G(y, u, v); + point[2] = YUV2B(y, u, v); } } - cvShowImage(vdout, &output_img); - free(output_img.imageData); + + CvMat mat = cvMat(height, width, CV_8UC3, img_data); + + CvSize sz = {.height = height, .width = width}; + + IplImage* header = cvCreateImageHeader(sz, 1, 3); + IplImage* img = cvGetImage(&mat, header); + cvShowImage(vdout, img); + cvWaitKey(1); + free(img_data); } void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, int16_t const *pcm, @@ -140,7 +149,7 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, return; - alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, + alBufferData(bufid, channels == 1 ? AL_FORMAT_MONO16 : AL_FORMAT_STEREO16, pcm, sample_count * 2, sampling_rate); alSourceQueueBuffers(adout, 1, &bufid); @@ -244,45 +253,42 @@ int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) } int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) -{ - /* I use vpx image coz i'm noob TODO use opencv conversion */ - vpx_image_t vpx_img; - vpx_img.w = vpx_img.h = vpx_img.d_w = vpx_img.d_h = 0; +{ + int32_t strides[3] = { 1280, 640, 640 }; + uint8_t* planes[3] = { + malloc(img->height * img->width), + malloc(img->height * img->width / 2), + malloc(img->height * img->width / 2), + }; - const int w = img->width; - const int h = img->height; + int x_chroma_shift = 1; + int y_chroma_shift = 1; - vpx_img_alloc(&vpx_img, VPX_IMG_FMT_VPXI420, w, h, 1); - - int y = 0; - for (; y < h; ++y) - { - int x = 0; - for (; x < w; ++x) - { - uint8_t b = img->imageData[(x + y * w) * 3 + 0]; - uint8_t g = img->imageData[(x + y * w) * 3 + 1]; - uint8_t r = img->imageData[(x + y * w) * 3 + 2]; + int x, y; + for (y = 0; y < img->height; ++y) { + for (x = 0; x < img->width; ++x) { + uint8_t r = img->imageData[(x + y * img->width) * 3 + 0]; + uint8_t g = img->imageData[(x + y * img->width) * 3 + 1]; + uint8_t b = img->imageData[(x + y * img->width) * 3 + 2]; - vpx_img.planes[VPX_PLANE_Y][x + y * vpx_img.stride[VPX_PLANE_Y]] = ((66 * r + 129 * g + 25 * b) >> 8) + 16; - - if (!(x % (1 << vpx_img.x_chroma_shift)) && !(y % (1 << vpx_img.y_chroma_shift))) - { - const int i = x / (1 << vpx_img.x_chroma_shift); - const int j = y / (1 << vpx_img.y_chroma_shift); - vpx_img.planes[VPX_PLANE_U][i + j * vpx_img.stride[VPX_PLANE_U]] = ((112 * r + -94 * g + -18 * b) >> 8) + 128; - vpx_img.planes[VPX_PLANE_V][i + j * vpx_img.stride[VPX_PLANE_V]] = ((-38 * r + -74 * g + 112 * b) >> 8) + 128; + planes[0][x + y * strides[0]] = RGB2Y(r, g, b); + if (!(x % (1 << x_chroma_shift)) && !(y % (1 << y_chroma_shift))) { + const int i = x / (1 << x_chroma_shift); + const int j = y / (1 << y_chroma_shift); + planes[1][i + j * strides[1]] = RGB2U(r, g, b); + planes[2][i + j * strides[2]] = RGB2V(r, g, b); } } } - int rc = toxav_send_video_frame(av, friend_number, w, h, - vpx_img.planes[VPX_PLANE_Y], - vpx_img.planes[VPX_PLANE_U], - vpx_img.planes[VPX_PLANE_V], NULL); - vpx_img_free(&vpx_img); - return rc; +// int rc = toxav_send_video_frame(av, friend_number, img->width, img->height, planes[0], planes[1], planes[2], NULL); + t_toxav_receive_video_frame_cb(av, 0, img->width, img->height, (const uint8_t**) planes, strides, NULL); + free(planes[0]); + free(planes[1]); + free(planes[2]); +// return rc; + return 0; } ALCdevice* open_audio_device(const char* audio_out_dev_name) @@ -403,6 +409,32 @@ int main (int argc, char** argv) } } + + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); + + CvCapture* capture = cvCreateFileCapture(vf_name); + if (!capture) { + printf("Failed to open video file: %s\n", vf_name); + exit(1); + } + + IplImage* frame; + time_t start_time = time(NULL); + + int frame_rate = cvGetCaptureProperty(capture, CV_CAP_PROP_FPS); + while(start_time + 4 > time(NULL)) { + frame = cvQueryFrame( capture ); + if (!frame) + break; + + send_opencv_img(NULL, 0, frame); + c_sleep(1); + } + + cvReleaseCapture(&capture); + cvDestroyWindow(vdout); + + return 0; const char* audio_out_dev_name = NULL; int i = 0; @@ -706,19 +738,19 @@ int main (int argc, char** argv) time_t expected_time = af_info.frames / af_info.samplerate + 2; while ( start_time + expected_time > time(NULL) ) { - int frame_size = (af_info.samplerate * frame_duration / 1000); + int frame_size = (af_info.samplerate * frame_duration / 1000) * af_info.channels; int64_t count = sf_read_short(af_handle, PCM, frame_size); if (count > 0) { - t_toxav_receive_audio_frame_cb(BobAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); -// TOXAV_ERR_SEND_FRAME rc; -// if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { -// printf("Error sending frame of size %ld: %d\n", count, rc); -// exit(1); -// } +// t_toxav_receive_audio_frame_cb(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL); + TOXAV_ERR_SEND_FRAME rc; + if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { + printf("Error sending frame of size %ld: %d\n", count, rc); + exit(1); + } } - c_sleep(frame_duration); -// iterate_tox(bootstrap, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); +// c_sleep(frame_duration); } @@ -744,6 +776,36 @@ int main (int argc, char** argv) } if (TEST_TRANSFER_V) { + printf("\nTrying video enc/dec...\n"); + + memset(&AliceCC, 0, sizeof(CallControl)); + memset(&BobCC, 0, sizeof(CallControl)); + + { /* Call */ + TOXAV_ERR_CALL rc; + toxav_call(AliceAV, 0, 0, 500000, &rc); + + if (rc != TOXAV_ERR_CALL_OK) { + printf("toxav_call failed: %d\n", rc); + exit(1); + } + } + + while (!BobCC.incoming) + iterate_tox(bootstrap, AliceAV, BobAV); + + { /* Answer */ + TOXAV_ERR_ANSWER rc; + toxav_answer(BobAV, 0, 0, 500000, &rc); + + if (rc != TOXAV_ERR_ANSWER_OK) { + printf("toxav_answer failed: %d\n", rc); + exit(1); + } + } + + iterate_tox(bootstrap, AliceAV, BobAV); + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); CvCapture* capture = cvCreateFileCapture(vf_name); @@ -755,15 +817,35 @@ int main (int argc, char** argv) IplImage* frame; time_t start_time = time(NULL); + int frame_rate = cvGetCaptureProperty(capture, CV_CAP_PROP_FPS); while(start_time + 10 > time(NULL)) { frame = cvQueryFrame( capture ); if (!frame) break; +// cvShowImage(vdout, frame); +// cvWaitKey(frame_rate); + send_opencv_img(AliceAV, 0, frame); + iterate_tox(bootstrap, AliceAV, BobAV); } cvReleaseCapture(&capture); cvDestroyWindow(vdout); + + { /* Hangup */ + TOXAV_ERR_CALL_CONTROL rc; + toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); + + if (rc != TOXAV_ERR_CALL_CONTROL_OK) { + printf("toxav_call_control failed: %d\n", rc); + exit(1); + } + } + + iterate_tox(bootstrap, AliceAV, BobAV); + assert(BobCC.state == TOXAV_CALL_STATE_END); + + printf("Success!"); } diff --git a/toxav/codec.c b/toxav/codec.c index b0f2ed31..4ae5a10b 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -337,12 +337,14 @@ void cs_do(CSession *cs) pthread_mutex_unlock(cs->queue_mutex); if (success == 2) { + LOGGER_DEBUG("OPUS correction"); rc = opus_decode(cs->audio_decoder, NULL, 0, tmp, - cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000, 1); + (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) * + cs->last_packet_channel_count, 1); } else { /* Get values from packet and decode. - * It also checks for validity of an opus packet - */ + * It also checks for validity of an opus packet + */ rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; @@ -351,6 +353,9 @@ void cs_do(CSession *cs) cs->last_packet_frame_duration = ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) / cs->last_packet_sampling_rate; + + /* TODO FIXME WARNING calculate properly according to propper channel count */ + cs->last_packet_frame_duration /= cs->last_packet_channel_count; } else { LOGGER_WARNING("Failed to load packet values!"); rtp_free_msg(NULL, msg); @@ -433,7 +438,7 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) */ int status; - cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be mono */ + cs->audio_decoder = opus_decoder_create(48000, 1, &status ); /* NOTE: Must be mono */ if ( status != OPUS_OK ) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); @@ -472,6 +477,10 @@ CSession *cs_new(uint32_t peer_video_frame_piece_size) vpx_codec_destroy(cs->v_decoder); goto AUDIO_DECODER_CLEANUP; } + + if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) ) + goto FAILURE; + /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/ /* Initialize encoders with default values */ diff --git a/toxav/toxav.c b/toxav/toxav.c index 62fed33f..9febd3a9 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -700,9 +700,8 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc goto END; } - LOGGER_DEBUG("Sending audio frame size: %d; channels: %d; srate: %d", sample_count, channels, sampling_rate); - uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; + uint8_t dest[sample_count * channels * sizeof(int16_t)]; int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); if (vrc < 0) {