From e65efc8936ee029f735a22105843335c18026ad2 Mon Sep 17 00:00:00 2001 From: mannol Date: Sat, 28 Mar 2015 03:36:31 +0100 Subject: Improvement in test and removed some bloat from CS --- toxav/Makefile.inc | 1 + toxav/av_test.c | 602 ++++++++++++++++++++++++----------------------------- toxav/codec.c | 187 ++++------------- toxav/codec.h | 16 +- toxav/toxav.c | 11 +- 5 files changed, 327 insertions(+), 490 deletions(-) (limited to 'toxav') diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc index 1dbabbcd..8e2be25e 100644 --- a/toxav/Makefile.inc +++ b/toxav/Makefile.inc @@ -52,6 +52,7 @@ av_test_LDADD = $(LIBSODIUM_LDFLAGS) \ $(NACL_OBJECTS) \ -lopenal \ -lopencv_calib3d -lopencv_contrib -lopencv_core -lopencv_features2d -lopencv_flann -lopencv_gpu -lopencv_highgui -lopencv_imgproc -lopencv_legacy -lopencv_ml -lopencv_objdetect -lopencv_ocl -lopencv_photo -lopencv_stitching -lopencv_superres -lopencv_ts -lopencv_video -lopencv_videostab \ + -lsndfile \ $(NACL_LIBS) diff --git a/toxav/av_test.c b/toxav/av_test.c index 249e0ab7..5ca53eb9 100644 --- a/toxav/av_test.c +++ b/toxav/av_test.c @@ -1,35 +1,32 @@ #include "toxav.h" #include "../toxcore/tox.h" -#ifdef __APPLE__ -# include -# include -#else -# include -# include -/* compatibility with older versions of OpenAL */ -# ifndef ALC_ALL_DEVICES_SPECIFIER -# include -# endif /* ALC_ALL_DEVICES_SPECIFIER */ -#endif /* __APPLE__ */ +/* For playing audio data */ +#include +#include +/* Processing wav's */ +#include + +/* For reading and displaying video data */ #include #include +/* For converting images TODO remove */ #include + +#include #include #include #include #include #include - -#if defined(_WIN32) || defined(__WIN32__) || defined (WIN32) -#define c_sleep(x) Sleep(1*x) -#else +#include #include + + #define c_sleep(x) usleep(1000*x) -#endif /* Enable/disable tests */ @@ -39,17 +36,17 @@ #define TEST_REJECT 0 #define TEST_CANCEL 0 #define TEST_MUTE_UNMUTE 0 -#define TEST_TRANSFER_A 0 -#define TEST_TRANSFER_V 1 +#define TEST_TRANSFER_A 1 +#define TEST_TRANSFER_V 0 typedef struct { bool incoming; uint32_t state; - uint32_t output_source; } CallControl; -const char* video_test_window = "AV Test"; +const char* vdout = "AV Test"; +uint32_t adout; const char* stringify_state(TOXAV_CALL_STATE s) { @@ -65,10 +62,7 @@ const char* stringify_state(TOXAV_CALL_STATE s) }; return strings [s]; -}; - - -int device_play_frame(uint32_t source, const int16_t* PCM, size_t size); +} /** * Callbacks @@ -104,9 +98,11 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, const uint8_t* vData = planes[VPX_PLANE_U]; // convert from planar to packed - for (int y = 0; y < height; ++y) + int y = 0; + for (; y < height; ++y) { - for (int x = 0; x < width; ++x) + int x = 0; + for (; x < width; ++x) { uint8_t Y = planes[VPX_PLANE_Y][x + y * bpl]; uint8_t U = planes[VPX_PLANE_V][x/(1 << 1) + y/(1 << 1)*cxbpl]; @@ -117,7 +113,7 @@ void t_toxav_receive_video_frame_cb(ToxAV *av, uint32_t friend_number, } } - cvShowImage(video_test_window, &output_img); + cvShowImage(vdout, &output_img); free(output_img.imageData); } void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, @@ -127,7 +123,32 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number, uint32_t sampling_rate, void *user_data) { - device_play_frame(((CallControl*)user_data)->output_source, pcm, sample_count); + uint32_t bufid; + int32_t processed, queued; + alGetSourcei(adout, AL_BUFFERS_PROCESSED, &processed); + alGetSourcei(adout, AL_BUFFERS_QUEUED, &queued); + + if(processed) { + uint32_t bufids[processed]; + alSourceUnqueueBuffers(adout, processed, bufids); + alDeleteBuffers(processed - 1, bufids + 1); + bufid = bufids[0]; + } +// else if(queued < 16) + alGenBuffers(1, &bufid); +// else +// return; + + + alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, + pcm, sample_count * channels * 2, sampling_rate); + alSourceQueueBuffers(adout, 1, &bufid); + + int32_t state; + alGetSourcei(adout, AL_SOURCE_STATE, &state); + + if(state != AL_PLAYING) + alSourcePlay(adout); } void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t *data, uint16_t length, void *userdata) { @@ -139,8 +160,20 @@ void t_accept_friend_request_cb(Tox *m, const uint8_t *public_key, const uint8_t /** */ -void prepare(Tox* Bsn, Tox* Alice, Tox* Bob) +void initialize_tox(Tox** bootstrap, ToxAV** AliceAV, CallControl* AliceCC, ToxAV** BobAV, CallControl* BobCC) { + Tox* Alice; + Tox* Bob; + + *bootstrap = tox_new(0); + Alice = tox_new(0); + Bob = tox_new(0); + + assert(bootstrap && Alice && Bob); + + printf("Created 3 instances of Tox\n"); + + printf("Preparing network...\n"); long long unsigned int cur_time = time(NULL); uint32_t to_compare = 974536; @@ -154,11 +187,11 @@ void prepare(Tox* Bsn, Tox* Alice, Tox* Bob) uint8_t off = 1; while (1) { - tox_do(Bsn); + tox_do(*bootstrap); tox_do(Alice); tox_do(Bob); - if (tox_isconnected(Bsn) && tox_isconnected(Alice) && tox_isconnected(Bob) && off) { + if (tox_isconnected(*bootstrap) && tox_isconnected(Alice) && tox_isconnected(Bob) && off) { printf("Toxes are online, took %llu seconds\n", time(NULL) - cur_time); off = 0; } @@ -169,25 +202,32 @@ void prepare(Tox* Bsn, Tox* Alice, Tox* Bob) c_sleep(20); } - printf("All set after %llu seconds!\n", time(NULL) - cur_time); -} -void prepareAV(ToxAV* AliceAV, void* AliceUD, ToxAV* BobAV, void* BobUD) -{ + + TOXAV_ERR_NEW rc; + *AliceAV = toxav_new(Alice, &rc); + assert(rc == TOXAV_ERR_NEW_OK); + + *BobAV = toxav_new(Bob, &rc); + assert(rc == TOXAV_ERR_NEW_OK); + /* Alice */ - toxav_callback_call(AliceAV, t_toxav_call_cb, AliceUD); - toxav_callback_call_state(AliceAV, t_toxav_call_state_cb, AliceUD); - toxav_callback_receive_video_frame(AliceAV, t_toxav_receive_video_frame_cb, AliceUD); - toxav_callback_receive_audio_frame(AliceAV, t_toxav_receive_audio_frame_cb, AliceUD); + toxav_callback_call(*AliceAV, t_toxav_call_cb, AliceCC); + toxav_callback_call_state(*AliceAV, t_toxav_call_state_cb, AliceCC); + toxav_callback_receive_video_frame(*AliceAV, t_toxav_receive_video_frame_cb, AliceCC); + toxav_callback_receive_audio_frame(*AliceAV, t_toxav_receive_audio_frame_cb, AliceCC); /* Bob */ - toxav_callback_call(BobAV, t_toxav_call_cb, BobUD); - toxav_callback_call_state(BobAV, t_toxav_call_state_cb, BobUD); - toxav_callback_receive_video_frame(BobAV, t_toxav_receive_video_frame_cb, BobUD); - toxav_callback_receive_audio_frame(BobAV, t_toxav_receive_audio_frame_cb, BobUD); + toxav_callback_call(*BobAV, t_toxav_call_cb, BobCC); + toxav_callback_call_state(*BobAV, t_toxav_call_state_cb, BobCC); + toxav_callback_receive_video_frame(*BobAV, t_toxav_receive_video_frame_cb, BobCC); + toxav_callback_receive_audio_frame(*BobAV, t_toxav_receive_audio_frame_cb, BobCC); + + printf("Created 2 instances of ToxAV\n"); + printf("All set after %llu seconds!\n", time(NULL) - cur_time); } -void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) +int iterate_tox(Tox* bootstrap, ToxAV* AliceAV, ToxAV* BobAV) { - tox_do(Bsn); + tox_do(bootstrap); tox_do(toxav_get_tox(AliceAV)); tox_do(toxav_get_tox(BobAV)); @@ -197,114 +237,16 @@ void iterate(Tox* Bsn, ToxAV* AliceAV, ToxAV* BobAV) int mina = MIN(tox_do_interval(toxav_get_tox(AliceAV)), toxav_iteration_interval(AliceAV)); int minb = MIN(tox_do_interval(toxav_get_tox(BobAV)), toxav_iteration_interval(BobAV)); - c_sleep(MIN(mina, minb)); -} - -int device_read_frame(ALCdevice* device, int32_t frame_dur, int16_t* PCM, size_t max_size) -{ - int f_size = (8000 * frame_dur / 1000); - - if (max_size < f_size) - return -1; - - /* Don't block if not enough data */ - int32_t samples; - alcGetIntegerv(device, ALC_CAPTURE_SAMPLES, sizeof(int32_t), &samples); - if (samples < f_size) - return 0; - - alcCaptureSamples(device, PCM, f_size); - return f_size; -} - -int device_play_frame(uint32_t source, const int16_t* PCM, size_t size) -{ - uint32_t bufid; - int32_t processed, queued; - alGetSourcei(source, AL_BUFFERS_PROCESSED, &processed); - alGetSourcei(source, AL_BUFFERS_QUEUED, &queued); - - if(processed) { - uint32_t bufids[processed]; - alSourceUnqueueBuffers(source, processed, bufids); - alDeleteBuffers(processed - 1, bufids + 1); - bufid = bufids[0]; - } - else if(queued < 16) - alGenBuffers(1, &bufid); - else - return 0; + int rc = MIN(mina, minb); + c_sleep(rc); - - alBufferData(bufid, AL_FORMAT_STEREO16, PCM, size * 2 * 2, 48000); - alSourceQueueBuffers(source, 1, &bufid); - - int32_t state; - alGetSourcei(source, AL_SOURCE_STATE, &state); - - if(state != AL_PLAYING) - alSourcePlay(source); - return 1; -} - -int print_devices() -{ - const char* default_input; - const char* default_output; - - const char *device; - - printf("Default input device: %s\n", alcGetString(NULL, ALC_CAPTURE_DEFAULT_DEVICE_SPECIFIER)); - printf("Default output device: %s\n", alcGetString(NULL, ALC_DEFAULT_DEVICE_SPECIFIER)); - - printf("\n"); - - printf("Input devices:\n"); - - int i = 0; - for(device = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER); *device; - device += strlen( device ) + 1, ++i) { - printf("%d) %s\n", i, device); - } - - printf("\n"); - printf("Output devices:\n"); - - i = 0; - for(device = alcGetString(NULL, ALC_DEVICE_SPECIFIER); *device; - device += strlen( device ) + 1, ++i) { - printf("%d) %s\n", i, device); - } - - return 0; -} - -int print_help(const char* name, int rc) -{ - fprintf(stderr, "Usage: %s [-h] \n", name); - return rc; -} - -long get_device_idx(const char* arg) -{ - if (strcmp(arg, "-") == 0) - return -1; /* Default */ - - char *p; - long res = strtol(arg, &p, 10); - - if (*p) { - fprintf(stderr, "Invalid device!"); - exit(1); - } - - return res; + return rc; } int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) { /* I use vpx image coz i'm noob TODO use opencv conversion */ - vpx_image vpx_img; + vpx_image_t vpx_img; vpx_img.w = vpx_img.h = vpx_img.d_w = vpx_img.d_h = 0; const int w = img->width; @@ -312,9 +254,11 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) vpx_img_alloc(&vpx_img, VPX_IMG_FMT_VPXI420, w, h, 1); - for (int y = 0; y < h; ++y) + int y = 0; + for (; y < h; ++y) { - for (int x = 0; x < w; ++x) + int x = 0; + for (; x < w; ++x) { uint8_t b = img->imageData[(x + y * w) * 3 + 0]; uint8_t g = img->imageData[(x + y * w) * 3 + 1]; @@ -341,113 +285,118 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img) return rc; } -int main (int argc, char** argv) +int print_audio_devices() { - /* AV files for testing */ - const char* audio_in = ""; - const char* video_in = ""; - long audio_out_dev = 0; + const char *device; + + printf("Default output device: %s\n", alcGetString(NULL, ALC_DEFAULT_DEVICE_SPECIFIER)); + printf("Output devices:\n"); -AGAIN: - switch (getopt(argc, argv, "a:v:o:")) - { - case 'a': - audio_in = optarg; - goto AGAIN; - break; - case 'v': - video_in = optarg; - goto AGAIN; - break; - case 'o': - char *d; - audio_out_dev = strtol(optarg, &d, 10); - if (*d) { - fprintf(stderr, "Invalid value for argument: 'o'"); - return 1; - } - goto AGAIN; - break; - case '?': - return 1; - break; - case -1: - break; + int i = 0; + for(device = alcGetString(NULL, ALC_DEVICE_SPECIFIER); *device; + device += strlen( device ) + 1, ++i) { + printf("%d) %s\n", i, device); } - + return 0; +} + +int print_help (const char* name) +{ + printf("Usage: %s -[a:v:o:dh]\n" + "-a video input file\n" + "-v video input file\n" + "-o output audio device index\n" + "-d print output audio devices\n" + "-h print this help\n", name); return 0; +} + +int main (int argc, char** argv) +{ + struct stat st; - if (argc == 2) { - if (strcmp(argv[1], "-d") == 0 || strcmp(argv[1], "--devices") == 0) { - return print_devices(); - } - if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0) { - return print_help(argv[0], 0); - } - } - - if (argc != 3) { - fprintf(stderr, "Invalid input!\n"); - return print_help(argv[0], 1); - } - - int i; - - const char* in_device_name = ""; - const char* out_device_name = ""; - - { - long dev = get_device_idx(argv[1]); - if (dev == -1) - in_device_name = alcGetString(NULL, ALC_CAPTURE_DEFAULT_DEVICE_SPECIFIER); - else - { - const char* tmp; - i = -1; - for(tmp = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER); *tmp && i != dev; - tmp += strlen( tmp ) + 1, ++i) - in_device_name = tmp; - } - - printf("Input device: %s\n", in_device_name); - } - - { - long dev = get_device_idx(argv[1]); - if (dev == -1) - out_device_name = alcGetString(NULL, ALC_DEFAULT_DEVICE_SPECIFIER); - else - { - const char* tmp; - i = -1; - for(tmp = alcGetString(NULL, ALC_DEVICE_SPECIFIER); *tmp && i != dev; - tmp += strlen( tmp ) + 1, ++i) - out_device_name = tmp; - } - - printf("Output device: %s\n", out_device_name); - } - - ALCdevice* out_device; - ALCcontext* out_ctx; - uint32_t source; - uint32_t buffers[5]; + /* AV files for testing */ + const char* af_name = NULL; + const char* vf_name = NULL; + long audio_out_dev_idx = 0; + + /* Pasre settings */ + CHECK_ARG: switch (getopt(argc, argv, "a:v:o:dh")) { + case 'a': + af_name = optarg; + goto CHECK_ARG; + case 'v': + vf_name = optarg; + goto CHECK_ARG; + case 'o': { + char *d; + audio_out_dev_idx = strtol(optarg, &d, 10); + if (*d) { + printf("Invalid value for argument: 'o'"); + exit(1); + } + goto CHECK_ARG; + } + case 'd': + return print_audio_devices(); + case 'h': + return print_help(argv[0]); + case '?': + exit(1); + case -1:; + } + + { /* Check files */ + if (!af_name) { + printf("Required audio input file!\n"); + exit(1); + } + + if (!vf_name) { + printf("Required video input file!\n"); + exit(1); + } + + /* Check for files */ + if(stat(af_name, &st) != 0 || !S_ISREG(st.st_mode)) + { + printf("%s doesn't seem to be a regular file!\n", af_name); + exit(1); + } + + if(stat(vf_name, &st) != 0 || !S_ISREG(st.st_mode)) + { + printf("%s doesn't seem to be a regular file!\n", vf_name); + exit(1); + } + } + + ALCdevice* audio_out_device; { /* Open output device */ - out_device = alcOpenDevice(out_device_name); - if ( !out_device ) { - fprintf(stderr, "Failed to open playback device: %s: %d\n", out_device_name, alGetError()); - return 1; + const char* audio_out_dev_name = NULL; + + int i = 0; + for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; + audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i) + if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1)) + break; + + audio_out_device = alcOpenDevice(audio_out_dev_name); + if ( !audio_out_device ) { + printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError()); + exit(1); } - out_ctx = alcCreateContext(out_device, NULL); + ALCcontext* out_ctx = alcCreateContext(audio_out_device, NULL); alcMakeContextCurrent(out_ctx); + uint32_t buffers[5]; alGenBuffers(5, buffers); - alGenSources((uint32_t)1, &source); - alSourcei(source, AL_LOOPING, AL_FALSE); + alGenSources((uint32_t)1, &adout); + alSourcei(adout, AL_LOOPING, AL_FALSE); uint16_t zeros[10000]; memset(zeros, 0, 10000); @@ -455,45 +404,31 @@ AGAIN: for ( i = 0; i < 5; ++i ) alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, 10000, 48000); - alSourceQueueBuffers(source, 5, buffers); - alSourcePlay(source); + alSourceQueueBuffers(adout, 5, buffers); + alSourcePlay(adout); + + printf("Using audio device: %s\n", audio_out_dev_name); } - ALCdevice* in_device; - - { /* Open input device */ - in_device = alcCaptureOpenDevice(in_device_name, 48000, AL_FORMAT_STEREO16, 10000); - if ( !in_device ) { - fprintf(stderr, "Failed to open capture device: %s: %d\n", in_device_name, alGetError()); - return 1; - } - - alcCaptureStart(in_device); - } + printf("Using audio file: %s\n", af_name); + printf("Using video file: %s\n", vf_name); + - Tox *Bsn = tox_new(0); - Tox *Alice = tox_new(0); - Tox *Bob = tox_new(0); - assert(Bsn && Alice && Bob); - prepare(Bsn, Alice, Bob); + /* START TOX NETWORK */ + + Tox *bootstrap; + ToxAV *AliceAV; + ToxAV *BobAV; + + CallControl AliceCC; + CallControl BobCC; + + initialize_tox(&bootstrap, &AliceAV, &AliceCC, &BobAV, &BobCC); - ToxAV *AliceAV, *BobAV; - CallControl AliceCC, BobCC; - { - TOXAV_ERR_NEW rc; - AliceAV = toxav_new(Alice, &rc); - assert(rc == TOXAV_ERR_NEW_OK); - - BobAV = toxav_new(Bob, &rc); - assert(rc == TOXAV_ERR_NEW_OK); - - prepareAV(AliceAV, &AliceCC, BobAV, &BobCC); - printf("Created 2 instances of ToxAV\n"); - } #define REGULAR_CALL_FLOW(A_BR, V_BR) \ @@ -539,7 +474,7 @@ AGAIN: } \ } \ \ - iterate(Bsn, AliceAV, BobAV); \ + iterate(bootstrap, AliceAV, BobAV); \ } \ printf("Success!\n");\ } while(0) @@ -578,7 +513,7 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* Reject */ { @@ -592,7 +527,7 @@ AGAIN: } while (AliceCC.state != TOXAV_CALL_STATE_END) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); printf("Success!\n"); } @@ -614,7 +549,7 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* Cancel */ { @@ -629,7 +564,7 @@ AGAIN: /* Alice will not receive end state */ while (BobCC.state != TOXAV_CALL_STATE_END) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); printf("Success!\n"); } @@ -652,7 +587,7 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* At first try all stuff while in invalid state */ assert(!toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); @@ -670,39 +605,39 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); /* Pause and Resume */ printf("Pause and Resume\n"); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_PAUSE, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_PAUSED); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_RESUME, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & (TOXAV_CALL_STATE_SENDING_A | TOXAV_CALL_STATE_SENDING_V)); /* Mute/Unmute single */ printf("Mute/Unmute single\n"); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); /* Mute/Unmute both */ printf("Mute/Unmute both\n"); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_A); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state ^ TOXAV_CALL_STATE_RECEIVING_V); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_AUDIO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_A); assert(toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_TOGGLE_MUTE_VIDEO, NULL)); - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state & TOXAV_CALL_STATE_RECEIVING_V); { @@ -715,21 +650,22 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_END); printf("Success!\n"); } if (TEST_TRANSFER_A) { /* Audio encoding/decoding and transfer */ + SNDFILE* af_handle; + SF_INFO af_info; + printf("\nTrying audio enc/dec...\n"); memset(&AliceCC, 0, sizeof(CallControl)); memset(&BobCC, 0, sizeof(CallControl)); - - AliceCC.output_source = BobCC.output_source = source; - { + { /* Call */ TOXAV_ERR_CALL rc; toxav_call(AliceAV, 0, 48, 0, &rc); @@ -740,11 +676,11 @@ AGAIN: } while (!BobCC.incoming) - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); - { + { /* Answer */ TOXAV_ERR_ANSWER rc; - toxav_answer(BobAV, 0, 48, 0, &rc); + toxav_answer(BobAV, 0, 64, 0, &rc); if (rc != TOXAV_ERR_ANSWER_OK) { printf("toxav_answer failed: %d\n", rc); @@ -752,26 +688,44 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); - - int16_t PCM[10000]; - time_t start_time = time(NULL); + iterate_tox(bootstrap, AliceAV, BobAV); + /* Open audio file */ + af_handle = sf_open(af_name, SFM_READ, &af_info); + if (af_handle == NULL) + { + printf("Failed to open the file.\n"); + exit(1); + } + /* Run for 5 seconds */ - while ( start_time + 10 > time(NULL) ) { - int frame_size = device_read_frame(in_device, 20, PCM, sizeof(PCM)); - if (frame_size > 0) { - TOXAV_ERR_SEND_FRAME rc; - if (toxav_send_audio_frame(AliceAV, 0, PCM, frame_size, 2, 8000, &rc) == false) { - printf("Error sending frame of size %d: %d\n", frame_size, rc); - exit (1); - } - } - - iterate(Bsn, AliceAV, BobAV); + + uint32_t frame_duration = 10; + int16_t PCM[10000]; + + time_t start_time = time(NULL); + time_t expected_time = af_info.frames / af_info.samplerate + 2; + + while ( start_time + expected_time > time(NULL) ) { + int frame_size = (af_info.samplerate * frame_duration / 1000); + + int64_t count = sf_read_short(af_handle, PCM, frame_size); + if (count > 0) { + TOXAV_ERR_SEND_FRAME rc; + if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { + printf("Error sending frame of size %ld: %d\n", count, rc); + exit(1); + } + } + + iterate_tox(bootstrap, AliceAV, BobAV); } + + printf("Played file in: %lu\n", time(NULL) - start_time); + + sf_close(af_handle); - { + { /* Hangup */ TOXAV_ERR_CALL_CONTROL rc; toxav_call_control(AliceAV, 0, TOXAV_CALL_CONTROL_CANCEL, &rc); @@ -781,24 +735,19 @@ AGAIN: } } - iterate(Bsn, AliceAV, BobAV); + iterate_tox(bootstrap, AliceAV, BobAV); assert(BobCC.state == TOXAV_CALL_STATE_END); printf("Success!"); } if (TEST_TRANSFER_V) { - if (strlen(video_in) == 0) { - printf("Skipping video test...\n"); - goto CONTINUE; - } + cvNamedWindow(vdout, CV_WINDOW_AUTOSIZE); - cvNamedWindow(video_test_window, CV_WINDOW_AUTOSIZE); - - CvCapture* capture = cvCreateFileCapture(video_in); + CvCapture* capture = cvCreateFileCapture(vf_name); if (!capture) { - printf("No file named: %s\n", video_in); - return 1; + printf("Failed to open video file: %s\n", vf_name); + exit(1); } IplImage* frame; @@ -812,21 +761,20 @@ AGAIN: } cvReleaseCapture(&capture); - cvDestroyWindow(video_test_window); - - CONTINUE:; + cvDestroyWindow(vdout); } + Tox* Alice = toxav_get_tox(AliceAV); + Tox* Bob = toxav_get_tox(BobAV); toxav_kill(BobAV); toxav_kill(AliceAV); tox_kill(Bob); tox_kill(Alice); - tox_kill(Bsn); + tox_kill(bootstrap); printf("\nTest successful!\n"); - alcCloseDevice(out_device); - alcCaptureCloseDevice(in_device); + alcCloseDevice(audio_out_device); return 0; } diff --git a/toxav/codec.c b/toxav/codec.c index a72b2764..b9cbbc06 100644 --- a/toxav/codec.c +++ b/toxav/codec.c @@ -38,7 +38,7 @@ #include "rtp.h" #include "codec.h" -#define DEFAULT_JBUF 6 +#define DEFAULT_JBUF 3 /* Good quality encode. */ #define MAX_DECODE_TIME_US 0 @@ -175,6 +175,8 @@ static int jbuf_write(JitterBuffer *q, RTPMessage *m) unsigned int num = sequnum % q->size; if ((uint32_t)(sequnum - q->bottom) > q->size) { + LOGGER_DEBUG("Clearing jitter: %p", q); + jbuf_clear(q); q->bottom = sequnum - q->capacity; q->queue[num] = m; @@ -193,7 +195,7 @@ static int jbuf_write(JitterBuffer *q, RTPMessage *m) return 0; } -/* Success is 0 when there is nothing to dequeue, +/* success is set to 0 when there is nothing to dequeue, * 1 when there's a good packet, * 2 when there's a lost packet */ static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success) @@ -237,62 +239,6 @@ static int convert_bw_to_sampling_rate(int bw) } -int cs_set_receiving_audio_bitrate(CSSession *cs, int32_t rate) -{ - if (cs->audio_decoder == NULL) - return -1; - - int rc = opus_decoder_ctl(cs->audio_decoder, OPUS_SET_BITRATE(rate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while setting decoder ctl: %s", opus_strerror(rc)); - return -1; - } - - LOGGER_DEBUG("Set new decoder bitrate to: %d", rate); - return 0; -} - -int cs_set_receiving_audio_sampling_rate(CSSession* cs, int32_t rate) -{ - /* TODO Find a better way? */ - if (cs->audio_decoder == NULL) - return -1; - - if (cs->decoder_sample_rate == rate) - return 0; - - int channels = cs->decoder_channels; - - cs_disable_audio_receiving(cs); - - cs->decoder_sample_rate = rate; - cs->decoder_channels = channels; - - LOGGER_DEBUG("Set new encoder sampling rate: %d", rate); - return cs_enable_audio_receiving(cs); -} - -int cs_set_receiving_audio_channels(CSSession* cs, int32_t count) -{ - /* TODO Find a better way? */ - if (cs->audio_decoder == NULL) - return -1; - - if (cs->decoder_channels == count) - return 0; - - int srate = cs->decoder_sample_rate; - cs_disable_audio_receiving(cs); - - cs->decoder_channels = count; - cs->decoder_sample_rate = srate; - - LOGGER_DEBUG("Set new encoder channel count: %d", count); - return cs_enable_audio_receiving(cs); -} - - /* PUBLIC */ void cs_do(CSSession *cs) @@ -310,17 +256,19 @@ void cs_do(CSSession *cs) pthread_mutex_lock(cs->queue_mutex); + /********************* AUDIO *********************/ if (cs->audio_decoder) { /* If receiving enabled */ RTPMessage *msg; - uint16_t fsize = 10000; /* Should be enough for all normal frequences */ - int16_t tmp[fsize * 2]; + /* The maximum for 120 ms 48 KHz audio */ + int16_t tmp[20000]; while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { pthread_mutex_unlock(cs->queue_mutex); if (success == 2) { - rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1); + rc = opus_decode(cs->audio_decoder, NULL, 0, tmp, + cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000, 1); } else { /* Get values from packet and decode. * It also checks for validity of an opus packet @@ -328,7 +276,7 @@ void cs_do(CSSession *cs) rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); if (rc != -1) { cs->last_packet_sampling_rate = rc; - cs->last_pack_channels = opus_packet_get_nb_channels(msg->data); + cs->last_packet_channels = opus_packet_get_nb_channels(msg->data); cs->last_packet_frame_duration = ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) @@ -339,11 +287,7 @@ void cs_do(CSSession *cs) continue; } - cs_set_receiving_audio_sampling_rate(cs, cs->last_packet_sampling_rate); - cs_set_receiving_audio_channels(cs, cs->last_pack_channels); - - LOGGER_DEBUG("Decoding packet of length: %d", msg->length); - rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0); + rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, sizeof(tmp), 0); rtp_free_msg(NULL, msg); } @@ -351,15 +295,28 @@ void cs_do(CSSession *cs) LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); } else if (cs->acb.first) { /* Play */ - LOGGER_DEBUG("Playing audio frame size: %d chans: %d srate: %d", rc, cs->last_pack_channels, cs->last_packet_sampling_rate); + + LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc, + cs->last_packet_channels, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); + + /* According to https://tools.ietf.org/html/rfc6716#section-2.1.2 + * Every encoder can encode both mono and stereo data so we must + * determine which format is selected. + */ + + if (cs->last_packet_channels == 2) { + /* The packet is encoded with stereo encoder */ + } + cs->acb.first(cs->agent, cs->friend_id, tmp, rc, - cs->last_pack_channels, cs->last_packet_sampling_rate, cs->acb.second); + cs->last_packet_channels, cs->last_packet_sampling_rate, cs->acb.second); } pthread_mutex_lock(cs->queue_mutex); } } + /********************* VIDEO *********************/ if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { /* Decode video */ buffer_read(cs->vbuf_raw, &p); @@ -411,8 +368,6 @@ CSSession *cs_new(uint32_t peer_video_frame_piece_size) cs->peer_video_frame_piece_size = peer_video_frame_piece_size; - cs->decoder_sample_rate = 48000; - cs->decoder_channels = 2; return cs; } @@ -538,6 +493,8 @@ int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate) return 0; } + + int cs_enable_video_sending(CSSession* cs, uint32_t bitrate) { if (cs->v_encoding) @@ -619,6 +576,8 @@ FAILURE: return -1; } + + void cs_disable_video_sending(CSSession* cs) { if (cs->v_encoding) { @@ -663,64 +622,13 @@ int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate) return 0; } -int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate) -{ - /* TODO Find a better way? */ - if (cs->audio_encoder == NULL) - return -1; - - if (cs->encoder_sample_rate == rate) - return 0; - - int rc = OPUS_OK; - int bitrate = 0; - int channels = cs->encoder_channels; - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_BITRATE(&bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - cs_disable_audio_sending(cs); - cs->encoder_sample_rate = rate; - - LOGGER_DEBUG("Set new encoder sampling rate: %d", rate); - return cs_enable_audio_sending(cs, bitrate, channels); -} -int cs_set_sending_audio_channels(CSSession* cs, int32_t count) -{ - /* TODO Find a better way? */ - if (cs->audio_encoder == NULL) - return -1; - - if (cs->encoder_channels == count) - return 0; - - int rc = OPUS_OK; - int bitrate = 0; - - rc = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_BITRATE(&bitrate)); - - if ( rc != OPUS_OK ) { - LOGGER_ERROR("Error while getting encoder ctl: %s", opus_strerror(rc)); - return -1; - } - - cs_disable_audio_sending(cs); - - LOGGER_DEBUG("Set new encoder channel count: %d", count); - return cs_enable_audio_sending(cs, bitrate, count); -} void cs_disable_audio_sending(CSSession* cs) { if ( cs->audio_encoder ) { opus_encoder_destroy(cs->audio_encoder); cs->audio_encoder = NULL; - cs->encoder_channels = 0; } } @@ -732,26 +640,22 @@ void cs_disable_audio_receiving(CSSession* cs) jbuf_free(cs->j_buf); cs->j_buf = NULL; - /* It's used for measuring iteration interval so this has to be some value. - * To avoid unecessary checking we set this to 500 - */ - cs->last_packet_frame_duration = 500; - cs->decoder_sample_rate = 48000; - cs->decoder_channels = 2; + /* These need to be set in order to properly + * do error correction with opus */ + cs->last_packet_frame_duration = 120; + cs->last_packet_sampling_rate = 48000; } } -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels) + + +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate) { if (cs->audio_encoder) return 0; - if (!cs->encoder_sample_rate) - cs->encoder_sample_rate = 48000; - cs->encoder_channels = channels; - int rc = OPUS_OK; - cs->audio_encoder = opus_encoder_create(cs->encoder_sample_rate, channels, OPUS_APPLICATION_AUDIO, &rc); + cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc); if ( rc != OPUS_OK ) { LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); @@ -785,7 +689,7 @@ int cs_enable_audio_receiving(CSSession* cs) return 0; int rc; - cs->audio_decoder = opus_decoder_create(cs->decoder_sample_rate, cs->decoder_channels, &rc ); + cs->audio_decoder = opus_decoder_create(48000, 2, &rc ); if ( rc != OPUS_OK ) { LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); @@ -800,10 +704,11 @@ int cs_enable_audio_receiving(CSSession* cs) return -1; } - /* It's used for measuring iteration interval so this has to be some value. - * To avoid unecessary checking we set this to 500 - */ - cs->last_packet_frame_duration = 500; + + /* These need to be set in order to properly + * do error correction with opus */ + cs->last_packet_frame_duration = 120; + cs->last_packet_sampling_rate = 48000; return 0; } @@ -813,8 +718,8 @@ int cs_enable_audio_receiving(CSSession* cs) /* Called from RTP */ void queue_message(RTPSession *session, RTPMessage *msg) { - /* This function is unregistered during call termination befor destroing - * Codec session so no need to check for validity of cs + /* This function is unregistered during call termination befor destroying + * Codec session so no need to check for validity of cs TODO properly check video cycle */ CSSession *cs = session->cs; diff --git a/toxav/codec.h b/toxav/codec.h index 5e015f39..526a80d5 100644 --- a/toxav/codec.h +++ b/toxav/codec.h @@ -107,25 +107,15 @@ typedef struct CSSession_s { /* audio encoding */ OpusEncoder *audio_encoder; - int32_t encoder_channels; - int32_t encoder_sample_rate; /* audio decoding */ OpusDecoder *audio_decoder; - int32_t decoder_channels; - int32_t decoder_sample_rate; - int32_t last_pack_channels; + int32_t last_packet_channels; int32_t last_packet_sampling_rate; int32_t last_packet_frame_duration; struct JitterBuffer_s *j_buf; - /* Voice activity detection */ - uint32_t EVAD_tolerance; /* In frames */ - uint32_t EVAD_tolerance_cr; - - - /* OTHER * * @@ -171,10 +161,8 @@ void cs_disable_video_receiving(CSSession* cs); * AUDIO HANDLING */ int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate); -int cs_set_sending_audio_sampling_rate(CSSession* cs, int32_t rate); -int cs_set_sending_audio_channels(CSSession* cs, int32_t count); -int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate, int channels); +int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate); int cs_enable_audio_receiving(CSSession* cs); void cs_disable_audio_sending(CSSession* cs); diff --git a/toxav/toxav.c b/toxav/toxav.c index fcc6e86a..84a0c43a 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -208,7 +208,7 @@ void toxav_iterate(ToxAV* av) return; uint64_t start = current_time_monotonic(); - uint32_t rc = 0; + uint32_t rc = 500; pthread_mutex_lock(av->mutex); ToxAVCall* i = av->calls[av->calls_head]; @@ -225,7 +225,7 @@ void toxav_iterate(ToxAV* av) pthread_mutex_lock(av->mutex); } - av->interval = rc < av->dmssa ? 0 : rc - av->dmssa; + av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); av->dmsst += current_time_monotonic() - start; if (++av->dmssc == 3) { @@ -650,9 +650,6 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc } { /* Encode and send */ - cs_set_sending_audio_channels(call->cs, channels); - cs_set_sending_audio_sampling_rate(call->cs, sampling_rate); - uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); @@ -667,8 +664,6 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc LOGGER_WARNING("Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } - - LOGGER_DEBUG("Sent packet of size: %d (o %d)", vrc, sample_count); } pthread_mutex_unlock(call->mutex_audio_sending); @@ -956,7 +951,7 @@ bool call_prepare_transmission(ToxAVCall* call) call->rtps[audio_index]->cs = call->cs; /* Only enable sending if bitrate is defined */ - if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000, 2) != 0) { + if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000) != 0) { LOGGER_WARNING("Failed to enable audio sending!"); goto FAILURE; } -- cgit v1.2.3