summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormannol <eniz_vukovic@hotmail.com>2015-03-29 01:10:34 +0100
committermannol <eniz_vukovic@hotmail.com>2015-03-29 01:10:34 +0100
commitfdaad0b7c008b65646ca5cbd62ac0d305617ecfe (patch)
treee7f3e4e7a39916c37ff5af4fc9224a562826217e
parente65efc8936ee029f735a22105843335c18026ad2 (diff)
A little CS cleanup
-rw-r--r--toxav/av_test.c108
-rw-r--r--toxav/codec.c479
-rw-r--r--toxav/codec.h81
-rw-r--r--toxav/rtp.h2
-rw-r--r--toxav/toxav.c160
-rw-r--r--toxav/toxav.h10
6 files changed, 363 insertions, 477 deletions
diff --git a/toxav/av_test.c b/toxav/av_test.c
index 5ca53eb9..f0ad6a01 100644
--- a/toxav/av_test.c
+++ b/toxav/av_test.c
@@ -134,14 +134,14 @@ void t_toxav_receive_audio_frame_cb(ToxAV *av, uint32_t friend_number,
134 alDeleteBuffers(processed - 1, bufids + 1); 134 alDeleteBuffers(processed - 1, bufids + 1);
135 bufid = bufids[0]; 135 bufid = bufids[0];
136 } 136 }
137// else if(queued < 16) 137 else if(queued < 16)
138 alGenBuffers(1, &bufid); 138 alGenBuffers(1, &bufid);
139// else 139 else
140// return; 140 return;
141 141
142 142
143 alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16, 143 alBufferData(bufid, channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16,
144 pcm, sample_count * channels * 2, sampling_rate); 144 pcm, sample_count * 2, sampling_rate);
145 alSourceQueueBuffers(adout, 1, &bufid); 145 alSourceQueueBuffers(adout, 1, &bufid);
146 146
147 int32_t state; 147 int32_t state;
@@ -285,6 +285,36 @@ int send_opencv_img(ToxAV* av, uint32_t friend_number, const IplImage* img)
285 return rc; 285 return rc;
286} 286}
287 287
288ALCdevice* open_audio_device(const char* audio_out_dev_name)
289{
290 ALCdevice* rc;
291 rc = alcOpenDevice(audio_out_dev_name);
292 if ( !rc ) {
293 printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError());
294 exit(1);
295 }
296
297 ALCcontext* out_ctx = alcCreateContext(rc, NULL);
298 alcMakeContextCurrent(out_ctx);
299
300 uint32_t buffers[10];
301 alGenBuffers(10, buffers);
302 alGenSources((uint32_t)1, &adout);
303 alSourcei(adout, AL_LOOPING, AL_FALSE);
304
305 int16_t zeros[10000];
306 memset(zeros, 0, sizeof(zeros));
307
308 int i;
309 for ( i = 0; i < 10; ++i )
310 alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, sizeof(zeros), 48000);
311
312 alSourceQueueBuffers(adout, 10, buffers);
313 alSourcePlay(adout);
314
315 return rc;
316}
317
288int print_audio_devices() 318int print_audio_devices()
289{ 319{
290 const char *device; 320 const char *device;
@@ -373,49 +403,18 @@ int main (int argc, char** argv)
373 } 403 }
374 } 404 }
375 405
376 ALCdevice* audio_out_device; 406 const char* audio_out_dev_name = NULL;
377 407
378 { /* Open output device */ 408 int i = 0;
379 const char* audio_out_dev_name = NULL; 409 for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx;
380 410 audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i)
381 int i = 0; 411 if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1))
382 for(audio_out_dev_name = alcGetString(NULL, ALC_DEVICE_SPECIFIER); i < audio_out_dev_idx; 412 break;
383 audio_out_dev_name += strlen( audio_out_dev_name ) + 1, ++i)
384 if (!(audio_out_dev_name + strlen( audio_out_dev_name ) + 1))
385 break;
386
387 audio_out_device = alcOpenDevice(audio_out_dev_name);
388 if ( !audio_out_device ) {
389 printf("Failed to open playback device: %s: %d\n", audio_out_dev_name, alGetError());
390 exit(1);
391 }
392
393 ALCcontext* out_ctx = alcCreateContext(audio_out_device, NULL);
394 alcMakeContextCurrent(out_ctx);
395
396 uint32_t buffers[5];
397 alGenBuffers(5, buffers);
398 alGenSources((uint32_t)1, &adout);
399 alSourcei(adout, AL_LOOPING, AL_FALSE);
400
401 uint16_t zeros[10000];
402 memset(zeros, 0, 10000);
403
404 for ( i = 0; i < 5; ++i )
405 alBufferData(buffers[i], AL_FORMAT_STEREO16, zeros, 10000, 48000);
406
407 alSourceQueueBuffers(adout, 5, buffers);
408 alSourcePlay(adout);
409
410 printf("Using audio device: %s\n", audio_out_dev_name);
411 }
412 413
414 printf("Using audio device: %s\n", audio_out_dev_name);
413 printf("Using audio file: %s\n", af_name); 415 printf("Using audio file: %s\n", af_name);
414 printf("Using video file: %s\n", vf_name); 416 printf("Using video file: %s\n", vf_name);
415 417
416
417
418
419 418
420 /* START TOX NETWORK */ 419 /* START TOX NETWORK */
421 420
@@ -697,11 +696,11 @@ int main (int argc, char** argv)
697 printf("Failed to open the file.\n"); 696 printf("Failed to open the file.\n");
698 exit(1); 697 exit(1);
699 } 698 }
699 ALCdevice* audio_out_device = open_audio_device(audio_out_dev_name);
700 700
701 /* Run for 5 seconds */
702 701
703 uint32_t frame_duration = 10; 702 uint32_t frame_duration = 10;
704 int16_t PCM[10000]; 703 int16_t PCM[5760];
705 704
706 time_t start_time = time(NULL); 705 time_t start_time = time(NULL);
707 time_t expected_time = af_info.frames / af_info.samplerate + 2; 706 time_t expected_time = af_info.frames / af_info.samplerate + 2;
@@ -711,18 +710,21 @@ int main (int argc, char** argv)
711 710
712 int64_t count = sf_read_short(af_handle, PCM, frame_size); 711 int64_t count = sf_read_short(af_handle, PCM, frame_size);
713 if (count > 0) { 712 if (count > 0) {
714 TOXAV_ERR_SEND_FRAME rc; 713 t_toxav_receive_audio_frame_cb(BobAV, 0, PCM, count, af_info.channels, af_info.samplerate, NULL);
715 if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) { 714// TOXAV_ERR_SEND_FRAME rc;
716 printf("Error sending frame of size %ld: %d\n", count, rc); 715// if (toxav_send_audio_frame(AliceAV, 0, PCM, count, af_info.channels, af_info.samplerate, &rc) == false) {
717 exit(1); 716// printf("Error sending frame of size %ld: %d\n", count, rc);
718 } 717// exit(1);
718// }
719 } 719 }
720 720 c_sleep(frame_duration);
721 iterate_tox(bootstrap, AliceAV, BobAV); 721// iterate_tox(bootstrap, AliceAV, BobAV);
722 } 722 }
723
723 724
724 printf("Played file in: %lu\n", time(NULL) - start_time); 725 printf("Played file in: %lu\n", time(NULL) - start_time);
725 726
727 alcCloseDevice(audio_out_device);
726 sf_close(af_handle); 728 sf_close(af_handle);
727 729
728 { /* Hangup */ 730 { /* Hangup */
@@ -774,7 +776,5 @@ int main (int argc, char** argv)
774 tox_kill(bootstrap); 776 tox_kill(bootstrap);
775 777
776 printf("\nTest successful!\n"); 778 printf("\nTest successful!\n");
777
778 alcCloseDevice(audio_out_device);
779 return 0; 779 return 0;
780} 780}
diff --git a/toxav/codec.c b/toxav/codec.c
index b9cbbc06..b0f2ed31 100644
--- a/toxav/codec.c
+++ b/toxav/codec.c
@@ -238,10 +238,80 @@ static int convert_bw_to_sampling_rate(int bw)
238 } 238 }
239} 239}
240 240
241OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count)
242{
243 int status = OPUS_OK;
244 OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status);
245
246 if ( status != OPUS_OK ) {
247 LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status));
248 return NULL;
249 }
250
251 status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate));
252
253 if ( status != OPUS_OK ) {
254 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
255 goto FAILURE;
256 }
257
258 status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10));
259
260 if ( status != OPUS_OK ) {
261 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
262 goto FAILURE;
263 }
264
265 return rc;
266
267FAILURE:
268 opus_encoder_destroy(rc);
269 return NULL;
270}
271
272bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate)
273{
274 assert(dest);
275
276 vpx_codec_enc_cfg_t cfg;
277 int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
278
279 if (rc != VPX_CODEC_OK) {
280 LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
281 return false;
282 }
283
284 rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0,
285 VPX_ENCODER_ABI_VERSION);
286
287 if ( rc != VPX_CODEC_OK) {
288 LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
289 return false;
290 }
291
292 cfg.rc_target_bitrate = bitrate;
293 cfg.g_w = 800;
294 cfg.g_h = 600;
295 cfg.g_pass = VPX_RC_ONE_PASS;
296 cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
297 cfg.g_lag_in_frames = 0;
298 cfg.kf_min_dist = 0;
299 cfg.kf_max_dist = 48;
300 cfg.kf_mode = VPX_KF_AUTO;
301
302 rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8);
303
304 if ( rc != VPX_CODEC_OK) {
305 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
306 vpx_codec_destroy(dest);
307 }
308
309 return true;
310}
241 311
242/* PUBLIC */ 312/* PUBLIC */
243 313
244void cs_do(CSSession *cs) 314void cs_do(CSession *cs)
245{ 315{
246 /* Codec session should always be protected by call mutex so no need to check for cs validity 316 /* Codec session should always be protected by call mutex so no need to check for cs validity
247 */ 317 */
@@ -261,7 +331,7 @@ void cs_do(CSSession *cs)
261 RTPMessage *msg; 331 RTPMessage *msg;
262 332
263 /* The maximum for 120 ms 48 KHz audio */ 333 /* The maximum for 120 ms 48 KHz audio */
264 int16_t tmp[20000]; 334 int16_t tmp[5760];
265 335
266 while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { 336 while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) {
267 pthread_mutex_unlock(cs->queue_mutex); 337 pthread_mutex_unlock(cs->queue_mutex);
@@ -276,7 +346,7 @@ void cs_do(CSSession *cs)
276 rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); 346 rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data));
277 if (rc != -1) { 347 if (rc != -1) {
278 cs->last_packet_sampling_rate = rc; 348 cs->last_packet_sampling_rate = rc;
279 cs->last_packet_channels = opus_packet_get_nb_channels(msg->data); 349 cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data);
280 350
281 cs->last_packet_frame_duration = 351 cs->last_packet_frame_duration =
282 ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 ) 352 ( opus_packet_get_samples_per_frame(msg->data, cs->last_packet_sampling_rate) * 1000 )
@@ -287,7 +357,7 @@ void cs_do(CSSession *cs)
287 continue; 357 continue;
288 } 358 }
289 359
290 rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, sizeof(tmp), 0); 360 rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, 5760, 0);
291 rtp_free_msg(NULL, msg); 361 rtp_free_msg(NULL, msg);
292 } 362 }
293 363
@@ -295,21 +365,11 @@ void cs_do(CSSession *cs)
295 LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); 365 LOGGER_WARNING("Decoding error: %s", opus_strerror(rc));
296 } else if (cs->acb.first) { 366 } else if (cs->acb.first) {
297 /* Play */ 367 /* Play */
298
299 LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc, 368 LOGGER_DEBUG("Playing audio frame size: %d; channels: %d; srate: %d; duration %d", rc,
300 cs->last_packet_channels, cs->last_packet_sampling_rate, cs->last_packet_frame_duration); 369 cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->last_packet_frame_duration);
301
302 /* According to https://tools.ietf.org/html/rfc6716#section-2.1.2
303 * Every encoder can encode both mono and stereo data so we must
304 * determine which format is selected.
305 */
306
307 if (cs->last_packet_channels == 2) {
308 /* The packet is encoded with stereo encoder */
309 }
310 370
311 cs->acb.first(cs->agent, cs->friend_id, tmp, rc, 371 cs->acb.first(cs->av, cs->friend_id, tmp, rc,
312 cs->last_packet_channels, cs->last_packet_sampling_rate, cs->acb.second); 372 cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second);
313 } 373 }
314 374
315 pthread_mutex_lock(cs->queue_mutex); 375 pthread_mutex_lock(cs->queue_mutex);
@@ -336,7 +396,7 @@ void cs_do(CSSession *cs)
336 /* Play decoded images */ 396 /* Play decoded images */
337 for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) { 397 for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) {
338 if (cs->vcb.first) 398 if (cs->vcb.first)
339 cs->vcb.first(cs->agent, cs->friend_id, dest->d_w, dest->d_h, 399 cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h,
340 (const uint8_t**)dest->planes, dest->stride, cs->vcb.second); 400 (const uint8_t**)dest->planes, dest->stride, cs->vcb.second);
341 401
342 vpx_img_free(dest); 402 vpx_img_free(dest);
@@ -349,29 +409,103 @@ void cs_do(CSSession *cs)
349 pthread_mutex_unlock(cs->queue_mutex); 409 pthread_mutex_unlock(cs->queue_mutex);
350} 410}
351 411
352CSSession *cs_new(uint32_t peer_video_frame_piece_size) 412CSession *cs_new(uint32_t peer_video_frame_piece_size)
353{ 413{
354 CSSession *cs = calloc(sizeof(CSSession), 1); 414 CSession *cs = calloc(sizeof(CSession), 1);
355 415
356 if (!cs) { 416 if (!cs) {
357 LOGGER_WARNING("Allocation failed! Application might misbehave!"); 417 LOGGER_WARNING("Allocation failed! Application might misbehave!");
358 return NULL; 418 return NULL;
359 } 419 }
360 420
361
362 if (create_recursive_mutex(cs->queue_mutex) != 0) { 421 if (create_recursive_mutex(cs->queue_mutex) != 0) {
363 LOGGER_WARNING("Failed to create recursive mutex!"); 422 LOGGER_WARNING("Failed to create recursive mutex!");
364 free(cs); 423 free(cs);
365 return NULL; 424 return NULL;
366 } 425 }
367 426
427 /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
428 /* Create decoders and set up their values
429 */
430
431 /*
432 * AUDIO
433 */
434
435 int status;
436 cs->audio_decoder = opus_decoder_create(48000, 2, &status ); /* NOTE: Must be mono */
437
438 if ( status != OPUS_OK ) {
439 LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status));
440 goto FAILURE;
441 }
442
443 /* These need to be set in order to properly
444 * do error correction with opus */
445 cs->last_packet_frame_duration = 120;
446 cs->last_packet_sampling_rate = 48000;
447
448 if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) {
449 LOGGER_WARNING("Jitter buffer creaton failed!");
450 opus_decoder_destroy(cs->audio_decoder);
451 goto FAILURE;
452 }
453
454 /*
455 * VIDEO
456 */
457 int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE,
458 NULL, 0, VPX_DECODER_ABI_VERSION);
459
460 if ( rc != VPX_CODEC_OK) {
461 LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
462 goto AUDIO_DECODER_CLEANUP;
463 }
464
465 if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) {
466 vpx_codec_destroy(cs->v_decoder);
467 goto AUDIO_DECODER_CLEANUP;
468 }
469
470 if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) {
471 free(cs->frame_buf);
472 vpx_codec_destroy(cs->v_decoder);
473 goto AUDIO_DECODER_CLEANUP;
474 }
475 /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
476
477 /* Initialize encoders with default values */
478 cs->audio_encoder = create_audio_encoder(48000, 48000, 2);
479 if (cs->audio_encoder == NULL)
480 goto VIDEO_DECODER_CLEANUP;
481
482 cs->last_encoding_bitrate = 48000;
483 cs->last_encoding_sampling_rate = 48000;
484 cs->last_encoding_channel_count = 2;
485
486 if (!create_video_encoder(cs->v_encoder, 500000)) {
487 opus_encoder_destroy(cs->audio_encoder);
488 goto VIDEO_DECODER_CLEANUP;
489 }
368 490
369 cs->peer_video_frame_piece_size = peer_video_frame_piece_size; 491 cs->peer_video_frame_piece_size = peer_video_frame_piece_size;
370 492
371 return cs; 493 return cs;
494
495VIDEO_DECODER_CLEANUP:
496 buffer_free(cs->vbuf_raw);
497 free(cs->frame_buf);
498 vpx_codec_destroy(cs->v_decoder);
499AUDIO_DECODER_CLEANUP:
500 opus_decoder_destroy(cs->audio_decoder);
501 jbuf_free(cs->j_buf);
502FAILURE:
503 pthread_mutex_destroy(cs->queue_mutex);
504 free(cs);
505 return NULL;
372} 506}
373 507
374void cs_kill(CSSession *cs) 508void cs_kill(CSession *cs)
375{ 509{
376 if (!cs) 510 if (!cs)
377 return; 511 return;
@@ -380,10 +514,13 @@ void cs_kill(CSSession *cs)
380 * the callback is unregistered before cs_kill is called. 514 * the callback is unregistered before cs_kill is called.
381 */ 515 */
382 516
383 cs_disable_audio_sending(cs); 517 vpx_codec_destroy(cs->v_encoder);
384 cs_disable_audio_receiving(cs); 518 vpx_codec_destroy(cs->v_decoder);
385 cs_disable_video_sending(cs); 519 opus_encoder_destroy(cs->audio_encoder);
386 cs_disable_video_receiving(cs); 520 opus_decoder_destroy(cs->audio_decoder);
521 buffer_free(cs->vbuf_raw);
522 jbuf_free(cs->j_buf);
523 free(cs->frame_buf);
387 524
388 pthread_mutex_destroy(cs->queue_mutex); 525 pthread_mutex_destroy(cs->queue_mutex);
389 526
@@ -391,15 +528,13 @@ void cs_kill(CSSession *cs)
391 free(cs); 528 free(cs);
392} 529}
393 530
394 531void cs_init_video_splitter_cycle(CSession* cs)
395
396void cs_init_video_splitter_cycle(CSSession* cs)
397{ 532{
398 cs->split_video_frame[0] = cs->frameid_out++; 533 cs->split_video_frame[0] = cs->frameid_out++;
399 cs->split_video_frame[1] = 0; 534 cs->split_video_frame[1] = 0;
400} 535}
401 536
402int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16_t length) 537int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length)
403{ 538{
404 cs->processing_video_frame = payload; 539 cs->processing_video_frame = payload;
405 cs->processing_video_frame_size = length; 540 cs->processing_video_frame_size = length;
@@ -407,7 +542,7 @@ int cs_update_video_splitter_cycle(CSSession *cs, const uint8_t *payload, uint16
407 return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1; 542 return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1;
408} 543}
409 544
410const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size) 545const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size)
411{ 546{
412 if (!cs || !size) return NULL; 547 if (!cs || !size) return NULL;
413 548
@@ -433,295 +568,61 @@ const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size)
433 return cs->split_video_frame; 568 return cs->split_video_frame;
434} 569}
435 570
436 571int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height)
437
438int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height)
439{ 572{
440 if (!cs->v_encoding)
441 return -1;
442
443 /* TODO FIXME reference is safe? */
444 vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc; 573 vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc;
574 if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height)
575 return 0; /* Nothing changed */
445 576
446 if (cfg.g_w == width && cfg.g_h == height) 577 cfg.rc_target_bitrate = bitrate;
447 return 0;
448/*
449 if (width * height > cs->max_width * cs->max_height) {
450 vpx_codec_ctx_t v_encoder = cs->v_encoder;
451
452 if (init_video_encoder(cs, width, height, cs->video_bitrate) == -1) {
453 cs->v_encoder = v_encoder;
454 return cs_ErrorSettingVideoResolution;
455 }
456
457 vpx_codec_destroy(&v_encoder);
458 return 0;
459 }*/
460
461 LOGGER_DEBUG("New video resolution: %u %u", width, height);
462 cfg.g_w = width; 578 cfg.g_w = width;
463 cfg.g_h = height; 579 cfg.g_h = height;
464 int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg);
465
466 if ( rc != VPX_CODEC_OK) {
467 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
468 return cs_ErrorSettingVideoResolution;
469 }
470
471 return 0;
472}
473
474int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate)
475{
476 if (!cs->v_encoding)
477 return -1;
478
479 /* TODO FIXME reference is safe? */
480 vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc;
481 if (cfg.rc_target_bitrate == bitrate)
482 return 0;
483
484 LOGGER_DEBUG("New video bitrate: %u", bitrate);
485 cfg.rc_target_bitrate = bitrate;
486 580
487 int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg); 581 int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg);
488 if ( rc != VPX_CODEC_OK) { 582 if ( rc != VPX_CODEC_OK) {
489 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); 583 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
490 return cs_ErrorSettingVideoBitrate;
491 }
492
493 return 0;
494}
495
496
497
498int cs_enable_video_sending(CSSession* cs, uint32_t bitrate)
499{
500 if (cs->v_encoding)
501 return 0;
502
503 vpx_codec_enc_cfg_t cfg;
504 int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
505
506 if (rc != VPX_CODEC_OK) {
507 LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
508 return -1;
509 }
510
511 rc = vpx_codec_enc_init_ver(cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0,
512 VPX_ENCODER_ABI_VERSION);
513
514 if ( rc != VPX_CODEC_OK) {
515 LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
516 return -1; 584 return -1;
517 } 585 }
518
519 /* So that we can use cs_disable_video_sending to clean up */
520 cs->v_encoding = true;
521
522 if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) )
523 goto FAILURE;
524
525 cfg.rc_target_bitrate = bitrate;
526 cfg.g_w = 800;
527 cfg.g_h = 600;
528 cfg.g_pass = VPX_RC_ONE_PASS;
529 cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
530 cfg.g_lag_in_frames = 0;
531 cfg.kf_min_dist = 0;
532 cfg.kf_max_dist = 48;
533 cfg.kf_mode = VPX_KF_AUTO;
534
535
536 rc = vpx_codec_control(cs->v_encoder, VP8E_SET_CPUUSED, 8);
537
538 if ( rc != VPX_CODEC_OK) {
539 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
540 goto FAILURE;
541 }
542
543 return 0;
544
545FAILURE:
546 cs_disable_video_sending(cs);
547 return -1;
548}
549 586
550int cs_enable_video_receiving(CSSession* cs)
551{
552 if (cs->v_decoding)
553 return 0;
554
555 int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE,
556 NULL, 0, VPX_DECODER_ABI_VERSION);
557
558 if ( rc != VPX_CODEC_OK) {
559 LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
560 return -1;
561 }
562
563 /* So that we can use cs_disable_video_sending to clean up */
564 cs->v_decoding = true;
565
566 if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) )
567 goto FAILURE;
568
569 if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) )
570 goto FAILURE;
571
572 return 0; 587 return 0;
573
574FAILURE:
575 cs_disable_video_receiving(cs);
576 return -1;
577} 588}
578 589
579 590int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels)
580
581void cs_disable_video_sending(CSSession* cs)
582{ 591{
583 if (cs->v_encoding) { 592 /* Values are checked in toxav.c */
584 cs->v_encoding = false;
585
586 free(cs->split_video_frame);
587 cs->split_video_frame = NULL;
588
589 vpx_codec_destroy(cs->v_encoder);
590 }
591}
592
593void cs_disable_video_receiving(CSSession* cs)
594{
595 if (cs->v_decoding) {
596 cs->v_decoding = false;
597
598 buffer_free(cs->vbuf_raw);
599 cs->vbuf_raw = NULL;
600 free(cs->frame_buf);
601 cs->frame_buf = NULL;
602
603 vpx_codec_destroy(cs->v_decoder);
604 }
605}
606
607
608
609int cs_set_sending_audio_bitrate(CSSession *cs, int32_t rate)
610{
611 if (cs->audio_encoder == NULL)
612 return -1;
613
614 int rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(rate));
615
616 if ( rc != OPUS_OK ) {
617 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
618 return -1;
619 }
620 593
621 LOGGER_DEBUG("Set new encoder bitrate to: %d", rate); 594 if (cs->last_encoding_sampling_rate != sampling_rate || cs->last_encoding_channel_count != channels) {
622 return 0; 595 OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels);
623} 596 if (new_encoder == NULL)
624 597 return -1;
625 598
626
627void cs_disable_audio_sending(CSSession* cs)
628{
629 if ( cs->audio_encoder ) {
630 opus_encoder_destroy(cs->audio_encoder); 599 opus_encoder_destroy(cs->audio_encoder);
631 cs->audio_encoder = NULL; 600 cs->audio_encoder = new_encoder;
632 } 601 } else if (cs->last_encoding_bitrate == bitrate)
633} 602 return 0; /* Nothing changed */
634 603 else {
635void cs_disable_audio_receiving(CSSession* cs) 604 int status = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate));
636{
637 if ( cs->audio_decoder ) {
638 opus_decoder_destroy(cs->audio_decoder);
639 cs->audio_decoder = NULL;
640 jbuf_free(cs->j_buf);
641 cs->j_buf = NULL;
642 605
643 /* These need to be set in order to properly 606 if ( status != OPUS_OK ) {
644 * do error correction with opus */ 607 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
645 cs->last_packet_frame_duration = 120; 608 return -1;
646 cs->last_packet_sampling_rate = 48000; 609 }
647 }
648}
649
650
651
652int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate)
653{
654 if (cs->audio_encoder)
655 return 0;
656
657 int rc = OPUS_OK;
658 cs->audio_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_AUDIO, &rc);
659
660 if ( rc != OPUS_OK ) {
661 LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc));
662 return -1;
663 }
664
665 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate));
666
667 if ( rc != OPUS_OK ) {
668 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
669 goto FAILURE;
670 }
671
672 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10));
673
674 if ( rc != OPUS_OK ) {
675 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
676 goto FAILURE;
677 } 610 }
678
679 return 0;
680
681FAILURE:
682 cs_disable_audio_sending(cs);
683 return -1;
684}
685 611
686int cs_enable_audio_receiving(CSSession* cs) 612 cs->last_encoding_bitrate = bitrate;
687{ 613 cs->last_encoding_sampling_rate = sampling_rate;
688 if (cs->audio_decoder) 614 cs->last_encoding_channel_count = channels;
689 return 0;
690
691 int rc;
692 cs->audio_decoder = opus_decoder_create(48000, 2, &rc );
693
694 if ( rc != OPUS_OK ) {
695 LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc));
696 return -1;
697 }
698
699
700 if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) {
701 LOGGER_WARNING("Jitter buffer creaton failed!");
702 opus_decoder_destroy(cs->audio_decoder);
703 cs->audio_decoder = NULL;
704 return -1;
705 }
706
707
708 /* These need to be set in order to properly
709 * do error correction with opus */
710 cs->last_packet_frame_duration = 120;
711 cs->last_packet_sampling_rate = 48000;
712
713 return 0; 615 return 0;
714} 616}
715 617
716 618
717
718/* Called from RTP */ 619/* Called from RTP */
719void queue_message(RTPSession *session, RTPMessage *msg) 620void queue_message(RTPSession *session, RTPMessage *msg)
720{ 621{
721 /* This function is unregistered during call termination befor destroying 622 /* This function is unregistered during call termination befor destroying
722 * Codec session so no need to check for validity of cs TODO properly check video cycle 623 * Codec session so no need to check for validity of cs TODO properly check video cycle
723 */ 624 */
724 CSSession *cs = session->cs; 625 CSession *cs = session->cs;
725 626
726 if (!cs) 627 if (!cs)
727 return; 628 return;
diff --git a/toxav/codec.h b/toxav/codec.h
index 526a80d5..b13203f1 100644
--- a/toxav/codec.h
+++ b/toxav/codec.h
@@ -42,32 +42,7 @@
42 42
43#define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; } 43#define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; }
44 44
45typedef void (*CSAudioCallback) (void *agent, int32_t call_idx, const int16_t *PCM, uint16_t size, void *data); 45typedef struct CSession_s {
46typedef void (*CSVideoCallback) (void *agent, int32_t call_idx, const vpx_image_t *img, void *data);
47
48/**
49 * Codec capabilities
50 */
51typedef enum {
52 cs_AudioEncoding = 1 << 0,
53 cs_AudioDecoding = 1 << 1,
54 cs_VideoEncoding = 1 << 2,
55 cs_VideoDecoding = 1 << 3
56} CSCapabilities;
57
58/**
59 * Codec errors.
60 */
61typedef enum {
62 cs_ErrorSettingVideoResolution = -30,
63 cs_ErrorSettingVideoBitrate = -31,
64 cs_ErrorSplittingVideoPayload = -32,
65} CSError;
66
67/**
68 * Codec session - controling codec
69 */
70typedef struct CSSession_s {
71 46
72 /* VIDEO 47 /* VIDEO
73 * 48 *
@@ -76,12 +51,10 @@ typedef struct CSSession_s {
76 51
77 /* video encoding */ 52 /* video encoding */
78 vpx_codec_ctx_t v_encoder[1]; 53 vpx_codec_ctx_t v_encoder[1];
79 bool v_encoding;
80 uint32_t frame_counter; 54 uint32_t frame_counter;
81 55
82 /* video decoding */ 56 /* video decoding */
83 vpx_codec_ctx_t v_decoder[1]; 57 vpx_codec_ctx_t v_decoder[1];
84 bool v_decoding;
85 void *vbuf_raw; /* Un-decoded data */ 58 void *vbuf_raw; /* Un-decoded data */
86 59
87 /* Data handling */ 60 /* Data handling */
@@ -107,10 +80,13 @@ typedef struct CSSession_s {
107 80
108 /* audio encoding */ 81 /* audio encoding */
109 OpusEncoder *audio_encoder; 82 OpusEncoder *audio_encoder;
83 int32_t last_encoding_sampling_rate;
84 int32_t last_encoding_channel_count;
85 int32_t last_encoding_bitrate;
110 86
111 /* audio decoding */ 87 /* audio decoding */
112 OpusDecoder *audio_decoder; 88 OpusDecoder *audio_decoder;
113 int32_t last_packet_channels; 89 int32_t last_packet_channel_count;
114 int32_t last_packet_sampling_rate; 90 int32_t last_packet_sampling_rate;
115 int32_t last_packet_frame_duration; 91 int32_t last_packet_frame_duration;
116 struct JitterBuffer_s *j_buf; 92 struct JitterBuffer_s *j_buf;
@@ -120,55 +96,28 @@ typedef struct CSSession_s {
120 * 96 *
121 * 97 *
122 */ 98 */
123 void *agent; /* Pointer to ToxAV TODO make this pointer to ToxAV*/ 99 ToxAV *av;
124 int32_t friend_id; 100 int32_t friend_id;
125 101
126 PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */ 102 PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */
127 PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */ 103 PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */
128 104
129 pthread_mutex_t queue_mutex[1]; 105 pthread_mutex_t queue_mutex[1];
130} CSSession; 106} CSession;
131 107
132 108
133/** 109void cs_do(CSession *cs);
134 * Generic
135 */
136void cs_do(CSSession *cs);
137
138/* Make sure to be called BEFORE corresponding rtp_new */ 110/* Make sure to be called BEFORE corresponding rtp_new */
139CSSession *cs_new(uint32_t peer_mvfpsz); 111CSession *cs_new(uint32_t peer_mvfpsz);
140/* Make sure to be called AFTER corresponding rtp_kill */ 112/* Make sure to be called AFTER corresponding rtp_kill */
141void cs_kill(CSSession *cs); 113void cs_kill(CSession *cs);
142
143
144/**
145 * VIDEO HANDLING
146 */
147void cs_init_video_splitter_cycle(CSSession *cs);
148int cs_update_video_splitter_cycle(CSSession* cs, const uint8_t* payload, uint16_t length);
149const uint8_t *cs_iterate_split_video_frame(CSSession *cs, uint16_t *size);
150
151int cs_set_sending_video_resolution(CSSession *cs, uint16_t width, uint16_t height);
152int cs_set_sending_video_bitrate(CSSession *cs, uint32_t bitrate);
153
154int cs_enable_video_sending(CSSession* cs, uint32_t bitrate);
155int cs_enable_video_receiving(CSSession* cs);
156
157void cs_disable_video_sending(CSSession* cs);
158void cs_disable_video_receiving(CSSession* cs);
159
160/**
161 * AUDIO HANDLING
162 */
163int cs_set_sending_audio_bitrate(CSSession* cs, int32_t rate);
164
165int cs_enable_audio_sending(CSSession* cs, uint32_t bitrate);
166int cs_enable_audio_receiving(CSSession* cs);
167
168void cs_disable_audio_sending(CSSession* cs);
169void cs_disable_audio_receiving(CSSession* cs);
170 114
115void cs_init_video_splitter_cycle(CSession *cs);
116int cs_update_video_splitter_cycle(CSession* cs, const uint8_t* payload, uint16_t length);
117const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size);
171 118
119int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height);
120int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels);
172 121
173 122
174/* Internal. Called from rtp_handle_message */ 123/* Internal. Called from rtp_handle_message */
diff --git a/toxav/rtp.h b/toxav/rtp.h
index 2950941b..3bfbb7af 100644
--- a/toxav/rtp.h
+++ b/toxav/rtp.h
@@ -101,7 +101,7 @@ typedef struct {
101 101
102 int dest; 102 int dest;
103 103
104 struct CSSession_s *cs; 104 struct CSession_s *cs;
105 Messenger *m; 105 Messenger *m;
106 106
107} RTPSession; 107} RTPSession;
diff --git a/toxav/toxav.c b/toxav/toxav.c
index 84a0c43a..62fed33f 100644
--- a/toxav/toxav.c
+++ b/toxav/toxav.c
@@ -43,7 +43,7 @@ enum {
43typedef struct ToxAVCall_s { 43typedef struct ToxAVCall_s {
44 ToxAV* av; 44 ToxAV* av;
45 RTPSession *rtps[2]; /* Audio is first and video is second */ 45 RTPSession *rtps[2]; /* Audio is first and video is second */
46 CSSession *cs; 46 CSession *cs;
47 47
48 pthread_mutex_t mutex_audio_sending[1]; 48 pthread_mutex_t mutex_audio_sending[1];
49 pthread_mutex_t mutex_video_sending[1]; 49 pthread_mutex_t mutex_video_sending[1];
@@ -54,8 +54,8 @@ typedef struct ToxAVCall_s {
54 MSICall* msi_call; 54 MSICall* msi_call;
55 uint32_t friend_id; 55 uint32_t friend_id;
56 56
57 uint32_t s_audio_b; /* Sending audio bitrate */ 57 uint32_t audio_bit_rate; /* Sending audio bitrate */
58 uint32_t s_video_b; /* Sending video bitrate */ 58 uint32_t video_bit_rate; /* Sending video bitrate */
59 59
60 uint8_t last_self_capabilities; 60 uint8_t last_self_capabilities;
61 uint8_t last_peer_capabilities; 61 uint8_t last_peer_capabilities;
@@ -244,8 +244,8 @@ bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint
244 return false; 244 return false;
245 } 245 }
246 246
247 call->s_audio_b = audio_bit_rate; 247 call->audio_bit_rate = audio_bit_rate;
248 call->s_video_b = video_bit_rate; 248 call->video_bit_rate = video_bit_rate;
249 249
250 call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; 250 call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo;
251 251
@@ -302,8 +302,8 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui
302 goto END; 302 goto END;
303 } 303 }
304 304
305 call->s_audio_b = audio_bit_rate; 305 call->audio_bit_rate = audio_bit_rate;
306 call->s_video_b = video_bit_rate; 306 call->video_bit_rate = video_bit_rate;
307 307
308 call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; 308 call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo;
309 309
@@ -479,12 +479,70 @@ END:
479 479
480bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error) 480bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, TOXAV_ERR_BIT_RATE* error)
481{ 481{
482 /* TODO */ 482 TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK;
483 ToxAVCall* call;
484
485 if (m_friend_exists(av->m, friend_number) == 0) {
486 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND;
487 goto END;
488 }
489
490 if (audio_bitrate_invalid(audio_bit_rate)) {
491 rc = TOXAV_ERR_BIT_RATE_INVALID;
492 goto END;
493 }
494
495 pthread_mutex_lock(av->mutex);
496 call = call_get(av, friend_number);
497 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
498 pthread_mutex_unlock(av->mutex);
499 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL;
500 goto END;
501 }
502
503 /* NOTE: no need to lock*/
504 call->audio_bit_rate = audio_bit_rate;
505 pthread_mutex_unlock(av->mutex);
506
507END:
508 if (error)
509 *error = rc;
510
511 return rc == TOXAV_ERR_BIT_RATE_OK;
483} 512}
484 513
485bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error) 514bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_bit_rate, TOXAV_ERR_BIT_RATE* error)
486{ 515{
487 /* TODO */ 516 TOXAV_ERR_BIT_RATE rc = TOXAV_ERR_BIT_RATE_OK;
517 ToxAVCall* call;
518
519 if (m_friend_exists(av->m, friend_number) == 0) {
520 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND;
521 goto END;
522 }
523
524 if (video_bitrate_invalid(video_bit_rate)) {
525 rc = TOXAV_ERR_BIT_RATE_INVALID;
526 goto END;
527 }
528
529 pthread_mutex_lock(av->mutex);
530 call = call_get(av, friend_number);
531 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
532 pthread_mutex_unlock(av->mutex);
533 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL;
534 goto END;
535 }
536
537 /* NOTE: no need to lock*/
538 call->video_bit_rate = video_bit_rate;
539 pthread_mutex_unlock(av->mutex);
540
541END:
542 if (error)
543 *error = rc;
544
545 return rc == TOXAV_ERR_BIT_RATE_OK;
488} 546}
489 547
490void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) 548void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data)
@@ -507,7 +565,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
507 565
508 pthread_mutex_lock(av->mutex); 566 pthread_mutex_lock(av->mutex);
509 call = call_get(av, friend_number); 567 call = call_get(av, friend_number);
510 if (call == NULL || !call->active) { 568 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
511 pthread_mutex_unlock(av->mutex); 569 pthread_mutex_unlock(av->mutex);
512 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; 570 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
513 goto END; 571 goto END;
@@ -516,20 +574,13 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
516 pthread_mutex_lock(call->mutex_video_sending); 574 pthread_mutex_lock(call->mutex_video_sending);
517 pthread_mutex_unlock(av->mutex); 575 pthread_mutex_unlock(av->mutex);
518 576
519 if (call->msi_call->state != msi_CallActive) {
520 /* TODO */
521 pthread_mutex_unlock(call->mutex_video_sending);
522 rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED;
523 goto END;
524 }
525
526 if ( y == NULL || u == NULL || v == NULL ) { 577 if ( y == NULL || u == NULL || v == NULL ) {
527 pthread_mutex_unlock(call->mutex_video_sending); 578 pthread_mutex_unlock(call->mutex_video_sending);
528 rc = TOXAV_ERR_SEND_FRAME_NULL; 579 rc = TOXAV_ERR_SEND_FRAME_NULL;
529 goto END; 580 goto END;
530 } 581 }
531 582
532 if ( cs_set_sending_video_resolution(call->cs, width, height) != 0 ) { 583 if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) {
533 pthread_mutex_unlock(call->mutex_video_sending); 584 pthread_mutex_unlock(call->mutex_video_sending);
534 rc = TOXAV_ERR_SEND_FRAME_INVALID; 585 rc = TOXAV_ERR_SEND_FRAME_INVALID;
535 goto END; 586 goto END;
@@ -550,7 +601,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
550 int vrc = vpx_codec_encode(call->cs->v_encoder, &img, 601 int vrc = vpx_codec_encode(call->cs->v_encoder, &img,
551 call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); 602 call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US);
552 603
553 vpx_img_free(&img); /* FIXME don't free? */ 604 vpx_img_free(&img);
554 if ( vrc != VPX_CODEC_OK) { 605 if ( vrc != VPX_CODEC_OK) {
555 pthread_mutex_unlock(call->mutex_video_sending); 606 pthread_mutex_unlock(call->mutex_video_sending);
556 LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); 607 LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc));
@@ -621,7 +672,7 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc
621 672
622 pthread_mutex_lock(av->mutex); 673 pthread_mutex_lock(av->mutex);
623 call = call_get(av, friend_number); 674 call = call_get(av, friend_number);
624 if (call == NULL || !call->active) { 675 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
625 pthread_mutex_unlock(av->mutex); 676 pthread_mutex_unlock(av->mutex);
626 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; 677 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
627 goto END; 678 goto END;
@@ -630,33 +681,34 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc
630 pthread_mutex_lock(call->mutex_audio_sending); 681 pthread_mutex_lock(call->mutex_audio_sending);
631 pthread_mutex_unlock(av->mutex); 682 pthread_mutex_unlock(av->mutex);
632 683
633 if (call->msi_call->state != msi_CallActive) {
634 /* TODO */
635 pthread_mutex_unlock(call->mutex_audio_sending);
636 rc = TOXAV_ERR_SEND_FRAME_NOT_REQUESTED;
637 goto END;
638 }
639
640 if ( pcm == NULL ) { 684 if ( pcm == NULL ) {
641 pthread_mutex_unlock(call->mutex_audio_sending); 685 pthread_mutex_unlock(call->mutex_audio_sending);
642 rc = TOXAV_ERR_SEND_FRAME_NULL; 686 rc = TOXAV_ERR_SEND_FRAME_NULL;
643 goto END; 687 goto END;
644 } 688 }
645 689
646 if ( channels != 1 && channels != 2 ) { 690 if ( channels > 2 ) {
647 pthread_mutex_unlock(call->mutex_audio_sending); 691 pthread_mutex_unlock(call->mutex_audio_sending);
648 rc = TOXAV_ERR_SEND_FRAME_INVALID; 692 rc = TOXAV_ERR_SEND_FRAME_INVALID;
649 goto END; 693 goto END;
650 } 694 }
651 695
652 { /* Encode and send */ 696 { /* Encode and send */
697 if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) {
698 pthread_mutex_unlock(call->mutex_audio_sending);
699 rc = TOXAV_ERR_SEND_FRAME_INVALID;
700 goto END;
701 }
702
703
704 LOGGER_DEBUG("Sending audio frame size: %d; channels: %d; srate: %d", sample_count, channels, sampling_rate);
653 uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */]; 705 uint8_t dest[sample_count * channels * 2 /* sizeof(uint16_t) */];
654 int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest)); 706 int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, dest, sizeof (dest));
655 707
656 if (vrc < 0) { 708 if (vrc < 0) {
657 LOGGER_WARNING("Failed to encode frame"); 709 LOGGER_WARNING("Failed to encode frame");
658 rc = TOXAV_ERR_SEND_FRAME_INVALID;
659 pthread_mutex_unlock(call->mutex_audio_sending); 710 pthread_mutex_unlock(call->mutex_audio_sending);
711 rc = TOXAV_ERR_SEND_FRAME_INVALID;
660 goto END; 712 goto END;
661 } 713 }
662 714
@@ -914,19 +966,19 @@ bool call_prepare_transmission(ToxAVCall* call)
914 } 966 }
915 967
916 if (pthread_mutex_init(call->mutex_audio_sending, NULL) != 0) 968 if (pthread_mutex_init(call->mutex_audio_sending, NULL) != 0)
917 goto MUTEX_INIT_ERROR; 969 return false;
918 970
919 if (pthread_mutex_init(call->mutex_video_sending, NULL) != 0) { 971 if (pthread_mutex_init(call->mutex_video_sending, NULL) != 0) {
920 pthread_mutex_destroy(call->mutex_audio_sending); 972 goto AUDIO_SENDING_MUTEX_CLEANUP;
921 goto MUTEX_INIT_ERROR;
922 } 973 }
923 974
924 if (pthread_mutex_init(call->mutex_decoding, NULL) != 0) { 975 if (pthread_mutex_init(call->mutex_decoding, NULL) != 0) {
925 pthread_mutex_destroy(call->mutex_audio_sending); 976 goto VIDEO_SENDING_MUTEX_CLEANUP;
926 pthread_mutex_destroy(call->mutex_video_sending);
927 goto MUTEX_INIT_ERROR;
928 } 977 }
929 978
979 /* Creates both audio and video encoders and decoders with some default values.
980 * Make sure to reconfigure encoders dynamically when sending data
981 */
930 call->cs = cs_new(call->msi_call->peer_vfpsz); 982 call->cs = cs_new(call->msi_call->peer_vfpsz);
931 983
932 if ( !call->cs ) { 984 if ( !call->cs ) {
@@ -934,13 +986,13 @@ bool call_prepare_transmission(ToxAVCall* call)
934 goto FAILURE; 986 goto FAILURE;
935 } 987 }
936 988
937 call->cs->agent = av; 989 call->cs->av = av;
938 call->cs->friend_id = call->friend_id; 990 call->cs->friend_id = call->friend_id;
939 991
940 memcpy(&call->cs->acb, &av->acb, sizeof(av->acb)); 992 memcpy(&call->cs->acb, &av->acb, sizeof(av->acb));
941 memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb)); 993 memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb));
942 994
943 { /* Prepare audio */ 995 { /* Prepare audio RTP */
944 call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); 996 call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id);
945 997
946 if ( !call->rtps[audio_index] ) { 998 if ( !call->rtps[audio_index] ) {
@@ -950,24 +1002,13 @@ bool call_prepare_transmission(ToxAVCall* call)
950 1002
951 call->rtps[audio_index]->cs = call->cs; 1003 call->rtps[audio_index]->cs = call->cs;
952 1004
953 /* Only enable sending if bitrate is defined */
954 if (call->s_audio_b > 0 && cs_enable_audio_sending(call->cs, call->s_audio_b * 1000) != 0) {
955 LOGGER_WARNING("Failed to enable audio sending!");
956 goto FAILURE;
957 }
958
959 if (cs_enable_audio_receiving(call->cs) != 0) {
960 LOGGER_WARNING("Failed to enable audio receiving!");
961 goto FAILURE;
962 }
963
964 if (rtp_start_receiving(call->rtps[audio_index]) != 0) { 1005 if (rtp_start_receiving(call->rtps[audio_index]) != 0) {
965 LOGGER_WARNING("Failed to enable audio receiving!"); 1006 LOGGER_WARNING("Failed to enable audio receiving!");
966 goto FAILURE; 1007 goto FAILURE;
967 } 1008 }
968 } 1009 }
969 1010
970 { /* Prepare video */ 1011 { /* Prepare video RTP */
971 call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); 1012 call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id);
972 1013
973 if ( !call->rtps[video_index] ) { 1014 if ( !call->rtps[video_index] ) {
@@ -977,17 +1018,6 @@ bool call_prepare_transmission(ToxAVCall* call)
977 1018
978 call->rtps[video_index]->cs = call->cs; 1019 call->rtps[video_index]->cs = call->cs;
979 1020
980 /* Only enable sending if bitrate is defined */
981 if (call->s_video_b > 0 && cs_enable_video_sending(call->cs, call->s_video_b) != 0) {
982 LOGGER_WARNING("Failed to enable video sending!");
983 goto FAILURE;
984 }
985
986 if (cs_enable_video_receiving(call->cs) != 0) {
987 LOGGER_WARNING("Failed to enable video receiving!");
988 goto FAILURE;
989 }
990
991 if (rtp_start_receiving(call->rtps[video_index]) != 0) { 1021 if (rtp_start_receiving(call->rtps[video_index]) != 0) {
992 LOGGER_WARNING("Failed to enable audio receiving!"); 1022 LOGGER_WARNING("Failed to enable audio receiving!");
993 goto FAILURE; 1023 goto FAILURE;
@@ -1004,13 +1034,11 @@ FAILURE:
1004 call->rtps[video_index] = NULL; 1034 call->rtps[video_index] = NULL;
1005 cs_kill(call->cs); 1035 cs_kill(call->cs);
1006 call->cs = NULL; 1036 call->cs = NULL;
1007 pthread_mutex_destroy(call->mutex_audio_sending);
1008 pthread_mutex_destroy(call->mutex_video_sending);
1009 pthread_mutex_destroy(call->mutex_decoding); 1037 pthread_mutex_destroy(call->mutex_decoding);
1010 return false; 1038VIDEO_SENDING_MUTEX_CLEANUP:
1011 1039 pthread_mutex_destroy(call->mutex_video_sending);
1012MUTEX_INIT_ERROR: 1040AUDIO_SENDING_MUTEX_CLEANUP:
1013 LOGGER_ERROR("Mutex initialization failed!\n"); 1041 pthread_mutex_destroy(call->mutex_audio_sending);
1014 return false; 1042 return false;
1015} 1043}
1016 1044
diff --git a/toxav/toxav.h b/toxav/toxav.h
index 48bb6b8c..ae95c61b 100644
--- a/toxav/toxav.h
+++ b/toxav/toxav.h
@@ -324,7 +324,15 @@ typedef enum TOXAV_ERR_BIT_RATE {
324 /** 324 /**
325 * The bit rate passed was not one of the supported values. 325 * The bit rate passed was not one of the supported values.
326 */ 326 */
327 TOXAV_ERR_BIT_RATE_INVALID 327 TOXAV_ERR_BIT_RATE_INVALID,
328 /**
329 * The friend_number passed did not designate a valid friend.
330 */
331 TOXAV_ERR_BIT_RATE_FRIEND_NOT_FOUND,
332 /**
333 * This client is currently not in a call with the friend.
334 */
335 TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL
328} TOXAV_ERR_BIT_RATE; 336} TOXAV_ERR_BIT_RATE;
329/** 337/**
330 * Set the audio bit rate to be used in subsequent audio frames. 338 * Set the audio bit rate to be used in subsequent audio frames.