summaryrefslogtreecommitdiff
path: root/toxav/codec.c
diff options
context:
space:
mode:
Diffstat (limited to 'toxav/codec.c')
-rw-r--r--toxav/codec.c688
1 files changed, 0 insertions, 688 deletions
diff --git a/toxav/codec.c b/toxav/codec.c
deleted file mode 100644
index de802526..00000000
--- a/toxav/codec.c
+++ /dev/null
@@ -1,688 +0,0 @@
1/** codec.c
2 *
3 * Audio and video codec intitialization, encoding/decoding and playback
4 *
5 * Copyright (C) 2013 Tox project All Rights Reserved.
6 *
7 * This file is part of Tox.
8 *
9 * Tox is free software: you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation, either version 3 of the License, or
12 * (at your option) any later version.
13 *
14 * Tox is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with Tox. If not, see <http://www.gnu.org/licenses/>.
21 *
22 */
23
24
25#ifdef HAVE_CONFIG_H
26#include "config.h"
27#endif /* HAVE_CONFIG_H */
28
29#include "../toxcore/logger.h"
30#include "../toxcore/util.h"
31
32#include <stdio.h>
33#include <stdlib.h>
34#include <math.h>
35#include <assert.h>
36#include <time.h>
37
38#include "msi.h"
39#include "rtp.h"
40#include "codec.h"
41
42/* Good quality encode. */
43#define MAX_DECODE_TIME_US 0
44
45// TODO this has to be exchanged in msi
46#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */
47#define VIDEOFRAME_PIECE_SIZE 0x500 /* 1.25 KiB*/
48#define VIDEOFRAME_HEADER_SIZE 0x2
49
50/* FIXME: Might not be enough */
51#define VIDEO_DECODE_BUFFER_SIZE 20
52
53#define ARRAY(TYPE__) struct { uint16_t size; TYPE__ data[]; }
54
55typedef ARRAY(uint8_t) Payload;
56
57typedef struct {
58 uint16_t size; /* Max size */
59 uint16_t start;
60 uint16_t end;
61 Payload **packets;
62} PayloadBuffer;
63
64static _Bool buffer_full(const PayloadBuffer *b)
65{
66 return (b->end + 1) % b->size == b->start;
67}
68
69static _Bool buffer_empty(const PayloadBuffer *b)
70{
71 return b->end == b->start;
72}
73
74static void buffer_write(PayloadBuffer *b, Payload *p)
75{
76 b->packets[b->end] = p;
77 b->end = (b->end + 1) % b->size;
78
79 if (b->end == b->start) b->start = (b->start + 1) % b->size; /* full, overwrite */
80}
81
82static void buffer_read(PayloadBuffer *b, Payload **p)
83{
84 *p = b->packets[b->start];
85 b->start = (b->start + 1) % b->size;
86}
87
88static void buffer_clear(PayloadBuffer *b)
89{
90 while (!buffer_empty(b)) {
91 Payload *p;
92 buffer_read(b, &p);
93 free(p);
94 }
95}
96
97static PayloadBuffer *buffer_new(int size)
98{
99 PayloadBuffer *buf = calloc(sizeof(PayloadBuffer), 1);
100
101 if (!buf) return NULL;
102
103 buf->size = size + 1; /* include empty elem */
104
105 if (!(buf->packets = calloc(buf->size, sizeof(Payload *)))) {
106 free(buf);
107 return NULL;
108 }
109
110 return buf;
111}
112
113static void buffer_free(PayloadBuffer *b)
114{
115 if (b) {
116 buffer_clear(b);
117 free(b->packets);
118 free(b);
119 }
120}
121
122/* JITTER BUFFER WORK */
123typedef struct _JitterBuffer {
124 RTPMessage **queue;
125 uint32_t size;
126 uint32_t capacity;
127 uint16_t bottom;
128 uint16_t top;
129} JitterBuffer;
130
131static JitterBuffer *jbuf_new(uint32_t capacity)
132{
133 unsigned int size = 1;
134
135 while (size <= (capacity * 4)) {
136 size *= 2;
137 }
138
139 JitterBuffer *q;
140
141 if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL;
142
143 if (!(q->queue = calloc(sizeof(RTPMessage *), size))) {
144 free(q);
145 return NULL;
146 }
147
148 q->size = size;
149 q->capacity = capacity;
150 return q;
151}
152
153static void jbuf_clear(JitterBuffer *q)
154{
155 for (; q->bottom != q->top; ++q->bottom) {
156 if (q->queue[q->bottom % q->size]) {
157 rtp_free_msg(NULL, q->queue[q->bottom % q->size]);
158 q->queue[q->bottom % q->size] = NULL;
159 }
160 }
161}
162
163static void jbuf_free(JitterBuffer *q)
164{
165 if (!q) return;
166
167 jbuf_clear(q);
168 free(q->queue);
169 free(q);
170}
171
172static int jbuf_write(JitterBuffer *q, RTPMessage *m)
173{
174 uint16_t sequnum = m->header->sequnum;
175
176 unsigned int num = sequnum % q->size;
177
178 if ((uint32_t)(sequnum - q->bottom) > q->size) {
179 jbuf_clear(q);
180 q->bottom = sequnum - q->capacity;
181 q->queue[num] = m;
182 q->top = sequnum + 1;
183 return 0;
184 }
185
186 if (q->queue[num])
187 return -1;
188
189 q->queue[num] = m;
190
191 if ((sequnum - q->bottom) >= (q->top - q->bottom))
192 q->top = sequnum + 1;
193
194 return 0;
195}
196
197/* Success is 0 when there is nothing to dequeue,
198 * 1 when there's a good packet,
199 * 2 when there's a lost packet */
200static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success)
201{
202 if (q->top == q->bottom) {
203 *success = 0;
204 return NULL;
205 }
206
207 unsigned int num = q->bottom % q->size;
208
209 if (q->queue[num]) {
210 RTPMessage *ret = q->queue[num];
211 q->queue[num] = NULL;
212 ++q->bottom;
213 *success = 1;
214 return ret;
215 }
216
217 if ((uint32_t)(q->top - q->bottom) > q->capacity) {
218 ++q->bottom;
219 *success = 2;
220 return NULL;
221 }
222
223 *success = 0;
224 return NULL;
225}
226
227static int init_video_decoder(CSSession *cs)
228{
229 int rc = vpx_codec_dec_init_ver(&cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION);
230
231 if ( rc != VPX_CODEC_OK) {
232 LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
233 return -1;
234 }
235
236 return 0;
237}
238
239static int init_audio_decoder(CSSession *cs)
240{
241 int rc;
242 cs->audio_decoder = opus_decoder_create(cs->audio_decoder_sample_rate, cs->audio_decoder_channels, &rc );
243
244 if ( rc != OPUS_OK ) {
245 LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc));
246 return -1;
247 }
248
249 return 0;
250}
251
252static int init_video_encoder(CSSession *cs, uint16_t max_width, uint16_t max_height, uint32_t video_bitrate)
253{
254 vpx_codec_enc_cfg_t cfg;
255 int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
256
257 if (rc != VPX_CODEC_OK) {
258 LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
259 return -1;
260 }
261
262 cfg.rc_target_bitrate = video_bitrate;
263 cfg.g_w = max_width;
264 cfg.g_h = max_height;
265 cfg.g_pass = VPX_RC_ONE_PASS;
266 cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
267 cfg.g_lag_in_frames = 0;
268 cfg.kf_min_dist = 0;
269 cfg.kf_max_dist = 48;
270 cfg.kf_mode = VPX_KF_AUTO;
271
272 rc = vpx_codec_enc_init_ver(&cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION);
273
274 if ( rc != VPX_CODEC_OK) {
275 LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
276 return -1;
277 }
278
279 rc = vpx_codec_control(&cs->v_encoder, VP8E_SET_CPUUSED, 8);
280
281 if ( rc != VPX_CODEC_OK) {
282 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
283 return -1;
284 }
285
286 cs->max_width = max_width;
287 cs->max_height = max_height;
288 cs->video_bitrate = video_bitrate;
289
290 return 0;
291}
292
293static int init_audio_encoder(CSSession *cs)
294{
295 int rc = OPUS_OK;
296 cs->audio_encoder = opus_encoder_create(cs->audio_encoder_sample_rate,
297 cs->audio_encoder_channels, OPUS_APPLICATION_AUDIO, &rc);
298
299 if ( rc != OPUS_OK ) {
300 LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc));
301 return -1;
302 }
303
304 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_encoder_bitrate));
305
306 if ( rc != OPUS_OK ) {
307 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
308 return -1;
309 }
310
311 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10));
312
313 if ( rc != OPUS_OK ) {
314 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
315 return -1;
316 }
317
318 return 0;
319}
320
321/* PUBLIC */
322int cs_split_video_payload(CSSession *cs, const uint8_t *payload, uint16_t length)
323{
324 if (!cs || !length || length > cs->max_video_frame_size) {
325 LOGGER_ERROR("Invalid CodecState or video frame size: %u", length);
326 return cs_ErrorSplittingVideoPayload;
327 }
328
329 cs->split_video_frame[0] = cs->frameid_out++;
330 cs->split_video_frame[1] = 0;
331 cs->processing_video_frame = payload;
332 cs->processing_video_frame_size = length;
333
334 return ((length - 1) / cs->video_frame_piece_size) + 1;
335}
336
337const uint8_t *cs_get_split_video_frame(CSSession *cs, uint16_t *size)
338{
339 if (!cs || !size) return NULL;
340
341 if (cs->processing_video_frame_size > cs->video_frame_piece_size) {
342 memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE,
343 cs->processing_video_frame,
344 cs->video_frame_piece_size);
345
346 cs->processing_video_frame += cs->video_frame_piece_size;
347 cs->processing_video_frame_size -= cs->video_frame_piece_size;
348
349 *size = cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE;
350 } else {
351 memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE,
352 cs->processing_video_frame,
353 cs->processing_video_frame_size);
354
355 *size = cs->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE;
356 }
357
358 cs->split_video_frame[1]++;
359
360 return cs->split_video_frame;
361}
362
363void cs_do(CSSession *cs)
364{
365 /* Codec session should always be protected by call mutex so no need to check for cs validity
366 */
367
368 if (!cs) return;
369
370 Payload *p;
371 int rc;
372
373 int success = 0;
374
375 pthread_mutex_lock(cs->queue_mutex);
376 RTPMessage *msg;
377
378 while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) {
379 pthread_mutex_unlock(cs->queue_mutex);
380
381 uint16_t fsize = ((cs->audio_decoder_sample_rate * cs->audio_decoder_frame_duration) / 1000);
382 int16_t tmp[fsize * cs->audio_decoder_channels];
383
384 if (success == 2) {
385 rc = opus_decode(cs->audio_decoder, 0, 0, tmp, fsize, 1);
386 } else {
387 rc = opus_decode(cs->audio_decoder, msg->data, msg->length, tmp, fsize, 0);
388 rtp_free_msg(NULL, msg);
389 }
390
391 if (rc < 0) {
392 LOGGER_WARNING("Decoding error: %s", opus_strerror(rc));
393 } else if (cs->acb.first) {
394 /* Play */
395 cs->acb.first(cs->agent, cs->call_idx, tmp, rc, cs->acb.second);
396 }
397
398 pthread_mutex_lock(cs->queue_mutex);
399 }
400
401 if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) {
402 /* Decode video */
403 buffer_read(cs->vbuf_raw, &p);
404
405 /* Leave space for (possibly) other thread to queue more data after we read it here */
406 pthread_mutex_unlock(cs->queue_mutex);
407
408 rc = vpx_codec_decode(&cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US);
409 free(p);
410
411 if (rc != VPX_CODEC_OK) {
412 LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc));
413 } else {
414 vpx_codec_iter_t iter = NULL;
415 vpx_image_t *dest = vpx_codec_get_frame(&cs->v_decoder, &iter);
416
417 /* Play decoded images */
418 for (; dest; dest = vpx_codec_get_frame(&cs->v_decoder, &iter)) {
419 if (cs->vcb.first)
420 cs->vcb.first(cs->agent, cs->call_idx, dest, cs->vcb.second);
421
422 vpx_img_free(dest);
423 }
424 }
425
426 return;
427 }
428
429 pthread_mutex_unlock(cs->queue_mutex);
430}
431
432int cs_set_video_encoder_resolution(CSSession *cs, uint16_t width, uint16_t height)
433{
434 vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc;
435
436 if (cfg.g_w == width && cfg.g_h == height)
437 return 0;
438
439 if (width * height > cs->max_width * cs->max_height) {
440 vpx_codec_ctx_t v_encoder = cs->v_encoder;
441
442 if (init_video_encoder(cs, width, height, cs->video_bitrate) == -1) {
443 cs->v_encoder = v_encoder;
444 return cs_ErrorSettingVideoResolution;
445 }
446
447 vpx_codec_destroy(&v_encoder);
448 return 0;
449 }
450
451 LOGGER_DEBUG("New video resolution: %u %u", width, height);
452 cfg.g_w = width;
453 cfg.g_h = height;
454 int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg);
455
456 if ( rc != VPX_CODEC_OK) {
457 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
458 return cs_ErrorSettingVideoResolution;
459 }
460
461 return 0;
462}
463
464int cs_set_video_encoder_bitrate(CSSession *cs, uint32_t video_bitrate)
465{
466 vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc;
467
468 if (cfg.rc_target_bitrate == video_bitrate)
469 return 0;
470
471 LOGGER_DEBUG("New video bitrate: %u", video_bitrate);
472 cfg.rc_target_bitrate = video_bitrate;
473 int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg);
474
475 if ( rc != VPX_CODEC_OK) {
476 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
477 return cs_ErrorSettingVideoBitrate;
478 }
479
480 cs->video_bitrate = video_bitrate;
481 return 0;
482}
483
484CSSession *cs_new(const ToxAvCSettings *cs_self, const ToxAvCSettings *cs_peer, uint32_t jbuf_size, int has_video)
485{
486 CSSession *cs = calloc(sizeof(CSSession), 1);
487
488 if (!cs) {
489 LOGGER_WARNING("Allocation failed! Application might misbehave!");
490 return NULL;
491 }
492
493 if (create_recursive_mutex(cs->queue_mutex) != 0) {
494 LOGGER_WARNING("Failed to create recursive mutex!");
495 free(cs);
496 return NULL;
497 }
498
499 if ( !(cs->j_buf = jbuf_new(jbuf_size)) ) {
500 LOGGER_WARNING("Jitter buffer creaton failed!");
501 goto error;
502 }
503
504 cs->audio_encoder_bitrate = cs_self->audio_bitrate;
505 cs->audio_encoder_sample_rate = cs_self->audio_sample_rate;
506 cs->audio_encoder_channels = cs_self->audio_channels;
507 cs->audio_encoder_frame_duration = cs_self->audio_frame_duration;
508
509 cs->audio_decoder_bitrate = cs_peer->audio_bitrate;
510 cs->audio_decoder_sample_rate = cs_peer->audio_sample_rate;
511 cs->audio_decoder_channels = cs_peer->audio_channels;
512 cs->audio_decoder_frame_duration = cs_peer->audio_frame_duration;
513
514
515 cs->capabilities |= ( 0 == init_audio_encoder(cs) ) ? cs_AudioEncoding : 0;
516 cs->capabilities |= ( 0 == init_audio_decoder(cs) ) ? cs_AudioDecoding : 0;
517
518 if ( !(cs->capabilities & cs_AudioEncoding) || !(cs->capabilities & cs_AudioDecoding) ) goto error;
519
520 if ((cs->support_video = has_video)) {
521 cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE;
522 cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE;
523
524 cs->capabilities |= ( 0 == init_video_encoder(cs, cs_self->max_video_width,
525 cs_self->max_video_height, cs_self->video_bitrate) ) ? cs_VideoEncoding : 0;
526 cs->capabilities |= ( 0 == init_video_decoder(cs) ) ? cs_VideoDecoding : 0;
527
528 if ( !(cs->capabilities & cs_VideoEncoding) || !(cs->capabilities & cs_VideoDecoding) ) goto error;
529
530 if ( !(cs->frame_buf = calloc(cs->max_video_frame_size, 1)) ) goto error;
531
532 if ( !(cs->split_video_frame = calloc(cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE, 1)) )
533 goto error;
534
535 if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) goto error;
536 }
537
538 return cs;
539
540error:
541 LOGGER_WARNING("Error initializing codec session! Application might misbehave!");
542
543 pthread_mutex_destroy(cs->queue_mutex);
544
545 if ( cs->audio_encoder ) opus_encoder_destroy(cs->audio_encoder);
546
547 if ( cs->audio_decoder ) opus_decoder_destroy(cs->audio_decoder);
548
549
550 if (has_video) {
551 if ( cs->capabilities & cs_VideoDecoding ) vpx_codec_destroy(&cs->v_decoder);
552
553 if ( cs->capabilities & cs_VideoEncoding ) vpx_codec_destroy(&cs->v_encoder);
554
555 buffer_free(cs->vbuf_raw);
556
557 free(cs->frame_buf);
558 free(cs->split_video_frame);
559 }
560
561 jbuf_free(cs->j_buf);
562 free(cs);
563
564 return NULL;
565}
566
567void cs_kill(CSSession *cs)
568{
569 if (!cs) return;
570
571 /* queue_message will not be called since it's unregistered before cs_kill is called */
572 pthread_mutex_destroy(cs->queue_mutex);
573
574
575 if ( cs->audio_encoder )
576 opus_encoder_destroy(cs->audio_encoder);
577
578 if ( cs->audio_decoder )
579 opus_decoder_destroy(cs->audio_decoder);
580
581 if ( cs->capabilities & cs_VideoDecoding )
582 vpx_codec_destroy(&cs->v_decoder);
583
584 if ( cs->capabilities & cs_VideoEncoding )
585 vpx_codec_destroy(&cs->v_encoder);
586
587 jbuf_free(cs->j_buf);
588 buffer_free(cs->vbuf_raw);
589 free(cs->frame_buf);
590 free(cs->split_video_frame);
591
592 LOGGER_DEBUG("Terminated codec state: %p", cs);
593 free(cs);
594}
595
596
597
598
599/* Called from RTP */
600void queue_message(RTPSession *session, RTPMessage *msg)
601{
602 /* This function is unregistered during call termination befor destroing
603 * Codec session so no need to check for validity of cs
604 */
605 CSSession *cs = session->cs;
606
607 if (!cs) return;
608
609 /* Audio */
610 if (session->payload_type == msi_TypeAudio % 128) {
611 pthread_mutex_lock(cs->queue_mutex);
612 int ret = jbuf_write(cs->j_buf, msg);
613 pthread_mutex_unlock(cs->queue_mutex);
614
615 if (ret == -1) {
616 rtp_free_msg(NULL, msg);
617 }
618 }
619 /* Video */
620 else {
621 uint8_t *packet = msg->data;
622 uint32_t packet_size = msg->length;
623
624 if (packet_size < VIDEOFRAME_HEADER_SIZE)
625 goto end;
626
627 uint8_t diff = packet[0] - cs->frameid_in;
628
629 if (diff != 0) {
630 if (diff < 225) { /* New frame */
631 /* Flush last frames' data and get ready for this frame */
632 Payload *p = malloc(sizeof(Payload) + cs->frame_size);
633
634 if (p) {
635 pthread_mutex_lock(cs->queue_mutex);
636
637 if (buffer_full(cs->vbuf_raw)) {
638 LOGGER_DEBUG("Dropped video frame");
639 Payload *tp;
640 buffer_read(cs->vbuf_raw, &tp);
641 free(tp);
642 } else {
643 p->size = cs->frame_size;
644 memcpy(p->data, cs->frame_buf, cs->frame_size);
645 }
646
647 buffer_write(cs->vbuf_raw, p);
648 pthread_mutex_unlock(cs->queue_mutex);
649 } else {
650 LOGGER_WARNING("Allocation failed! Program might misbehave!");
651 goto end;
652 }
653
654 cs->last_timestamp = msg->header->timestamp;
655 cs->frameid_in = packet[0];
656 memset(cs->frame_buf, 0, cs->frame_size);
657 cs->frame_size = 0;
658
659 } else { /* Old frame; drop */
660 LOGGER_DEBUG("Old packet: %u", packet[0]);
661 goto end;
662 }
663 }
664
665 uint8_t piece_number = packet[1];
666
667 uint32_t length_before_piece = ((piece_number - 1) * cs->video_frame_piece_size);
668 uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE);
669
670 if (framebuf_new_length > cs->max_video_frame_size) {
671 goto end;
672 }
673
674 /* Otherwise it's part of the frame so just process */
675 /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */
676
677 memcpy(cs->frame_buf + length_before_piece,
678 packet + VIDEOFRAME_HEADER_SIZE,
679 packet_size - VIDEOFRAME_HEADER_SIZE);
680
681 if (framebuf_new_length > cs->frame_size) {
682 cs->frame_size = framebuf_new_length;
683 }
684
685end:
686 rtp_free_msg(NULL, msg);
687 }
688}
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-13 19:06:22 +0000