summaryrefslogtreecommitdiff
path: root/toxav/codec.c
diff options
context:
space:
mode:
Diffstat (limited to 'toxav/codec.c')
-rw-r--r--toxav/codec.c660
1 files changed, 524 insertions, 136 deletions
diff --git a/toxav/codec.c b/toxav/codec.c
index 2f00319f..46fd6ba7 100644
--- a/toxav/codec.c
+++ b/toxav/codec.c
@@ -32,33 +32,124 @@
32#include <stdlib.h> 32#include <stdlib.h>
33#include <math.h> 33#include <math.h>
34#include <assert.h> 34#include <assert.h>
35#include <time.h>
35 36
37#include "msi.h"
36#include "rtp.h" 38#include "rtp.h"
37#include "codec.h" 39#include "codec.h"
38 40
39JitterBuffer *create_queue(unsigned int capacity) 41/* Assume 24 fps*/
42#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000)
43#define MAX_DECODE_TIME_US 0
44
45// TODO this has to be exchanged in msi
46#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */
47#define VIDEOFRAME_PIECE_SIZE 0x500 /* 1.25 KiB*/
48#define VIDEOFRAME_HEADER_SIZE 0x2
49
50/* FIXME: Might not be enough */
51#define VIDEO_DECODE_BUFFER_SIZE 20
52
53#define ARRAY(TYPE__) struct { uint16_t size; TYPE__ data[]; }
54#define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; }
55
56typedef ARRAY(uint8_t) Payload;
57
58static PAIR(CSVideoCallback, void*) vpcallback;
59static PAIR(CSAudioCallback, void*) apcallback;
60
61typedef struct {
62 uint16_t size; /* Max size */
63 uint16_t start;
64 uint16_t end;
65 Payload **packets;
66} PayloadBuffer;
67
68static _Bool buffer_full(const PayloadBuffer *b)
40{ 69{
41 unsigned int size = 1; 70 return (b->end + 1) % b->size == b->start;
71}
72
73static _Bool buffer_empty(const PayloadBuffer *b)
74{
75 return b->end == b->start;
76}
77
78static void buffer_write(PayloadBuffer *b, Payload *p)
79{
80 b->packets[b->end] = p;
81 b->end = (b->end + 1) % b->size;
82 if (b->end == b->start) b->start = (b->start + 1) % b->size; /* full, overwrite */
83}
84
85static void buffer_read(PayloadBuffer *b, Payload **p)
86{
87 *p = b->packets[b->start];
88 b->start = (b->start + 1) % b->size;
89}
90
91static void buffer_clear(PayloadBuffer *b)
92{
93 while (!buffer_empty(b)) {
94 Payload* p;
95 buffer_read(b, &p);
96 free(p);
97 }
98}
99
100static PayloadBuffer* buffer_new(int size)
101{
102 PayloadBuffer *buf = calloc(sizeof(PayloadBuffer), 1);
103 if (!buf) return NULL;
104
105 buf->size = size + 1; /* include empty elem */
106 if (!(buf->packets = calloc(buf->size, sizeof(Payload*)))) {
107 free(buf);
108 return NULL;
109 }
110 return buf;
111}
42 112
113static void buffer_free(PayloadBuffer *b)
114{
115 if (b) {
116 buffer_clear(b);
117 free(b->packets);
118 free(b);
119 }
120}
121
122/* JITTER BUFFER WORK */
123typedef struct _JitterBuffer {
124 RTPMessage **queue;
125 uint32_t size;
126 uint32_t capacity;
127 uint16_t bottom;
128 uint16_t top;
129} JitterBuffer;
130
131static JitterBuffer *jbuf_new(uint32_t capacity)
132{
133 unsigned int size = 1;
43 while (size <= capacity) { 134 while (size <= capacity) {
44 size *= 2; 135 size *= 2;
45 } 136 }
46 137
47 JitterBuffer *q; 138 JitterBuffer *q;
48 139
49 if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL; 140 if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL;
50 141
51 if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { 142 if (!(q->queue = calloc(sizeof(RTPMessage *), size))) {
52 free(q); 143 free(q);
53 return NULL; 144 return NULL;
54 } 145 }
55 146
56 q->size = size; 147 q->size = size;
57 q->capacity = capacity; 148 q->capacity = capacity;
58 return q; 149 return q;
59} 150}
60 151
61static void clear_queue(JitterBuffer *q) 152static void jbuf_clear(JitterBuffer *q)
62{ 153{
63 for (; q->bottom != q->top; ++q->bottom) { 154 for (; q->bottom != q->top; ++q->bottom) {
64 if (q->queue[q->bottom % q->size]) { 155 if (q->queue[q->bottom % q->size]) {
@@ -68,48 +159,50 @@ static void clear_queue(JitterBuffer *q)
68 } 159 }
69} 160}
70 161
71void terminate_queue(JitterBuffer *q) 162static void jbuf_free(JitterBuffer *q)
72{ 163{
73 if (!q) return; 164 if (!q) return;
74 165
75 clear_queue(q); 166 jbuf_clear(q);
76 free(q->queue); 167 free(q->queue);
77 free(q); 168 free(q);
78} 169}
79 170
80void queue(JitterBuffer *q, RTPMessage *pk) 171static void jbuf_write(JitterBuffer *q, RTPMessage *m)
81{ 172{
82 uint16_t sequnum = pk->header->sequnum; 173 uint16_t sequnum = m->header->sequnum;
83 174
84 unsigned int num = sequnum % q->size; 175 unsigned int num = sequnum % q->size;
85 176
86 if ((uint32_t)(sequnum - q->bottom) > q->size) { 177 if ((uint32_t)(sequnum - q->bottom) > q->size) {
87 clear_queue(q); 178 jbuf_clear(q);
88 q->bottom = sequnum; 179 q->bottom = sequnum;
89 q->queue[num] = pk; 180 q->queue[num] = m;
90 q->top = sequnum + 1; 181 q->top = sequnum + 1;
91 return; 182 return;
92 } 183 }
93 184
94 if (q->queue[num]) 185 if (q->queue[num])
95 return; 186 return;
96 187
97 q->queue[num] = pk; 188 q->queue[num] = m;
98 189
99 if ((sequnum - q->bottom) >= (q->top - q->bottom)) 190 if ((sequnum - q->bottom) >= (q->top - q->bottom))
100 q->top = sequnum + 1; 191 q->top = sequnum + 1;
101} 192}
102 193
103/* success is 0 when there is nothing to dequeue, 1 when there's a good packet, 2 when there's a lost packet */ 194/* Success is 0 when there is nothing to dequeue,
104RTPMessage *dequeue(JitterBuffer *q, int *success) 195 * 1 when there's a good packet,
196 * 2 when there's a lost packet */
197static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success)
105{ 198{
106 if (q->top == q->bottom) { 199 if (q->top == q->bottom) {
107 *success = 0; 200 *success = 0;
108 return NULL; 201 return NULL;
109 } 202 }
110 203
111 unsigned int num = q->bottom % q->size; 204 unsigned int num = q->bottom % q->size;
112 205
113 if (q->queue[num]) { 206 if (q->queue[num]) {
114 RTPMessage *ret = q->queue[num]; 207 RTPMessage *ret = q->queue[num];
115 q->queue[num] = NULL; 208 q->queue[num] = NULL;
@@ -117,19 +210,18 @@ RTPMessage *dequeue(JitterBuffer *q, int *success)
117 *success = 1; 210 *success = 1;
118 return ret; 211 return ret;
119 } 212 }
120 213
121 if ((uint32_t)(q->top - q->bottom) > q->capacity) { 214 if ((uint32_t)(q->top - q->bottom) > q->capacity) {
122 ++q->bottom; 215 ++q->bottom;
123 *success = 2; 216 *success = 2;
124 return NULL; 217 return NULL;
125 } 218 }
126 219
127 *success = 0; 220 *success = 0;
128 return NULL; 221 return NULL;
129} 222}
130 223
131 224static int init_video_decoder(CSSession *cs)
132int init_video_decoder(CodecState *cs)
133{ 225{
134 int rc = vpx_codec_dec_init_ver(&cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION); 226 int rc = vpx_codec_dec_init_ver(&cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION);
135 227
@@ -141,72 +233,28 @@ int init_video_decoder(CodecState *cs)
141 return 0; 233 return 0;
142} 234}
143 235
144int init_audio_decoder(CodecState *cs, uint32_t audio_channels) 236static int init_audio_decoder(CSSession *cs)
145{ 237{
146 int rc; 238 int rc;
147 cs->audio_decoder = opus_decoder_create(cs->audio_sample_rate, audio_channels, &rc ); 239 cs->audio_decoder = opus_decoder_create(cs->audio_decoder_sample_rate, cs->audio_decoder_channels, &rc );
148 240
149 if ( rc != OPUS_OK ) { 241 if ( rc != OPUS_OK ) {
150 LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); 242 LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc));
151 return -1; 243 return -1;
152 } 244 }
153
154 cs->audio_decoder_channels = audio_channels;
155 return 0; 245 return 0;
156} 246}
157 247
158int reconfigure_video_encoder_resolution(CodecState *cs, uint16_t width, uint16_t height) 248static int init_video_encoder(CSSession *cs, uint16_t max_width, uint16_t max_height, uint32_t video_bitrate)
159{
160 vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc;
161
162 if (cfg.g_w == width && cfg.g_h == height)
163 return 0;
164
165 if (width * height > cs->max_width * cs->max_height)
166 return -1;
167
168 LOGGER_DEBUG("New video resolution: %u %u", width, height);
169 cfg.g_w = width;
170 cfg.g_h = height;
171 int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg);
172
173 if ( rc != VPX_CODEC_OK) {
174 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
175 return -1;
176 }
177
178 return 0;
179}
180
181int reconfigure_video_encoder_bitrate(CodecState *cs, uint32_t video_bitrate)
182{
183 vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc;
184
185 if (cfg.rc_target_bitrate == video_bitrate)
186 return 0;
187
188 LOGGER_DEBUG("New video bitrate: %u", video_bitrate);
189 cfg.rc_target_bitrate = video_bitrate;
190 int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg);
191
192 if ( rc != VPX_CODEC_OK) {
193 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
194 return -1;
195 }
196
197 return 0;
198}
199
200int init_video_encoder(CodecState *cs, uint16_t max_width, uint16_t max_height, uint32_t video_bitrate)
201{ 249{
202 vpx_codec_enc_cfg_t cfg; 250 vpx_codec_enc_cfg_t cfg;
203 int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); 251 int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
204 252
205 if (rc != VPX_CODEC_OK) { 253 if (rc != VPX_CODEC_OK) {
206 LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); 254 LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
207 return -1; 255 return -1;
208 } 256 }
209 257
210 cfg.rc_target_bitrate = video_bitrate; 258 cfg.rc_target_bitrate = video_bitrate;
211 cfg.g_w = max_width; 259 cfg.g_w = max_width;
212 cfg.g_h = max_height; 260 cfg.g_h = max_height;
@@ -216,130 +264,375 @@ int init_video_encoder(CodecState *cs, uint16_t max_width, uint16_t max_height,
216 cfg.kf_min_dist = 0; 264 cfg.kf_min_dist = 0;
217 cfg.kf_max_dist = 5; 265 cfg.kf_max_dist = 5;
218 cfg.kf_mode = VPX_KF_AUTO; 266 cfg.kf_mode = VPX_KF_AUTO;
219 267
220 cs->max_width = max_width; 268 cs->max_width = max_width;
221 cs->max_height = max_height; 269 cs->max_height = max_height;
222 cs->bitrate = video_bitrate; 270
223
224 rc = vpx_codec_enc_init_ver(&cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION); 271 rc = vpx_codec_enc_init_ver(&cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION);
225 272
226 if ( rc != VPX_CODEC_OK) { 273 if ( rc != VPX_CODEC_OK) {
227 LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); 274 LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
228 return -1; 275 return -1;
229 } 276 }
230 277
231 rc = vpx_codec_control(&cs->v_encoder, VP8E_SET_CPUUSED, 8); 278 rc = vpx_codec_control(&cs->v_encoder, VP8E_SET_CPUUSED, 8);
232 279
233 if ( rc != VPX_CODEC_OK) { 280 if ( rc != VPX_CODEC_OK) {
234 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); 281 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
235 return -1; 282 return -1;
236 } 283 }
237 284
238 return 0; 285 return 0;
239} 286}
240 287
241int init_audio_encoder(CodecState *cs, uint32_t audio_channels) 288static int init_audio_encoder(CSSession *cs)
242{ 289{
243 int rc = OPUS_OK; 290 int rc = OPUS_OK;
244 cs->audio_encoder = opus_encoder_create(cs->audio_sample_rate, audio_channels, OPUS_APPLICATION_AUDIO, &rc); 291 cs->audio_encoder = opus_encoder_create(cs->audio_encoder_sample_rate,
245 292 cs->audio_encoder_channels, OPUS_APPLICATION_AUDIO, &rc);
293
246 if ( rc != OPUS_OK ) { 294 if ( rc != OPUS_OK ) {
247 LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); 295 LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc));
248 return -1; 296 return -1;
249 } 297 }
250 298
251 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_bitrate)); 299 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_encoder_bitrate));
252 300
253 if ( rc != OPUS_OK ) { 301 if ( rc != OPUS_OK ) {
254 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); 302 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
255 return -1; 303 return -1;
256 } 304 }
257 305
258 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); 306 rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10));
259 307
260 if ( rc != OPUS_OK ) { 308 if ( rc != OPUS_OK ) {
261 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); 309 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc));
262 return -1; 310 return -1;
263 } 311 }
264
265 cs->audio_encoder_channels = audio_channels;
266 return 0; 312 return 0;
267} 313}
268 314
269 315static float calculate_sum_sq (int16_t *n, uint16_t k)
270CodecState *codec_init_session ( uint32_t audio_bitrate,
271 uint16_t audio_frame_duration,
272 uint32_t audio_sample_rate,
273 uint32_t encoder_audio_channels,
274 uint32_t decoder_audio_channels,
275 uint32_t audio_VAD_tolerance_ms,
276 uint16_t max_video_width,
277 uint16_t max_video_height,
278 uint32_t video_bitrate )
279{ 316{
280 CodecState *retu = calloc(sizeof(CodecState), 1); 317 float result = 0;
318 uint16_t i = 0;
319
320 for ( ; i < k; i ++) result += (float) (n[i] * n[i]);
321
322 return result;
323}
281 324
282 if (!retu) return NULL;
283 325
284 retu->audio_bitrate = audio_bitrate;
285 retu->audio_sample_rate = audio_sample_rate;
286 326
287 /* Encoders */ 327/* PUBLIC */
288 if (!max_video_width || !max_video_height) { /* Disable video */ 328int cs_split_video_payload(CSSession* cs, const uint8_t* payload, uint16_t length)
289 /*video_width = 320; 329{
290 video_height = 240; */ 330 if (!cs || !length || length > cs->max_video_frame_size) {
331 LOGGER_ERROR("Invalid CodecState or video frame size: %u", length);
332 return -1;
333 }
334
335 cs->split_video_frame[0] = cs->frameid_out++;
336 cs->split_video_frame[1] = 0;
337 cs->processing_video_frame = payload;
338 cs->processing_video_frame_size = length;
339
340 return ((length - 1) / cs->video_frame_piece_size) + 1;
341}
342
343const uint8_t* cs_get_split_video_frame(CSSession* cs, uint16_t* size)
344{
345 if (!cs || !size) return NULL;
346
347 if (cs->processing_video_frame_size > cs->video_frame_piece_size) {
348 memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE,
349 cs->processing_video_frame,
350 cs->video_frame_piece_size);
351
352 cs->processing_video_frame += cs->video_frame_piece_size;
353 cs->processing_video_frame_size -= cs->video_frame_piece_size;
354
355 *size = cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE;
291 } else { 356 } else {
292 retu->capabilities |= ( 0 == init_video_encoder(retu, max_video_width, max_video_height, 357 memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE,
293 video_bitrate) ) ? v_encoding : 0; 358 cs->processing_video_frame,
294 retu->capabilities |= ( 0 == init_video_decoder(retu) ) ? v_decoding : 0; 359 cs->processing_video_frame_size);
360
361 *size = cs->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE;
362 }
363
364 cs->split_video_frame[1]++;
365
366 return cs->split_video_frame;
367}
368
369void cs_do(CSSession* cs)
370{
371 if (!cs) return;
372
373 pthread_mutex_lock(cs->queue_mutex);
374 /*
375 if (!cs->active) {
376 pthread_mutex_unlock(cs->queue_mutex);
377 return;
378 }
379
380 /* Iterate over whole buffers and call playback callback * /
381 if (cs->abuf_ready) while (!DecodedAudioBuffer_empty(cs->abuf_ready)) {
382 DecodedAudio* p;
383 DecodedAudioBuffer_read(cs->abuf_ready, &p);
384 if (apcallback.first)
385 apcallback.first(cs->agent, cs->call_idx, p->data, p->size, apcallback.second);
386
387 free(p);
388 }
389
390 if (cs->vbuf_ready) while (!DecodedVideoBuffer_empty(cs->vbuf_ready)) {
391 vpx_image_t* p;
392 DecodedVideoBuffer_read(cs->vbuf_ready, &p);
393 if (vpcallback.first)
394 vpcallback.first(cs->agent, cs->call_idx, p, vpcallback.second);
395
396 vpx_img_free(p);
397 }
398 */
399
400 Payload* p;
401 int rc;
402
403 if (cs->abuf_raw && !buffer_empty(cs->abuf_raw)) {
404 /* Decode audio */
405 buffer_read(cs->abuf_raw, &p);
406
407 uint16_t fsize = (cs->audio_decoder_channels *
408 (cs->audio_decoder_sample_rate * cs->audio_decoder_frame_duration) / 1000);
409 int16_t tmp[fsize];
410
411 rc = opus_decode(cs->audio_decoder, p->data, p->size, tmp, fsize, (p->size == 0));
412 free(p);
413
414 if (rc < 0)
415 LOGGER_WARNING("Decoding error: %s", opus_strerror(rc));
416 else
417 /* Play */
418 apcallback.first(cs->agent, cs->call_idx, tmp, rc, apcallback.second);
419 }
420
421 if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) {
422 /* Decode video */
423 buffer_read(cs->vbuf_raw, &p);
424
425 rc = vpx_codec_decode(&cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US);
426 free(p);
427
428 if (rc != VPX_CODEC_OK) {
429 LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc));
430 } else {
431 vpx_codec_iter_t iter = NULL;
432 vpx_image_t* dest = vpx_codec_get_frame(&cs->v_decoder, &iter);
433
434 /* Play decoded images */
435 for (; dest; dest = vpx_codec_get_frame(&cs->v_decoder, &iter)) {
436 vpcallback.first(cs->agent, cs->call_idx, dest, vpcallback.second);
437 vpx_img_free(dest);
438 }
439 }
295 } 440 }
441
442 pthread_mutex_unlock(cs->queue_mutex);
443}
296 444
297 retu->capabilities |= ( 0 == init_audio_encoder(retu, encoder_audio_channels) ) ? a_encoding : 0; 445void cs_register_audio_callback(CSAudioCallback cb, void* data)
298 retu->capabilities |= ( 0 == init_audio_decoder(retu, decoder_audio_channels) ) ? a_decoding : 0; 446{
447 apcallback.first = cb;
448 apcallback.second = data;
449}
299 450
300 if ( retu->capabilities == 0 ) { /* everything failed */ 451void cs_register_video_callback(CSVideoCallback cb, void* data)
301 free (retu); 452{
302 return NULL; 453 vpcallback.first = cb;
454 vpcallback.second = data;
455}
456
457int cs_set_video_encoder_resolution(CSSession *cs, uint16_t width, uint16_t height)
458{
459 vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc;
460
461 if (cfg.g_w == width && cfg.g_h == height)
462 return 0;
463
464 if (width * height > cs->max_width * cs->max_height)
465 return -1;
466
467 LOGGER_DEBUG("New video resolution: %u %u", width, height);
468 cfg.g_w = width;
469 cfg.g_h = height;
470 int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg);
471
472 if ( rc != VPX_CODEC_OK) {
473 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
474 return -1;
303 } 475 }
304 476
477 return 0;
478}
305 479
306 retu->EVAD_tolerance = audio_VAD_tolerance_ms > audio_frame_duration ? 480int cs_set_video_encoder_bitrate(CSSession *cs, uint32_t video_bitrate)
307 audio_VAD_tolerance_ms / audio_frame_duration : audio_frame_duration; 481{
482 vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc;
483
484 if (cfg.rc_target_bitrate == video_bitrate)
485 return 0;
308 486
309 return retu; 487 LOGGER_DEBUG("New video bitrate: %u", video_bitrate);
488 cfg.rc_target_bitrate = video_bitrate;
489 int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg);
490
491 if ( rc != VPX_CODEC_OK) {
492 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
493 return -1;
494 }
495
496 return 0;
497}
498
499CSSession *cs_new(const ToxAvCSettings *cs_self, const ToxAvCSettings *cs_peer, uint32_t jbuf_size, int has_video)
500{
501 CSSession *cs = calloc(sizeof(CSSession), 1);
502
503 if (!cs) {
504 LOGGER_WARNING("Allocation failed! Application might misbehave!");
505 return NULL;
506 }
507
508 if ( !(cs->j_buf = jbuf_new(jbuf_size)) ) {
509 LOGGER_WARNING("Jitter buffer creaton failed!");
510 goto error;
511 }
512
513 cs->audio_encoder_bitrate = cs_self->audio_bitrate;
514 cs->audio_encoder_sample_rate = cs_self->audio_sample_rate;
515 cs->audio_encoder_channels = cs_self->audio_channels;
516 cs->audio_encoder_frame_duration = cs_self->audio_frame_duration;
517
518 cs->audio_decoder_bitrate = cs_peer->audio_bitrate;
519 cs->audio_decoder_sample_rate = cs_peer->audio_sample_rate;
520 cs->audio_decoder_channels = cs_peer->audio_channels;
521 cs->audio_decoder_frame_duration = cs_peer->audio_frame_duration;
522
523
524 cs->capabilities |= ( 0 == init_audio_encoder(cs) ) ? a_encoding : 0;
525 cs->capabilities |= ( 0 == init_audio_decoder(cs) ) ? a_decoding : 0;
526
527 if ( !(cs->capabilities & a_encoding) || !(cs->capabilities & a_decoding) ) goto error;
528
529 if ( !(cs->abuf_raw = buffer_new(jbuf_size)) ) goto error;
530
531 pthread_mutexattr_t attr;
532 if (pthread_mutexattr_init(&attr) != 0) goto error;
533 if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0) {
534 if (pthread_mutexattr_destroy(&attr) != 0)
535 LOGGER_WARNING("Failed to destroy mutex attribute!");
536 goto error;
537 }
538
539 if (pthread_mutex_init(cs->queue_mutex, &attr) != 0) {
540 pthread_mutexattr_destroy(&attr);
541 goto error;
542 }
543
544 if ((cs->support_video = has_video)) {
545 cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE;
546 cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE;
547
548 cs->capabilities |= ( 0 == init_video_encoder(cs, cs_self->max_video_width,
549 cs_self->max_video_height, cs_self->video_bitrate) ) ? v_encoding : 0;
550 cs->capabilities |= ( 0 == init_video_decoder(cs) ) ? v_decoding : 0;
551
552 if ( !(cs->capabilities & v_encoding) || !(cs->capabilities & v_decoding) ) goto error;
553
554 if ( !(cs->frame_buf = calloc(cs->max_video_frame_size, 1)) ) goto error;
555 if ( !(cs->split_video_frame = calloc(cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE, 1)) )
556 goto error;
557
558 if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) goto error;
559 }
560
561 if (pthread_mutexattr_destroy(&attr) != 0)
562 LOGGER_WARNING("Failed to destroy mutex attribute!");
563
564
565 cs->active = 1;
566 return cs;
567
568error:
569 LOGGER_WARNING("Error initializing codec session! Application might misbehave!");
570
571 buffer_free(cs->abuf_raw);
572
573 if ( cs->audio_encoder ) opus_encoder_destroy(cs->audio_encoder);
574 if ( cs->audio_decoder ) opus_decoder_destroy(cs->audio_decoder);
575
576
577 if(has_video) {
578 if ( cs->capabilities & v_decoding ) vpx_codec_destroy(&cs->v_decoder);
579 if ( cs->capabilities & v_encoding ) vpx_codec_destroy(&cs->v_encoder);
580
581 buffer_free(cs->vbuf_raw);
582
583 free(cs->frame_buf);
584 free(cs->split_video_frame);
585 }
586
587 jbuf_free(cs->j_buf);
588 free(cs);
589
590 return NULL;
310} 591}
311 592
312void codec_terminate_session ( CodecState *cs ) 593void cs_kill(CSSession *cs)
313{ 594{
314 if (!cs) return; 595 if (!cs) return;
315 596
597 /* Lock running mutex and signal that cs is no longer active */
598 pthread_mutex_lock(cs->queue_mutex);
599 cs->active = 0;
600
601 /* Wait threads to close */
602 pthread_mutex_unlock(cs->queue_mutex);
603 pthread_mutex_lock(cs->queue_mutex);
604 pthread_mutex_unlock(cs->queue_mutex);
605
606 pthread_mutex_destroy(cs->queue_mutex);
607
608
316 if ( cs->audio_encoder ) 609 if ( cs->audio_encoder )
317 opus_encoder_destroy(cs->audio_encoder); 610 opus_encoder_destroy(cs->audio_encoder);
318 611
319 if ( cs->audio_decoder ) 612 if ( cs->audio_decoder )
320 opus_decoder_destroy(cs->audio_decoder); 613 opus_decoder_destroy(cs->audio_decoder);
321 614
322 if ( cs->capabilities & v_decoding ) 615 if ( cs->capabilities & v_decoding )
323 vpx_codec_destroy(&cs->v_decoder); 616 vpx_codec_destroy(&cs->v_decoder);
324 617
325 if ( cs->capabilities & v_encoding ) 618 if ( cs->capabilities & v_encoding )
326 vpx_codec_destroy(&cs->v_encoder); 619 vpx_codec_destroy(&cs->v_encoder);
327 620
621 jbuf_free(cs->j_buf);
622 buffer_free(cs->abuf_raw);
623 buffer_free(cs->vbuf_raw);
624 free(cs->frame_buf);
625
328 LOGGER_DEBUG("Terminated codec state: %p", cs); 626 LOGGER_DEBUG("Terminated codec state: %p", cs);
329 free(cs); 627 free(cs);
330} 628}
331 629
332static float calculate_sum_sq (int16_t *n, uint16_t k) 630void cs_set_vad_treshold(CSSession* cs, uint32_t treshold, uint16_t frame_duration)
333{ 631{
334 float result = 0; 632 cs->EVAD_tolerance = treshold > frame_duration ? treshold / frame_duration : frame_duration;
335 uint16_t i = 0;
336
337 for ( ; i < k; i ++) result += (float) (n[i] * n[i]);
338
339 return result;
340} 633}
341 634
342int energy_VAD(CodecState *cs, int16_t *PCM, uint16_t frame_size, float energy) 635int cs_calculate_vad(CSSession *cs, int16_t *PCM, uint16_t frame_size, float energy)
343{ 636{
344 float frame_energy = sqrt(calculate_sum_sq(PCM, frame_size)) / frame_size; 637 float frame_energy = sqrt(calculate_sum_sq(PCM, frame_size)) / frame_size;
345 638
@@ -355,3 +648,98 @@ int energy_VAD(CodecState *cs, int16_t *PCM, uint16_t frame_size, float energy)
355 648
356 return 0; 649 return 0;
357} 650}
651
652
653
654
655/* Called from RTP */
656void queue_message(RTPSession *session, RTPMessage *msg)
657{
658 CSSession *cs = session->cs;
659 if (!cs || !cs->active) return;
660
661 /* Audio */
662 if (session->payload_type == type_audio % 128) {
663 jbuf_write(cs->j_buf, msg);
664
665 pthread_mutex_lock(cs->queue_mutex);
666 int success = 0;
667 while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) {
668 Payload* p;
669
670 if (success == 2) {
671 p = malloc(sizeof(Payload));
672 if (p) p->size = 0;
673
674 } else {
675 p = malloc(sizeof(Payload) + msg->length);
676
677 if (p) {
678 p->size = msg->length;
679 memcpy(p->data, msg->data, msg->length);
680 }
681
682 rtp_free_msg(NULL, msg);
683 }
684
685 if (p) {
686 buffer_write(cs->abuf_raw, p);
687 } else {
688 LOGGER_WARNING("Allocation failed! Program might misbehave!");
689 }
690 }
691 pthread_mutex_unlock(cs->queue_mutex);
692 }
693 /* Video */
694 else {
695 uint8_t *packet = msg->data;
696 uint32_t packet_size = msg->length;
697
698 if (packet_size < VIDEOFRAME_HEADER_SIZE)
699 goto end;
700
701 if (packet[0] > cs->frameid_in || (msg->header->timestamp > cs->last_timestamp)) { /* New frame */
702 /* Flush last frames' data and get ready for this frame */
703 Payload* p = malloc(sizeof(Payload) + cs->frame_size);
704
705 if (p) {
706 pthread_mutex_lock(cs->queue_mutex);
707 if (buffer_full(cs->vbuf_raw)) {
708 LOGGER_DEBUG("Dropped video frame");
709 Payload* tp;
710 buffer_read(cs->vbuf_raw, &tp);
711 free(tp);
712 }
713 else {
714 p->size = cs->frame_size;
715 memcpy(p->data, cs->frame_buf, cs->frame_size);
716 }
717 buffer_write(cs->vbuf_raw, p);
718 pthread_mutex_unlock(cs->queue_mutex);
719 } else {
720 LOGGER_WARNING("Allocation failed! Program might misbehave!");
721 goto end;
722 }
723
724 cs->last_timestamp = msg->header->timestamp;
725 cs->frameid_in = packet[0];
726 memset(cs->frame_buf, 0, cs->frame_size);
727 cs->frame_size = 0;
728
729 } else if (packet[0] < cs->frameid_in) { /* Old frame; drop */
730 LOGGER_DEBUG("Old packet: %u", packet[0]);
731 goto end;
732 }
733
734 /* Otherwise it's part of the frame so just process */
735 /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */
736 memcpy(cs->frame_buf + cs->frame_size,
737 packet + VIDEOFRAME_HEADER_SIZE,
738 packet_size - VIDEOFRAME_HEADER_SIZE);
739
740 cs->frame_size += packet_size - VIDEOFRAME_HEADER_SIZE;
741
742 end:
743 rtp_free_msg(NULL, msg);
744 }
745} \ No newline at end of file