diff options
author | mannol <eniz_vukovic@hotmail.com> | 2014-11-18 00:46:46 +0100 |
---|---|---|
committer | mannol <eniz_vukovic@hotmail.com> | 2014-11-18 00:46:46 +0100 |
commit | 386c9748d48d3bb4513e8e5c32e2b30a4d6a00d4 (patch) | |
tree | 55d0fb2e9fb6e1149317b9de355c28dd86c57014 /toxav/codec.c | |
parent | 4e6f993e7d22865ee2ac90bd7dd3ff25b078c55c (diff) |
av refactor
Diffstat (limited to 'toxav/codec.c')
-rw-r--r-- | toxav/codec.c | 660 |
1 files changed, 524 insertions, 136 deletions
diff --git a/toxav/codec.c b/toxav/codec.c index 2f00319f..46fd6ba7 100644 --- a/toxav/codec.c +++ b/toxav/codec.c | |||
@@ -32,33 +32,124 @@ | |||
32 | #include <stdlib.h> | 32 | #include <stdlib.h> |
33 | #include <math.h> | 33 | #include <math.h> |
34 | #include <assert.h> | 34 | #include <assert.h> |
35 | #include <time.h> | ||
35 | 36 | ||
37 | #include "msi.h" | ||
36 | #include "rtp.h" | 38 | #include "rtp.h" |
37 | #include "codec.h" | 39 | #include "codec.h" |
38 | 40 | ||
39 | JitterBuffer *create_queue(unsigned int capacity) | 41 | /* Assume 24 fps*/ |
42 | #define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) | ||
43 | #define MAX_DECODE_TIME_US 0 | ||
44 | |||
45 | // TODO this has to be exchanged in msi | ||
46 | #define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */ | ||
47 | #define VIDEOFRAME_PIECE_SIZE 0x500 /* 1.25 KiB*/ | ||
48 | #define VIDEOFRAME_HEADER_SIZE 0x2 | ||
49 | |||
50 | /* FIXME: Might not be enough */ | ||
51 | #define VIDEO_DECODE_BUFFER_SIZE 20 | ||
52 | |||
53 | #define ARRAY(TYPE__) struct { uint16_t size; TYPE__ data[]; } | ||
54 | #define PAIR(TYPE1__, TYPE2__) struct { TYPE1__ first; TYPE2__ second; } | ||
55 | |||
56 | typedef ARRAY(uint8_t) Payload; | ||
57 | |||
58 | static PAIR(CSVideoCallback, void*) vpcallback; | ||
59 | static PAIR(CSAudioCallback, void*) apcallback; | ||
60 | |||
61 | typedef struct { | ||
62 | uint16_t size; /* Max size */ | ||
63 | uint16_t start; | ||
64 | uint16_t end; | ||
65 | Payload **packets; | ||
66 | } PayloadBuffer; | ||
67 | |||
68 | static _Bool buffer_full(const PayloadBuffer *b) | ||
40 | { | 69 | { |
41 | unsigned int size = 1; | 70 | return (b->end + 1) % b->size == b->start; |
71 | } | ||
72 | |||
73 | static _Bool buffer_empty(const PayloadBuffer *b) | ||
74 | { | ||
75 | return b->end == b->start; | ||
76 | } | ||
77 | |||
78 | static void buffer_write(PayloadBuffer *b, Payload *p) | ||
79 | { | ||
80 | b->packets[b->end] = p; | ||
81 | b->end = (b->end + 1) % b->size; | ||
82 | if (b->end == b->start) b->start = (b->start + 1) % b->size; /* full, overwrite */ | ||
83 | } | ||
84 | |||
85 | static void buffer_read(PayloadBuffer *b, Payload **p) | ||
86 | { | ||
87 | *p = b->packets[b->start]; | ||
88 | b->start = (b->start + 1) % b->size; | ||
89 | } | ||
90 | |||
91 | static void buffer_clear(PayloadBuffer *b) | ||
92 | { | ||
93 | while (!buffer_empty(b)) { | ||
94 | Payload* p; | ||
95 | buffer_read(b, &p); | ||
96 | free(p); | ||
97 | } | ||
98 | } | ||
99 | |||
100 | static PayloadBuffer* buffer_new(int size) | ||
101 | { | ||
102 | PayloadBuffer *buf = calloc(sizeof(PayloadBuffer), 1); | ||
103 | if (!buf) return NULL; | ||
104 | |||
105 | buf->size = size + 1; /* include empty elem */ | ||
106 | if (!(buf->packets = calloc(buf->size, sizeof(Payload*)))) { | ||
107 | free(buf); | ||
108 | return NULL; | ||
109 | } | ||
110 | return buf; | ||
111 | } | ||
42 | 112 | ||
113 | static void buffer_free(PayloadBuffer *b) | ||
114 | { | ||
115 | if (b) { | ||
116 | buffer_clear(b); | ||
117 | free(b->packets); | ||
118 | free(b); | ||
119 | } | ||
120 | } | ||
121 | |||
122 | /* JITTER BUFFER WORK */ | ||
123 | typedef struct _JitterBuffer { | ||
124 | RTPMessage **queue; | ||
125 | uint32_t size; | ||
126 | uint32_t capacity; | ||
127 | uint16_t bottom; | ||
128 | uint16_t top; | ||
129 | } JitterBuffer; | ||
130 | |||
131 | static JitterBuffer *jbuf_new(uint32_t capacity) | ||
132 | { | ||
133 | unsigned int size = 1; | ||
43 | while (size <= capacity) { | 134 | while (size <= capacity) { |
44 | size *= 2; | 135 | size *= 2; |
45 | } | 136 | } |
46 | 137 | ||
47 | JitterBuffer *q; | 138 | JitterBuffer *q; |
48 | 139 | ||
49 | if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL; | 140 | if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL; |
50 | 141 | ||
51 | if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { | 142 | if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { |
52 | free(q); | 143 | free(q); |
53 | return NULL; | 144 | return NULL; |
54 | } | 145 | } |
55 | 146 | ||
56 | q->size = size; | 147 | q->size = size; |
57 | q->capacity = capacity; | 148 | q->capacity = capacity; |
58 | return q; | 149 | return q; |
59 | } | 150 | } |
60 | 151 | ||
61 | static void clear_queue(JitterBuffer *q) | 152 | static void jbuf_clear(JitterBuffer *q) |
62 | { | 153 | { |
63 | for (; q->bottom != q->top; ++q->bottom) { | 154 | for (; q->bottom != q->top; ++q->bottom) { |
64 | if (q->queue[q->bottom % q->size]) { | 155 | if (q->queue[q->bottom % q->size]) { |
@@ -68,48 +159,50 @@ static void clear_queue(JitterBuffer *q) | |||
68 | } | 159 | } |
69 | } | 160 | } |
70 | 161 | ||
71 | void terminate_queue(JitterBuffer *q) | 162 | static void jbuf_free(JitterBuffer *q) |
72 | { | 163 | { |
73 | if (!q) return; | 164 | if (!q) return; |
74 | 165 | ||
75 | clear_queue(q); | 166 | jbuf_clear(q); |
76 | free(q->queue); | 167 | free(q->queue); |
77 | free(q); | 168 | free(q); |
78 | } | 169 | } |
79 | 170 | ||
80 | void queue(JitterBuffer *q, RTPMessage *pk) | 171 | static void jbuf_write(JitterBuffer *q, RTPMessage *m) |
81 | { | 172 | { |
82 | uint16_t sequnum = pk->header->sequnum; | 173 | uint16_t sequnum = m->header->sequnum; |
83 | 174 | ||
84 | unsigned int num = sequnum % q->size; | 175 | unsigned int num = sequnum % q->size; |
85 | 176 | ||
86 | if ((uint32_t)(sequnum - q->bottom) > q->size) { | 177 | if ((uint32_t)(sequnum - q->bottom) > q->size) { |
87 | clear_queue(q); | 178 | jbuf_clear(q); |
88 | q->bottom = sequnum; | 179 | q->bottom = sequnum; |
89 | q->queue[num] = pk; | 180 | q->queue[num] = m; |
90 | q->top = sequnum + 1; | 181 | q->top = sequnum + 1; |
91 | return; | 182 | return; |
92 | } | 183 | } |
93 | 184 | ||
94 | if (q->queue[num]) | 185 | if (q->queue[num]) |
95 | return; | 186 | return; |
96 | 187 | ||
97 | q->queue[num] = pk; | 188 | q->queue[num] = m; |
98 | 189 | ||
99 | if ((sequnum - q->bottom) >= (q->top - q->bottom)) | 190 | if ((sequnum - q->bottom) >= (q->top - q->bottom)) |
100 | q->top = sequnum + 1; | 191 | q->top = sequnum + 1; |
101 | } | 192 | } |
102 | 193 | ||
103 | /* success is 0 when there is nothing to dequeue, 1 when there's a good packet, 2 when there's a lost packet */ | 194 | /* Success is 0 when there is nothing to dequeue, |
104 | RTPMessage *dequeue(JitterBuffer *q, int *success) | 195 | * 1 when there's a good packet, |
196 | * 2 when there's a lost packet */ | ||
197 | static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success) | ||
105 | { | 198 | { |
106 | if (q->top == q->bottom) { | 199 | if (q->top == q->bottom) { |
107 | *success = 0; | 200 | *success = 0; |
108 | return NULL; | 201 | return NULL; |
109 | } | 202 | } |
110 | 203 | ||
111 | unsigned int num = q->bottom % q->size; | 204 | unsigned int num = q->bottom % q->size; |
112 | 205 | ||
113 | if (q->queue[num]) { | 206 | if (q->queue[num]) { |
114 | RTPMessage *ret = q->queue[num]; | 207 | RTPMessage *ret = q->queue[num]; |
115 | q->queue[num] = NULL; | 208 | q->queue[num] = NULL; |
@@ -117,19 +210,18 @@ RTPMessage *dequeue(JitterBuffer *q, int *success) | |||
117 | *success = 1; | 210 | *success = 1; |
118 | return ret; | 211 | return ret; |
119 | } | 212 | } |
120 | 213 | ||
121 | if ((uint32_t)(q->top - q->bottom) > q->capacity) { | 214 | if ((uint32_t)(q->top - q->bottom) > q->capacity) { |
122 | ++q->bottom; | 215 | ++q->bottom; |
123 | *success = 2; | 216 | *success = 2; |
124 | return NULL; | 217 | return NULL; |
125 | } | 218 | } |
126 | 219 | ||
127 | *success = 0; | 220 | *success = 0; |
128 | return NULL; | 221 | return NULL; |
129 | } | 222 | } |
130 | 223 | ||
131 | 224 | static int init_video_decoder(CSSession *cs) | |
132 | int init_video_decoder(CodecState *cs) | ||
133 | { | 225 | { |
134 | int rc = vpx_codec_dec_init_ver(&cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION); | 226 | int rc = vpx_codec_dec_init_ver(&cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION); |
135 | 227 | ||
@@ -141,72 +233,28 @@ int init_video_decoder(CodecState *cs) | |||
141 | return 0; | 233 | return 0; |
142 | } | 234 | } |
143 | 235 | ||
144 | int init_audio_decoder(CodecState *cs, uint32_t audio_channels) | 236 | static int init_audio_decoder(CSSession *cs) |
145 | { | 237 | { |
146 | int rc; | 238 | int rc; |
147 | cs->audio_decoder = opus_decoder_create(cs->audio_sample_rate, audio_channels, &rc ); | 239 | cs->audio_decoder = opus_decoder_create(cs->audio_decoder_sample_rate, cs->audio_decoder_channels, &rc ); |
148 | 240 | ||
149 | if ( rc != OPUS_OK ) { | 241 | if ( rc != OPUS_OK ) { |
150 | LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); | 242 | LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(rc)); |
151 | return -1; | 243 | return -1; |
152 | } | 244 | } |
153 | |||
154 | cs->audio_decoder_channels = audio_channels; | ||
155 | return 0; | 245 | return 0; |
156 | } | 246 | } |
157 | 247 | ||
158 | int reconfigure_video_encoder_resolution(CodecState *cs, uint16_t width, uint16_t height) | 248 | static int init_video_encoder(CSSession *cs, uint16_t max_width, uint16_t max_height, uint32_t video_bitrate) |
159 | { | ||
160 | vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc; | ||
161 | |||
162 | if (cfg.g_w == width && cfg.g_h == height) | ||
163 | return 0; | ||
164 | |||
165 | if (width * height > cs->max_width * cs->max_height) | ||
166 | return -1; | ||
167 | |||
168 | LOGGER_DEBUG("New video resolution: %u %u", width, height); | ||
169 | cfg.g_w = width; | ||
170 | cfg.g_h = height; | ||
171 | int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg); | ||
172 | |||
173 | if ( rc != VPX_CODEC_OK) { | ||
174 | LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); | ||
175 | return -1; | ||
176 | } | ||
177 | |||
178 | return 0; | ||
179 | } | ||
180 | |||
181 | int reconfigure_video_encoder_bitrate(CodecState *cs, uint32_t video_bitrate) | ||
182 | { | ||
183 | vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc; | ||
184 | |||
185 | if (cfg.rc_target_bitrate == video_bitrate) | ||
186 | return 0; | ||
187 | |||
188 | LOGGER_DEBUG("New video bitrate: %u", video_bitrate); | ||
189 | cfg.rc_target_bitrate = video_bitrate; | ||
190 | int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg); | ||
191 | |||
192 | if ( rc != VPX_CODEC_OK) { | ||
193 | LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); | ||
194 | return -1; | ||
195 | } | ||
196 | |||
197 | return 0; | ||
198 | } | ||
199 | |||
200 | int init_video_encoder(CodecState *cs, uint16_t max_width, uint16_t max_height, uint32_t video_bitrate) | ||
201 | { | 249 | { |
202 | vpx_codec_enc_cfg_t cfg; | 250 | vpx_codec_enc_cfg_t cfg; |
203 | int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); | 251 | int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); |
204 | 252 | ||
205 | if (rc != VPX_CODEC_OK) { | 253 | if (rc != VPX_CODEC_OK) { |
206 | LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); | 254 | LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc)); |
207 | return -1; | 255 | return -1; |
208 | } | 256 | } |
209 | 257 | ||
210 | cfg.rc_target_bitrate = video_bitrate; | 258 | cfg.rc_target_bitrate = video_bitrate; |
211 | cfg.g_w = max_width; | 259 | cfg.g_w = max_width; |
212 | cfg.g_h = max_height; | 260 | cfg.g_h = max_height; |
@@ -216,130 +264,375 @@ int init_video_encoder(CodecState *cs, uint16_t max_width, uint16_t max_height, | |||
216 | cfg.kf_min_dist = 0; | 264 | cfg.kf_min_dist = 0; |
217 | cfg.kf_max_dist = 5; | 265 | cfg.kf_max_dist = 5; |
218 | cfg.kf_mode = VPX_KF_AUTO; | 266 | cfg.kf_mode = VPX_KF_AUTO; |
219 | 267 | ||
220 | cs->max_width = max_width; | 268 | cs->max_width = max_width; |
221 | cs->max_height = max_height; | 269 | cs->max_height = max_height; |
222 | cs->bitrate = video_bitrate; | 270 | |
223 | |||
224 | rc = vpx_codec_enc_init_ver(&cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION); | 271 | rc = vpx_codec_enc_init_ver(&cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION); |
225 | 272 | ||
226 | if ( rc != VPX_CODEC_OK) { | 273 | if ( rc != VPX_CODEC_OK) { |
227 | LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); | 274 | LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); |
228 | return -1; | 275 | return -1; |
229 | } | 276 | } |
230 | 277 | ||
231 | rc = vpx_codec_control(&cs->v_encoder, VP8E_SET_CPUUSED, 8); | 278 | rc = vpx_codec_control(&cs->v_encoder, VP8E_SET_CPUUSED, 8); |
232 | 279 | ||
233 | if ( rc != VPX_CODEC_OK) { | 280 | if ( rc != VPX_CODEC_OK) { |
234 | LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); | 281 | LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
235 | return -1; | 282 | return -1; |
236 | } | 283 | } |
237 | 284 | ||
238 | return 0; | 285 | return 0; |
239 | } | 286 | } |
240 | 287 | ||
241 | int init_audio_encoder(CodecState *cs, uint32_t audio_channels) | 288 | static int init_audio_encoder(CSSession *cs) |
242 | { | 289 | { |
243 | int rc = OPUS_OK; | 290 | int rc = OPUS_OK; |
244 | cs->audio_encoder = opus_encoder_create(cs->audio_sample_rate, audio_channels, OPUS_APPLICATION_AUDIO, &rc); | 291 | cs->audio_encoder = opus_encoder_create(cs->audio_encoder_sample_rate, |
245 | 292 | cs->audio_encoder_channels, OPUS_APPLICATION_AUDIO, &rc); | |
293 | |||
246 | if ( rc != OPUS_OK ) { | 294 | if ( rc != OPUS_OK ) { |
247 | LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); | 295 | LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(rc)); |
248 | return -1; | 296 | return -1; |
249 | } | 297 | } |
250 | 298 | ||
251 | rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_bitrate)); | 299 | rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_encoder_bitrate)); |
252 | 300 | ||
253 | if ( rc != OPUS_OK ) { | 301 | if ( rc != OPUS_OK ) { |
254 | LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); | 302 | LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); |
255 | return -1; | 303 | return -1; |
256 | } | 304 | } |
257 | 305 | ||
258 | rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); | 306 | rc = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); |
259 | 307 | ||
260 | if ( rc != OPUS_OK ) { | 308 | if ( rc != OPUS_OK ) { |
261 | LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); | 309 | LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(rc)); |
262 | return -1; | 310 | return -1; |
263 | } | 311 | } |
264 | |||
265 | cs->audio_encoder_channels = audio_channels; | ||
266 | return 0; | 312 | return 0; |
267 | } | 313 | } |
268 | 314 | ||
269 | 315 | static float calculate_sum_sq (int16_t *n, uint16_t k) | |
270 | CodecState *codec_init_session ( uint32_t audio_bitrate, | ||
271 | uint16_t audio_frame_duration, | ||
272 | uint32_t audio_sample_rate, | ||
273 | uint32_t encoder_audio_channels, | ||
274 | uint32_t decoder_audio_channels, | ||
275 | uint32_t audio_VAD_tolerance_ms, | ||
276 | uint16_t max_video_width, | ||
277 | uint16_t max_video_height, | ||
278 | uint32_t video_bitrate ) | ||
279 | { | 316 | { |
280 | CodecState *retu = calloc(sizeof(CodecState), 1); | 317 | float result = 0; |
318 | uint16_t i = 0; | ||
319 | |||
320 | for ( ; i < k; i ++) result += (float) (n[i] * n[i]); | ||
321 | |||
322 | return result; | ||
323 | } | ||
281 | 324 | ||
282 | if (!retu) return NULL; | ||
283 | 325 | ||
284 | retu->audio_bitrate = audio_bitrate; | ||
285 | retu->audio_sample_rate = audio_sample_rate; | ||
286 | 326 | ||
287 | /* Encoders */ | 327 | /* PUBLIC */ |
288 | if (!max_video_width || !max_video_height) { /* Disable video */ | 328 | int cs_split_video_payload(CSSession* cs, const uint8_t* payload, uint16_t length) |
289 | /*video_width = 320; | 329 | { |
290 | video_height = 240; */ | 330 | if (!cs || !length || length > cs->max_video_frame_size) { |
331 | LOGGER_ERROR("Invalid CodecState or video frame size: %u", length); | ||
332 | return -1; | ||
333 | } | ||
334 | |||
335 | cs->split_video_frame[0] = cs->frameid_out++; | ||
336 | cs->split_video_frame[1] = 0; | ||
337 | cs->processing_video_frame = payload; | ||
338 | cs->processing_video_frame_size = length; | ||
339 | |||
340 | return ((length - 1) / cs->video_frame_piece_size) + 1; | ||
341 | } | ||
342 | |||
343 | const uint8_t* cs_get_split_video_frame(CSSession* cs, uint16_t* size) | ||
344 | { | ||
345 | if (!cs || !size) return NULL; | ||
346 | |||
347 | if (cs->processing_video_frame_size > cs->video_frame_piece_size) { | ||
348 | memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, | ||
349 | cs->processing_video_frame, | ||
350 | cs->video_frame_piece_size); | ||
351 | |||
352 | cs->processing_video_frame += cs->video_frame_piece_size; | ||
353 | cs->processing_video_frame_size -= cs->video_frame_piece_size; | ||
354 | |||
355 | *size = cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE; | ||
291 | } else { | 356 | } else { |
292 | retu->capabilities |= ( 0 == init_video_encoder(retu, max_video_width, max_video_height, | 357 | memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE, |
293 | video_bitrate) ) ? v_encoding : 0; | 358 | cs->processing_video_frame, |
294 | retu->capabilities |= ( 0 == init_video_decoder(retu) ) ? v_decoding : 0; | 359 | cs->processing_video_frame_size); |
360 | |||
361 | *size = cs->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE; | ||
362 | } | ||
363 | |||
364 | cs->split_video_frame[1]++; | ||
365 | |||
366 | return cs->split_video_frame; | ||
367 | } | ||
368 | |||
369 | void cs_do(CSSession* cs) | ||
370 | { | ||
371 | if (!cs) return; | ||
372 | |||
373 | pthread_mutex_lock(cs->queue_mutex); | ||
374 | /* | ||
375 | if (!cs->active) { | ||
376 | pthread_mutex_unlock(cs->queue_mutex); | ||
377 | return; | ||
378 | } | ||
379 | |||
380 | /* Iterate over whole buffers and call playback callback * / | ||
381 | if (cs->abuf_ready) while (!DecodedAudioBuffer_empty(cs->abuf_ready)) { | ||
382 | DecodedAudio* p; | ||
383 | DecodedAudioBuffer_read(cs->abuf_ready, &p); | ||
384 | if (apcallback.first) | ||
385 | apcallback.first(cs->agent, cs->call_idx, p->data, p->size, apcallback.second); | ||
386 | |||
387 | free(p); | ||
388 | } | ||
389 | |||
390 | if (cs->vbuf_ready) while (!DecodedVideoBuffer_empty(cs->vbuf_ready)) { | ||
391 | vpx_image_t* p; | ||
392 | DecodedVideoBuffer_read(cs->vbuf_ready, &p); | ||
393 | if (vpcallback.first) | ||
394 | vpcallback.first(cs->agent, cs->call_idx, p, vpcallback.second); | ||
395 | |||
396 | vpx_img_free(p); | ||
397 | } | ||
398 | */ | ||
399 | |||
400 | Payload* p; | ||
401 | int rc; | ||
402 | |||
403 | if (cs->abuf_raw && !buffer_empty(cs->abuf_raw)) { | ||
404 | /* Decode audio */ | ||
405 | buffer_read(cs->abuf_raw, &p); | ||
406 | |||
407 | uint16_t fsize = (cs->audio_decoder_channels * | ||
408 | (cs->audio_decoder_sample_rate * cs->audio_decoder_frame_duration) / 1000); | ||
409 | int16_t tmp[fsize]; | ||
410 | |||
411 | rc = opus_decode(cs->audio_decoder, p->data, p->size, tmp, fsize, (p->size == 0)); | ||
412 | free(p); | ||
413 | |||
414 | if (rc < 0) | ||
415 | LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); | ||
416 | else | ||
417 | /* Play */ | ||
418 | apcallback.first(cs->agent, cs->call_idx, tmp, rc, apcallback.second); | ||
419 | } | ||
420 | |||
421 | if (cs->vbuf_raw && !buffer_empty(cs->vbuf_raw)) { | ||
422 | /* Decode video */ | ||
423 | buffer_read(cs->vbuf_raw, &p); | ||
424 | |||
425 | rc = vpx_codec_decode(&cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); | ||
426 | free(p); | ||
427 | |||
428 | if (rc != VPX_CODEC_OK) { | ||
429 | LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc)); | ||
430 | } else { | ||
431 | vpx_codec_iter_t iter = NULL; | ||
432 | vpx_image_t* dest = vpx_codec_get_frame(&cs->v_decoder, &iter); | ||
433 | |||
434 | /* Play decoded images */ | ||
435 | for (; dest; dest = vpx_codec_get_frame(&cs->v_decoder, &iter)) { | ||
436 | vpcallback.first(cs->agent, cs->call_idx, dest, vpcallback.second); | ||
437 | vpx_img_free(dest); | ||
438 | } | ||
439 | } | ||
295 | } | 440 | } |
441 | |||
442 | pthread_mutex_unlock(cs->queue_mutex); | ||
443 | } | ||
296 | 444 | ||
297 | retu->capabilities |= ( 0 == init_audio_encoder(retu, encoder_audio_channels) ) ? a_encoding : 0; | 445 | void cs_register_audio_callback(CSAudioCallback cb, void* data) |
298 | retu->capabilities |= ( 0 == init_audio_decoder(retu, decoder_audio_channels) ) ? a_decoding : 0; | 446 | { |
447 | apcallback.first = cb; | ||
448 | apcallback.second = data; | ||
449 | } | ||
299 | 450 | ||
300 | if ( retu->capabilities == 0 ) { /* everything failed */ | 451 | void cs_register_video_callback(CSVideoCallback cb, void* data) |
301 | free (retu); | 452 | { |
302 | return NULL; | 453 | vpcallback.first = cb; |
454 | vpcallback.second = data; | ||
455 | } | ||
456 | |||
457 | int cs_set_video_encoder_resolution(CSSession *cs, uint16_t width, uint16_t height) | ||
458 | { | ||
459 | vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc; | ||
460 | |||
461 | if (cfg.g_w == width && cfg.g_h == height) | ||
462 | return 0; | ||
463 | |||
464 | if (width * height > cs->max_width * cs->max_height) | ||
465 | return -1; | ||
466 | |||
467 | LOGGER_DEBUG("New video resolution: %u %u", width, height); | ||
468 | cfg.g_w = width; | ||
469 | cfg.g_h = height; | ||
470 | int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg); | ||
471 | |||
472 | if ( rc != VPX_CODEC_OK) { | ||
473 | LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); | ||
474 | return -1; | ||
303 | } | 475 | } |
304 | 476 | ||
477 | return 0; | ||
478 | } | ||
305 | 479 | ||
306 | retu->EVAD_tolerance = audio_VAD_tolerance_ms > audio_frame_duration ? | 480 | int cs_set_video_encoder_bitrate(CSSession *cs, uint32_t video_bitrate) |
307 | audio_VAD_tolerance_ms / audio_frame_duration : audio_frame_duration; | 481 | { |
482 | vpx_codec_enc_cfg_t cfg = *cs->v_encoder.config.enc; | ||
483 | |||
484 | if (cfg.rc_target_bitrate == video_bitrate) | ||
485 | return 0; | ||
308 | 486 | ||
309 | return retu; | 487 | LOGGER_DEBUG("New video bitrate: %u", video_bitrate); |
488 | cfg.rc_target_bitrate = video_bitrate; | ||
489 | int rc = vpx_codec_enc_config_set(&cs->v_encoder, &cfg); | ||
490 | |||
491 | if ( rc != VPX_CODEC_OK) { | ||
492 | LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); | ||
493 | return -1; | ||
494 | } | ||
495 | |||
496 | return 0; | ||
497 | } | ||
498 | |||
499 | CSSession *cs_new(const ToxAvCSettings *cs_self, const ToxAvCSettings *cs_peer, uint32_t jbuf_size, int has_video) | ||
500 | { | ||
501 | CSSession *cs = calloc(sizeof(CSSession), 1); | ||
502 | |||
503 | if (!cs) { | ||
504 | LOGGER_WARNING("Allocation failed! Application might misbehave!"); | ||
505 | return NULL; | ||
506 | } | ||
507 | |||
508 | if ( !(cs->j_buf = jbuf_new(jbuf_size)) ) { | ||
509 | LOGGER_WARNING("Jitter buffer creaton failed!"); | ||
510 | goto error; | ||
511 | } | ||
512 | |||
513 | cs->audio_encoder_bitrate = cs_self->audio_bitrate; | ||
514 | cs->audio_encoder_sample_rate = cs_self->audio_sample_rate; | ||
515 | cs->audio_encoder_channels = cs_self->audio_channels; | ||
516 | cs->audio_encoder_frame_duration = cs_self->audio_frame_duration; | ||
517 | |||
518 | cs->audio_decoder_bitrate = cs_peer->audio_bitrate; | ||
519 | cs->audio_decoder_sample_rate = cs_peer->audio_sample_rate; | ||
520 | cs->audio_decoder_channels = cs_peer->audio_channels; | ||
521 | cs->audio_decoder_frame_duration = cs_peer->audio_frame_duration; | ||
522 | |||
523 | |||
524 | cs->capabilities |= ( 0 == init_audio_encoder(cs) ) ? a_encoding : 0; | ||
525 | cs->capabilities |= ( 0 == init_audio_decoder(cs) ) ? a_decoding : 0; | ||
526 | |||
527 | if ( !(cs->capabilities & a_encoding) || !(cs->capabilities & a_decoding) ) goto error; | ||
528 | |||
529 | if ( !(cs->abuf_raw = buffer_new(jbuf_size)) ) goto error; | ||
530 | |||
531 | pthread_mutexattr_t attr; | ||
532 | if (pthread_mutexattr_init(&attr) != 0) goto error; | ||
533 | if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) != 0) { | ||
534 | if (pthread_mutexattr_destroy(&attr) != 0) | ||
535 | LOGGER_WARNING("Failed to destroy mutex attribute!"); | ||
536 | goto error; | ||
537 | } | ||
538 | |||
539 | if (pthread_mutex_init(cs->queue_mutex, &attr) != 0) { | ||
540 | pthread_mutexattr_destroy(&attr); | ||
541 | goto error; | ||
542 | } | ||
543 | |||
544 | if ((cs->support_video = has_video)) { | ||
545 | cs->max_video_frame_size = MAX_VIDEOFRAME_SIZE; | ||
546 | cs->video_frame_piece_size = VIDEOFRAME_PIECE_SIZE; | ||
547 | |||
548 | cs->capabilities |= ( 0 == init_video_encoder(cs, cs_self->max_video_width, | ||
549 | cs_self->max_video_height, cs_self->video_bitrate) ) ? v_encoding : 0; | ||
550 | cs->capabilities |= ( 0 == init_video_decoder(cs) ) ? v_decoding : 0; | ||
551 | |||
552 | if ( !(cs->capabilities & v_encoding) || !(cs->capabilities & v_decoding) ) goto error; | ||
553 | |||
554 | if ( !(cs->frame_buf = calloc(cs->max_video_frame_size, 1)) ) goto error; | ||
555 | if ( !(cs->split_video_frame = calloc(cs->video_frame_piece_size + VIDEOFRAME_HEADER_SIZE, 1)) ) | ||
556 | goto error; | ||
557 | |||
558 | if ( !(cs->vbuf_raw = buffer_new(VIDEO_DECODE_BUFFER_SIZE)) ) goto error; | ||
559 | } | ||
560 | |||
561 | if (pthread_mutexattr_destroy(&attr) != 0) | ||
562 | LOGGER_WARNING("Failed to destroy mutex attribute!"); | ||
563 | |||
564 | |||
565 | cs->active = 1; | ||
566 | return cs; | ||
567 | |||
568 | error: | ||
569 | LOGGER_WARNING("Error initializing codec session! Application might misbehave!"); | ||
570 | |||
571 | buffer_free(cs->abuf_raw); | ||
572 | |||
573 | if ( cs->audio_encoder ) opus_encoder_destroy(cs->audio_encoder); | ||
574 | if ( cs->audio_decoder ) opus_decoder_destroy(cs->audio_decoder); | ||
575 | |||
576 | |||
577 | if(has_video) { | ||
578 | if ( cs->capabilities & v_decoding ) vpx_codec_destroy(&cs->v_decoder); | ||
579 | if ( cs->capabilities & v_encoding ) vpx_codec_destroy(&cs->v_encoder); | ||
580 | |||
581 | buffer_free(cs->vbuf_raw); | ||
582 | |||
583 | free(cs->frame_buf); | ||
584 | free(cs->split_video_frame); | ||
585 | } | ||
586 | |||
587 | jbuf_free(cs->j_buf); | ||
588 | free(cs); | ||
589 | |||
590 | return NULL; | ||
310 | } | 591 | } |
311 | 592 | ||
312 | void codec_terminate_session ( CodecState *cs ) | 593 | void cs_kill(CSSession *cs) |
313 | { | 594 | { |
314 | if (!cs) return; | 595 | if (!cs) return; |
315 | 596 | ||
597 | /* Lock running mutex and signal that cs is no longer active */ | ||
598 | pthread_mutex_lock(cs->queue_mutex); | ||
599 | cs->active = 0; | ||
600 | |||
601 | /* Wait threads to close */ | ||
602 | pthread_mutex_unlock(cs->queue_mutex); | ||
603 | pthread_mutex_lock(cs->queue_mutex); | ||
604 | pthread_mutex_unlock(cs->queue_mutex); | ||
605 | |||
606 | pthread_mutex_destroy(cs->queue_mutex); | ||
607 | |||
608 | |||
316 | if ( cs->audio_encoder ) | 609 | if ( cs->audio_encoder ) |
317 | opus_encoder_destroy(cs->audio_encoder); | 610 | opus_encoder_destroy(cs->audio_encoder); |
318 | 611 | ||
319 | if ( cs->audio_decoder ) | 612 | if ( cs->audio_decoder ) |
320 | opus_decoder_destroy(cs->audio_decoder); | 613 | opus_decoder_destroy(cs->audio_decoder); |
321 | 614 | ||
322 | if ( cs->capabilities & v_decoding ) | 615 | if ( cs->capabilities & v_decoding ) |
323 | vpx_codec_destroy(&cs->v_decoder); | 616 | vpx_codec_destroy(&cs->v_decoder); |
324 | 617 | ||
325 | if ( cs->capabilities & v_encoding ) | 618 | if ( cs->capabilities & v_encoding ) |
326 | vpx_codec_destroy(&cs->v_encoder); | 619 | vpx_codec_destroy(&cs->v_encoder); |
327 | 620 | ||
621 | jbuf_free(cs->j_buf); | ||
622 | buffer_free(cs->abuf_raw); | ||
623 | buffer_free(cs->vbuf_raw); | ||
624 | free(cs->frame_buf); | ||
625 | |||
328 | LOGGER_DEBUG("Terminated codec state: %p", cs); | 626 | LOGGER_DEBUG("Terminated codec state: %p", cs); |
329 | free(cs); | 627 | free(cs); |
330 | } | 628 | } |
331 | 629 | ||
332 | static float calculate_sum_sq (int16_t *n, uint16_t k) | 630 | void cs_set_vad_treshold(CSSession* cs, uint32_t treshold, uint16_t frame_duration) |
333 | { | 631 | { |
334 | float result = 0; | 632 | cs->EVAD_tolerance = treshold > frame_duration ? treshold / frame_duration : frame_duration; |
335 | uint16_t i = 0; | ||
336 | |||
337 | for ( ; i < k; i ++) result += (float) (n[i] * n[i]); | ||
338 | |||
339 | return result; | ||
340 | } | 633 | } |
341 | 634 | ||
342 | int energy_VAD(CodecState *cs, int16_t *PCM, uint16_t frame_size, float energy) | 635 | int cs_calculate_vad(CSSession *cs, int16_t *PCM, uint16_t frame_size, float energy) |
343 | { | 636 | { |
344 | float frame_energy = sqrt(calculate_sum_sq(PCM, frame_size)) / frame_size; | 637 | float frame_energy = sqrt(calculate_sum_sq(PCM, frame_size)) / frame_size; |
345 | 638 | ||
@@ -355,3 +648,98 @@ int energy_VAD(CodecState *cs, int16_t *PCM, uint16_t frame_size, float energy) | |||
355 | 648 | ||
356 | return 0; | 649 | return 0; |
357 | } | 650 | } |
651 | |||
652 | |||
653 | |||
654 | |||
655 | /* Called from RTP */ | ||
656 | void queue_message(RTPSession *session, RTPMessage *msg) | ||
657 | { | ||
658 | CSSession *cs = session->cs; | ||
659 | if (!cs || !cs->active) return; | ||
660 | |||
661 | /* Audio */ | ||
662 | if (session->payload_type == type_audio % 128) { | ||
663 | jbuf_write(cs->j_buf, msg); | ||
664 | |||
665 | pthread_mutex_lock(cs->queue_mutex); | ||
666 | int success = 0; | ||
667 | while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) { | ||
668 | Payload* p; | ||
669 | |||
670 | if (success == 2) { | ||
671 | p = malloc(sizeof(Payload)); | ||
672 | if (p) p->size = 0; | ||
673 | |||
674 | } else { | ||
675 | p = malloc(sizeof(Payload) + msg->length); | ||
676 | |||
677 | if (p) { | ||
678 | p->size = msg->length; | ||
679 | memcpy(p->data, msg->data, msg->length); | ||
680 | } | ||
681 | |||
682 | rtp_free_msg(NULL, msg); | ||
683 | } | ||
684 | |||
685 | if (p) { | ||
686 | buffer_write(cs->abuf_raw, p); | ||
687 | } else { | ||
688 | LOGGER_WARNING("Allocation failed! Program might misbehave!"); | ||
689 | } | ||
690 | } | ||
691 | pthread_mutex_unlock(cs->queue_mutex); | ||
692 | } | ||
693 | /* Video */ | ||
694 | else { | ||
695 | uint8_t *packet = msg->data; | ||
696 | uint32_t packet_size = msg->length; | ||
697 | |||
698 | if (packet_size < VIDEOFRAME_HEADER_SIZE) | ||
699 | goto end; | ||
700 | |||
701 | if (packet[0] > cs->frameid_in || (msg->header->timestamp > cs->last_timestamp)) { /* New frame */ | ||
702 | /* Flush last frames' data and get ready for this frame */ | ||
703 | Payload* p = malloc(sizeof(Payload) + cs->frame_size); | ||
704 | |||
705 | if (p) { | ||
706 | pthread_mutex_lock(cs->queue_mutex); | ||
707 | if (buffer_full(cs->vbuf_raw)) { | ||
708 | LOGGER_DEBUG("Dropped video frame"); | ||
709 | Payload* tp; | ||
710 | buffer_read(cs->vbuf_raw, &tp); | ||
711 | free(tp); | ||
712 | } | ||
713 | else { | ||
714 | p->size = cs->frame_size; | ||
715 | memcpy(p->data, cs->frame_buf, cs->frame_size); | ||
716 | } | ||
717 | buffer_write(cs->vbuf_raw, p); | ||
718 | pthread_mutex_unlock(cs->queue_mutex); | ||
719 | } else { | ||
720 | LOGGER_WARNING("Allocation failed! Program might misbehave!"); | ||
721 | goto end; | ||
722 | } | ||
723 | |||
724 | cs->last_timestamp = msg->header->timestamp; | ||
725 | cs->frameid_in = packet[0]; | ||
726 | memset(cs->frame_buf, 0, cs->frame_size); | ||
727 | cs->frame_size = 0; | ||
728 | |||
729 | } else if (packet[0] < cs->frameid_in) { /* Old frame; drop */ | ||
730 | LOGGER_DEBUG("Old packet: %u", packet[0]); | ||
731 | goto end; | ||
732 | } | ||
733 | |||
734 | /* Otherwise it's part of the frame so just process */ | ||
735 | /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */ | ||
736 | memcpy(cs->frame_buf + cs->frame_size, | ||
737 | packet + VIDEOFRAME_HEADER_SIZE, | ||
738 | packet_size - VIDEOFRAME_HEADER_SIZE); | ||
739 | |||
740 | cs->frame_size += packet_size - VIDEOFRAME_HEADER_SIZE; | ||
741 | |||
742 | end: | ||
743 | rtp_free_msg(NULL, msg); | ||
744 | } | ||
745 | } \ No newline at end of file | ||