diff options
Diffstat (limited to 'toxav/audio.c')
-rw-r--r-- | toxav/audio.c | 424 |
1 files changed, 424 insertions, 0 deletions
diff --git a/toxav/audio.c b/toxav/audio.c new file mode 100644 index 00000000..f6993a1d --- /dev/null +++ b/toxav/audio.c | |||
@@ -0,0 +1,424 @@ | |||
1 | /** audio.c | ||
2 | * | ||
3 | * Copyright (C) 2013-2015 Tox project All Rights Reserved. | ||
4 | * | ||
5 | * This file is part of Tox. | ||
6 | * | ||
7 | * Tox is free software: you can redistribute it and/or modify | ||
8 | * it under the terms of the GNU General Public License as published by | ||
9 | * the Free Software Foundation, either version 3 of the License, or | ||
10 | * (at your option) any later version. | ||
11 | * | ||
12 | * Tox is distributed in the hope that it will be useful, | ||
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
15 | * GNU General Public License for more details. | ||
16 | * | ||
17 | * You should have received a copy of the GNU General Public License | ||
18 | * along with Tox. If not, see <http://www.gnu.org/licenses/>. | ||
19 | * | ||
20 | */ | ||
21 | |||
22 | #include <stdlib.h> | ||
23 | |||
24 | #include "audio.h" | ||
25 | #include "rtp.h" | ||
26 | |||
27 | #include "../toxcore/logger.h" | ||
28 | |||
29 | static struct JitterBuffer *jbuf_new(uint32_t capacity); | ||
30 | static void jbuf_clear(struct JitterBuffer *q); | ||
31 | static void jbuf_free(struct JitterBuffer *q); | ||
32 | static int jbuf_write(struct JitterBuffer *q, RTPMessage *m); | ||
33 | static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success); | ||
34 | OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count); | ||
35 | bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, uint8_t new_ch, | ||
36 | int32_t *old_br, int32_t *old_sr, int32_t *old_ch); | ||
37 | bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels); | ||
38 | |||
39 | |||
40 | |||
41 | ACSession* ac_new(ToxAV* av, uint32_t friend_number, toxav_audio_receive_frame_cb *cb, void *cb_data) | ||
42 | { | ||
43 | ACSession *ac = calloc(sizeof(ACSession), 1); | ||
44 | |||
45 | if (!ac) { | ||
46 | LOGGER_WARNING("Allocation failed! Application might misbehave!"); | ||
47 | return NULL; | ||
48 | } | ||
49 | |||
50 | if (create_recursive_mutex(ac->queue_mutex) != 0) { | ||
51 | LOGGER_WARNING("Failed to create recursive mutex!"); | ||
52 | free(ac); | ||
53 | return NULL; | ||
54 | } | ||
55 | |||
56 | int status; | ||
57 | ac->decoder = opus_decoder_create(48000, 2, &status ); | ||
58 | |||
59 | if ( status != OPUS_OK ) { | ||
60 | LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status)); | ||
61 | goto BASE_CLEANUP; | ||
62 | } | ||
63 | |||
64 | if ( !(ac->j_buf = jbuf_new(3)) ) { | ||
65 | LOGGER_WARNING("Jitter buffer creaton failed!"); | ||
66 | opus_decoder_destroy(ac->decoder); | ||
67 | goto BASE_CLEANUP; | ||
68 | } | ||
69 | |||
70 | /* Initialize encoders with default values */ | ||
71 | ac->encoder = create_audio_encoder(48000, 48000, 2); | ||
72 | if (ac->encoder == NULL) | ||
73 | goto DECODER_CLEANUP; | ||
74 | |||
75 | ac->test_encoder = create_audio_encoder(48000, 48000, 2); | ||
76 | if (ac->test_encoder == NULL) { | ||
77 | opus_encoder_destroy(ac->encoder); | ||
78 | goto DECODER_CLEANUP; | ||
79 | } | ||
80 | |||
81 | ac->last_encoding_bit_rate = 48000; | ||
82 | ac->last_encoding_sampling_rate = 48000; | ||
83 | ac->last_encoding_channel_count = 2; | ||
84 | |||
85 | ac->last_test_encoding_bit_rate = 48000; | ||
86 | ac->last_test_encoding_sampling_rate = 48000; | ||
87 | ac->last_test_encoding_channel_count = 2; | ||
88 | |||
89 | ac->last_decoding_channel_count = 2; | ||
90 | ac->last_decoding_sampling_rate = 48000; | ||
91 | ac->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */ | ||
92 | |||
93 | /* These need to be set in order to properly | ||
94 | * do error correction with opus */ | ||
95 | ac->last_packet_frame_duration = 120; | ||
96 | ac->last_packet_sampling_rate = 48000; | ||
97 | ac->last_packet_channel_count = 1; | ||
98 | |||
99 | ac->av = av; | ||
100 | ac->friend_number = friend_number; | ||
101 | ac->acb.first = cb; | ||
102 | ac->acb.second = cb_data; | ||
103 | |||
104 | return ac; | ||
105 | |||
106 | DECODER_CLEANUP: | ||
107 | opus_decoder_destroy(ac->decoder); | ||
108 | jbuf_free(ac->j_buf); | ||
109 | BASE_CLEANUP: | ||
110 | pthread_mutex_destroy(ac->queue_mutex); | ||
111 | free(ac); | ||
112 | return NULL; | ||
113 | } | ||
114 | void ac_kill(ACSession* ac) | ||
115 | { | ||
116 | if (!ac) | ||
117 | return; | ||
118 | |||
119 | opus_encoder_destroy(ac->encoder); | ||
120 | opus_decoder_destroy(ac->decoder); | ||
121 | jbuf_free(ac->j_buf); | ||
122 | |||
123 | pthread_mutex_destroy(ac->queue_mutex); | ||
124 | |||
125 | LOGGER_DEBUG("Terminated audio handler: %p", ac); | ||
126 | free(ac); | ||
127 | } | ||
128 | void ac_do(ACSession* ac) | ||
129 | { | ||
130 | if (!ac) | ||
131 | return; | ||
132 | |||
133 | /* Enough space for the maximum frame size (120 ms 48 KHz audio) */ | ||
134 | int16_t tmp[5760 * 2]; | ||
135 | |||
136 | RTPMessage *msg; | ||
137 | int rc = 0; | ||
138 | |||
139 | pthread_mutex_lock(ac->queue_mutex); | ||
140 | while ((msg = jbuf_read(ac->j_buf, &rc)) || rc == 2) { | ||
141 | pthread_mutex_unlock(ac->queue_mutex); | ||
142 | |||
143 | if (rc == 2) { | ||
144 | LOGGER_DEBUG("OPUS correction"); | ||
145 | int fs = (ac->last_packet_sampling_rate * ac->last_packet_frame_duration) / 1000; | ||
146 | rc = opus_decode(ac->decoder, NULL, 0, tmp, fs, 1); | ||
147 | } else { | ||
148 | /* Get values from packet and decode. */ | ||
149 | /* NOTE: This didn't work very well | ||
150 | rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data)); | ||
151 | if (rc != -1) { | ||
152 | cs->last_packet_sampling_rate = rc; | ||
153 | } else { | ||
154 | LOGGER_WARNING("Failed to load packet values!"); | ||
155 | rtp_free_msg(msg); | ||
156 | continue; | ||
157 | }*/ | ||
158 | |||
159 | |||
160 | /* Pick up sampling rate from packet */ | ||
161 | memcpy(&ac->last_packet_sampling_rate, msg->data, 4); | ||
162 | ac->last_packet_sampling_rate = ntohl(ac->last_packet_sampling_rate); | ||
163 | |||
164 | ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); | ||
165 | |||
166 | /** NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, | ||
167 | * it didn't work quite well. | ||
168 | */ | ||
169 | if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) { | ||
170 | LOGGER_WARNING("Failed to reconfigure decoder!"); | ||
171 | rtp_free_msg(msg); | ||
172 | continue; | ||
173 | } | ||
174 | |||
175 | rc = opus_decode(ac->decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0); | ||
176 | rtp_free_msg(msg); | ||
177 | } | ||
178 | |||
179 | if (rc < 0) { | ||
180 | LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); | ||
181 | } else if (ac->acb.first) { | ||
182 | ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate; | ||
183 | |||
184 | ac->acb.first(ac->av, ac->friend_number, tmp, rc * ac->last_packet_channel_count, | ||
185 | ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second); | ||
186 | } | ||
187 | |||
188 | return; | ||
189 | } | ||
190 | pthread_mutex_unlock(ac->queue_mutex); | ||
191 | } | ||
192 | int ac_queue_message(void* acp, struct RTPMessage_s *msg) | ||
193 | { | ||
194 | if (!acp || !msg) | ||
195 | return -1; | ||
196 | |||
197 | if ((msg->header->marker_payloadt & 0x7f) == (rtp_TypeAudio + 2) % 128) { | ||
198 | LOGGER_WARNING("Got dummy!"); | ||
199 | rtp_free_msg(msg); | ||
200 | return 0; | ||
201 | } | ||
202 | |||
203 | if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeAudio % 128) { | ||
204 | LOGGER_WARNING("Invalid payload type!"); | ||
205 | rtp_free_msg(msg); | ||
206 | return -1; | ||
207 | } | ||
208 | |||
209 | ACSession* ac = acp; | ||
210 | |||
211 | pthread_mutex_lock(ac->queue_mutex); | ||
212 | int rc = jbuf_write(ac->j_buf, msg); | ||
213 | pthread_mutex_unlock(ac->queue_mutex); | ||
214 | |||
215 | if (rc == -1) { | ||
216 | LOGGER_WARNING("Could not queue the message!"); | ||
217 | rtp_free_msg(msg); | ||
218 | return -1; | ||
219 | } | ||
220 | |||
221 | return 0; | ||
222 | } | ||
223 | int ac_reconfigure_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels) | ||
224 | { | ||
225 | if (!ac || !reconfigure_audio_encoder(&ac->encoder, bit_rate, sampling_rate, channels, | ||
226 | &ac->last_encoding_bit_rate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) | ||
227 | return -1; | ||
228 | |||
229 | LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bit_rate, sampling_rate, channels); | ||
230 | return 0; | ||
231 | } | ||
232 | int ac_reconfigure_test_encoder(ACSession* ac, int32_t bit_rate, int32_t sampling_rate, uint8_t channels) | ||
233 | { | ||
234 | if (!ac || !reconfigure_audio_encoder(&ac->test_encoder, bit_rate, sampling_rate, channels, | ||
235 | &ac->last_encoding_bit_rate, &ac->last_encoding_sampling_rate, &ac->last_encoding_channel_count)) | ||
236 | return -1; | ||
237 | |||
238 | LOGGER_DEBUG ("Reconfigured test audio encoder br: %d sr: %d cc:%d", bit_rate, sampling_rate, channels); | ||
239 | return 0; | ||
240 | } | ||
241 | |||
242 | |||
243 | |||
244 | struct JitterBuffer { | ||
245 | RTPMessage **queue; | ||
246 | uint32_t size; | ||
247 | uint32_t capacity; | ||
248 | uint16_t bottom; | ||
249 | uint16_t top; | ||
250 | }; | ||
251 | |||
252 | static struct JitterBuffer *jbuf_new(uint32_t capacity) | ||
253 | { | ||
254 | unsigned int size = 1; | ||
255 | |||
256 | while (size <= (capacity * 4)) { | ||
257 | size *= 2; | ||
258 | } | ||
259 | |||
260 | struct JitterBuffer *q; | ||
261 | |||
262 | if ( !(q = calloc(sizeof(struct JitterBuffer), 1)) ) return NULL; | ||
263 | |||
264 | if (!(q->queue = calloc(sizeof(RTPMessage *), size))) { | ||
265 | free(q); | ||
266 | return NULL; | ||
267 | } | ||
268 | |||
269 | q->size = size; | ||
270 | q->capacity = capacity; | ||
271 | return q; | ||
272 | } | ||
273 | static void jbuf_clear(struct JitterBuffer *q) | ||
274 | { | ||
275 | for (; q->bottom != q->top; ++q->bottom) { | ||
276 | if (q->queue[q->bottom % q->size]) { | ||
277 | rtp_free_msg(q->queue[q->bottom % q->size]); | ||
278 | q->queue[q->bottom % q->size] = NULL; | ||
279 | } | ||
280 | } | ||
281 | } | ||
282 | static void jbuf_free(struct JitterBuffer *q) | ||
283 | { | ||
284 | if (!q) return; | ||
285 | |||
286 | jbuf_clear(q); | ||
287 | free(q->queue); | ||
288 | free(q); | ||
289 | } | ||
290 | static int jbuf_write(struct JitterBuffer *q, RTPMessage *m) | ||
291 | { | ||
292 | uint16_t sequnum = m->header->sequnum; | ||
293 | |||
294 | unsigned int num = sequnum % q->size; | ||
295 | |||
296 | if ((uint32_t)(sequnum - q->bottom) > q->size) { | ||
297 | LOGGER_DEBUG("Clearing filled jitter buffer: %p", q); | ||
298 | |||
299 | jbuf_clear(q); | ||
300 | q->bottom = sequnum - q->capacity; | ||
301 | q->queue[num] = m; | ||
302 | q->top = sequnum + 1; | ||
303 | return 0; | ||
304 | } | ||
305 | |||
306 | if (q->queue[num]) | ||
307 | return -1; | ||
308 | |||
309 | q->queue[num] = m; | ||
310 | |||
311 | if ((sequnum - q->bottom) >= (q->top - q->bottom)) | ||
312 | q->top = sequnum + 1; | ||
313 | |||
314 | return 0; | ||
315 | } | ||
316 | static RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success) | ||
317 | { | ||
318 | if (q->top == q->bottom) { | ||
319 | *success = 0; | ||
320 | return NULL; | ||
321 | } | ||
322 | |||
323 | unsigned int num = q->bottom % q->size; | ||
324 | |||
325 | if (q->queue[num]) { | ||
326 | RTPMessage *ret = q->queue[num]; | ||
327 | q->queue[num] = NULL; | ||
328 | ++q->bottom; | ||
329 | *success = 1; | ||
330 | return ret; | ||
331 | } | ||
332 | |||
333 | if ((uint32_t)(q->top - q->bottom) > q->capacity) { | ||
334 | ++q->bottom; | ||
335 | *success = 2; | ||
336 | return NULL; | ||
337 | } | ||
338 | |||
339 | *success = 0; | ||
340 | return NULL; | ||
341 | } | ||
342 | OpusEncoder* create_audio_encoder (int32_t bit_rate, int32_t sampling_rate, int32_t channel_count) | ||
343 | { | ||
344 | int status = OPUS_OK; | ||
345 | OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status); | ||
346 | |||
347 | if ( status != OPUS_OK ) { | ||
348 | LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status)); | ||
349 | return NULL; | ||
350 | } | ||
351 | |||
352 | status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bit_rate)); | ||
353 | |||
354 | if ( status != OPUS_OK ) { | ||
355 | LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); | ||
356 | goto FAILURE; | ||
357 | } | ||
358 | |||
359 | status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10)); | ||
360 | |||
361 | if ( status != OPUS_OK ) { | ||
362 | LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); | ||
363 | goto FAILURE; | ||
364 | } | ||
365 | |||
366 | return rc; | ||
367 | |||
368 | FAILURE: | ||
369 | opus_encoder_destroy(rc); | ||
370 | return NULL; | ||
371 | } | ||
372 | bool reconfigure_audio_encoder(OpusEncoder** e, int32_t new_br, int32_t new_sr, uint8_t new_ch, | ||
373 | int32_t* old_br, int32_t* old_sr, int32_t* old_ch) | ||
374 | { | ||
375 | /* Values are checked in toxav.c */ | ||
376 | if (*old_sr != new_sr || *old_ch != new_ch) { | ||
377 | OpusEncoder* new_encoder = create_audio_encoder(new_br, new_sr, new_ch); | ||
378 | if (new_encoder == NULL) | ||
379 | return false; | ||
380 | |||
381 | opus_encoder_destroy(*e); | ||
382 | *e = new_encoder; | ||
383 | } else if (*old_br == new_br) | ||
384 | return true; /* Nothing changed */ | ||
385 | else { | ||
386 | int status = opus_encoder_ctl(*e, OPUS_SET_BITRATE(new_br)); | ||
387 | |||
388 | if ( status != OPUS_OK ) { | ||
389 | LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status)); | ||
390 | return false; | ||
391 | } | ||
392 | } | ||
393 | |||
394 | *old_br = new_br; | ||
395 | *old_sr = new_sr; | ||
396 | *old_ch = new_ch; | ||
397 | |||
398 | return true; | ||
399 | } | ||
400 | bool reconfigure_audio_decoder(ACSession* ac, int32_t sampling_rate, int8_t channels) | ||
401 | { | ||
402 | if (sampling_rate != ac->last_decoding_sampling_rate || channels != ac->last_decoding_channel_count) { | ||
403 | if (current_time_monotonic() - ac->last_decoder_reconfiguration < 500) | ||
404 | return false; | ||
405 | |||
406 | int status; | ||
407 | OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status ); | ||
408 | if ( status != OPUS_OK ) { | ||
409 | LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status)); | ||
410 | return false; | ||
411 | } | ||
412 | |||
413 | ac->last_decoding_sampling_rate = sampling_rate; | ||
414 | ac->last_decoding_channel_count = channels; | ||
415 | ac->last_decoder_reconfiguration = current_time_monotonic(); | ||
416 | |||
417 | opus_decoder_destroy(ac->decoder); | ||
418 | ac->decoder = new_dec; | ||
419 | |||
420 | LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels); | ||
421 | } | ||
422 | |||
423 | return true; | ||
424 | } \ No newline at end of file | ||