summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormannol <eniz_vukovic@hotmail.com>2015-04-22 02:09:37 +0200
committermannol <eniz_vukovic@hotmail.com>2015-04-22 02:09:37 +0200
commit1bfd93e64a2a6d3bf9c90a9aa89abd29f3d826a7 (patch)
tree094af1cad749bef83678071476075160d956bfeb
parent3fd0ee5f0873924b4881b0e33eb1c17ea877ab4a (diff)
Finished refactoring
Diffstat
-rw-r--r--toxav/Makefile.inc2
-rw-r--r--toxav/audio.c62
-rw-r--r--toxav/audio.h3
-rw-r--r--toxav/av_test.c7
-rw-r--r--toxav/codec.c686
-rw-r--r--toxav/codec.h125
-rw-r--r--toxav/msi.c50
-rw-r--r--toxav/rtp.c41
-rw-r--r--toxav/rtp.h46
-rw-r--r--toxav/toxav.c371
-rw-r--r--toxav/video.c70
-rw-r--r--toxav/video.h3
12 files changed, 338 insertions, 1128 deletions
diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc
index d9adb4fe..0434a3c6 100644
--- a/toxav/Makefile.inc
+++ b/toxav/Makefile.inc
@@ -10,8 +10,6 @@ libtoxav_la_SOURCES = ../toxav/rtp.h \
10 ../toxav/msi.c \ 10 ../toxav/msi.c \
11 ../toxav/group.h \ 11 ../toxav/group.h \
12 ../toxav/group.c \ 12 ../toxav/group.c \
13 ../toxav/codec.h \
14 ../toxav/codec.c \
15 ../toxav/audio.h \ 13 ../toxav/audio.h \
16 ../toxav/audio.c \ 14 ../toxav/audio.c \
17 ../toxav/video.h \ 15 ../toxav/video.h \
diff --git a/toxav/audio.c b/toxav/audio.c
index f3e969e9..dc85452a 100644
--- a/toxav/audio.c
+++ b/toxav/audio.c
@@ -83,6 +83,7 @@ ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *c
83 * do error correction with opus */ 83 * do error correction with opus */
84 ac->last_packet_frame_duration = 120; 84 ac->last_packet_frame_duration = 120;
85 ac->last_packet_sampling_rate = 48000; 85 ac->last_packet_sampling_rate = 48000;
86 ac->last_packet_channel_count = 1;
86 87
87 ac->av = av; 88 ac->av = av;
88 ac->friend_id = friend_id; 89 ac->friend_id = friend_id;
@@ -119,7 +120,7 @@ void ac_do(ACSession* ac)
119 return; 120 return;
120 121
121 /* Enough space for the maximum frame size (120 ms 48 KHz audio) */ 122 /* Enough space for the maximum frame size (120 ms 48 KHz audio) */
122 int16_t tmp[5760]; 123 int16_t tmp[5760 * 2];
123 124
124 RTPMessage *msg; 125 RTPMessage *msg;
125 int rc = 0; 126 int rc = 0;
@@ -130,9 +131,8 @@ void ac_do(ACSession* ac)
130 131
131 if (rc == 2) { 132 if (rc == 2) {
132 LOGGER_DEBUG("OPUS correction"); 133 LOGGER_DEBUG("OPUS correction");
133 rc = opus_decode(ac->decoder, NULL, 0, tmp, 134 int fs = (ac->last_packet_sampling_rate * ac->last_packet_frame_duration) / 1000;
134 (ac->last_packet_sampling_rate * ac->last_packet_frame_duration / 1000) / 135 rc = opus_decode(ac->decoder, NULL, 0, tmp, fs, 1);
135 ac->last_packet_channel_count, 1);
136 } else { 136 } else {
137 /* Get values from packet and decode. */ 137 /* Get values from packet and decode. */
138 /* NOTE: This didn't work very well 138 /* NOTE: This didn't work very well
@@ -152,10 +152,9 @@ void ac_do(ACSession* ac)
152 152
153 ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4); 153 ac->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4);
154 154
155 /* 155 /** NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa,
156 * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa, 156 * it didn't work quite well.
157 * it didn't work quite well. 157 */
158 */
159 if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) { 158 if (!reconfigure_audio_decoder(ac, ac->last_packet_sampling_rate, ac->last_packet_channel_count)) {
160 LOGGER_WARNING("Failed to reconfigure decoder!"); 159 LOGGER_WARNING("Failed to reconfigure decoder!");
161 rtp_free_msg(NULL, msg); 160 rtp_free_msg(NULL, msg);
@@ -169,7 +168,7 @@ void ac_do(ACSession* ac)
169 if (rc < 0) { 168 if (rc < 0) {
170 LOGGER_WARNING("Decoding error: %s", opus_strerror(rc)); 169 LOGGER_WARNING("Decoding error: %s", opus_strerror(rc));
171 } else if (ac->acb.first) { 170 } else if (ac->acb.first) {
172 ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate * ac->last_packet_channel_count; 171 ac->last_packet_frame_duration = (rc * 1000) / ac->last_packet_sampling_rate;
173 172
174 ac->acb.first(ac->av, ac->friend_id, tmp, rc * ac->last_packet_channel_count, 173 ac->acb.first(ac->av, ac->friend_id, tmp, rc * ac->last_packet_channel_count,
175 ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second); 174 ac->last_packet_channel_count, ac->last_packet_sampling_rate, ac->acb.second);
@@ -179,6 +178,37 @@ void ac_do(ACSession* ac)
179 } 178 }
180 pthread_mutex_unlock(ac->queue_mutex); 179 pthread_mutex_unlock(ac->queue_mutex);
181} 180}
181int ac_queue_message(void* acp, struct RTPMessage_s *msg)
182{
183 if (!acp || !msg)
184 return -1;
185
186 if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyAudio % 128) {
187 LOGGER_WARNING("Got dummy!");
188 rtp_free_msg(NULL, msg);
189 return 0;
190 }
191
192 if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeAudio % 128) {
193 LOGGER_WARNING("Invalid payload type!");
194 rtp_free_msg(NULL, msg);
195 return -1;
196 }
197
198 ACSession* ac = acp;
199
200 pthread_mutex_lock(ac->queue_mutex);
201 int rc = jbuf_write(ac->j_buf, msg);
202 pthread_mutex_unlock(ac->queue_mutex);
203
204 if (rc == -1) {
205 LOGGER_WARNING("Could not queue the message!");
206 rtp_free_msg(NULL, msg);
207 return -1;
208 }
209
210 return 0;
211}
182int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels) 212int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels)
183{ 213{
184 if (!ac) 214 if (!ac)
@@ -210,21 +240,7 @@ int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate
210 LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels); 240 LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels);
211 return 0; 241 return 0;
212} 242}
213/* called from rtp */
214void ac_queue_message(void* acp, RTPMessage *msg)
215{
216 if (!acp || !msg)
217 return;
218
219 ACSession* ac = acp;
220
221 pthread_mutex_lock(ac->queue_mutex);
222 int ret = jbuf_write(ac->j_buf, msg);
223 pthread_mutex_unlock(ac->queue_mutex);
224 243
225 if (ret == -1)
226 rtp_free_msg(NULL, msg);
227}
228 244
229 245
230/* JITTER BUFFER WORK */ 246/* JITTER BUFFER WORK */
diff --git a/toxav/audio.h b/toxav/audio.h
index 62a28cdf..2cb0d8f6 100644
--- a/toxav/audio.h
+++ b/toxav/audio.h
@@ -29,6 +29,8 @@
29 29
30#include "../toxcore/util.h" 30#include "../toxcore/util.h"
31 31
32struct RTPMessage_s;
33
32typedef struct ACSession_s { 34typedef struct ACSession_s {
33 /* encoding */ 35 /* encoding */
34 OpusEncoder *encoder; 36 OpusEncoder *encoder;
@@ -56,5 +58,6 @@ typedef struct ACSession_s {
56ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data); 58ACSession* ac_new(ToxAV* av, uint32_t friend_id, toxav_receive_audio_frame_cb *cb, void *cb_data);
57void ac_kill(ACSession* ac); 59void ac_kill(ACSession* ac);
58void ac_do(ACSession* ac); 60void ac_do(ACSession* ac);
61int ac_queue_message(void *acp, struct RTPMessage_s *msg);
59int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels); 62int ac_reconfigure_encoder(ACSession* ac, int32_t bitrate, int32_t sampling_rate, uint8_t channels);
60#endif /* AUDIO_H */ \ No newline at end of file 63#endif /* AUDIO_H */ \ No newline at end of file
diff --git a/toxav/av_test.c b/toxav/av_test.c
index 994f9cf6..dce63184 100644
--- a/toxav/av_test.c
+++ b/toxav/av_test.c
@@ -77,8 +77,8 @@
77#define TEST_REJECT 0 77#define TEST_REJECT 0
78#define TEST_CANCEL 0 78#define TEST_CANCEL 0
79#define TEST_MUTE_UNMUTE 0 79#define TEST_MUTE_UNMUTE 0
80#define TEST_TRANSFER_A 1 80#define TEST_TRANSFER_A 0
81#define TEST_TRANSFER_V 0 81#define TEST_TRANSFER_V 1
82 82
83 83
84typedef struct { 84typedef struct {
@@ -329,6 +329,9 @@ void* iterate_toxav (void * data)
329 toxav_iterate(data_cast->BobAV); 329 toxav_iterate(data_cast->BobAV);
330 int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV)); 330 int rc = MIN(toxav_iteration_interval(data_cast->AliceAV), toxav_iteration_interval(data_cast->BobAV));
331 331
332 printf("\rIteration interval: %d ", rc);
333 fflush(stdout);
334
332#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1 335#if defined TEST_TRANSFER_V && TEST_TRANSFER_V == 1
333 cvWaitKey(rc); 336 cvWaitKey(rc);
334#else 337#else
diff --git a/toxav/codec.c b/toxav/codec.c
deleted file mode 100644
index 57e43c67..00000000
--- a/toxav/codec.c
+++ /dev/null
@@ -1,686 +0,0 @@
1/** codec.c
2 *
3 * Copyright (C) 2013-2015 Tox project All Rights Reserved.
4 *
5 * This file is part of Tox.
6 *
7 * Tox is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * Tox is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with Tox. If not, see <http://www.gnu.org/licenses/>.
19 *
20 */
21
22
23#ifdef HAVE_CONFIG_H
24#include "config.h"
25#endif /* HAVE_CONFIG_H */
26
27#include "../toxcore/logger.h"
28#include "../toxcore/util.h"
29
30#include <stdio.h>
31#include <stdlib.h>
32#include <stdbool.h>
33#include <math.h>
34#include <assert.h>
35#include <time.h>
36
37#include "msi.h"
38#include "rtp.h"
39#include "codec.h"
40
41#define DEFAULT_JBUF 3
42
43/* Good quality encode. */
44#define MAX_DECODE_TIME_US 0
45
46#define MAX_VIDEOFRAME_SIZE 0x40000 /* 256KiB */
47#define VIDEOFRAME_HEADER_SIZE 0x2
48
49/* FIXME: Might not be enough? NOTE: I think it is enough */
50#define VIDEO_DECODE_BUFFER_SIZE 20
51
52#define ARRAY(TYPE__) struct { uint16_t size; TYPE__ data[]; }
53
54typedef ARRAY(uint8_t) Payload;
55
56/* JITTER BUFFER WORK */
57typedef struct JitterBuffer_s {
58 RTPMessage **queue;
59 uint32_t size;
60 uint32_t capacity;
61 uint16_t bottom;
62 uint16_t top;
63} JitterBuffer;
64
65static JitterBuffer *jbuf_new(uint32_t capacity)
66{
67 unsigned int size = 1;
68
69 while (size <= (capacity * 4)) {
70 size *= 2;
71 }
72
73 JitterBuffer *q;
74
75 if ( !(q = calloc(sizeof(JitterBuffer), 1)) ) return NULL;
76
77 if (!(q->queue = calloc(sizeof(RTPMessage *), size))) {
78 free(q);
79 return NULL;
80 }
81
82 q->size = size;
83 q->capacity = capacity;
84 return q;
85}
86
87static void jbuf_clear(JitterBuffer *q)
88{
89 for (; q->bottom != q->top; ++q->bottom) {
90 if (q->queue[q->bottom % q->size]) {
91 rtp_free_msg(NULL, q->queue[q->bottom % q->size]);
92 q->queue[q->bottom % q->size] = NULL;
93 }
94 }
95}
96
97static void jbuf_free(JitterBuffer *q)
98{
99 if (!q) return;
100
101 jbuf_clear(q);
102 free(q->queue);
103 free(q);
104}
105
106static int jbuf_write(JitterBuffer *q, RTPMessage *m)
107{
108 uint16_t sequnum = m->header->sequnum;
109
110 unsigned int num = sequnum % q->size;
111
112 if ((uint32_t)(sequnum - q->bottom) > q->size) {
113 LOGGER_DEBUG("Clearing filled jitter buffer: %p", q);
114
115 jbuf_clear(q);
116 q->bottom = sequnum - q->capacity;
117 q->queue[num] = m;
118 q->top = sequnum + 1;
119 return 0;
120 }
121
122 if (q->queue[num])
123 return -1;
124
125 q->queue[num] = m;
126
127 if ((sequnum - q->bottom) >= (q->top - q->bottom))
128 q->top = sequnum + 1;
129
130 return 0;
131}
132
133/* success is set to 0 when there is nothing to dequeue,
134 * 1 when there's a good packet,
135 * 2 when there's a lost packet */
136static RTPMessage *jbuf_read(JitterBuffer *q, int32_t *success)
137{
138 if (q->top == q->bottom) {
139 *success = 0;
140 return NULL;
141 }
142
143 unsigned int num = q->bottom % q->size;
144
145 if (q->queue[num]) {
146 RTPMessage *ret = q->queue[num];
147 q->queue[num] = NULL;
148 ++q->bottom;
149 *success = 1;
150 return ret;
151 }
152
153 if ((uint32_t)(q->top - q->bottom) > q->capacity) {
154 ++q->bottom;
155 *success = 2;
156 return NULL;
157 }
158
159 *success = 0;
160 return NULL;
161}
162
163static int convert_bw_to_sampling_rate(int bw)
164{
165 switch(bw)
166 {
167 case OPUS_BANDWIDTH_NARROWBAND: return 8000;
168 case OPUS_BANDWIDTH_MEDIUMBAND: return 12000;
169 case OPUS_BANDWIDTH_WIDEBAND: return 16000;
170 case OPUS_BANDWIDTH_SUPERWIDEBAND: return 24000;
171 case OPUS_BANDWIDTH_FULLBAND: return 48000;
172 default: return -1;
173 }
174}
175
176OpusEncoder* create_audio_encoder (int32_t bitrate, int32_t sampling_rate, int32_t channel_count)
177{
178 int status = OPUS_OK;
179 OpusEncoder* rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_AUDIO, &status);
180
181 if ( status != OPUS_OK ) {
182 LOGGER_ERROR("Error while starting audio encoder: %s", opus_strerror(status));
183 return NULL;
184 }
185
186 status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bitrate));
187
188 if ( status != OPUS_OK ) {
189 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
190 goto FAILURE;
191 }
192
193 status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(10));
194
195 if ( status != OPUS_OK ) {
196 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
197 goto FAILURE;
198 }
199
200 return rc;
201
202FAILURE:
203 opus_encoder_destroy(rc);
204 return NULL;
205}
206
207bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate)
208{
209 assert(dest);
210
211 vpx_codec_enc_cfg_t cfg;
212 int rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
213
214 if (rc != VPX_CODEC_OK) {
215 LOGGER_ERROR("Failed to get config: %s", vpx_codec_err_to_string(rc));
216 return false;
217 }
218
219 rc = vpx_codec_enc_init_ver(dest, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0,
220 VPX_ENCODER_ABI_VERSION);
221
222 if ( rc != VPX_CODEC_OK) {
223 LOGGER_ERROR("Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
224 return false;
225 }
226
227 cfg.rc_target_bitrate = bitrate;
228 cfg.g_w = 800;
229 cfg.g_h = 600;
230 cfg.g_pass = VPX_RC_ONE_PASS;
231 cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
232 cfg.g_lag_in_frames = 0;
233 cfg.kf_min_dist = 0;
234 cfg.kf_max_dist = 48;
235 cfg.kf_mode = VPX_KF_AUTO;
236
237 rc = vpx_codec_control(dest, VP8E_SET_CPUUSED, 8);
238
239 if ( rc != VPX_CODEC_OK) {
240 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
241 vpx_codec_destroy(dest);
242 }
243
244 return true;
245}
246
247bool reconfigure_audio_decoder(CSession* cs, int32_t sampling_rate, int8_t channels)
248{
249 if (sampling_rate != cs->last_decoding_sampling_rate || channels != cs->last_decoding_channel_count) {
250 if (current_time_monotonic() - cs->last_decoder_reconfiguration < 500)
251 return false;
252
253 int status;
254 OpusDecoder* new_dec = opus_decoder_create(sampling_rate, channels, &status );
255 if ( status != OPUS_OK ) {
256 LOGGER_ERROR("Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status));
257 return false;
258 }
259
260 cs->last_decoding_sampling_rate = sampling_rate;
261 cs->last_decoding_channel_count = channels;
262 cs->last_decoder_reconfiguration = current_time_monotonic();
263
264 opus_decoder_destroy(cs->audio_decoder);
265 cs->audio_decoder = new_dec;
266
267 LOGGER_DEBUG("Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels);
268 }
269
270 return true;
271}
272
273/* PUBLIC */
274void cs_do(CSession *cs)
275{
276 /* Codec session should always be protected by call mutex so no need to check for cs validity
277 */
278
279 if (!cs)
280 return;
281
282 Payload *p;
283 int rc;
284
285 int success = 0;
286
287 LOGGED_LOCK(cs->queue_mutex);
288
289 /********************* AUDIO *********************/
290 if (cs->audio_decoder) {
291 RTPMessage *msg;
292
293 /* The maximum for 120 ms 48 KHz audio */
294 int16_t tmp[5760];
295
296 while ((msg = jbuf_read(cs->j_buf, &success)) || success == 2) {
297 LOGGED_UNLOCK(cs->queue_mutex);
298
299 if (success == 2) {
300 LOGGER_DEBUG("OPUS correction");
301 rc = opus_decode(cs->audio_decoder, NULL, 0, tmp,
302 (cs->last_packet_sampling_rate * cs->last_packet_frame_duration / 1000) /
303 cs->last_packet_channel_count, 1);
304 } else {
305 /* Get values from packet and decode. */
306 /* NOTE: This didn't work very well
307 rc = convert_bw_to_sampling_rate(opus_packet_get_bandwidth(msg->data));
308 if (rc != -1) {
309 cs->last_packet_sampling_rate = rc;
310 } else {
311 LOGGER_WARNING("Failed to load packet values!");
312 rtp_free_msg(NULL, msg);
313 continue;
314 }*/
315
316
317 /* Pick up sampling rate from packet */
318 memcpy(&cs->last_packet_sampling_rate, msg->data, 4);
319 cs->last_packet_sampling_rate = ntohl(cs->last_packet_sampling_rate);
320
321 cs->last_packet_channel_count = opus_packet_get_nb_channels(msg->data + 4);
322
323 /*
324 * NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa,
325 * it didn't work quite well.
326 */
327 if (!reconfigure_audio_decoder(cs, cs->last_packet_sampling_rate, cs->last_packet_channel_count)) {
328 LOGGER_WARNING("Failed to reconfigure decoder!");
329 rtp_free_msg(NULL, msg);
330 continue;
331 }
332
333 rc = opus_decode(cs->audio_decoder, msg->data + 4, msg->length - 4, tmp, 5760, 0);
334 rtp_free_msg(NULL, msg);
335 }
336
337 if (rc < 0) {
338 LOGGER_WARNING("Decoding error: %s", opus_strerror(rc));
339 } else if (cs->acb.first) {
340 cs->last_packet_frame_duration = (rc * 1000) / cs->last_packet_sampling_rate * cs->last_packet_channel_count;
341
342 cs->acb.first(cs->av, cs->friend_id, tmp, rc * cs->last_packet_channel_count,
343 cs->last_packet_channel_count, cs->last_packet_sampling_rate, cs->acb.second);
344
345 }
346
347 LOGGED_LOCK(cs->queue_mutex);
348 }
349 }
350
351 /********************* VIDEO *********************/
352 if (cs->vbuf_raw && !rb_empty(cs->vbuf_raw)) {
353 /* Decode video */
354 rb_read(cs->vbuf_raw, (void**)&p);
355
356 /* Leave space for (possibly) other thread to queue more data after we read it here */
357 LOGGED_UNLOCK(cs->queue_mutex);
358
359 rc = vpx_codec_decode(cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US);
360 free(p);
361
362 if (rc != VPX_CODEC_OK) {
363 LOGGER_ERROR("Error decoding video: %s", vpx_codec_err_to_string(rc));
364 } else {
365 vpx_codec_iter_t iter = NULL;
366 vpx_image_t *dest = vpx_codec_get_frame(cs->v_decoder, &iter);
367
368 /* Play decoded images */
369 for (; dest; dest = vpx_codec_get_frame(cs->v_decoder, &iter)) {
370 if (cs->vcb.first)
371 cs->vcb.first(cs->av, cs->friend_id, dest->d_w, dest->d_h,
372 (const uint8_t*)dest->planes[0], (const uint8_t*)dest->planes[1], (const uint8_t*)dest->planes[2],
373 dest->stride[0], dest->stride[1], dest->stride[2], cs->vcb.second);
374
375 vpx_img_free(dest);
376 }
377 }
378
379 return;
380 }
381
382 LOGGED_UNLOCK(cs->queue_mutex);
383}
384CSession *cs_new(uint32_t peer_video_frame_piece_size)
385{
386 CSession *cs = calloc(sizeof(CSession), 1);
387
388 if (!cs) {
389 LOGGER_WARNING("Allocation failed! Application might misbehave!");
390 return NULL;
391 }
392
393 if (create_recursive_mutex(cs->queue_mutex) != 0) {
394 LOGGER_WARNING("Failed to create recursive mutex!");
395 free(cs);
396 return NULL;
397 }
398
399 /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
400 /* Create decoders and set up their values
401 */
402
403 /*
404 * AUDIO
405 */
406
407 int status;
408 cs->audio_decoder = opus_decoder_create(48000, 2, &status );
409
410 if ( status != OPUS_OK ) {
411 LOGGER_ERROR("Error while starting audio decoder: %s", opus_strerror(status));
412 goto FAILURE;
413 }
414
415 cs->last_decoding_channel_count = 2;
416 cs->last_decoding_sampling_rate = 48000;
417 cs->last_decoder_reconfiguration = 0; /* Make it possible to reconfigure straight away */
418
419 /* These need to be set in order to properly
420 * do error correction with opus */
421 cs->last_packet_frame_duration = 120;
422 cs->last_packet_sampling_rate = 48000;
423
424 if ( !(cs->j_buf = jbuf_new(DEFAULT_JBUF)) ) {
425 LOGGER_WARNING("Jitter buffer creaton failed!");
426 opus_decoder_destroy(cs->audio_decoder);
427 goto FAILURE;
428 }
429
430 /*
431 * VIDEO
432 */
433 int rc = vpx_codec_dec_init_ver(cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE,
434 NULL, 0, VPX_DECODER_ABI_VERSION);
435
436 if ( rc != VPX_CODEC_OK) {
437 LOGGER_ERROR("Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
438 goto AUDIO_DECODER_CLEANUP;
439 }
440
441 if ( !(cs->frame_buf = calloc(MAX_VIDEOFRAME_SIZE, 1)) ) {
442 vpx_codec_destroy(cs->v_decoder);
443 goto AUDIO_DECODER_CLEANUP;
444 }
445
446 if ( !(cs->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE)) ) {
447 free(cs->frame_buf);
448 vpx_codec_destroy(cs->v_decoder);
449 goto AUDIO_DECODER_CLEANUP;
450 }
451
452 if ( !(cs->split_video_frame = calloc(VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE, 1)) )
453 goto FAILURE;
454
455 cs->linfts = current_time_monotonic();
456 cs->lcfd = 60;
457 /*++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
458
459 /* Initialize encoders with default values */
460 cs->audio_encoder = create_audio_encoder(48000, 48000, 2);
461 if (cs->audio_encoder == NULL)
462 goto VIDEO_DECODER_CLEANUP;
463
464 cs->last_encoding_bitrate = 48000;
465 cs->last_encoding_sampling_rate = 48000;
466 cs->last_encoding_channel_count = 2;
467
468 if (!create_video_encoder(cs->v_encoder, 500000)) {
469 opus_encoder_destroy(cs->audio_encoder);
470 goto VIDEO_DECODER_CLEANUP;
471 }
472
473 cs->peer_video_frame_piece_size = peer_video_frame_piece_size;
474
475 return cs;
476
477VIDEO_DECODER_CLEANUP:
478 rb_free(cs->vbuf_raw);
479 free(cs->frame_buf);
480 vpx_codec_destroy(cs->v_decoder);
481AUDIO_DECODER_CLEANUP:
482 opus_decoder_destroy(cs->audio_decoder);
483 jbuf_free(cs->j_buf);
484FAILURE:
485 pthread_mutex_destroy(cs->queue_mutex);
486 free(cs);
487 return NULL;
488}
489void cs_kill(CSession *cs)
490{
491 if (!cs)
492 return;
493
494 /* NOTE: queue_message() will not be called since
495 * the callback is unregistered before cs_kill is called.
496 */
497
498 opus_encoder_destroy(cs->audio_encoder);
499 opus_decoder_destroy(cs->audio_decoder);
500 jbuf_free(cs->j_buf);
501 vpx_codec_destroy(cs->v_encoder);
502 vpx_codec_destroy(cs->v_decoder);
503 rb_free(cs->vbuf_raw);
504 free(cs->frame_buf);
505 free(cs->split_video_frame);
506
507 pthread_mutex_destroy(cs->queue_mutex);
508
509 LOGGER_DEBUG("Terminated codec state: %p", cs);
510 free(cs);
511}
512void cs_init_video_splitter_cycle(CSession* cs)
513{
514 cs->split_video_frame[0] = cs->frameid_out++;
515 cs->split_video_frame[1] = 0;
516}
517int cs_update_video_splitter_cycle(CSession *cs, const uint8_t *payload, uint16_t length)
518{
519 cs->processing_video_frame = payload;
520 cs->processing_video_frame_size = length;
521
522 return ((length - 1) / VIDEOFRAME_PIECE_SIZE) + 1;
523}
524const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size)
525{
526 if (!cs || !size) return NULL;
527
528 if (cs->processing_video_frame_size > VIDEOFRAME_PIECE_SIZE) {
529 memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE,
530 cs->processing_video_frame,
531 VIDEOFRAME_PIECE_SIZE);
532
533 cs->processing_video_frame += VIDEOFRAME_PIECE_SIZE;
534 cs->processing_video_frame_size -= VIDEOFRAME_PIECE_SIZE;
535
536 *size = VIDEOFRAME_PIECE_SIZE + VIDEOFRAME_HEADER_SIZE;
537 } else {
538 memcpy(cs->split_video_frame + VIDEOFRAME_HEADER_SIZE,
539 cs->processing_video_frame,
540 cs->processing_video_frame_size);
541
542 *size = cs->processing_video_frame_size + VIDEOFRAME_HEADER_SIZE;
543 }
544
545 cs->split_video_frame[1]++;
546
547 return cs->split_video_frame;
548}
549int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height)
550{
551 vpx_codec_enc_cfg_t cfg = *cs->v_encoder[0].config.enc;
552 if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height)
553 return 0; /* Nothing changed */
554
555 cfg.rc_target_bitrate = bitrate;
556 cfg.g_w = width;
557 cfg.g_h = height;
558
559 int rc = vpx_codec_enc_config_set(cs->v_encoder, &cfg);
560 if ( rc != VPX_CODEC_OK) {
561 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
562 return -1;
563 }
564
565 return 0;
566}
567int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels)
568{
569 /* Values are checked in toxav.c */
570
571 if (cs->last_encoding_sampling_rate != sampling_rate || cs->last_encoding_channel_count != channels) {
572 OpusEncoder* new_encoder = create_audio_encoder(bitrate, sampling_rate, channels);
573 if (new_encoder == NULL)
574 return -1;
575
576 opus_encoder_destroy(cs->audio_encoder);
577 cs->audio_encoder = new_encoder;
578 } else if (cs->last_encoding_bitrate == bitrate)
579 return 0; /* Nothing changed */
580 else {
581 int status = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(bitrate));
582
583 if ( status != OPUS_OK ) {
584 LOGGER_ERROR("Error while setting encoder ctl: %s", opus_strerror(status));
585 return -1;
586 }
587 }
588
589 cs->last_encoding_bitrate = bitrate;
590 cs->last_encoding_sampling_rate = sampling_rate;
591 cs->last_encoding_channel_count = channels;
592
593 LOGGER_DEBUG ("Reconfigured audio encoder br: %d sr: %d cc:%d", bitrate, sampling_rate, channels);
594 return 0;
595}
596/* Called from RTP */
597void queue_message(RTPSession *session, RTPMessage *msg)
598{
599 CSession *cs = session->cs;
600
601 if (!cs)
602 return;
603
604 /* Audio */
605 if (session->payload_type == rtp_TypeAudio % 128) {
606 LOGGED_LOCK(cs->queue_mutex);
607 int ret = jbuf_write(cs->j_buf, msg);
608 LOGGED_UNLOCK(cs->queue_mutex);
609
610 if (ret == -1) {
611 rtp_free_msg(NULL, msg);
612 }
613 }
614 /* Video */
615 else {
616 uint8_t *packet = msg->data;
617 uint32_t packet_size = msg->length;
618
619 if (packet_size < VIDEOFRAME_HEADER_SIZE)
620 goto end;
621
622 uint8_t diff = packet[0] - cs->frameid_in;
623
624 if (diff != 0) {
625 if (diff < 225) { /* New frame */
626 /* Flush last frames' data and get ready for this frame */
627 Payload *p = malloc(sizeof(Payload) + cs->frame_size);
628
629 if (p) {
630 LOGGED_LOCK(cs->queue_mutex);
631
632 if (rb_full(cs->vbuf_raw)) {
633 LOGGER_DEBUG("Dropped video frame");
634 Payload *tp;
635 rb_read(cs->vbuf_raw, (void**)&tp);
636 free(tp);
637 } else {
638 p->size = cs->frame_size;
639 memcpy(p->data, cs->frame_buf, cs->frame_size);
640 }
641
642 /* Calculate time took for peer to send us this frame */
643 uint32_t t_lcfd = current_time_monotonic() - cs->linfts;
644 cs->lcfd = t_lcfd > 100 ? cs->lcfd : t_lcfd;
645 cs->linfts = current_time_monotonic();
646
647 rb_write(cs->vbuf_raw, p);
648 LOGGED_UNLOCK(cs->queue_mutex);
649 } else {
650 LOGGER_WARNING("Allocation failed! Program might misbehave!");
651 goto end;
652 }
653
654 cs->frameid_in = packet[0];
655 memset(cs->frame_buf, 0, cs->frame_size);
656 cs->frame_size = 0;
657
658 } else { /* Old frame; drop */
659 LOGGER_DEBUG("Old packet: %u", packet[0]);
660 goto end;
661 }
662 }
663
664 uint8_t piece_number = packet[1];
665
666 uint32_t length_before_piece = ((piece_number - 1) * cs->peer_video_frame_piece_size);
667 uint32_t framebuf_new_length = length_before_piece + (packet_size - VIDEOFRAME_HEADER_SIZE);
668
669 if (framebuf_new_length > MAX_VIDEOFRAME_SIZE) {
670 goto end;
671 }
672
673 /* Otherwise it's part of the frame so just process */
674 /* LOGGER_DEBUG("Video Packet: %u %u", packet[0], packet[1]); */
675
676 memcpy(cs->frame_buf + length_before_piece,
677 packet + VIDEOFRAME_HEADER_SIZE,
678 packet_size - VIDEOFRAME_HEADER_SIZE);
679
680 if (framebuf_new_length > cs->frame_size)
681 cs->frame_size = framebuf_new_length;
682
683end:
684 rtp_free_msg(NULL, msg);
685 }
686}
diff --git a/toxav/codec.h b/toxav/codec.h
deleted file mode 100644
index 497016eb..00000000
--- a/toxav/codec.h
+++ /dev/null
@@ -1,125 +0,0 @@
1/** codec.h
2 *
3 * Copyright (C) 2013-2015 Tox project All Rights Reserved.
4 *
5 * This file is part of Tox.
6 *
7 * Tox is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * Tox is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with Tox. If not, see <http://www.gnu.org/licenses/>.
19 *
20 */
21
22#ifndef CODEC_H
23#define CODEC_H
24
25#include "toxav.h"
26#include "rtp.h"
27
28#include "../toxcore/util.h"
29
30#include <stdio.h>
31#include <math.h>
32#include <pthread.h>
33
34#include <vpx/vpx_decoder.h>
35#include <vpx/vpx_encoder.h>
36#include <vpx/vp8dx.h>
37#include <vpx/vp8cx.h>
38#include <vpx/vpx_image.h>
39#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx())
40#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx())
41
42/* Audio encoding/decoding */
43#include <opus.h>
44
45typedef struct CSession_s {
46
47 /* VIDEO
48 *
49 *
50 */
51
52 /* video encoding */
53 vpx_codec_ctx_t v_encoder[1];
54 uint32_t frame_counter;
55
56 /* video decoding */
57 vpx_codec_ctx_t v_decoder[1];
58 void *vbuf_raw; /* Un-decoded data */
59
60 /* Data handling */
61 uint8_t *frame_buf; /* buffer for split video payloads */
62 uint32_t frame_size; /* largest address written to in frame_buf for current input frame */
63 uint8_t frameid_in, frameid_out; /* id of input and output video frame */
64 uint64_t linfts; /* Last received frame time stamp */
65 uint32_t lcfd; /* Last calculated frame duration for incoming video payload */
66
67 /* Limits */
68 uint32_t peer_video_frame_piece_size;
69
70 /* Splitting */
71 uint8_t *split_video_frame;
72 const uint8_t *processing_video_frame;
73 uint16_t processing_video_frame_size;
74
75
76
77 /* AUDIO
78 *
79 *
80 */
81
82 /* audio encoding */
83 OpusEncoder *audio_encoder;
84 int32_t last_encoding_sampling_rate;
85 int32_t last_encoding_channel_count;
86 int32_t last_encoding_bitrate;
87
88 /* audio decoding */
89 OpusDecoder *audio_decoder;
90 int32_t last_packet_channel_count;
91 int32_t last_packet_sampling_rate;
92 int32_t last_packet_frame_duration;
93 int32_t last_decoding_sampling_rate;
94 int32_t last_decoding_channel_count;
95 uint64_t last_decoder_reconfiguration;
96 struct JitterBuffer_s *j_buf;
97
98
99 /* OTHER
100 *
101 *
102 */
103 ToxAV *av;
104 int32_t friend_id;
105
106 PAIR(toxav_receive_audio_frame_cb *, void *) acb; /* Audio frame receive callback */
107 PAIR(toxav_receive_video_frame_cb *, void *) vcb; /* Video frame receive callback */
108
109 pthread_mutex_t queue_mutex[1];
110} CSession;
111
112
113void cs_do(CSession *cs);
114/* Make sure to be called BEFORE corresponding rtp_new */
115CSession *cs_new(uint32_t peer_mvfpsz);
116/* Make sure to be called AFTER corresponding rtp_kill */
117void cs_kill(CSession *cs);
118
119void cs_init_video_splitter_cycle(CSession *cs);
120int cs_update_video_splitter_cycle(CSession* cs, const uint8_t* payload, uint16_t length);
121const uint8_t *cs_iterate_split_video_frame(CSession *cs, uint16_t *size);
122
123int cs_reconfigure_video_encoder(CSession* cs, int32_t bitrate, uint16_t width, uint16_t height);
124int cs_reconfigure_audio_encoder(CSession* cs, int32_t bitrate, int32_t sampling_rate, uint8_t channels);
125#endif /* CODEC_H */
diff --git a/toxav/msi.c b/toxav/msi.c
index b7926e07..0bd04c56 100644
--- a/toxav/msi.c
+++ b/toxav/msi.c
@@ -99,9 +99,9 @@ void handle_msi_packet ( Messenger *m, uint32_t friend_id, const uint8_t *data,
99 */ 99 */
100void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id) 100void msi_register_callback ( MSISession *session, MSICallbackType callback, MSICallbackID id)
101{ 101{
102 LOGGED_LOCK(session->mutex); 102 pthread_mutex_lock(session->mutex);
103 session->callbacks[id] = callback; 103 session->callbacks[id] = callback;
104 LOGGED_UNLOCK(session->mutex); 104 pthread_mutex_unlock(session->mutex);
105} 105}
106MSISession *msi_new ( Messenger *messenger ) 106MSISession *msi_new ( Messenger *messenger )
107{ 107{
@@ -141,7 +141,7 @@ int msi_kill ( MSISession *session )
141 } 141 }
142 142
143 m_callback_msi_packet((struct Messenger *) session->messenger, NULL, NULL); 143 m_callback_msi_packet((struct Messenger *) session->messenger, NULL, NULL);
144 LOGGED_LOCK(session->mutex); 144 pthread_mutex_lock(session->mutex);
145 145
146 if (session->calls) { 146 if (session->calls) {
147 MSIMessage msg; 147 MSIMessage msg;
@@ -154,7 +154,7 @@ int msi_kill ( MSISession *session )
154 } 154 }
155 } 155 }
156 156
157 LOGGED_UNLOCK(session->mutex); 157 pthread_mutex_unlock(session->mutex);
158 pthread_mutex_destroy(session->mutex); 158 pthread_mutex_destroy(session->mutex);
159 159
160 LOGGER_DEBUG("Terminated session: %p", session); 160 LOGGER_DEBUG("Terminated session: %p", session);
@@ -165,17 +165,17 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_
165{ 165{
166 LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id); 166 LOGGER_DEBUG("Session: %p Inviting friend: %u", session, friend_id);
167 167
168 LOGGED_LOCK(session->mutex); 168 pthread_mutex_lock(session->mutex);
169 if (get_call(session, friend_id) != NULL) { 169 if (get_call(session, friend_id) != NULL) {
170 LOGGER_ERROR("Already in a call"); 170 LOGGER_ERROR("Already in a call");
171 LOGGED_UNLOCK(session->mutex); 171 pthread_mutex_unlock(session->mutex);
172 return -1; 172 return -1;
173 } 173 }
174 174
175 (*call) = new_call ( session, friend_id ); 175 (*call) = new_call ( session, friend_id );
176 176
177 if ( *call == NULL ) { 177 if ( *call == NULL ) {
178 LOGGED_UNLOCK(session->mutex); 178 pthread_mutex_unlock(session->mutex);
179 return -1; 179 return -1;
180 } 180 }
181 181
@@ -195,7 +195,7 @@ int msi_invite ( MSISession *session, MSICall **call, uint32_t friend_id, uint8_
195 (*call)->state = msi_CallRequesting; 195 (*call)->state = msi_CallRequesting;
196 196
197 LOGGER_DEBUG("Invite sent"); 197 LOGGER_DEBUG("Invite sent");
198 LOGGED_UNLOCK(session->mutex); 198 pthread_mutex_unlock(session->mutex);
199 return 0; 199 return 0;
200} 200}
201int msi_hangup ( MSICall* call ) 201int msi_hangup ( MSICall* call )
@@ -203,7 +203,7 @@ int msi_hangup ( MSICall* call )
203 LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id); 203 LOGGER_DEBUG("Session: %p Hanging up call with friend: %u", call->session, call->friend_id);
204 204
205 MSISession* session = call->session; 205 MSISession* session = call->session;
206 LOGGED_LOCK(session->mutex); 206 pthread_mutex_lock(session->mutex);
207 207
208 MSIMessage msg; 208 MSIMessage msg;
209 msg_init(&msg, requ_pop); 209 msg_init(&msg, requ_pop);
@@ -211,7 +211,7 @@ int msi_hangup ( MSICall* call )
211 send_message ( session->messenger, call->friend_id, &msg ); 211 send_message ( session->messenger, call->friend_id, &msg );
212 212
213 kill_call(call); 213 kill_call(call);
214 LOGGED_UNLOCK(session->mutex); 214 pthread_mutex_unlock(session->mutex);
215 return 0; 215 return 0;
216} 216}
217int msi_answer ( MSICall* call, uint8_t capabilities ) 217int msi_answer ( MSICall* call, uint8_t capabilities )
@@ -219,13 +219,13 @@ int msi_answer ( MSICall* call, uint8_t capabilities )
219 LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id); 219 LOGGER_DEBUG("Session: %p Answering call from: %u", call->session, call->friend_id);
220 220
221 MSISession* session = call->session; 221 MSISession* session = call->session;
222 LOGGED_LOCK(session->mutex); 222 pthread_mutex_lock(session->mutex);
223 223
224 if ( call->state != msi_CallRequested ) { 224 if ( call->state != msi_CallRequested ) {
225 /* Though sending in invalid state will not cause anything wierd 225 /* Though sending in invalid state will not cause anything wierd
226 * Its better to not do it like a maniac */ 226 * Its better to not do it like a maniac */
227 LOGGER_ERROR("Call is in invalid state!"); 227 LOGGER_ERROR("Call is in invalid state!");
228 LOGGED_UNLOCK(session->mutex); 228 pthread_mutex_unlock(session->mutex);
229 return -1; 229 return -1;
230 } 230 }
231 231
@@ -243,7 +243,7 @@ int msi_answer ( MSICall* call, uint8_t capabilities )
243 send_message ( session->messenger, call->friend_id, &msg ); 243 send_message ( session->messenger, call->friend_id, &msg );
244 244
245 call->state = msi_CallActive; 245 call->state = msi_CallActive;
246 LOGGED_UNLOCK(session->mutex); 246 pthread_mutex_unlock(session->mutex);
247 247
248 return 0; 248 return 0;
249} 249}
@@ -252,7 +252,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities )
252 LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id); 252 LOGGER_DEBUG("Session: %p Trying to change capabilities to friend %u", call->session, call->friend_id);
253 253
254 MSISession* session = call->session; 254 MSISession* session = call->session;
255 LOGGED_LOCK(session->mutex); 255 pthread_mutex_lock(session->mutex);
256 256
257 if ( call->state != msi_CallActive ) { 257 if ( call->state != msi_CallActive ) {
258 /* Sending capabilities change can cause error on other side if 258 /* Sending capabilities change can cause error on other side if
@@ -263,7 +263,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities )
263 * like new. TODO: explain this better 263 * like new. TODO: explain this better
264 */ 264 */
265 LOGGER_ERROR("Call is in invalid state!"); 265 LOGGER_ERROR("Call is in invalid state!");
266 LOGGED_UNLOCK(session->mutex); 266 pthread_mutex_unlock(session->mutex);
267 return -1; 267 return -1;
268 } 268 }
269 269
@@ -277,7 +277,7 @@ int msi_change_capabilities( MSICall* call, uint8_t capabilities )
277 277
278 send_message ( call->session->messenger, call->friend_id, &msg ); 278 send_message ( call->session->messenger, call->friend_id, &msg );
279 279
280 LOGGED_UNLOCK(session->mutex); 280 pthread_mutex_unlock(session->mutex);
281 return 0; 281 return 0;
282} 282}
283 283
@@ -349,7 +349,7 @@ int msg_parse_in ( MSIMessage *dest, const uint8_t *data, uint16_t length )
349 case IDVFPSZ: 349 case IDVFPSZ:
350 CHECK_SIZE(it, size_constraint, 2); 350 CHECK_SIZE(it, size_constraint, 2);
351 SET_UINT16(it, dest->vfpsz); 351 SET_UINT16(it, dest->vfpsz);
352 dest->vfpsz = ntohs(dest->vfpsz); 352 dest->vfpsz.value = ntohs(dest->vfpsz.value);
353 353
354 if (dest->vfpsz.value > 1200) { 354 if (dest->vfpsz.value > 1200) {
355 LOGGER_ERROR("Invalid vfpsz param"); 355 LOGGER_ERROR("Invalid vfpsz param");
@@ -425,7 +425,7 @@ int send_message ( Messenger* m, uint32_t friend_id, const MSIMessage *msg )
425 } 425 }
426 426
427 if (msg->vfpsz.exists) { 427 if (msg->vfpsz.exists) {
428 uint16_t nb_vfpsz = htons(msg->vfpsz); 428 uint16_t nb_vfpsz = htons(msg->vfpsz.value);
429 it = msg_parse_header_out(IDVFPSZ, it, &nb_vfpsz, 429 it = msg_parse_header_out(IDVFPSZ, it, &nb_vfpsz,
430 sizeof(nb_vfpsz), &size); 430 sizeof(nb_vfpsz), &size);
431 } 431 }
@@ -588,17 +588,17 @@ void on_peer_status(Messenger* m, uint32_t friend_id, uint8_t status, void* data
588 case 0: { /* Friend is now offline */ 588 case 0: { /* Friend is now offline */
589 LOGGER_DEBUG("Friend %d is now offline", friend_id); 589 LOGGER_DEBUG("Friend %d is now offline", friend_id);
590 590
591 LOGGED_LOCK(session->mutex); 591 pthread_mutex_lock(session->mutex);
592 MSICall* call = get_call(session, friend_id); 592 MSICall* call = get_call(session, friend_id);
593 593
594 if (call == NULL) { 594 if (call == NULL) {
595 LOGGED_UNLOCK(session->mutex); 595 pthread_mutex_unlock(session->mutex);
596 return; 596 return;
597 } 597 }
598 598
599 invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */ 599 invoke_callback(call, msi_OnPeerTimeout); /* Failure is ignored */
600 kill_call(call); 600 kill_call(call);
601 LOGGED_UNLOCK(session->mutex); 601 pthread_mutex_unlock(session->mutex);
602 } 602 }
603 break; 603 break;
604 604
@@ -766,20 +766,20 @@ void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data,
766 LOGGER_DEBUG("Successfully parsed message"); 766 LOGGER_DEBUG("Successfully parsed message");
767 } 767 }
768 768
769 LOGGED_LOCK(session->mutex); 769 pthread_mutex_lock(session->mutex);
770 MSICall *call = get_call(session, friend_id); 770 MSICall *call = get_call(session, friend_id);
771 771
772 if (call == NULL) { 772 if (call == NULL) {
773 if (msg.request.value != requ_push) { 773 if (msg.request.value != requ_push) {
774 send_error(m, friend_id, msi_EStrayMessage); 774 send_error(m, friend_id, msi_EStrayMessage);
775 LOGGED_UNLOCK(session->mutex); 775 pthread_mutex_unlock(session->mutex);
776 return; 776 return;
777 } 777 }
778 778
779 call = new_call(session, friend_id); 779 call = new_call(session, friend_id);
780 if (call == NULL) { 780 if (call == NULL) {
781 send_error(m, friend_id, msi_ESystem); 781 send_error(m, friend_id, msi_ESystem);
782 LOGGED_UNLOCK(session->mutex); 782 pthread_mutex_unlock(session->mutex);
783 return; 783 return;
784 } 784 }
785 } 785 }
@@ -789,5 +789,5 @@ void handle_msi_packet ( Messenger* m, uint32_t friend_id, const uint8_t* data,
789 else 789 else
790 handle_pop(call, &msg); /* always kills the call */ 790 handle_pop(call, &msg); /* always kills the call */
791 791
792 LOGGED_UNLOCK(session->mutex); 792 pthread_mutex_unlock(session->mutex);
793} 793}
diff --git a/toxav/rtp.c b/toxav/rtp.c
index 9ef41b35..9657da67 100644
--- a/toxav/rtp.c
+++ b/toxav/rtp.c
@@ -68,12 +68,6 @@ typedef struct RTCPSession_s {
68} RTCPSession; 68} RTCPSession;
69 69
70 70
71
72/* These are defined externally */
73void ac_queue_message(void *acp, RTPMessage *msg);
74void vc_queue_message(void *vcp, RTPMessage *msg);
75
76
77RTPHeader *parse_header_in ( const uint8_t *payload, int length ); 71RTPHeader *parse_header_in ( const uint8_t *payload, int length );
78RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length ); 72RTPExtHeader *parse_ext_header_in ( const uint8_t *payload, uint16_t length );
79RTPMessage *msg_parse ( const uint8_t *data, int length ); 73RTPMessage *msg_parse ( const uint8_t *data, int length );
@@ -100,7 +94,7 @@ RTPSession *rtp_new ( int payload_type, Messenger *messenger, int friend_num )
100 94
101 retu->tstate = rtp_StateNormal; 95 retu->tstate = rtp_StateNormal;
102 retu->m = messenger; 96 retu->m = messenger;
103 retu->dest = friend_num; 97 retu->friend_id = friend_num;
104 98
105 if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) { 99 if ( !(retu->csrc = calloc(1, sizeof(uint32_t))) ) {
106 LOGGER_WARNING("Alloc failed! Program might misbehave!"); 100 LOGGER_WARNING("Alloc failed! Program might misbehave!");
@@ -155,7 +149,7 @@ void rtp_do(RTPSession *session)
155 return; 149 return;
156 150
157 if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) { 151 if (current_time_monotonic() - session->rtcp_session->last_sent_report_ts >= RTCP_REPORT_INTERVAL_MS) {
158 send_rtcp_report(session->rtcp_session, session->m, session->dest); 152 send_rtcp_report(session->rtcp_session, session->m, session->friend_id);
159 } 153 }
160 154
161 if (rb_full(session->rtcp_session->pl_stats)) { 155 if (rb_full(session->rtcp_session->pl_stats)) {
@@ -202,15 +196,15 @@ int rtp_start_receiving(RTPSession* session)
202 if (session == NULL) 196 if (session == NULL)
203 return -1; 197 return -1;
204 198
205 if (m_callback_rtp_packet(session->m, session->dest, session->prefix, 199 if (m_callback_rtp_packet(session->m, session->friend_id, session->prefix,
206 handle_rtp_packet, session) == -1) { 200 handle_rtp_packet, session) == -1) {
207 LOGGER_WARNING("Failed to register rtp receive handler"); 201 LOGGER_WARNING("Failed to register rtp receive handler");
208 return -1; 202 return -1;
209 } 203 }
210 if (m_callback_rtp_packet(session->m, session->dest, session->rtcp_session->prefix, 204 if (m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix,
211 handle_rtcp_packet, session->rtcp_session) == -1) { 205 handle_rtcp_packet, session->rtcp_session) == -1) {
212 LOGGER_WARNING("Failed to register rtcp receive handler"); 206 LOGGER_WARNING("Failed to register rtcp receive handler");
213 m_callback_rtp_packet(session->m, session->dest, session->prefix, NULL, NULL); 207 m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL);
214 return -1; 208 return -1;
215 } 209 }
216 210
@@ -221,8 +215,8 @@ int rtp_stop_receiving(RTPSession* session)
221 if (session == NULL) 215 if (session == NULL)
222 return -1; 216 return -1;
223 217
224 m_callback_rtp_packet(session->m, session->dest, session->prefix, NULL, NULL); 218 m_callback_rtp_packet(session->m, session->friend_id, session->prefix, NULL, NULL);
225 m_callback_rtp_packet(session->m, session->dest, session->rtcp_session->prefix, NULL, NULL); /* RTCP */ 219 m_callback_rtp_packet(session->m, session->friend_id, session->rtcp_session->prefix, NULL, NULL); /* RTCP */
226 220
227 return 0; 221 return 0;
228} 222}
@@ -253,7 +247,7 @@ int rtp_send_msg ( RTPSession *session, const uint8_t *data, uint16_t length )
253 memcpy ( it, data, length ); 247 memcpy ( it, data, length );
254 248
255 249
256 if ( -1 == send_custom_lossy_packet(session->m, session->dest, parsed, parsed_len) ) { 250 if ( -1 == send_custom_lossy_packet(session->m, session->friend_id, parsed, parsed_len) ) {
257 LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno)); 251 LOGGER_WARNING("Failed to send full packet (len: %d)! std error: %s", length, strerror(errno));
258 return -1; 252 return -1;
259 } 253 }
@@ -546,7 +540,6 @@ void send_rtcp_report(RTCPSession* session, Messenger* m, uint32_t friendnumber)
546} 540}
547int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) 541int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object )
548{ 542{
549 /* TODO on message callback */
550 RTPSession *session = object; 543 RTPSession *session = object;
551 RTPMessage *msg; 544 RTPMessage *msg;
552 545
@@ -578,20 +571,12 @@ int handle_rtp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data
578 571
579 session->rtcp_session->last_received_packets ++; 572 session->rtcp_session->last_received_packets ++;
580 573
581 /* Check if this session can handle the packet */ 574 if (session->mcb)
582 if (session->payload_type != session->prefix % 128) { 575 return session->mcb (session->cs, msg);
583 LOGGER_WARNING("Friend %d sent invalid payload type!", session->dest); 576 else {
584 rtp_free_msg(msg); 577 rtp_free_msg(session, msg);
585 return -1; 578 return 0;
586 } 579 }
587
588 /* Handle */
589 if (session->payload_type == rtp_TypeAudio % 128)
590 ac_queue_message(session->cs, msg);
591 else /* It can only be video */
592 vc_queue_message(session->cs, msg);
593
594 return 0;
595} 580}
596int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object ) 581int handle_rtcp_packet ( Messenger* m, uint32_t friendnumber, const uint8_t* data, uint16_t length, void* object )
597{ 582{
diff --git a/toxav/rtp.h b/toxav/rtp.h
index a28ae7bc..3056b54e 100644
--- a/toxav/rtp.h
+++ b/toxav/rtp.h
@@ -42,11 +42,13 @@
42#define MAX_RTP_SIZE 65535 42#define MAX_RTP_SIZE 65535
43 43
44/** 44/**
45 * Payload type identifier. Also used as rtp callback prefix. 45 * Payload type identifier. Also used as rtp callback prefix. (Not dummies)
46 */ 46 */
47enum { 47enum {
48 rtp_TypeAudio = 192, 48 rtp_TypeAudio = 192,
49 rtp_TypeVideo 49 rtp_TypeVideo,
50 rtp_TypeDummyAudio,
51 rtp_TypeDummyVideo,
50}; 52};
51 53
52typedef enum { 54typedef enum {
@@ -79,7 +81,7 @@ typedef struct {
79/** 81/**
80 * Standard rtp message. 82 * Standard rtp message.
81 */ 83 */
82typedef struct { 84typedef struct RTPMessage_s {
83 RTPHeader *header; 85 RTPHeader *header;
84 RTPExtHeader *ext_header; 86 RTPExtHeader *ext_header;
85 87
@@ -91,34 +93,36 @@ typedef struct {
91 * RTP control session. 93 * RTP control session.
92 */ 94 */
93typedef struct { 95typedef struct {
94 uint8_t version; 96 uint8_t version;
95 uint8_t padding; 97 uint8_t padding;
96 uint8_t extension; 98 uint8_t extension;
97 uint8_t cc; 99 uint8_t cc;
98 uint8_t marker; 100 uint8_t marker;
99 uint8_t payload_type; 101 uint8_t payload_type;
100 uint16_t sequnum; /* Sending sequence number */ 102 uint16_t sequnum; /* Sending sequence number */
101 uint16_t rsequnum; /* Receiving sequence number */ 103 uint16_t rsequnum; /* Receiving sequence number */
102 uint32_t rtimestamp; 104 uint32_t rtimestamp;
103 uint32_t ssrc; 105 uint32_t ssrc;
104 uint32_t *csrc; 106 uint32_t *csrc;
105 107
106 /* If some additional data must be sent via message 108 /* If some additional data must be sent via message
107 * apply it here. Only by allocating this member you will be 109 * apply it here. Only by allocating this member you will be
108 * automatically placing it within a message. 110 * automatically placing it within a message.
109 */ 111 */
110 RTPExtHeader *ext_header; 112 RTPExtHeader *ext_header;
111 113
112 /* Msg prefix for core to know when recving */ 114 /* Msg prefix for core to know when recving */
113 uint8_t prefix; 115 uint8_t prefix;
114
115 int dest;
116 116
117 Messenger *m;
118 int friend_id;
119 RTPTransmissionState tstate;
117 struct RTCPSession_s *rtcp_session; 120 struct RTCPSession_s *rtcp_session;
118 struct CSession_s *cs; 121
119 Messenger *m; 122
123 void *cs;
124 int (*mcb) (void*, RTPMessage* msg);
120 125
121 RTPTransmissionState tstate;
122} RTPSession; 126} RTPSession;
123 127
124/** 128/**
diff --git a/toxav/toxav.c b/toxav/toxav.c
index 6f712af9..5cb614d4 100644
--- a/toxav/toxav.c
+++ b/toxav/toxav.c
@@ -23,7 +23,8 @@
23#include "config.h" 23#include "config.h"
24#endif /* HAVE_CONFIG_H */ 24#endif /* HAVE_CONFIG_H */
25 25
26#include "msi.h" /* Includes codec.h, rtp.h and toxav.h */ 26#include "msi.h"
27#include "rtp.h"
27 28
28#include "../toxcore/Messenger.h" 29#include "../toxcore/Messenger.h"
29#include "../toxcore/logger.h" 30#include "../toxcore/logger.h"
@@ -35,20 +36,17 @@
35 36
36#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) 37#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000)
37 38
38enum {
39 audio_index,
40 video_index,
41};
42 39
43typedef struct ToxAVCall_s { 40typedef struct ToxAVCall_s {
44 ToxAV* av; 41 ToxAV* av;
45 RTPSession *rtps[2]; /* Audio is first and video is second */
46 CSession *cs;
47 42
48 pthread_mutex_t mutex_audio_sending[1]; 43 pthread_mutex_t mutex_audio[1];
49 pthread_mutex_t mutex_video_sending[1]; 44 PAIR(RTPSession *, ACSession *) audio;
50 /* Only audio or video can be decoded at the time */ 45
51 pthread_mutex_t mutex_decoding[1]; 46 pthread_mutex_t mutex_video[1];
47 PAIR(RTPSession *, VCSession *) video;
48
49 pthread_mutex_t mutex[1];
52 50
53 bool active; 51 bool active;
54 MSICall* msi_call; 52 MSICall* msi_call;
@@ -57,8 +55,8 @@ typedef struct ToxAVCall_s {
57 uint32_t audio_bit_rate; /* Sending audio bitrate */ 55 uint32_t audio_bit_rate; /* Sending audio bitrate */
58 uint32_t video_bit_rate; /* Sending video bitrate */ 56 uint32_t video_bit_rate; /* Sending video bitrate */
59 57
60 uint8_t last_self_capabilities; 58 /** Required for monitoring */
61 uint8_t last_peer_capabilities; 59 uint8_t previous_self_capabilities;
62 60
63 /** Quality control */ 61 /** Quality control */
64 uint64_t time_audio_good; 62 uint64_t time_audio_good;
@@ -181,7 +179,7 @@ void toxav_kill(ToxAV* av)
181{ 179{
182 if (av == NULL) 180 if (av == NULL)
183 return; 181 return;
184 LOGGED_LOCK(av->mutex); 182 pthread_mutex_lock(av->mutex);
185 183
186 msi_kill(av->msi); 184 msi_kill(av->msi);
187 185
@@ -194,7 +192,7 @@ void toxav_kill(ToxAV* av)
194 } 192 }
195 } 193 }
196 194
197 LOGGED_UNLOCK(av->mutex); 195 pthread_mutex_unlock(av->mutex);
198 pthread_mutex_destroy(av->mutex); 196 pthread_mutex_destroy(av->mutex);
199 free(av); 197 free(av);
200} 198}
@@ -212,9 +210,9 @@ uint32_t toxav_iteration_interval(const ToxAV* av)
212 210
213void toxav_iterate(ToxAV* av) 211void toxav_iterate(ToxAV* av)
214{ 212{
215 LOGGED_LOCK(av->mutex); 213 pthread_mutex_lock(av->mutex);
216 if (av->calls == NULL) { 214 if (av->calls == NULL) {
217 LOGGED_UNLOCK(av->mutex); 215 pthread_mutex_unlock(av->mutex);
218 return; 216 return;
219 } 217 }
220 218
@@ -224,30 +222,36 @@ void toxav_iterate(ToxAV* av)
224 ToxAVCall* i = av->calls[av->calls_head]; 222 ToxAVCall* i = av->calls[av->calls_head];
225 for (; i; i = i->next) { 223 for (; i; i = i->next) {
226 if (i->active) { 224 if (i->active) {
227 LOGGED_LOCK(i->mutex_decoding); 225 pthread_mutex_lock(i->mutex);
228 LOGGED_UNLOCK(av->mutex); 226 pthread_mutex_unlock(av->mutex);
227
228 rtp_do(i->audio.first);
229 ac_do(i->audio.second);
230
231 rtp_do(i->video.first);
232 vc_do(i->video.second);
229 233
230 cs_do(i->cs);
231 rtp_do(i->rtps[audio_index]);
232 rtp_do(i->rtps[video_index]);
233 qc_do(i); 234 qc_do(i);
234 235
235 if (i->last_self_capabilities & msi_CapRAudio) /* Receiving audio */ 236 if (i->msi_call->self_capabilities & msi_CapRAudio &&
236 rc = MIN(i->cs->last_packet_frame_duration, rc); 237 i->msi_call->peer_capabilities & msi_CapSAudio)
237 if (i->last_self_capabilities & msi_CapRVideo) /* Receiving video */ 238 rc = MIN(i->audio.second->last_packet_frame_duration, rc);
238 rc = MIN(i->cs->lcfd, rc); /* TODO handle on/off */ 239
240 if (i->msi_call->self_capabilities & msi_CapRVideo &&
241 i->msi_call->peer_capabilities & msi_CapSVideo)
242 rc = MIN(i->video.second->lcfd, rc);
239 243
240 uint32_t fid = i->friend_id; 244 uint32_t fid = i->friend_id;
241 245
242 LOGGED_UNLOCK(i->mutex_decoding); 246 pthread_mutex_unlock(i->mutex);
243 LOGGED_LOCK(av->mutex); 247 pthread_mutex_lock(av->mutex);
244 248
245 /* In case this call is popped from container stop iteration */ 249 /* In case this call is popped from container stop iteration */
246 if (call_get(av, fid) != i) 250 if (call_get(av, fid) != i)
247 break; 251 break;
248 } 252 }
249 } 253 }
250 LOGGED_UNLOCK(av->mutex); 254 pthread_mutex_unlock(av->mutex);
251 255
252 av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa); 256 av->interval = rc < av->dmssa ? 0 : (rc - av->dmssa);
253 av->dmsst += current_time_monotonic() - start; 257 av->dmsst += current_time_monotonic() - start;
@@ -261,46 +265,46 @@ void toxav_iterate(ToxAV* av)
261 265
262bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error) 266bool toxav_call(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_CALL* error)
263{ 267{
264 LOGGED_LOCK(av->mutex); 268 pthread_mutex_lock(av->mutex);
265 ToxAVCall* call = call_new(av, friend_number, error); 269 ToxAVCall* call = call_new(av, friend_number, error);
266 if (call == NULL) { 270 if (call == NULL) {
267 LOGGED_UNLOCK(av->mutex); 271 pthread_mutex_unlock(av->mutex);
268 return false; 272 return false;
269 } 273 }
270 274
271 call->audio_bit_rate = audio_bit_rate; 275 call->audio_bit_rate = audio_bit_rate;
272 call->video_bit_rate = video_bit_rate; 276 call->video_bit_rate = video_bit_rate;
273 277
274 call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; 278 call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo;
275 279
276 call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; 280 call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0;
277 call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; 281 call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0;
278 282
279 if (msi_invite(av->msi, &call->msi_call, friend_number, call->last_self_capabilities) != 0) { 283 if (msi_invite(av->msi, &call->msi_call, friend_number, call->previous_self_capabilities) != 0) {
280 call_remove(call); 284 call_remove(call);
281 if (error) 285 if (error)
282 *error = TOXAV_ERR_CALL_MALLOC; 286 *error = TOXAV_ERR_CALL_MALLOC;
283 LOGGED_UNLOCK(av->mutex); 287 pthread_mutex_unlock(av->mutex);
284 return false; 288 return false;
285 } 289 }
286 290
287 call->msi_call->av_call = call; 291 call->msi_call->av_call = call;
288 LOGGED_UNLOCK(av->mutex); 292 pthread_mutex_unlock(av->mutex);
289 293
290 return true; 294 return true;
291} 295}
292 296
293void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data) 297void toxav_callback_call(ToxAV* av, toxav_call_cb* function, void* user_data)
294{ 298{
295 LOGGED_LOCK(av->mutex); 299 pthread_mutex_lock(av->mutex);
296 av->ccb.first = function; 300 av->ccb.first = function;
297 av->ccb.second = user_data; 301 av->ccb.second = user_data;
298 LOGGED_UNLOCK(av->mutex); 302 pthread_mutex_unlock(av->mutex);
299} 303}
300 304
301bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error) 305bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate, TOXAV_ERR_ANSWER* error)
302{ 306{
303 LOGGED_LOCK(av->mutex); 307 pthread_mutex_lock(av->mutex);
304 308
305 TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK; 309 TOXAV_ERR_ANSWER rc = TOXAV_ERR_ANSWER_OK;
306 if (m_friend_exists(av->m, friend_number) == 0) { 310 if (m_friend_exists(av->m, friend_number) == 0) {
@@ -329,17 +333,17 @@ bool toxav_answer(ToxAV* av, uint32_t friend_number, uint32_t audio_bit_rate, ui
329 call->audio_bit_rate = audio_bit_rate; 333 call->audio_bit_rate = audio_bit_rate;
330 call->video_bit_rate = video_bit_rate; 334 call->video_bit_rate = video_bit_rate;
331 335
332 call->last_self_capabilities = msi_CapRAudio | msi_CapRVideo; 336 call->previous_self_capabilities = msi_CapRAudio | msi_CapRVideo;
333 337
334 call->last_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0; 338 call->previous_self_capabilities |= audio_bit_rate > 0 ? msi_CapSAudio : 0;
335 call->last_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0; 339 call->previous_self_capabilities |= video_bit_rate > 0 ? msi_CapSVideo : 0;
336 340
337 if (msi_answer(call->msi_call, call->last_self_capabilities) != 0) 341 if (msi_answer(call->msi_call, call->previous_self_capabilities) != 0)
338 rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */ 342 rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING; /* the only reason for msi_answer to fail */
339 343
340 344
341END: 345END:
342 LOGGED_UNLOCK(av->mutex); 346 pthread_mutex_unlock(av->mutex);
343 347
344 if (error) 348 if (error)
345 *error = rc; 349 *error = rc;
@@ -349,15 +353,15 @@ END:
349 353
350void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data) 354void toxav_callback_call_state(ToxAV* av, toxav_call_state_cb* function, void* user_data)
351{ 355{
352 LOGGED_LOCK(av->mutex); 356 pthread_mutex_lock(av->mutex);
353 av->scb.first = function; 357 av->scb.first = function;
354 av->scb.second = user_data; 358 av->scb.second = user_data;
355 LOGGED_UNLOCK(av->mutex); 359 pthread_mutex_unlock(av->mutex);
356} 360}
357 361
358bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error) 362bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL control, TOXAV_ERR_CALL_CONTROL* error)
359{ 363{
360 LOGGED_LOCK(av->mutex); 364 pthread_mutex_lock(av->mutex);
361 TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK; 365 TOXAV_ERR_CALL_CONTROL rc = TOXAV_ERR_CALL_CONTROL_OK;
362 366
363 if (m_friend_exists(av->m, friend_number) == 0) { 367 if (m_friend_exists(av->m, friend_number) == 0) {
@@ -381,18 +385,18 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co
381 385
382 /* Only act if paused and had media transfer active before */ 386 /* Only act if paused and had media transfer active before */
383 if (call->msi_call->self_capabilities == 0 && 387 if (call->msi_call->self_capabilities == 0 &&
384 call->last_self_capabilities ) { 388 call->previous_self_capabilities ) {
385 389
386 if (msi_change_capabilities(call->msi_call, 390 if (msi_change_capabilities(call->msi_call,
387 call->last_self_capabilities) == -1) { 391 call->previous_self_capabilities) == -1) {
388 /* The only reason for this function to fail is invalid state 392 /* The only reason for this function to fail is invalid state
389 * ( not active ) */ 393 * ( not active ) */
390 rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL; 394 rc = TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL;
391 goto END; 395 goto END;
392 } 396 }
393 397
394 rtp_start_receiving(call->rtps[audio_index]); 398 rtp_start_receiving(call->audio.first);
395 rtp_start_receiving(call->rtps[video_index]); 399 rtp_start_receiving(call->video.first);
396 } 400 }
397 } break; 401 } break;
398 402
@@ -404,7 +408,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co
404 408
405 /* Only act if not already paused */ 409 /* Only act if not already paused */
406 if (call->msi_call->self_capabilities) { 410 if (call->msi_call->self_capabilities) {
407 call->last_self_capabilities = call->msi_call->self_capabilities; 411 call->previous_self_capabilities = call->msi_call->self_capabilities;
408 412
409 if (msi_change_capabilities(call->msi_call, 0) == -1 ) { 413 if (msi_change_capabilities(call->msi_call, 0) == -1 ) {
410 /* The only reason for this function to fail is invalid state 414 /* The only reason for this function to fail is invalid state
@@ -413,8 +417,8 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co
413 goto END; 417 goto END;
414 } 418 }
415 419
416 rtp_stop_receiving(call->rtps[audio_index]); 420 rtp_stop_receiving(call->audio.first);
417 rtp_stop_receiving(call->rtps[video_index]); 421 rtp_stop_receiving(call->video.first);
418 } 422 }
419 } break; 423 } break;
420 424
@@ -442,7 +446,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co
442 goto END; 446 goto END;
443 } 447 }
444 448
445 rtp_stop_receiving(call->rtps[audio_index]); 449 rtp_stop_receiving(call->audio.first);
446 } else { 450 } else {
447 /* This call was already muted so notify the friend that he can 451 /* This call was already muted so notify the friend that he can
448 * start sending audio again 452 * start sending audio again
@@ -455,7 +459,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co
455 goto END; 459 goto END;
456 } 460 }
457 461
458 rtp_start_receiving(call->rtps[audio_index]); 462 rtp_start_receiving(call->audio.first);
459 } 463 }
460 } break; 464 } break;
461 465
@@ -474,7 +478,7 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co
474 goto END; 478 goto END;
475 } 479 }
476 480
477 rtp_stop_receiving(call->rtps[video_index]); 481 rtp_stop_receiving(call->video.first);
478 } else { 482 } else {
479 /* This call was already muted so notify the friend that he can 483 /* This call was already muted so notify the friend that he can
480 * start sending video again 484 * start sending video again
@@ -487,13 +491,13 @@ bool toxav_call_control(ToxAV* av, uint32_t friend_number, TOXAV_CALL_CONTROL co
487 goto END; 491 goto END;
488 } 492 }
489 493
490 rtp_start_receiving(call->rtps[video_index]); 494 rtp_start_receiving(call->video.first);
491 } 495 }
492 } break; 496 } break;
493 } 497 }
494 498
495END: 499END:
496 LOGGED_UNLOCK(av->mutex); 500 pthread_mutex_unlock(av->mutex);
497 501
498 if (error) 502 if (error)
499 *error = rc; 503 *error = rc;
@@ -516,19 +520,19 @@ bool toxav_set_audio_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t audio_
516 goto END; 520 goto END;
517 } 521 }
518 522
519 LOGGED_LOCK(av->mutex); 523 pthread_mutex_lock(av->mutex);
520 call = call_get(av, friend_number); 524 call = call_get(av, friend_number);
521 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { 525 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
522 LOGGED_UNLOCK(av->mutex); 526 pthread_mutex_unlock(av->mutex);
523 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; 527 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL;
524 goto END; 528 goto END;
525 } 529 }
526 530
527 /* Decoding mutex is locked because of quality control */ 531 /* Decoding mutex is locked because of quality control */
528 LOGGED_LOCK(call->mutex_decoding); 532 pthread_mutex_lock(call->mutex);
529 call->audio_bit_rate = audio_bit_rate; 533 call->audio_bit_rate = audio_bit_rate;
530 LOGGED_UNLOCK(call->mutex_decoding); 534 pthread_mutex_unlock(call->mutex);
531 LOGGED_UNLOCK(av->mutex); 535 pthread_mutex_unlock(av->mutex);
532 536
533END: 537END:
534 if (error) 538 if (error)
@@ -552,19 +556,19 @@ bool toxav_set_video_bit_rate(ToxAV* av, uint32_t friend_number, uint32_t video_
552 goto END; 556 goto END;
553 } 557 }
554 558
555 LOGGED_LOCK(av->mutex); 559 pthread_mutex_lock(av->mutex);
556 call = call_get(av, friend_number); 560 call = call_get(av, friend_number);
557 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { 561 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
558 LOGGED_UNLOCK(av->mutex); 562 pthread_mutex_unlock(av->mutex);
559 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL; 563 rc = TOXAV_ERR_BIT_RATE_FRIEND_NOT_IN_CALL;
560 goto END; 564 goto END;
561 } 565 }
562 566
563 /* Decoding mutex is locked because of quality control */ 567 /* Decoding mutex is locked because of quality control */
564 LOGGED_LOCK(call->mutex_decoding); 568 pthread_mutex_lock(call->mutex);
565 call->video_bit_rate = video_bit_rate; 569 call->video_bit_rate = video_bit_rate;
566 LOGGED_UNLOCK(call->mutex_decoding); 570 pthread_mutex_unlock(call->mutex);
567 LOGGED_UNLOCK(av->mutex); 571 pthread_mutex_unlock(av->mutex);
568 572
569END: 573END:
570 if (error) 574 if (error)
@@ -575,10 +579,10 @@ END:
575 579
576void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data) 580void toxav_callback_video_frame_request(ToxAV* av, toxav_video_frame_request_cb* function, void* user_data)
577{ 581{
578 LOGGED_LOCK(av->mutex); 582 pthread_mutex_lock(av->mutex);
579 av->rvcb.first = function; 583 av->rvcb.first = function;
580 av->rvcb.second = user_data; 584 av->rvcb.second = user_data;
581 LOGGED_UNLOCK(av->mutex); 585 pthread_mutex_unlock(av->mutex);
582} 586}
583 587
584bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error) 588bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t* y, const uint8_t* u, const uint8_t* v, TOXAV_ERR_SEND_FRAME* error)
@@ -591,25 +595,25 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
591 goto END; 595 goto END;
592 } 596 }
593 597
594 LOGGED_LOCK(av->mutex); 598 pthread_mutex_lock(av->mutex);
595 call = call_get(av, friend_number); 599 call = call_get(av, friend_number);
596 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { 600 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
597 LOGGED_UNLOCK(av->mutex); 601 pthread_mutex_unlock(av->mutex);
598 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; 602 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
599 goto END; 603 goto END;
600 } 604 }
601 605
602 LOGGED_LOCK(call->mutex_video_sending); 606 pthread_mutex_lock(call->mutex_video);
603 LOGGED_UNLOCK(av->mutex); 607 pthread_mutex_unlock(av->mutex);
604 608
605 if ( y == NULL || u == NULL || v == NULL ) { 609 if ( y == NULL || u == NULL || v == NULL ) {
606 LOGGED_UNLOCK(call->mutex_video_sending); 610 pthread_mutex_unlock(call->mutex_video);
607 rc = TOXAV_ERR_SEND_FRAME_NULL; 611 rc = TOXAV_ERR_SEND_FRAME_NULL;
608 goto END; 612 goto END;
609 } 613 }
610 614
611 if ( cs_reconfigure_video_encoder(call->cs, call->video_bit_rate, width, height) != 0 ) { 615 if ( vc_reconfigure_encoder(call->video.second, call->video_bit_rate, width, height) != 0 ) {
612 LOGGED_UNLOCK(call->mutex_video_sending); 616 pthread_mutex_unlock(call->mutex_video);
613 rc = TOXAV_ERR_SEND_FRAME_INVALID; 617 rc = TOXAV_ERR_SEND_FRAME_INVALID;
614 goto END; 618 goto END;
615 } 619 }
@@ -626,29 +630,29 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
626 memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2)); 630 memcpy(img.planes[VPX_PLANE_U], u, (width/2) * (height/2));
627 memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2)); 631 memcpy(img.planes[VPX_PLANE_V], v, (width/2) * (height/2));
628 632
629 int vrc = vpx_codec_encode(call->cs->v_encoder, &img, 633 int vrc = vpx_codec_encode(call->video.second->v_encoder, &img,
630 call->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US); 634 call->video.second->frame_counter, 1, 0, MAX_ENCODE_TIME_US);
631 635
632 vpx_img_free(&img); 636 vpx_img_free(&img);
633 if ( vrc != VPX_CODEC_OK) { 637 if ( vrc != VPX_CODEC_OK) {
634 LOGGED_UNLOCK(call->mutex_video_sending); 638 pthread_mutex_unlock(call->mutex_video);
635 LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc)); 639 LOGGER_ERROR("Could not encode video frame: %s\n", vpx_codec_err_to_string(vrc));
636 rc = TOXAV_ERR_SEND_FRAME_INVALID; 640 rc = TOXAV_ERR_SEND_FRAME_INVALID;
637 goto END; 641 goto END;
638 } 642 }
639 } 643 }
640 644
641 ++call->cs->frame_counter; 645 ++call->video.second->frame_counter;
642 646
643 { /* Split and send */ 647 { /* Split and send */
644 vpx_codec_iter_t iter = NULL; 648 vpx_codec_iter_t iter = NULL;
645 const vpx_codec_cx_pkt_t *pkt; 649 const vpx_codec_cx_pkt_t *pkt;
646 650
647 cs_init_video_splitter_cycle(call->cs); 651 vc_init_video_splitter_cycle(call->video.second);
648 652
649 while ( (pkt = vpx_codec_get_cx_data(call->cs->v_encoder, &iter)) ) { 653 while ( (pkt = vpx_codec_get_cx_data(call->video.second->v_encoder, &iter)) ) {
650 if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { 654 if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
651 int parts = cs_update_video_splitter_cycle(call->cs, pkt->data.frame.buf, 655 int parts = vc_update_video_splitter_cycle(call->video.second, pkt->data.frame.buf,
652 pkt->data.frame.sz); 656 pkt->data.frame.sz);
653 657
654 if (parts < 0) /* Should never happen though */ 658 if (parts < 0) /* Should never happen though */
@@ -659,10 +663,10 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
659 663
660 int i; 664 int i;
661 for (i = 0; i < parts; i++) { 665 for (i = 0; i < parts; i++) {
662 iter = cs_iterate_split_video_frame(call->cs, &part_size); 666 iter = vc_iterate_split_video_frame(call->video.second, &part_size);
663 667
664 if (rtp_send_msg(call->rtps[video_index], iter, part_size) < 0) { 668 if (rtp_send_msg(call->video.first, iter, part_size) < 0) {
665 LOGGED_UNLOCK(call->mutex_video_sending); 669 pthread_mutex_unlock(call->mutex_video);
666 LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno)); 670 LOGGER_WARNING("Could not send video frame: %s\n", strerror(errno));
667 goto END; 671 goto END;
668 } 672 }
@@ -671,7 +675,7 @@ bool toxav_send_video_frame(ToxAV* av, uint32_t friend_number, uint16_t width, u
671 } 675 }
672 } 676 }
673 677
674 LOGGED_UNLOCK(call->mutex_video_sending); 678 pthread_mutex_unlock(call->mutex_video);
675 679
676END: 680END:
677 if (error) 681 if (error)
@@ -682,10 +686,10 @@ END:
682 686
683void toxav_callback_audio_frame_request(ToxAV* av, toxav_audio_frame_request_cb* function, void* user_data) 687void toxav_callback_audio_frame_request(ToxAV* av, toxav_audio_frame_request_cb* function, void* user_data)
684{ 688{
685 LOGGED_LOCK(av->mutex); 689 pthread_mutex_lock(av->mutex);
686 av->racb.first = function; 690 av->racb.first = function;
687 av->racb.second = user_data; 691 av->racb.second = user_data;
688 LOGGED_UNLOCK(av->mutex); 692 pthread_mutex_unlock(av->mutex);
689} 693}
690 694
691bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error) 695bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pcm, size_t sample_count, uint8_t channels, uint32_t sampling_rate, TOXAV_ERR_SEND_FRAME* error)
@@ -698,32 +702,32 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc
698 goto END; 702 goto END;
699 } 703 }
700 704
701 LOGGED_LOCK(av->mutex); 705 pthread_mutex_lock(av->mutex);
702 call = call_get(av, friend_number); 706 call = call_get(av, friend_number);
703 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) { 707 if (call == NULL || !call->active || call->msi_call->state != msi_CallActive) {
704 LOGGED_UNLOCK(av->mutex); 708 pthread_mutex_unlock(av->mutex);
705 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL; 709 rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
706 goto END; 710 goto END;
707 } 711 }
708 712
709 LOGGED_LOCK(call->mutex_audio_sending); 713 pthread_mutex_lock(call->mutex_audio);
710 LOGGED_UNLOCK(av->mutex); 714 pthread_mutex_unlock(av->mutex);
711 715
712 if ( pcm == NULL ) { 716 if ( pcm == NULL ) {
713 LOGGED_UNLOCK(call->mutex_audio_sending); 717 pthread_mutex_unlock(call->mutex_audio);
714 rc = TOXAV_ERR_SEND_FRAME_NULL; 718 rc = TOXAV_ERR_SEND_FRAME_NULL;
715 goto END; 719 goto END;
716 } 720 }
717 721
718 if ( channels > 2 ) { 722 if ( channels > 2 ) {
719 LOGGED_UNLOCK(call->mutex_audio_sending); 723 pthread_mutex_unlock(call->mutex_audio);
720 rc = TOXAV_ERR_SEND_FRAME_INVALID; 724 rc = TOXAV_ERR_SEND_FRAME_INVALID;
721 goto END; 725 goto END;
722 } 726 }
723 727
724 { /* Encode and send */ 728 { /* Encode and send */
725 if (cs_reconfigure_audio_encoder(call->cs, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) { 729 if (ac_reconfigure_encoder(call->audio.second, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) {
726 LOGGED_UNLOCK(call->mutex_audio_sending); 730 pthread_mutex_unlock(call->mutex_audio);
727 rc = TOXAV_ERR_SEND_FRAME_INVALID; 731 rc = TOXAV_ERR_SEND_FRAME_INVALID;
728 goto END; 732 goto END;
729 } 733 }
@@ -732,12 +736,12 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc
732 736
733 sampling_rate = htonl(sampling_rate); 737 sampling_rate = htonl(sampling_rate);
734 memcpy(dest, &sampling_rate, sizeof(sampling_rate)); 738 memcpy(dest, &sampling_rate, sizeof(sampling_rate));
735 int vrc = opus_encode(call->cs->audio_encoder, pcm, sample_count, 739 int vrc = opus_encode(call->audio.second->encoder, pcm, sample_count,
736 dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate)); 740 dest + sizeof(sampling_rate), sizeof(dest) - sizeof(sampling_rate));
737 741
738 if (vrc < 0) { 742 if (vrc < 0) {
739 LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc)); 743 LOGGER_WARNING("Failed to encode frame %s", opus_strerror(vrc));
740 LOGGED_UNLOCK(call->mutex_audio_sending); 744 pthread_mutex_unlock(call->mutex_audio);
741 rc = TOXAV_ERR_SEND_FRAME_INVALID; 745 rc = TOXAV_ERR_SEND_FRAME_INVALID;
742 goto END; 746 goto END;
743 } 747 }
@@ -745,13 +749,13 @@ bool toxav_send_audio_frame(ToxAV* av, uint32_t friend_number, const int16_t* pc
745// LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels, 749// LOGGER_DEBUG("Sending encoded audio frame size: %d; channels: %d; srate: %d", vrc, channels,
746// ntohl(sampling_rate)); 750// ntohl(sampling_rate));
747 751
748 if (rtp_send_msg(call->rtps[audio_index], dest, vrc + sizeof(sampling_rate)) != 0) { 752 if (rtp_send_msg(call->audio.first, dest, vrc + sizeof(sampling_rate)) != 0) {
749 LOGGER_WARNING("Failed to send audio packet"); 753 LOGGER_WARNING("Failed to send audio packet");
750 rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; 754 rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
751 } 755 }
752 } 756 }
753 757
754 LOGGED_UNLOCK(call->mutex_audio_sending); 758 pthread_mutex_unlock(call->mutex_audio);
755 759
756END: 760END:
757 if (error) 761 if (error)
@@ -762,18 +766,18 @@ END:
762 766
763void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data) 767void toxav_callback_receive_video_frame(ToxAV* av, toxav_receive_video_frame_cb* function, void* user_data)
764{ 768{
765 LOGGED_LOCK(av->mutex); 769 pthread_mutex_lock(av->mutex);
766 av->vcb.first = function; 770 av->vcb.first = function;
767 av->vcb.second = user_data; 771 av->vcb.second = user_data;
768 LOGGED_UNLOCK(av->mutex); 772 pthread_mutex_unlock(av->mutex);
769} 773}
770 774
771void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data) 775void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb* function, void* user_data)
772{ 776{
773 LOGGED_LOCK(av->mutex); 777 pthread_mutex_lock(av->mutex);
774 av->acb.first = function; 778 av->acb.first = function;
775 av->acb.second = user_data; 779 av->acb.second = user_data;
776 LOGGED_UNLOCK(av->mutex); 780 pthread_mutex_unlock(av->mutex);
777} 781}
778 782
779 783
@@ -785,12 +789,12 @@ void toxav_callback_receive_audio_frame(ToxAV* av, toxav_receive_audio_frame_cb*
785int callback_invite(void* toxav_inst, MSICall* call) 789int callback_invite(void* toxav_inst, MSICall* call)
786{ 790{
787 ToxAV* toxav = toxav_inst; 791 ToxAV* toxav = toxav_inst;
788 LOGGED_LOCK(toxav->mutex); 792 pthread_mutex_lock(toxav->mutex);
789 793
790 ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL); 794 ToxAVCall* av_call = call_new(toxav, call->friend_id, NULL);
791 if (av_call == NULL) { 795 if (av_call == NULL) {
792 LOGGER_WARNING("Failed to initialize call..."); 796 LOGGER_WARNING("Failed to initialize call...");
793 LOGGED_UNLOCK(toxav->mutex); 797 pthread_mutex_unlock(toxav->mutex);
794 return -1; 798 return -1;
795 } 799 }
796 800
@@ -801,72 +805,72 @@ int callback_invite(void* toxav_inst, MSICall* call)
801 toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio, 805 toxav->ccb.first(toxav, call->friend_id, call->peer_capabilities & msi_CapSAudio,
802 call->peer_capabilities & msi_CapSVideo, toxav->ccb.second); 806 call->peer_capabilities & msi_CapSVideo, toxav->ccb.second);
803 807
804 LOGGED_UNLOCK(toxav->mutex); 808 pthread_mutex_unlock(toxav->mutex);
805 return 0; 809 return 0;
806} 810}
807 811
808int callback_start(void* toxav_inst, MSICall* call) 812int callback_start(void* toxav_inst, MSICall* call)
809{ 813{
810 ToxAV* toxav = toxav_inst; 814 ToxAV* toxav = toxav_inst;
811 LOGGED_LOCK(toxav->mutex); 815 pthread_mutex_lock(toxav->mutex);
812 816
813 ToxAVCall* av_call = call_get(toxav, call->friend_id); 817 ToxAVCall* av_call = call_get(toxav, call->friend_id);
814 818
815 if (av_call == NULL) { 819 if (av_call == NULL) {
816 /* Should this ever happen? */ 820 /* Should this ever happen? */
817 LOGGED_UNLOCK(toxav->mutex); 821 pthread_mutex_unlock(toxav->mutex);
818 return -1; 822 return -1;
819 } 823 }
820 824
821 if (!call_prepare_transmission(av_call)) { 825 if (!call_prepare_transmission(av_call)) {
822 callback_error(toxav_inst, call); 826 callback_error(toxav_inst, call);
823 call_remove(av_call); 827 call_remove(av_call);
824 LOGGED_UNLOCK(toxav->mutex); 828 pthread_mutex_unlock(toxav->mutex);
825 return -1; 829 return -1;
826 } 830 }
827 831
828 invoke_call_state(toxav, call->friend_id, call->peer_capabilities); 832 invoke_call_state(toxav, call->friend_id, call->peer_capabilities);
829 833
830 LOGGED_UNLOCK(toxav->mutex); 834 pthread_mutex_unlock(toxav->mutex);
831 return 0; 835 return 0;
832} 836}
833 837
834int callback_end(void* toxav_inst, MSICall* call) 838int callback_end(void* toxav_inst, MSICall* call)
835{ 839{
836 ToxAV* toxav = toxav_inst; 840 ToxAV* toxav = toxav_inst;
837 LOGGED_LOCK(toxav->mutex); 841 pthread_mutex_lock(toxav->mutex);
838 842
839 invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_END); 843 invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_END);
840 844
841 call_kill_transmission(call->av_call); 845 call_kill_transmission(call->av_call);
842 call_remove(call->av_call); 846 call_remove(call->av_call);
843 847
844 LOGGED_UNLOCK(toxav->mutex); 848 pthread_mutex_unlock(toxav->mutex);
845 return 0; 849 return 0;
846} 850}
847 851
848int callback_error(void* toxav_inst, MSICall* call) 852int callback_error(void* toxav_inst, MSICall* call)
849{ 853{
850 ToxAV* toxav = toxav_inst; 854 ToxAV* toxav = toxav_inst;
851 LOGGED_LOCK(toxav->mutex); 855 pthread_mutex_lock(toxav->mutex);
852 856
853 invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR); 857 invoke_call_state(toxav, call->friend_id, TOXAV_CALL_STATE_ERROR);
854 858
855 call_kill_transmission(call->av_call); 859 call_kill_transmission(call->av_call);
856 call_remove(call->av_call); 860 call_remove(call->av_call);
857 861
858 LOGGED_UNLOCK(toxav->mutex); 862 pthread_mutex_unlock(toxav->mutex);
859 return 0; 863 return 0;
860} 864}
861 865
862int callback_capabilites(void* toxav_inst, MSICall* call) 866int callback_capabilites(void* toxav_inst, MSICall* call)
863{ 867{
864 ToxAV* toxav = toxav_inst; 868 ToxAV* toxav = toxav_inst;
865 LOGGED_LOCK(toxav->mutex); 869 pthread_mutex_lock(toxav->mutex);
866 870
867 invoke_call_state(toxav, call->friend_id, call->peer_capabilities); 871 invoke_call_state(toxav, call->friend_id, call->peer_capabilities);
868 872
869 LOGGED_UNLOCK(toxav->mutex); 873 pthread_mutex_unlock(toxav->mutex);
870 return 0; 874 return 0;
871} 875}
872 876
@@ -982,10 +986,8 @@ ToxAVCall* call_get(ToxAV* av, uint32_t friend_number)
982 986
983void qc_do(ToxAVCall* call) 987void qc_do(ToxAVCall* call)
984{ 988{
985 /* Please NOTE: The quality control is rather basic, 989 /*
986 * advanced algorithms will be applied in the future 990 switch(call->audio.first->tstate) {
987 */
988 switch(call->rtps[audio_index]->tstate) {
989 case rtp_StateBad: 991 case rtp_StateBad:
990 LOGGER_DEBUG("Suggesting lower bitrate for audio..."); 992 LOGGER_DEBUG("Suggesting lower bitrate for audio...");
991 call->time_audio_good = 0; 993 call->time_audio_good = 0;
@@ -1007,9 +1009,9 @@ void qc_do(ToxAVCall* call)
1007 case rtp_StateNormal: 1009 case rtp_StateNormal:
1008 call->time_audio_good = 0; 1010 call->time_audio_good = 0;
1009 break; 1011 break;
1010 } 1012 }*/
1011 1013 /*
1012 switch(call->rtps[video_index]->tstate) { 1014 switch(call->video.first->tstate) {
1013 case rtp_StateBad: 1015 case rtp_StateBad:
1014 LOGGER_DEBUG("Suggesting lower bitrate for video..."); 1016 LOGGER_DEBUG("Suggesting lower bitrate for video...");
1015 call->time_video_good = 0; 1017 call->time_video_good = 0;
@@ -1030,8 +1032,7 @@ void qc_do(ToxAVCall* call)
1030 case rtp_StateNormal: 1032 case rtp_StateNormal:
1031 call->time_video_good = 0; 1033 call->time_video_good = 0;
1032 break; 1034 break;
1033 } 1035 }*/
1034
1035} 1036}
1036 1037
1037void call_remove(ToxAVCall* call) 1038void call_remove(ToxAVCall* call)
@@ -1086,61 +1087,50 @@ bool call_prepare_transmission(ToxAVCall* call)
1086 return true; 1087 return true;
1087 } 1088 }
1088 1089
1089 if (create_recursive_mutex(call->mutex_audio_sending) != 0) 1090 if (create_recursive_mutex(call->mutex_audio) != 0)
1090 return false; 1091 return false;
1091 1092
1092 if (create_recursive_mutex(call->mutex_video_sending) != 0) { 1093 if (create_recursive_mutex(call->mutex_video) != 0) {
1093 goto AUDIO_SENDING_MUTEX_CLEANUP; 1094 goto AUDIO_SENDING_MUTEX_CLEANUP;
1094 } 1095 }
1095 1096
1096 if (create_recursive_mutex(call->mutex_decoding) != 0) { 1097 if (create_recursive_mutex(call->mutex) != 0) {
1097 goto VIDEO_SENDING_MUTEX_CLEANUP; 1098 goto VIDEO_SENDING_MUTEX_CLEANUP;
1098 } 1099 }
1099 1100
1100 /* Creates both audio and video encoders and decoders with some default values.
1101 * Make sure to reconfigure encoders dynamically when sending data
1102 */
1103 call->cs = cs_new(call->msi_call->peer_vfpsz);
1104
1105 if ( !call->cs ) {
1106 LOGGER_ERROR("Error while starting Codec State!\n");
1107 goto FAILURE;
1108 }
1109
1110 call->cs->av = av;
1111 call->cs->friend_id = call->friend_id;
1112
1113 memcpy(&call->cs->acb, &av->acb, sizeof(av->acb));
1114 memcpy(&call->cs->vcb, &av->vcb, sizeof(av->vcb));
1115 1101
1116 { /* Prepare audio RTP */ 1102 { /* Prepare audio */
1117 call->rtps[audio_index] = rtp_new(rtp_TypeAudio, av->m, call->friend_id); 1103 call->audio.first = rtp_new(rtp_TypeAudio, av->m, call->friend_id);
1104 call->audio.second = ac_new(av, call->friend_id, av->acb.first, av->acb.second);
1118 1105
1119 if ( !call->rtps[audio_index] ) { 1106 if ( !call->audio.first || !call->audio.second ) {
1120 LOGGER_ERROR("Error while starting audio RTP session!\n"); 1107 LOGGER_ERROR("Error while starting audio!\n");
1121 goto FAILURE; 1108 goto FAILURE;
1122 } 1109 }
1123 1110
1124 call->rtps[audio_index]->cs = call->cs; 1111 call->audio.first->cs = call->audio.second;
1112 call->audio.first->mcb = ac_queue_message;
1125 1113
1126 if (rtp_start_receiving(call->rtps[audio_index]) != 0) { 1114 if (rtp_start_receiving(call->audio.first) != 0) {
1127 LOGGER_WARNING("Failed to enable audio receiving!"); 1115 LOGGER_WARNING("Failed to enable audio receiving!");
1128 goto FAILURE; 1116 goto FAILURE;
1129 } 1117 }
1130 } 1118 }
1131 1119
1132 { /* Prepare video RTP */ 1120 { /* Prepare video */
1133 call->rtps[video_index] = rtp_new(rtp_TypeVideo, av->m, call->friend_id); 1121 call->video.first = rtp_new(rtp_TypeVideo, av->m, call->friend_id);
1122 call->video.second = vc_new(av, call->friend_id, av->vcb.first, av->vcb.second, call->msi_call->peer_vfpsz);
1134 1123
1135 if ( !call->rtps[video_index] ) { 1124 if ( !call->video.first || !call->video.second ) {
1136 LOGGER_ERROR("Error while starting video RTP session!\n"); 1125 LOGGER_ERROR("Error while starting video!\n");
1137 goto FAILURE; 1126 goto FAILURE;
1138 } 1127 }
1139 1128
1140 call->rtps[video_index]->cs = call->cs; 1129 call->video.first->cs = call->video.second;
1130 call->video.first->mcb = vc_queue_message;
1141 1131
1142 if (rtp_start_receiving(call->rtps[video_index]) != 0) { 1132 if (rtp_start_receiving(call->video.first) != 0) {
1143 LOGGER_WARNING("Failed to enable audio receiving!"); 1133 LOGGER_WARNING("Failed to enable video receiving!");
1144 goto FAILURE; 1134 goto FAILURE;
1145 } 1135 }
1146 } 1136 }
@@ -1149,17 +1139,19 @@ bool call_prepare_transmission(ToxAVCall* call)
1149 return true; 1139 return true;
1150 1140
1151FAILURE: 1141FAILURE:
1152 rtp_kill(call->rtps[audio_index]); 1142 rtp_kill(call->audio.first);
1153 call->rtps[audio_index] = NULL; 1143 ac_kill(call->audio.second);
1154 rtp_kill(call->rtps[video_index]); 1144 call->audio.first = NULL;
1155 call->rtps[video_index] = NULL; 1145 call->audio.second = NULL;
1156 cs_kill(call->cs); 1146 rtp_kill(call->video.first);
1157 call->cs = NULL; 1147 vc_kill(call->video.second);
1158 pthread_mutex_destroy(call->mutex_decoding); 1148 call->video.first = NULL;
1149 call->video.second = NULL;
1150 pthread_mutex_destroy(call->mutex);
1159VIDEO_SENDING_MUTEX_CLEANUP: 1151VIDEO_SENDING_MUTEX_CLEANUP:
1160 pthread_mutex_destroy(call->mutex_video_sending); 1152 pthread_mutex_destroy(call->mutex_video);
1161AUDIO_SENDING_MUTEX_CLEANUP: 1153AUDIO_SENDING_MUTEX_CLEANUP:
1162 pthread_mutex_destroy(call->mutex_audio_sending); 1154 pthread_mutex_destroy(call->mutex_audio);
1163 return false; 1155 return false;
1164} 1156}
1165 1157
@@ -1170,23 +1162,24 @@ void call_kill_transmission(ToxAVCall* call)
1170 1162
1171 call->active = 0; 1163 call->active = 0;
1172 1164
1173 LOGGED_LOCK(call->mutex_audio_sending); 1165 pthread_mutex_lock(call->mutex_audio);
1174 LOGGED_UNLOCK(call->mutex_audio_sending); 1166 pthread_mutex_unlock(call->mutex_audio);
1175 LOGGED_LOCK(call->mutex_video_sending); 1167 pthread_mutex_lock(call->mutex_video);
1176 LOGGED_UNLOCK(call->mutex_video_sending); 1168 pthread_mutex_unlock(call->mutex_video);
1177 LOGGED_LOCK(call->mutex_decoding); 1169 pthread_mutex_lock(call->mutex);
1178 LOGGED_UNLOCK(call->mutex_decoding); 1170 pthread_mutex_unlock(call->mutex);
1179 1171
1180 1172 rtp_kill(call->audio.first);
1181 rtp_kill(call->rtps[audio_index]); 1173 ac_kill(call->audio.second);
1182 call->rtps[audio_index] = NULL; 1174 call->audio.first = NULL;
1183 rtp_kill(call->rtps[video_index]); 1175 call->audio.second = NULL;
1184 call->rtps[video_index] = NULL; 1176
1185 1177 rtp_kill(call->video.first);
1186 cs_kill(call->cs); 1178 vc_kill(call->video.second);
1187 call->cs = NULL; 1179 call->video.first = NULL;
1188 1180 call->video.second = NULL;
1189 pthread_mutex_destroy(call->mutex_audio_sending); 1181
1190 pthread_mutex_destroy(call->mutex_video_sending); 1182 pthread_mutex_destroy(call->mutex_audio);
1191 pthread_mutex_destroy(call->mutex_decoding); 1183 pthread_mutex_destroy(call->mutex_video);
1184 pthread_mutex_destroy(call->mutex);
1192} 1185}
diff --git a/toxav/video.c b/toxav/video.c
index d51cfd4a..039fc2a0 100644
--- a/toxav/video.c
+++ b/toxav/video.c
@@ -24,6 +24,7 @@
24 24
25#include "video.h" 25#include "video.h"
26#include "msi.h" 26#include "msi.h"
27#include "rtp.h"
27 28
28#include "../toxcore/logger.h" 29#include "../toxcore/logger.h"
29#include "../toxcore/network.h" 30#include "../toxcore/network.h"
@@ -78,7 +79,9 @@ VCSession* vc_new(ToxAV* av, uint32_t friend_id, toxav_receive_video_frame_cb* c
78 79
79 vc->linfts = current_time_monotonic(); 80 vc->linfts = current_time_monotonic();
80 vc->lcfd = 60; 81 vc->lcfd = 60;
81 82 vc->vcb.first = cb;
83 vc->vcb.second = cb_data;
84 vc->friend_id = friend_id;
82 vc->peer_video_frame_piece_size = mvfpsz; 85 vc->peer_video_frame_piece_size = mvfpsz;
83 86
84 return vc; 87 return vc;
@@ -187,35 +190,25 @@ const uint8_t* vc_iterate_split_video_frame(VCSession* vc, uint16_t* size)
187 190
188 return vc->split_video_frame; 191 return vc->split_video_frame;
189} 192}
190int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height) 193int vc_queue_message(void* vcp, struct RTPMessage_s *msg)
191{
192 if (!vc)
193 return;
194
195 vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc;
196 if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height)
197 return 0; /* Nothing changed */
198
199 cfg.rc_target_bitrate = bitrate;
200 cfg.g_w = width;
201 cfg.g_h = height;
202
203 int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg);
204 if ( rc != VPX_CODEC_OK) {
205 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
206 return -1;
207 }
208
209 return 0;
210}
211/* Called from RTP */
212void vc_queue_message(void* vcp, RTPMessage *msg)
213{ 194{
214 /* This function does the reconstruction of video packets. 195 /* This function does the reconstruction of video packets.
215 * See more info about video splitting in docs 196 * See more info about video splitting in docs
216 */ 197 */
217 if (!vcp || !msg) 198 if (!vcp || !msg)
218 return; 199 return -1;
200
201 if ((msg->header->marker_payloadt & 0x7f) == rtp_TypeDummyVideo % 128) {
202 LOGGER_WARNING("Got dummy!");
203 rtp_free_msg(NULL, msg);
204 return 0;
205 }
206
207 if ((msg->header->marker_payloadt & 0x7f) != rtp_TypeVideo % 128) {
208 LOGGER_WARNING("Invalid payload type!");
209 rtp_free_msg(NULL, msg);
210 return -1;
211 }
219 212
220 VCSession* vc = vcp; 213 VCSession* vc = vcp;
221 214
@@ -233,7 +226,7 @@ void vc_queue_message(void* vcp, RTPMessage *msg)
233 Payload *p = malloc(sizeof(Payload) + vc->frame_size); 226 Payload *p = malloc(sizeof(Payload) + vc->frame_size);
234 227
235 if (p) { 228 if (p) {
236 LOGGED_LOCK(vc->queue_mutex); 229 pthread_mutex_lock(vc->queue_mutex);
237 230
238 if (rb_full(vc->vbuf_raw)) { 231 if (rb_full(vc->vbuf_raw)) {
239 LOGGER_DEBUG("Dropped video frame"); 232 LOGGER_DEBUG("Dropped video frame");
@@ -251,7 +244,7 @@ void vc_queue_message(void* vcp, RTPMessage *msg)
251 vc->linfts = current_time_monotonic(); 244 vc->linfts = current_time_monotonic();
252 245
253 rb_write(vc->vbuf_raw, p); 246 rb_write(vc->vbuf_raw, p);
254 LOGGED_UNLOCK(vc->queue_mutex); 247 pthread_mutex_unlock(vc->queue_mutex);
255 } else { 248 } else {
256 LOGGER_WARNING("Allocation failed! Program might misbehave!"); 249 LOGGER_WARNING("Allocation failed! Program might misbehave!");
257 goto end; 250 goto end;
@@ -288,9 +281,32 @@ void vc_queue_message(void* vcp, RTPMessage *msg)
288 281
289end: 282end:
290 rtp_free_msg(NULL, msg); 283 rtp_free_msg(NULL, msg);
284 return 0;
285}
286int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height)
287{
288 if (!vc)
289 return;
290
291 vpx_codec_enc_cfg_t cfg = *vc->v_encoder[0].config.enc;
292 if (cfg.rc_target_bitrate == bitrate && cfg.g_w == width && cfg.g_h == height)
293 return 0; /* Nothing changed */
294
295 cfg.rc_target_bitrate = bitrate;
296 cfg.g_w = width;
297 cfg.g_h = height;
298
299 int rc = vpx_codec_enc_config_set(vc->v_encoder, &cfg);
300 if ( rc != VPX_CODEC_OK) {
301 LOGGER_ERROR("Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
302 return -1;
303 }
304
305 return 0;
291} 306}
292 307
293 308
309
294bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate) 310bool create_video_encoder (vpx_codec_ctx_t* dest, int32_t bitrate)
295{ 311{
296 assert(dest); 312 assert(dest);
diff --git a/toxav/video.h b/toxav/video.h
index c1678ad2..ed264f36 100644
--- a/toxav/video.h
+++ b/toxav/video.h
@@ -36,6 +36,8 @@
36 36
37#include "../toxcore/util.h" 37#include "../toxcore/util.h"
38 38
39struct RTPMessage_s;
40
39typedef struct VCSession_s { 41typedef struct VCSession_s {
40 42
41 /* encoding */ 43 /* encoding */
@@ -76,6 +78,7 @@ void vc_do(VCSession* vc);
76void vc_init_video_splitter_cycle(VCSession* vc); 78void vc_init_video_splitter_cycle(VCSession* vc);
77int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length); 79int vc_update_video_splitter_cycle(VCSession* vc, const uint8_t* payload, uint16_t length);
78const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size); 80const uint8_t *vc_iterate_split_video_frame(VCSession* vc, uint16_t *size);
81int vc_queue_message(void *vcp, struct RTPMessage_s *msg);
79int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height); 82int vc_reconfigure_encoder(VCSession* vc, int32_t bitrate, uint16_t width, uint16_t height);
80 83
81#endif /* VIDEO_H */ \ No newline at end of file 84#endif /* VIDEO_H */ \ No newline at end of file
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-23 02:11:21 +0000