summaryrefslogtreecommitdiff
path: root/toxav
diff options
context:
space:
mode:
Diffstat (limited to 'toxav')
-rw-r--r--toxav/Makefile.inc134
-rw-r--r--toxav/media.c151
-rw-r--r--toxav/media.h78
-rw-r--r--toxav/msi.c40
-rw-r--r--toxav/msi.h9
-rwxr-xr-xtoxav/phone.c401
-rw-r--r--toxav/rtp.c55
-rw-r--r--toxav/rtp.h18
-rw-r--r--toxav/toxav.c128
-rw-r--r--toxav/toxav.h47
10 files changed, 551 insertions, 510 deletions
diff --git a/toxav/Makefile.inc b/toxav/Makefile.inc
index 203d8a07..0472d361 100644
--- a/toxav/Makefile.inc
+++ b/toxav/Makefile.inc
@@ -1,105 +1,43 @@
1if BUILD_AV 1if BUILD_AV
2 2
3lib_LTLIBRARIES += libtoxrtp.la \ 3lib_LTLIBRARIES += libtoxav.la
4 libtoxmsi.la \ 4libtoxav_la_include_HEADERS = ../toxav/toxav.h
5 libtoxmedia.la 5libtoxav_la_includedir = $(includedir)/tox
6
7libtoxav_la_SOURCES = ../toxav/rtp.h \
8 ../toxav/rtp.c \
9 ../toxav/msi.h \
10 ../toxav/msi.c \
11 ../toxav/media.h \
12 ../toxav/media.c \
13 ../toxav/toxav.h \
14 ../toxav/toxav.c
15
16
17libtoxav_la_CFLAGS = -I../toxcore \
18 -I../toxav \
19 $(NACL_CFLAGS) \
20 $(OPUS_CFLAGS) \
21 $(VPX_CFLAGS)
22
23libtoxav_la_LDFLAGS = $(TOXAV_LT_LDFLAGS) \
24 $(NACL_LDFLAGS) \
25 $(EXTRA_LT_LDFLAGS)
26
27libtoxav_la_LIBS = $(NACL_LIBS) \
28 $(OPUS_LIBS) \
29 $(VPX_LIBS)
6 30
7
8# ****** RTP ****** #
9
10libtoxrtp_la_include_HEADERS = \
11 ../toxav/toxrtp.h
12
13libtoxrtp_la_includedir = $(includedir)/tox
14
15libtoxrtp_la_SOURCES = ../toxav/toxrtp.h \
16 ../toxav/toxrtp.c
17
18libtoxrtp_la_CFLAGS = -I../toxcore \
19 -I../toxav \
20 $(NACL_CFLAGS)
21
22libtoxrtp_la_LDFLAGS = $(TOXRTP_LT_LDFLAGS) \
23 $(NACL_LDFLAGS) \
24 $(EXTRA_LT_LDFLAGS)
25
26libtoxrtp_la_LIBS = libtoxcore.la \
27 $(NACL_LIBS)
28
29
30 31
32endif
31 33
32 34
33# ****** MSI ****** #
34
35libtoxmsi_la_include_HEADERS = \
36 ../toxav/toxmsi.h
37
38libtoxmsi_la_includedir = $(includedir)/tox
39
40libtoxmsi_la_SOURCES = ../toxav/toxmsi.h \
41 ../toxav/toxmsi.c
42
43libtoxmsi_la_CFLAGS = -I../toxcore \
44 -I../toxav \
45 $(NACL_CFLAGS)
46
47libtoxmsi_la_LDFLAGS = $(TOXMSI_LT_LDFLAGS) \
48 $(EXTRA_LT_LDFLAGS) \
49 $(NACL_LDFLAGS)
50
51libtoxmsi_la_LIBS = libtoxcore.la \
52 $(NACL_LIBS)
53
54 35
55 36
56 37
57 38
58# ****** MEDIA ****** # 39if BUILD_PHONE
59
60libtoxmedia_la_include_HEADERS = \
61 ../toxav/toxmedia.h
62
63libtoxmedia_la_includedir = $(includedir)/tox
64
65libtoxmedia_la_SOURCES = ../toxav/toxmedia.h \
66 ../toxav/toxmedia.c
67 40
68libtoxmedia_la_CFLAGS = -I../toxcore \
69 -I../toxav \
70 $(AVFORMAT_CFLAGS) \
71 $(AVCODEC_CFLAGS) \
72 $(AVUTIL_CFLAGS) \
73 $(AVDEVICE_CFLAGS) \
74 $(SWSCALE_CFLAGS) \
75 $(SDL_CFLAGS) \
76 $(OPENAL_CFLAGS) \
77 $(NACL_CFLAGS) \
78 $(OPUS_CFLAGS)
79
80
81libtoxmedia_la_LDFLAGS = $(TOXMSI_LT_LDFLAGS) \
82 $(TOXRTP_LT_LDFLAGS) \
83 $(EXTRA_LT_LDFLAGS) \
84 $(NACL_LDFLAGS)
85
86
87libtoxmedia_la_LIBS = libtoxcore.la \
88 $(NACL_LDFLAGS) \
89 $(AVFORMAT_LIBS) \
90 $(AVCODEC_LIBS) \
91 $(AVUTIL_LIBS) \
92 $(AVDEVICE_LIBS) \
93 $(SWSCALE_LIBS) \
94 $(SDL_LIBS) \
95 $(OPENAL_LIBS) \
96 $(NACL_LIBS) \
97 $(OPUS_LIBS)
98
99
100
101
102# ***** PHONE ***** #
103 41
104noinst_PROGRAMS += phone 42noinst_PROGRAMS += phone
105 43
@@ -113,25 +51,17 @@ phone_CFLAGS = -I../toxcore \
113 $(AVDEVICE_CFLAGS) \ 51 $(AVDEVICE_CFLAGS) \
114 $(SWSCALE_CFLAGS) \ 52 $(SWSCALE_CFLAGS) \
115 $(SDL_CFLAGS) \ 53 $(SDL_CFLAGS) \
116 $(OPENAL_CFLAGS) \ 54 $(OPENAL_CFLAGS)
117 $(NACL_CFLAGS) \
118 $(OPUS_CFLAGS)
119
120 55
121phone_LDADD = libtoxrtp.la \ 56phone_LDADD = libtoxav.la \
122 libtoxmsi.la \
123 libtoxmedia.la \
124 libtoxcore.la \ 57 libtoxcore.la \
125 $(NACL_LDFLAGS) \
126 $(AVFORMAT_LIBS) \ 58 $(AVFORMAT_LIBS) \
127 $(AVCODEC_LIBS) \ 59 $(AVCODEC_LIBS) \
128 $(AVUTIL_LIBS) \ 60 $(AVUTIL_LIBS) \
129 $(AVDEVICE_LIBS) \ 61 $(AVDEVICE_LIBS) \
130 $(SWSCALE_LIBS) \ 62 $(SWSCALE_LIBS) \
131 $(SDL_LIBS) \ 63 $(SDL_LIBS) \
132 $(OPENAL_LIBS) \ 64 $(OPENAL_LIBS)
133 $(NACL_LIBS) \
134 $(OPUS_LIBS)
135 65
136 66
137endif \ No newline at end of file 67endif \ No newline at end of file
diff --git a/toxav/media.c b/toxav/media.c
index e18c1803..a327c751 100644
--- a/toxav/media.c
+++ b/toxav/media.c
@@ -28,13 +28,8 @@
28#endif /* HAVE_CONFIG_H */ 28#endif /* HAVE_CONFIG_H */
29 29
30#include <stdio.h> 30#include <stdio.h>
31#include <stdlib.h>
31#include <math.h> 32#include <math.h>
32#include <libavcodec/avcodec.h>
33#include <libavformat/avformat.h>
34#include <libswscale/swscale.h>
35#include <libavdevice/avdevice.h>
36#include <libavutil/opt.h>
37#include <opus/opus.h>
38#include <assert.h> 33#include <assert.h>
39 34
40#include "rtp.h" 35#include "rtp.h"
@@ -206,25 +201,11 @@ int queue(struct jitter_buffer *q, RTPMessage *pk)
206 201
207int init_video_decoder(CodecState *cs) 202int init_video_decoder(CodecState *cs)
208{ 203{
209 cs->video_decoder = avcodec_find_decoder(VIDEO_CODEC); 204 if (vpx_codec_dec_init_ver(&cs->v_decoder, VIDEO_CODEC_DECODER_INTERFACE, NULL, 0, VPX_DECODER_ABI_VERSION) != VPX_CODEC_OK) {
210
211 if (!cs->video_decoder) {
212 fprintf(stderr, "Init video_decoder failed!\n"); 205 fprintf(stderr, "Init video_decoder failed!\n");
213 return -1; 206 return -1;
214 } 207 }
215 208
216 cs->video_decoder_ctx = avcodec_alloc_context3(cs->video_decoder);
217
218 if (!cs->video_decoder_ctx) {
219 fprintf(stderr, "Init video_decoder_ctx failed!\n");
220 return -1;
221 }
222
223 if (avcodec_open2(cs->video_decoder_ctx, cs->video_decoder, NULL) < 0) {
224 fprintf(stderr, "Opening video decoder failed!\n");
225 return -1;
226 }
227
228 return 0; 209 return 0;
229} 210}
230 211
@@ -242,97 +223,32 @@ int init_audio_decoder(CodecState *cs, uint32_t audio_channels)
242} 223}
243 224
244 225
245int init_video_encoder(CodecState *cs, const char* webcam, const char* video_driver, uint32_t video_bitrate) 226int init_video_encoder(CodecState *cs, uint16_t width, uint16_t height, uint32_t video_bitrate)
246{ 227{
247 cs->video_input_format = av_find_input_format(video_driver); 228 vpx_codec_enc_cfg_t cfg;
248 229 int res = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0);
249 if (avformat_open_input(&cs->video_format_ctx, webcam, cs->video_input_format, NULL) != 0) { 230 if(res) {
250 fprintf(stderr, "Opening video_input_format failed!\n"); 231 printf("Failed to get config: %s\n", vpx_codec_err_to_string(res));
251 return -1;
252 }
253
254 avformat_find_stream_info(cs->video_format_ctx, NULL);
255 av_dump_format(cs->video_format_ctx, 0, webcam, 0);
256
257 int i;
258
259 for (i = 0; i < cs->video_format_ctx->nb_streams; ++i) {
260 if (cs->video_format_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
261 cs->video_stream = i;
262 break;
263 }
264 }
265
266 cs->webcam_decoder_ctx = cs->video_format_ctx->streams[cs->video_stream]->codec;
267 cs->webcam_decoder = avcodec_find_decoder(cs->webcam_decoder_ctx->codec_id);
268
269 if (cs->webcam_decoder == NULL) {
270 fprintf(stderr, "Unsupported codec!\n");
271 return -1;
272 }
273
274 if (cs->webcam_decoder_ctx == NULL) {
275 fprintf(stderr, "Init webcam_decoder_ctx failed!\n");
276 return -1;
277 }
278
279 if (avcodec_open2(cs->webcam_decoder_ctx, cs->webcam_decoder, NULL) < 0) {
280 fprintf(stderr, "Opening webcam decoder failed!\n");
281 return -1; 232 return -1;
282 } 233 }
283 234
284 cs->video_encoder = avcodec_find_encoder(VIDEO_CODEC); 235 cfg.rc_target_bitrate = video_bitrate;
285 236 cfg.g_w = width;
286 if (!cs->video_encoder) { 237 cfg.g_h = height;
287 fprintf(stderr, "Init video_encoder failed!\n"); 238 if(vpx_codec_enc_init_ver(&cs->v_encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0, VPX_ENCODER_ABI_VERSION) != VPX_CODEC_OK) {
288 return -1; 239 fprintf(stderr, "Failed to initialize encoder\n");
289 }
290
291 cs->video_encoder_ctx = avcodec_alloc_context3(cs->video_encoder);
292
293 if (!cs->video_encoder_ctx) {
294 fprintf(stderr, "Init video_encoder_ctx failed!\n");
295 return -1;
296 }
297
298 cs->video_encoder_ctx->bit_rate = video_bitrate;
299 cs->video_encoder_ctx->rc_min_rate = cs->video_encoder_ctx->rc_max_rate = cs->video_encoder_ctx->bit_rate;
300 av_opt_set_double(cs->video_encoder_ctx->priv_data, "max-intra-rate", 90, 0);
301 av_opt_set(cs->video_encoder_ctx->priv_data, "quality", "realtime", 0);
302
303 cs->video_encoder_ctx->thread_count = 4;
304 cs->video_encoder_ctx->rc_buffer_aggressivity = 0.95;
305 cs->video_encoder_ctx->rc_buffer_size = video_bitrate * 6;
306 cs->video_encoder_ctx->profile = 3;
307 cs->video_encoder_ctx->qmax = 54;
308 cs->video_encoder_ctx->qmin = 4;
309 AVRational myrational = {1, 25};
310 cs->video_encoder_ctx->time_base = myrational;
311 cs->video_encoder_ctx->gop_size = 99999;
312 cs->video_encoder_ctx->pix_fmt = PIX_FMT_YUV420P;
313 cs->video_encoder_ctx->width = cs->webcam_decoder_ctx->width;
314 cs->video_encoder_ctx->height = cs->webcam_decoder_ctx->height;
315
316 if (avcodec_open2(cs->video_encoder_ctx, cs->video_encoder, NULL) < 0) {
317 fprintf(stderr, "Opening video encoder failed!\n");
318 return -1; 240 return -1;
319 } 241 }
320
321 return 0; 242 return 0;
322} 243}
323 244
324int init_audio_encoder(CodecState *cs) 245int init_audio_encoder(CodecState *cs, uint32_t audio_channels)
325{ 246{
326 int err = OPUS_OK; 247 int err = OPUS_OK;
327 cs->audio_encoder = opus_encoder_create(cs->audio_sample_rate, 1, OPUS_APPLICATION_VOIP, &err); 248 cs->audio_encoder = opus_encoder_create(cs->audio_sample_rate, audio_channels, OPUS_APPLICATION_AUDIO, &err);
328
329 err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_bitrate)); 249 err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_bitrate));
330 err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); 250 err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10));
331 err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); 251
332
333 /* NOTE: What do we do with this? */
334 int nfo;
335 err = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_LOOKAHEAD(&nfo));
336 252
337 return err == OPUS_OK ? 0 : -1; 253 return err == OPUS_OK ? 0 : -1;
338} 254}
@@ -342,30 +258,26 @@ CodecState* codec_init_session ( uint32_t audio_bitrate,
342 uint16_t audio_frame_duration, 258 uint16_t audio_frame_duration,
343 uint32_t audio_sample_rate, 259 uint32_t audio_sample_rate,
344 uint32_t audio_channels, 260 uint32_t audio_channels,
345 uint32_t video_bitrate, 261 uint16_t video_width,
346 const char* webcam, 262 uint16_t video_height,
347 const char* webcam_driver ) 263 uint32_t video_bitrate )
348{ 264{
349 CodecState* _retu = av_calloc(sizeof(CodecState), 1); 265 CodecState* _retu = calloc(sizeof(CodecState), 1);
350 assert(_retu); 266 assert(_retu);
351 267
352
353 avdevice_register_all();
354 avcodec_register_all();
355 av_register_all();
356
357
358 _retu->audio_bitrate = audio_bitrate; 268 _retu->audio_bitrate = audio_bitrate;
359 _retu->audio_sample_rate = audio_sample_rate; 269 _retu->audio_sample_rate = audio_sample_rate;
360 270
361 pthread_mutex_init(&_retu->ctrl_mutex, NULL);
362
363
364 /* Encoders */ 271 /* Encoders */
365 if ( 0 == init_video_encoder(_retu, webcam, webcam_driver, video_bitrate) ) 272 if (!video_width || !video_height) {
273 video_width = 320;
274 video_height = 240;
275 }
276
277 if ( 0 == init_video_encoder(_retu, video_width, video_height, video_bitrate) )
366 printf("Video encoder initialized!\n"); 278 printf("Video encoder initialized!\n");
367 279
368 if ( 0 == init_audio_encoder(_retu) ) 280 if ( 0 == init_audio_encoder(_retu, audio_channels) )
369 printf("Audio encoder initialized!\n"); 281 printf("Audio encoder initialized!\n");
370 282
371 283
@@ -391,7 +303,8 @@ void codec_terminate_session ( CodecState* cs )
391 opus_decoder_destroy(cs->audio_decoder); 303 opus_decoder_destroy(cs->audio_decoder);
392 printf("Terminated decoder!\n"); 304 printf("Terminated decoder!\n");
393 } 305 }
394 306
395 /* TODO: Terminate video */ 307 /* TODO: Terminate video */
396 308 vpx_codec_destroy(&cs->v_decoder);
309 vpx_codec_destroy(&cs->v_encoder);
397} 310}
diff --git a/toxav/media.h b/toxav/media.h
index ef2de27c..4230de8f 100644
--- a/toxav/media.h
+++ b/toxav/media.h
@@ -27,66 +27,27 @@
27 27
28#include <stdio.h> 28#include <stdio.h>
29#include <math.h> 29#include <math.h>
30#include "../toxcore/tox.h" 30#include <pthread.h>
31 31
32/* Video encoding/decoding */ 32#include <vpx/vpx_decoder.h>
33#include <libavcodec/avcodec.h> 33#include <vpx/vpx_encoder.h>
34#include <libavformat/avformat.h> 34#include <vpx/vp8dx.h>
35#include <libswscale/swscale.h> 35#include <vpx/vp8cx.h>
36#include <libavdevice/avdevice.h> 36#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx())
37#include <libavutil/opt.h> 37#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx())
38 38
39/* Audio encoding/decoding */ 39/* Audio encoding/decoding */
40#include <opus/opus.h> 40#include <opus/opus.h>
41 41
42/* ffmpeg VP8 codec ID */
43#define VIDEO_CODEC AV_CODEC_ID_VP8
44
45/* ffmpeg Opus codec ID */
46#define AUDIO_CODEC AV_CODEC_ID_OPUS
47
48/* default video bitrate in bytes/s */
49#define VIDEO_BITRATE 10*1000
50
51/* default audio bitrate in bytes/s */
52#define AUDIO_BITRATE 64000
53
54/* audio frame duration in miliseconds */
55#define AUDIO_FRAME_DURATION 20
56
57/* audio sample rate recommended to be 48kHz for Opus */
58#define AUDIO_SAMPLE_RATE 48000
59
60/* the amount of samples in one audio frame */
61#define AUDIO_FRAME_SIZE AUDIO_SAMPLE_RATE*AUDIO_FRAME_DURATION/1000
62
63/* the quit event for SDL */
64#define FF_QUIT_EVENT (SDL_USEREVENT + 2)
65
66#ifdef __linux__
67#define VIDEO_DRIVER "video4linux2"
68#define DEFAULT_WEBCAM "/dev/video0"
69#endif
70
71#if defined(_WIN32) || defined(__WIN32__) || defined (WIN32)
72#define VIDEO_DRIVER "vfwcap"
73#define DEFAULT_WEBCAM "0"
74#endif
75 42
76typedef struct _CodecState{ 43typedef struct _CodecState{
77 44
78 /* video encoding */ 45 /* video encoding */
79 AVInputFormat *video_input_format; 46 vpx_codec_ctx_t v_encoder;
80 AVFormatContext *video_format_ctx; 47 uint32_t frame_counter;
81 uint8_t video_stream;
82 AVCodecContext *webcam_decoder_ctx;
83 AVCodec *webcam_decoder;
84 AVCodecContext *video_encoder_ctx;
85 AVCodec *video_encoder;
86 48
87 /* video decoding */ 49 /* video decoding */
88 AVCodecContext *video_decoder_ctx; 50 vpx_codec_ctx_t v_decoder;
89 AVCodec *video_decoder;
90 51
91 /* audio encoding */ 52 /* audio encoding */
92 OpusEncoder *audio_encoder; 53 OpusEncoder *audio_encoder;
@@ -95,11 +56,6 @@ typedef struct _CodecState{
95 56
96 /* audio decoding */ 57 /* audio decoding */
97 OpusDecoder *audio_decoder; 58 OpusDecoder *audio_decoder;
98
99 pthread_mutex_t ctrl_mutex;
100
101
102 uint32_t frame_rate;
103 59
104} CodecState; 60} CodecState;
105 61
@@ -112,13 +68,13 @@ int queue(struct jitter_buffer *q, RTPMessage *pk);
112RTPMessage *dequeue(struct jitter_buffer *q, int *success); 68RTPMessage *dequeue(struct jitter_buffer *q, int *success);
113 69
114 70
115CodecState* codec_init_session( uint32_t audio_bitrate, 71CodecState* codec_init_session ( uint32_t audio_bitrate,
116 uint16_t audio_frame_duration, 72 uint16_t audio_frame_duration,
117 uint32_t audio_sample_rate, 73 uint32_t audio_sample_rate,
118 uint32_t audio_channels, 74 uint32_t audio_channels,
119 uint32_t video_bitrate, 75 uint16_t video_width,
120 const char* webcam, 76 uint16_t video_height,
121 const char* webcam_driver ); 77 uint32_t video_bitrate );
122 78
123void codec_terminate_session(CodecState* cs); 79void codec_terminate_session(CodecState* cs);
124 80
diff --git a/toxav/msi.c b/toxav/msi.c
index 014a904f..f00029ba 100644
--- a/toxav/msi.c
+++ b/toxav/msi.c
@@ -593,7 +593,7 @@ int send_message ( MSISession* session, MSIMessage* msg, uint32_t to )
593 uint8_t _msg_string_final [MSI_MAXMSG_SIZE]; 593 uint8_t _msg_string_final [MSI_MAXMSG_SIZE];
594 uint16_t _length = message_to_string ( msg, _msg_string_final ); 594 uint16_t _length = message_to_string ( msg, _msg_string_final );
595 595
596 return m_msi_packet((struct Messenger*) session->messenger_handle, to, _msg_string_final, _length) ? 0 : -1; 596 return m_msi_packet(session->messenger_handle, to, _msg_string_final, _length) ? 0 : -1;
597} 597}
598 598
599 599
@@ -616,7 +616,27 @@ void flush_peer_type ( MSISession* session, MSIMessage* msg, int peer_id ) {
616 } else {} /* Error */ 616 } else {} /* Error */
617} 617}
618 618
619 619void handle_remote_connection_change(Messenger* messenger, int friend_num, uint8_t status, void* session_p)
620{
621 MSISession* session = session_p;
622
623 switch ( status )
624 {
625 case 0: /* Went offline */
626 {
627 if ( session->call ) {
628 int i = 0;
629 for ( ; i < session->call->peer_count; i ++ )
630 if ( session->call->peers[i] == friend_num ) {
631 msi_stopcall(session); /* Stop the call for now */
632 return;
633 }
634 }
635 } break;
636
637 default: break;
638 }
639}
620 640
621/** 641/**
622 * @brief Sends error response to peer. 642 * @brief Sends error response to peer.
@@ -694,8 +714,8 @@ void* handle_timeout ( void* arg )
694 714
695 } 715 }
696 716
697 ( *callbacks[MSI_OnTimeout] ) ( _session->agent_handler ); 717 ( *callbacks[MSI_OnRequestTimeout] ) ( _session->agent_handler );
698 ( *callbacks[MSI_OnEnding ] ) ( _session->agent_handler ); 718 ( *callbacks[MSI_OnEnding ] ) ( _session->agent_handler );
699 719
700 return NULL; 720 return NULL;
701} 721}
@@ -774,7 +794,7 @@ int terminate_call ( MSISession* session ) {
774 794
775 795
776 /* Check event loop and cancel timed events if there are any 796 /* Check event loop and cancel timed events if there are any
777 * Notice: This has to be done before possibly 797 * NOTE: This has to be done before possibly
778 * locking the mutex the second time 798 * locking the mutex the second time
779 */ 799 */
780 event.timer_release ( session->call->request_timer_id ); 800 event.timer_release ( session->call->request_timer_id );
@@ -797,7 +817,7 @@ int terminate_call ( MSISession* session ) {
797 pthread_mutex_destroy ( &_call->mutex ); 817 pthread_mutex_destroy ( &_call->mutex );
798 818
799 free ( _call ); 819 free ( _call );
800 820
801 return 0; 821 return 0;
802} 822}
803 823
@@ -1136,7 +1156,7 @@ void msi_register_callback ( MSICallback callback, MSICallbackID id )
1136 * @return MSISession* The created session. 1156 * @return MSISession* The created session.
1137 * @retval NULL Error occured. 1157 * @retval NULL Error occured.
1138 */ 1158 */
1139MSISession* msi_init_session ( Tox* messenger, const uint8_t* ua_name ) { 1159MSISession* msi_init_session ( Messenger* messenger, const uint8_t* ua_name ) {
1140 assert ( messenger ); 1160 assert ( messenger );
1141 1161
1142 MSISession* _retu = calloc ( sizeof ( MSISession ), 1 ); 1162 MSISession* _retu = calloc ( sizeof ( MSISession ), 1 );
@@ -1152,8 +1172,10 @@ MSISession* msi_init_session ( Tox* messenger, const uint8_t* ua_name ) {
1152 _retu->call_timeout = 30000; /* default value? */ 1172 _retu->call_timeout = 30000; /* default value? */
1153 1173
1154 1174
1155 m_callback_msi_packet((struct Messenger*) messenger, msi_handle_packet, _retu ); 1175 m_callback_msi_packet(messenger, msi_handle_packet, _retu );
1156 1176
1177 /* This is called when remote terminates session */
1178 m_callback_connectionstatus_internal_av(messenger, handle_remote_connection_change, _retu);
1157 1179
1158 return _retu; 1180 return _retu;
1159} 1181}
@@ -1351,7 +1373,7 @@ int msi_stopcall ( MSISession* session ) {
1351 return -1; 1373 return -1;
1352 1374
1353 /* just terminate it */ 1375 /* just terminate it */
1354 1376
1355 terminate_call ( session ); 1377 terminate_call ( session );
1356 1378
1357 return 0; 1379 return 0;
diff --git a/toxav/msi.h b/toxav/msi.h
index 20d6671d..4487dae6 100644
--- a/toxav/msi.h
+++ b/toxav/msi.h
@@ -26,9 +26,10 @@
26#define __TOXMSI 26#define __TOXMSI
27 27
28#include <inttypes.h> 28#include <inttypes.h>
29#include "../toxcore/tox.h"
30#include <pthread.h> 29#include <pthread.h>
31 30
31#include "../toxcore/Messenger.h"
32
32/* define size for call_id */ 33/* define size for call_id */
33#define CALL_ID_LEN 12 34#define CALL_ID_LEN 12
34 35
@@ -106,7 +107,7 @@ typedef struct _MSISession {
106 const uint8_t* ua_name; 107 const uint8_t* ua_name;
107 108
108 void* agent_handler; /* Pointer to an object that is handling msi */ 109 void* agent_handler; /* Pointer to an object that is handling msi */
109 Tox* messenger_handle; 110 Messenger* messenger_handle;
110 111
111 uint32_t frequ; 112 uint32_t frequ;
112 uint32_t call_timeout; /* Time of the timeout for some action to end; 0 if infinite */ 113 uint32_t call_timeout; /* Time of the timeout for some action to end; 0 if infinite */
@@ -133,7 +134,7 @@ typedef enum {
133 134
134 /* Protocol */ 135 /* Protocol */
135 MSI_OnError, 136 MSI_OnError,
136 MSI_OnTimeout 137 MSI_OnRequestTimeout
137 138
138} MSICallbackID; 139} MSICallbackID;
139 140
@@ -156,7 +157,7 @@ void msi_register_callback(MSICallback callback, MSICallbackID id);
156 * @return MSISession* The created session. 157 * @return MSISession* The created session.
157 * @retval NULL Error occured. 158 * @retval NULL Error occured.
158 */ 159 */
159MSISession* msi_init_session ( Tox* messenger, const uint8_t* ua_name ); 160MSISession* msi_init_session ( Messenger* messenger, const uint8_t* ua_name );
160 161
161 162
162/** 163/**
diff --git a/toxav/phone.c b/toxav/phone.c
index 4f078e2b..7806727d 100755
--- a/toxav/phone.c
+++ b/toxav/phone.c
@@ -44,18 +44,41 @@
44#include <unistd.h> 44#include <unistd.h>
45#include <assert.h> 45#include <assert.h>
46#include <math.h> 46#include <math.h>
47#include <AL/al.h>
48#include <AL/alc.h>
49#include <SDL/SDL.h>
50#include <SDL/SDL_thread.h>
51#include <pthread.h> 47#include <pthread.h>
52#include <opus/opus.h>
53 48
54#include "media.h" 49//#include "media.h"
55#include "toxav.h" 50#include "toxav.h"
56#include "../toxcore/event.h" 51#include "../toxcore/event.h"
57#include "../toxcore/tox.h" 52#include "../toxcore/tox.h"
58 53
54#ifdef TOX_FFMPEG
55/* Video encoding/decoding */
56#include <libavcodec/avcodec.h>
57#include <libavformat/avformat.h>
58#include <libswscale/swscale.h>
59#include <libavdevice/avdevice.h>
60#include <libavutil/opt.h>
61#endif
62
63#include <AL/al.h>
64#include <AL/alc.h>
65#include <SDL/SDL.h>
66#include <SDL/SDL_thread.h>
67
68/* the quit event for SDL */
69#define FF_QUIT_EVENT (SDL_USEREVENT + 2)
70
71#ifdef __linux__
72#define VIDEO_DRIVER "video4linux2"
73#define DEFAULT_WEBCAM "/dev/video0"
74#endif
75
76#if defined(_WIN32) || defined(__WIN32__) || defined (WIN32)
77#define VIDEO_DRIVER "vfwcap"
78#define DEFAULT_WEBCAM "0"
79#endif
80
81
59/* Define client version */ 82/* Define client version */
60#define _USERAGENT "v.0.3.0" 83#define _USERAGENT "v.0.3.0"
61 84
@@ -96,6 +119,13 @@ typedef struct av_session_s {
96 av_friend_t* _friends; 119 av_friend_t* _friends;
97 int _friend_cout; 120 int _friend_cout;
98 char _my_public_id[200]; 121 char _my_public_id[200];
122#ifdef TOX_FFMPEG
123 AVInputFormat *video_input_format;
124 AVFormatContext *video_format_ctx;
125 uint8_t video_stream;
126 AVCodecContext *webcam_decoder_ctx;
127 AVCodec *webcam_decoder;
128#endif
99} av_session_t; 129} av_session_t;
100 130
101 131
@@ -236,8 +266,8 @@ static void fraddr_to_str(uint8_t *id_bin, char *id_str)
236/* 266/*
237 * How av stuff _should_ look like 267 * How av stuff _should_ look like
238 */ 268 */
239 269/*
240int display_received_frame(av_session_t* _phone, AVFrame *r_video_frame) 270int display_received_frame(av_session_t* _phone, vpx_image_t *image)
241{ 271{
242 CodecState* cs = get_cs_temp(_phone->av); 272 CodecState* cs = get_cs_temp(_phone->av);
243 AVPicture pict; 273 AVPicture pict;
@@ -249,8 +279,8 @@ int display_received_frame(av_session_t* _phone, AVFrame *r_video_frame)
249 pict.linesize[0] = _phone->video_picture.bmp->pitches[0]; 279 pict.linesize[0] = _phone->video_picture.bmp->pitches[0];
250 pict.linesize[1] = _phone->video_picture.bmp->pitches[2]; 280 pict.linesize[1] = _phone->video_picture.bmp->pitches[2];
251 pict.linesize[2] = _phone->video_picture.bmp->pitches[1]; 281 pict.linesize[2] = _phone->video_picture.bmp->pitches[1];
252 282 */
253 /* Convert the image into YUV format that SDL uses */ 283 /* Convert the image into YUV format that SDL uses *//*
254 sws_scale(_phone->sws_SDL_r_ctx, (uint8_t const * const *)r_video_frame->data, r_video_frame->linesize, 0, 284 sws_scale(_phone->sws_SDL_r_ctx, (uint8_t const * const *)r_video_frame->data, r_video_frame->linesize, 0,
255 cs->video_decoder_ctx->height, pict.data, pict.linesize ); 285 cs->video_decoder_ctx->height, pict.data, pict.linesize );
256 286
@@ -263,60 +293,65 @@ int display_received_frame(av_session_t* _phone, AVFrame *r_video_frame)
263 SDL_DisplayYUVOverlay(_phone->video_picture.bmp, &rect); 293 SDL_DisplayYUVOverlay(_phone->video_picture.bmp, &rect);
264 return 1; 294 return 1;
265} 295}
266 296*/
297#ifdef TOX_FFMPEG
267void *encode_video_thread(void *arg) 298void *encode_video_thread(void *arg)
268{ 299{
269 INFO("Started encode video thread!"); 300 INFO("Started encode video thread!");
270 301
271 av_session_t* _phone = arg; 302 av_session_t* _phone = arg;
272 303
273 _phone->running_encvid = 1; 304 _phone->running_encvid = 1;
274 305 //CodecState *cs = get_cs_temp(_phone->av);
275 CodecState *cs = get_cs_temp(_phone->av);
276 AVPacket pkt1, *packet = &pkt1; 306 AVPacket pkt1, *packet = &pkt1;
277 int p = 0; 307 //int p = 0;
278 int got_packet; 308 //int got_packet;
279 int video_frame_finished; 309 int video_frame_finished;
280 AVFrame *s_video_frame; 310 AVFrame *s_video_frame;
281 AVFrame *webcam_frame; 311 AVFrame *webcam_frame;
282 s_video_frame = avcodec_alloc_frame(); 312 s_video_frame = avcodec_alloc_frame();
283 webcam_frame = avcodec_alloc_frame(); 313 webcam_frame = avcodec_alloc_frame();
284 AVPacket enc_video_packet; 314 //AVPacket enc_video_packet;
285 315
286 uint8_t *buffer; 316 uint8_t *buffer;
287 int numBytes; 317 int numBytes;
288 /* Determine required buffer size and allocate buffer */ 318 /* Determine required buffer size and allocate buffer */
289 numBytes = avpicture_get_size(PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height); 319 numBytes = avpicture_get_size(PIX_FMT_YUV420P, _phone->webcam_decoder_ctx->width, _phone->webcam_decoder_ctx->height);
290 buffer = (uint8_t *)av_calloc(numBytes * sizeof(uint8_t),1); 320 buffer = (uint8_t *)av_calloc(numBytes * sizeof(uint8_t),1);
291 avpicture_fill((AVPicture *)s_video_frame, buffer, PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width, 321 avpicture_fill((AVPicture *)s_video_frame, buffer, PIX_FMT_YUV420P, _phone->webcam_decoder_ctx->width,
292 cs->webcam_decoder_ctx->height); 322 _phone->webcam_decoder_ctx->height);
293 _phone->sws_ctx = sws_getContext(cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height, 323 _phone->sws_ctx = sws_getContext(_phone->webcam_decoder_ctx->width, _phone->webcam_decoder_ctx->height,
294 cs->webcam_decoder_ctx->pix_fmt, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height, PIX_FMT_YUV420P, 324 _phone->webcam_decoder_ctx->pix_fmt, _phone->webcam_decoder_ctx->width, _phone->webcam_decoder_ctx->height, PIX_FMT_YUV420P,
295 SWS_BILINEAR, NULL, NULL, NULL); 325 SWS_BILINEAR, NULL, NULL, NULL);
296 326
327
328 vpx_image_t *image =
329 vpx_img_alloc(NULL, VPX_IMG_FMT_I420, _phone->webcam_decoder_ctx->width, _phone->webcam_decoder_ctx->height, 1);
330
331 //uint32_t frame_counter = 0;
297 while (_phone->running_encvid) { 332 while (_phone->running_encvid) {
298 333
299 if (av_read_frame(cs->video_format_ctx, packet) < 0) { 334 if (av_read_frame(_phone->video_format_ctx, packet) < 0) {
300 printf("error reading frame\n"); 335 printf("error reading frame\n");
301 336
302 if (cs->video_format_ctx->pb->error != 0) 337 if (_phone->video_format_ctx->pb->error != 0)
303 break; 338 break;
304 339
305 continue; 340 continue;
306 } 341 }
307 342
308 if (packet->stream_index == cs->video_stream) { 343 if (packet->stream_index == _phone->video_stream) {
309 if (avcodec_decode_video2(cs->webcam_decoder_ctx, webcam_frame, &video_frame_finished, packet) < 0) { 344 if (avcodec_decode_video2(_phone->webcam_decoder_ctx, webcam_frame, &video_frame_finished, packet) < 0) {
310 printf("couldn't decode\n"); 345 printf("couldn't decode\n");
311 continue; 346 continue;
312 } 347 }
313 348
314 av_free_packet(packet); 349 av_free_packet(packet);
315 sws_scale(_phone->sws_ctx, (uint8_t const * const *)webcam_frame->data, webcam_frame->linesize, 0, 350 sws_scale(_phone->sws_ctx, (uint8_t const * const *)webcam_frame->data, webcam_frame->linesize, 0,
316 cs->webcam_decoder_ctx->height, s_video_frame->data, s_video_frame->linesize); 351 _phone->webcam_decoder_ctx->height, s_video_frame->data, s_video_frame->linesize);
317 /* create a new I-frame every 60 frames */ 352 /* create a new I-frame every 60 frames */
318 ++p; 353 //++p;
319 354 /*
320 if (p == 60) { 355 if (p == 60) {
321 356
322 s_video_frame->pict_type = AV_PICTURE_TYPE_BI ; 357 s_video_frame->pict_type = AV_PICTURE_TYPE_BI ;
@@ -325,53 +360,66 @@ void *encode_video_thread(void *arg)
325 p = 0; 360 p = 0;
326 } else { 361 } else {
327 s_video_frame->pict_type = AV_PICTURE_TYPE_P ; 362 s_video_frame->pict_type = AV_PICTURE_TYPE_P ;
328 } 363 }*/
329 364
330 if (video_frame_finished) { 365 if (video_frame_finished) {
331 366 memcpy(image->planes[VPX_PLANE_Y], s_video_frame->data[0], s_video_frame->linesize[0] * _phone->webcam_decoder_ctx->height);
332 if (avcodec_encode_video2(cs->video_encoder_ctx, &enc_video_packet, s_video_frame, &got_packet) < 0) { 367 memcpy(image->planes[VPX_PLANE_U], s_video_frame->data[1], s_video_frame->linesize[1] * _phone->webcam_decoder_ctx->height / 2);
368 memcpy(image->planes[VPX_PLANE_V], s_video_frame->data[2], s_video_frame->linesize[2] * _phone->webcam_decoder_ctx->height / 2);
369 toxav_send_video (_phone->av, image);
370 //if (avcodec_encode_video2(cs->video_encoder_ctx, &enc_video_packet, s_video_frame, &got_packet) < 0) {
371 /*if (vpx_codec_encode(&cs->v_encoder, image, frame_counter, 1, 0, 0) != VPX_CODEC_OK) {
333 printf("could not encode video frame\n"); 372 printf("could not encode video frame\n");
334 continue; 373 continue;
335 } 374 }
375 ++frame_counter;
336 376
337 if (!got_packet) { 377 vpx_codec_iter_t iter = NULL;
338 continue; 378 vpx_codec_cx_pkt_t *pkt;
339 } 379 while( (pkt = vpx_codec_get_cx_data(&cs->v_encoder, &iter)) ) {
380 if (pkt->kind == VPX_CODEC_CX_FRAME_PKT)
381 toxav_send_rtp_payload(_phone->av, TypeVideo, pkt->data.frame.buf, pkt->data.frame.sz);
382 }*/
383 //if (!got_packet) {
384 // continue;
385 //}
340 386
341 if (!enc_video_packet.data) fprintf(stderr, "video packet data is NULL\n"); 387 //if (!enc_video_packet.data) fprintf(stderr, "video packet data is NULL\n");
342 388
343 toxav_send_rtp_payload(_phone->av, TypeVideo, enc_video_packet.data, enc_video_packet.size); 389 //toxav_send_rtp_payload(_phone->av, TypeVideo, enc_video_packet.data, enc_video_packet.size);
344 390
345 av_free_packet(&enc_video_packet); 391 //av_free_packet(&enc_video_packet);
346 } 392 }
347 } else { 393 } else {
348 av_free_packet(packet); 394 av_free_packet(packet);
349 } 395 }
350 } 396 }
351 397
398 vpx_img_free(image);
399
352 /* clean up codecs */ 400 /* clean up codecs */
353 pthread_mutex_lock(&cs->ctrl_mutex); 401 //pthread_mutex_lock(&cs->ctrl_mutex);
354 av_free(buffer); 402 av_free(buffer);
355 av_free(webcam_frame); 403 av_free(webcam_frame);
356 av_free(s_video_frame); 404 av_free(s_video_frame);
357 sws_freeContext(_phone->sws_ctx); 405 sws_freeContext(_phone->sws_ctx);
358 avcodec_close(cs->webcam_decoder_ctx); 406 //avcodec_close(webcam_decoder_ctx);
359 avcodec_close(cs->video_encoder_ctx); 407 //avcodec_close(cs->video_encoder_ctx);
360 pthread_mutex_unlock(&cs->ctrl_mutex); 408 //pthread_mutex_unlock(&cs->ctrl_mutex);
361 409
362 _phone->running_encvid = -1; 410 _phone->running_encvid = -1;
363 411
364 pthread_exit ( NULL ); 412 pthread_exit ( NULL );
365} 413}
414#endif
366 415
367void *encode_audio_thread(void *arg) 416void *encode_audio_thread(void *arg)
368{ 417{
369 INFO("Started encode audio thread!"); 418 INFO("Started encode audio thread!");
370 av_session_t* _phone = arg; 419 av_session_t* _phone = arg;
371 _phone->running_encaud = 1; 420 _phone->running_encaud = 1;
372 421
373 unsigned char encoded_data[4096]; 422 int ret = 0;
374 int encoded_size = 0;
375 int16_t frame[4096]; 423 int16_t frame[4096];
376 int frame_size = AUDIO_FRAME_SIZE; 424 int frame_size = AUDIO_FRAME_SIZE;
377 ALint sample = 0; 425 ALint sample = 0;
@@ -383,94 +431,164 @@ void *encode_audio_thread(void *arg)
383 if (sample >= frame_size) { 431 if (sample >= frame_size) {
384 alcCaptureSamples((ALCdevice*)_phone->audio_capture_device, frame, frame_size); 432 alcCaptureSamples((ALCdevice*)_phone->audio_capture_device, frame, frame_size);
385 433
386 encoded_size = toxav_encode_audio(_phone->av, frame, frame_size, encoded_data); 434 ret = toxav_send_audio(_phone->av, frame, frame_size);
387 435
388 if (encoded_size <= 0) { 436 if (ret < 0)
389 printf("Could not encode audio packet\n"); 437 printf("Could not encode or send audio packet\n");
390 } else { 438
391 if ( -1 == toxav_send_rtp_payload(_phone->av, TypeAudio, encoded_data, encoded_size) )
392 assert(0);
393 }
394 } else { 439 } else {
395 usleep(1000); 440 usleep(1000);
396 } 441 }
397 } 442 }
398 443
399 /* clean up codecs * 444 /* clean up codecs *
400 pthread_mutex_lock(&cs->ctrl_mutex);*/ 445 pthread_mutex_lock(&cs->ctrl_mutex);* /
401 alcCaptureStop((ALCdevice*)_phone->audio_capture_device); 446 alcCaptureStop((ALCdevice*)_phone->audio_capture_device);
402 alcCaptureCloseDevice((ALCdevice*)_phone->audio_capture_device); 447 alcCaptureCloseDevice((ALCdevice*)_phone->audio_capture_device);
403 /*pthread_mutex_unlock(&cs->ctrl_mutex);*/ 448 / *pthread_mutex_unlock(&cs->ctrl_mutex);*/
404 _phone->running_encaud = -1; 449 _phone->running_encaud = -1;
405 pthread_exit ( NULL ); 450 pthread_exit ( NULL );
406} 451}
407 452
453void convert_to_rgb(vpx_image_t *img, unsigned char *out)
454{
455 const int w = img->d_w;
456 const int w2 = w/2;
457 const int pstride = w*3;
458 const int h = img->d_h;
459 const int h2 = h/2;
460
461 const int strideY = img->stride[0];
462 const int strideU = img->stride[1];
463 const int strideV = img->stride[2];
464 int posy, posx;
465 for (posy = 0; posy < h2; posy++) {
466 unsigned char *dst = out + pstride * (posy * 2);
467 unsigned char *dst2 = out + pstride * (posy * 2 + 1);
468 const unsigned char *srcY = img->planes[0] + strideY * posy * 2;
469 const unsigned char *srcY2 = img->planes[0] + strideY * (posy * 2 + 1);
470 const unsigned char *srcU = img->planes[1] + strideU * posy;
471 const unsigned char *srcV = img->planes[2] + strideV * posy;
472
473 for (posx = 0; posx < w2; posx++) {
474 unsigned char Y,U,V;
475 short R,G,B;
476 short iR,iG,iB;
477
478 U = *(srcU++); V = *(srcV++);
479 iR = (351 * (V-128)) / 256;
480 iG = - (179 * (V-128)) / 256 - (86 * (U-128)) / 256;
481 iB = (444 * (U-128)) / 256;
482
483 Y = *(srcY++);
484 R = Y + iR ; G = Y + iG ; B = Y + iB ;
485 R = (R<0?0:(R>255?255:R)); G = (G<0?0:(G>255?255:G)); B = (B<0?0:(B>255?255:B));
486 *(dst++) = R; *(dst++) = G; *(dst++) = B;
487
488 Y = *(srcY2++);
489 R = Y + iR ; G = Y + iG ; B = Y + iB ;
490 R = (R<0?0:(R>255?255:R)); G = (G<0?0:(G>255?255:G)); B = (B<0?0:(B>255?255:B));
491 *(dst2++) = R; *(dst2++) = G; *(dst2++) = B;
492
493 Y = *(srcY++) ;
494 R = Y + iR ; G = Y + iG ; B = Y + iB ;
495 R = (R<0?0:(R>255?255:R)); G = (G<0?0:(G>255?255:G)); B = (B<0?0:(B>255?255:B));
496 *(dst++) = R; *(dst++) = G; *(dst++) = B;
497
498 Y = *(srcY2++);
499 R = Y + iR ; G = Y + iG ; B = Y + iB ;
500 R = (R<0?0:(R>255?255:R)); G = (G<0?0:(G>255?255:G)); B = (B<0?0:(B>255?255:B));
501 *(dst2++) = R; *(dst2++) = G; *(dst2++) = B;
502 }
503 }
504}
505
506#define mask32(BYTE) (*(uint32_t *)(uint8_t [4]){ [BYTE] = 0xff })
507
408void *decode_video_thread(void *arg) 508void *decode_video_thread(void *arg)
409{ 509{
410 INFO("Started decode video thread!"); 510 INFO("Started decode video thread!");
411 av_session_t* _phone = arg; 511 av_session_t* _phone = arg;
412 _phone->running_decvid = 1; 512 _phone->running_decvid = 1;
413 513
414 CodecState *cs = get_cs_temp(_phone->av); 514 //CodecState *cs = get_cs_temp(_phone->av);
415 cs->video_stream = 0; 515 //cs->video_stream = 0;
416 516
417 int recved_size; 517 //int recved_size;
418 uint8_t dest[RTP_PAYLOAD_SIZE]; 518 //uint8_t dest[RTP_PAYLOAD_SIZE];
419 519
420 int dec_frame_finished; 520 //int dec_frame_finished;
421 AVFrame *r_video_frame; 521 //AVFrame *r_video_frame;
422 r_video_frame = avcodec_alloc_frame(); 522 //r_video_frame = avcodec_alloc_frame();
423 AVPacket dec_video_packet; 523 //AVPacket dec_video_packet;
424 av_new_packet (&dec_video_packet, 65536); 524 //av_new_packet (&dec_video_packet, 65536);
425 int width = 0; 525 int width = 0;
426 int height = 0; 526 int height = 0;
427 527
428 while (_phone->running_decvid) { 528 while (_phone->running_decvid) {
429 529 //recved_size = toxav_recv_rtp_payload(_phone->av, TypeVideo, dest);
430 recved_size = toxav_recv_rtp_payload(_phone->av, TypeVideo, 1, dest); 530 //if (recved_size) {
431 531 vpx_image_t *image;
432 if (recved_size) { 532 if (toxav_recv_video(_phone->av, &image) == 0) {
433 memcpy(dec_video_packet.data, dest, recved_size); 533 //memcpy(dec_video_packet.data, dest, recved_size);
434 dec_video_packet.size = recved_size; 534 //dec_video_packet.size = recved_size;
435 535
436 avcodec_decode_video2(cs->video_decoder_ctx, r_video_frame, &dec_frame_finished, &dec_video_packet); 536 //avcodec_decode_video2(cs->video_decoder_ctx, r_video_frame, &dec_frame_finished, &dec_video_packet);
437 537
438 if (dec_frame_finished) { 538 //if (dec_frame_finished) {
439 539
440 /* Check if size has changed */ 540 /* Check if size has changed */
441 if (cs->video_decoder_ctx->width != width || cs->video_decoder_ctx->height != height) { 541 if (image->d_w != width || image->d_h != height) {
442 542
443 width = cs->video_decoder_ctx->width; 543 width = image->d_w;
444 height = cs->video_decoder_ctx->height; 544 height = image->d_h;
445 545
446 printf("w: %d h: %d \n", width, height); 546 printf("w: %d h: %d \n", width, height);
447 547
448 screen = SDL_SetVideoMode(width, height, 0, 0); 548 screen = SDL_SetVideoMode(width, height, 0, 0);
449 549
450 if (_phone->video_picture.bmp) 550 //if (_phone->video_picture.bmp)
451 SDL_FreeYUVOverlay(_phone->video_picture.bmp); 551 // SDL_FreeYUVOverlay(_phone->video_picture.bmp);
452 552
453 _phone->video_picture.bmp = SDL_CreateYUVOverlay(width, height, SDL_YV12_OVERLAY, screen); 553 //_phone->video_picture.bmp = SDL_CreateYUVOverlay(width, height, SDL_YV12_OVERLAY, screen);
454 _phone->sws_SDL_r_ctx = sws_getContext(width, height, cs->video_decoder_ctx->pix_fmt, width, height, PIX_FMT_YUV420P, 554 // _phone->sws_SDL_r_ctx = sws_getContext(width, height, cs->video_decoder_ctx->pix_fmt, width, height, PIX_FMT_YUV420P,
455 SWS_BILINEAR, NULL, NULL, NULL); 555 // SWS_BILINEAR, NULL, NULL, NULL);
456 } 556 }
557 uint8_t *rgb_image = malloc(width*height*3);
558 convert_to_rgb(image, rgb_image);
559 SDL_Surface* img_surface = SDL_CreateRGBSurfaceFrom(rgb_image, width, height, 24, width * 3, mask32(0), mask32(1), mask32(2), 0);
560 if(SDL_BlitSurface(img_surface, NULL, screen, NULL) == 0)
561 SDL_UpdateRect(screen, 0, 0, 0, 0);
562 /*
563 SDL_LockYUVOverlay(_phone->video_picture.bmp);
564 memcpy(_phone->video_picture.bmp->pixels[0], image->planes[VPX_PLANE_Y], _phone->video_picture.bmp->pitches[0] * height);
565 memcpy(_phone->video_picture.bmp->pixels[1], image->planes[VPX_PLANE_V], _phone->video_picture.bmp->pitches[1] * height / 2);
566 memcpy(_phone->video_picture.bmp->pixels[2], image->planes[VPX_PLANE_U], _phone->video_picture.bmp->pitches[2] * height / 2);
457 567
458 display_received_frame(_phone, r_video_frame); 568 SDL_Rect rect;
459 } else { 569 rect.x = 0;
570 rect.y = 0;
571 rect.w = width;
572 rect.h = height;
573 SDL_DisplayYUVOverlay(_phone->video_picture.bmp, &rect);*/
574 free(rgb_image);
575 //display_received_frame(_phone, image);
576
577 } //else {
460 /* TODO: request the sender to create a new i-frame immediatly */ 578 /* TODO: request the sender to create a new i-frame immediatly */
461 printf("Bad video packet\n"); 579 //printf("Bad video packet\n");
462 } 580 //}
463 } 581 //}
464 582
465 usleep(1000); 583 usleep(1000);
466 } 584 }
467 585
468 /* clean up codecs */ 586 /* clean up codecs */
469 av_free(r_video_frame); 587 //av_free(r_video_frame);
470 588
471 pthread_mutex_lock(&cs->ctrl_mutex); 589 //pthread_mutex_lock(&cs->ctrl_mutex);
472 avcodec_close(cs->video_decoder_ctx); 590 //avcodec_close(cs->video_decoder_ctx);
473 pthread_mutex_unlock(&cs->ctrl_mutex); 591 //pthread_mutex_unlock(&cs->ctrl_mutex);
474 592
475 _phone->running_decvid = -1; 593 _phone->running_decvid = -1;
476 594
@@ -483,11 +601,11 @@ void *decode_audio_thread(void *arg)
483 av_session_t* _phone = arg; 601 av_session_t* _phone = arg;
484 _phone->running_decaud = 1; 602 _phone->running_decaud = 1;
485 603
486 int recved_size; 604 //int recved_size;
487 uint8_t dest [RTP_PAYLOAD_SIZE]; 605 //uint8_t dest [RTP_PAYLOAD_SIZE];
488 606
489 int frame_size = AUDIO_FRAME_SIZE; 607 int frame_size = AUDIO_FRAME_SIZE;
490 int data_size; 608 //int data_size;
491 609
492 ALCdevice *dev; 610 ALCdevice *dev;
493 ALCcontext *ctx; 611 ALCcontext *ctx;
@@ -507,7 +625,7 @@ void *decode_audio_thread(void *arg)
507 625
508 uint16_t zeros[frame_size]; 626 uint16_t zeros[frame_size];
509 memset(zeros, 0, frame_size); 627 memset(zeros, 0, frame_size);
510 opus_int16 PCM[frame_size]; 628 int16_t PCM[frame_size];
511 629
512 int i; 630 int i;
513 for (i = 0; i < openal_buffers; ++i) { 631 for (i = 0; i < openal_buffers; ++i) {
@@ -527,28 +645,15 @@ void *decode_audio_thread(void *arg)
527 while (_phone->running_decaud) { 645 while (_phone->running_decaud) {
528 646
529 alGetSourcei(source, AL_BUFFERS_PROCESSED, &ready); 647 alGetSourcei(source, AL_BUFFERS_PROCESSED, &ready);
530 648 if (ready <= 0)
531 recved_size = toxav_recv_rtp_payload(_phone->av, TypeAudio, ready, dest); 649 continue;
532 650
533 if ( recved_size == ErrorAudioPacketLost ) { 651 dec_frame_len = toxav_recv_audio(_phone->av, frame_size, PCM);
534 printf("Lost packet\n"); 652
535 dec_frame_len = toxav_decode_audio(_phone->av, NULL, 0, frame_size, PCM);
536
537 } else if ( recved_size ) {
538 dec_frame_len = toxav_decode_audio(_phone->av, dest, recved_size, frame_size, PCM);
539 }
540
541
542 /* Play the packet */ 653 /* Play the packet */
543 if (dec_frame_len) { 654 if (dec_frame_len > 0) {
544 alGetSourcei(source, AL_BUFFERS_PROCESSED, &ready);
545
546 if (ready <= 0)
547 continue;
548
549 alSourceUnqueueBuffers(source, 1, &buffer); 655 alSourceUnqueueBuffers(source, 1, &buffer);
550 data_size = av_samples_get_buffer_size(NULL, 1, dec_frame_len, AV_SAMPLE_FMT_S16, 1); 656 alBufferData(buffer, AL_FORMAT_MONO16, PCM, dec_frame_len * 2 * 1, 48000);
551 alBufferData(buffer, AL_FORMAT_MONO16, PCM, data_size, 48000);
552 int error = alGetError(); 657 int error = alGetError();
553 658
554 if (error != AL_NO_ERROR) { 659 if (error != AL_NO_ERROR) {
@@ -573,16 +678,16 @@ void *decode_audio_thread(void *arg)
573 678
574 679
575ending: 680ending:
576 /* clean up codecs * / 681 /* clean up codecs */
577 pthread_mutex_lock(&cs->ctrl_mutex); 682 //pthread_mutex_lock(&cs->ctrl_mutex);
578 683 /*
579 alDeleteSources(1, &source); 684 alDeleteSources(1, &source);
580 alDeleteBuffers(openal_buffers, buffers); 685 alDeleteBuffers(openal_buffers, buffers);
581 alcMakeContextCurrent(NULL); 686 alcMakeContextCurrent(NULL);
582 alcDestroyContext(ctx); 687 alcDestroyContext(ctx);
583 alcCloseDevice(dev); 688 alcCloseDevice(dev);
584 689 */
585 pthread_mutex_unlock(&cs->ctrl_mutex); */ 690 //pthread_mutex_unlock(&cs->ctrl_mutex);
586 691
587 _phone->running_decaud = -1; 692 _phone->running_decaud = -1;
588 693
@@ -604,7 +709,7 @@ int phone_startmedia_loop ( ToxAv* arg )
604 /* 709 /*
605 * Rise all threads 710 * Rise all threads
606 */ 711 */
607 712#ifdef TOX_FFMPEG
608 /* Only checks for last peer */ 713 /* Only checks for last peer */
609 if ( toxav_get_peer_transmission_type(arg, 0) == TypeVideo && 714 if ( toxav_get_peer_transmission_type(arg, 0) == TypeVideo &&
610 0 > event.rise(encode_video_thread, toxav_get_agent_handler(arg)) ) 715 0 > event.rise(encode_video_thread, toxav_get_agent_handler(arg)) )
@@ -612,7 +717,7 @@ int phone_startmedia_loop ( ToxAv* arg )
612 INFO("Error while starting encode_video_thread()"); 717 INFO("Error while starting encode_video_thread()");
613 return -1; 718 return -1;
614 } 719 }
615 720#endif
616 /* Always send audio */ 721 /* Always send audio */
617 if ( 0 > event.rise(encode_audio_thread, toxav_get_agent_handler(arg)) ) 722 if ( 0 > event.rise(encode_audio_thread, toxav_get_agent_handler(arg)) )
618 { 723 {
@@ -779,7 +884,6 @@ av_session_t* av_init_session()
779 } 884 }
780 885
781 _retu->_friends = NULL; 886 _retu->_friends = NULL;
782 _retu->av = toxav_new(_retu->_messenger, _retu, _USERAGENT);
783 887
784 888
785 const ALchar *_device_list = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER); 889 const ALchar *_device_list = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER);
@@ -821,13 +925,58 @@ av_session_t* av_init_session()
821 printf("Could not start capture device! %d\n", alcGetError((ALCdevice*)_retu->audio_capture_device)); 925 printf("Could not start capture device! %d\n", alcGetError((ALCdevice*)_retu->audio_capture_device));
822 return 0; 926 return 0;
823 } 927 }
824 928 uint16_t height = 0, width = 0;
825 929#ifdef TOX_FFMPEG
930 avdevice_register_all();
931 avcodec_register_all();
932 av_register_all();
933
934 _retu->video_input_format = av_find_input_format(VIDEO_DRIVER);
935 if (avformat_open_input(&_retu->video_format_ctx, DEFAULT_WEBCAM, _retu->video_input_format, NULL) != 0) {
936 fprintf(stderr, "Opening video_input_format failed!\n");
937 //return -1;
938 return NULL;
939 }
940
941 avformat_find_stream_info(_retu->video_format_ctx, NULL);
942 av_dump_format(_retu->video_format_ctx, 0, DEFAULT_WEBCAM, 0);
943
944 for (i = 0; i < _retu->video_format_ctx->nb_streams; ++i) {
945 if (_retu->video_format_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
946 _retu->video_stream = i;
947 break;
948 }
949 }
950
951 _retu->webcam_decoder_ctx = _retu->video_format_ctx->streams[_retu->video_stream]->codec;
952 _retu->webcam_decoder = avcodec_find_decoder(_retu->webcam_decoder_ctx->codec_id);
953
954 if (_retu->webcam_decoder == NULL) {
955 fprintf(stderr, "Unsupported codec!\n");
956 //return -1;
957 return NULL;
958 }
959
960 if (_retu->webcam_decoder_ctx == NULL) {
961 fprintf(stderr, "Init webcam_decoder_ctx failed!\n");
962 //return -1;
963 return NULL;
964 }
965
966 if (avcodec_open2(_retu->webcam_decoder_ctx, _retu->webcam_decoder, NULL) < 0) {
967 fprintf(stderr, "Opening webcam decoder failed!\n");
968 //return -1;
969 return NULL;
970 }
971 width = _retu->webcam_decoder_ctx->width;
972 height = _retu->webcam_decoder_ctx->height;
973#endif
826 uint8_t _byte_address[TOX_FRIEND_ADDRESS_SIZE]; 974 uint8_t _byte_address[TOX_FRIEND_ADDRESS_SIZE];
827 tox_get_address(_retu->_messenger, _byte_address ); 975 tox_get_address(_retu->_messenger, _byte_address );
828 fraddr_to_str( _byte_address, _retu->_my_public_id ); 976 fraddr_to_str( _byte_address, _retu->_my_public_id );
829 977
830 978
979 _retu->av = toxav_new(_retu->_messenger, _retu, _USERAGENT, width, height);
831 980
832 /* ------------------ */ 981 /* ------------------ */
833 982
@@ -842,7 +991,7 @@ av_session_t* av_init_session()
842 toxav_register_callstate_callback(callback_recv_ending, OnEnding); 991 toxav_register_callstate_callback(callback_recv_ending, OnEnding);
843 992
844 toxav_register_callstate_callback(callback_recv_error, OnError); 993 toxav_register_callstate_callback(callback_recv_error, OnError);
845 toxav_register_callstate_callback(callback_requ_timeout, OnTimeout); 994 toxav_register_callstate_callback(callback_requ_timeout, OnRequestTimeout);
846 995
847 /* ------------------ */ 996 /* ------------------ */
848 997
@@ -1013,9 +1162,9 @@ void do_phone ( av_session_t* _phone )
1013 { 1162 {
1014 ToxAvError rc; 1163 ToxAvError rc;
1015 1164
1016 if ( _len > 1 && _line[2] == 'v' ) 1165 if ( _len > 1 && _line[2] == 'v' ) {
1017 rc = toxav_answer(_phone->av, TypeVideo); 1166 rc = toxav_answer(_phone->av, TypeVideo);
1018 else 1167 } else
1019 rc = toxav_answer(_phone->av, TypeAudio); 1168 rc = toxav_answer(_phone->av, TypeAudio);
1020 1169
1021 if ( rc == ErrorInvalidState ) { 1170 if ( rc == ErrorInvalidState ) {
diff --git a/toxav/rtp.c b/toxav/rtp.c
index e23fa132..80fcc630 100644
--- a/toxav/rtp.c
+++ b/toxav/rtp.c
@@ -30,10 +30,6 @@
30#include <assert.h> 30#include <assert.h>
31#include <stdlib.h> 31#include <stdlib.h>
32 32
33#include "../toxcore/util.h"
34#include "../toxcore/network.h"
35#include "../toxcore/net_crypto.h"
36#include "../toxcore/Messenger.h"
37 33
38#define PAYLOAD_ID_VALUE_OPUS 1 34#define PAYLOAD_ID_VALUE_OPUS 1
39#define PAYLOAD_ID_VALUE_VP8 2 35#define PAYLOAD_ID_VALUE_VP8 2
@@ -241,8 +237,8 @@ RTPHeader* extract_header ( const uint8_t* payload, int length )
241 237
242 _retu->flags = *_it; ++_it; 238 _retu->flags = *_it; ++_it;
243 239
244 /* This indicates if the first 2 bytes are valid. 240 /* This indicates if the first 2 bits are valid.
245 * Now it my happen that this is out of order but 241 * Now it may happen that this is out of order but
246 * it cuts down chances of parsing some invalid value 242 * it cuts down chances of parsing some invalid value
247 */ 243 */
248 244
@@ -299,7 +295,7 @@ RTPHeader* extract_header ( const uint8_t* payload, int length )
299 * @return RTPExtHeader* Extracted extension header. 295 * @return RTPExtHeader* Extracted extension header.
300 * @retval NULL Error occurred while extracting extension header. 296 * @retval NULL Error occurred while extracting extension header.
301 */ 297 */
302RTPExtHeader* extract_ext_header ( const uint8_t* payload, size_t length ) 298RTPExtHeader* extract_ext_header ( const uint8_t* payload, uint16_t length )
303{ 299{
304 const uint8_t* _it = payload; 300 const uint8_t* _it = payload;
305 301
@@ -551,7 +547,7 @@ int rtp_handle_packet ( void* object, IP_Port ip_port, uint8_t* data, uint32_t l
551 /* Hopefully this goes well 547 /* Hopefully this goes well
552 * NOTE: Is this even used? 548 * NOTE: Is this even used?
553 */ 549 */
554 memcpy(&_msg->from, &ip_port, sizeof(tox_IP_Port)); 550 memcpy(&_msg->from, &ip_port, sizeof(IP_Port));
555 551
556 /* Check if message came in late */ 552 /* Check if message came in late */
557 if ( check_late_message(_session, _msg) < 0 ) { /* Not late */ 553 if ( check_late_message(_session, _msg) < 0 ) { /* Not late */
@@ -689,7 +685,7 @@ int rtp_release_session_recv ( RTPSession* session )
689 685
690 686
691/** 687/**
692 * @brief Get's oldes message in the list. 688 * @brief Gets oldest message in the list.
693 * 689 *
694 * @param session Where the list is. 690 * @param session Where the list is.
695 * @return RTPMessage* The message. You _must_ call rtp_msg_free() to free it. 691 * @return RTPMessage* The message. You _must_ call rtp_msg_free() to free it.
@@ -727,7 +723,7 @@ RTPMessage* rtp_recv_msg ( RTPSession* session )
727 * @retval -1 On error. 723 * @retval -1 On error.
728 * @retval 0 On success. 724 * @retval 0 On success.
729 */ 725 */
730int rtp_send_msg ( RTPSession* session, Tox* messenger, const uint8_t* data, uint16_t length ) 726int rtp_send_msg ( RTPSession* session, Messenger* messenger, const uint8_t* data, uint16_t length )
731{ 727{
732 RTPMessage* msg = rtp_new_message (session, data, length); 728 RTPMessage* msg = rtp_new_message (session, data, length);
733 729
@@ -743,8 +739,8 @@ int rtp_send_msg ( RTPSession* session, Tox* messenger, const uint8_t* data, uin
743 increase_nonce ( _calculated, msg->header->sequnum ); 739 increase_nonce ( _calculated, msg->header->sequnum );
744 740
745 /* Need to skip 2 bytes that are for sequnum */ 741 /* Need to skip 2 bytes that are for sequnum */
746 int encrypted_length = encrypt_data_symmetric( 742 int encrypted_length = encrypt_data_symmetric( /* TODO: msg->length - 2 (fix this properly)*/
747 (uint8_t*) session->encrypt_key, _calculated, msg->data + 2, msg->length - 2, _send_data + 3 ); 743 (uint8_t*) session->encrypt_key, _calculated, msg->data + 2, msg->length, _send_data + 3 );
748 744
749 int full_length = encrypted_length + 3; 745 int full_length = encrypted_length + 3;
750 746
@@ -752,7 +748,8 @@ int rtp_send_msg ( RTPSession* session, Tox* messenger, const uint8_t* data, uin
752 _send_data[2] = msg->data[1]; 748 _send_data[2] = msg->data[1];
753 749
754 750
755 if ( full_length != sendpacket ( ((Messenger*)messenger)->net, *((IP_Port*) &session->dest), _send_data, full_length) ) { 751 /*if ( full_length != sendpacket ( messenger->net, *((IP_Port*) &session->dest), _send_data, full_length) ) {*/
752 if ( full_length != send_custom_user_packet(messenger, session->dest, _send_data, full_length) ) {
756 printf("Rtp error: %s\n", strerror(errno)); 753 printf("Rtp error: %s\n", strerror(errno));
757 return -1; 754 return -1;
758 } 755 }
@@ -816,30 +813,26 @@ void rtp_free_msg ( RTPSession* session, RTPMessage* msg )
816 * @retval NULL Error occurred. 813 * @retval NULL Error occurred.
817 */ 814 */
818RTPSession* rtp_init_session ( int payload_type, 815RTPSession* rtp_init_session ( int payload_type,
819 Tox* messenger, 816 Messenger* messenger,
820 int friend_num, 817 int friend_num,
821 const uint8_t* encrypt_key, 818 const uint8_t* encrypt_key,
822 const uint8_t* decrypt_key, 819 const uint8_t* decrypt_key,
823 const uint8_t* encrypt_nonce, 820 const uint8_t* encrypt_nonce,
824 const uint8_t* decrypt_nonce 821 const uint8_t* decrypt_nonce )
825)
826{ 822{
827 Messenger* _messenger_casted = (Messenger*) messenger;
828
829 IP_Port _dest = get_friend_ipport(_messenger_casted, friend_num );
830
831 /* This should be enough eh? */
832 if ( _dest.port == 0) {
833 return NULL;
834 }
835
836 RTPSession* _retu = calloc(1, sizeof(RTPSession)); 823 RTPSession* _retu = calloc(1, sizeof(RTPSession));
837 assert(_retu); 824 assert(_retu);
838 825
839 networking_registerhandler(_messenger_casted->net, payload_type, rtp_handle_packet, _retu); 826 /*networking_registerhandler(messenger->net, payload_type, rtp_handle_packet, _retu);*/
827 if ( -1 == custom_user_packet_registerhandler(messenger, friend_num, payload_type, rtp_handle_packet, _retu) )
828 {
829 fprintf(stderr, "Error setting custom register handler for rtp session\n");
830 free(_retu);
831 return NULL;
832 }
840 833
841 _retu->version = RTP_VERSION; /* It's always 2 */ 834 _retu->version = RTP_VERSION; /* It's always 2 */
842 _retu->padding = 0; /* If some additional data is needed about the packet */ 835 _retu->padding = 0; /* If some additional data is needed about the packet */
843 _retu->extension = 0; /* If extension to header is needed */ 836 _retu->extension = 0; /* If extension to header is needed */
844 _retu->cc = 1; /* Amount of contributors */ 837 _retu->cc = 1; /* Amount of contributors */
845 _retu->csrc = NULL; /* Container */ 838 _retu->csrc = NULL; /* Container */
@@ -847,7 +840,7 @@ RTPSession* rtp_init_session ( int payload_type,
847 _retu->marker = 0; 840 _retu->marker = 0;
848 _retu->payload_type = payload_table[payload_type]; 841 _retu->payload_type = payload_table[payload_type];
849 842
850 _retu->dest = *((tox_IP_Port*)&_dest); 843 _retu->dest = friend_num;
851 844
852 _retu->rsequnum = _retu->sequnum = 1; 845 _retu->rsequnum = _retu->sequnum = 1;
853 846
@@ -894,12 +887,12 @@ RTPSession* rtp_init_session ( int payload_type,
894 * @retval -1 Error occurred. 887 * @retval -1 Error occurred.
895 * @retval 0 Success. 888 * @retval 0 Success.
896 */ 889 */
897int rtp_terminate_session ( RTPSession* session, Tox* messenger ) 890int rtp_terminate_session ( RTPSession* session, Messenger* messenger )
898{ 891{
899 if ( !session ) 892 if ( !session )
900 return -1; 893 return -1;
901 894
902 networking_registerhandler(((Messenger*)messenger)->net, session->prefix, NULL, NULL); 895 custom_user_packet_registerhandler(messenger, session->dest, session->prefix, NULL, NULL);
903 896
904 free ( session->ext_header ); 897 free ( session->ext_header );
905 free ( session->csrc ); 898 free ( session->csrc );
diff --git a/toxav/rtp.h b/toxav/rtp.h
index 4b0d681f..03f6a873 100644
--- a/toxav/rtp.h
+++ b/toxav/rtp.h
@@ -28,10 +28,14 @@
28#define RTP_VERSION 2 28#define RTP_VERSION 2
29#include <inttypes.h> 29#include <inttypes.h>
30#include <pthread.h> 30#include <pthread.h>
31#include "../toxcore/tox.h" 31
32#include "../toxcore/util.h"
33#include "../toxcore/network.h"
34#include "../toxcore/net_crypto.h"
35#include "../toxcore/Messenger.h"
32 36
33#define MAX_SEQU_NUM 65535 37#define MAX_SEQU_NUM 65535
34#define MAX_RTP_SIZE 10400 38#define MAX_RTP_SIZE 65535
35 39
36/** 40/**
37 * @brief Standard rtp header 41 * @brief Standard rtp header
@@ -72,7 +76,7 @@ typedef struct _RTPMessage {
72 76
73 uint8_t data[MAX_RTP_SIZE]; 77 uint8_t data[MAX_RTP_SIZE];
74 uint32_t length; 78 uint32_t length;
75 tox_IP_Port from; 79 IP_Port from;
76 80
77 struct _RTPMessage* next; 81 struct _RTPMessage* next;
78} RTPMessage; 82} RTPMessage;
@@ -128,7 +132,7 @@ typedef struct _RTPSession {
128 uint8_t prefix; 132 uint8_t prefix;
129 133
130 pthread_mutex_t mutex; 134 pthread_mutex_t mutex;
131 tox_IP_Port dest; 135 int dest;
132 136
133} RTPSession; 137} RTPSession;
134 138
@@ -164,7 +168,7 @@ RTPMessage* rtp_recv_msg ( RTPSession* session );
164 * @retval -1 On error. 168 * @retval -1 On error.
165 * @retval 0 On success. 169 * @retval 0 On success.
166 */ 170 */
167int rtp_send_msg ( RTPSession* session, Tox* messenger, const uint8_t* data, uint16_t length ); 171int rtp_send_msg ( RTPSession* session, Messenger* messenger, const uint8_t* data, uint16_t length );
168 172
169 173
170/** 174/**
@@ -192,7 +196,7 @@ void rtp_free_msg ( RTPSession* session, RTPMessage* msg );
192 * @retval NULL Error occurred. 196 * @retval NULL Error occurred.
193 */ 197 */
194RTPSession* rtp_init_session ( int payload_type, 198RTPSession* rtp_init_session ( int payload_type,
195 Tox* messenger, 199 Messenger* messenger,
196 int friend_num, 200 int friend_num,
197 const uint8_t* encrypt_key, 201 const uint8_t* encrypt_key,
198 const uint8_t* decrypt_key, 202 const uint8_t* decrypt_key,
@@ -209,7 +213,7 @@ RTPSession* rtp_init_session ( int payload_type,
209 * @retval -1 Error occurred. 213 * @retval -1 Error occurred.
210 * @retval 0 Success. 214 * @retval 0 Success.
211 */ 215 */
212int rtp_terminate_session ( RTPSession* session, Tox* messenger ); 216int rtp_terminate_session ( RTPSession* session, Messenger* messenger );
213 217
214 218
215 219
diff --git a/toxav/toxav.c b/toxav/toxav.c
index 8757d7fd..5683795e 100644
--- a/toxav/toxav.c
+++ b/toxav/toxav.c
@@ -26,16 +26,16 @@
26#include "config.h" 26#include "config.h"
27#endif /* HAVE_CONFIG_H */ 27#endif /* HAVE_CONFIG_H */
28 28
29#include "toxav.h" 29#include "media.h"
30#include "../toxcore/tox.h"
31#include "rtp.h" 30#include "rtp.h"
32#include "msi.h" 31#include "msi.h"
33#include "media.h"
34 32
35#include <stdlib.h> 33#include <stdlib.h>
36#include <string.h> 34#include <string.h>
37#include <assert.h> 35#include <assert.h>
38 36
37#include "toxav.h"
38
39#define inline__ inline __attribute__((always_inline)) 39#define inline__ inline __attribute__((always_inline))
40 40
41static const uint8_t audio_index = 0, video_index = 1; 41static const uint8_t audio_index = 0, video_index = 1;
@@ -50,7 +50,7 @@ typedef enum {
50 50
51typedef struct _ToxAv 51typedef struct _ToxAv
52{ 52{
53 Tox* messenger; 53 Messenger* messenger;
54 54
55 MSISession* msi_session; /** Main msi session */ 55 MSISession* msi_session; /** Main msi session */
56 56
@@ -89,21 +89,23 @@ typedef struct _ToxAv
89 89
90 90
91 91
92ToxAv* toxav_new( Tox* messenger, void* useragent, const char* ua_name ) 92ToxAv* toxav_new( Tox* messenger, void* useragent, const char* ua_name , uint16_t video_width, uint16_t video_height)
93{ 93{
94 ToxAv* av = calloc ( sizeof(ToxAv), 1); 94 ToxAv* av = calloc ( sizeof(ToxAv), 1);
95 95 if (av == NULL)
96 av->msi_session = msi_init_session(messenger, (const unsigned char*) ua_name ); 96 return NULL;
97
98 av->messenger = (Messenger *)messenger;
99
100 av->msi_session = msi_init_session(av->messenger, (const unsigned char*) ua_name );
97 av->msi_session->agent_handler = av; 101 av->msi_session->agent_handler = av;
98 102
99 av->rtp_sessions[0] = av->rtp_sessions [1] = NULL; 103 av->rtp_sessions[0] = av->rtp_sessions [1] = NULL;
100 104
101 av->messenger = messenger;
102
103 /* NOTE: This should be user defined or? */ 105 /* NOTE: This should be user defined or? */
104 av->j_buf = create_queue(20); 106 av->j_buf = create_queue(20);
105 107
106 av->cs = codec_init_session(AUDIO_BITRATE, AUDIO_FRAME_DURATION, AUDIO_SAMPLE_RATE, 1, VIDEO_BITRATE, DEFAULT_WEBCAM, VIDEO_DRIVER); 108 av->cs = codec_init_session(AUDIO_BITRATE, AUDIO_FRAME_DURATION, AUDIO_SAMPLE_RATE, AUDIO_CHANNELS, video_width, video_height, VIDEO_BITRATE);
107 109
108 av->agent_handler = useragent; 110 av->agent_handler = useragent;
109 111
@@ -273,7 +275,7 @@ inline__ int toxav_send_rtp_payload ( ToxAv* av, ToxAvCallType type, const uint8
273 else return -1; 275 else return -1;
274} 276}
275 277
276inline__ int toxav_recv_rtp_payload ( ToxAv* av, ToxAvCallType type, int ready, uint8_t* dest ) 278inline__ int toxav_recv_rtp_payload ( ToxAv* av, ToxAvCallType type, uint8_t* dest )
277{ 279{
278 if ( !dest ) return ErrorInternal; 280 if ( !dest ) return ErrorInternal;
279 281
@@ -283,20 +285,19 @@ inline__ int toxav_recv_rtp_payload ( ToxAv* av, ToxAvCallType type, int ready,
283 285
284 if ( type == TypeAudio ) { 286 if ( type == TypeAudio ) {
285 287
286 message = rtp_recv_msg(av->rtp_sessions[audio_index]); 288 do {
289 message = rtp_recv_msg(av->rtp_sessions[audio_index]);
290
291 if (message) {
292 /* push the packet into the queue */
293 queue(av->j_buf, message);
294 }
295 } while(message);
287 296
288 if (message) { 297 int success = 0;
289 /* push the packet into the queue */ 298 message = dequeue(av->j_buf, &success);
290 queue(av->j_buf, message); 299
291 } 300 if ( success == 2) return ErrorAudioPacketLost;
292
293 if (ready) {
294 int success = 0;
295 message = dequeue(av->j_buf, &success);
296
297 if ( success == 2) return ErrorAudioPacketLost;
298 }
299 else return 0;
300 } 301 }
301 else { 302 else {
302 message = rtp_recv_msg(av->rtp_sessions[video_index]); 303 message = rtp_recv_msg(av->rtp_sessions[video_index]);
@@ -315,19 +316,75 @@ inline__ int toxav_recv_rtp_payload ( ToxAv* av, ToxAvCallType type, int ready,
315 return 0; 316 return 0;
316} 317}
317 318
318inline__ int toxav_decode_audio ( ToxAv* av, const uint8_t* payload, uint16_t length, int frame_size, short int* dest ) 319inline__ int toxav_recv_video ( ToxAv* av, vpx_image_t **output)
319{ 320{
320 if ( !dest ) return ErrorInternal; 321 if ( !output ) return ErrorInternal;
322 uint8_t packet [RTP_PAYLOAD_SIZE];
323 int recved_size = 0;
324 do {
325 recved_size = toxav_recv_rtp_payload(av, TypeVideo, packet);
326 if (recved_size > 0) {
327 printf("decode: %s\n", vpx_codec_err_to_string(vpx_codec_decode(&av->cs->v_decoder, packet, recved_size, NULL, 0)));
328 }
329 }while (recved_size > 0);
330 vpx_codec_iter_t iter = NULL;
331 vpx_image_t *img;
332 img = vpx_codec_get_frame(&av->cs->v_decoder, &iter);
333 if (img == NULL)
334 return ErrorInternal;
321 335
322 return opus_decode(av->cs->audio_decoder, payload, length, dest, frame_size, payload ? 0 : 1); 336 *output = img;
337 return 0;
323} 338}
324 339
325inline__ int toxav_encode_audio ( ToxAv* av, const short int* frame, int frame_size, uint8_t* dest ) 340inline__ int toxav_send_video ( ToxAv* av, vpx_image_t *input)
326{ 341{
327 if ( !dest ) 342 if (vpx_codec_encode(&av->cs->v_encoder, input, av->cs->frame_counter, 1, 0, MAX_ENCODE_TIME_US) != VPX_CODEC_OK) {
343 printf("could not encode video frame\n");
328 return ErrorInternal; 344 return ErrorInternal;
345 }
346 ++av->cs->frame_counter;
347
348 vpx_codec_iter_t iter = NULL;
349 const vpx_codec_cx_pkt_t *pkt;
350 int sent = 0;
351 while( (pkt = vpx_codec_get_cx_data(&av->cs->v_encoder, &iter)) ) {
352 if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
353 if (toxav_send_rtp_payload(av, TypeVideo, pkt->data.frame.buf, pkt->data.frame.sz) != -1)
354 ++sent;
355 }
356 }
357 if (sent > 0)
358 return 0;
329 359
330 return opus_encode(av->cs->audio_encoder, frame, frame_size, dest, RTP_PAYLOAD_SIZE); 360 return ErrorInternal;
361}
362
363inline__ int toxav_recv_audio ( ToxAv* av, int frame_size, int16_t* dest )
364{
365 if ( !dest ) return ErrorInternal;
366 uint8_t packet [RTP_PAYLOAD_SIZE];
367
368 int recved_size = toxav_recv_rtp_payload(av, TypeAudio, packet);
369
370 if ( recved_size == ErrorAudioPacketLost ) {
371 printf("Lost packet\n");
372 return opus_decode(av->cs->audio_decoder, NULL, 0, dest, frame_size, 1);
373 } else if ( recved_size ){
374 return opus_decode(av->cs->audio_decoder, packet, recved_size, dest, frame_size, 0);
375 } else {
376 return ErrorInternal;
377 }
378}
379
380inline__ int toxav_send_audio ( ToxAv* av, const int16_t* frame, int frame_size)
381{
382 uint8_t temp_data[RTP_PAYLOAD_SIZE];
383 int32_t ret = opus_encode(av->cs->audio_encoder, frame, frame_size, temp_data, sizeof(temp_data));
384 if (ret <= 0)
385 return ErrorInternal;
386
387 return toxav_send_rtp_payload(av, TypeAudio, temp_data, ret);
331} 388}
332 389
333int toxav_get_peer_transmission_type ( ToxAv* av, int peer ) 390int toxav_get_peer_transmission_type ( ToxAv* av, int peer )
@@ -343,10 +400,3 @@ void* toxav_get_agent_handler ( ToxAv* av )
343{ 400{
344 return av->agent_handler; 401 return av->agent_handler;
345} 402}
346
347
348/* Only temporary */
349void* get_cs_temp(ToxAv* av)
350{
351 return av->cs;
352}
diff --git a/toxav/toxav.h b/toxav/toxav.h
index 96a666a2..63dbf162 100644
--- a/toxav/toxav.h
+++ b/toxav/toxav.h
@@ -27,6 +27,9 @@
27#define __TOXAV 27#define __TOXAV
28#include <inttypes.h> 28#include <inttypes.h>
29 29
30/* vpx_image_t */
31#include <vpx/vpx_image.h>
32
30typedef void* ( *ToxAVCallback ) ( void* arg ); 33typedef void* ( *ToxAVCallback ) ( void* arg );
31typedef struct _ToxAv ToxAv; 34typedef struct _ToxAv ToxAv;
32 35
@@ -35,7 +38,29 @@ typedef struct _ToxAv ToxAv;
35typedef struct Tox Tox; 38typedef struct Tox Tox;
36#endif 39#endif
37 40
38#define RTP_PAYLOAD_SIZE 10400 41#define RTP_PAYLOAD_SIZE 65535
42
43/* Default video bitrate in bytes/s */
44#define VIDEO_BITRATE 10*1000*100
45
46/* Default audio bitrate in bits/s */
47#define AUDIO_BITRATE 64000
48
49/* Number of audio channels. */
50#define AUDIO_CHANNELS 1
51
52/* Audio frame duration in miliseconds */
53#define AUDIO_FRAME_DURATION 20
54
55/* Audio sample rate recommended to be 48kHz for Opus */
56#define AUDIO_SAMPLE_RATE 48000
57
58/* The amount of samples in one audio frame */
59#define AUDIO_FRAME_SIZE AUDIO_SAMPLE_RATE*AUDIO_FRAME_DURATION/1000
60
61/* Assume 60 fps*/
62#define MAX_ENCODE_TIME_US ((1000 / 60) * 1000)
63
39 64
40/** 65/**
41 * @brief Callbacks ids that handle the call states 66 * @brief Callbacks ids that handle the call states
@@ -55,7 +80,7 @@ typedef enum {
55 80
56 /* Protocol */ 81 /* Protocol */
57 OnError, 82 OnError,
58 OnTimeout 83 OnRequestTimeout
59 84
60} ToxAvCallbackID; 85} ToxAvCallbackID;
61 86
@@ -86,7 +111,7 @@ typedef enum {
86} ToxAvError; 111} ToxAvError;
87 112
88 113
89ToxAv* toxav_new(Tox* messenger, void* useragent, const char* ua_name); 114ToxAv* toxav_new(Tox* messenger, void* useragent, const char* ua_name, uint16_t video_width, uint16_t video_height) ;
90void toxav_kill(ToxAv* av); 115void toxav_kill(ToxAv* av);
91 116
92void toxav_register_callstate_callback (ToxAVCallback callback, ToxAvCallbackID id); 117void toxav_register_callstate_callback (ToxAVCallback callback, ToxAvCallbackID id);
@@ -103,27 +128,25 @@ int toxav_prepare_transmission(ToxAv* av);
103int toxav_kill_transmission(ToxAv* av); 128int toxav_kill_transmission(ToxAv* av);
104 129
105 130
106int toxav_send_rtp_payload(ToxAv* av, ToxAvCallType type, const uint8_t* payload, uint16_t length); 131
107 132
108/* Return length of received packet. Returns 0 if nothing recved. Dest has to have 133/* Return length of received packet. Returns 0 if nothing recved. Dest has to have
109 * MAX_RTP_PAYLOAD_SIZE space available. Returns -1 if packet is not ready (ready < 1) for deque. 134 * MAX_RTP_PAYLOAD_SIZE space available. Returns -1 if packet is not ready (ready < 1) for deque.
110 * For video packets set 'ready' at _any_ value. 135 * For video packets set 'ready' at _any_ value.
111 */ 136 */
112int toxav_recv_rtp_payload(ToxAv* av, ToxAvCallType type, int ready, uint8_t* dest);
113
114
115 137
138/* returns 0 on success */
139int toxav_recv_video ( ToxAv* av, vpx_image_t **output);
116 140
117int toxav_decode_audio( ToxAv* av, const uint8_t* payload, uint16_t length, int frame_size, short int* dest ); 141int toxav_recv_audio( ToxAv* av, int frame_size, int16_t* dest );
118 142
119/* Please make sure 'dest' has enough storage for RTP_PAYLOAD_SIZE length of data */ 143int toxav_send_video ( ToxAv* av, vpx_image_t *input);
120int toxav_encode_audio( ToxAv* av, const short int* frame, int frame_size, uint8_t* dest ); 144/* Encode and send audio frame. */
145int toxav_send_audio ( ToxAv* av, const int16_t* frame, int frame_size);
121 146
122 147
123 148
124int toxav_get_peer_transmission_type ( ToxAv* av, int peer ); 149int toxav_get_peer_transmission_type ( ToxAv* av, int peer );
125void* toxav_get_agent_handler ( ToxAv* av ); 150void* toxav_get_agent_handler ( ToxAv* av );
126 151
127/* Use this to get handle of CodecState from ToxAv struct */
128void* get_cs_temp( ToxAv* av );
129#endif /* __TOXAV */ \ No newline at end of file 152#endif /* __TOXAV */ \ No newline at end of file
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-28 21:34:35 +0000