diff options
Diffstat (limited to 'toxmsi')
-rw-r--r-- | toxmsi/AV_codec.c | 823 | ||||
-rw-r--r-- | toxmsi/AV_codec.h | 168 | ||||
-rw-r--r-- | toxmsi/phone.c | 645 |
3 files changed, 1636 insertions, 0 deletions
diff --git a/toxmsi/AV_codec.c b/toxmsi/AV_codec.c new file mode 100644 index 00000000..6730dca9 --- /dev/null +++ b/toxmsi/AV_codec.c | |||
@@ -0,0 +1,823 @@ | |||
1 | /* AV_codec.c | ||
2 | // * | ||
3 | * Audio and video codec intitialisation, encoding/decoding and playback | ||
4 | * | ||
5 | * Copyright (C) 2013 Tox project All Rights Reserved. | ||
6 | * | ||
7 | * This file is part of Tox. | ||
8 | * | ||
9 | * Tox is free software: you can redistribute it and/or modify | ||
10 | * it under the terms of the GNU General Public License as published by | ||
11 | * the Free Software Foundation, either version 3 of the License, or | ||
12 | * (at your option) any later version. | ||
13 | * | ||
14 | * Tox is distributed in the hope that it will be useful, | ||
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
17 | * GNU General Public License for more details. | ||
18 | * | ||
19 | * You should have received a copy of the GNU General Public License | ||
20 | * along with Tox. If not, see <http://www.gnu.org/licenses/>. | ||
21 | * | ||
22 | */ | ||
23 | |||
24 | /*----------------------------------------------------------------------------------*/ | ||
25 | |||
26 | #ifdef HAVE_CONFIG_H | ||
27 | #include "config.h" | ||
28 | #endif /* HAVE_CONFIG_H */ | ||
29 | |||
30 | #include <stdio.h> | ||
31 | #include <math.h> | ||
32 | #include <libavcodec/avcodec.h> | ||
33 | #include <libavformat/avformat.h> | ||
34 | #include <libswscale/swscale.h> | ||
35 | #include <libavdevice/avdevice.h> | ||
36 | #include <libavutil/opt.h> | ||
37 | #include <AL/al.h> | ||
38 | #include <AL/alc.h> | ||
39 | #include <SDL.h> | ||
40 | #include <SDL_thread.h> | ||
41 | #include <pthread.h> | ||
42 | #include <opus/opus.h> | ||
43 | |||
44 | #include "toxmsi.h" | ||
45 | #include "toxmsi_message.h" | ||
46 | #include "toxrtp_message.h" | ||
47 | #include "toxrtp/tests/test_helper.h" | ||
48 | #include "phone.h" | ||
49 | #include "AV_codec.h" | ||
50 | |||
51 | int display_received_frame(codec_state *cs, AVFrame *r_video_frame) | ||
52 | { | ||
53 | AVPicture pict; | ||
54 | SDL_LockYUVOverlay(cs->video_picture.bmp); | ||
55 | |||
56 | pict.data[0] = cs->video_picture.bmp->pixels[0]; | ||
57 | pict.data[1] = cs->video_picture.bmp->pixels[2]; | ||
58 | pict.data[2] = cs->video_picture.bmp->pixels[1]; | ||
59 | pict.linesize[0] = cs->video_picture.bmp->pitches[0]; | ||
60 | pict.linesize[1] = cs->video_picture.bmp->pitches[2]; | ||
61 | pict.linesize[2] = cs->video_picture.bmp->pitches[1]; | ||
62 | |||
63 | /* Convert the image into YUV format that SDL uses */ | ||
64 | sws_scale(cs->sws_SDL_r_ctx, (uint8_t const * const *)r_video_frame->data, r_video_frame->linesize, 0, | ||
65 | cs->video_decoder_ctx->height, pict.data, pict.linesize ); | ||
66 | |||
67 | SDL_UnlockYUVOverlay(cs->video_picture.bmp); | ||
68 | SDL_Rect rect; | ||
69 | rect.x = 0; | ||
70 | rect.y = 0; | ||
71 | rect.w = cs->video_decoder_ctx->width; | ||
72 | rect.h = cs->video_decoder_ctx->height; | ||
73 | SDL_DisplayYUVOverlay(cs->video_picture.bmp, &rect); | ||
74 | return 1; | ||
75 | } | ||
76 | |||
77 | struct jitter_buffer { | ||
78 | rtp_msg_t **queue; | ||
79 | uint16_t capacity; | ||
80 | uint16_t size; | ||
81 | uint16_t front; | ||
82 | uint16_t rear; | ||
83 | uint8_t queue_ready; | ||
84 | uint16_t current_id; | ||
85 | uint32_t current_ts; | ||
86 | uint8_t id_set; | ||
87 | }; | ||
88 | |||
89 | struct jitter_buffer *create_queue(int capacity) | ||
90 | { | ||
91 | struct jitter_buffer *q; | ||
92 | q = (struct jitter_buffer *)calloc(sizeof(struct jitter_buffer),1); | ||
93 | q->queue = (rtp_msg_t **)calloc((sizeof(rtp_msg_t) * capacity),1); | ||
94 | int i = 0; | ||
95 | |||
96 | for (i = 0; i < capacity; ++i) { | ||
97 | q->queue[i] = NULL; | ||
98 | } | ||
99 | |||
100 | q->size = 0; | ||
101 | q->capacity = capacity; | ||
102 | q->front = 0; | ||
103 | q->rear = -1; | ||
104 | q->queue_ready = 0; | ||
105 | q->current_id = 0; | ||
106 | q->current_ts = 0; | ||
107 | q->id_set = 0; | ||
108 | return q; | ||
109 | } | ||
110 | |||
111 | /* returns 1 if 'a' has a higher sequence number than 'b' */ | ||
112 | uint8_t sequence_number_older(uint16_t sn_a, uint16_t sn_b, uint32_t ts_a, uint32_t ts_b) | ||
113 | { | ||
114 | /* should be stable enough */ | ||
115 | return (sn_a > sn_b || ts_a > ts_b); | ||
116 | } | ||
117 | |||
118 | /* success is 0 when there is nothing to dequeue, 1 when there's a good packet, 2 when there's a lost packet */ | ||
119 | rtp_msg_t *dequeue(struct jitter_buffer *q, int *success) | ||
120 | { | ||
121 | if (q->size == 0 || q->queue_ready == 0) { | ||
122 | q->queue_ready = 0; | ||
123 | *success = 0; | ||
124 | return NULL; | ||
125 | } | ||
126 | |||
127 | int front = q->front; | ||
128 | |||
129 | if (q->id_set == 0) { | ||
130 | q->current_id = q->queue[front]->_header->_sequence_number; | ||
131 | q->current_ts = q->queue[front]->_header->_timestamp; | ||
132 | q->id_set = 1; | ||
133 | } else { | ||
134 | int next_id = q->queue[front]->_header->_sequence_number; | ||
135 | int next_ts = q->queue[front]->_header->_timestamp; | ||
136 | |||
137 | /* if this packet is indeed the expected packet */ | ||
138 | if (next_id == (q->current_id + 1) % _MAX_SEQU_NUM) { | ||
139 | q->current_id = next_id; | ||
140 | q->current_ts = next_ts; | ||
141 | } else { | ||
142 | if (sequence_number_older(next_id, q->current_id, next_ts, q->current_ts)) { | ||
143 | printf("nextid: %d current: %d\n", next_id, q->current_id); | ||
144 | q->current_id = (q->current_id + 1) % _MAX_SEQU_NUM; | ||
145 | *success = 2; /* tell the decoder the packet is lost */ | ||
146 | return NULL; | ||
147 | } else { | ||
148 | /* packet too old */ | ||
149 | printf("packet too old\n"); | ||
150 | *success = 0; | ||
151 | return NULL; | ||
152 | } | ||
153 | } | ||
154 | } | ||
155 | |||
156 | q->size--; | ||
157 | q->front++; | ||
158 | |||
159 | if (q->front == q->capacity) | ||
160 | q->front = 0; | ||
161 | |||
162 | *success = 1; | ||
163 | q->current_id = q->queue[front]->_header->_sequence_number; | ||
164 | q->current_ts = q->queue[front]->_header->_timestamp; | ||
165 | return q->queue[front]; | ||
166 | } | ||
167 | |||
168 | int empty_queue(struct jitter_buffer *q) | ||
169 | { | ||
170 | while (q->size > 0) { | ||
171 | q->size--; | ||
172 | /* FIXME: */ | ||
173 | /* rtp_free_msg(cs->_rtp_video, q->queue[q->front]); */ | ||
174 | q->front++; | ||
175 | |||
176 | if (q->front == q->capacity) | ||
177 | q->front = 0; | ||
178 | } | ||
179 | |||
180 | q->id_set = 0; | ||
181 | q->queue_ready = 0; | ||
182 | return 0; | ||
183 | } | ||
184 | |||
185 | int queue(struct jitter_buffer *q, rtp_msg_t *pk) | ||
186 | { | ||
187 | if (q->size == q->capacity) { | ||
188 | printf("buffer full, emptying buffer...\n"); | ||
189 | empty_queue(q); | ||
190 | return 0; | ||
191 | } | ||
192 | |||
193 | if (q->size > 8) | ||
194 | q->queue_ready = 1; | ||
195 | |||
196 | ++q->size; | ||
197 | ++q->rear; | ||
198 | |||
199 | if (q->rear == q->capacity) | ||
200 | q->rear = 0; | ||
201 | |||
202 | q->queue[q->rear] = pk; | ||
203 | |||
204 | int a; | ||
205 | int b; | ||
206 | int j; | ||
207 | a = q->rear; | ||
208 | |||
209 | for (j = 0; j < q->size - 1; ++j) { | ||
210 | b = a - 1; | ||
211 | |||
212 | if (b < 0) | ||
213 | b += q->capacity; | ||
214 | |||
215 | if (sequence_number_older(q->queue[b]->_header->_sequence_number, q->queue[a]->_header->_sequence_number, | ||
216 | q->queue[b]->_header->_timestamp, q->queue[a]->_header->_timestamp)) { | ||
217 | rtp_msg_t *temp; | ||
218 | temp = q->queue[a]; | ||
219 | q->queue[a] = q->queue[b]; | ||
220 | q->queue[b] = temp; | ||
221 | printf("had to swap\n"); | ||
222 | } else { | ||
223 | break; | ||
224 | } | ||
225 | |||
226 | a -= 1; | ||
227 | |||
228 | if (a < 0) | ||
229 | a += q->capacity; | ||
230 | } | ||
231 | |||
232 | if (pk) | ||
233 | return 1; | ||
234 | |||
235 | return 0; | ||
236 | } | ||
237 | |||
238 | int init_receive_audio(codec_state *cs) | ||
239 | { | ||
240 | int err = OPUS_OK; | ||
241 | cs->audio_decoder = opus_decoder_create(48000, 1, &err); | ||
242 | opus_decoder_init(cs->audio_decoder, 48000, 1); | ||
243 | printf("init audio decoder successful\n"); | ||
244 | return 1; | ||
245 | } | ||
246 | |||
247 | int init_receive_video(codec_state *cs) | ||
248 | { | ||
249 | cs->video_decoder = avcodec_find_decoder(VIDEO_CODEC); | ||
250 | |||
251 | if (!cs->video_decoder) { | ||
252 | printf("init video_decoder failed\n"); | ||
253 | return 0; | ||
254 | } | ||
255 | |||
256 | cs->video_decoder_ctx = avcodec_alloc_context3(cs->video_decoder); | ||
257 | |||
258 | if (!cs->video_decoder_ctx) { | ||
259 | printf("init video_decoder_ctx failed\n"); | ||
260 | return 0; | ||
261 | } | ||
262 | |||
263 | if (avcodec_open2(cs->video_decoder_ctx, cs->video_decoder, NULL) < 0) { | ||
264 | printf("opening video decoder failed\n"); | ||
265 | return 0; | ||
266 | } | ||
267 | |||
268 | printf("init video decoder successful\n"); | ||
269 | return 1; | ||
270 | } | ||
271 | |||
272 | int init_send_video(codec_state *cs) | ||
273 | { | ||
274 | cs->video_input_format = av_find_input_format(VIDEO_DRIVER); | ||
275 | |||
276 | if (avformat_open_input(&cs->video_format_ctx, DEFAULT_WEBCAM, cs->video_input_format, NULL) != 0) { | ||
277 | printf("opening video_input_format failed\n"); | ||
278 | return 0; | ||
279 | } | ||
280 | |||
281 | avformat_find_stream_info(cs->video_format_ctx, NULL); | ||
282 | av_dump_format(cs->video_format_ctx, 0, DEFAULT_WEBCAM, 0); | ||
283 | |||
284 | int i; | ||
285 | |||
286 | for (i = 0; i < cs->video_format_ctx->nb_streams; ++i) { | ||
287 | if (cs->video_format_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) { | ||
288 | cs->video_stream = i; | ||
289 | break; | ||
290 | } | ||
291 | } | ||
292 | |||
293 | cs->webcam_decoder_ctx = cs->video_format_ctx->streams[cs->video_stream]->codec; | ||
294 | cs->webcam_decoder = avcodec_find_decoder(cs->webcam_decoder_ctx->codec_id); | ||
295 | |||
296 | if (cs->webcam_decoder == NULL) { | ||
297 | printf("Unsupported codec\n"); | ||
298 | return 0; | ||
299 | } | ||
300 | |||
301 | if (cs->webcam_decoder_ctx == NULL) { | ||
302 | printf("init webcam_decoder_ctx failed\n"); | ||
303 | return 0; | ||
304 | } | ||
305 | |||
306 | if (avcodec_open2(cs->webcam_decoder_ctx, cs->webcam_decoder, NULL) < 0) { | ||
307 | printf("opening webcam decoder failed\n"); | ||
308 | return 0; | ||
309 | } | ||
310 | |||
311 | cs->video_encoder = avcodec_find_encoder(VIDEO_CODEC); | ||
312 | |||
313 | if (!cs->video_encoder) { | ||
314 | printf("init video_encoder failed\n"); | ||
315 | return 0; | ||
316 | } | ||
317 | |||
318 | cs->video_encoder_ctx = avcodec_alloc_context3(cs->video_encoder); | ||
319 | |||
320 | if (!cs->video_encoder_ctx) { | ||
321 | printf("init video_encoder_ctx failed\n"); | ||
322 | return 0; | ||
323 | } | ||
324 | |||
325 | cs->video_encoder_ctx->bit_rate = VIDEO_BITRATE; | ||
326 | cs->video_encoder_ctx->rc_min_rate = cs->video_encoder_ctx->rc_max_rate = cs->video_encoder_ctx->bit_rate; | ||
327 | av_opt_set_double(cs->video_encoder_ctx->priv_data, "max-intra-rate", 90, 0); | ||
328 | av_opt_set(cs->video_encoder_ctx->priv_data, "quality", "realtime", 0); | ||
329 | |||
330 | cs->video_encoder_ctx->thread_count = 4; | ||
331 | cs->video_encoder_ctx->rc_buffer_aggressivity = 0.95; | ||
332 | cs->video_encoder_ctx->rc_buffer_size = VIDEO_BITRATE * 6; | ||
333 | cs->video_encoder_ctx->profile = 3; | ||
334 | cs->video_encoder_ctx->qmax = 54; | ||
335 | cs->video_encoder_ctx->qmin = 4; | ||
336 | AVRational myrational = {1, 25}; | ||
337 | cs->video_encoder_ctx->time_base = myrational; | ||
338 | cs->video_encoder_ctx->gop_size = 99999; | ||
339 | cs->video_encoder_ctx->pix_fmt = PIX_FMT_YUV420P; | ||
340 | cs->video_encoder_ctx->width = cs->webcam_decoder_ctx->width; | ||
341 | cs->video_encoder_ctx->height = cs->webcam_decoder_ctx->height; | ||
342 | |||
343 | if (avcodec_open2(cs->video_encoder_ctx, cs->video_encoder, NULL) < 0) { | ||
344 | printf("opening video encoder failed\n"); | ||
345 | return 0; | ||
346 | } | ||
347 | |||
348 | printf("init video encoder successful\n"); | ||
349 | return 1; | ||
350 | } | ||
351 | |||
352 | int init_send_audio(codec_state *cs) | ||
353 | { | ||
354 | cs->support_send_audio = 0; | ||
355 | |||
356 | const ALchar *pDeviceList = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER); | ||
357 | int i = 0; | ||
358 | const ALchar *device_names[20]; | ||
359 | |||
360 | if (pDeviceList) { | ||
361 | printf("\nAvailable Capture Devices are:\n"); | ||
362 | |||
363 | while (*pDeviceList) { | ||
364 | device_names[i] = pDeviceList; | ||
365 | printf("%d) %s\n", i, device_names[i]); | ||
366 | pDeviceList += strlen(pDeviceList) + 1; | ||
367 | ++i; | ||
368 | } | ||
369 | } | ||
370 | |||
371 | printf("enter capture device number: \n"); | ||
372 | char dev[2]; | ||
373 | fgets(dev, sizeof(dev), stdin); | ||
374 | cs->audio_capture_device = alcCaptureOpenDevice(device_names[dev[0] - 48], AUDIO_SAMPLE_RATE, AL_FORMAT_MONO16, | ||
375 | AUDIO_FRAME_SIZE * 4); | ||
376 | |||
377 | if (alcGetError(cs->audio_capture_device) != AL_NO_ERROR) { | ||
378 | printf("could not start capture device! %d\n", alcGetError(cs->audio_capture_device)); | ||
379 | return 0; | ||
380 | } | ||
381 | |||
382 | int err = OPUS_OK; | ||
383 | cs->audio_bitrate = AUDIO_BITRATE; | ||
384 | cs->audio_encoder = opus_encoder_create(AUDIO_SAMPLE_RATE, 1, OPUS_APPLICATION_VOIP, &err); | ||
385 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_bitrate)); | ||
386 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); | ||
387 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); | ||
388 | |||
389 | opus_encoder_init(cs->audio_encoder, AUDIO_SAMPLE_RATE, 1, OPUS_APPLICATION_VOIP); | ||
390 | |||
391 | int nfo; | ||
392 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_LOOKAHEAD(&nfo)); | ||
393 | /* printf("Encoder lookahead delay : %d\n", nfo); */ | ||
394 | printf("init audio encoder successful\n"); | ||
395 | |||
396 | return 1; | ||
397 | } | ||
398 | |||
399 | int init_encoder(codec_state *cs) | ||
400 | { | ||
401 | avdevice_register_all(); | ||
402 | avcodec_register_all(); | ||
403 | avdevice_register_all(); | ||
404 | av_register_all(); | ||
405 | |||
406 | pthread_mutex_init(&cs->rtp_msg_mutex_lock, NULL); | ||
407 | pthread_mutex_init(&cs->avcodec_mutex_lock, NULL); | ||
408 | |||
409 | cs->support_send_video = init_send_video(cs); | ||
410 | cs->support_send_audio = init_send_audio(cs); | ||
411 | |||
412 | cs->send_audio = 1; | ||
413 | cs->send_video = 1; | ||
414 | |||
415 | return 1; | ||
416 | } | ||
417 | |||
418 | int init_decoder(codec_state *cs) | ||
419 | { | ||
420 | avdevice_register_all(); | ||
421 | avcodec_register_all(); | ||
422 | avdevice_register_all(); | ||
423 | av_register_all(); | ||
424 | |||
425 | cs->receive_video = 0; | ||
426 | cs->receive_audio = 0; | ||
427 | |||
428 | cs->support_receive_video = init_receive_video(cs); | ||
429 | cs->support_receive_audio = init_receive_audio(cs); | ||
430 | |||
431 | cs->receive_audio = 1; | ||
432 | cs->receive_video = 1; | ||
433 | |||
434 | return 1; | ||
435 | } | ||
436 | |||
437 | int video_encoder_refresh(codec_state *cs, int bps) | ||
438 | { | ||
439 | if (cs->video_encoder_ctx) | ||
440 | avcodec_close(cs->video_encoder_ctx); | ||
441 | |||
442 | cs->video_encoder = avcodec_find_encoder(VIDEO_CODEC); | ||
443 | |||
444 | if (!cs->video_encoder) { | ||
445 | printf("init video_encoder failed\n"); | ||
446 | return -1; | ||
447 | } | ||
448 | |||
449 | cs->video_encoder_ctx = avcodec_alloc_context3(cs->video_encoder); | ||
450 | |||
451 | if (!cs->video_encoder_ctx) { | ||
452 | printf("init video_encoder_ctx failed\n"); | ||
453 | return -1; | ||
454 | } | ||
455 | |||
456 | cs->video_encoder_ctx->bit_rate = bps; | ||
457 | cs->video_encoder_ctx->rc_min_rate = cs->video_encoder_ctx->rc_max_rate = cs->video_encoder_ctx->bit_rate; | ||
458 | av_opt_set_double(cs->video_encoder_ctx->priv_data, "max-intra-rate", 90, 0); | ||
459 | av_opt_set(cs->video_encoder_ctx->priv_data, "quality", "realtime", 0); | ||
460 | |||
461 | cs->video_encoder_ctx->thread_count = 4; | ||
462 | cs->video_encoder_ctx->rc_buffer_aggressivity = 0.95; | ||
463 | cs->video_encoder_ctx->rc_buffer_size = bps * 6; | ||
464 | cs->video_encoder_ctx->profile = 0; | ||
465 | cs->video_encoder_ctx->qmax = 54; | ||
466 | cs->video_encoder_ctx->qmin = 4; | ||
467 | AVRational myrational = {1, 25}; | ||
468 | cs->video_encoder_ctx->time_base = myrational; | ||
469 | cs->video_encoder_ctx->gop_size = 99999; | ||
470 | cs->video_encoder_ctx->pix_fmt = PIX_FMT_YUV420P; | ||
471 | cs->video_encoder_ctx->width = cs->webcam_decoder_ctx->width; | ||
472 | cs->video_encoder_ctx->height = cs->webcam_decoder_ctx->height; | ||
473 | |||
474 | if (avcodec_open2(cs->video_encoder_ctx, cs->video_encoder, NULL) < 0) { | ||
475 | printf("opening video encoder failed\n"); | ||
476 | return -1; | ||
477 | } | ||
478 | return 0; | ||
479 | } | ||
480 | |||
481 | void *encode_video_thread(void *arg) | ||
482 | { | ||
483 | codec_state *cs = (codec_state *)arg; | ||
484 | AVPacket pkt1, *packet = &pkt1; | ||
485 | int p = 0; | ||
486 | int err; | ||
487 | int got_packet; | ||
488 | rtp_msg_t *s_video_msg; | ||
489 | int video_frame_finished; | ||
490 | AVFrame *s_video_frame; | ||
491 | AVFrame *webcam_frame; | ||
492 | s_video_frame = avcodec_alloc_frame(); | ||
493 | webcam_frame = avcodec_alloc_frame(); | ||
494 | AVPacket enc_video_packet; | ||
495 | |||
496 | uint8_t *buffer; | ||
497 | int numBytes; | ||
498 | /* Determine required buffer size and allocate buffer */ | ||
499 | numBytes = avpicture_get_size(PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height); | ||
500 | buffer = (uint8_t *)av_calloc(numBytes * sizeof(uint8_t),1); | ||
501 | avpicture_fill((AVPicture *)s_video_frame, buffer, PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width, | ||
502 | cs->webcam_decoder_ctx->height); | ||
503 | cs->sws_ctx = sws_getContext(cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height, | ||
504 | cs->webcam_decoder_ctx->pix_fmt, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height, PIX_FMT_YUV420P, | ||
505 | SWS_BILINEAR, NULL, NULL, NULL); | ||
506 | |||
507 | while (!cs->quit && cs->send_video) { | ||
508 | |||
509 | if (av_read_frame(cs->video_format_ctx, packet) < 0) { | ||
510 | printf("error reading frame\n"); | ||
511 | |||
512 | if (cs->video_format_ctx->pb->error != 0) | ||
513 | break; | ||
514 | |||
515 | continue; | ||
516 | } | ||
517 | |||
518 | if (packet->stream_index == cs->video_stream) { | ||
519 | if (avcodec_decode_video2(cs->webcam_decoder_ctx, webcam_frame, &video_frame_finished, packet) < 0) { | ||
520 | printf("couldn't decode\n"); | ||
521 | continue; | ||
522 | } | ||
523 | |||
524 | av_free_packet(packet); | ||
525 | sws_scale(cs->sws_ctx, (uint8_t const * const *)webcam_frame->data, webcam_frame->linesize, 0, | ||
526 | cs->webcam_decoder_ctx->height, s_video_frame->data, s_video_frame->linesize); | ||
527 | /* create a new I-frame every 60 frames */ | ||
528 | ++p; | ||
529 | |||
530 | if (p == 60) { | ||
531 | |||
532 | s_video_frame->pict_type = AV_PICTURE_TYPE_BI ; | ||
533 | } else if (p == 61) { | ||
534 | s_video_frame->pict_type = AV_PICTURE_TYPE_I ; | ||
535 | p = 0; | ||
536 | } else { | ||
537 | s_video_frame->pict_type = AV_PICTURE_TYPE_P ; | ||
538 | } | ||
539 | |||
540 | if (video_frame_finished) { | ||
541 | err = avcodec_encode_video2(cs->video_encoder_ctx, &enc_video_packet, s_video_frame, &got_packet); | ||
542 | |||
543 | if (err < 0) { | ||
544 | printf("could not encode video frame\n"); | ||
545 | continue; | ||
546 | } | ||
547 | |||
548 | if (!got_packet) { | ||
549 | continue; | ||
550 | } | ||
551 | |||
552 | pthread_mutex_lock(&cs->rtp_msg_mutex_lock); | ||
553 | THREADLOCK() | ||
554 | |||
555 | if (!enc_video_packet.data) fprintf(stderr, "video packet data is NULL\n"); | ||
556 | |||
557 | s_video_msg = rtp_msg_new ( cs->_rtp_video, enc_video_packet.data, enc_video_packet.size ) ; | ||
558 | |||
559 | if (!s_video_msg) { | ||
560 | printf("invalid message\n"); | ||
561 | } | ||
562 | |||
563 | rtp_send_msg ( cs->_rtp_video, s_video_msg, cs->_networking ); | ||
564 | THREADUNLOCK() | ||
565 | pthread_mutex_unlock(&cs->rtp_msg_mutex_lock); | ||
566 | av_free_packet(&enc_video_packet); | ||
567 | } | ||
568 | } else { | ||
569 | av_free_packet(packet); | ||
570 | } | ||
571 | } | ||
572 | |||
573 | /* clean up codecs */ | ||
574 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
575 | av_free(buffer); | ||
576 | av_free(webcam_frame); | ||
577 | av_free(s_video_frame); | ||
578 | sws_freeContext(cs->sws_ctx); | ||
579 | avcodec_close(cs->webcam_decoder_ctx); | ||
580 | avcodec_close(cs->video_encoder_ctx); | ||
581 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
582 | pthread_exit ( NULL ); | ||
583 | } | ||
584 | |||
585 | void *encode_audio_thread(void *arg) | ||
586 | { | ||
587 | codec_state *cs = (codec_state *)arg; | ||
588 | rtp_msg_t *s_audio_msg; | ||
589 | unsigned char encoded_data[4096]; | ||
590 | int encoded_size = 0; | ||
591 | int16_t frame[4096]; | ||
592 | int frame_size = AUDIO_FRAME_SIZE; | ||
593 | ALint sample = 0; | ||
594 | alcCaptureStart(cs->audio_capture_device); | ||
595 | |||
596 | while (!cs->quit && cs->send_audio) { | ||
597 | alcGetIntegerv(cs->audio_capture_device, ALC_CAPTURE_SAMPLES, (ALCsizei)sizeof(ALint), &sample); | ||
598 | |||
599 | if (sample >= frame_size) { | ||
600 | alcCaptureSamples(cs->audio_capture_device, frame, frame_size); | ||
601 | encoded_size = opus_encode(cs->audio_encoder, frame, frame_size, encoded_data, 480); | ||
602 | |||
603 | if (encoded_size <= 0) { | ||
604 | printf("Could not encode audio packet\n"); | ||
605 | } else { | ||
606 | pthread_mutex_lock(&cs->rtp_msg_mutex_lock); | ||
607 | THREADLOCK() | ||
608 | rtp_set_payload_type(cs->_rtp_audio, 96); | ||
609 | s_audio_msg = rtp_msg_new (cs->_rtp_audio, encoded_data, encoded_size) ; | ||
610 | rtp_send_msg ( cs->_rtp_audio, s_audio_msg, cs->_networking ); | ||
611 | pthread_mutex_unlock(&cs->rtp_msg_mutex_lock); | ||
612 | THREADUNLOCK() | ||
613 | } | ||
614 | } else { | ||
615 | usleep(1000); | ||
616 | } | ||
617 | } | ||
618 | |||
619 | /* clean up codecs */ | ||
620 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
621 | alcCaptureStop(cs->audio_capture_device); | ||
622 | alcCaptureCloseDevice(cs->audio_capture_device); | ||
623 | |||
624 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
625 | pthread_exit ( NULL ); | ||
626 | } | ||
627 | |||
628 | |||
629 | int video_decoder_refresh(codec_state *cs, int width, int height) | ||
630 | { | ||
631 | printf("need to refresh\n"); | ||
632 | screen = SDL_SetVideoMode(width, height, 0, 0); | ||
633 | |||
634 | if (cs->video_picture.bmp) | ||
635 | SDL_FreeYUVOverlay(cs->video_picture.bmp); | ||
636 | |||
637 | cs->video_picture.bmp = SDL_CreateYUVOverlay(width, height, SDL_YV12_OVERLAY, screen); | ||
638 | cs->sws_SDL_r_ctx = sws_getContext(width, height, cs->video_decoder_ctx->pix_fmt, width, height, PIX_FMT_YUV420P, | ||
639 | SWS_BILINEAR, NULL, NULL, NULL); | ||
640 | return 1; | ||
641 | } | ||
642 | |||
643 | void *decode_video_thread(void *arg) | ||
644 | { | ||
645 | codec_state *cs = (codec_state *)arg; | ||
646 | cs->video_stream = 0; | ||
647 | rtp_msg_t *r_msg; | ||
648 | int dec_frame_finished; | ||
649 | AVFrame *r_video_frame; | ||
650 | r_video_frame = avcodec_alloc_frame(); | ||
651 | AVPacket dec_video_packet; | ||
652 | av_new_packet (&dec_video_packet, 65536); | ||
653 | int width = 0; | ||
654 | int height = 0; | ||
655 | |||
656 | while (!cs->quit && cs->receive_video) { | ||
657 | r_msg = rtp_recv_msg ( cs->_rtp_video ); | ||
658 | |||
659 | if (r_msg) { | ||
660 | memcpy(dec_video_packet.data, r_msg->_data, r_msg->_length); | ||
661 | dec_video_packet.size = r_msg->_length; | ||
662 | avcodec_decode_video2(cs->video_decoder_ctx, r_video_frame, &dec_frame_finished, &dec_video_packet); | ||
663 | |||
664 | if (dec_frame_finished) { | ||
665 | if (cs->video_decoder_ctx->width != width || cs->video_decoder_ctx->height != height) { | ||
666 | width = cs->video_decoder_ctx->width; | ||
667 | height = cs->video_decoder_ctx->height; | ||
668 | printf("w: %d h%d \n", width, height); | ||
669 | video_decoder_refresh(cs, width, height); | ||
670 | } | ||
671 | |||
672 | display_received_frame(cs, r_video_frame); | ||
673 | } else { | ||
674 | /* TODO: request the sender to create a new i-frame immediatly */ | ||
675 | printf("bad video packet\n"); | ||
676 | } | ||
677 | |||
678 | rtp_free_msg(cs->_rtp_video, r_msg); | ||
679 | } | ||
680 | |||
681 | usleep(1000); | ||
682 | } | ||
683 | |||
684 | printf("vend\n"); | ||
685 | /* clean up codecs */ | ||
686 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
687 | av_free(r_video_frame); | ||
688 | avcodec_close(cs->video_decoder_ctx); | ||
689 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
690 | pthread_exit ( NULL ); | ||
691 | } | ||
692 | |||
693 | void *decode_audio_thread(void *arg) | ||
694 | { | ||
695 | codec_state *cs = (codec_state *)arg; | ||
696 | rtp_msg_t *r_msg; | ||
697 | |||
698 | int frame_size = AUDIO_FRAME_SIZE; | ||
699 | int data_size; | ||
700 | |||
701 | ALCdevice *dev; | ||
702 | ALCcontext *ctx; | ||
703 | ALuint source, *buffers; | ||
704 | dev = alcOpenDevice(NULL); | ||
705 | ctx = alcCreateContext(dev, NULL); | ||
706 | alcMakeContextCurrent(ctx); | ||
707 | int openal_buffers = 5; | ||
708 | |||
709 | buffers = calloc(sizeof(ALuint) * openal_buffers,1); | ||
710 | alGenBuffers(openal_buffers, buffers); | ||
711 | alGenSources((ALuint)1, &source); | ||
712 | alSourcei(source, AL_LOOPING, AL_FALSE); | ||
713 | |||
714 | ALuint buffer; | ||
715 | ALint val; | ||
716 | |||
717 | ALenum error; | ||
718 | uint16_t zeros[frame_size]; | ||
719 | int i; | ||
720 | |||
721 | for (i = 0; i < frame_size; i++) { | ||
722 | zeros[i] = 0; | ||
723 | } | ||
724 | |||
725 | for (i = 0; i < openal_buffers; ++i) { | ||
726 | alBufferData(buffers[i], AL_FORMAT_MONO16, zeros, frame_size, 48000); | ||
727 | } | ||
728 | |||
729 | alSourceQueueBuffers(source, openal_buffers, buffers); | ||
730 | alSourcePlay(source); | ||
731 | |||
732 | if (alGetError() != AL_NO_ERROR) { | ||
733 | fprintf(stderr, "Error starting audio\n"); | ||
734 | cs->quit = 1; | ||
735 | } | ||
736 | |||
737 | struct jitter_buffer *j_buf = NULL; | ||
738 | |||
739 | j_buf = create_queue(20); | ||
740 | |||
741 | int success = 0; | ||
742 | |||
743 | int dec_frame_len; | ||
744 | |||
745 | opus_int16 PCM[frame_size]; | ||
746 | |||
747 | while (!cs->quit && cs->receive_audio) { | ||
748 | THREADLOCK() | ||
749 | r_msg = rtp_recv_msg ( cs->_rtp_audio ); | ||
750 | |||
751 | if (r_msg) { | ||
752 | /* push the packet into the queue */ | ||
753 | queue(j_buf, r_msg); | ||
754 | } | ||
755 | |||
756 | /* grab a packet from the queue */ | ||
757 | success = 0; | ||
758 | alGetSourcei(source, AL_BUFFERS_PROCESSED, &val); | ||
759 | |||
760 | if (val > 0) | ||
761 | r_msg = dequeue(j_buf, &success); | ||
762 | |||
763 | if (success > 0) { | ||
764 | /* good packet */ | ||
765 | if (success == 1) { | ||
766 | dec_frame_len = opus_decode(cs->audio_decoder, r_msg->_data, r_msg->_length, PCM, frame_size, 0); | ||
767 | rtp_free_msg(cs->_rtp_audio, r_msg); | ||
768 | } | ||
769 | |||
770 | /* lost packet */ | ||
771 | if (success == 2) { | ||
772 | printf("lost packet\n"); | ||
773 | dec_frame_len = opus_decode(cs->audio_decoder, NULL, 0, PCM, frame_size, 1); | ||
774 | } | ||
775 | |||
776 | if (dec_frame_len > 0) { | ||
777 | alGetSourcei(source, AL_BUFFERS_PROCESSED, &val); | ||
778 | |||
779 | if (val <= 0) | ||
780 | continue; | ||
781 | |||
782 | alSourceUnqueueBuffers(source, 1, &buffer); | ||
783 | data_size = av_samples_get_buffer_size(NULL, 1, dec_frame_len, AV_SAMPLE_FMT_S16, 1); | ||
784 | alBufferData(buffer, AL_FORMAT_MONO16, PCM, data_size, 48000); | ||
785 | int error = alGetError(); | ||
786 | |||
787 | if (error != AL_NO_ERROR) { | ||
788 | fprintf(stderr, "Error setting buffer %d\n", error); | ||
789 | break; | ||
790 | } | ||
791 | |||
792 | alSourceQueueBuffers(source, 1, &buffer); | ||
793 | |||
794 | if (alGetError() != AL_NO_ERROR) { | ||
795 | fprintf(stderr, "error: could not buffer audio\n"); | ||
796 | break; | ||
797 | } | ||
798 | |||
799 | alGetSourcei(source, AL_SOURCE_STATE, &val); | ||
800 | |||
801 | if (val != AL_PLAYING) | ||
802 | alSourcePlay(source); | ||
803 | |||
804 | |||
805 | } | ||
806 | } | ||
807 | |||
808 | THREADUNLOCK() | ||
809 | usleep(1000); | ||
810 | } | ||
811 | |||
812 | /* clean up codecs */ | ||
813 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
814 | |||
815 | /* clean up openal */ | ||
816 | alDeleteSources(1, &source); | ||
817 | alDeleteBuffers(openal_buffers, buffers); | ||
818 | alcMakeContextCurrent(NULL); | ||
819 | alcDestroyContext(ctx); | ||
820 | alcCloseDevice(dev); | ||
821 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
822 | pthread_exit ( NULL ); | ||
823 | } \ No newline at end of file | ||
diff --git a/toxmsi/AV_codec.h b/toxmsi/AV_codec.h new file mode 100644 index 00000000..97a2abbb --- /dev/null +++ b/toxmsi/AV_codec.h | |||
@@ -0,0 +1,168 @@ | |||
1 | /* AV_codec.h | ||
2 | * | ||
3 | * Audio and video codec intitialisation, encoding/decoding and playback | ||
4 | * | ||
5 | * Copyright (C) 2013 Tox project All Rights Reserved. | ||
6 | * | ||
7 | * This file is part of Tox. | ||
8 | * | ||
9 | * Tox is free software: you can redistribute it and/or modify | ||
10 | * it under the terms of the GNU General Public License as published by | ||
11 | * the Free Software Foundation, either version 3 of the License, or | ||
12 | * (at your option) any later version. | ||
13 | * | ||
14 | * Tox is distributed in the hope that it will be useful, | ||
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
17 | * GNU General Public License for more details. | ||
18 | * | ||
19 | * You should have received a copy of the GNU General Public License | ||
20 | * along with Tox. If not, see <http://www.gnu.org/licenses/>. | ||
21 | * | ||
22 | */ | ||
23 | |||
24 | /*----------------------------------------------------------------------------------*/ | ||
25 | #ifndef _AVCODEC_H_ | ||
26 | #define _AVCODEC_H_ | ||
27 | |||
28 | #include <stdio.h> | ||
29 | #include <math.h> | ||
30 | #include <libavcodec/avcodec.h> | ||
31 | #include <libavformat/avformat.h> | ||
32 | #include <libswscale/swscale.h> | ||
33 | #include <libavdevice/avdevice.h> | ||
34 | #include <libavutil/opt.h> | ||
35 | #include <pthread.h> | ||
36 | #include <AL/al.h> | ||
37 | #include <AL/alc.h> | ||
38 | #include "toxrtp.h" | ||
39 | #include "tox.h" | ||
40 | |||
41 | #include <SDL.h> | ||
42 | #include <opus/opus.h> | ||
43 | |||
44 | /* ffmpeg VP8 codec ID */ | ||
45 | #define VIDEO_CODEC AV_CODEC_ID_VP8 | ||
46 | |||
47 | /* ffmpeg Opus codec ID */ | ||
48 | #define AUDIO_CODEC AV_CODEC_ID_OPUS | ||
49 | |||
50 | /* default video bitrate in bytes/s */ | ||
51 | #define VIDEO_BITRATE 10*1000 | ||
52 | |||
53 | /* default audio bitrate in bytes/s */ | ||
54 | #define AUDIO_BITRATE 64000 | ||
55 | |||
56 | /* audio frame duration in miliseconds */ | ||
57 | #define AUDIO_FRAME_DURATION 20 | ||
58 | |||
59 | /* audio sample rate recommended to be 48kHz for Opus */ | ||
60 | #define AUDIO_SAMPLE_RATE 48000 | ||
61 | |||
62 | /* the amount of samples in one audio frame */ | ||
63 | #define AUDIO_FRAME_SIZE AUDIO_SAMPLE_RATE*AUDIO_FRAME_DURATION/1000 | ||
64 | |||
65 | /* the quit event for SDL */ | ||
66 | #define FF_QUIT_EVENT (SDL_USEREVENT + 2) | ||
67 | |||
68 | #ifdef __linux__ | ||
69 | #define VIDEO_DRIVER "video4linux2" | ||
70 | #define DEFAULT_WEBCAM "/dev/video0" | ||
71 | #endif | ||
72 | |||
73 | #ifdef WIN32 | ||
74 | #define VIDEO_DRIVER "vfwcap" | ||
75 | #define DEFAULT_WEBCAM "0" | ||
76 | #endif | ||
77 | |||
78 | SDL_Surface *screen; | ||
79 | |||
80 | typedef struct { | ||
81 | SDL_Overlay *bmp; | ||
82 | int width, height; | ||
83 | } VideoPicture; | ||
84 | |||
85 | |||
86 | typedef struct { | ||
87 | uint8_t send_audio; | ||
88 | uint8_t receive_audio; | ||
89 | uint8_t send_video; | ||
90 | uint8_t receive_video; | ||
91 | |||
92 | uint8_t support_send_audio; | ||
93 | uint8_t support_send_video; | ||
94 | uint8_t support_receive_audio; | ||
95 | uint8_t support_receive_video; | ||
96 | |||
97 | /* video encoding */ | ||
98 | AVInputFormat *video_input_format; | ||
99 | AVFormatContext *video_format_ctx; | ||
100 | uint8_t video_stream; | ||
101 | AVCodecContext *webcam_decoder_ctx; | ||
102 | AVCodec *webcam_decoder; | ||
103 | AVCodecContext *video_encoder_ctx; | ||
104 | AVCodec *video_encoder; | ||
105 | |||
106 | /* video decoding */ | ||
107 | AVCodecContext *video_decoder_ctx; | ||
108 | AVCodec *video_decoder; | ||
109 | |||
110 | /* audio encoding */ | ||
111 | ALCdevice *audio_capture_device; | ||
112 | OpusEncoder *audio_encoder; | ||
113 | int audio_bitrate; | ||
114 | |||
115 | /* audio decoding */ | ||
116 | OpusDecoder *audio_decoder; | ||
117 | |||
118 | uint8_t req_video_refresh; | ||
119 | |||
120 | /* context for converting image format to something SDL can use*/ | ||
121 | struct SwsContext *sws_SDL_r_ctx; | ||
122 | |||
123 | /* context for converting webcam image format to something the video encoder can use */ | ||
124 | struct SwsContext *sws_ctx; | ||
125 | |||
126 | /* rendered video picture, ready for display */ | ||
127 | VideoPicture video_picture; | ||
128 | |||
129 | rtp_session_t *_rtp_video; | ||
130 | rtp_session_t *_rtp_audio; | ||
131 | int socket; | ||
132 | Networking_Core *_networking; | ||
133 | |||
134 | pthread_t encode_audio_thread; | ||
135 | pthread_t encode_video_thread; | ||
136 | |||
137 | pthread_t decode_audio_thread; | ||
138 | pthread_t decode_video_thread; | ||
139 | |||
140 | pthread_mutex_t rtp_msg_mutex_lock; | ||
141 | pthread_mutex_t avcodec_mutex_lock; | ||
142 | |||
143 | uint8_t quit; | ||
144 | SDL_Event SDL_event; | ||
145 | |||
146 | msi_session_t *_msi; | ||
147 | uint32_t _frame_rate; | ||
148 | uint16_t _send_port, _recv_port; | ||
149 | int _tox_sock; | ||
150 | //pthread_id _medialoop_id; | ||
151 | |||
152 | } codec_state; | ||
153 | |||
154 | int display_received_frame(codec_state *cs, AVFrame *r_video_frame); | ||
155 | int init_receive_audio(codec_state *cs); | ||
156 | int init_decoder(codec_state *cs); | ||
157 | int init_send_video(codec_state *cs); | ||
158 | int init_send_audio(codec_state *cs); | ||
159 | int init_encoder(codec_state *cs); | ||
160 | int video_encoder_refresh(codec_state *cs, int bps); | ||
161 | void *encode_video_thread(void *arg); | ||
162 | void *encode_audio_thread(void *arg); | ||
163 | int video_decoder_refresh(codec_state *cs, int width, int height); | ||
164 | int handle_rtp_video_packet(codec_state *cs, rtp_msg_t *r_msg); | ||
165 | void *decode_video_thread(void *arg); | ||
166 | void *decode_audio_thread(void *arg); | ||
167 | |||
168 | #endif \ No newline at end of file | ||
diff --git a/toxmsi/phone.c b/toxmsi/phone.c new file mode 100644 index 00000000..010f26aa --- /dev/null +++ b/toxmsi/phone.c | |||
@@ -0,0 +1,645 @@ | |||
1 | #ifdef HAVE_CONFIG_H | ||
2 | #include "config.h" | ||
3 | #endif /* HAVE_CONFIG_H */ | ||
4 | |||
5 | #define _BSD_SOURCE | ||
6 | #define _GNU_SOURCE | ||
7 | |||
8 | #define _CT_PHONE | ||
9 | |||
10 | #ifdef _CT_PHONE | ||
11 | #include "phone.h" | ||
12 | #include <stdarg.h> | ||
13 | #include <unistd.h> | ||
14 | #include <stdio.h> | ||
15 | #include <string.h> | ||
16 | #include <stdlib.h> | ||
17 | #include <termios.h> | ||
18 | #include <pthread.h> | ||
19 | #include "AV_codec.h" | ||
20 | |||
21 | |||
22 | |||
23 | void INFO (const char* _format, ...) | ||
24 | { | ||
25 | printf("\r[!] "); | ||
26 | va_list _arg; | ||
27 | va_start (_arg, _format); | ||
28 | vfprintf (stdout, _format, _arg); | ||
29 | va_end (_arg); | ||
30 | printf("\n\r >> "); | ||
31 | fflush(stdout); | ||
32 | } | ||
33 | |||
34 | int rtp_handlepacket ( void* _object, tox_IP_Port ip_port, uint8_t* data, uint32_t length ) | ||
35 | { | ||
36 | phone_t* _phone = _object; | ||
37 | rtp_msg_t* _msg; | ||
38 | uint8_t _payload_id; | ||
39 | |||
40 | if ( _phone->_msi->_call && _phone->_msi->_call->_state == call_active ){ | ||
41 | |||
42 | _msg = rtp_msg_parse ( NULL, data + 1, length - 1 ); /* ignore marker byte */ | ||
43 | |||
44 | if ( !_msg ) | ||
45 | return 0; | ||
46 | |||
47 | _payload_id = rtp_header_get_setting_payload_type(_msg->_header); | ||
48 | |||
49 | if ( _payload_id == _PAYLOAD_OPUS && _phone->_rtp_audio ) | ||
50 | rtp_store_msg(_phone->_rtp_audio, _msg); | ||
51 | else if ( _payload_id == _PAYLOAD_VP8 && _phone->_rtp_video ) | ||
52 | rtp_store_msg(_phone->_rtp_video, _msg); | ||
53 | else rtp_free_msg( NULL, _msg); | ||
54 | } | ||
55 | |||
56 | return SUCCESS; | ||
57 | } | ||
58 | int msi_handlepacket ( void* _object, tox_IP_Port ip_port, uint8_t* data, uint32_t length ) | ||
59 | { | ||
60 | msi_session_t* _session = _object; | ||
61 | msi_msg_t* _msg; | ||
62 | |||
63 | _msg = msi_parse_msg ( data + 1 ); /* ignore marker byte */ | ||
64 | |||
65 | if ( _msg ) { | ||
66 | /* my current solution for "hole punching" */ | ||
67 | _session->_friend_id = ip_port; | ||
68 | } else { | ||
69 | return FAILURE; | ||
70 | } | ||
71 | |||
72 | /* place message in a session */ | ||
73 | msi_store_msg(_session, _msg); | ||
74 | |||
75 | return SUCCESS; | ||
76 | } | ||
77 | |||
78 | void* phone_receivepacket ( void* _phone_p ) | ||
79 | { | ||
80 | phone_t* _phone = _phone_p; | ||
81 | |||
82 | |||
83 | networking_registerhandler(_phone->_networking, MSI_PACKET, msi_handlepacket, _phone->_msi); | ||
84 | networking_registerhandler(_phone->_networking, RTP_PACKET, rtp_handlepacket, _phone); | ||
85 | |||
86 | /* Now start main networking loop */ | ||
87 | while ( _phone->_networking ) { /* so not thread safe */ | ||
88 | networking_poll(_phone->_networking); | ||
89 | usleep(10000); | ||
90 | } | ||
91 | |||
92 | pthread_exit ( NULL ); | ||
93 | } | ||
94 | |||
95 | /* Media transport callback */ | ||
96 | typedef struct hmtc_args_s { | ||
97 | rtp_session_t** _rtp_audio; | ||
98 | rtp_session_t** _rtp_video; | ||
99 | call_type* _local_type_call; | ||
100 | call_state* _this_call; | ||
101 | void *_core_handler; | ||
102 | } hmtc_args_t; | ||
103 | |||
104 | void* phone_handle_media_transport_poll ( void* _hmtc_args_p ) | ||
105 | { | ||
106 | rtp_msg_t* _audio_msg, * _video_msg; | ||
107 | |||
108 | hmtc_args_t* _hmtc_args = _hmtc_args_p; | ||
109 | |||
110 | rtp_session_t* _rtp_audio = *_hmtc_args->_rtp_audio; | ||
111 | rtp_session_t* _rtp_video = *_hmtc_args->_rtp_video; | ||
112 | |||
113 | call_type* _type = _hmtc_args->_local_type_call; | ||
114 | void* _core_handler = _hmtc_args->_core_handler; | ||
115 | |||
116 | |||
117 | call_state* _this_call = _hmtc_args->_this_call; | ||
118 | |||
119 | while ( *_this_call == call_active ) { | ||
120 | |||
121 | // THREADLOCK() | ||
122 | |||
123 | _audio_msg = rtp_recv_msg ( _rtp_audio ); | ||
124 | _video_msg = rtp_recv_msg ( _rtp_video ); | ||
125 | |||
126 | if ( _audio_msg ) { | ||
127 | /* Do whatever with msg */ | ||
128 | puts("audio"); | ||
129 | /* Do whatever with msg | ||
130 | puts(_audio_msg->_data);*/ | ||
131 | rtp_free_msg ( _rtp_audio, _audio_msg ); | ||
132 | } | ||
133 | |||
134 | if ( _video_msg ) { | ||
135 | /* Do whatever with msg */ | ||
136 | puts("video"); | ||
137 | /* Do whatever with msg | ||
138 | puts(_video_msg->_data); */ | ||
139 | rtp_free_msg ( _rtp_video, _video_msg ); | ||
140 | _video_msg = NULL; | ||
141 | } | ||
142 | /* -------------------- */ | ||
143 | |||
144 | _audio_msg = rtp_msg_new ( _rtp_audio, (const uint8_t*)"audio\0", 6 ) ; | ||
145 | rtp_send_msg ( _rtp_audio, _audio_msg, _core_handler ); | ||
146 | _audio_msg = NULL; | ||
147 | |||
148 | if ( *_type == type_video ){ /* if local call send video */ | ||
149 | _video_msg = rtp_msg_new ( _rtp_video, (const uint8_t*)"video\0", 6 ) ; | ||
150 | rtp_send_msg ( _rtp_video, _video_msg, _core_handler ); | ||
151 | _video_msg = NULL; | ||
152 | } | ||
153 | |||
154 | //THREADUNLOCK() | ||
155 | |||
156 | usleep ( 10000 ); | ||
157 | /* -------------------- */ | ||
158 | } | ||
159 | |||
160 | //THREADLOCK() | ||
161 | |||
162 | if ( _audio_msg ){ | ||
163 | rtp_free_msg(_rtp_audio, _audio_msg); | ||
164 | } | ||
165 | |||
166 | if ( _video_msg ) { | ||
167 | rtp_free_msg(_rtp_video, _video_msg); | ||
168 | } | ||
169 | |||
170 | rtp_release_session_recv(_rtp_video); | ||
171 | rtp_release_session_recv(_rtp_audio); | ||
172 | |||
173 | rtp_terminate_session(_rtp_audio); | ||
174 | rtp_terminate_session(_rtp_video); | ||
175 | |||
176 | *_hmtc_args->_rtp_audio = NULL; | ||
177 | *_hmtc_args->_rtp_video = NULL; | ||
178 | |||
179 | free(_hmtc_args_p); | ||
180 | |||
181 | //THREADUNLOCK() | ||
182 | |||
183 | INFO("Media thread finished!"); | ||
184 | |||
185 | pthread_exit ( NULL ); | ||
186 | } | ||
187 | |||
188 | pthread_t phone_startmedia_loop ( phone_t* _phone ) | ||
189 | { | ||
190 | if ( !_phone ){ | ||
191 | return 0; | ||
192 | } | ||
193 | |||
194 | int _status; | ||
195 | |||
196 | uint8_t _prefix = RTP_PACKET; | ||
197 | |||
198 | pthread_t _rtp_tid; | ||
199 | int _rtp_thread_running = 1; | ||
200 | |||
201 | _phone->_rtp_audio = rtp_init_session ( -1, 1 ); | ||
202 | rtp_set_prefix ( _phone->_rtp_audio, &_prefix, 1 ); | ||
203 | rtp_add_receiver ( _phone->_rtp_audio, &_phone->_msi->_friend_id ); | ||
204 | rtp_set_payload_type(_phone->_rtp_audio, _PAYLOAD_OPUS); | ||
205 | |||
206 | _phone->_rtp_video = rtp_init_session ( -1, 1 ); | ||
207 | rtp_set_prefix ( _phone->_rtp_video, &_prefix, 1 ); | ||
208 | rtp_add_receiver ( _phone->_rtp_video, &_phone->_msi->_friend_id ); | ||
209 | rtp_set_payload_type(_phone->_rtp_video, _PAYLOAD_VP8); | ||
210 | |||
211 | |||
212 | |||
213 | hmtc_args_t* rtp_targs = calloc(sizeof(hmtc_args_t),1); | ||
214 | |||
215 | |||
216 | rtp_targs->_rtp_audio = &_phone->_rtp_audio; | ||
217 | rtp_targs->_rtp_video = &_phone->_rtp_video; | ||
218 | rtp_targs->_local_type_call = &_phone->_msi->_call->_type_local; | ||
219 | rtp_targs->_this_call = &_phone->_msi->_call->_state; | ||
220 | rtp_targs->_core_handler = _phone->_networking; | ||
221 | |||
222 | codec_state *cs; | ||
223 | cs=_phone->cs; | ||
224 | //_status = pthread_create ( &_rtp_tid, NULL, phone_handle_media_transport_poll, rtp_targs ); | ||
225 | cs->_rtp_audio=_phone->_rtp_audio; | ||
226 | cs->_rtp_video=_phone->_rtp_video; | ||
227 | cs->_networking=_phone->_networking; | ||
228 | cs->socket=_phone->_tox_sock; | ||
229 | cs->quit = 0; | ||
230 | |||
231 | printf("support: %d %d\n",cs->support_send_audio,cs->support_send_video); | ||
232 | |||
233 | if(cs->support_send_audio&&cs->support_send_video) /* quick fix */ | ||
234 | pthread_create(&_phone->cs->encode_audio_thread, NULL, encode_audio_thread, _phone->cs); | ||
235 | if(cs->support_receive_audio) | ||
236 | pthread_create(&_phone->cs->decode_audio_thread, NULL, decode_audio_thread, _phone->cs); | ||
237 | |||
238 | if(cs->support_send_video) | ||
239 | pthread_create(&_phone->cs->encode_video_thread, NULL, encode_video_thread, _phone->cs); | ||
240 | if(cs->support_receive_video) | ||
241 | pthread_create(&_phone->cs->decode_video_thread, NULL, decode_video_thread, _phone->cs); | ||
242 | // | ||
243 | return 1; | ||
244 | |||
245 | |||
246 | |||
247 | |||
248 | } | ||
249 | |||
250 | |||
251 | /* Some example callbacks */ | ||
252 | |||
253 | MCBTYPE callback_recv_invite ( MCBARGS ) | ||
254 | { | ||
255 | const char* _call_type; | ||
256 | |||
257 | msi_session_t* _msi = _arg; | ||
258 | |||
259 | /* Get the last one */ | ||
260 | call_type _type = _msi->_call->_type_peer[_msi->_call->_participants - 1]; | ||
261 | |||
262 | switch ( _type ){ | ||
263 | case type_audio: | ||
264 | _call_type = "audio"; | ||
265 | break; | ||
266 | case type_video: | ||
267 | _call_type = "video"; | ||
268 | break; | ||
269 | } | ||
270 | |||
271 | INFO( "Incoming %s call!", _call_type ); | ||
272 | |||
273 | } | ||
274 | MCBTYPE callback_recv_trying ( MCBARGS ) | ||
275 | { | ||
276 | INFO ( "Trying..."); | ||
277 | } | ||
278 | MCBTYPE callback_recv_ringing ( MCBARGS ) | ||
279 | { | ||
280 | INFO ( "Ringing!" ); | ||
281 | } | ||
282 | MCBTYPE callback_recv_starting ( MCBARGS ) | ||
283 | { | ||
284 | msi_session_t* _session = _arg; | ||
285 | if ( !phone_startmedia_loop(_session->_agent_handler) ){ | ||
286 | INFO("Starting call failed!"); | ||
287 | } else { | ||
288 | INFO ("Call started! ( press h to hangup )"); | ||
289 | } | ||
290 | } | ||
291 | MCBTYPE callback_recv_ending ( MCBARGS ) | ||
292 | { | ||
293 | msi_session_t* _session = _arg; | ||
294 | phone_t * _phone = _session->_agent_handler; | ||
295 | _phone->cs->quit=1; | ||
296 | if(_phone->cs->encode_video_thread) | ||
297 | pthread_join(_phone->cs->encode_video_thread,NULL); | ||
298 | if(_phone->cs->encode_audio_thread) | ||
299 | pthread_join(_phone->cs->encode_audio_thread,NULL); | ||
300 | if(_phone->cs->decode_audio_thread) | ||
301 | pthread_join(_phone->cs->decode_audio_thread,NULL); | ||
302 | if(_phone->cs->decode_video_thread) | ||
303 | pthread_join(_phone->cs->decode_video_thread,NULL); | ||
304 | SDL_Quit(); | ||
305 | printf("all A/V threads successfully shut down\n"); | ||
306 | |||
307 | INFO ( "Call ended!" ); | ||
308 | } | ||
309 | |||
310 | MCBTYPE callback_recv_error ( MCBARGS ) | ||
311 | { | ||
312 | msi_session_t* _session = _arg; | ||
313 | |||
314 | INFO( "Error: %s", _session->_last_error_str ); | ||
315 | } | ||
316 | |||
317 | MCBTYPE callback_call_started ( MCBARGS ) | ||
318 | { | ||
319 | msi_session_t* _session = _arg; | ||
320 | if ( !phone_startmedia_loop(_session->_agent_handler) ){ | ||
321 | INFO("Starting call failed!"); | ||
322 | } else { | ||
323 | INFO ("Call started! ( press h to hangup )"); | ||
324 | } | ||
325 | |||
326 | } | ||
327 | MCBTYPE callback_call_canceled ( MCBARGS ) | ||
328 | { | ||
329 | INFO ( "Call canceled!" ); | ||
330 | } | ||
331 | MCBTYPE callback_call_rejected ( MCBARGS ) | ||
332 | { | ||
333 | INFO ( "Call rejected!\n" ); | ||
334 | } | ||
335 | MCBTYPE callback_call_ended ( MCBARGS ) | ||
336 | { | ||
337 | |||
338 | msi_session_t* _session = _arg; | ||
339 | phone_t * _phone = _session->_agent_handler; | ||
340 | _phone->cs->quit=1; | ||
341 | if(_phone->cs->encode_video_thread) | ||
342 | pthread_join(_phone->cs->encode_video_thread,NULL); | ||
343 | if(_phone->cs->encode_audio_thread) | ||
344 | pthread_join(_phone->cs->encode_audio_thread,NULL); | ||
345 | if(_phone->cs->decode_audio_thread) | ||
346 | pthread_join(_phone->cs->decode_audio_thread,NULL); | ||
347 | if(_phone->cs->decode_video_thread) | ||
348 | pthread_join(_phone->cs->decode_video_thread,NULL); | ||
349 | SDL_Quit(); | ||
350 | printf("all A/V threads successfully shut down\n"); | ||
351 | |||
352 | INFO ( "Call ended!" ); | ||
353 | } | ||
354 | |||
355 | MCBTYPE callback_requ_timeout ( MCBARGS ) | ||
356 | { | ||
357 | INFO( "No answer! " ); | ||
358 | } | ||
359 | |||
360 | |||
361 | phone_t* initPhone(uint16_t _listen_port, uint16_t _send_port) | ||
362 | { | ||
363 | phone_t* _retu = calloc(sizeof(phone_t),1); | ||
364 | _retu->cs = av_calloc(sizeof(codec_state),1); | ||
365 | |||
366 | /* Initialize our mutex */ | ||
367 | pthread_mutex_init ( &_mutex, NULL ); | ||
368 | |||
369 | IP_Port _local; | ||
370 | ip_init(&_local.ip, 0); | ||
371 | _local.ip.ip4.uint32 = htonl ( INADDR_ANY ); | ||
372 | |||
373 | /* Bind local receive port to any address */ | ||
374 | _retu->_networking = new_networking ( _local.ip, _listen_port ); | ||
375 | |||
376 | if ( !_retu->_networking ) { | ||
377 | fprintf ( stderr, "new_networking() failed!\n" ); | ||
378 | return NULL; | ||
379 | } | ||
380 | |||
381 | _retu->_send_port = _send_port; | ||
382 | _retu->_recv_port = _listen_port; | ||
383 | |||
384 | _retu->_tox_sock = _retu->_networking->sock; | ||
385 | |||
386 | _retu->_rtp_audio = NULL; | ||
387 | _retu->_rtp_video = NULL; | ||
388 | |||
389 | |||
390 | /* Initialize msi */ | ||
391 | _retu->_msi = msi_init_session ( _retu->_networking, (const uint8_t*)_USERAGENT ); | ||
392 | |||
393 | if ( !_retu->_msi ) { | ||
394 | fprintf ( stderr, "msi_init_session() failed\n" ); | ||
395 | return NULL; | ||
396 | } | ||
397 | |||
398 | /* Initiate codecs */ | ||
399 | init_encoder(_retu->cs); | ||
400 | init_decoder(_retu->cs); | ||
401 | |||
402 | _retu->_msi->_agent_handler = _retu; | ||
403 | /* Initiate callbacks */ | ||
404 | msi_register_callback_send ( sendpacket ); /* Using core's send */ | ||
405 | |||
406 | msi_register_callback_call_started ( callback_call_started ); | ||
407 | msi_register_callback_call_canceled ( callback_call_canceled ); | ||
408 | msi_register_callback_call_rejected ( callback_call_rejected ); | ||
409 | msi_register_callback_call_ended ( callback_call_ended ); | ||
410 | |||
411 | msi_register_callback_recv_invite ( callback_recv_invite ); | ||
412 | msi_register_callback_recv_ringing ( callback_recv_ringing ); | ||
413 | msi_register_callback_recv_starting ( callback_recv_starting ); | ||
414 | msi_register_callback_recv_ending ( callback_recv_ending ); | ||
415 | msi_register_callback_recv_error(callback_recv_error); | ||
416 | |||
417 | msi_register_callback_requ_timeout ( callback_requ_timeout ); | ||
418 | /* ------------------ */ | ||
419 | |||
420 | /* Now start msi main loop. It's a must! | ||
421 | * Define a frequency in ms; 10 ms is just fine | ||
422 | */ | ||
423 | msi_start_main_loop ( _retu->_msi, 10 ); | ||
424 | |||
425 | return _retu; | ||
426 | } | ||
427 | |||
428 | pthread_t phone_startmain_loop(phone_t* _phone) | ||
429 | { | ||
430 | int _status; | ||
431 | /* Start receive thread */ | ||
432 | pthread_t _recv_thread, _phone_loop_thread; | ||
433 | _status = pthread_create ( &_recv_thread, NULL, phone_receivepacket, _phone ); | ||
434 | |||
435 | if ( _status < 0 ) { | ||
436 | printf ( "Error while starting handle call: %d, %s\n", errno, strerror ( errno ) ); | ||
437 | return 0; | ||
438 | } | ||
439 | |||
440 | _status = pthread_detach ( _recv_thread ); | ||
441 | |||
442 | if ( _status < 0 ) { | ||
443 | printf ( "Error while starting handle call: %d, %s\n", errno, strerror ( errno ) ); | ||
444 | return 0; | ||
445 | } | ||
446 | |||
447 | _status = pthread_create ( &_phone_loop_thread, NULL, phone_poll, _phone ); | ||
448 | |||
449 | if ( _status < 0 ) { | ||
450 | printf ( "Error while starting main phone loop: %d, %s\n", errno, strerror ( errno ) ); | ||
451 | return 0; | ||
452 | } | ||
453 | |||
454 | _status = pthread_join ( _phone_loop_thread, NULL ); | ||
455 | |||
456 | if ( _status < 0 ) { | ||
457 | printf ( "Error while starting main phone loop: %d, %s\n", errno, strerror ( errno ) ); | ||
458 | return 0; | ||
459 | } | ||
460 | |||
461 | return _phone_loop_thread; | ||
462 | } | ||
463 | |||
464 | void* phone_poll ( void* _p_phone ) | ||
465 | { | ||
466 | phone_t* _phone = _p_phone; | ||
467 | |||
468 | int _status = SUCCESS; | ||
469 | |||
470 | char _line[100]; | ||
471 | size_t _len; | ||
472 | |||
473 | |||
474 | char _dest[17]; /* For parsing destination ip */ | ||
475 | memset(_dest, '\0', 17); | ||
476 | |||
477 | INFO("Welcome to tox_phone version: " _USERAGENT "\n" | ||
478 | "Usage: \n" | ||
479 | "c [a/v] (type) [0.0.0.0] (dest ip) (calls dest ip)\n" | ||
480 | "h (if call is active hang up)\n" | ||
481 | "a [a/v] (answer incoming call: a - audio / v - audio + video (audio is default))\n" | ||
482 | "r (reject incoming call)\n" | ||
483 | "q (quit)\n" | ||
484 | "================================================================================" | ||
485 | ); | ||
486 | |||
487 | while ( 1 ) | ||
488 | { | ||
489 | fgets(_line, sizeof(_line), stdin); | ||
490 | int i; | ||
491 | for (i = 0; i < 100; i++) { | ||
492 | if (_line[i] == '\n') { | ||
493 | _line[i] = '\0'; | ||
494 | } | ||
495 | } | ||
496 | _len = strlen(_line); | ||
497 | |||
498 | if ( !_len ){ | ||
499 | printf(" >> "); fflush(stdout); | ||
500 | continue; | ||
501 | } | ||
502 | |||
503 | if ( _len > 1 && _line[1] != ' ' && _line[1] != '\n' ){ | ||
504 | INFO("Invalid input!"); | ||
505 | continue; | ||
506 | } | ||
507 | |||
508 | switch (_line[0]){ | ||
509 | |||
510 | case 'c': | ||
511 | { | ||
512 | if ( _phone->_msi->_call ){ | ||
513 | INFO("Already in a call"); | ||
514 | break; | ||
515 | } | ||
516 | |||
517 | call_type _ctype; | ||
518 | if ( _len < 11 ){ | ||
519 | INFO("Invalid input; usage: c a/v 0.0.0.0"); | ||
520 | break; | ||
521 | } | ||
522 | else if ( _line[2] == 'a' || _line[2] != 'v' ){ /* default and audio */ | ||
523 | _ctype = type_audio; | ||
524 | } | ||
525 | else { /* video */ | ||
526 | _ctype = type_video; | ||
527 | } | ||
528 | |||
529 | strcpy(_dest, _line + 4 ); | ||
530 | _status = t_setipport(_dest, _phone->_send_port, &(_phone->_msi->_friend_id)); | ||
531 | |||
532 | if ( _status < 0 ){ | ||
533 | INFO("Could not resolve address!"); | ||
534 | } else { | ||
535 | /* Set timeout */ | ||
536 | msi_invite ( _phone->_msi, _ctype, 30 * 1000 ); | ||
537 | INFO("Calling!"); | ||
538 | } | ||
539 | |||
540 | t_memset((uint8_t*)_dest, '\0', 17); | ||
541 | |||
542 | } break; | ||
543 | case 'h': | ||
544 | { | ||
545 | if ( !_phone->_msi->_call ){ | ||
546 | break; | ||
547 | } | ||
548 | |||
549 | msi_hangup(_phone->_msi); | ||
550 | |||
551 | INFO("Hung up..."); | ||
552 | |||
553 | } break; | ||
554 | case 'a': | ||
555 | { | ||
556 | if ( _phone->_msi->_call && _phone->_msi->_call->_state != call_starting ) { | ||
557 | break; | ||
558 | } | ||
559 | |||
560 | if ( _len > 1 && _line[2] == 'v' ) | ||
561 | msi_answer(_phone->_msi, type_video); | ||
562 | else | ||
563 | msi_answer(_phone->_msi, type_audio); | ||
564 | |||
565 | } break; | ||
566 | case 'r': | ||
567 | { | ||
568 | if ( _phone->_msi->_call && _phone->_msi->_call->_state != call_starting ){ | ||
569 | break; | ||
570 | } | ||
571 | |||
572 | msi_reject(_phone->_msi); | ||
573 | |||
574 | INFO("Call Rejected..."); | ||
575 | |||
576 | } break; | ||
577 | case 'q': | ||
578 | { | ||
579 | INFO("Quitting!"); | ||
580 | pthread_exit(NULL); | ||
581 | } | ||
582 | default: | ||
583 | { | ||
584 | INFO("Invalid command!"); | ||
585 | } break; | ||
586 | |||
587 | } | ||
588 | usleep(1000); | ||
589 | } | ||
590 | |||
591 | pthread_exit(NULL); | ||
592 | } | ||
593 | |||
594 | int quitPhone(phone_t* _phone) | ||
595 | { | ||
596 | if ( _phone->_msi->_call ){ | ||
597 | msi_hangup(_phone->_msi); /* Hangup the phone first */ | ||
598 | } | ||
599 | |||
600 | msi_terminate_session(_phone->_msi); | ||
601 | pthread_mutex_destroy ( &_mutex ); | ||
602 | |||
603 | printf("\r[i] Quit!\n"); | ||
604 | return SUCCESS; | ||
605 | } | ||
606 | |||
607 | /* ---------------------- */ | ||
608 | |||
609 | int print_help ( const char* _name ) | ||
610 | { | ||
611 | printf ( "Usage: %s -m (mode) -r/s ( for setting the ports on test version )\n", _name ); | ||
612 | return FAILURE; | ||
613 | } | ||
614 | |||
615 | int main ( int argc, char* argv [] ) | ||
616 | { | ||
617 | arg_t* _args = parse_args ( argc, argv ); | ||
618 | |||
619 | const char* _mode = find_arg_duble ( _args, "-m" ); | ||
620 | uint16_t _listen_port; | ||
621 | uint16_t _send_port; | ||
622 | |||
623 | if ( !_mode ) | ||
624 | return print_help ( argv[0] ); | ||
625 | |||
626 | if ( _mode[0] == 'r' ) { | ||
627 | _send_port = 31000; | ||
628 | _listen_port = 31001; | ||
629 | } else if ( _mode[0] == 's' ) { | ||
630 | _send_port = 31001; | ||
631 | _listen_port = 31000; | ||
632 | } else return print_help ( argv[0] ); | ||
633 | |||
634 | phone_t* _phone = initPhone(_listen_port, _send_port); | ||
635 | |||
636 | if ( _phone ){ | ||
637 | phone_startmain_loop(_phone); | ||
638 | |||
639 | quitPhone(_phone); | ||
640 | } | ||
641 | |||
642 | return SUCCESS; | ||
643 | } | ||
644 | |||
645 | #endif /* _CT_PHONE */ | ||