diff options
author | Martijn <martijnvdc@gmail.com> | 2013-10-13 16:14:27 +0200 |
---|---|---|
committer | BtbN <btbn@btbn.de> | 2013-10-13 16:14:27 +0200 |
commit | 1b971de651278429eea312f3240e1c5b8fbc67a4 (patch) | |
tree | 83e57945f3e7e9b0a305947935aed9d49735d2ab /toxmsi/AV_codec.c | |
parent | f2497b65897f034e736c234242fb08b2fc3b4a24 (diff) |
tox A/V: encode/decode and display/playback
Diffstat (limited to 'toxmsi/AV_codec.c')
-rw-r--r-- | toxmsi/AV_codec.c | 823 |
1 files changed, 823 insertions, 0 deletions
diff --git a/toxmsi/AV_codec.c b/toxmsi/AV_codec.c new file mode 100644 index 00000000..6730dca9 --- /dev/null +++ b/toxmsi/AV_codec.c | |||
@@ -0,0 +1,823 @@ | |||
1 | /* AV_codec.c | ||
2 | // * | ||
3 | * Audio and video codec intitialisation, encoding/decoding and playback | ||
4 | * | ||
5 | * Copyright (C) 2013 Tox project All Rights Reserved. | ||
6 | * | ||
7 | * This file is part of Tox. | ||
8 | * | ||
9 | * Tox is free software: you can redistribute it and/or modify | ||
10 | * it under the terms of the GNU General Public License as published by | ||
11 | * the Free Software Foundation, either version 3 of the License, or | ||
12 | * (at your option) any later version. | ||
13 | * | ||
14 | * Tox is distributed in the hope that it will be useful, | ||
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
17 | * GNU General Public License for more details. | ||
18 | * | ||
19 | * You should have received a copy of the GNU General Public License | ||
20 | * along with Tox. If not, see <http://www.gnu.org/licenses/>. | ||
21 | * | ||
22 | */ | ||
23 | |||
24 | /*----------------------------------------------------------------------------------*/ | ||
25 | |||
26 | #ifdef HAVE_CONFIG_H | ||
27 | #include "config.h" | ||
28 | #endif /* HAVE_CONFIG_H */ | ||
29 | |||
30 | #include <stdio.h> | ||
31 | #include <math.h> | ||
32 | #include <libavcodec/avcodec.h> | ||
33 | #include <libavformat/avformat.h> | ||
34 | #include <libswscale/swscale.h> | ||
35 | #include <libavdevice/avdevice.h> | ||
36 | #include <libavutil/opt.h> | ||
37 | #include <AL/al.h> | ||
38 | #include <AL/alc.h> | ||
39 | #include <SDL.h> | ||
40 | #include <SDL_thread.h> | ||
41 | #include <pthread.h> | ||
42 | #include <opus/opus.h> | ||
43 | |||
44 | #include "toxmsi.h" | ||
45 | #include "toxmsi_message.h" | ||
46 | #include "toxrtp_message.h" | ||
47 | #include "toxrtp/tests/test_helper.h" | ||
48 | #include "phone.h" | ||
49 | #include "AV_codec.h" | ||
50 | |||
51 | int display_received_frame(codec_state *cs, AVFrame *r_video_frame) | ||
52 | { | ||
53 | AVPicture pict; | ||
54 | SDL_LockYUVOverlay(cs->video_picture.bmp); | ||
55 | |||
56 | pict.data[0] = cs->video_picture.bmp->pixels[0]; | ||
57 | pict.data[1] = cs->video_picture.bmp->pixels[2]; | ||
58 | pict.data[2] = cs->video_picture.bmp->pixels[1]; | ||
59 | pict.linesize[0] = cs->video_picture.bmp->pitches[0]; | ||
60 | pict.linesize[1] = cs->video_picture.bmp->pitches[2]; | ||
61 | pict.linesize[2] = cs->video_picture.bmp->pitches[1]; | ||
62 | |||
63 | /* Convert the image into YUV format that SDL uses */ | ||
64 | sws_scale(cs->sws_SDL_r_ctx, (uint8_t const * const *)r_video_frame->data, r_video_frame->linesize, 0, | ||
65 | cs->video_decoder_ctx->height, pict.data, pict.linesize ); | ||
66 | |||
67 | SDL_UnlockYUVOverlay(cs->video_picture.bmp); | ||
68 | SDL_Rect rect; | ||
69 | rect.x = 0; | ||
70 | rect.y = 0; | ||
71 | rect.w = cs->video_decoder_ctx->width; | ||
72 | rect.h = cs->video_decoder_ctx->height; | ||
73 | SDL_DisplayYUVOverlay(cs->video_picture.bmp, &rect); | ||
74 | return 1; | ||
75 | } | ||
76 | |||
77 | struct jitter_buffer { | ||
78 | rtp_msg_t **queue; | ||
79 | uint16_t capacity; | ||
80 | uint16_t size; | ||
81 | uint16_t front; | ||
82 | uint16_t rear; | ||
83 | uint8_t queue_ready; | ||
84 | uint16_t current_id; | ||
85 | uint32_t current_ts; | ||
86 | uint8_t id_set; | ||
87 | }; | ||
88 | |||
89 | struct jitter_buffer *create_queue(int capacity) | ||
90 | { | ||
91 | struct jitter_buffer *q; | ||
92 | q = (struct jitter_buffer *)calloc(sizeof(struct jitter_buffer),1); | ||
93 | q->queue = (rtp_msg_t **)calloc((sizeof(rtp_msg_t) * capacity),1); | ||
94 | int i = 0; | ||
95 | |||
96 | for (i = 0; i < capacity; ++i) { | ||
97 | q->queue[i] = NULL; | ||
98 | } | ||
99 | |||
100 | q->size = 0; | ||
101 | q->capacity = capacity; | ||
102 | q->front = 0; | ||
103 | q->rear = -1; | ||
104 | q->queue_ready = 0; | ||
105 | q->current_id = 0; | ||
106 | q->current_ts = 0; | ||
107 | q->id_set = 0; | ||
108 | return q; | ||
109 | } | ||
110 | |||
111 | /* returns 1 if 'a' has a higher sequence number than 'b' */ | ||
112 | uint8_t sequence_number_older(uint16_t sn_a, uint16_t sn_b, uint32_t ts_a, uint32_t ts_b) | ||
113 | { | ||
114 | /* should be stable enough */ | ||
115 | return (sn_a > sn_b || ts_a > ts_b); | ||
116 | } | ||
117 | |||
118 | /* success is 0 when there is nothing to dequeue, 1 when there's a good packet, 2 when there's a lost packet */ | ||
119 | rtp_msg_t *dequeue(struct jitter_buffer *q, int *success) | ||
120 | { | ||
121 | if (q->size == 0 || q->queue_ready == 0) { | ||
122 | q->queue_ready = 0; | ||
123 | *success = 0; | ||
124 | return NULL; | ||
125 | } | ||
126 | |||
127 | int front = q->front; | ||
128 | |||
129 | if (q->id_set == 0) { | ||
130 | q->current_id = q->queue[front]->_header->_sequence_number; | ||
131 | q->current_ts = q->queue[front]->_header->_timestamp; | ||
132 | q->id_set = 1; | ||
133 | } else { | ||
134 | int next_id = q->queue[front]->_header->_sequence_number; | ||
135 | int next_ts = q->queue[front]->_header->_timestamp; | ||
136 | |||
137 | /* if this packet is indeed the expected packet */ | ||
138 | if (next_id == (q->current_id + 1) % _MAX_SEQU_NUM) { | ||
139 | q->current_id = next_id; | ||
140 | q->current_ts = next_ts; | ||
141 | } else { | ||
142 | if (sequence_number_older(next_id, q->current_id, next_ts, q->current_ts)) { | ||
143 | printf("nextid: %d current: %d\n", next_id, q->current_id); | ||
144 | q->current_id = (q->current_id + 1) % _MAX_SEQU_NUM; | ||
145 | *success = 2; /* tell the decoder the packet is lost */ | ||
146 | return NULL; | ||
147 | } else { | ||
148 | /* packet too old */ | ||
149 | printf("packet too old\n"); | ||
150 | *success = 0; | ||
151 | return NULL; | ||
152 | } | ||
153 | } | ||
154 | } | ||
155 | |||
156 | q->size--; | ||
157 | q->front++; | ||
158 | |||
159 | if (q->front == q->capacity) | ||
160 | q->front = 0; | ||
161 | |||
162 | *success = 1; | ||
163 | q->current_id = q->queue[front]->_header->_sequence_number; | ||
164 | q->current_ts = q->queue[front]->_header->_timestamp; | ||
165 | return q->queue[front]; | ||
166 | } | ||
167 | |||
168 | int empty_queue(struct jitter_buffer *q) | ||
169 | { | ||
170 | while (q->size > 0) { | ||
171 | q->size--; | ||
172 | /* FIXME: */ | ||
173 | /* rtp_free_msg(cs->_rtp_video, q->queue[q->front]); */ | ||
174 | q->front++; | ||
175 | |||
176 | if (q->front == q->capacity) | ||
177 | q->front = 0; | ||
178 | } | ||
179 | |||
180 | q->id_set = 0; | ||
181 | q->queue_ready = 0; | ||
182 | return 0; | ||
183 | } | ||
184 | |||
185 | int queue(struct jitter_buffer *q, rtp_msg_t *pk) | ||
186 | { | ||
187 | if (q->size == q->capacity) { | ||
188 | printf("buffer full, emptying buffer...\n"); | ||
189 | empty_queue(q); | ||
190 | return 0; | ||
191 | } | ||
192 | |||
193 | if (q->size > 8) | ||
194 | q->queue_ready = 1; | ||
195 | |||
196 | ++q->size; | ||
197 | ++q->rear; | ||
198 | |||
199 | if (q->rear == q->capacity) | ||
200 | q->rear = 0; | ||
201 | |||
202 | q->queue[q->rear] = pk; | ||
203 | |||
204 | int a; | ||
205 | int b; | ||
206 | int j; | ||
207 | a = q->rear; | ||
208 | |||
209 | for (j = 0; j < q->size - 1; ++j) { | ||
210 | b = a - 1; | ||
211 | |||
212 | if (b < 0) | ||
213 | b += q->capacity; | ||
214 | |||
215 | if (sequence_number_older(q->queue[b]->_header->_sequence_number, q->queue[a]->_header->_sequence_number, | ||
216 | q->queue[b]->_header->_timestamp, q->queue[a]->_header->_timestamp)) { | ||
217 | rtp_msg_t *temp; | ||
218 | temp = q->queue[a]; | ||
219 | q->queue[a] = q->queue[b]; | ||
220 | q->queue[b] = temp; | ||
221 | printf("had to swap\n"); | ||
222 | } else { | ||
223 | break; | ||
224 | } | ||
225 | |||
226 | a -= 1; | ||
227 | |||
228 | if (a < 0) | ||
229 | a += q->capacity; | ||
230 | } | ||
231 | |||
232 | if (pk) | ||
233 | return 1; | ||
234 | |||
235 | return 0; | ||
236 | } | ||
237 | |||
238 | int init_receive_audio(codec_state *cs) | ||
239 | { | ||
240 | int err = OPUS_OK; | ||
241 | cs->audio_decoder = opus_decoder_create(48000, 1, &err); | ||
242 | opus_decoder_init(cs->audio_decoder, 48000, 1); | ||
243 | printf("init audio decoder successful\n"); | ||
244 | return 1; | ||
245 | } | ||
246 | |||
247 | int init_receive_video(codec_state *cs) | ||
248 | { | ||
249 | cs->video_decoder = avcodec_find_decoder(VIDEO_CODEC); | ||
250 | |||
251 | if (!cs->video_decoder) { | ||
252 | printf("init video_decoder failed\n"); | ||
253 | return 0; | ||
254 | } | ||
255 | |||
256 | cs->video_decoder_ctx = avcodec_alloc_context3(cs->video_decoder); | ||
257 | |||
258 | if (!cs->video_decoder_ctx) { | ||
259 | printf("init video_decoder_ctx failed\n"); | ||
260 | return 0; | ||
261 | } | ||
262 | |||
263 | if (avcodec_open2(cs->video_decoder_ctx, cs->video_decoder, NULL) < 0) { | ||
264 | printf("opening video decoder failed\n"); | ||
265 | return 0; | ||
266 | } | ||
267 | |||
268 | printf("init video decoder successful\n"); | ||
269 | return 1; | ||
270 | } | ||
271 | |||
272 | int init_send_video(codec_state *cs) | ||
273 | { | ||
274 | cs->video_input_format = av_find_input_format(VIDEO_DRIVER); | ||
275 | |||
276 | if (avformat_open_input(&cs->video_format_ctx, DEFAULT_WEBCAM, cs->video_input_format, NULL) != 0) { | ||
277 | printf("opening video_input_format failed\n"); | ||
278 | return 0; | ||
279 | } | ||
280 | |||
281 | avformat_find_stream_info(cs->video_format_ctx, NULL); | ||
282 | av_dump_format(cs->video_format_ctx, 0, DEFAULT_WEBCAM, 0); | ||
283 | |||
284 | int i; | ||
285 | |||
286 | for (i = 0; i < cs->video_format_ctx->nb_streams; ++i) { | ||
287 | if (cs->video_format_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) { | ||
288 | cs->video_stream = i; | ||
289 | break; | ||
290 | } | ||
291 | } | ||
292 | |||
293 | cs->webcam_decoder_ctx = cs->video_format_ctx->streams[cs->video_stream]->codec; | ||
294 | cs->webcam_decoder = avcodec_find_decoder(cs->webcam_decoder_ctx->codec_id); | ||
295 | |||
296 | if (cs->webcam_decoder == NULL) { | ||
297 | printf("Unsupported codec\n"); | ||
298 | return 0; | ||
299 | } | ||
300 | |||
301 | if (cs->webcam_decoder_ctx == NULL) { | ||
302 | printf("init webcam_decoder_ctx failed\n"); | ||
303 | return 0; | ||
304 | } | ||
305 | |||
306 | if (avcodec_open2(cs->webcam_decoder_ctx, cs->webcam_decoder, NULL) < 0) { | ||
307 | printf("opening webcam decoder failed\n"); | ||
308 | return 0; | ||
309 | } | ||
310 | |||
311 | cs->video_encoder = avcodec_find_encoder(VIDEO_CODEC); | ||
312 | |||
313 | if (!cs->video_encoder) { | ||
314 | printf("init video_encoder failed\n"); | ||
315 | return 0; | ||
316 | } | ||
317 | |||
318 | cs->video_encoder_ctx = avcodec_alloc_context3(cs->video_encoder); | ||
319 | |||
320 | if (!cs->video_encoder_ctx) { | ||
321 | printf("init video_encoder_ctx failed\n"); | ||
322 | return 0; | ||
323 | } | ||
324 | |||
325 | cs->video_encoder_ctx->bit_rate = VIDEO_BITRATE; | ||
326 | cs->video_encoder_ctx->rc_min_rate = cs->video_encoder_ctx->rc_max_rate = cs->video_encoder_ctx->bit_rate; | ||
327 | av_opt_set_double(cs->video_encoder_ctx->priv_data, "max-intra-rate", 90, 0); | ||
328 | av_opt_set(cs->video_encoder_ctx->priv_data, "quality", "realtime", 0); | ||
329 | |||
330 | cs->video_encoder_ctx->thread_count = 4; | ||
331 | cs->video_encoder_ctx->rc_buffer_aggressivity = 0.95; | ||
332 | cs->video_encoder_ctx->rc_buffer_size = VIDEO_BITRATE * 6; | ||
333 | cs->video_encoder_ctx->profile = 3; | ||
334 | cs->video_encoder_ctx->qmax = 54; | ||
335 | cs->video_encoder_ctx->qmin = 4; | ||
336 | AVRational myrational = {1, 25}; | ||
337 | cs->video_encoder_ctx->time_base = myrational; | ||
338 | cs->video_encoder_ctx->gop_size = 99999; | ||
339 | cs->video_encoder_ctx->pix_fmt = PIX_FMT_YUV420P; | ||
340 | cs->video_encoder_ctx->width = cs->webcam_decoder_ctx->width; | ||
341 | cs->video_encoder_ctx->height = cs->webcam_decoder_ctx->height; | ||
342 | |||
343 | if (avcodec_open2(cs->video_encoder_ctx, cs->video_encoder, NULL) < 0) { | ||
344 | printf("opening video encoder failed\n"); | ||
345 | return 0; | ||
346 | } | ||
347 | |||
348 | printf("init video encoder successful\n"); | ||
349 | return 1; | ||
350 | } | ||
351 | |||
352 | int init_send_audio(codec_state *cs) | ||
353 | { | ||
354 | cs->support_send_audio = 0; | ||
355 | |||
356 | const ALchar *pDeviceList = alcGetString(NULL, ALC_CAPTURE_DEVICE_SPECIFIER); | ||
357 | int i = 0; | ||
358 | const ALchar *device_names[20]; | ||
359 | |||
360 | if (pDeviceList) { | ||
361 | printf("\nAvailable Capture Devices are:\n"); | ||
362 | |||
363 | while (*pDeviceList) { | ||
364 | device_names[i] = pDeviceList; | ||
365 | printf("%d) %s\n", i, device_names[i]); | ||
366 | pDeviceList += strlen(pDeviceList) + 1; | ||
367 | ++i; | ||
368 | } | ||
369 | } | ||
370 | |||
371 | printf("enter capture device number: \n"); | ||
372 | char dev[2]; | ||
373 | fgets(dev, sizeof(dev), stdin); | ||
374 | cs->audio_capture_device = alcCaptureOpenDevice(device_names[dev[0] - 48], AUDIO_SAMPLE_RATE, AL_FORMAT_MONO16, | ||
375 | AUDIO_FRAME_SIZE * 4); | ||
376 | |||
377 | if (alcGetError(cs->audio_capture_device) != AL_NO_ERROR) { | ||
378 | printf("could not start capture device! %d\n", alcGetError(cs->audio_capture_device)); | ||
379 | return 0; | ||
380 | } | ||
381 | |||
382 | int err = OPUS_OK; | ||
383 | cs->audio_bitrate = AUDIO_BITRATE; | ||
384 | cs->audio_encoder = opus_encoder_create(AUDIO_SAMPLE_RATE, 1, OPUS_APPLICATION_VOIP, &err); | ||
385 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_BITRATE(cs->audio_bitrate)); | ||
386 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_COMPLEXITY(10)); | ||
387 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); | ||
388 | |||
389 | opus_encoder_init(cs->audio_encoder, AUDIO_SAMPLE_RATE, 1, OPUS_APPLICATION_VOIP); | ||
390 | |||
391 | int nfo; | ||
392 | err = opus_encoder_ctl(cs->audio_encoder, OPUS_GET_LOOKAHEAD(&nfo)); | ||
393 | /* printf("Encoder lookahead delay : %d\n", nfo); */ | ||
394 | printf("init audio encoder successful\n"); | ||
395 | |||
396 | return 1; | ||
397 | } | ||
398 | |||
399 | int init_encoder(codec_state *cs) | ||
400 | { | ||
401 | avdevice_register_all(); | ||
402 | avcodec_register_all(); | ||
403 | avdevice_register_all(); | ||
404 | av_register_all(); | ||
405 | |||
406 | pthread_mutex_init(&cs->rtp_msg_mutex_lock, NULL); | ||
407 | pthread_mutex_init(&cs->avcodec_mutex_lock, NULL); | ||
408 | |||
409 | cs->support_send_video = init_send_video(cs); | ||
410 | cs->support_send_audio = init_send_audio(cs); | ||
411 | |||
412 | cs->send_audio = 1; | ||
413 | cs->send_video = 1; | ||
414 | |||
415 | return 1; | ||
416 | } | ||
417 | |||
418 | int init_decoder(codec_state *cs) | ||
419 | { | ||
420 | avdevice_register_all(); | ||
421 | avcodec_register_all(); | ||
422 | avdevice_register_all(); | ||
423 | av_register_all(); | ||
424 | |||
425 | cs->receive_video = 0; | ||
426 | cs->receive_audio = 0; | ||
427 | |||
428 | cs->support_receive_video = init_receive_video(cs); | ||
429 | cs->support_receive_audio = init_receive_audio(cs); | ||
430 | |||
431 | cs->receive_audio = 1; | ||
432 | cs->receive_video = 1; | ||
433 | |||
434 | return 1; | ||
435 | } | ||
436 | |||
437 | int video_encoder_refresh(codec_state *cs, int bps) | ||
438 | { | ||
439 | if (cs->video_encoder_ctx) | ||
440 | avcodec_close(cs->video_encoder_ctx); | ||
441 | |||
442 | cs->video_encoder = avcodec_find_encoder(VIDEO_CODEC); | ||
443 | |||
444 | if (!cs->video_encoder) { | ||
445 | printf("init video_encoder failed\n"); | ||
446 | return -1; | ||
447 | } | ||
448 | |||
449 | cs->video_encoder_ctx = avcodec_alloc_context3(cs->video_encoder); | ||
450 | |||
451 | if (!cs->video_encoder_ctx) { | ||
452 | printf("init video_encoder_ctx failed\n"); | ||
453 | return -1; | ||
454 | } | ||
455 | |||
456 | cs->video_encoder_ctx->bit_rate = bps; | ||
457 | cs->video_encoder_ctx->rc_min_rate = cs->video_encoder_ctx->rc_max_rate = cs->video_encoder_ctx->bit_rate; | ||
458 | av_opt_set_double(cs->video_encoder_ctx->priv_data, "max-intra-rate", 90, 0); | ||
459 | av_opt_set(cs->video_encoder_ctx->priv_data, "quality", "realtime", 0); | ||
460 | |||
461 | cs->video_encoder_ctx->thread_count = 4; | ||
462 | cs->video_encoder_ctx->rc_buffer_aggressivity = 0.95; | ||
463 | cs->video_encoder_ctx->rc_buffer_size = bps * 6; | ||
464 | cs->video_encoder_ctx->profile = 0; | ||
465 | cs->video_encoder_ctx->qmax = 54; | ||
466 | cs->video_encoder_ctx->qmin = 4; | ||
467 | AVRational myrational = {1, 25}; | ||
468 | cs->video_encoder_ctx->time_base = myrational; | ||
469 | cs->video_encoder_ctx->gop_size = 99999; | ||
470 | cs->video_encoder_ctx->pix_fmt = PIX_FMT_YUV420P; | ||
471 | cs->video_encoder_ctx->width = cs->webcam_decoder_ctx->width; | ||
472 | cs->video_encoder_ctx->height = cs->webcam_decoder_ctx->height; | ||
473 | |||
474 | if (avcodec_open2(cs->video_encoder_ctx, cs->video_encoder, NULL) < 0) { | ||
475 | printf("opening video encoder failed\n"); | ||
476 | return -1; | ||
477 | } | ||
478 | return 0; | ||
479 | } | ||
480 | |||
481 | void *encode_video_thread(void *arg) | ||
482 | { | ||
483 | codec_state *cs = (codec_state *)arg; | ||
484 | AVPacket pkt1, *packet = &pkt1; | ||
485 | int p = 0; | ||
486 | int err; | ||
487 | int got_packet; | ||
488 | rtp_msg_t *s_video_msg; | ||
489 | int video_frame_finished; | ||
490 | AVFrame *s_video_frame; | ||
491 | AVFrame *webcam_frame; | ||
492 | s_video_frame = avcodec_alloc_frame(); | ||
493 | webcam_frame = avcodec_alloc_frame(); | ||
494 | AVPacket enc_video_packet; | ||
495 | |||
496 | uint8_t *buffer; | ||
497 | int numBytes; | ||
498 | /* Determine required buffer size and allocate buffer */ | ||
499 | numBytes = avpicture_get_size(PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height); | ||
500 | buffer = (uint8_t *)av_calloc(numBytes * sizeof(uint8_t),1); | ||
501 | avpicture_fill((AVPicture *)s_video_frame, buffer, PIX_FMT_YUV420P, cs->webcam_decoder_ctx->width, | ||
502 | cs->webcam_decoder_ctx->height); | ||
503 | cs->sws_ctx = sws_getContext(cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height, | ||
504 | cs->webcam_decoder_ctx->pix_fmt, cs->webcam_decoder_ctx->width, cs->webcam_decoder_ctx->height, PIX_FMT_YUV420P, | ||
505 | SWS_BILINEAR, NULL, NULL, NULL); | ||
506 | |||
507 | while (!cs->quit && cs->send_video) { | ||
508 | |||
509 | if (av_read_frame(cs->video_format_ctx, packet) < 0) { | ||
510 | printf("error reading frame\n"); | ||
511 | |||
512 | if (cs->video_format_ctx->pb->error != 0) | ||
513 | break; | ||
514 | |||
515 | continue; | ||
516 | } | ||
517 | |||
518 | if (packet->stream_index == cs->video_stream) { | ||
519 | if (avcodec_decode_video2(cs->webcam_decoder_ctx, webcam_frame, &video_frame_finished, packet) < 0) { | ||
520 | printf("couldn't decode\n"); | ||
521 | continue; | ||
522 | } | ||
523 | |||
524 | av_free_packet(packet); | ||
525 | sws_scale(cs->sws_ctx, (uint8_t const * const *)webcam_frame->data, webcam_frame->linesize, 0, | ||
526 | cs->webcam_decoder_ctx->height, s_video_frame->data, s_video_frame->linesize); | ||
527 | /* create a new I-frame every 60 frames */ | ||
528 | ++p; | ||
529 | |||
530 | if (p == 60) { | ||
531 | |||
532 | s_video_frame->pict_type = AV_PICTURE_TYPE_BI ; | ||
533 | } else if (p == 61) { | ||
534 | s_video_frame->pict_type = AV_PICTURE_TYPE_I ; | ||
535 | p = 0; | ||
536 | } else { | ||
537 | s_video_frame->pict_type = AV_PICTURE_TYPE_P ; | ||
538 | } | ||
539 | |||
540 | if (video_frame_finished) { | ||
541 | err = avcodec_encode_video2(cs->video_encoder_ctx, &enc_video_packet, s_video_frame, &got_packet); | ||
542 | |||
543 | if (err < 0) { | ||
544 | printf("could not encode video frame\n"); | ||
545 | continue; | ||
546 | } | ||
547 | |||
548 | if (!got_packet) { | ||
549 | continue; | ||
550 | } | ||
551 | |||
552 | pthread_mutex_lock(&cs->rtp_msg_mutex_lock); | ||
553 | THREADLOCK() | ||
554 | |||
555 | if (!enc_video_packet.data) fprintf(stderr, "video packet data is NULL\n"); | ||
556 | |||
557 | s_video_msg = rtp_msg_new ( cs->_rtp_video, enc_video_packet.data, enc_video_packet.size ) ; | ||
558 | |||
559 | if (!s_video_msg) { | ||
560 | printf("invalid message\n"); | ||
561 | } | ||
562 | |||
563 | rtp_send_msg ( cs->_rtp_video, s_video_msg, cs->_networking ); | ||
564 | THREADUNLOCK() | ||
565 | pthread_mutex_unlock(&cs->rtp_msg_mutex_lock); | ||
566 | av_free_packet(&enc_video_packet); | ||
567 | } | ||
568 | } else { | ||
569 | av_free_packet(packet); | ||
570 | } | ||
571 | } | ||
572 | |||
573 | /* clean up codecs */ | ||
574 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
575 | av_free(buffer); | ||
576 | av_free(webcam_frame); | ||
577 | av_free(s_video_frame); | ||
578 | sws_freeContext(cs->sws_ctx); | ||
579 | avcodec_close(cs->webcam_decoder_ctx); | ||
580 | avcodec_close(cs->video_encoder_ctx); | ||
581 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
582 | pthread_exit ( NULL ); | ||
583 | } | ||
584 | |||
585 | void *encode_audio_thread(void *arg) | ||
586 | { | ||
587 | codec_state *cs = (codec_state *)arg; | ||
588 | rtp_msg_t *s_audio_msg; | ||
589 | unsigned char encoded_data[4096]; | ||
590 | int encoded_size = 0; | ||
591 | int16_t frame[4096]; | ||
592 | int frame_size = AUDIO_FRAME_SIZE; | ||
593 | ALint sample = 0; | ||
594 | alcCaptureStart(cs->audio_capture_device); | ||
595 | |||
596 | while (!cs->quit && cs->send_audio) { | ||
597 | alcGetIntegerv(cs->audio_capture_device, ALC_CAPTURE_SAMPLES, (ALCsizei)sizeof(ALint), &sample); | ||
598 | |||
599 | if (sample >= frame_size) { | ||
600 | alcCaptureSamples(cs->audio_capture_device, frame, frame_size); | ||
601 | encoded_size = opus_encode(cs->audio_encoder, frame, frame_size, encoded_data, 480); | ||
602 | |||
603 | if (encoded_size <= 0) { | ||
604 | printf("Could not encode audio packet\n"); | ||
605 | } else { | ||
606 | pthread_mutex_lock(&cs->rtp_msg_mutex_lock); | ||
607 | THREADLOCK() | ||
608 | rtp_set_payload_type(cs->_rtp_audio, 96); | ||
609 | s_audio_msg = rtp_msg_new (cs->_rtp_audio, encoded_data, encoded_size) ; | ||
610 | rtp_send_msg ( cs->_rtp_audio, s_audio_msg, cs->_networking ); | ||
611 | pthread_mutex_unlock(&cs->rtp_msg_mutex_lock); | ||
612 | THREADUNLOCK() | ||
613 | } | ||
614 | } else { | ||
615 | usleep(1000); | ||
616 | } | ||
617 | } | ||
618 | |||
619 | /* clean up codecs */ | ||
620 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
621 | alcCaptureStop(cs->audio_capture_device); | ||
622 | alcCaptureCloseDevice(cs->audio_capture_device); | ||
623 | |||
624 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
625 | pthread_exit ( NULL ); | ||
626 | } | ||
627 | |||
628 | |||
629 | int video_decoder_refresh(codec_state *cs, int width, int height) | ||
630 | { | ||
631 | printf("need to refresh\n"); | ||
632 | screen = SDL_SetVideoMode(width, height, 0, 0); | ||
633 | |||
634 | if (cs->video_picture.bmp) | ||
635 | SDL_FreeYUVOverlay(cs->video_picture.bmp); | ||
636 | |||
637 | cs->video_picture.bmp = SDL_CreateYUVOverlay(width, height, SDL_YV12_OVERLAY, screen); | ||
638 | cs->sws_SDL_r_ctx = sws_getContext(width, height, cs->video_decoder_ctx->pix_fmt, width, height, PIX_FMT_YUV420P, | ||
639 | SWS_BILINEAR, NULL, NULL, NULL); | ||
640 | return 1; | ||
641 | } | ||
642 | |||
643 | void *decode_video_thread(void *arg) | ||
644 | { | ||
645 | codec_state *cs = (codec_state *)arg; | ||
646 | cs->video_stream = 0; | ||
647 | rtp_msg_t *r_msg; | ||
648 | int dec_frame_finished; | ||
649 | AVFrame *r_video_frame; | ||
650 | r_video_frame = avcodec_alloc_frame(); | ||
651 | AVPacket dec_video_packet; | ||
652 | av_new_packet (&dec_video_packet, 65536); | ||
653 | int width = 0; | ||
654 | int height = 0; | ||
655 | |||
656 | while (!cs->quit && cs->receive_video) { | ||
657 | r_msg = rtp_recv_msg ( cs->_rtp_video ); | ||
658 | |||
659 | if (r_msg) { | ||
660 | memcpy(dec_video_packet.data, r_msg->_data, r_msg->_length); | ||
661 | dec_video_packet.size = r_msg->_length; | ||
662 | avcodec_decode_video2(cs->video_decoder_ctx, r_video_frame, &dec_frame_finished, &dec_video_packet); | ||
663 | |||
664 | if (dec_frame_finished) { | ||
665 | if (cs->video_decoder_ctx->width != width || cs->video_decoder_ctx->height != height) { | ||
666 | width = cs->video_decoder_ctx->width; | ||
667 | height = cs->video_decoder_ctx->height; | ||
668 | printf("w: %d h%d \n", width, height); | ||
669 | video_decoder_refresh(cs, width, height); | ||
670 | } | ||
671 | |||
672 | display_received_frame(cs, r_video_frame); | ||
673 | } else { | ||
674 | /* TODO: request the sender to create a new i-frame immediatly */ | ||
675 | printf("bad video packet\n"); | ||
676 | } | ||
677 | |||
678 | rtp_free_msg(cs->_rtp_video, r_msg); | ||
679 | } | ||
680 | |||
681 | usleep(1000); | ||
682 | } | ||
683 | |||
684 | printf("vend\n"); | ||
685 | /* clean up codecs */ | ||
686 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
687 | av_free(r_video_frame); | ||
688 | avcodec_close(cs->video_decoder_ctx); | ||
689 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
690 | pthread_exit ( NULL ); | ||
691 | } | ||
692 | |||
693 | void *decode_audio_thread(void *arg) | ||
694 | { | ||
695 | codec_state *cs = (codec_state *)arg; | ||
696 | rtp_msg_t *r_msg; | ||
697 | |||
698 | int frame_size = AUDIO_FRAME_SIZE; | ||
699 | int data_size; | ||
700 | |||
701 | ALCdevice *dev; | ||
702 | ALCcontext *ctx; | ||
703 | ALuint source, *buffers; | ||
704 | dev = alcOpenDevice(NULL); | ||
705 | ctx = alcCreateContext(dev, NULL); | ||
706 | alcMakeContextCurrent(ctx); | ||
707 | int openal_buffers = 5; | ||
708 | |||
709 | buffers = calloc(sizeof(ALuint) * openal_buffers,1); | ||
710 | alGenBuffers(openal_buffers, buffers); | ||
711 | alGenSources((ALuint)1, &source); | ||
712 | alSourcei(source, AL_LOOPING, AL_FALSE); | ||
713 | |||
714 | ALuint buffer; | ||
715 | ALint val; | ||
716 | |||
717 | ALenum error; | ||
718 | uint16_t zeros[frame_size]; | ||
719 | int i; | ||
720 | |||
721 | for (i = 0; i < frame_size; i++) { | ||
722 | zeros[i] = 0; | ||
723 | } | ||
724 | |||
725 | for (i = 0; i < openal_buffers; ++i) { | ||
726 | alBufferData(buffers[i], AL_FORMAT_MONO16, zeros, frame_size, 48000); | ||
727 | } | ||
728 | |||
729 | alSourceQueueBuffers(source, openal_buffers, buffers); | ||
730 | alSourcePlay(source); | ||
731 | |||
732 | if (alGetError() != AL_NO_ERROR) { | ||
733 | fprintf(stderr, "Error starting audio\n"); | ||
734 | cs->quit = 1; | ||
735 | } | ||
736 | |||
737 | struct jitter_buffer *j_buf = NULL; | ||
738 | |||
739 | j_buf = create_queue(20); | ||
740 | |||
741 | int success = 0; | ||
742 | |||
743 | int dec_frame_len; | ||
744 | |||
745 | opus_int16 PCM[frame_size]; | ||
746 | |||
747 | while (!cs->quit && cs->receive_audio) { | ||
748 | THREADLOCK() | ||
749 | r_msg = rtp_recv_msg ( cs->_rtp_audio ); | ||
750 | |||
751 | if (r_msg) { | ||
752 | /* push the packet into the queue */ | ||
753 | queue(j_buf, r_msg); | ||
754 | } | ||
755 | |||
756 | /* grab a packet from the queue */ | ||
757 | success = 0; | ||
758 | alGetSourcei(source, AL_BUFFERS_PROCESSED, &val); | ||
759 | |||
760 | if (val > 0) | ||
761 | r_msg = dequeue(j_buf, &success); | ||
762 | |||
763 | if (success > 0) { | ||
764 | /* good packet */ | ||
765 | if (success == 1) { | ||
766 | dec_frame_len = opus_decode(cs->audio_decoder, r_msg->_data, r_msg->_length, PCM, frame_size, 0); | ||
767 | rtp_free_msg(cs->_rtp_audio, r_msg); | ||
768 | } | ||
769 | |||
770 | /* lost packet */ | ||
771 | if (success == 2) { | ||
772 | printf("lost packet\n"); | ||
773 | dec_frame_len = opus_decode(cs->audio_decoder, NULL, 0, PCM, frame_size, 1); | ||
774 | } | ||
775 | |||
776 | if (dec_frame_len > 0) { | ||
777 | alGetSourcei(source, AL_BUFFERS_PROCESSED, &val); | ||
778 | |||
779 | if (val <= 0) | ||
780 | continue; | ||
781 | |||
782 | alSourceUnqueueBuffers(source, 1, &buffer); | ||
783 | data_size = av_samples_get_buffer_size(NULL, 1, dec_frame_len, AV_SAMPLE_FMT_S16, 1); | ||
784 | alBufferData(buffer, AL_FORMAT_MONO16, PCM, data_size, 48000); | ||
785 | int error = alGetError(); | ||
786 | |||
787 | if (error != AL_NO_ERROR) { | ||
788 | fprintf(stderr, "Error setting buffer %d\n", error); | ||
789 | break; | ||
790 | } | ||
791 | |||
792 | alSourceQueueBuffers(source, 1, &buffer); | ||
793 | |||
794 | if (alGetError() != AL_NO_ERROR) { | ||
795 | fprintf(stderr, "error: could not buffer audio\n"); | ||
796 | break; | ||
797 | } | ||
798 | |||
799 | alGetSourcei(source, AL_SOURCE_STATE, &val); | ||
800 | |||
801 | if (val != AL_PLAYING) | ||
802 | alSourcePlay(source); | ||
803 | |||
804 | |||
805 | } | ||
806 | } | ||
807 | |||
808 | THREADUNLOCK() | ||
809 | usleep(1000); | ||
810 | } | ||
811 | |||
812 | /* clean up codecs */ | ||
813 | pthread_mutex_lock(&cs->avcodec_mutex_lock); | ||
814 | |||
815 | /* clean up openal */ | ||
816 | alDeleteSources(1, &source); | ||
817 | alDeleteBuffers(openal_buffers, buffers); | ||
818 | alcMakeContextCurrent(NULL); | ||
819 | alcDestroyContext(ctx); | ||
820 | alcCloseDevice(dev); | ||
821 | pthread_mutex_unlock(&cs->avcodec_mutex_lock); | ||
822 | pthread_exit ( NULL ); | ||
823 | } \ No newline at end of file | ||