diff options
author | irungentoo <irungentoo@gmail.com> | 2014-08-01 13:21:03 -0400 |
---|---|---|
committer | irungentoo <irungentoo@gmail.com> | 2014-08-01 13:21:03 -0400 |
commit | 055640611fca4b6e35e8d81d5e6cf809c2cbb078 (patch) | |
tree | 1b7483822f5f1b44de64318366e4e8a0a84b04e1 | |
parent | 946a09f57b223f4b8bf5bd6e65fb23d089b9959e (diff) | |
parent | 8ee3f645b158a85dcb921e471225b48c20f5559a (diff) |
Merge branch 'notsecure-split-video'
-rw-r--r-- | auto_tests/toxav_many_test.c | 1 | ||||
-rw-r--r-- | toxav/toxav.c | 290 |
2 files changed, 258 insertions, 33 deletions
diff --git a/auto_tests/toxav_many_test.c b/auto_tests/toxav_many_test.c index df2be467..299e701d 100644 --- a/auto_tests/toxav_many_test.c +++ b/auto_tests/toxav_many_test.c | |||
@@ -127,6 +127,7 @@ static void callback_audio(ToxAv *av, int32_t call_index, int16_t *data, int len | |||
127 | static void callback_video(ToxAv *av, int32_t call_index, vpx_image_t *img) | 127 | static void callback_video(ToxAv *av, int32_t call_index, vpx_image_t *img) |
128 | { | 128 | { |
129 | } | 129 | } |
130 | |||
130 | void register_callbacks(ToxAv *av, void *data) | 131 | void register_callbacks(ToxAv *av, void *data) |
131 | { | 132 | { |
132 | toxav_register_callstate_callback(av, callback_call_started, av_OnStart, data); | 133 | toxav_register_callstate_callback(av, callback_call_started, av_OnStart, data); |
diff --git a/toxav/toxav.c b/toxav/toxav.c index 0aa88919..75e93bac 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c | |||
@@ -90,6 +90,14 @@ typedef struct _CallSpecific { | |||
90 | pthread_mutex_t mutex; | 90 | pthread_mutex_t mutex; |
91 | } CallSpecific; | 91 | } CallSpecific; |
92 | 92 | ||
93 | typedef struct { | ||
94 | int32_t call_index; | ||
95 | uint32_t size; | ||
96 | uint8_t data[0]; | ||
97 | } DECODE_PACKET; | ||
98 | |||
99 | #define VIDEO_DECODE_QUEUE_SIZE 2 | ||
100 | #define AUDIO_DECODE_QUEUE_SIZE 8 | ||
93 | 101 | ||
94 | struct _ToxAv { | 102 | struct _ToxAv { |
95 | Messenger *messenger; | 103 | Messenger *messenger; |
@@ -100,8 +108,18 @@ struct _ToxAv { | |||
100 | void (*video_callback)(ToxAv *, int32_t, vpx_image_t *); | 108 | void (*video_callback)(ToxAv *, int32_t, vpx_image_t *); |
101 | 109 | ||
102 | uint32_t max_calls; | 110 | uint32_t max_calls; |
111 | |||
112 | /* used in the "decode on another thread" system */ | ||
113 | volatile _Bool exit, decoding; | ||
114 | uint8_t video_decode_read, video_decode_write, audio_decode_read, audio_decode_write; | ||
115 | pthread_mutex_t decode_cond_mutex; | ||
116 | pthread_cond_t decode_cond; | ||
117 | DECODE_PACKET *volatile video_decode_queue[VIDEO_DECODE_QUEUE_SIZE]; | ||
118 | DECODE_PACKET *volatile audio_decode_queue[AUDIO_DECODE_QUEUE_SIZE]; | ||
103 | }; | 119 | }; |
104 | 120 | ||
121 | static void *toxav_decoding(void *arg); | ||
122 | |||
105 | static MSICSettings msicsettings_cast (const ToxAvCSettings *from) | 123 | static MSICSettings msicsettings_cast (const ToxAvCSettings *from) |
106 | { | 124 | { |
107 | MSICSettings csettings; | 125 | MSICSettings csettings; |
@@ -162,6 +180,12 @@ ToxAv *toxav_new( Tox *messenger, int32_t max_calls) | |||
162 | av->calls = calloc(sizeof(CallSpecific), max_calls); | 180 | av->calls = calloc(sizeof(CallSpecific), max_calls); |
163 | av->max_calls = max_calls; | 181 | av->max_calls = max_calls; |
164 | 182 | ||
183 | pthread_mutex_init(&av->decode_cond_mutex, NULL); | ||
184 | pthread_cond_init(&av->decode_cond, NULL); | ||
185 | |||
186 | pthread_t temp; | ||
187 | pthread_create(&temp, NULL, toxav_decoding, av); | ||
188 | |||
165 | return av; | 189 | return av; |
166 | } | 190 | } |
167 | 191 | ||
@@ -173,9 +197,39 @@ ToxAv *toxav_new( Tox *messenger, int32_t max_calls) | |||
173 | */ | 197 | */ |
174 | void toxav_kill ( ToxAv *av ) | 198 | void toxav_kill ( ToxAv *av ) |
175 | { | 199 | { |
176 | int i = 0; | 200 | int i; |
201 | DECODE_PACKET *p; | ||
202 | |||
203 | av->exit = 1; | ||
204 | pthread_mutex_lock(&av->decode_cond_mutex); | ||
205 | pthread_cond_signal(&av->decode_cond); | ||
177 | 206 | ||
178 | for (; i < av->max_calls; i ++) { | 207 | if (av->exit) { |
208 | pthread_cond_wait(&av->decode_cond, &av->decode_cond_mutex); | ||
209 | } | ||
210 | |||
211 | pthread_mutex_unlock(&av->decode_cond_mutex); | ||
212 | |||
213 | pthread_mutex_destroy(&av->decode_cond_mutex); | ||
214 | pthread_cond_destroy(&av->decode_cond); | ||
215 | |||
216 | for (i = 0; i != VIDEO_DECODE_QUEUE_SIZE; i++) { | ||
217 | p = av->video_decode_queue[i]; | ||
218 | |||
219 | if (p) { | ||
220 | free(p); | ||
221 | } | ||
222 | } | ||
223 | |||
224 | for (i = 0; i != AUDIO_DECODE_QUEUE_SIZE; i++) { | ||
225 | p = av->audio_decode_queue[i]; | ||
226 | |||
227 | if (p) { | ||
228 | free(p); | ||
229 | } | ||
230 | } | ||
231 | |||
232 | for (i = 0; i < av->max_calls; i ++) { | ||
179 | if ( av->calls[i].crtps[audio_index] ) | 233 | if ( av->calls[i].crtps[audio_index] ) |
180 | rtp_terminate_session(av->calls[i].crtps[audio_index], av->msi_session->messenger_handle); | 234 | rtp_terminate_session(av->calls[i].crtps[audio_index], av->msi_session->messenger_handle); |
181 | 235 | ||
@@ -514,9 +568,37 @@ int toxav_kill_transmission ( ToxAv *av, int32_t call_index ) | |||
514 | call->crtps[video_index] = NULL; | 568 | call->crtps[video_index] = NULL; |
515 | terminate_queue(call->j_buf); | 569 | terminate_queue(call->j_buf); |
516 | call->j_buf = NULL; | 570 | call->j_buf = NULL; |
571 | |||
572 | |||
573 | pthread_mutex_lock(&av->decode_cond_mutex); | ||
574 | int i; | ||
575 | DECODE_PACKET *p; | ||
576 | |||
577 | for (i = 0; i != VIDEO_DECODE_QUEUE_SIZE; i++) { | ||
578 | p = av->video_decode_queue[i]; | ||
579 | |||
580 | if (p && p->call_index == call_index) { | ||
581 | free(p); | ||
582 | av->video_decode_queue[i] = NULL; | ||
583 | } | ||
584 | } | ||
585 | |||
586 | for (i = 0; i != AUDIO_DECODE_QUEUE_SIZE; i++) { | ||
587 | p = av->audio_decode_queue[i]; | ||
588 | |||
589 | if (p && p->call_index == call_index) { | ||
590 | free(p); | ||
591 | av->audio_decode_queue[i] = NULL; | ||
592 | } | ||
593 | } | ||
594 | |||
595 | while (av->decoding) {} //use a pthread condition? | ||
596 | |||
517 | codec_terminate_session(call->cs); | 597 | codec_terminate_session(call->cs); |
518 | call->cs = NULL; | 598 | call->cs = NULL; |
519 | 599 | ||
600 | pthread_mutex_unlock(&av->decode_cond_mutex); | ||
601 | |||
520 | pthread_mutex_unlock(&call->mutex); | 602 | pthread_mutex_unlock(&call->mutex); |
521 | pthread_mutex_destroy(&call->mutex); | 603 | pthread_mutex_destroy(&call->mutex); |
522 | 604 | ||
@@ -835,6 +917,120 @@ int toxav_has_activity(ToxAv *av, int32_t call_index, int16_t *PCM, uint16_t fra | |||
835 | return energy_VAD(av->calls[call_index].cs, PCM, frame_size, ref_energy); | 917 | return energy_VAD(av->calls[call_index].cs, PCM, frame_size, ref_energy); |
836 | } | 918 | } |
837 | 919 | ||
920 | |||
921 | static void decode_video(ToxAv *av, DECODE_PACKET *p) | ||
922 | { | ||
923 | CallSpecific *call = &av->calls[p->call_index]; | ||
924 | |||
925 | int rc = vpx_codec_decode(&call->cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US); | ||
926 | |||
927 | if (rc != VPX_CODEC_OK) { | ||
928 | LOGGER_ERROR("Error decoding video: %u %s\n", i, vpx_codec_err_to_string(rc)); | ||
929 | } | ||
930 | |||
931 | vpx_codec_iter_t iter = NULL; | ||
932 | vpx_image_t *img; | ||
933 | img = vpx_codec_get_frame(&call->cs->v_decoder, &iter); | ||
934 | |||
935 | if (img && av->video_callback) { | ||
936 | av->video_callback(av, p->call_index, img); | ||
937 | } else { | ||
938 | LOGGER_WARNING("Video packet dropped due to missing callback or no image!"); | ||
939 | } | ||
940 | |||
941 | free(p); | ||
942 | } | ||
943 | |||
944 | static void decode_audio(ToxAv *av, DECODE_PACKET *p) | ||
945 | { | ||
946 | int32_t call_index = p->call_index; | ||
947 | CallSpecific *call = &av->calls[call_index]; | ||
948 | |||
949 | // ToxAvCSettings csettings; | ||
950 | // toxav_get_peer_csettings(av, call_index, 0, &csettings); | ||
951 | |||
952 | int frame_size = 10000; /* FIXME: not static? */ | ||
953 | int16_t dest[frame_size]; | ||
954 | |||
955 | int dec_size = opus_decode(call->cs->audio_decoder, p->data, p->size, dest, frame_size, (p->size == 0)); | ||
956 | free(p); | ||
957 | |||
958 | if (dec_size < 0) { | ||
959 | LOGGER_WARNING("Decoding error: %s", opus_strerror(dec_size)); | ||
960 | return; | ||
961 | } | ||
962 | |||
963 | if ( av->audio_callback ) | ||
964 | av->audio_callback(av, call_index, dest, dec_size); | ||
965 | else | ||
966 | LOGGER_WARNING("Audio packet dropped due to missing callback!"); | ||
967 | } | ||
968 | |||
969 | static void *toxav_decoding(void *arg) | ||
970 | { | ||
971 | ToxAv *av = arg; | ||
972 | |||
973 | while (!av->exit) { | ||
974 | DECODE_PACKET *p; | ||
975 | _Bool video = 0; | ||
976 | |||
977 | av->decoding = 0; | ||
978 | pthread_mutex_lock(&av->decode_cond_mutex); | ||
979 | uint8_t r; | ||
980 | |||
981 | /* first check for available packets, otherwise wait for condition*/ | ||
982 | r = av->audio_decode_read; | ||
983 | p = av->audio_decode_queue[r]; | ||
984 | |||
985 | if (!p) { | ||
986 | r = av->video_decode_read; | ||
987 | p = av->video_decode_queue[r]; | ||
988 | |||
989 | if (!p) { | ||
990 | pthread_cond_wait(&av->decode_cond, &av->decode_cond_mutex); | ||
991 | r = av->audio_decode_read; | ||
992 | p = av->audio_decode_queue[r]; | ||
993 | |||
994 | if (!p) { | ||
995 | r = av->video_decode_read; | ||
996 | p = av->video_decode_queue[r]; | ||
997 | video = 1; | ||
998 | } | ||
999 | } else { | ||
1000 | video = 1; | ||
1001 | } | ||
1002 | } | ||
1003 | |||
1004 | if (video) { | ||
1005 | if (p) { | ||
1006 | av->video_decode_queue[r] = NULL; | ||
1007 | av->video_decode_read = (r + 1) % VIDEO_DECODE_QUEUE_SIZE; | ||
1008 | } | ||
1009 | } else { | ||
1010 | av->audio_decode_queue[r] = NULL; | ||
1011 | av->audio_decode_read = (r + 1) % AUDIO_DECODE_QUEUE_SIZE; | ||
1012 | } | ||
1013 | |||
1014 | av->decoding = 1; | ||
1015 | pthread_mutex_unlock(&av->decode_cond_mutex); | ||
1016 | |||
1017 | if (p) { | ||
1018 | if (video) { | ||
1019 | decode_video(av, p); | ||
1020 | } else { | ||
1021 | decode_audio(av, p); | ||
1022 | } | ||
1023 | } | ||
1024 | } | ||
1025 | |||
1026 | pthread_mutex_lock(&av->decode_cond_mutex); | ||
1027 | av->exit = 0; | ||
1028 | pthread_cond_signal(&av->decode_cond); | ||
1029 | pthread_mutex_unlock(&av->decode_cond_mutex); | ||
1030 | |||
1031 | return NULL; | ||
1032 | } | ||
1033 | |||
838 | void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg) | 1034 | void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg) |
839 | { | 1035 | { |
840 | ToxAv *av = _session->av; | 1036 | ToxAv *av = _session->av; |
@@ -846,31 +1042,48 @@ void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg) | |||
846 | if (_session->payload_type == type_audio % 128) { | 1042 | if (_session->payload_type == type_audio % 128) { |
847 | queue(call->j_buf, _msg); | 1043 | queue(call->j_buf, _msg); |
848 | 1044 | ||
849 | int success = 0, dec_size; | 1045 | int success = 0; |
850 | |||
851 | ToxAvCSettings csettings; | ||
852 | toxav_get_peer_csettings(av, call_index, 0, &csettings); | ||
853 | |||
854 | int frame_size = 10000; /* FIXME: not static? */ | ||
855 | int16_t dest[frame_size]; | ||
856 | 1046 | ||
857 | while ((_msg = dequeue(call->j_buf, &success)) || success == 2) { | 1047 | while ((_msg = dequeue(call->j_buf, &success)) || success == 2) { |
1048 | DECODE_PACKET *p; | ||
1049 | |||
858 | if (success == 2) { | 1050 | if (success == 2) { |
859 | dec_size = opus_decode(call->cs->audio_decoder, NULL, 0, dest, frame_size, 1); | 1051 | p = malloc(sizeof(DECODE_PACKET)); |
1052 | |||
1053 | if (p) { | ||
1054 | p->call_index = call_index; | ||
1055 | p->size = 0; | ||
1056 | } | ||
860 | } else { | 1057 | } else { |
861 | dec_size = opus_decode(call->cs->audio_decoder, _msg->data, _msg->length, dest, frame_size, 0); | 1058 | p = malloc(sizeof(DECODE_PACKET) + _msg->length); |
1059 | |||
1060 | if (p) { | ||
1061 | p->call_index = call_index; | ||
1062 | p->size = _msg->length; | ||
1063 | memcpy(p->data, _msg->data, _msg->length); | ||
1064 | } | ||
1065 | |||
862 | rtp_free_msg(NULL, _msg); | 1066 | rtp_free_msg(NULL, _msg); |
863 | } | 1067 | } |
864 | 1068 | ||
865 | if (dec_size < 0) { | 1069 | if (p) { |
866 | LOGGER_WARNING("Decoding error: %s", opus_strerror(dec_size)); | 1070 | /* do the decoding on another thread */ |
867 | continue; | 1071 | pthread_mutex_lock(&av->decode_cond_mutex); |
868 | } | 1072 | uint8_t w = av->audio_decode_write; |
1073 | |||
1074 | if (av->audio_decode_queue[w] == NULL) { | ||
1075 | av->audio_decode_queue[w] = p; | ||
1076 | av->audio_decode_write = (w + 1) % AUDIO_DECODE_QUEUE_SIZE; | ||
1077 | pthread_cond_signal(&av->decode_cond); | ||
1078 | } else { | ||
1079 | printf("dropped audio frame\n"); | ||
1080 | free(p); | ||
1081 | } | ||
869 | 1082 | ||
870 | if ( av->audio_callback ) | 1083 | pthread_mutex_unlock(&av->decode_cond_mutex); |
871 | av->audio_callback(av, call_index, dest, dec_size); | 1084 | } else { |
872 | else | 1085 | //malloc failed |
873 | LOGGER_WARNING("Audio packet dropped due to missing callback!"); | 1086 | } |
874 | } | 1087 | } |
875 | 1088 | ||
876 | } else { | 1089 | } else { |
@@ -887,14 +1100,34 @@ void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg) | |||
887 | /* piece of current frame */ | 1100 | /* piece of current frame */ |
888 | } else if (i > 0 && i < 128) { | 1101 | } else if (i > 0 && i < 128) { |
889 | /* recieved a piece of a frame ahead, flush current frame and start reading this new frame */ | 1102 | /* recieved a piece of a frame ahead, flush current frame and start reading this new frame */ |
890 | int rc = vpx_codec_decode(&call->cs->v_decoder, call->frame_buf, call->frame_limit, NULL, MAX_DECODE_TIME_US); | 1103 | DECODE_PACKET *p = malloc(sizeof(DECODE_PACKET) + call->frame_limit); |
1104 | |||
1105 | if (p) { | ||
1106 | p->call_index = call_index; | ||
1107 | p->size = call->frame_limit; | ||
1108 | memcpy(p->data, call->frame_buf, call->frame_limit); | ||
1109 | |||
1110 | /* do the decoding on another thread */ | ||
1111 | pthread_mutex_lock(&av->decode_cond_mutex); | ||
1112 | uint8_t w = av->video_decode_write; | ||
1113 | |||
1114 | if (av->video_decode_queue[w] == NULL) { | ||
1115 | av->video_decode_queue[w] = p; | ||
1116 | av->video_decode_write = (w + 1) % VIDEO_DECODE_QUEUE_SIZE; | ||
1117 | pthread_cond_signal(&av->decode_cond); | ||
1118 | } else { | ||
1119 | printf("dropped video frame\n"); | ||
1120 | free(p); | ||
1121 | } | ||
1122 | |||
1123 | pthread_mutex_unlock(&av->decode_cond_mutex); | ||
1124 | } else { | ||
1125 | //malloc failed | ||
1126 | } | ||
1127 | |||
891 | call->frame_id = packet[0]; | 1128 | call->frame_id = packet[0]; |
892 | memset(call->frame_buf, 0, call->frame_limit); | 1129 | memset(call->frame_buf, 0, call->frame_limit); |
893 | call->frame_limit = 0; | 1130 | call->frame_limit = 0; |
894 | |||
895 | if (rc != VPX_CODEC_OK) { | ||
896 | LOGGER_ERROR("Error decoding video: %u %s\n", i, vpx_codec_err_to_string(rc)); | ||
897 | } | ||
898 | } else { | 1131 | } else { |
899 | /* old packet, dont read */ | 1132 | /* old packet, dont read */ |
900 | LOGGER_DEBUG("Old packet: %u\n", i); | 1133 | LOGGER_DEBUG("Old packet: %u\n", i); |
@@ -919,15 +1152,6 @@ void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg) | |||
919 | 1152 | ||
920 | end: | 1153 | end: |
921 | ; | 1154 | ; |
922 | vpx_codec_iter_t iter = NULL; | ||
923 | vpx_image_t *img; | ||
924 | img = vpx_codec_get_frame(&call->cs->v_decoder, &iter); | ||
925 | |||
926 | if (img && av->video_callback) { | ||
927 | av->video_callback(av, call_index, img); | ||
928 | } else | ||
929 | LOGGER_WARNING("Video packet dropped due to missing callback or no image!"); | ||
930 | |||
931 | rtp_free_msg(NULL, _msg); | 1155 | rtp_free_msg(NULL, _msg); |
932 | } | 1156 | } |
933 | } | 1157 | } |