summaryrefslogtreecommitdiff
path: root/toxav/toxav.c
diff options
context:
space:
mode:
Diffstat (limited to 'toxav/toxav.c')
-rw-r--r--toxav/toxav.c290
1 files changed, 257 insertions, 33 deletions
diff --git a/toxav/toxav.c b/toxav/toxav.c
index 0aa88919..75e93bac 100644
--- a/toxav/toxav.c
+++ b/toxav/toxav.c
@@ -90,6 +90,14 @@ typedef struct _CallSpecific {
90 pthread_mutex_t mutex; 90 pthread_mutex_t mutex;
91} CallSpecific; 91} CallSpecific;
92 92
93typedef struct {
94 int32_t call_index;
95 uint32_t size;
96 uint8_t data[0];
97} DECODE_PACKET;
98
99#define VIDEO_DECODE_QUEUE_SIZE 2
100#define AUDIO_DECODE_QUEUE_SIZE 8
93 101
94struct _ToxAv { 102struct _ToxAv {
95 Messenger *messenger; 103 Messenger *messenger;
@@ -100,8 +108,18 @@ struct _ToxAv {
100 void (*video_callback)(ToxAv *, int32_t, vpx_image_t *); 108 void (*video_callback)(ToxAv *, int32_t, vpx_image_t *);
101 109
102 uint32_t max_calls; 110 uint32_t max_calls;
111
112 /* used in the "decode on another thread" system */
113 volatile _Bool exit, decoding;
114 uint8_t video_decode_read, video_decode_write, audio_decode_read, audio_decode_write;
115 pthread_mutex_t decode_cond_mutex;
116 pthread_cond_t decode_cond;
117 DECODE_PACKET *volatile video_decode_queue[VIDEO_DECODE_QUEUE_SIZE];
118 DECODE_PACKET *volatile audio_decode_queue[AUDIO_DECODE_QUEUE_SIZE];
103}; 119};
104 120
121static void *toxav_decoding(void *arg);
122
105static MSICSettings msicsettings_cast (const ToxAvCSettings *from) 123static MSICSettings msicsettings_cast (const ToxAvCSettings *from)
106{ 124{
107 MSICSettings csettings; 125 MSICSettings csettings;
@@ -162,6 +180,12 @@ ToxAv *toxav_new( Tox *messenger, int32_t max_calls)
162 av->calls = calloc(sizeof(CallSpecific), max_calls); 180 av->calls = calloc(sizeof(CallSpecific), max_calls);
163 av->max_calls = max_calls; 181 av->max_calls = max_calls;
164 182
183 pthread_mutex_init(&av->decode_cond_mutex, NULL);
184 pthread_cond_init(&av->decode_cond, NULL);
185
186 pthread_t temp;
187 pthread_create(&temp, NULL, toxav_decoding, av);
188
165 return av; 189 return av;
166} 190}
167 191
@@ -173,9 +197,39 @@ ToxAv *toxav_new( Tox *messenger, int32_t max_calls)
173 */ 197 */
174void toxav_kill ( ToxAv *av ) 198void toxav_kill ( ToxAv *av )
175{ 199{
176 int i = 0; 200 int i;
201 DECODE_PACKET *p;
202
203 av->exit = 1;
204 pthread_mutex_lock(&av->decode_cond_mutex);
205 pthread_cond_signal(&av->decode_cond);
177 206
178 for (; i < av->max_calls; i ++) { 207 if (av->exit) {
208 pthread_cond_wait(&av->decode_cond, &av->decode_cond_mutex);
209 }
210
211 pthread_mutex_unlock(&av->decode_cond_mutex);
212
213 pthread_mutex_destroy(&av->decode_cond_mutex);
214 pthread_cond_destroy(&av->decode_cond);
215
216 for (i = 0; i != VIDEO_DECODE_QUEUE_SIZE; i++) {
217 p = av->video_decode_queue[i];
218
219 if (p) {
220 free(p);
221 }
222 }
223
224 for (i = 0; i != AUDIO_DECODE_QUEUE_SIZE; i++) {
225 p = av->audio_decode_queue[i];
226
227 if (p) {
228 free(p);
229 }
230 }
231
232 for (i = 0; i < av->max_calls; i ++) {
179 if ( av->calls[i].crtps[audio_index] ) 233 if ( av->calls[i].crtps[audio_index] )
180 rtp_terminate_session(av->calls[i].crtps[audio_index], av->msi_session->messenger_handle); 234 rtp_terminate_session(av->calls[i].crtps[audio_index], av->msi_session->messenger_handle);
181 235
@@ -514,9 +568,37 @@ int toxav_kill_transmission ( ToxAv *av, int32_t call_index )
514 call->crtps[video_index] = NULL; 568 call->crtps[video_index] = NULL;
515 terminate_queue(call->j_buf); 569 terminate_queue(call->j_buf);
516 call->j_buf = NULL; 570 call->j_buf = NULL;
571
572
573 pthread_mutex_lock(&av->decode_cond_mutex);
574 int i;
575 DECODE_PACKET *p;
576
577 for (i = 0; i != VIDEO_DECODE_QUEUE_SIZE; i++) {
578 p = av->video_decode_queue[i];
579
580 if (p && p->call_index == call_index) {
581 free(p);
582 av->video_decode_queue[i] = NULL;
583 }
584 }
585
586 for (i = 0; i != AUDIO_DECODE_QUEUE_SIZE; i++) {
587 p = av->audio_decode_queue[i];
588
589 if (p && p->call_index == call_index) {
590 free(p);
591 av->audio_decode_queue[i] = NULL;
592 }
593 }
594
595 while (av->decoding) {} //use a pthread condition?
596
517 codec_terminate_session(call->cs); 597 codec_terminate_session(call->cs);
518 call->cs = NULL; 598 call->cs = NULL;
519 599
600 pthread_mutex_unlock(&av->decode_cond_mutex);
601
520 pthread_mutex_unlock(&call->mutex); 602 pthread_mutex_unlock(&call->mutex);
521 pthread_mutex_destroy(&call->mutex); 603 pthread_mutex_destroy(&call->mutex);
522 604
@@ -835,6 +917,120 @@ int toxav_has_activity(ToxAv *av, int32_t call_index, int16_t *PCM, uint16_t fra
835 return energy_VAD(av->calls[call_index].cs, PCM, frame_size, ref_energy); 917 return energy_VAD(av->calls[call_index].cs, PCM, frame_size, ref_energy);
836} 918}
837 919
920
921static void decode_video(ToxAv *av, DECODE_PACKET *p)
922{
923 CallSpecific *call = &av->calls[p->call_index];
924
925 int rc = vpx_codec_decode(&call->cs->v_decoder, p->data, p->size, NULL, MAX_DECODE_TIME_US);
926
927 if (rc != VPX_CODEC_OK) {
928 LOGGER_ERROR("Error decoding video: %u %s\n", i, vpx_codec_err_to_string(rc));
929 }
930
931 vpx_codec_iter_t iter = NULL;
932 vpx_image_t *img;
933 img = vpx_codec_get_frame(&call->cs->v_decoder, &iter);
934
935 if (img && av->video_callback) {
936 av->video_callback(av, p->call_index, img);
937 } else {
938 LOGGER_WARNING("Video packet dropped due to missing callback or no image!");
939 }
940
941 free(p);
942}
943
944static void decode_audio(ToxAv *av, DECODE_PACKET *p)
945{
946 int32_t call_index = p->call_index;
947 CallSpecific *call = &av->calls[call_index];
948
949 // ToxAvCSettings csettings;
950 // toxav_get_peer_csettings(av, call_index, 0, &csettings);
951
952 int frame_size = 10000; /* FIXME: not static? */
953 int16_t dest[frame_size];
954
955 int dec_size = opus_decode(call->cs->audio_decoder, p->data, p->size, dest, frame_size, (p->size == 0));
956 free(p);
957
958 if (dec_size < 0) {
959 LOGGER_WARNING("Decoding error: %s", opus_strerror(dec_size));
960 return;
961 }
962
963 if ( av->audio_callback )
964 av->audio_callback(av, call_index, dest, dec_size);
965 else
966 LOGGER_WARNING("Audio packet dropped due to missing callback!");
967}
968
969static void *toxav_decoding(void *arg)
970{
971 ToxAv *av = arg;
972
973 while (!av->exit) {
974 DECODE_PACKET *p;
975 _Bool video = 0;
976
977 av->decoding = 0;
978 pthread_mutex_lock(&av->decode_cond_mutex);
979 uint8_t r;
980
981 /* first check for available packets, otherwise wait for condition*/
982 r = av->audio_decode_read;
983 p = av->audio_decode_queue[r];
984
985 if (!p) {
986 r = av->video_decode_read;
987 p = av->video_decode_queue[r];
988
989 if (!p) {
990 pthread_cond_wait(&av->decode_cond, &av->decode_cond_mutex);
991 r = av->audio_decode_read;
992 p = av->audio_decode_queue[r];
993
994 if (!p) {
995 r = av->video_decode_read;
996 p = av->video_decode_queue[r];
997 video = 1;
998 }
999 } else {
1000 video = 1;
1001 }
1002 }
1003
1004 if (video) {
1005 if (p) {
1006 av->video_decode_queue[r] = NULL;
1007 av->video_decode_read = (r + 1) % VIDEO_DECODE_QUEUE_SIZE;
1008 }
1009 } else {
1010 av->audio_decode_queue[r] = NULL;
1011 av->audio_decode_read = (r + 1) % AUDIO_DECODE_QUEUE_SIZE;
1012 }
1013
1014 av->decoding = 1;
1015 pthread_mutex_unlock(&av->decode_cond_mutex);
1016
1017 if (p) {
1018 if (video) {
1019 decode_video(av, p);
1020 } else {
1021 decode_audio(av, p);
1022 }
1023 }
1024 }
1025
1026 pthread_mutex_lock(&av->decode_cond_mutex);
1027 av->exit = 0;
1028 pthread_cond_signal(&av->decode_cond);
1029 pthread_mutex_unlock(&av->decode_cond_mutex);
1030
1031 return NULL;
1032}
1033
838void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg) 1034void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg)
839{ 1035{
840 ToxAv *av = _session->av; 1036 ToxAv *av = _session->av;
@@ -846,31 +1042,48 @@ void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg)
846 if (_session->payload_type == type_audio % 128) { 1042 if (_session->payload_type == type_audio % 128) {
847 queue(call->j_buf, _msg); 1043 queue(call->j_buf, _msg);
848 1044
849 int success = 0, dec_size; 1045 int success = 0;
850
851 ToxAvCSettings csettings;
852 toxav_get_peer_csettings(av, call_index, 0, &csettings);
853
854 int frame_size = 10000; /* FIXME: not static? */
855 int16_t dest[frame_size];
856 1046
857 while ((_msg = dequeue(call->j_buf, &success)) || success == 2) { 1047 while ((_msg = dequeue(call->j_buf, &success)) || success == 2) {
1048 DECODE_PACKET *p;
1049
858 if (success == 2) { 1050 if (success == 2) {
859 dec_size = opus_decode(call->cs->audio_decoder, NULL, 0, dest, frame_size, 1); 1051 p = malloc(sizeof(DECODE_PACKET));
1052
1053 if (p) {
1054 p->call_index = call_index;
1055 p->size = 0;
1056 }
860 } else { 1057 } else {
861 dec_size = opus_decode(call->cs->audio_decoder, _msg->data, _msg->length, dest, frame_size, 0); 1058 p = malloc(sizeof(DECODE_PACKET) + _msg->length);
1059
1060 if (p) {
1061 p->call_index = call_index;
1062 p->size = _msg->length;
1063 memcpy(p->data, _msg->data, _msg->length);
1064 }
1065
862 rtp_free_msg(NULL, _msg); 1066 rtp_free_msg(NULL, _msg);
863 } 1067 }
864 1068
865 if (dec_size < 0) { 1069 if (p) {
866 LOGGER_WARNING("Decoding error: %s", opus_strerror(dec_size)); 1070 /* do the decoding on another thread */
867 continue; 1071 pthread_mutex_lock(&av->decode_cond_mutex);
868 } 1072 uint8_t w = av->audio_decode_write;
1073
1074 if (av->audio_decode_queue[w] == NULL) {
1075 av->audio_decode_queue[w] = p;
1076 av->audio_decode_write = (w + 1) % AUDIO_DECODE_QUEUE_SIZE;
1077 pthread_cond_signal(&av->decode_cond);
1078 } else {
1079 printf("dropped audio frame\n");
1080 free(p);
1081 }
869 1082
870 if ( av->audio_callback ) 1083 pthread_mutex_unlock(&av->decode_cond_mutex);
871 av->audio_callback(av, call_index, dest, dec_size); 1084 } else {
872 else 1085 //malloc failed
873 LOGGER_WARNING("Audio packet dropped due to missing callback!"); 1086 }
874 } 1087 }
875 1088
876 } else { 1089 } else {
@@ -887,14 +1100,34 @@ void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg)
887 /* piece of current frame */ 1100 /* piece of current frame */
888 } else if (i > 0 && i < 128) { 1101 } else if (i > 0 && i < 128) {
889 /* recieved a piece of a frame ahead, flush current frame and start reading this new frame */ 1102 /* recieved a piece of a frame ahead, flush current frame and start reading this new frame */
890 int rc = vpx_codec_decode(&call->cs->v_decoder, call->frame_buf, call->frame_limit, NULL, MAX_DECODE_TIME_US); 1103 DECODE_PACKET *p = malloc(sizeof(DECODE_PACKET) + call->frame_limit);
1104
1105 if (p) {
1106 p->call_index = call_index;
1107 p->size = call->frame_limit;
1108 memcpy(p->data, call->frame_buf, call->frame_limit);
1109
1110 /* do the decoding on another thread */
1111 pthread_mutex_lock(&av->decode_cond_mutex);
1112 uint8_t w = av->video_decode_write;
1113
1114 if (av->video_decode_queue[w] == NULL) {
1115 av->video_decode_queue[w] = p;
1116 av->video_decode_write = (w + 1) % VIDEO_DECODE_QUEUE_SIZE;
1117 pthread_cond_signal(&av->decode_cond);
1118 } else {
1119 printf("dropped video frame\n");
1120 free(p);
1121 }
1122
1123 pthread_mutex_unlock(&av->decode_cond_mutex);
1124 } else {
1125 //malloc failed
1126 }
1127
891 call->frame_id = packet[0]; 1128 call->frame_id = packet[0];
892 memset(call->frame_buf, 0, call->frame_limit); 1129 memset(call->frame_buf, 0, call->frame_limit);
893 call->frame_limit = 0; 1130 call->frame_limit = 0;
894
895 if (rc != VPX_CODEC_OK) {
896 LOGGER_ERROR("Error decoding video: %u %s\n", i, vpx_codec_err_to_string(rc));
897 }
898 } else { 1131 } else {
899 /* old packet, dont read */ 1132 /* old packet, dont read */
900 LOGGER_DEBUG("Old packet: %u\n", i); 1133 LOGGER_DEBUG("Old packet: %u\n", i);
@@ -919,15 +1152,6 @@ void toxav_handle_packet(RTPSession *_session, RTPMessage *_msg)
919 1152
920end: 1153end:
921 ; 1154 ;
922 vpx_codec_iter_t iter = NULL;
923 vpx_image_t *img;
924 img = vpx_codec_get_frame(&call->cs->v_decoder, &iter);
925
926 if (img && av->video_callback) {
927 av->video_callback(av, call_index, img);
928 } else
929 LOGGER_WARNING("Video packet dropped due to missing callback or no image!");
930
931 rtp_free_msg(NULL, _msg); 1155 rtp_free_msg(NULL, _msg);
932 } 1156 }
933} 1157}