Improve video key frame sending.

This change does not include the addition of VP9. We do that in a
separate pull request.

Changes:

* fix the video bug (video frames larger than 65KBytes) by sending full
  frame length in alternate header field
* improve video frame reconstruction logic with slots
* configure video encoder and decoder to be multihtreaded
* set error resilience flags on video codec
* change encoder and decoder softdeadline

1 files changed, 67 insertions, 23 deletions

diff --git a/toxav/toxav.c b/toxav/toxav.c
index c48fc192..4e45b747 100644
--- a/toxav/toxav.c
+++ b/toxav/toxav.c
@@ -36,7 +36,18 @@
 #include <stdlib.h>
 #include <string.h>
 
-#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000)
+// TODO: don't hardcode this, let the application choose it
+// VPX Info: Time to spend encoding, in microseconds (it's a *soft* deadline)
+#define WANTED_MAX_ENCODER_FPS (40)
+#define MAX_ENCODE_TIME_US (1000000 / WANTED_MAX_ENCODER_FPS) // to allow x fps
+
+#define VIDEO_SEND_X_KEYFRAMES_FIRST 7 // force the first n frames to be keyframes!
+
+/*
+VPX_DL_REALTIME       (1)       deadline parameter analogous to VPx REALTIME mode.
+VPX_DL_GOOD_QUALITY   (1000000) deadline parameter analogous to VPx GOOD QUALITY mode.
+VPX_DL_BEST_QUALITY   (0)       deadline parameter analogous to VPx BEST QUALITY mode.
+*/
 
 typedef struct ToxAVCall_s {
     ToxAV *av;
@@ -752,13 +763,12 @@ bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pc
             goto END;
         }
 
-        if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), av->m->log) != 0) {
+        if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), false, av->m->log) != 0) {
             LOGGER_WARNING(av->m->log, "Failed to send audio packet");
             rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
         }
     }
 
-
     pthread_mutex_unlock(call->mutex_audio);
 
 END:
@@ -769,12 +779,15 @@ END:
 
     return rc == TOXAV_ERR_SEND_FRAME_OK;
 }
+
 bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y,
                             const uint8_t *u, const uint8_t *v, TOXAV_ERR_SEND_FRAME *error)
 {
     TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK;
     ToxAVCall *call;
 
+    int vpx_encode_flags = 0;
+
     if (m_friend_exists(av->m, friend_number) == 0) {
         rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND;
         goto END;
@@ -810,12 +823,28 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u
         goto END;
     }
 
-    if (vc_reconfigure_encoder(call->video.second, call->video_bit_rate * 1000, width, height) != 0) {
+    if (vc_reconfigure_encoder(call->video.second, call->video_bit_rate * 1000, width, height, -1) != 0) {
         pthread_mutex_unlock(call->mutex_video);
         rc = TOXAV_ERR_SEND_FRAME_INVALID;
         goto END;
     }
 
+    if (call->video.first->ssrc < VIDEO_SEND_X_KEYFRAMES_FIRST) {
+        // Key frame flag for first frames
+        vpx_encode_flags = VPX_EFLAG_FORCE_KF;
+        LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d only-i-frame mode", call->video.first->ssrc);
+
+        call->video.first->ssrc++;
+    } else if (call->video.first->ssrc == VIDEO_SEND_X_KEYFRAMES_FIRST) {
+        // normal keyframe placement
+        vpx_encode_flags = 0;
+        LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d normal mode", call->video.first->ssrc);
+
+        call->video.first->ssrc++;
+    }
+
+    // we start with I-frames (full frames) and then switch to normal mode later
+
     { /* Encode */
         vpx_image_t img;
         img.w = img.h = img.d_w = img.d_h = 0;
@@ -829,7 +858,7 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u
         memcpy(img.planes[VPX_PLANE_V], v, (width / 2) * (height / 2));
 
         vpx_codec_err_t vrc = vpx_codec_encode(call->video.second->encoder, &img,
-                                               call->video.second->frame_counter, 1, 0, MAX_ENCODE_TIME_US);
+                                               call->video.second->frame_counter, 1, vpx_encode_flags, MAX_ENCODE_TIME_US);
 
         vpx_img_free(&img);
 
@@ -847,22 +876,31 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u
         vpx_codec_iter_t iter = nullptr;
         const vpx_codec_cx_pkt_t *pkt;
 
-        while ((pkt = vpx_codec_get_cx_data(call->video.second->encoder, &iter))) {
+        while ((pkt = vpx_codec_get_cx_data(call->video.second->encoder, &iter)) != nullptr) {
             if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
-                const uint8_t *buf = (const uint8_t *)pkt->data.frame.buf;
-                const uint8_t *end = buf + pkt->data.frame.sz;
-
-                while (buf < end) {
-                    uint16_t size = MIN(UINT16_MAX, end - buf);
-
-                    if (rtp_send_data(call->video.first, buf, size, av->m->log) < 0) {
-                        pthread_mutex_unlock(call->mutex_video);
-                        LOGGER_WARNING(av->m->log, "Could not send video frame: %s\n", strerror(errno));
-                        rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
-                        goto END;
-                    }
-
-                    buf += size;
+                const bool is_keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
+
+                // https://www.webmproject.org/docs/webm-sdk/structvpx__codec__cx__pkt.html
+                // pkt->data.frame.sz -> size_t
+                const uint32_t frame_length_in_bytes = pkt->data.frame.sz;
+
+                const int res = rtp_send_data(
+                                    call->video.first,
+                                    (const uint8_t *)pkt->data.frame.buf,
+                                    frame_length_in_bytes,
+                                    is_keyframe,
+                                    av->m->log);
+
+                LOGGER_DEBUG(av->m->log, "+ _sending_FRAME_TYPE_==%s bytes=%d frame_len=%d", is_keyframe ? "K" : ".",
+                             (int)pkt->data.frame.sz, (int)frame_length_in_bytes);
+                LOGGER_DEBUG(av->m->log, "+ _sending_FRAME_ b0=%d b1=%d", ((const uint8_t *)pkt->data.frame.buf)[0],
+                             ((const uint8_t *)pkt->data.frame.buf)[1]);
+
+                if (res < 0) {
+                    pthread_mutex_unlock(call->mutex_video);
+                    LOGGER_WARNING(av->m->log, "Could not send video frame: %s", strerror(errno));
+                    rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
+                    goto END;
                 }
             }
         }
@@ -878,6 +916,7 @@ END:
 
     return rc == TOXAV_ERR_SEND_FRAME_OK;
 }
+
 void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb *callback, void *user_data)
 {
     pthread_mutex_lock(av->mutex);
@@ -885,6 +924,7 @@ void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb
     av->acb.second = user_data;
     pthread_mutex_unlock(av->mutex);
 }
+
 void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb *callback, void *user_data)
 {
     pthread_mutex_lock(av->mutex);
@@ -893,7 +933,6 @@ void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb
     pthread_mutex_unlock(av->mutex);
 }
 
-
 /*******************************************************************************
  *
  * :: Internal
@@ -913,7 +952,8 @@ void callback_bwc(BWController *bwc, uint32_t friend_number, float loss, void *u
 
     LOGGER_DEBUG(call->av->m->log, "Reported loss of %f%%", loss * 100);
 
-    if (loss < .01f) {
+    /* if less than 10% data loss we do nothing! */
+    if (loss < 0.1f) {
         return;
     }
 
@@ -1079,6 +1119,7 @@ bool invoke_call_state_callback(ToxAV *av, uint32_t friend_number, uint32_t stat
 
     return true;
 }
+
 ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error)
 {
     /* Assumes mutex locked */
@@ -1100,7 +1141,6 @@ ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error)
         goto END;
     }
 
-
     call = (ToxAVCall *)calloc(sizeof(ToxAVCall), 1);
 
     if (call == nullptr) {
@@ -1161,6 +1201,7 @@ END:
 
     return call;
 }
+
 ToxAVCall *call_get(ToxAV *av, uint32_t friend_number)
 {
     /* Assumes mutex locked */
@@ -1170,6 +1211,7 @@ ToxAVCall *call_get(ToxAV *av, uint32_t friend_number)
 
     return av->calls[friend_number];
 }
+
 ToxAVCall *call_remove(ToxAVCall *call)
 {
     if (call == nullptr) {
@@ -1217,6 +1259,7 @@ CLEAR:
 
     return nullptr;
 }
+
 bool call_prepare_transmission(ToxAVCall *call)
 {
     /* Assumes mutex locked */
@@ -1305,6 +1348,7 @@ FAILURE_3:
     pthread_mutex_destroy(call->mutex_audio);
     return false;
 }
+
 void call_kill_transmission(ToxAVCall *call)
 {
     if (call == nullptr || call->active == 0) {