Welcome! Log In Create A New Profile

Advanced

[PATCH] Add optional "&exact=1" CGI arg to show video between keyframes

Tracey Jaquith
June 03, 2021 03:56PM
# HG changeset patch
# User Tracey Jaquith <tracey@archive.org>
# Date 1622678642 0
# Thu Jun 03 00:04:02 2021 +0000
# Node ID 5da9c62fa61016600f2c59982ae184e2811be427
# Parent 5f765427c17ac8cf753967387562201cf4f78dc4
Add optional "&exact=1" CGI arg to show video between keyframes.

archive.org has been using mod_h264_streaming with an "&exact=1" patch from me since 2013.
We just moved to nginx mp4 module and are using this patch.
The technique is to find the keyframe just before the desired "start" time, and send
that down the wire so video playback can start immediately.
Next calculate how many video samples are between the keyframe and desired "start" time
and update the STTS atom where those samples move the duration from (typically) 1001 to 1.
This way, initial unwanted video frames play at ~1/30,000s -- so visually the
video & audio start playing immediately.

You can see an example before/after here (nginx binary built with mp4 module + patch):

https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30
https://pi.archive.org/0/items/CSPAN_20160425_022500_2011_White_House_Correspondents_Dinner.mp4?start=12&end=30&exact=1

Tested on linux and macosx.

I realize two var declarations should style-wise get moved "up" in the lower hooked in method.
However, to minimize the changeset/patch, I figured we should at least start here and see what you folks think.

(this is me: https://github.com/traceypooh )

diff -r 5f765427c17a -r 5da9c62fa610 src/http/modules/ngx_http_mp4_module.c
--- a/src/http/modules/ngx_http_mp4_module.c Tue Jun 01 17:37:51 2021 +0300
+++ b/src/http/modules/ngx_http_mp4_module.c Thu Jun 03 00:04:02 2021 +0000
@@ -2045,6 +2045,109 @@
u_char duration[4];
} ngx_mp4_stts_entry_t;

+typedef struct {
+ uint32_t speedup_samples;
+ ngx_uint_t speedup_seconds;
+} ngx_mp4_exact_t;
+
+static void
+exact_video_adjustment(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, ngx_mp4_exact_t *exact)
+{
+ // will parse STTS -- time-to-sample atom
+ ngx_str_t value;
+ ngx_buf_t *stts_data;
+ ngx_buf_t *atom;
+ ngx_mp4_stts_entry_t *stts_entry, *stts_end;
+ uint32_t count, duration, j, n, sample_keyframe, sample_num;
+ uint64_t sample_time, seconds, start_seconds_closest_keyframe;
+ uint8_t is_keyframe;
+
+ exact->speedup_samples = 0;
+ exact->speedup_seconds = 0;
+
+ // if '&exact=1' CGI arg isn't present, do nothing
+ if (!(ngx_http_arg(mp4->request, (u_char *) "exact", 5, &value) == NGX_OK)) {
+ return;
+ }
+
+ if (!trak->sync_samples_entries) {
+ // Highly unlikely video STSS got parsed and _every_ sample is a keyframe.
+ // However, if the case, we don't need to adjust the video at all.
+ return;
+ }
+
+ // check HDLR atom to see if this trak is video or audio
+ atom = trak->out[NGX_HTTP_MP4_HDLR_ATOM].buf;
+ // 'vide' or 'soun'
+ if (!(atom->pos[16] == 'v' &&
+ atom->pos[17] == 'i' &&
+ atom->pos[18] == 'd' &&
+ atom->pos[19] == 'e')) {
+ return; // do nothing if not video
+ }
+
+
+ stts_data = trak->out[NGX_HTTP_MP4_STTS_DATA].buf;
+ stts_entry = (ngx_mp4_stts_entry_t *) stts_data->pos;
+ stts_end = (ngx_mp4_stts_entry_t *) stts_data->last;
+
+ sample_num = 0; // they start at one <shrug>
+ sample_time = 0;
+ start_seconds_closest_keyframe = 0;
+ while (stts_entry < stts_end) {
+ // STTS === time-to-sample atom
+ // each entry is 4B and [sample count][sample duration] (since durations can vary)
+ count = ngx_mp4_get_32value(stts_entry->count);
+ duration = ngx_mp4_get_32value(stts_entry->duration);
+
+ for (j = 0; j < count; j++) {
+ sample_num++;
+
+ // search STSS sync sample entries to see if this sample is a keyframe
+ is_keyframe = (trak->sync_samples_entries ? 0 : 1);
+ for (n = 0; n < trak->sync_samples_entries; n++) {
+ // each one of this these are a video sample number keyframe
+ sample_keyframe = ngx_mp4_get_32value(trak->stss_data_buf.pos + (n * 4));
+ if (sample_keyframe == sample_num) {
+ is_keyframe = 1;
+ break;
+ }
+ if (sample_keyframe > sample_num) {
+ break;
+ }
+ }
+
+ seconds = sample_time * 1000 / trak->timescale;
+ sample_time += duration;
+
+ if (seconds > mp4->start) {
+ goto found;
+ }
+
+ if (is_keyframe) {
+ start_seconds_closest_keyframe = seconds;
+ exact->speedup_samples = 0;
+ } else {
+ exact->speedup_samples++;
+ }
+ }
+
+ stts_entry++;
+ }
+
+ found:
+
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact_video_adjustment() new keyframe start: %d, speedup first %d samples",
+ start_seconds_closest_keyframe,
+ exact->speedup_samples);
+
+ // NOTE: begin 1 start position before keyframe to ensure first video frame emitted is always
+ // a keyframe
+ exact->speedup_seconds = mp4->start - start_seconds_closest_keyframe
+ - (start_seconds_closest_keyframe ? 1 : 0);
+}
+

static ngx_int_t
ngx_http_mp4_read_stts_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size)
@@ -2164,10 +2267,14 @@
ngx_buf_t *data;
ngx_uint_t start_sample, entries, start_sec;
ngx_mp4_stts_entry_t *entry, *end;
+ ngx_mp4_exact_t exact;

if (start) {
start_sec = mp4->start;

+ exact_video_adjustment(mp4, trak, &exact);
+ start_sec -= exact.speedup_seconds;
+
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
"mp4 stts crop start_time:%ui", start_sec);

@@ -2230,6 +2337,42 @@

if (start) {
ngx_mp4_set_32value(entry->count, count - rest);
+
+ if (exact.speedup_samples) {
+ // We're going to prepend an entry with duration=1 for the frames we want to "not see".
+ // MOST of the time, we're taking a single element entry array and making it two.
+ uint32_t current_count = ngx_mp4_get_32value(entry->count);
+ ngx_mp4_stts_entry_t* entries_array = ngx_palloc(mp4->request->pool,
+ (1 + entries) * sizeof(ngx_mp4_stts_entry_t));
+ if (entries_array == NULL) {
+ return NGX_ERROR;
+ }
+ ngx_copy(&(entries_array[1]), entry, entries * sizeof(ngx_mp4_stts_entry_t));
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact split in 2 video STTS entry from count:%d", current_count);
+
+ if (current_count <= exact.speedup_samples)
+ return NGX_ERROR;
+
+ entry = &(entries_array[1]);
+ ngx_mp4_set_32value(entry->count, current_count - exact.speedup_samples);
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact split new[1]: count:%d duration:%d",
+ ngx_mp4_get_32value(entry->count),
+ ngx_mp4_get_32value(entry->duration));
+ entry--;
+ ngx_mp4_set_32value(entry->count, exact.speedup_samples);
+ ngx_mp4_set_32value(entry->duration, 1);
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0,
+ "exact split new[0]: count:%d duration:1",
+ ngx_mp4_get_32value(entry->count));
+
+ data->last = (u_char *) (entry + 2);
+
+ entries++;
+ }
+
data->pos = (u_char *) entry;
trak->time_to_sample_entries = entries;
trak->start_sample = start_sample;
_______________________________________________
nginx-devel mailing list
nginx-devel@nginx.org
http://mailman.nginx.org/mailman/listinfo/nginx-devel
Subject Author Views Posted

[PATCH] Add optional "&exact=1" CGI arg to show video between keyframes

Tracey Jaquith 181 June 03, 2021 03:56PM

Re: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes

Maxim Dounin 27 June 04, 2021 01:40PM

Re: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes

Tracey Jaquith 33 June 04, 2021 10:20PM

Re: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes

Roman Arutyunyan 31 June 09, 2021 08:12AM

Re: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes

Tracey Jaquith 32 June 12, 2021 11:46PM

Re: [PATCH] Add optional "&exact=1" CGI arg to show video between keyframes

Tracey Jaquith 40 June 13, 2021 05:54AM



Sorry, you do not have permission to post/reply in this forum.

Online Users

Guests: 82
Record Number of Users: 6 on February 13, 2018
Record Number of Guests: 421 on December 02, 2018
Powered by nginx      Powered by FreeBSD      PHP Powered      Powered by MariaDB      ipv6 ready