Sync audio with precision / delay with Firefox setInterval and setting currentTime

I am writing a JS application that highlights words in sync with pre-recorded audio. The process is pretty straightforward. I have a list of cases where each word starts in a sound and I call a JS function every 0.1 seconds to check the audio.currentTime and see if it has reached the next word threshold. I tried to use the timeUpdate audio event and don't get it often enough for different purposes for my own purposes. Words are missing.

I've tested my code on Moto E on mobile Chrome, iPad (mobile Safari), Kindle Fire (silk browser or whatever it's called now), and on Windows in the latest versions of Chrome, Firefox and IE 11.

Everything works very well with one exception: Firefox is lagging behind. The sync works fine at first in FF, but every time I write audio at a new time, Firefox seems to lag behind, and on page 10 or so (even if I just slipped right to it by clicking my "next page" , several times), the lag is noticeable. The narrator will finish one word and FF will just start highlighting the previous one.

Here's the relevant code:

This is called after the AJAX call to get the book information:

window.setInterval( bundleFunctions.execute, 100 );

      

An object bundleFunctions

is just a simple way to call a dynamic group of functions at the same time.

The bundleFunctions

added two functions. I am concerned that highlightWord

.

var lasthighlight = -1;
function highlightWord() {

    var debug = false;

        // find the last word time that comes before the current time.

    var playerSound     = document.getElementById( "playerSound" );     // audio tag

        // wordTimes is an array of times where each word starts in the recorded audio

    var indexToHighlight = Math.max.apply( Math, $.map( wordTimes, function( time, index ) {

            // only highlight words that start before the current time with a buffer equal to the interval at which this function is called
            // don't highlight word for more than 2 seconds.

        if ( time - .1 > playerSound.currentTime || playerSound.currentTime - time > 2 ) {      
            return null;

            // if this is the last or first wordTime or the preceding word time
            // is less than the current time, return the index of that word time

        } else {
            return index;
        }
    }));

        // minimize lag by not touching the DOM if there no new word to highlight

    if ( typeof indexToHighlight != "undefined" ) {
        if ( lasthighlight != indexToHighlight ) {
            lasthighlight = indexToHighlight;

            if ( debug ) {
                console.log( "highlighting word [ " + $( ".word.use:eq(" + indexToHighlight + ")" ).text() + "] at this index: [" + indexToHighlight + "]" );
                console.log( indexToHighlight );
            }
            $word = $( ".word:eq(" + indexToHighlight + ")");
            $( ".word" ).css( "color", "black" );
            $word.css( "color", "red" );
        } 
    }
}

      

You can see that I added some code to try and minimize DOM manipulation in the hopes that this is the reason why FF is falling behind, but it didn't make any difference. Note that there are no more than 40 words on the page at a time, so this is not the case, but this is very heavy DOM manipulation and the lag seems to be directly related to searching in the audio, not any problem that the FF JS engine is handling when handling your workload.

Another function, called by bundleFunction, checks if we have reached the end of the page. Here's the function ( override

lets me skip the next page manually):

function checkAdvancePage( override ) {
    var debug = false;
    override = override === true;
    if ( playerSound.currentTime >= timeToSeconds( book.pages.page[ curPage ][ "_f" ]) || override ) {

        if ( book.pages.page.length >= curPage + 2 ) {
            if ( debug ) {
                console.log( "advancing page from [" + curPage + "]" );
            }

            curPage++;

            if ( debug ) {
                console.log( "to [" + curPage + "]" );
            }


                // this book has another page. Keep going.

            startPage( book );

        } else {

                // this book is on its last page

            endScreen( book );
        }
    }
}

      

As you can see, this is pretty damn basic. 99.9% of the time it will evaluate one line of code - "currentTime> is the current page end time?", Return false and you're done.

Finally, here's the code where, by the way, the code looks in the audio at the start of a new page when pages change. The oddly structured book object is like this because it is converted from XML:

var soundLoaded = false;
var isPlaying   = false;
function queueAudio( book ) {
    var debug           = false;

    var playerSound     = document.getElementById( "playerSound" );     // audio tag
    var playing         = false;
    playerSound.addEventListener( 'canplaythrough', function queue() {

        if ( debug ) {
            console.log( "tracing this page" );
            console.log( book.pages.page[ curPage ]);   
        }

        playerSound.currentTime = timeToSeconds( book.pages.page[ curPage ]["_s" ]);
        playerSound.removeEventListener( 'canplaythrough', queue );
        bundleFunctions.add( checkAdvancePage );
        soundLoaded             = true;
    }); 


    if ( !soundLoaded ) {
        var playerSource    = document.getElementById( "playerSource" );    // source tag
        playerSource.src    = BOOKS_DIR + book.track[ "_src" ];
        playerSound.load();
        playerSound.play();         // has to be called here to link a mobile tap to the play command
        isPlaying = true;
    } else {
        playerSound.currentTime = timeToSeconds( book.pages.page[ curPage ]["_s" ]);
    }

        // manual click to next page. Only add the listener once.
    if ( typeof( document.getElementById( "next" ).onclick ) == "undefined" ) {
        $( "#next" ).click( function() {
            console.log( "next page clicked" );
            checkAdvancePage( true );
        });
    }

        // watches current time of audio and advances to the next page when we get to the end of the audio for that page

    function checkAdvancePage( override ) {
        var debug = false;
        override = override === true;  // if true, advance the page regardless of currentTime
        if ( playerSound.currentTime >= timeToSeconds( book.pages.page[ curPage ][ "_f" ]) || override ) {
            playerSound.removeEventListener( 'timeupdate', checkAdvancePage );

            if ( book.pages.page.length >= curPage + 2 ) {
                if ( debug ) {
                    console.log( "advancing page from [" + curPage + "]" );
                }

                curPage++;

                if ( debug ) {
                    console.log( "to [" + curPage + "]" );
                }


                    // this book has another page. Keep going.

                startPage( book );

            } else {

                    // this book is on its last page

                endScreen( book );
            }
        }
    }
}

      

This is acceptable if the word does not stand out at the moment it is spoken. What's unacceptable is that the word starts to stand out after it has been spoken, which starts happening in fast chunks of audio around 10 pages.

The fastest words are spoken in about 0.1 seconds. It would be nice if from time to time one of those ultra-fast words were skipped while the backlight stayed in sync.

Thanks in advance for any help you can provide.

+3


source to share





All Articles