H5实时解码和播放音频

音频和视频的格式不明确. 我们熟悉的Flv，Mp4，Mov都是包装格式，可以理解为容器h5支持的音频格式，就像盒子一样. 内部是编码的音频和视频数据，这些音频和视频数据具有自己的编码格式，例如AAC，H264，H265等.

今天的演示是解码从实时流中获取的音频编码数据并使用H5音频API进行播放的过程.

这些格式是

speexaacmp3

这些格式都有开源解码库，但它们都是C库. 在H5中，它们需要编译为js并由emscripten执行.

#ifdef USE_SPEEX
#include <speex/speex.h>
#endif
#ifdef USE_AAC
#include "aacDecoder/include/neaacdec.h"
// #include "libfdk-aac/libAACdec/include/aacdecoder_lib.h"
#endif
#ifdef USE_MP3
#include "libmad/mad.h"
//#include "libid3tag/tag.h"
#endif

int bufferLength;
int bufferFilled;
u8 *outputBuffer;
#ifdef USE_AAC
    faacDecHandle faacHandle;
#endif
#ifdef USE_SPEEX
    i16 *audioOutput;
    void *speexState;
    SpeexBits speexBits;
#endif
#ifdef USE_MP3
    MP3Decoder mp3Decoder;
#endif

bufferLength用于指定缓冲区的长度，bufferFilled用于指示缓冲区中未使用的数据，而outputBuffer用于存储解码后的数据.

MP3Decoder是您自己编写的类，您需要定义这些成员

mad_stream inputStream;
mad_frame frame;
mad_synth synth;

    outputBuffer = (u8 *)malloc(bufferLength);
#ifdef USE_SPEEX
    audioOutput = (i16 *)malloc(640);
    auto mode = speex_lib_get_mode(SPEEX_MODEID_WB);
    speexState = speex_decoder_init(mode);
    speex_bits_init(&speexBits);
#endif
#ifdef USE_AAC
    faacHandle = faacDecOpen();
#endif

初始化mp3

mad_stream_init(&inputStream);
mad_frame_init(&frame);
mad_synth_init(&synth);

协议解压缩后，输入对象包含原始音频数据（RTMP协议格式或Flv格式）. 尽管缓冲区大小由您自己定义，但您必须遵循以下规则

aac: 1024的倍数（AAC帧播放时间= 10241000/44100 = 22.32ms）

speex: 320的倍数（3201000/16000 = 20ms）

MP3: 576的倍数（两通道1152 * 1000/44100 = 26.122ms）

基于这些数据h5支持的音频格式，您可以估计由缓冲区大小引起的音频延迟，然后需要与视频延迟同步.

#ifdef USE_SPEEX
    if (input.length() <= 11)
    {
        memset(output, 0, 640);
    }
    else
    {
        speex_bits_read_from(&speexBits, (const char *)input, 52);
        speex_decode_int(speexState, &speexBits, audioOutput);
        memcpy(output, audioOutput, 640);
    }
    return 640;
#endif

#ifdef USE_AAC
    //0 = AAC sequence header ,1 = AAC raw 
    if (input.readB<1, u8>())
    {
        faacDecFrameInfo frame_info;
        auto pcm_data = faacDecDecode(faacHandle, &frame_info, (unsigned char *)input.point(), input.length());
        if (frame_info.error > 0)
        {
            emscripten_log(1, "!!%s\n", NeAACDecGetErrorMessage(frame_info.error));
        }
        else
        {
        int samplesBytes = frame_info.samples << 1;
        memcpy(output, pcm_data, samplesBytes);
        return samplesBytes;
        }
    }
    else
    {
        unsigned long samplerate;
        unsigned char channels;
        auto config = faacDecGetCurrentConfiguration(faacHandle);
        config->defObjectType = LTP;
        faacDecSetConfiguration(faacHandle,config);
        faacDecInit2(faacHandle, (unsigned char *)input.point(), 4, &samplerate, &channels);
        emscripten_log(0, "aac samplerate:%d channels:%d", samplerate, channels);
    }
#endif

mp3更为复杂，此处未发布任何代码，主要是因为mad库无法直接调用其提供的API，实时流中的MP3数据格式和mp3文件格式不同. 如果这篇文章很热门，我会详细说明.

#ifdef USE_AAC
    faacDecClose(faacHandle);
#endif
#ifdef USE_SPEEX
    speex_decoder_destroy(speexState);
    speex_bits_destroy(&speexBits);
    free(audioOutput);
#endif
    free(outputBuffer);

mp3

mad_synth_finish(&synth);
mad_frame_finish(&frame);

window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new window.AudioContext();

var audioBuffers = []
var audioBuffer = context.createBuffer(channels, frameCount, samplerate);

                var playNextBuffer = function() {
                    isPlaying = false;
                    if (audioBuffers.length) {
                        playAudio(audioBuffers.shift());
                    }
                    if (audioBuffers.length > 1) audioBuffers.shift();
                    //console.log(audioBuffers.length)
                };
                var copyAudioOutputArray = resampled ? function(target) {
                    for (var i = 0; i < allFrameCount; i++) {
                        var j = i << 1;
                        target[j] = target[j + 1] = audioOutputArray[i] / 32768;
                    }
                } : function(target) {
                    for (var i = 0; i < allFrameCount; i++) {
                        target[i] = audioOutputArray[i] / 32768;
                    }
                };
                var copyToCtxBuffer = channels > 1 ? function(fromBuffer) {
                    for (var channel = 0; channel < channels; channel++) {
                        var nowBuffering = audioBuffer.getChannelData(channel);
                        if (fromBuffer) {
                            for (var i = 0; i < frameCount; i++) {
                                nowBuffering[i] = fromBuffer[i * (channel + 1)];
                            }
                        } else {
                            for (var i = 0; i < frameCount; i++) {
                                nowBuffering[i] = audioOutputArray[i * (channel + 1)] / 32768;
                            }
                        }
                    }
                } : function(fromBuffer) {
                    var nowBuffering = audioBuffer.getChannelData(0);
                    if (fromBuffer) nowBuffering.set(fromBuffer);
                    else copyAudioOutputArray(nowBuffering);
                };
                var playAudio = function(fromBuffer) {
                    if (isPlaying) {
                        var buffer = new Float32Array(resampled ? allFrameCount * 2 : allFrameCount);
                        copyAudioOutputArray(buffer);
                        audioBuffers.push(buffer);
                        return;
                    }
                    isPlaying = true;
                    copyToCtxBuffer(fromBuffer);
                    var source = context.createBufferSource();
                    source.buffer = audioBuffer;
                    source.connect(context.destination);
                    source.onended = playNextBuffer;
                    //setTimeout(playNextBuffer, audioBufferTime-audioBuffers.length*200);
                    source.start();
                };

使用playNextBuffer函数从缓冲区中获取数据

copyAudioOutputArray函数用于将音频数据转换为浮点数.

copyToCtxBuffer函数用于将音频数据复制到可以播放的缓冲区阵列中.

这些功能处理单声道和双声道

var resampled = samplerate < 22050;

对于频率小于22khz的数据，我们需要进行复制并将其模拟为22khz，因为H5仅支持大于22khz的数据.

本文来自电脑杂谈，转载请注明本文网址：
https://www.pc-fly.com/a/ruanjian/article-171567-1.html