将Android MediaCodec编码的H264数据包多路复用到RTMP中

Question

我来自一个线程编码H.264从摄像头与Android媒体编解码器。我的设置很相似。然而，我尝试用javacv编写mux编码帧和并通过RTMP广播它们。

rtmpclient.java

...
private volatile BlockingQueue<byte[]> mFrameQueue = new LinkedBlockingQueue<byte[]>(MAXIMUM_VIDEO_FRAME_BACKLOG);
...
private void startStream() throws FrameRecorder.Exception, IOException {
    if (TextUtils.isEmpty(mDestination)) {
        throw new IllegalStateException("Cannot start RtmpClient without destination");
    }

    if (mCamera == null) {
        throw new IllegalStateException("Cannot start RtmpClient without camera.");
    }

    Camera.Parameters cameraParams = mCamera.getParameters();

    mRecorder = new FFmpegFrameRecorder(
            mDestination,
            mVideoQuality.resX,
            mVideoQuality.resY,
            (mAudioQuality.channelType.equals(AudioQuality.CHANNEL_TYPE_STEREO) ? 2 : 1));

    mRecorder.setFormat("flv");

    mRecorder.setFrameRate(mVideoQuality.frameRate);
    mRecorder.setVideoBitrate(mVideoQuality.bitRate);
    mRecorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);

    mRecorder.setSampleRate(mAudioQuality.samplingRate);
    mRecorder.setAudioBitrate(mAudioQuality.bitRate);
    mRecorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC);

    mVideoStream = new VideoStream(mRecorder, mVideoQuality, mFrameQueue, mCamera);
    mAudioStream = new AudioStream(mRecorder, mAudioQuality);

    mRecorder.start();

    // Setup a bufferred preview callback
    setupCameraCallback(mCamera, mRtmpClient, DEFAULT_PREVIEW_CALLBACK_BUFFERS,
            mVideoQuality.resX * mVideoQuality.resY * ImageFormat.getBitsPerPixel(
                    cameraParams.getPreviewFormat())/8);

    try {
        mVideoStream.start();
        mAudioStream.start();
    }
    catch(Exception e) {
        e.printStackTrace();
        stopStream();
    }
}
...
@Override
public void onPreviewFrame(byte[] data, Camera camera) {
    boolean frameQueued = false;

    if (mRecorder == null || data == null) {
        return;
    }

    frameQueued = mFrameQueue.offer(data);

    // return the buffer to be reused - done in videostream
    //camera.addCallbackBuffer(data);
}
...

...
@Override
public void run() {
    try {
        mMediaCodec = MediaCodec.createEncoderByType("video/avc");
        MediaFormat mediaFormat = MediaFormat.createVideoFormat("video/avc", mVideoQuality.resX, mVideoQuality.resY);
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, mVideoQuality.bitRate);
        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, mVideoQuality.frameRate);
        mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar);
        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
        mMediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
        mMediaCodec.start();
    }
    catch(IOException e) {
        e.printStackTrace();
    }

    long startTimestamp = System.currentTimeMillis();
    long frameTimestamp = 0;
    byte[] rawFrame = null;

    try {
        while (!Thread.interrupted()) {
            rawFrame = mFrameQueue.take();

            frameTimestamp = 1000 * (System.currentTimeMillis() - startTimestamp);

            encodeFrame(rawFrame, frameTimestamp);

            // return the buffer to be reused
            mCamera.addCallbackBuffer(rawFrame);
        }
    }
    catch (InterruptedException ignore) {
        // ignore interrup while waiting
    }

    // Clean up video stream allocations
    try {
        mMediaCodec.stop();
        mMediaCodec.release();
        mOutputStream.flush();
        mOutputStream.close();
    } catch (Exception e){
        e.printStackTrace();
    }
}
...
private void encodeFrame(byte[] input, long timestamp) {
    try {
        ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers();
        ByteBuffer[] outputBuffers = mMediaCodec.getOutputBuffers();

        int inputBufferIndex = mMediaCodec.dequeueInputBuffer(0);

        if (inputBufferIndex >= 0) {
            ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
            inputBuffer.clear();
            inputBuffer.put(input);
            mMediaCodec.queueInputBuffer(inputBufferIndex, 0, input.length, timestamp, 0);
        }

        MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();

        int outputBufferIndex = mMediaCodec.dequeueOutputBuffer(bufferInfo, 0);

        if (outputBufferIndex >= 0) {
            while (outputBufferIndex >= 0) {
                ByteBuffer outputBuffer = outputBuffers[outputBufferIndex];

                // Should this be a direct byte buffer?
                byte[] outData = new byte[bufferInfo.size - bufferInfo.offset];
                outputBuffer.get(outData);

                mFrameRecorder.record(outData, bufferInfo.offset, outData.length, timestamp);

                mMediaCodec.releaseOutputBuffer(outputBufferIndex, false);
                outputBufferIndex = mMediaCodec.dequeueOutputBuffer(bufferInfo, 0);
            }
        }
        else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
            outputBuffers = mMediaCodec.getOutputBuffers();
        } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
            // ignore for now
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }

}
...

...
// Hackish codec copy frame recording function
public boolean record(byte[] encodedData, int offset, int length, long frameCount) throws Exception {
    int ret;

    if (encodedData == null) {
        return false;
    }

    av_init_packet(video_pkt);

    // this is why i wondered whether I should get outputbuffer data into direct byte buffer
    video_outbuf.put(encodedData, 0, encodedData.length);

    video_pkt.data(video_outbuf);
    video_pkt.size(video_outbuf_size);

    video_pkt.pts(frameCount);
    video_pkt.dts(frameCount);

    video_pkt.stream_index(video_st.index());

    synchronized (oc) {
        /* write the compressed frame in the media file */
        if (interleaved && audio_st != null) {
            if ((ret = av_interleaved_write_frame(oc, video_pkt)) < 0) {
                throw new Exception("av_interleaved_write_frame() error " + ret + " while writing interleaved video frame.");
            }
        } else {
            if ((ret = av_write_frame(oc, video_pkt)) < 0) {
                throw new Exception("av_write_frame() error " + ret + " while writing video frame.");
            }
        }
    }
    return (video_pkt.flags() & AV_PKT_FLAG_KEY) == 1;
}
...

ffprobe version 2.5.3 Copyright (c) 2007-2015 the FFmpeg developers
  built on Jan 19 2015 12:56:57 with gcc 4.1.2 (GCC) 20080704 (Red Hat 4.1.2-55)
  configuration: --prefix=/usr --bindir=/usr/bin --datadir=/usr/share/ffmpeg --incdir=/usr/include/ffmpeg --libdir=/usr/lib64 --mandir=/usr/share/man --arch=x86_64 --optflags='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic' --enable-bzlib --disable-crystalhd --enable-libass --enable-libdc1394 --enable-libfaac --enable-nonfree --disable-indev=jack --enable-libfreetype --enable-libgsm --enable-libmp3lame --enable-openal --enable-libopencv --enable-libopenjpeg --enable-libopus --enable-librtmp --enable-libtheora --enable-libvorbis --enable-libvpx --enable-libx264 --enable-libxvid --enable-x11grab --enable-avfilter --enable-avresample --enable-postproc --enable-pthreads --disable-static --enable-shared --enable-gpl --disable-debug --disable-stripping --enable-libcaca --shlibdir=/usr/lib64 --enable-runtime-cpudetect
  libavutil      54. 15.100 / 54. 15.100
  libavcodec     56. 13.100 / 56. 13.100
  libavformat    56. 15.102 / 56. 15.102
  libavdevice    56.  3.100 / 56.  3.100
  libavfilter     5.  2.103 /  5.  2.103
  libavresample   2.  1.  0 /  2.  1.  0
  libswscale      3.  1.101 /  3.  1.101
  libswresample   1.  1.100 /  1.  1.100
  libpostproc    53.  3.100 / 53.  3.100
Metadata:
  Server                NGINX RTMP (github.com/arut/nginx-rtmp-module)
  width                 320.00
  height                240.00
  displayWidth          320.00
  displayHeight         240.00
  duration              0.00
  framerate             0.00
  fps                   0.00
  videodatarate         261.00
  videocodecid          7.00
  audiodatarate         62.00
  audiocodecid          10.00
  profile
  level
[live_flv @ 0x1edb0820] Could not find codec parameters for stream 0 (Video: none, none, 267 kb/s): unknown codec
Consider increasing the value for the 'analyzeduration' and 'probesize' options
Input #0, live_flv, from 'rtmp://<server>/input/<stream id>':
  Metadata:
    Server          : NGINX RTMP (github.com/arut/nginx-rtmp-module)
    displayWidth    : 320
    displayHeight   : 240
    fps             : 0
    profile         :
    level           :
  Duration: 00:00:00.00, start: 16.768000, bitrate: N/A
    Stream #0:0: Video: none, none, 267 kb/s, 1k tbr, 1k tbn, 1k tbc
    Stream #0:1: Audio: aac (LC), 16000 Hz, mono, fltp, 63 kb/s
Unsupported codec with id 0 for input stream 0

我不是，无论如何，一个专家在H264或视频编码。我知道从MediaCodec出来的编码帧包含SPS NAL、PPS NAL和帧NAL单元。我还将MediaCodec输出写入了一个文件，并且能够回放它（我确实必须指定格式和帧率，否则它会播放得太快）。

我的假设是事情应该工作（看看我知道的有多少：））。知道了SPS和PPS被写出来，解码器应该知道足够多。然而，ffprobe无法识别编解码器、fps和其他视频信息。我需要将包标志信息传递给FFMPEGFrameRecorder.java：record（）函数吗？或者我应该使用直接缓冲区？任何建议都将被感激！我应该用一个提示来弄清楚事情。

PS：我知道有些编解码器使用平面和其他半平面颜色格式。如果我过了这一关，这种区别就会出现。此外，我没有使用Surface到MediaCodec的方式，因为我需要支持API17，它需要比这条路由更多的更改，我认为这有助于我理解更基本的流程。阿根，我很感激任何建议。如果有什么需要澄清的请告诉我。

更新#1

所以在做了更多的测试之后，我看到我的编码器输出以下帧：

000000016742800DDA0507E806D0A1350000000168CE06E2
0000000165B840A6F1E7F0EA24000AE73BEB5F51CC7000233A84240...
0000000141E2031364E387FD4F9BB3D67F51CC7000279B9F9CFE811...
0000000141E40304423FFFFF0B7867F89FAFFFFFFFFFFCBE8EF25E6...
0000000141E602899A3512EF8AEAD1379F0650CC3F905131504F839...
...

第一帧包含SP和PPS。据我所知，这些只有一次传送。其余为NAL类型1和5。因此，我的假设是，为了让ffprobe不仅在流开始时看到流信息，我应该捕获SPS和PPS帧，并在一定数量的帧之后，或者在每个I帧之前，定期地重新发送它们。你觉得呢？

Answer 1

据我所知，你不需要复用视频和音频流。当您发送公告消息时，您还可以指定将音频和视频流发送到哪个端口。您需要将它们单独打包并用RTP发送。更多信息见维基百科。https://en.wikipedia.org/wiki/real_time_streaming_protocol