安卓Unity3D Camera图像和音频采集推送代码

    安卓Unity3d 可以使用ReadPixels从当前Render Target读取图像，音频可以从AudioClip读取，具体调用GetData接口，读取到的可能是float类型，有些音频编码器可能需要sint16格式，这需要做一个转换。 读图像先用StartCoroutine创建协程，等待WaitForEndOfFrame, 之后调用ReadPixels就好，调用之前要创建一个和Camera一样的大小Texture2D, 创建代码如下：

 Texture2D image_texture = new Texture2D(w, h, texture_format_, false);

  启动停止协程，捕获图像代码如下：

// Github: https://github.com/daniulive/SmarterStreaming
// Copyright (C) [e-mail]

public class TestCameraCapture: MonoBehaviour
{
    Coroutine coroutine_ = null;

    void OnDisable()
    {
        coroutine_ = StartCoroutine(captureImage());
    }

    void OnEnable()
    {
       StopCoroutine(coroutine_);
       coroutine_ = null;
    }

    IEnumerator captureImage() 
    {

        yield return new WaitForEndOfFrame();
        image_texture.ReadPixels(new Rect(0, 0, w, h), 0, 0, false);

        // 编码图像
  }

}

  这个是Unity3D官方给出的代码:

using System.IO;
using UnityEngine;
using UnityEngine.Networking;
using System.Collections;

public class ExampleClass : MonoBehaviour
{
    // Take a shot immediately
    IEnumerator Start()
    {
        UploadPNG();
        yield return null;
    }

    IEnumerator UploadPNG()
    {
        // We should only read the screen buffer after rendering is complete
        yield return new WaitForEndOfFrame();

        // Create a texture the size of the screen, RGB24 format
        int width = Screen.width;
        int height = Screen.height;
        Texture2D tex = new Texture2D(width, height, TextureFormat.RGB24, false);

        // Read screen contents into the texture
        tex.ReadPixels(new Rect(0, 0, width, height), 0, 0);
        tex.Apply();

        // Encode texture into PNG
        byte[] bytes = tex.EncodeToPNG();
        Destroy(tex);

        // For testing purposes, also write to a file in the project folder
        // File.WriteAllBytes(Application.dataPath + "/../SavedScreen.png", bytes);


        // Create a Web Form
        WWWForm form = new WWWForm();
        form.AddField("frameCount", Time.frameCount.ToString());
        form.AddBinaryData("fileUpload", bytes);

        // Upload to a cgi script
        var w = UnityWebRequest.Post("http://localhost/cgi-bin/env.cgi?post", form);
        yield return w.SendWebRequest();
        if (w.result != UnityWebRequest.Result.Success)
            print(w.error);
        else
            print("Finished Uploading Screenshot");
        yield return null;
    }
}

   上面代码是读取图像后保存成PNG图片，只要把读取的纹理传给我写的C接口即可:

// Github: https://github.com/daniulive/SmarterStreaming
// Copyright (C) [e-mail]

 private int postLayerImageRGBA8888Native(long handle, int index, int left, int top,
                                            long rgba_plane, int offset, int row_stride, int width, int height,
                                            int is_vertical_flip, int is_horizontal_flip,
                                            int scale_width, int scale_height, int scale_filter_mode,
                                            int rotation_degree){

            return obj_.Call<int>("OnPostLayerImageRGBA8888Native", handle, index, left, top,
                rgba_plane, offset, row_stride, width, height,
                is_vertical_flip, is_horizontal_flip,
                scale_width, scale_height, scale_filter_mode, rotation_degree);
        }

        private int postLayerImageRGBX8888Native(long handle, int index, int left, int top,
                                            long rgbx_plane, int offset, int row_stride, int width, int height,
                                            int is_vertical_flip, int is_horizontal_flip,
                                            int scale_width, int scale_height, int scale_filter_mode,
                                            int rotation_degree){

            return obj_.Call<int>("OnPostLayerImageRGBX8888Native", handle, index, left, top,
                rgbx_plane, offset, row_stride, width, height,
                is_vertical_flip, is_horizontal_flip,
                scale_width, scale_height, scale_filter_mode, rotation_degree);
        }

读Camera和读Screen差不多，在我自己手机上硬编码视频可以跑到60帧，有些眼镜性能较弱，帧率稍微低些，不过基本都能满足需求, 实际开发中遇到不少问题，经验就是多看Unity3D官方文档，基本都能找到解决方案.