接着之前写的 音视频系列–MediaProjection+MediaCodec制作简单投屏效果，继续使用Camera2+MediaCodec来制作相互实时投屏效果，为后面的直播学习打下基础。

一、效果

---

模拟器作为客户端，手机作为服务端，用模拟器实时接收的手机投屏数据，如果是两个手机可以相互投屏。

整个过程使用Camera2来获取数据，获取的数据通过MediaCodec编码，使用H264协议，传输通过WebSocket来实现，接收端接收到数据之后解码，然后通过TextureView渲染，大概是这么一个流程，下面记录下实现过程。

二、界面初始化的相关

---

public class MainActivity extends AppCompatActivity implements SocketLive.SocketCallback{
    

...

//远程服务端的TextureView初始化
    private void initRemoteTextureView() {
    

        remoteTextureView = findViewById(R.id.remoteTextureView);
        remoteTextureView.setSurfaceTextureListener(new TextureView.SurfaceTextureListener() {
    
            @Override
            public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) {
    
                Surface remoteSurface = new Surface(surface);
                decodePlayerLiveH264 = new DecodePlayerLiveH264();
                decodePlayerLiveH264.initDecoder(remoteSurface);
            }

           ...
        });

    }

//请求相互连接模拟
    public void connect(View view) {
    
        localTextureView.startCapture(this);
    }

//onResume 打开本地相机
    @Override
    protected void onResume() {
    
        super.onResume();
        if (checkPermission()) {
    
            localTextureView.openCamera();
        }
    }

//onPause关闭本地相机
    @Override
    protected void onPause() {
    
        super.onPause();
        localTextureView.closeCamera();
    }

//onDestroy资源释放
    @Override
    protected void onDestroy() {
    
        super.onDestroy();
        localTextureView.onDestroy();
    }
...
//接收端接收到另一端的数据之后的回调，传给解码器解码，然后交给TextureView渲染
    @Override
    public void callBack(byte[] data) {
    
        if (decodePlayerLiveH264 != null) {
    
            decodePlayerLiveH264.callBack(data);
        }
    }
}

三、Camera2准备

---

3.1、相机初始化

private void initializeCameraManager() {
    
  //由于后面相机的参数需要传一个Handler，指明回调在那个线程，所以提前初始化
    startBackgroundThread();

    mCameraManager = (CameraManager) this.mContext.getSystemService(Context.CAMERA_SERVICE);
    try {
    
        //获取可用的相机列表
        String[] cameraIdList = mCameraManager.getCameraIdList();
        for (String cameraId : cameraIdList) {
    
            //获取该相机的CameraCharacteristics，它保存的相机相关的属性
            CameraCharacteristics cameraCharacteristics = mCameraManager.getCameraCharacteristics(cameraId);
            //获取相机的方向
            Integer facing = cameraCharacteristics.get(CameraCharacteristics.LENS_FACING);
            //如果是前置摄像头就continue，我们这里只用后置摄像头
            if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) {
    
                continue;
            }
            //保存cameraId
            this.mCameraId = cameraId;
            this.mBackCameraCharacteristics = cameraCharacteristics;
        }
    } catch (Exception e) {
    
        e.printStackTrace();
    }
}

3.2、打开相机

public void openCamera() {
    
    if (ActivityCompat.checkSelfPermission(mContext, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
    
        return;
    }
    try {
    
        prepareCameraOutputs();
        if (isAvailable()) {
    
            mCameraManager.openCamera(mCameraId, mStateCallback, mBackgroundHandler);
        } else {
    
            setSurfaceTextureListener(this);
        }
    } catch (CameraAccessException e) {
    
        e.printStackTrace();
    }
}

protected void prepareCameraOutputs() {
    
    //获取相机支持的流的参数的集合
    StreamConfigurationMap map = mBackCameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
    //寻找一个 最合适的尺寸
    mPreviewSize = getBestSupportedSize(new ArrayList<Size>(Arrays.asList(map.getOutputSizes(SurfaceTexture.class))));

    mImageReader = ImageReader.newInstance(mPreviewSize.getWidth(), mPreviewSize.getHeight(),
            ImageFormat.YUV_420_888, 2);
    mImageReader.setOnImageAvailableListener(this, mBackgroundHandler);
}

在打开相机之前先，先要去获取相机支持的最接近的预览尺寸，然后要实例化一个ImageReader，用来接收Camera2的可用数据。这些做完之后，需要监测TextureView是否可用，不可用先设置setSurfaceTextureListener监听，可用之后再来打开相机。

3.3、建立Camera2预览会话

private CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback() {
    
    @Override
    public void onOpened(@NonNull CameraDevice cameraDevice) {
    
        Camera2TextureView.this.mCameraDevice = cameraDevice;
        createCaptureSession();
    }

    @Override
    public void onDisconnected(@NonNull CameraDevice cameraDevice) {
    
        cameraDevice.close();
    }

    @Override
    public void onError(@NonNull CameraDevice cameraDevice, int error) {
    
        cameraDevice.close();
    }
};
private void createCaptureSession() {
    
    try {
    

        mSurfaceTexture = getSurfaceTexture();

        mSurfaceTexture.setDefaultBufferSize(mPreviewSize.getWidth(), mPreviewSize.getHeight());

        mWorkingSurface = new Surface(mSurfaceTexture);

        mPreviewRequestBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
        //添加接收数据回调的Target Surface
        mPreviewRequestBuilder.addTarget(mWorkingSurface);
        mPreviewRequestBuilder.addTarget(mImageReader.getSurface());

        mCameraDevice.createCaptureSession(Arrays.asList(mWorkingSurface, mImageReader.getSurface()),
                new CameraCaptureSession.StateCallback() {
    
                    @Override
                    public void onConfigured(@NonNull CameraCaptureSession cameraCaptureSession) {
    
                        updatePreview(cameraCaptureSession);
                    }

                    @Override
                    public void onConfigureFailed(
                            @NonNull CameraCaptureSession cameraCaptureSession) {
    
                        Log.d(TAG, "Fail while starting preview: ");
                    }
                }, null);

    } catch (Exception e) {
    
        Log.e(TAG, "Error while preparing surface for preview: ", e);
    }
}

private void updatePreview(CameraCaptureSession cameraCaptureSession) {
    
    if (null == mCameraDevice) {
    
        return;
    }
    mCaptureSession = cameraCaptureSession;
    //设置自动对焦模式为连续自动对焦
    mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
    //创建CaptureRequest
    mPreviewRequest = mPreviewRequestBuilder.build();
    try {
    
        //开始预览
        mCaptureSession.setRepeatingRequest(mPreviewRequest, null, mBackgroundHandler);
    } catch (CameraAccessException e) {
    
        e.printStackTrace();
    }
}

3.4、处理Camera2回调数据

public void onImageAvailable(ImageReader reader) {
    

    Image image= reader.acquireNextImage();
    Image.Plane[] planes =  image.getPlanes();
    // 重复使用同一批byte数组，减少gc频率
    if (y == null) {
    
        y = new byte[planes[0].getBuffer().limit() - planes[0].getBuffer().position()];
        u = new byte[planes[1].getBuffer().limit() - planes[1].getBuffer().position()];
        v = new byte[planes[2].getBuffer().limit() - planes[2].getBuffer().position()];
    }
    if (image.getPlanes()[0].getBuffer().remaining() == y.length) {
    
        planes[0].getBuffer().get(y);
        planes[1].getBuffer().get(u);
        planes[2].getBuffer().get(v);
    }

    if (nv21 == null) {
    
// 实例化一次
        nv21 = new byte[planes[0].getRowStride() * mPreviewSize.getHeight() * 3 / 2];
        nv21_rotated = new byte[planes[0].getRowStride() * mPreviewSize.getHeight() * 3 / 2];
    }
    ImageUtil.yuvToNv21(y, u, v, nv21, planes[0].getRowStride(), mPreviewSize.getHeight());
    ImageUtil.nv21_rotate_to_90(nv21, nv21_rotated, planes[0].getRowStride(), mPreviewSize.getHeight());
    byte[] temp = ImageUtil.nv21toNV12(nv21_rotated, nv12);

    if(encodePushLiveH264 != null){
    
        encodePushLiveH264.startLive();
        encodePushLiveH264.encodeFrame(temp);
    }

//注意这里用完之后要关闭
    image.close();
}

由于Camera2的yuv数据获取到数据需要自己手动拼接，所以这里是和Camera1不同的，获取到的数据拼接成nv21，需要转化成nv12，因为MediaCodec不支持nv21。

3.5、编码数据

///摄像头调用
public int encodeFrame(byte[] input) {
    
    int inputBufferIndex = mediaCodec.dequeueInputBuffer(100000);
    if (inputBufferIndex >= 0) {
    
        ByteBuffer inputBuffer = mediaCodec.getInputBuffer(inputBufferIndex);
        inputBuffer.clear();
        inputBuffer.put(input);
        long presentationTimeUs = computePresentationTime(frameIndex);
        mediaCodec.queueInputBuffer(inputBufferIndex, 0, input.length, presentationTimeUs, 0);
        frameIndex++;
    }
    MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
    int outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 100000);
    while (outputBufferIndex >= 0) {
    
        ByteBuffer outputBuffer = mediaCodec.getOutputBuffer(outputBufferIndex);
        dealFrame(outputBuffer, bufferInfo);
        mediaCodec.releaseOutputBuffer(outputBufferIndex, false);
        outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 0);
    }
    return 0;
}

3.6、处理数据

private void dealFrame(ByteBuffer bb, MediaCodec.BufferInfo bufferInfo) {
    
    int offset = 4;
    if (bb.get(2) == 0x01) {
    
        offset = 3;
    }
    int type = bb.get(offset) & 0x1F;
    if (type == NAL_SPS) {
    
        sps_pps_buf = new byte[bufferInfo.size];
        bb.get(sps_pps_buf);
    } else if (type == NAL_I) {
    
        final byte[] bytes = new byte[bufferInfo.size];
        bb.get(bytes);
        byte[] newBuf = new byte[sps_pps_buf.length + bytes.length];
        System.arraycopy(sps_pps_buf, 0, newBuf, 0, sps_pps_buf.length);
        System.arraycopy(bytes, 0, newBuf, sps_pps_buf.length, bytes.length);
        this.socketLive.sendData(newBuf);
    } else {
    
        final byte[] bytes = new byte[bufferInfo.size];
        bb.get(bytes);
        this.socketLive.sendData(bytes);
    }
}

1.h264中帧类型type在后位，所以与上0x1F获取后5位，通过这个type来获取帧类型。
2.上面数据处理的时候，对于sps直接保留下来就行了，I帧数据前面需要拼接sps,其它的就直接通过websocket传输。

四、WebSocket客户端和服务端

---

WebSocket配置

implementation "org.java-websocket:Java-WebSocket:1.4.0"

4.1、WebSocket服务端

1.WebSocket配置

private WebSocketServer webSocketServer = new WebSocketServer(new InetSocketAddress(配置端口，设置高一点，比如12001)) {
    
     @Override
     public void onOpen(WebSocket webSocket, ClientHandshake clientHandshake) {
    
         SocketLive.this.webSocket = webSocket;
         Log.i("TAG", "onOpen: 服务端 打开 socket ");
     }

     @Override
     public void onClose(WebSocket webSocket, int i, String s, boolean b) {
    
         Log.i(TAG, "onClose: 关闭 socket ");
     }

     @Override
     public void onMessage(WebSocket webSocket, String s) {
    

     }

//接收到对端数据然后解码渲染
     @Override
     public void onMessage(WebSocket conn, ByteBuffer bytes) {
    
         byte[] buf = new byte[bytes.remaining()];
         bytes.get(buf);
         socketCallback.callBack(buf);
     }

     @Override
     public void onError(WebSocket webSocket, Exception e) {
    
         Log.i(TAG, "onError:  " + e.toString());
     }

     @Override
     public void onStart() {
    

     }
};

2.开启和关闭

public void start() {
    
    webSocketServer.start();
}
public void close() {
    
    try {
    
        webSocket.close();
        webSocketServer.stop();
    } catch (IOException e) {
    
        e.printStackTrace();
    } catch (InterruptedException e) {
    
        e.printStackTrace();
    }
}

3.传输数据

public void sendData(byte[] bytes) {
    
    if (webSocket != null && webSocket.isOpen()) {
    
        Log.i("TAG", "sendData:  " + Arrays.toString(bytes));
        webSocket.send(bytes);
    }
}

4.2、WebScoket客户端

1.WebSocket配置

private class MyWebSocketClient extends WebSocketClient {
    

    public MyWebSocketClient(URI serverURI) {
    
        super(serverURI);
    }

    @Override
    public void onOpen(ServerHandshake serverHandshake) {
    
        Log.i(TAG, "打开 socket  onOpen: ");
    }

    @Override
    public void onMessage(String s) {
    
    }

//接收到对端数据
    @Override
    public void onMessage(ByteBuffer bytes) {
    
    //remaining是实际的数据长度
        byte[] buf = new byte[bytes.remaining()];
        bytes.get(buf);
        socketCallback.callBack(buf);

    }

    @Override
    public void onClose(int i, String s, boolean b) {
    
        Log.i(TAG, "onClose: ");
    }

    @Override
    public void onError(Exception e) {
    
        Log.i(TAG, "onError: ");
    }
}

public interface SocketCallback{
    
    void callBack(byte[] data);
}

2.WebSocket启动和连接

public void start() {
    
    try {
    
    //两个机子连同一个网络
        URI url = new URI("ws://xxx.xxx.xxx.xxx:12001");
        myWebSocketClient = new MyWebSocketClient(url);
        myWebSocketClient.connect();
    } catch (Exception e) {
    
        e.printStackTrace();
    }
}

五、MediaCodec解码推送过来的数据

---

public void callBack(byte[] data) {
    
    int index= mediaCodec.dequeueInputBuffer(100000);
    if (index >= 0) {
    
        ByteBuffer inputBuffer = mediaCodec.getInputBuffer(index);
        inputBuffer.clear();
        inputBuffer.put(data, 0, data.length);
        //dsp芯片解码 解码 的 传进去的 只需要保证编码顺序就好了  1000
        mediaCodec.queueInputBuffer(index,
                0, data.length, System.currentTimeMillis(), 0);
    }
    //        获取到解码后的数据  编码 ipbn
    MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
    int outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 100000);
    while (outputBufferIndex >=0) {
    
        mediaCodec.releaseOutputBuffer(outputBufferIndex, true);
        outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 0);
    }
}

详细代码位置