Трансляция живого видео между приложениями

Есть задача: транслировать живое видео с устройства (например, с телефона на Android), при этом читать поток должно другое приложение. Но организовать поток мне никак не удаётся: в лучшем случае удалённое приложение получает и отображает только первый кадр. При этом, если я правильно понимаю, приложение на устройстве поток отправляет.

Класс-отправитель:

using System;
using System.Collections;
using System.Net.Sockets;
using UnityEngine;
using TMPro;
using UnityEngine.UI;
using System.IO;
using System.Threading.Tasks;

public class Connector : MonoBehaviour
{
    TcpClient tcp;
    public TextMeshProUGUI label;
    public Button sendButton;
    public Button connectButton;

NetworkStream stream;
BinaryWriter writer;

bool connected = false;

void Awake() => Application.targetFrameRate = 30;

void Start()
{

}

void Update(){
    sendButton.gameObject.SetActive(connected);
    connectButton.gameObject.SetActive(!connected);
}

public void Connect(){
    try {
        label.text = "Connecting...";
        tcp = new TcpClient {NoDelay = true};
        tcp.Connect(
            Options.i.serverAddress, 
            Options.i.serverPort);
        label.text = "Connection established.";

        stream = tcp.GetStream();
        writer = new BinaryWriter(stream);

        connected = true;
    }
    catch (Exception ex){
        label.text = ex.Message;
    }
}

public void Send() => StartCoroutine(ProcessSending());

IEnumerator ProcessSending(){
    var endOfFrame = new WaitForEndOfFrame();
    bool readyToGetFrame;

    while (true){
        yield return endOfFrame;

        byte[] frameBytesLength = new byte[Options.i.sendRecieveCount];
        byte[] rawFrame = GetCameraLastImage();

        ByteLengthToFrameByteArray(rawFrame.Length, frameBytesLength);

        readyToGetFrame = false;
        Task.Run(() => {
            //Send total byte count first
            writer.Write(frameBytesLength, 0, frameBytesLength.Length);
            //label.text = "Sent Image byte Length: " + frameBytesLength.Length;

            //Send the image bytes
            writer.Write(rawFrame, 0, rawFrame.Length);
            label.text = "Sending Image byte array data : " + rawFrame.Length;

            readyToGetFrame = true;
        });

        while (!readyToGetFrame)
            yield return null;
    }
}

void ByteLengthToFrameByteArray(int byteLength, byte[] fullBytes)
{
    Array.Clear(fullBytes, 0, fullBytes.Length);
    byte[] bytesToSendCount = BitConverter.GetBytes(byteLength);
    bytesToSendCount.CopyTo(fullBytes, 0);
}

public byte[] GetCameraLastImage(){
    //получение изображения с камеры устройства
}
}

Класс-получатель:

using System;
using System.Net.Sockets;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.UI;

public class Listener : MonoBehaviour
{
    TcpListener server = null;
    TcpClient client = null;
    public RawImage projector;
    public AspectRatioFitter fitter;
bool running = false;

byte[] frameData { get; set; }
bool needUpdate = false;
bool readyToUpdate = true;

Texture2D texture;

void Start()
{
    texture = new Texture2D(1, 1, TextureFormat.RGB24, false, true);

    server = new TcpListener(Options.i.serverAddress, Options.i.serverPort);

    server.Start();
    Debug.Log("Waiting for connections...");
    running = true;
    Task.Run(async () => await Listen());
}

void Update(){
    if (!needUpdate)
        return;

    texture.LoadImage(frameData);
    texture.Apply();
    projector.texture = texture;
    projector.gameObject.SetActive(true);
    fitter.aspectRatio = (float)texture.width / texture.height;

    needUpdate = false;
    readyToUpdate = true;
}

async Task Listen()
{
    var strem =  client.GetStream();
    while (running){
        var size = Options.i.sendRecieveCount;
        bool disconnected = false;

        client = await server.AcceptTcpClientAsync();
        //HandleClient(await server.AcceptTcpClientAsync());
        var stream = client.GetStream();
        var imageBytesCount = new byte[size];

        var total = 0;
        do {
            var read = stream.Read(imageBytesCount, total, size - total);
            //Debug.LogFormat("Client recieved {0} bytes", total);
            if (read == 0)
            {
                disconnected = true;
                break;
            }
            total += read;
        } 
        while (total != size);

        var imageSize = disconnected
            ? -1
            : GetLength(imageBytesCount);

        await ReadStream(imageSize, stream);
    }
}

int GetLength(byte[] frameBytesLength)
{
    int byteLength = BitConverter.ToInt32(frameBytesLength, 0);
    return byteLength;
}

async Task ReadStream(int size, NetworkStream clientStream)
{
    bool disconnected = false;

    byte[] imageBytes = new byte[size];
    var total = 0;
    do
    {
        var read = clientStream.Read(imageBytes, total, size - total);
        if (read == 0)
        {
            disconnected = true;
            break;
        }
        total += read;
    } 
    while (total != size);
    Debug.LogFormat("{0} bytes recieved", total);

    //Display Image
    if (!disconnected)
    {
        //Display Image on the main Thread
        readyToUpdate = false;
        UpdateFrame(imageBytes);
    }

    //Wait until old Image is displayed
    while (!readyToUpdate)
        await Task.Delay(1);
        //System.Threading.Thread.Sleep(1);
}

private void UpdateFrame(byte[] rawFrame)
{
    frameData = rawFrame;
    needUpdate = true;
}
}

Что мне нужно сделать, чтобы поток заработал?


Ответы (1 шт):

Автор решения: eastwing

Итак, проблема была в глупой ошибке в моём коде, из-за которой обработка потока останавливалась, когда Listener получал 0 байт. Приведу исправленную версию кода. Пожалуйста, учтите, что этот код не следует копировать "как есть". В нём не рассматривается потеря соединения и не предусмотрен штатный разрыв соединения. Это просто набросок, пример того, как можно реализовать трансляцию, причём только в одну сторону и в режиме 1х1 :)

Listener:

using System;
using System.Net;
using System.Net.Sockets;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.UI;

public class Listener : MonoBehaviour
{
    TcpListener server = null;
    TcpClient client = null;
    public RawImage projector;
    public AspectRatioFitter fitter;

    bool running = false;

    byte[] frameData { get; set; }
    bool needUpdate = false;
    bool readyToUpdate = true;

    Texture2D texture;

    void Start()
    {
        texture = new Texture2D(1, 1, TextureFormat.RGB24, false, true);

        server = new TcpListener(Options.i.serverAddress, Options.i.serverPort);

        server.Start();
        Debug.Log("Waiting for connections...");
        running = true;
        Task.Run(async () => await Listen());
    }

    void Update(){
        if (!needUpdate)
            return;

        texture.LoadImage(frameData);
        texture.Apply();
        projector.texture = texture;
        projector.gameObject.SetActive(true);
        fitter.aspectRatio = (float)texture.width / texture.height;

        needUpdate = false;
        readyToUpdate = true;
    }

    async Task Listen()
    {
        client = await server.AcceptTcpClientAsync();
        var stream = client.GetStream();
        Debug.Log($"{((IPEndPoint)client.Client.RemoteEndPoint).Address} connected");

        while (running){
            var size = Options.i.sendRecieveCount;
            bool noData = false;

            var imageBytesCount = new byte[size];

            var total = 0;
            do {
                var read = stream.Read(imageBytesCount, total, size - total);
                noData = read == 0;

                total += read;
            } 
            while (total != size);

            var imageSize = noData
                ? -1
                : GetLength(imageBytesCount);

            await ReadStream(imageSize, stream);
        }
    }

    int GetLength(byte[] frameBytesLength)
    {
        int byteLength = BitConverter.ToInt32(frameBytesLength, 0);
        return byteLength;
    }

    async Task ReadStream(int size, NetworkStream clientStream)
    {
        bool disconnected = false;

        byte[] imageBytes = new byte[size];
        var total = 0;
        do
        {
            var read = clientStream.Read(imageBytes, total, size - total);
            if (read == 0)
            {
                disconnected = true;
                break;
            }
            total += read;
        } 
        while (total != size);
        Debug.LogFormat("{0} bytes recieved", total);

        //Display Image
        if (!disconnected)
        {
            //Display Image on the main Thread
            readyToUpdate = false;
            UpdateFrame(imageBytes);
        }

        //Wait until old Image is displayed
        while (!readyToUpdate)
            await Task.Delay(1);
    }

    private void UpdateFrame(byte[] rawFrame)
    {
        frameData = rawFrame;
        needUpdate = true;
    }
}

Connector / Sender:

using System;
using System.Collections;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using UnityEngine;
using TMPro;
using UnityEngine.UI;
using System.IO;
using System.Threading.Tasks;
using Unity.Collections;

public class Connector : MonoBehaviour
{
    TcpClient tcp;
    public TextMeshProUGUI label;
    public Button sendButton;
    public Button connectButton;

    public ARCameraBackground cameraBackground;
    public ARCameraManager cameraManager;
    public RenderTexture renderTexture;

    NetworkStream stream;
    BinaryWriter writer;

    bool connected = false;

    void Awake() => Application.targetFrameRate = 30;

    void Update(){
        sendButton.gameObject.SetActive(connected);
        connectButton.gameObject.SetActive(!connected);
    }

    public void Connect(){
        try {
            label.text = "Connecting...";
            tcp = new TcpClient {NoDelay = true};
            tcp.Connect(
                Options.i.serverAddress, 
                Options.i.serverPort);
            label.text = "Connection established.";

            stream = tcp.GetStream();
            writer = new BinaryWriter(stream);

            connected = true;
        }
        catch (Exception ex){
            label.text = ex.Message;
        }
    }

    public void Send() => StartCoroutine(ProcessSending());

    IEnumerator ProcessSending(){
        var endOfFrame = new WaitForEndOfFrame();
        bool readyToGetFrame;
        byte[] frameBytesLength = new byte[Options.i.sendRecieveCount];

        while (true){
            yield return endOfFrame;

            byte[] rawFrame = GetCameraLastImage();
            if (rawFrame == null)
                continue;

            ByteLengthToFrameByteArray(rawFrame.Length, frameBytesLength);

            readyToGetFrame = false;
            Task.Run(() => {
                writer.Write(frameBytesLength, 0, frameBytesLength.Length);

                //Send the image bytes
                writer.Write(rawFrame, 0, rawFrame.Length);
                label.text = "Sending Image byte array data : " + rawFrame.Length;

                readyToGetFrame = true;
            });

            while (!readyToGetFrame)
                yield return null;
        }
    }

    void ByteLengthToFrameByteArray(int byteLength, byte[] fullBytes)
    {
        //Clear old data
        Array.Clear(fullBytes, 0, fullBytes.Length);
        //Convert int to bytes
        byte[] bytesToSendCount = BitConverter.GetBytes(byteLength);
        //Copy result to fullBytes
        bytesToSendCount.CopyTo(fullBytes, 0);
    }

    unsafe byte[] GetCameraLastImage(){
        // получение картинки с камеры
    }
}

Пример значений из Options:

sendRecieveCount = 15;
serverAddress => IPAddress.Parse("192.168.1.5");
serverPort = 5665;
→ Ссылка