HoloLensにiPhoneのARカメラの位置情報を送信する　HoloLensアドベントカレンダー5日目

本日はHoloLens枠です。

今回はHoloLensとiPhoneを通信してiPhoneのデバイスのトランスフォームをHoloLensに送信することを試していきます。

〇iPhoneとHoloLensの通信部

データ送信部は次のようになります。　

  public void SendDataToClient(byte[] message)
    {
        try
        {
            if (_connectedTcpClient != null)
            {
                NetworkStream clientStream = _connectedTcpClient.GetStream();

                // データを直接送信する
                byte[] dataToSend = message;
                clientStream.Write(dataToSend, 0, dataToSend.Length);
                Debug.Log("Data Send");
            }
            else
            {
                Debug.LogWarning("No connected client to send data to.");
            }
        }
        catch (Exception e)
        {
            Debug.LogError($"Error sending data from server: {e.Message}");
        }
    }

このSendDataToClientの引数にデータを渡す形になります。

今回はMessagePackを使用しています。 MessagePackを使用することでJsonライクなバイトデータの圧縮が行えます。

[DataContract]
public class UnityCameraData
{
    [DataMember] public string header;

    [DataMember] public List<float> cameratransform;

    [DataMember] public List<float> camerarotation;
}

今回はMessagePackのカスタムリゾルバーを使用して独自のデータを定義しています。

デバイスの位置情報＝カメラ情報のためスケールを除くHeader,ポジション、ローテーションを定義しています。

Headerはデータの処理において複数のデータ型を送信した際の区別に使用されます。

このデータにシリアライズする部分が次になります。

    public void SendCameraTransformToClient()
{
    Transform cameraTransform = Camera.main.transform;

        Debug.Log(cameraTransform.transform.position);
    // to sharing
    Transform targetTransform = this.transform;

    Vector3 relativePosition = targetTransform.InverseTransformPoint(cameraTransform.position);
    Quaternion relativeRotation = Quaternion.Inverse(targetTransform.rotation) * cameraTransform.rotation;

    Dictionary<string, object> cameraData = new Dictionary<string, object>();
    cameraData.Add("header", "UCAM");

    cameraData.Add("cameratransform", new List<float>() {
        relativePosition.x,
        relativePosition.y,
        relativePosition.z
    });
    cameraData.Add("camerarotation", new List<float>() {
        -relativeRotation.eulerAngles.x,
        -relativeRotation.eulerAngles.y,
        -relativeRotation.eulerAngles.z
    });


    // MessagePackシリアライズ
    var options = MessagePackSerializerOptions.Standard.WithResolver(CompositeResolver.Create(
        new IMessagePackFormatter[] { new UnityCameraDataFormatter() },
        new IFormatterResolver[] { CustomUnityCameraResolver.Instance }
    ));

        byte[] serializedData = MessagePackSerializer.Serialize(cameraData, options);
        Debug.Log(BitConverter.ToString(serializedData).Replace("-", ""));
        SendDataToClient(serializedData);
}

MessaePackではMessagePackSerializer.Serialize(データ)を使用することでデータをシリアライズすることができますが、独自のデータクラスを使用する場合optionsを定義することができます。

今回はUnityCameraDataFormatterおよびCustomUnityCameraResolverを定義しています。

    var options = MessagePackSerializerOptions.Standard.WithResolver(CompositeResolver.Create(
        new IMessagePackFormatter[] { new UnityCameraDataFormatter() },
        new IFormatterResolver[] { CustomUnityCameraResolver.Instance }
    ));

ここでシリアライズ、およびデシリアライズの定義を行っています。

定義に関しては記事末尾のコードを参照ください。

こうして送信してデータからデータの冒頭部を4文字分取得してMessagePackで定義されたヘッダー情報を読み取り処理をしています。

   private void ProcessReceivedData(byte[] data, int length)
   {

       string header = Encoding.ASCII.GetString(data, 9, 4);
       ・・・
       else if (header == "UCAM")
       {
           // 受信したデータの16進数をログに出力
           Debug.Log("Received data in hex: " + BitConverter.ToString(data).Replace("-", ""));
           // カメラデータをデシリアライズ
           cameraData = DeserializeCameraData(data, length);
           MainThreadDispatcher.Enqueue(() => _isSetCamData = true);
           MainThreadDispatcher.Execute();
       }
       else
       {
           Debug.Log($"Received unknown data with header: {header}");
       }
   }

なおここでMainThreadDispatcher.Enqueue()を使用していますが、これは通常サブスレッドからメインスレッドの処理にアクセスすることができないため、それを回避するためのクラスとして定義しています。

public static class MainThreadDispatcher
{
    private static Queue<Action> _actions = new Queue<Action>();
    private static readonly object _lockObject = new object();

    public static void Enqueue(Action action)
    {
        lock (_lockObject)
        {
            _actions.Enqueue(action);
        }
    }

    public static void Execute()
    {
        lock (_lockObject)
        {
            while (_actions.Count > 0)
            {
                Action action = _actions.Dequeue();
                action?.Invoke();
            }
        }
    }
}

今回はデータを受け取った場合自身のbool値を変えてUpdate関数で検出、実際の処理に回しています。

  void Update()
  {
      if (_access)
      {
          lock (_lockObject)
          {
              _access = false;
          }
          gameObject.SetActive(!gameObject.activeSelf);
          //MainThreadDispatcher.Enqueue(() => multiPlayEvent.isReceiveEvent = true);
      }
      if (_isSetCamData) {
          lock (_lockObject) {
              _isSetCamData= false;
          }
          _multiUserEventManager.isCameraDateGet = true;
          _multiUserEventManager.cameraData = cameraData;

      }
  }

分量もですがトライアルアンドエラーも多かったので筆者自身も初めての分野で大変でしたが、何とか動きました。

〇サーバー側

サーバー側はClient側が起動する前に先に実行する必要があります。

今回はIphone側をサーバーにしています。

using System;
using System.IO;
using System.Collections;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.Text;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
using System.Threading;
using MessagePack;
using System.Runtime.Serialization;
using MessagePack.Formatters;
using MessagePack.Resolvers;
using System.Runtime.InteropServices.ComTypes;

public class Server : MonoBehaviour
{
    private TcpListener _tcpListener;
    private Thread _tcpListenerThread;
    private TcpClient _connectedTcpClient;
    public TextMeshProUGUI logText;

    private readonly Queue<string> _logQueue = new Queue<string>();
    private readonly object _lockObject = new object();


    [Tooltip("Port number")] public int _port = 9991; // Default is 9991

    private void Start()
    {
        _tcpListenerThread = new Thread(new ThreadStart(ListenForIncomingRequests));
        _tcpListenerThread.IsBackground = true;
        _tcpListenerThread.Start();

        // Start a coroutine to process logs from the main thread
        StartCoroutine(ProcessLogQueue());
    }


    private void Update()
    {
        // Process logs in the main thread
        MainThreadDispatcher.Execute();
    }

    private void ListenForIncomingRequests()
    {
        try
        {
            _tcpListener = new TcpListener(IPAddress.Any, _port);
            _tcpListener.Start();

            Debug.Log($"Server is listening on port {_port}");

            while (true)
            {
                _connectedTcpClient = _tcpListener.AcceptTcpClient();

                Thread clientThread = new Thread(new ParameterizedThreadStart(HandleClientComm));
                clientThread.Start(_connectedTcpClient);
            }
        }
        catch (Exception e)
        {
            Debug.LogError($"Error: {e.Message}");
        }
    }

    private void HandleClientComm(object clientObj)
    {
        TcpClient tcpClient = (TcpClient)clientObj;
        NetworkStream clientStream = tcpClient.GetStream();

        byte[] message = new byte[4096];
        int bytesRead;

        while (true)
        {
            bytesRead = 0;

            try
            {
                bytesRead = clientStream.Read(message, 0, 4096);
            }
            catch (Exception e)
            {
                Debug.LogError($"Error reading from client: {e.Message}");
                break;
            }
            if (bytesRead == 0)
                break;

            ProcessReceivedData(message, bytesRead);
        }

        tcpClient.Close();
    }

    private void ProcessReceivedData(byte[] data, int length)
    {
        string header = Encoding.ASCII.GetString(data, 0, 3);
        Debug.Log($"Received header: {header}");

        if (header == "LOG")
        {
            string logMessage = Encoding.ASCII.GetString(data, 3, length - 3);
            Debug.Log($"Received log message: {logMessage}");

            // Add log message to the queue for processing in the main thread
            lock (_lockObject)
            {
                _logQueue.Enqueue(logMessage);
            }
        }
        if (header == "TRF")
        {
            // カメラデータの部分を抽出
            byte[] cameraDataBytes = new byte[length - 3];
            Array.Copy(data, 3, cameraDataBytes, 0, cameraDataBytes.Length);
        }
        else
        {
            Debug.Log($"Received unknown data with header: {header}");
        }
    }

    public void SendCameraTransformToClient()
{
    Transform cameraTransform = Camera.main.transform;

        Debug.Log(cameraTransform.transform.position);
    // to sharing
    Transform targetTransform = this.transform;

    Vector3 relativePosition = targetTransform.InverseTransformPoint(cameraTransform.position);
    Quaternion relativeRotation = Quaternion.Inverse(targetTransform.rotation) * cameraTransform.rotation;

    Dictionary<string, object> cameraData = new Dictionary<string, object>();
    cameraData.Add("header", "UCAM");

    cameraData.Add("cameratransform", new List<float>() {
        relativePosition.x,
        relativePosition.y,
        relativePosition.z
    });
    cameraData.Add("camerarotation", new List<float>() {
        -relativeRotation.eulerAngles.x,
        -relativeRotation.eulerAngles.y,
        -relativeRotation.eulerAngles.z
    });


    // MessagePackシリアライズ
    var options = MessagePackSerializerOptions.Standard.WithResolver(CompositeResolver.Create(
        new IMessagePackFormatter[] { new UnityCameraDataFormatter() },
        new IFormatterResolver[] { CustomUnityCameraResolver.Instance }
    ));

        byte[] serializedData = MessagePackSerializer.Serialize(cameraData, options);
        Debug.Log(BitConverter.ToString(serializedData).Replace("-", ""));
        SendDataToClient(serializedData);
}
    public static byte[] SerializeTransform(Transform transform)
    {
        // カメラのTransform情報を含むすべてのデータを取得
        Vector3 position = transform.position;
        Quaternion rotation = transform.rotation;
        Vector3 scale = transform.localScale;

        // バイトに変換
        List<byte> bytes = new List<byte>();
        bytes.AddRange(BitConverter.GetBytes(position.x));
        bytes.AddRange(BitConverter.GetBytes(position.y));
        bytes.AddRange(BitConverter.GetBytes(position.z));
        bytes.AddRange(BitConverter.GetBytes(rotation.x));
        bytes.AddRange(BitConverter.GetBytes(rotation.y));
        bytes.AddRange(BitConverter.GetBytes(rotation.z));
        bytes.AddRange(BitConverter.GetBytes(rotation.w));
        bytes.AddRange(BitConverter.GetBytes(scale.x));
        bytes.AddRange(BitConverter.GetBytes(scale.y));
        bytes.AddRange(BitConverter.GetBytes(scale.z));

        return bytes.ToArray();
    }
    public void SendDataToClient(string message)
    {
        try
        {
            if (_connectedTcpClient != null)
            {
                NetworkStream clientStream = _connectedTcpClient.GetStream();

                // データを直接送信する
                byte[] dataToSend = Encoding.ASCII.GetBytes(message);

                clientStream.Write(dataToSend, 0, dataToSend.Length);
                Debug.Log("Data Send");
            }
            else
            {
                Debug.LogWarning("No connected client to send data to.");
            }
        }
        catch (Exception e)
        {
            Debug.LogError($"Error sending data from server: {e.Message}");
        }
    }

    public void SendDataToClient(byte[] message)
    {
        try
        {
            if (_connectedTcpClient != null)
            {
                NetworkStream clientStream = _connectedTcpClient.GetStream();

                // データを直接送信する
                byte[] dataToSend = message;
                clientStream.Write(dataToSend, 0, dataToSend.Length);
                Debug.Log("Data Send");
            }
            else
            {
                Debug.LogWarning("No connected client to send data to.");
            }
        }
        catch (Exception e)
        {
            Debug.LogError($"Error sending data from server: {e.Message}");
        }
    }
    private IEnumerator ProcessLogQueue()
    {
        while (true)
        {
            yield return null; // Wait for the next frame

            // Process logs in the main thread
            lock (_lockObject)
            {
                while (_logQueue.Count > 0)
                {
                    string logMessage = _logQueue.Dequeue();
                    AppendLogText(logMessage);
                }
            }
        }
    }

    private void AppendLogText(string message)
    {
        // Append the new log message to the existing log text using StringBuilder
        StringBuilder sb = new StringBuilder(logText.text);
        sb.AppendLine(message);
        logText.text = sb.ToString();
    }

    private void OnDestroy()
    {
        if (_tcpListener != null)
        {
            _tcpListener.Stop();
        }
    }
}



// Helper class to dispatch actions to the main thread
public static class MainThreadDispatcher
{
    private static Queue<Action> _actions = new Queue<Action>();
    private static readonly object _lockObject = new object();

    public static void Enqueue(Action action)
    {
        lock (_lockObject)
        {
            _actions.Enqueue(action);
        }
    }

    public static void Execute()
    {
        lock (_lockObject)
        {
            while (_actions.Count > 0)
            {
                Action action = _actions.Dequeue();
                action?.Invoke();
            }
        }
    }
}

[DataContract]
public class UnityCameraData
{
    [DataMember] public string header;

    [DataMember] public List<float> cameratransform;

    [DataMember] public List<float> camerarotation;
}

〇Clientの処理

using System;
using System.IO;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using JetBrains.Annotations;
using MessagePack;
using UnityEditor;
using UnityEngine;
using UnityEngine.Events;
using UnityEngine.UI;



public class Client : MonoBehaviour
{
    [CanBeNull] private TcpClient _client;
    private NetworkStream _stream;

    [Tooltip("Port number")] public int _port = 9991; // Default is 9991
    [Tooltip("Address of the server")] public string _ipAddress = "localhost";

    bool _access = false;
    bool _isSetCamData = false;
    public RawImage rawImage;

    public UnityCameraData cameraData;

    [SerializeField]
    MultiPlayEventManager _multiUserEventManager;

    public void StartConnection()
    {
        _client = new TcpClient(_ipAddress, _port);
        _stream = _client.GetStream();

        // Start a new thread to listen for incoming messages
        new Thread(() =>
        {
            var responseBytes = new byte[4096];//[536870912];
            try
            {
                using (MemoryStream ms = new MemoryStream())
                {
                    byte[] buffer = new byte[4096];
                    int bytesRead;

                    while ((bytesRead = _stream.Read(buffer, 0, buffer.Length)) > 0)
                    {
                        ms.Write(buffer, 0, bytesRead);

                        // Check if the received data ends with a specific delimiter
                        if (bytesRead < buffer.Length)
                        {
                            ProcessReceivedData(ms.ToArray(), (int)ms.Length);
                            ms.SetLength(0);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Debug.LogError($"Exception in StartConnection thread: {ex.Message}");
                // Handle the exception as needed
            }
            finally
            {
                // Clean up resources if necessary
                _client.Close();
            }
        }).Start();
    }

    private void ProcessReceivedData(byte[] data, int length)
    {

        string header = Encoding.ASCII.GetString(data, 9, 4);

        // Only the part related to processing received data has been kept
        if (header == "LOG")
        {
            // Extract the log message from the received data
            string logMessage = Encoding.ASCII.GetString(data, 13, length - 13);
            Debug.Log($"Received log message: {logMessage}");
        }
        if (header == "FUN")
        {
            string functionMessage = Encoding.ASCII.GetString(data, 13, length - 13);

            Debug.Log("functionMessage");

            MainThreadDispatcher.Enqueue(() => _access = true);
            MainThreadDispatcher.Execute();

        }
        else if (header == "IMG")
        {

            // Extract the Base64-encoded image data from the received data
            string base64ImageData = Encoding.ASCII.GetString(data, 13, length - 13);


            // Convert the Base64 string to byte array
            byte[] imageBytes = Convert.FromBase64String(base64ImageData);

            // 画像をメインスレッドで処理する
            MainThreadDispatcher.Enqueue(() =>
            {
                // 画像を表示などの処理を行う
                Texture2D receivedTexture = new Texture2D(2, 2);
                receivedTexture.LoadImage(imageBytes);

                // 例：受信した画像をRawImageに表示
                rawImage.texture = receivedTexture;
                Debug.Log("Received and displayed image.");
            });
        }

        else if (header == "UCAM")
        {
            // 受信したデータの16進数をログに出力
            Debug.Log("Received data in hex: " + BitConverter.ToString(data).Replace("-", ""));
            // カメラデータをデシリアライズ
            cameraData = DeserializeCameraData(data, length);
            MainThreadDispatcher.Enqueue(() => _isSetCamData = true);
            MainThreadDispatcher.Execute();
        }
        else
        {
            Debug.Log($"Received unknown data with header: {header}");
        }
    }
    UnityCameraData DeserializeCameraData(byte[] data, int length)
    {

        var options = MessagePackSerializerOptions.Standard.WithResolver(CustomCameraDataResolver.Instance);
        Debug.Log(options);
        return MessagePackSerializer.Deserialize<UnityCameraData>(data, options);
    }



    void Start()
    {
        StartConnection();
    }
    private readonly object _lockObject = new object();
    void Update()
    {
        if (_access)
        {
            lock (_lockObject)
            {
                _access = false;
            }
            gameObject.SetActive(!gameObject.activeSelf);
            //MainThreadDispatcher.Enqueue(() => multiPlayEvent.isReceiveEvent = true);
        }
        if (_isSetCamData) {
            lock (_lockObject) {
                _isSetCamData= false;
            }
            _multiUserEventManager.isCameraDateGet = true;
            _multiUserEventManager.cameraData = cameraData;

        }
    }

    public void RequestDebugLog(string logMessage)
    {
        byte[] logBytes = Encoding.ASCII.GetBytes(logMessage);
        byte[] dataToSend = new byte[logBytes.Length + 13];
        Encoding.ASCII.GetBytes("LOG").CopyTo(dataToSend, 0);
        logBytes.CopyTo(dataToSend, 3); // 修正: 3からコピーするように変更
        _stream.Write(dataToSend, 0, dataToSend.Length);
    }

    private void OnDestroy()
    {
        // Unity エディターモードでのみ実行されるコード
#if UNITY_EDITOR
        if (!EditorApplication.isPlayingOrWillChangePlaymode)
        {
            // エディターモードのクリーンアップ処理
            EditorApplication.ExitPlaymode();
            return;
        }
#endif

        // 実行モード時のクリーンアップ処理
        _stream.Close();
        _client.Close();
    }
}

〇リゾルバー

{
    public void Serialize(ref MessagePackWriter writer, UnityCameraData value, MessagePackSerializerOptions options)
    {
        writer.WriteArrayHeader(3);
        options.Resolver.GetFormatterWithVerify<string>().Serialize(ref writer, value.header, options);
        options.Resolver.GetFormatterWithVerify<List<float>>().Serialize(ref writer, value.cameratransform, options);
        options.Resolver.GetFormatterWithVerify<List<float>>().Serialize(ref writer, value.camerarotation, options);

    }
    public UnityCameraData Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
    {
        if (reader.TryReadNil())
        {
            return null;
        }
        options.Security.DepthStep(ref reader);
        int length = 0;
        try
        {
            length = reader.ReadMapHeader();
            Debug.Log($"Read Array Header. Length: {length}");
            if (length != 3)
            {
                throw new MessagePackSerializationException("Invalid map length.");
            }
        }
        catch (Exception ex)
        {
            Debug.LogError($"Error reading map header: {ex.Message}");
        }



        string header = null;
        List<float> cameratransform = null;
        List<float> camerarotation = null;
        for (int i = 0; i < length; i++)
        {
            string key = reader.ReadString();
            switch (key)
            {
                case "header":
                    header = options.Resolver.GetFormatterWithVerify<string>().Deserialize(ref reader, options);
                    break;
                case "cameratransform":
                    cameratransform = options.Resolver.GetFormatterWithVerify<List<float>>().Deserialize(ref reader, options);
                    break;
                case "camerarotation":
                    camerarotation = options.Resolver.GetFormatterWithVerify<List<float>>().Deserialize(ref reader, options);
                    break;
                default:
                    reader.Skip();
                    break;
            }
        }
        reader.Depth--;
        return new UnityCameraData() { header = header, cameratransform = cameratransform, camerarotation = camerarotation };
    }