Search Unity

  1. We are migrating the Unity Forums to Unity Discussions by the end of July. Read our announcement for more information and let us know if you have any questions.
    Dismiss Notice
  2. Dismiss Notice

Question TCP communication between Unity and Python

Discussion in 'Scripting' started by JourneytoUngoro, May 6, 2024.

  1. JourneytoUngoro

    JourneytoUngoro

    Joined:
    Jul 10, 2023
    Posts:
    24
    I'm trying to use mediapipe with unity so I can track my hands in AR. Python will act as server and when receiving data, it will send back the landmarks of my hands. Unity will be the client and send jpg image data from mobile camera and receive the landmarks data. It works well when I tested it in the loopback ip(127.0.0.1) but when I test it in the server ip, it connets to the server but right after spits out error saying that "unable to read data from the transport connection". Can anybody help? Thank you in advance.

    Code (CSharp):
    1. using System;
    2. using System.Collections.Generic;
    3. using System.Net.Sockets;
    4. using System.Text;
    5. using UnityEngine;
    6. using UnityEngine.UI;
    7. using System.Threading.Tasks;
    8. using System.Linq;
    9. using System.IO;
    10. using System.Drawing;
    11. using System.Threading;
    12.  
    13. public class CameraManager : MonoBehaviour
    14. {
    15.     public RawImage display;
    16.     public InputField inputField;
    17.     public int width = 1280;
    18.     public int height = 720;
    19.     public int frameRate = 30;
    20.     public string receivedData;
    21.  
    22.     WebCamTexture camTexture;
    23.     int currentIndex = 0;
    24.     bool isConnected = false;
    25.     float timeCount = 0.0f;
    26.  
    27.  
    28.     string serverIP;
    29.     int port = 8000;
    30.  
    31.     TcpClient client;
    32.     NetworkStream stream;
    33.     Thread receiveThread;
    34.  
    35.     private void Start()
    36.     {
    37.         if (camTexture != null)
    38.         {
    39.             display.texture = null;
    40.             camTexture.Stop();
    41.             camTexture = null;
    42.         }
    43.         WebCamDevice device = WebCamTexture.devices[currentIndex];
    44.         Debug.Log(device.name);
    45.         camTexture = new WebCamTexture(device.name, width, height);
    46.         display.texture = camTexture;
    47.         Debug.Log(camTexture.height);
    48.         camTexture.Play();
    49.     }
    50.  
    51.     private void Update()
    52.     {
    53.         if (isConnected)
    54.         {
    55.             if (timeCount > 1.0f / frameRate)
    56.             {
    57.                 SendDataAsync();
    58.                 timeCount = 0.0f;
    59.             }
    60.             timeCount += Time.deltaTime;
    61.  
    62.         }
    63.     }
    64.  
    65.     public void ConnectToServer()
    66.     {
    67.         serverIP = inputField.text;
    68.         Debug.Log(serverIP);
    69.         try
    70.         {
    71.             client = new TcpClient(serverIP, port);
    72.             stream = client.GetStream();
    73.             Debug.Log("Connected to server");
    74.             isConnected = true;
    75.             receiveThread = new Thread(new ThreadStart(ReceiveData));
    76.             receiveThread.IsBackground = true;
    77.             receiveThread.Start();
    78.         }
    79.         catch (Exception e)
    80.         {
    81.             Debug.LogError("Socket error: " + e.Message);
    82.         }
    83.     }
    84.  
    85.     async void SendDataAsync()
    86.     {
    87.         try
    88.         {
    89.             byte[] separator = Encoding.UTF8.GetBytes("<END>");
    90.             List<byte> sendData = new List<byte>();
    91.             sendData.AddRange(CompressImage(ConvertWebCamTextureToTexture2D(camTexture)));
    92.             sendData.AddRange(separator);
    93.  
    94.             if (stream.CanWrite)
    95.             {
    96.                 await stream.WriteAsync(sendData.ToArray(), 0, sendData.Count);
    97.             }
    98.         }
    99.         catch (Exception e)
    100.         {
    101.             Debug.LogError($"Error in SendDataAsync: {e.Message}");
    102.         }
    103.     }
    104.  
    105.     void ReceiveData()
    106.     {
    107.         while (true)
    108.         {
    109.             try
    110.             {
    111.                 byte[] lengthBuffer = new byte[4];
    112.                 int totalRead = 0;
    113.  
    114.                 int bytesRead = stream.Read(lengthBuffer, 0, lengthBuffer.Length);
    115.                 if (bytesRead == lengthBuffer.Length)
    116.                 {
    117.                     int messageLength = BitConverter.ToInt32(lengthBuffer.Reverse().ToArray(), 0);
    118.                     byte[] dataBuffer = new byte[messageLength];
    119.  
    120.                     while (totalRead < messageLength)
    121.                     {
    122.                         int read = stream.Read(dataBuffer, 0, messageLength);
    123.                         totalRead += read;
    124.                     }
    125.  
    126.                     receivedData = Encoding.UTF8.GetString(dataBuffer);
    127.                     Debug.Log($"Received data of length: {messageLength}");
    128.                     Debug.Log(receivedData);
    129.                 }
    130.             }
    131.             catch (Exception e)
    132.             {
    133.                 Debug.LogError("Exception: " + e.Message);
    134.             }
    135.         }
    136.  
    137.     }
    138.  
    139.     Texture2D ConvertWebCamTextureToTexture2D(WebCamTexture webCamTexture)
    140.     {
    141.         Texture2D texture = new Texture2D(webCamTexture.width, webCamTexture.height);
    142.         texture.SetPixels32(webCamTexture.GetPixels32());
    143.         texture.Apply();
    144.         return texture;
    145.     }
    146.  
    147.     byte[] CompressImage(Texture2D texture)
    148.     {
    149.         byte[] imageBytes = texture.EncodeToJPG();
    150.         return imageBytes;
    151.     }
    152. }
    Code (CSharp):
    1. import cv2
    2. from cvzone.HandTrackingModule import HandDetector
    3. import numpy as np
    4. import socket
    5. import struct
    6.  
    7. width, height = 1280, 720
    8.  
    9. detector = HandDetector(maxHands=1, detectionCon=0.5)
    10.  
    11. server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    12. server_socket.bind(('0.0.0.0', 8000))
    13. server_socket.listen(1)
    14. conn, addr = server_socket.accept()
    15.  
    16. separator = b"<END>"
    17. recv_data = b""
    18. while True:
    19.     packet = conn.recv(1280 * 720)
    20.     # 4096
    21.  
    22.     recv_data += packet
    23.     if separator in recv_data:
    24.  
    25.         img_data, _, recv_data = recv_data.partition(separator)
    26.  
    27.         nparr = np.frombuffer(img_data, np.uint8)
    28.         img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
    29.  
    30.         hands, img = detector.findHands(img)
    31.         cv2.imshow('Received Image', img)
    32.         data = []
    33.         if hands:
    34.             hand = hands[0]
    35.             lmList = hand['lmList']
    36.             for lm in lmList:
    37.                 data.extend([lm[0], height - lm[1], lm[2]])
    38.  
    39.         send_data = str.encode(str(data))
    40.         data_length = struct.pack('>I', len(send_data))
    41.         conn.sendall(data_length + send_data)
    42.  
    43.         if cv2.waitKey(1) & 0xFF == 27:
    44.             break
    45.  
     
  2. Kurt-Dekker

    Kurt-Dekker

    Joined:
    Mar 16, 2013
    Posts:
    39,432
    Networking, UnityWebRequest, WWW, Postman, curl, WebAPI, etc:

    https://forum.unity.com/threads/using-unity-for-subscription-lists.1070663/#post-6910289

    https://forum.unity.com/threads/unity-web-request-acting-stupid.1096396/#post-7060150

    And setting up a proxy can be very helpful too, in order to compare traffic:

    https://support.unity.com/hc/en-us/articles/115002917683-Using-Charles-Proxy-with-Unity