Search Unity

  1. We are migrating the Unity Forums to Unity Discussions. On July 12, the Unity Forums will become read-only. On July 15, Unity Discussions will become read-only until July 18, when the new design and the migrated forum contents will go live. Read our full announcement for more information and let us know if you have any questions.

Question Sending webcam image from python to Meta Quest 2

Discussion in 'VR' started by jarain78, Apr 12, 2023.

  1. jarain78

    jarain78

    Joined:
    Sep 21, 2015
    Posts:
    1
    Hello everyone,

    I'm working on an application for my Meta Quest 2 where I want to add images from a webcam as a texture to a plane. To achieve this, I have created a Python script that uses OpenCV to capture images from the camera. These images are then base64-encoded and sent to Meta Quest 2 via sockets. However, I'm having a problem: if I run both applications (Python and Unity) on the same machine, Unity can add the images as a texture to the plane without the issues. But when I try to do the same from Meta Quest 2, the blueprint appears black, and the submitted images are not textured in the plane.

    If anyone knows how to fix this problem, I would really appreciate your help. Attached are the codes I am using:

    Python:

    Code (Python):
    1.  
    2. import base64
    3. import pickle
    4. import socket
    5. import cv2
    6. import imutils
    7. import time
    8. import threading
    9. import sys
    10.  
    11. cam = cv2.VideoCapture(0)
    12.  
    13. # encode to jpeg format
    14. # encode param image quality 0 to 100. default:95
    15. # if you want to shrink data size, choose low image quality.
    16. encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90]
    17.  
    18. # Create a TCP/IP socket
    19. sock_image = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    20.  
    21. # Bind the socket to the port
    22. server_data = ('192.168.1.102', 8052)
    23. server_image = ('192.168.1.102', 8053)
    24.  
    25. print('starting up on {} port {}'.format(*server_data))
    26. sock_data.bind(server_data)
    27.  
    28. print('starting up on {} port {}'.format(*server_image))
    29. sock_image.bind(server_image)
    30.  
    31. # Listen for incoming connections
    32. sock_data.listen(2)
    33. sock_image.listen(2)
    34.  
    35.  
    36. # Custom Thread Class
    37. class ImageThread(threading.Thread):
    38.     def task_send_image(self):
    39.         cam_img = True
    40.         while True:
    41.             # Wait for a connection
    42.             print('waiting for a connection')
    43.             connection_img, client_address_img = sock_image.accept()
    44.             # t2.start()
    45.  
    46.             try:
    47.                 print('connection from', client_address_img)
    48.  
    49.                 # Receive the data in small chunks and retransmit it
    50.                 while True:
    51.                     if cam_img:
    52.                         ret, frame = cam.read()
    53.                         frame = imutils.resize(frame, width=320)
    54.                         frame = cv2.flip(frame, 0)
    55.                         result, image = cv2.imencode('.jpg', frame, encode_param)
    56.                         jpg_as_text = base64.b64encode(image)
    57.  
    58.                     else:
    59.                         frame = cv2.imread('robotImage.jpg')
    60.                         frame = imutils.resize(frame, width=320)
    61.                         frame = cv2.flip(frame, 0)
    62.                         result, image = cv2.imencode('.jpg', frame, encode_param)
    63.                         jpg_as_text = base64.b64encode(image)
    64.  
    65.                     # connection_img.sendall(jpg_as_text)
    66.                     connection_img.sendto(jpg_as_text, client_address_img)
    67.                     time.sleep(0.2)
    68.  
    69.             except KeyboardInterrupt:
    70.                 connection_img.close()
    71.             finally:
    72.                 # Clean up the connection
    73.                 connection_img.close()
    74.             raise Exception('Something bad happened')
    75.  
    76.     def run(self):
    77.         self.task_send_image()
    78.  
    79.     def join(self):
    80.         threading.Thread.join(self)
    81. # Driver function
    82. def main():
    83.     image_thread = ImageThread()
    84.  
    85.     image_thread.start()
    86.  
    87.  
    88. # Driver code
    89. if __name__ == '__main__':
    90.     main()
    91.  

    C# Code:

    Code (CSharp):
    1. using System;
    2. using System.Collections;
    3. using System.Collections.Generic;
    4. using System.Net.Sockets;
    5. using System.Text;
    6. using System.Threading;
    7. using UnityEngine;
    8. using UnityEngine.XR;
    9. using UnityEngine.XR.Interaction.Toolkit;
    10. using System.Collections.Generic;
    11. using System.Threading;
    12. using UnityEngine;
    13. using System;
    14.  
    15. public class TCPTestClient : MonoBehaviour
    16. {
    17.  
    18.     public String Server_Addres = "127.0.0.1";
    19.     public int DataPort = 8052;
    20.     public int ImagePort = 8053;
    21.  
    22.     string serverMessage;
    23.  
    24.     public GameObject renderImage;
    25.  
    26.     byte[] bytes_image;
    27.  
    28.     private bool serverComm = false;
    29.  
    30.     #region private members    
    31.     private TcpClient socketConnectionData;
    32.     private TcpClient socketConnectionImage;
    33.     private Thread clientReceiveThread;
    34.     #endregion
    35.  
    36.     Renderer renderer;
    37.     // Use this for initialization    
    38.     void Start()
    39.     {
    40.         renderer = GetComponent<Renderer>();
    41.         ConnectImageServer();
    42.  
    43.     }
    44.     // Update is called once per frame
    45.     void Update()
    46.     {
    47.         setImageAsTexture();
    48.     }
    49.  
    50.     /// <summary>    
    51.     /// Setup socket connection.    
    52.     /// </summary>    
    53.     private void ConnectDataServer()
    54.     {
    55.         try
    56.         {
    57.             socketConnectionData = new TcpClient(Server_Addres, DataPort);
    58.         }
    59.         catch (Exception e)
    60.         {
    61.             serverComm = false;
    62.             Debug.Log("On client connect exception " + e);
    63.         }
    64.     }
    65.  
    66.     /// <summary>    
    67.     /// Setup socket connection.    
    68.     /// </summary>    
    69.     private void ConnectImageServer()
    70.     {
    71.         try
    72.         {
    73.             socketConnectionImage = new TcpClient(Server_Addres, ImagePort);
    74.  
    75.             clientReceiveThread = new Thread(new ThreadStart(ListenForData));
    76.             clientReceiveThread.IsBackground = true;
    77.             clientReceiveThread.Start();
    78.             serverComm = true;
    79.         }
    80.         catch (Exception e)
    81.         {
    82.             serverComm = false;
    83.             Debug.Log("On client connect exception " + e);
    84.         }
    85.     }
    86.  
    87.     private void setImageAsTexture()
    88.     {
    89.         if (serverMessage != null && serverComm == true)
    90.         {
    91.             Texture2D tex = new Texture2D(100, 100, TextureFormat.BGRA32, true, false);
    92.             tex.hideFlags = HideFlags.HideAndDontSave;
    93.             tex.filterMode = FilterMode.Point;
    94.             tex.LoadImage(bytes_image);
    95.             renderImage.GetComponent<Renderer>().material.mainTexture = tex;
    96.  
    97.         }
    98.     }
    99.     public static byte[] DecodeUrlBase64(string s)
    100.     {
    101.         s = s.Replace('-', '+').Replace('_', '/').PadRight(4 * ((s.Length + 3) / 4), '=');
    102.         return Convert.FromBase64String(s);
    103.     }
    104.  
    105. }
    106.  

    I appreciate your help.
     
  2. potu1304

    potu1304

    Joined:
    Jul 28, 2012
    Posts:
    4
    Hi,
    I try to accomplish something similar. How exactly did you make it work? I send my webcam feed to Unity and try to display it on a Texture, similar to your code, but it is not working.
     
  3. pervy_sage2004

    pervy_sage2004

    Joined:
    Jun 17, 2024
    Posts:
    1

    hey were you able to do it i am stuck at same point