Search Unity

  1. Unity support for visionOS is now available. Learn more in our blog post.
    Dismiss Notice

Question Sending webcam image from python to Meta Quest 2

Discussion in 'VR' started by jarain78, Apr 12, 2023.

  1. jarain78

    jarain78

    Joined:
    Sep 21, 2015
    Posts:
    1
    Hello everyone,

    I'm working on an application for my Meta Quest 2 where I want to add images from a webcam as a texture to a plane. To achieve this, I have created a Python script that uses OpenCV to capture images from the camera. These images are then base64-encoded and sent to Meta Quest 2 via sockets. However, I'm having a problem: if I run both applications (Python and Unity) on the same machine, Unity can add the images as a texture to the plane without the issues. But when I try to do the same from Meta Quest 2, the blueprint appears black, and the submitted images are not textured in the plane.

    If anyone knows how to fix this problem, I would really appreciate your help. Attached are the codes I am using:

    Python:

    Code (Python):
    1.  
    2. import base64
    3. import pickle
    4. import socket
    5. import cv2
    6. import imutils
    7. import time
    8. import threading
    9. import sys
    10.  
    11. cam = cv2.VideoCapture(0)
    12.  
    13. # encode to jpeg format
    14. # encode param image quality 0 to 100. default:95
    15. # if you want to shrink data size, choose low image quality.
    16. encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90]
    17.  
    18. # Create a TCP/IP socket
    19. sock_image = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    20.  
    21. # Bind the socket to the port
    22. server_data = ('192.168.1.102', 8052)
    23. server_image = ('192.168.1.102', 8053)
    24.  
    25. print('starting up on {} port {}'.format(*server_data))
    26. sock_data.bind(server_data)
    27.  
    28. print('starting up on {} port {}'.format(*server_image))
    29. sock_image.bind(server_image)
    30.  
    31. # Listen for incoming connections
    32. sock_data.listen(2)
    33. sock_image.listen(2)
    34.  
    35.  
    36. # Custom Thread Class
    37. class ImageThread(threading.Thread):
    38.     def task_send_image(self):
    39.         cam_img = True
    40.         while True:
    41.             # Wait for a connection
    42.             print('waiting for a connection')
    43.             connection_img, client_address_img = sock_image.accept()
    44.             # t2.start()
    45.  
    46.             try:
    47.                 print('connection from', client_address_img)
    48.  
    49.                 # Receive the data in small chunks and retransmit it
    50.                 while True:
    51.                     if cam_img:
    52.                         ret, frame = cam.read()
    53.                         frame = imutils.resize(frame, width=320)
    54.                         frame = cv2.flip(frame, 0)
    55.                         result, image = cv2.imencode('.jpg', frame, encode_param)
    56.                         jpg_as_text = base64.b64encode(image)
    57.  
    58.                     else:
    59.                         frame = cv2.imread('robotImage.jpg')
    60.                         frame = imutils.resize(frame, width=320)
    61.                         frame = cv2.flip(frame, 0)
    62.                         result, image = cv2.imencode('.jpg', frame, encode_param)
    63.                         jpg_as_text = base64.b64encode(image)
    64.  
    65.                     # connection_img.sendall(jpg_as_text)
    66.                     connection_img.sendto(jpg_as_text, client_address_img)
    67.                     time.sleep(0.2)
    68.  
    69.             except KeyboardInterrupt:
    70.                 connection_img.close()
    71.             finally:
    72.                 # Clean up the connection
    73.                 connection_img.close()
    74.             raise Exception('Something bad happened')
    75.  
    76.     def run(self):
    77.         self.task_send_image()
    78.  
    79.     def join(self):
    80.         threading.Thread.join(self)
    81. # Driver function
    82. def main():
    83.     image_thread = ImageThread()
    84.  
    85.     image_thread.start()
    86.  
    87.  
    88. # Driver code
    89. if __name__ == '__main__':
    90.     main()
    91.  

    C# Code:

    Code (CSharp):
    1. using System;
    2. using System.Collections;
    3. using System.Collections.Generic;
    4. using System.Net.Sockets;
    5. using System.Text;
    6. using System.Threading;
    7. using UnityEngine;
    8. using UnityEngine.XR;
    9. using UnityEngine.XR.Interaction.Toolkit;
    10. using System.Collections.Generic;
    11. using System.Threading;
    12. using UnityEngine;
    13. using System;
    14.  
    15. public class TCPTestClient : MonoBehaviour
    16. {
    17.  
    18.     public String Server_Addres = "127.0.0.1";
    19.     public int DataPort = 8052;
    20.     public int ImagePort = 8053;
    21.  
    22.     string serverMessage;
    23.  
    24.     public GameObject renderImage;
    25.  
    26.     byte[] bytes_image;
    27.  
    28.     private bool serverComm = false;
    29.  
    30.     #region private members    
    31.     private TcpClient socketConnectionData;
    32.     private TcpClient socketConnectionImage;
    33.     private Thread clientReceiveThread;
    34.     #endregion
    35.  
    36.     Renderer renderer;
    37.     // Use this for initialization    
    38.     void Start()
    39.     {
    40.         renderer = GetComponent<Renderer>();
    41.         ConnectImageServer();
    42.  
    43.     }
    44.     // Update is called once per frame
    45.     void Update()
    46.     {
    47.         setImageAsTexture();
    48.     }
    49.  
    50.     /// <summary>    
    51.     /// Setup socket connection.    
    52.     /// </summary>    
    53.     private void ConnectDataServer()
    54.     {
    55.         try
    56.         {
    57.             socketConnectionData = new TcpClient(Server_Addres, DataPort);
    58.         }
    59.         catch (Exception e)
    60.         {
    61.             serverComm = false;
    62.             Debug.Log("On client connect exception " + e);
    63.         }
    64.     }
    65.  
    66.     /// <summary>    
    67.     /// Setup socket connection.    
    68.     /// </summary>    
    69.     private void ConnectImageServer()
    70.     {
    71.         try
    72.         {
    73.             socketConnectionImage = new TcpClient(Server_Addres, ImagePort);
    74.  
    75.             clientReceiveThread = new Thread(new ThreadStart(ListenForData));
    76.             clientReceiveThread.IsBackground = true;
    77.             clientReceiveThread.Start();
    78.             serverComm = true;
    79.         }
    80.         catch (Exception e)
    81.         {
    82.             serverComm = false;
    83.             Debug.Log("On client connect exception " + e);
    84.         }
    85.     }
    86.  
    87.     private void setImageAsTexture()
    88.     {
    89.         if (serverMessage != null && serverComm == true)
    90.         {
    91.             Texture2D tex = new Texture2D(100, 100, TextureFormat.BGRA32, true, false);
    92.             tex.hideFlags = HideFlags.HideAndDontSave;
    93.             tex.filterMode = FilterMode.Point;
    94.             tex.LoadImage(bytes_image);
    95.             renderImage.GetComponent<Renderer>().material.mainTexture = tex;
    96.  
    97.         }
    98.     }
    99.     public static byte[] DecodeUrlBase64(string s)
    100.     {
    101.         s = s.Replace('-', '+').Replace('_', '/').PadRight(4 * ((s.Length + 3) / 4), '=');
    102.         return Convert.FromBase64String(s);
    103.     }
    104.  
    105. }
    106.  

    I appreciate your help.