Search Unity

  1. Welcome to the Unity Forums! Please take the time to read our Code of Conduct to familiarize yourself with the forum rules and how to post constructively.

Texture2D.CreateExternalTexture from a webgl texture

Discussion in 'WebGL' started by hookmanuk, Apr 13, 2022.

  1. hookmanuk

    hookmanuk

    Joined:
    May 28, 2013
    Posts:
    20
    I'm creating a webgl texture in javascript, and trying to get back the handle pointer for the texture, so that I can use it within Texture2D.CreateExternalTexture.

    Unfortunately I can't find any examples on this online!

    This is the js code:
    Code (JavaScript):
    1. GetCanvasTexturePointer: function () {      
    2.         var canvas = document.querySelector('#iframecanvas').contentWindow.document.querySelector('canvas');      
    3.         var texture = null;
    4.        
    5.         if (canvas != null)
    6.         {                          
    7.             var canvasgl=document.querySelector("#glcanvas");
    8.             canvasgl.width=256;
    9.             canvasgl.height=256;
    10.  
    11.          
    12.             const gl = canvasgl.getContext("webgl");
    13.          
    14.             // Generate a handle on the GPU
    15.             texture = gl.createTexture();
    16.  
    17.             gl.bindTexture(gl.TEXTURE_2D, texture);              
    18.  
    19.             // Upload the image data to the GPU
    20.             gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas);
    21.         }      
    22.  
    23.         return texture;
    24.     }
    and this is my C# code:
    Code (CSharp):
    1. [DllImport("__Internal")]
    2. private static extern int GetCanvasTexturePointer();
    3.  
    4. int texint = GetCanvasTexturePointer();
    5. if (texint > 0)
    6. {                          
    7.     IntPtr intPtr = new IntPtr(texint);
    8.       Texture2D texture = Texture2D.CreateExternalTexture(256, 256, TextureFormat.RGBA32, false, false, intPtr);
    9. }
    Unfortunately I don't really know how to convert the js "texture" object (which is a type WebGlTexture), into an IntPtr for Unity to use.

    Can anyone help?
     
    hookmanuk, Apr 13, 2022
    #1

  2. brendanduncan_u3d

    brendanduncan_u3d

    Unity Technologies

    Joined:
    Jul 30, 2019
    Posts:
    292
    The problem here is that WebGL creates a texture object, and on the Unity side it uses texture IDs, as used by desktop OpenGL. Emscripten normally manages the mapping of texture IDs to WebGL texture objects, which it stores in the GL.textures array.

    What you might be able to do is generate the texture ID yourself by adding the texture to the GL.textures array. Something like (untested):
    Code (JavaScript):
    1. GetCanvasTexturePointer: function () {  
    2.         var canvas = document.querySelector('#iframecanvas').contentWindow.document.querySelector('canvas');  
    3.         var textureId = 0;
    4.      
    5.         if (canvas != null)
    6.         {                      
    7.             var canvasgl=document.querySelector("#glcanvas");
    8.             canvasgl.width=256;
    9.             canvasgl.height=256;
    10.  
    11.             const gl = canvasgl.getContext("webgl");
    12.        
    13.             // Generate a handle on the GPU
    14.             var textureObj = gl.createTexture(); // This is a WebGL texture object
    15.             GL.textures.push(textureObj); // "Register" the WebGL texture with Emscripten
    16.             textureId = GL.textures.length - 1; // The texture ID is it's index in the textures array
    17.  
    18.             gl.bindTexture(gl.TEXTURE_2D, textureObj);          
    19.  
    20.             // Upload the image data to the GPU
    21.             gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas);
    22.         }  
    23.  
    24.         return textureId;
    25. }
    Code (CSharp):
    1. [DllImport("__Internal")]
    2. private static extern int GetCanvasTexturePointer();
    3. int texint = GetCanvasTexturePointer();
    4. if (texint > 0)
    5. {                        
    6.     IntPtr intPtr = new IntPtr(texint);
    7.       Texture2D texture = Texture2D.CreateExternalTexture(256, 256, TextureFormat.RGBA32, false, false, intPtr);
    8. }
     
    brendanduncan_u3d, Apr 14, 2022
    #2

  3. hookmanuk

    hookmanuk

    Joined:
    May 28, 2013
    Posts:
    20
    Thanks for the reply... so that textureId does return a number, however it doesn't seem to contain the right texture!

    I have this extra code in C# running 30 times a second, to read the texture and apply it to my picture frame.

    Code (CSharp):
    1. public IEnumerator GetCanvasImages()
    2.         {
    3.             string imagebase64;
    4.             while (1 == 1)
    5.             {
    6.                 if (CanvasImage != null)
    7.                 {
    8.                     try
    9.                     {
    10.                         int texint = GetCanvasTexturePointer();                      
    11.  
    12.                         if (texint > 0)
    13.                         {
    14.                             PlaqueText.SetText(texint.ToString());
    15.  
    16.                             IntPtr intPtr = new IntPtr(texint);                          
    17.                             Texture2D texture = Texture2D.CreateExternalTexture(256, 256, TextureFormat.RGBA32, false, false, intPtr);
    18.                          
    19.                             CanvasImage.GetComponent<MeshRenderer>().material.SetTexture("_EmissionMap", texture);
    20.                             CanvasImage.GetComponent<MeshRenderer>().material.mainTexture = texture;                          
    21.                         }
    22.  
    23.                     }
    24.                     catch (Exception e)
    25.                     {
    26.                         Debug.Log(e);
    27.                     }
    28.                 }
    29.  
    30.                 yield return new WaitForSeconds(1f/30f);
    31.             }          
    32.         }
    This is showing an image, but not the canvas image...

    Below is a video of my webgl app, the top half shows the canvas I'm trying to read the image out of. When I click on the gallery pictureframe, the canvas starts using that new html code to render.
    At that point the gallery pictureframe should in theory update in sync with the canvas image.

    However as you can see it doesn't!
    One interesting thing to note, is if I move the camera from side to side the image does change, but to other random images in my scene... I assume this means that the pointer is pointing to the wrong image?

    I've also output the pointer id to the text plaque below the picture frame, which is incrementing at 30 fps, which makes sense with my current code.



    If anyone has any advice as to how to fix this, please let me know, thanks!
     
    hookmanuk, Apr 16, 2022
    #3

  4. hookmanuk

    hookmanuk

    Joined:
    May 28, 2013
    Posts:
    20
    Also of note, my previous solution was to call canvas.toDataURL() and send back the whole canvas image string from JS to C# and apply that as texture. That does work, so I know my core code is functional, it's just very slow as I'm doing it 30 times a second!
     
    hookmanuk, Apr 16, 2022
    #4

  5. brendanduncan_u3d

    brendanduncan_u3d

    Unity Technologies

    Joined:
    Jul 30, 2019
    Posts:
    292
    You might just be missing setting the wrap and filter parameters of the texture. Alternatively, you can create the texture from the C# side and pass its native ptr to JS. I just tested this and it worked.


    Code (CSharp):
    1. public class GetCanvasTexture : MonoBehaviour
    2. {
    3.     [DllImport("__Internal")]
    4.     private static extern void UpdateCanvasTexture(int id);
    5.  
    6.     // Start is called before the first frame update
    7.     void Start()
    8.     {
    9.         var texture = new Texture2D(256, 256);
    10.         int textureId = (int)texture.GetNativeTexturePtr();
    11.         UpdateCanvasTexture(textureId);
    12.  
    13.         GetComponent<Renderer>().material.SetTexture("_MainTex", texture);
    14.     }
    15. }

    Code (JavaScript):
    1. mergeInto(LibraryManager.library, {
    2.     UpdateCanvasTexture: function(id) {
    3.         // Create and fill a canvas with a gradient
    4.         var ct_canvas = document.createElement("canvas");
    5.         ct_canvas.width = 256;
    6.         ct_canvas.height = 256;
    7.         var ctx = ct_canvas.getContext("2d");
    8.         var grd = ctx.createLinearGradient(0, 0, 200, 0);
    9.         grd.addColorStop(0, "red");
    10.         grd.addColorStop(1, "white");
    11.         ctx.fillStyle = grd;
    12.         ctx.fillRect(0, 0, 256, 256);
    13.  
    14.         // Get the WebGL texture object from the Emscripten texture ID.
    15.         textureObj = GL.textures[id];
    16.  
    17.         // GLctx is the webgl context of the Unity canvas
    18.         GLctx.bindTexture(GLctx.TEXTURE_2D, textureObj);
    19.         GLctx.texParameteri(GLctx.TEXTURE_2D, GLctx.TEXTURE_WRAP_S, GLctx.CLAMP_TO_EDGE);
    20.         GLctx.texParameteri(GLctx.TEXTURE_2D, GLctx.TEXTURE_WRAP_T, GLctx.CLAMP_TO_EDGE);
    21.         GLctx.texParameteri(GLctx.TEXTURE_2D, GLctx.TEXTURE_MIN_FILTER, GLctx.LINEAR);
    22.  
    23.         // Upload the canvas image to the GPU texture.
    24.         GLctx.texSubImage2D(GLctx.TEXTURE_2D, 0, 0, 0, GLctx.RGBA, GLctx.UNSIGNED_BYTE, ct_canvas);
    25.     }
    26. });
    27.  
     
    brendanduncan_u3d, Apr 17, 2022
    #5
    awesomesaucelabs and hookmanuk like this.

  6. hookmanuk

    hookmanuk

    Joined:
    May 28, 2013
    Posts:
    20
    Thanks for your help!

    I got this working using your latest code. One other thing I did need to do was flip the Y scale to -1 on my objects, as the rendering of my textures appeared to be flipped coming from webgl back into unity.

    If anyone's interested in seeing the results, my NFT gallery is available online, click the feet icon to move, choose floor 4 for an example of rendering canvas into unity :)

    https://cnftgallery.hookman.co.uk/
     
    hookmanuk, Apr 26, 2022
    #6

  7. brendanduncan_u3d

    brendanduncan_u3d

    Unity Technologies

    Joined:
    Jul 30, 2019
    Posts:
    292
    Instead of scaling Y by -1, you can use GLctx.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); prior to the texSubImage2D call, and set it back to false afterword.
     
    brendanduncan_u3d, Apr 26, 2022
    #7
    hookmanuk likes this.