Search Unity

  1. Megacity Metro Demo now available. Download now.
    Dismiss Notice
  2. Unity support for visionOS is now available. Learn more in our blog post.
    Dismiss Notice

Question How do I produce point cloud with the same dimensions as 3d objects represented on Unity Cameras

Discussion in 'General Graphics' started by jiqimao52jp, Sep 5, 2022.

  1. jiqimao52jp

    jiqimao52jp

    Joined:
    Apr 26, 2022
    Posts:
    2
    I am looking at producing point cloud with the same dimensions as 3d objects rendered to Unity Cameras.
    So what I have been working on is to make point cloud based on RGB and Depth cameras' target textures.
    ※ Two 2D RenderTexutures look like as below
    • Depth Render texuture
    Color Format: R8G8B8A8_UNORM, Depth Buffer: At least 24 bits depth (with stencil)​
    • RGB Render texture
    Color Format: R16G16B16A16_SFLOAT, Depth Buffer: At least 16 bits depth (with stencil)​

    I populated the two of 2D RenderTexuture data in a geometry shader and seemed the point cloud had been successfully rendered.
    The point cloud is 3D representation with depth completion if looking directly in front of it, which is almost identical to the source object.
    ※ The source object is a animating square CubeField consisting of 100(10x10) 3d cubes.

    But when I set the X-axis of it to perpendicular to the screen, the point cloud's depth gone, looked thin as if it renderer on a piece of paper.

    So my question is if it is possible to render 3d point cloud with depth completion using 2d rendertexture.

    Code (CSharp):
    1. Shader:
    2.  
    3. Properties{
    4.     [NoScaleOffset] _Pos("Pos", 2D) = "white" {}
    5.     [NoScaleOffset] _Col("Col", 2D) = "white" {}
    6.     _PointSize("Point Size", Float) = 1.0
    7.     _Color("PointCloud Color", Color) = (0, 0, 0, 0)
    8.     [Toggle(USE_DISTANCE)]_UseDistance("Scale by distance?", float) = 0
    9. }
    10.  
    11. SubShader
    12. {
    13.     Cull Off
    14.     Pass
    15.     {
    16.         CGPROGRAM
    17.         #pragma vertex vert
    18.         #pragma geometry geom
    19.         #pragma fragment frag
    20.         #pragma shader_feature USE_DISTANCE
    21.         #include "UnityCG.cginc"
    22.  
    23.         struct appdata
    24.         {
    25.             float4 vertex : POSITION;
    26.             float2 uv : TEXCOORD0;
    27.         };
    28.  
    29.         struct v2f
    30.         {
    31.             float4 vertex : SV_POSITION;
    32.             float2 uv : TEXCOORD0;
    33.         };
    34.  
    35.         float _PointSize;
    36.         fixed4 _Color;
    37.  
    38.         sampler2D _Col;
    39.         float4 _Col_TexelSize;
    40.  
    41.         sampler2D _Pos;
    42.         float4 _Pos_TexelSize;
    43.  
    44.  
    45.         struct g2f
    46.         {
    47.             float4 vertex : SV_POSITION;
    48.             float2 uv : TEXCOORD0;
    49.         };
    50.  
    51.         [maxvertexcount(4)]
    52.         void geom(point v2f i[1], inout TriangleStream<g2f> triStream)
    53.         {
    54.             g2f o;
    55.             float4 v = i[0].vertex;
    56.  
    57.             // TODO: interpolate uvs on quad
    58.             float2 uv = i[0].uv;
    59.             float2 p = _PointSize * 0.001;
    60.             p.y *= _ScreenParams.x / _ScreenParams.y;
    61.  
    62.             o.vertex = UnityObjectToClipPos(v);
    63.             #ifdef USE_DISTANCE
    64.             o.vertex += float4(-p.x, p.y, 0, 0);
    65.             #else
    66.             o.vertex += float4(-p.x, p.y, 0, 0) * o.vertex.w;
    67.             #endif
    68.             o.uv = uv;
    69.             triStream.Append(o);
    70.  
    71.             o.vertex = UnityObjectToClipPos(v);
    72.             #ifdef USE_DISTANCE
    73.             o.vertex += float4(-p.x, -p.y, 0, 0);
    74.             #else
    75.             o.vertex += float4(-p.x, -p.y, 0, 0) * o.vertex.w;
    76.             #endif
    77.             o.uv = uv;
    78.             triStream.Append(o);
    79.  
    80.             o.vertex = UnityObjectToClipPos(v);
    81.             #ifdef USE_DISTANCE
    82.             o.vertex += float4(p.x, p.y, 0, 0);
    83.             #else
    84.             o.vertex += float4(p.x, p.y, 0, 0) * o.vertex.w;
    85.             #endif
    86.             o.uv = uv;
    87.             triStream.Append(o);
    88.  
    89.             o.vertex = UnityObjectToClipPos(v);
    90.             #ifdef USE_DISTANCE
    91.             o.vertex += float4(p.x, -p.y, 0, 0);
    92.             #else
    93.             o.vertex += float4(p.x, -p.y, 0, 0) * o.vertex.w;
    94.             #endif
    95.             o.uv = uv;
    96.             triStream.Append(o);
    97.  
    98.         }
    99.  
    100.         v2f vert(appdata v)
    101.         {
    102.             v2f o;
    103.             float dist = length(_WorldSpaceCameraPos - v.vertex);
    104.             o.vertex = v.vertex;
    105.             o.vertex.xyz = float3(o.vertex.x, o.vertex.y + 2.8, o.vertex.z - 4.2);
    106.             o.uv = v.uv;
    107.             return o;
    108.         }
    109.  
    110.         fixed4 frag(g2f i) : SV_Target
    111.         {
    112.             float2 uv = tex2D(_Pos, i.uv);
    113.             if (any(uv <= 0 || uv >= 1))
    114.                 discard;
    115.  
    116.             return tex2D(_Col, i.uv) * _Color;
    117.         }
    118.         ENDCG
    119.     }
    120. }
    PointCloud.png
     
    Last edited: Sep 7, 2022
  2. c0d3_m0nk3y

    c0d3_m0nk3y

    Joined:
    Oct 21, 2021
    Posts:
    651
    Didn't understand entirely what you are doing but I can see a few problems:
    - Line 103: v.vertex seems to be in object space since it's used as input for UnityObjectToClipPos. So you are mixing world space _WorldSpaceCameraPos with object space v.vertex.
    - Line 103: dist is not used
    - Line 105: You are adding an offset in object space, which doesn't make much sense. You probably want to add it in view space, clip space or NDC space.
    - Line 62, 71, 80, 89: You're doing the same matrix multiplication 4 times, which is not very efficient
    - Line 64, 73, 82, 91: I don't understand how that's supposed to work. You are adding an offset in clip space (before dividing by .w). You have to multiply with .w like in the lines 66, 75, 84, 93.

    I was wondering whether you have to return 6 vertices instead of 4 but I think, it's correct since you are not calling RestartStrip. I was also wondering whether any(uv <= 0 || uv >= 1) is the same as any(uv <= 0) || any(uv >= 1), but it probably is.
     
  3. jiqimao52jp

    jiqimao52jp

    Joined:
    Apr 26, 2022
    Posts:
    2
    Thanks a lot @c0d3_m0nk3y

    I am trying to make point cloud with same size/dimensions as 3d objects that rendered to Unity Cameras,
    to where no matter how I move/rotate camera, the point cloud could be well viewed,
    also the every single vertex of the point cloud shall be at the same coodinates with object (which called as CubeField).

    - Line 103: dist is not used
    - Line 105: Adding an offset in object space is to try to have the point cloud overlaid on CubeField.
    please take a look at the image file attached.
    - Line 51-98: I re-wrote this part to what it looks like as below.
    Code (CSharp):
    1. [maxvertexcount(4)]
    2. void geom(point v2f i[1], inout TriangleStream<g2f> triStream)
    3. {
    4.     float4 v = i[0].vertex;
    5.     float2 uv = i[0].uv;
    6.  
    7.     const float2 points[4] = { float2(1,-1) , float2(1,1) , float2(-1,-1) , float2(-1,1) };
    8.     float2 pmul = float2(_PointSize, _PointSize * (_ScreenParams.y / _ScreenParams.x)) * 0.001;
    9.  
    10.     g2f o;
    11.     for (int i = 0; i < 4; i++)
    12.     {
    13.         o.vertex = UnityObjectToClipPos(v);
    14.         o.vertex += float4(points[i] * pmul, 0, 0) * o.vertex.w;
    15.         o.uv = uv;
    16.         triStream.Append(o);
    17.     }
    18. }
    I was wondering if it is possible to make a point cloud with same size/dimensions as the object rendered a unity camera using 2D render texture.
    What if I use 3D render texture, would I possibly achieve the goal?
     

    Attached Files: