Search Unity

  1. Check out our Unite Austin 2017 YouTube playlist to catch up on what you missed. More videos coming soon.
    Dismiss Notice
  2. Unity 2017.2 is now released.
    Dismiss Notice
  3. The Unity Gear Store is here to help you look great at your next meetup, user group or conference. With all new Unity apparel, stickers and more!
    Dismiss Notice
  4. Introducing the Unity Essentials Packs! Find out more.
    Dismiss Notice
  5. Want to see the most recent patch releases? Take a peek at the patch release page.
    Dismiss Notice
  6. Unity 2017.3 beta is now available for download.
    Dismiss Notice

Using Projection Matrix to Create Holographic Effect

Discussion in 'Scripting' started by Ben-BearFish, Jan 14, 2015.

  1. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    I saw this video here:


    In the video they mention they modified Unity's projection matrix to achieve the effect of the image updating based on the player's physical position. I was wondering if anyone here knew how to achieve this? I'm most concerned about how to implement modifying the projection matrix in a script to properly skew the objects according to the camera's angle.
     
    Chrarc likes this.
  2. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    Did you try a
    http://forum.unity3d.com/search/?type=post

    Reposting some old code
    Code (CSharp):
    1.     public Transform[] Corners;
    2.     public Transform lookTarget;
    3.     public bool drawNearCone, drawFrustum;
    4.  
    5.     Camera theCam;
    6.  
    7.     void Start () {
    8.         theCam = camera;
    9.     }
    10.  
    11.     void Update () {
    12.         Vector3 pa, pb, pc, pd;
    13.         pa = Corners[0].position; //Bottom-Left
    14.         pb = Corners[1].position; //Bottom-Right
    15.         pc = Corners[2].position; //Top-Left
    16.         pd = Corners[3].position; //Top-Right
    17.  
    18.         Vector3 pe = theCam.transform.position;// eye position
    19.  
    20.         Vector3 vr = ( pb - pa ).normalized; // right axis of screen
    21.         Vector3 vu = ( pc - pa ).normalized; // up axis of screen
    22.         Vector3 vn = Vector3.Cross( vr, vu ).normalized; // normal vector of screen
    23.  
    24.         Vector3 va = pa - pe; // from pe to pa
    25.         Vector3 vb = pb - pe; // from pe to pb
    26.         Vector3 vc = pc - pe; // from pe to pc
    27.         Vector3 vd = pd - pe; // from pe to pd
    28.  
    29.         float n = -lookTarget.InverseTransformPoint( theCam.transform.position ).z; // distance to the near clip plane (screen)
    30.         float f = theCam.farClipPlane; // distance of far clipping plane
    31.         float d = Vector3.Dot( va, vn ); // distance from eye to screen
    32.         float l = Vector3.Dot( vr, va ) * n / d; // distance to left screen edge from the 'center'
    33.         float r = Vector3.Dot( vr, vb ) * n / d; // distance to right screen edge from 'center'
    34.         float b = Vector3.Dot( vu, va ) * n / d; // distance to bottom screen edge from 'center'
    35.         float t = Vector3.Dot( vu, vc ) * n / d; // distance to top screen edge from 'center'
    36.  
    37.         Matrix4x4 p = new Matrix4x4(); // Projection matrix
    38.         p[0, 0] = 2.0f * n / ( r - l );
    39.         p[0, 2] = ( r + l ) / ( r - l );
    40.         p[1, 1] = 2.0f * n / ( t - b );
    41.         p[1, 2] = ( t + b ) / ( t - b );
    42.         p[2, 2] = ( f + n ) / ( n - f );
    43.         p[2, 3] = 2.0f * f * n / ( n - f );
    44.         p[3, 2] = -1.0f;
    45.  
    46.         theCam.projectionMatrix = p; // Assign matrix to camera
    47.  
    48.         if ( drawNearCone ) { //Draw lines from the camera to the corners f the screen
    49.             Debug.DrawRay( theCam.transform.position, va, Color.blue );
    50.             Debug.DrawRay( theCam.transform.position, vb, Color.blue );
    51.             Debug.DrawRay( theCam.transform.position, vc, Color.blue );
    52.             Debug.DrawRay( theCam.transform.position, vd, Color.blue );
    53.         }
    54.  
    55.         if ( drawFrustum ) DrawFrustum( theCam ); //Draw actual camera frustum
    56.  
    57.     }
    58.  
    59.     Vector3 ThreePlaneIntersection ( Plane p1, Plane p2, Plane p3 ) { //get the intersection point of 3 planes
    60.         return ( ( -p1.distance * Vector3.Cross( p2.normal, p3.normal ) ) +
    61.                 ( -p2.distance * Vector3.Cross( p3.normal, p1.normal ) ) +
    62.                 ( -p3.distance * Vector3.Cross( p1.normal, p2.normal ) ) ) /
    63.             ( Vector3.Dot( p1.normal, Vector3.Cross( p2.normal, p3.normal ) ) );
    64.     }
    65.  
    66.     void DrawFrustum ( Camera cam ) {
    67.         Vector3[] nearCorners = new Vector3[4]; //Approx'd nearplane corners
    68.         Vector3[] farCorners = new Vector3[4]; //Approx'd farplane corners
    69.         Plane[] camPlanes = GeometryUtility.CalculateFrustumPlanes( cam ); //get planes from matrix
    70.         Plane temp = camPlanes[1]; camPlanes[1] = camPlanes[2]; camPlanes[2] = temp; //swap [1] and [2] so the order is better for the loop
    71.  
    72.         for ( int i = 0; i < 4; i++ ) {
    73.             nearCorners[i] = ThreePlaneIntersection( camPlanes[4], camPlanes[i], camPlanes[( i + 1 ) % 4] ); //near corners on the created projection matrix
    74.             farCorners[i] = ThreePlaneIntersection( camPlanes[5], camPlanes[i], camPlanes[( i + 1 ) % 4] ); //far corners on the created projection matrix
    75.         }
    76.  
    77.         for ( int i = 0; i < 4; i++ ) {
    78.             Debug.DrawLine( nearCorners[i], nearCorners[( i + 1 ) % 4], Color.red, Time.deltaTime, false ); //near corners on the created projection matrix
    79.             Debug.DrawLine( farCorners[i], farCorners[( i + 1 ) % 4], Color.red, Time.deltaTime, false ); //far corners on the created projection matrix
    80.             Debug.DrawLine( nearCorners[i], farCorners[i], Color.red, Time.deltaTime, false ); //sides of the created projection matrix
    81.         }
    82.     }
    83.  
    Create an empty GO for lookTarget, with 4 children to mark the corners, put this script on the camera.

    This looktarget construction represents where your screen is (so doesnt move), and then you move the camera around to track the users position (eyes/head)
     
    Chrarc likes this.
  3. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092

    @hpjohn thanks for the info.

    So let's say I have a character walking around a the scene. If I wanted to keep the character on screen at all times, would I want to make sure the character always stays next to the lookTarget and within the 4 corners that were created?
     
  4. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    The corners represent the real screen in virtual space. Everything needs to happen on the other side of it
    [​IMG]
     
  5. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    Oh, I see now. It seems your implementation depends on the corners being setup to make a vertical screen that is viewed from the side?

    If I wanted to have different views such as a top down view would I have to modify the projection or would could I just modify the corners. Currently, it seems the corners have to be setup in a very particular way.

    EDIT:
    It looks like simply rotating the camera itself fixes that. Thanks for sharing your implementation with me.
     
  6. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    Yeah, gotta have the rotations of the camera and lookTarget the same, so for a top down view, rotate both 90 degrees on x (think)
     
  7. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    Would you mind explaining the math behind your implementation? I feel this would help me better understand the scene. I was always under the impression that the projection matrix is applied to the 3D scene to map the 3d scene to 2D screen coordinates.

    With your example it seems weird to me that the screen plane is in front of the 3D scene rather the other way around.
     
    Last edited: Jan 15, 2015
  8. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    I don't get what you mean
    The screen plane (near clip plane) is always located somewhere between the camera transform and the things it's looking at.

    The projection matrix is just maths to map anything within a truncated rectangular-based pyramid to a set of 2d coordinates (technically still 3d)
    Usually the projection matrix is calculated from the values for FOV and aspect, but this always results in a symetrical frustum

    more reading: http://www.songho.ca/opengl/gl_projectionmatrix.html
     
  9. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    I just realized my wording was off. I was becoming somewhat confused with what the blue and red outlines were representing with regards to Unity's white outline view frustum. I forgot that now that we are overriding the projection the white frustum no longer mattered. Also, for some reason when I saw you mention screen plane and I was thinking screen coordinates.


    It'd be nice to be able to disable Unity's camera frustum outline in the editor.
     
    Last edited: Jan 15, 2015
  10. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    If you collapse the camera component it goes
     
  11. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    After messing with values and my models, it seems like with the final camera view my 3D models don't quite "pop out" the way it does in the video. Do you have any suggestions that might make the illusion seem better?
     
  12. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    What makes it look that way in the vid is that they are tracking the users position to change the view in-game. If you set up positional tracking, it'll look like that.
     
  13. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    Ok, I'll try to link it up to Kinect. Also, it seems my shadows aren't drawing with the custom projection matrix. Have you noticed this issue?
     
  14. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
  15. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    So, it seems like my issue is little different from others. I already have forward rendering setup. And shadows do show up, it's just when the camera moves away from the lookTarget they began to grow fainter and then the shadows disappear all together.

    Perhaps, the light has to be set in a very specific position to prevent this?

    After further tests, it seems like the directional light's position in the Unity scene does not affect how the light is shown in the scene.
     
    Last edited: Jan 16, 2015
  16. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    Position of a directional light will not have any effect. There is no concept of the source of a directional light, its just a direction.

    Maybe your far clip plane is too far
     
  17. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    It looks like for some reason the field of view on the camera affects the shadows, which is strange because I thought the field of view no longer is taken into account the projection matrix implementation.
     
  18. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    @hpjohn, could you tell me the best (recommended) way to setup the gameobject hierarchy? I just wanted to make sure that the results I'm getting are the intended results.

    Here's my gameobject hierarchy currently:

     
  19. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    Id be tempted to put looktarget as a child of the camera parent. If you ever need to rotate the virtual screen, you have to rotate the camera the same, so simpler to rotate the whole construction
     
  20. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    @hpjohn , I was trying to setup a scene with this camera where a model/mesh always stays in the center of the screen, and normally I'd use camera.transform.LookAt(target.transform), but with the custom projection, things really break down. Might you have any suggestions for this with the custom projection?
     
  21. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    you shouldnt need to do anything special, since the lookTarget object is always in the center of the virtual window, just put the model near that
     
  22. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    Hmm... I placed the model/mesh at the center in front of the lookTarget, but when the frustum skews for some reason the model seems to move to the side of the camera.

    Here's the camera center.
    modelCenter.png

    Here's the camera moved to the left.
    modelLeft.png

    I was hoping to keep the skewed perspective while keeping the model in the center of screen space.
     
  23. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    I see what you mean, but isn't that the desired effect?
    Don't forget, in order for the camera to skew that much, from the head tracking, the user is looking from far to one side of his monitor. The subject wouldn't be in the center of the screen any more
     
  24. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    It would normally be the desired effect, but for the particular case of the creative team I'm working with, they wanted the model to always stay in the center of screen space. The fact we're interacting with a theater projector which is projecting the screen on the floor and using the Kinect, the creative team wanted that behavior so the scene always appears to be "popping" out without moving. Right now the current skew setup makes it seem like the whole scene is moving.

    I guess they expect the camera to orbit & skew the model, rather than only skew it.

    With the custom projection matrix, because we're not taking into account the rotation, I'm not really sure how I'd go about achieving the correct 'orbit' rotation. Normally, Vector3.LookAt would handle something like this with a normal Unity projection camera, but it falls apart with the custom setup.
     
    Last edited: Feb 4, 2015
  25. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    I'll see what i can come up with tomorrow morning - if the object was in the exact spot of the lookTarget, that would work, except that the near clip plane cuts it in half, so just need to move the near plane closer, without moving the fixed window corners. might be possible
     
    Ben-BearFish likes this.
  26. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    Maybe like this then?
    [​IMG]
    Fortuantly an easy adjustment, On the script above, line 43, multiply the value by some float, about 0.5 (depends how much more space you need between the virtual floor plane and the camera) , this brings the near clip towards the camera, but keeps the frustrum pivoting around the corners

    p[2, 3] = ( 2.0f * f * n / ( n - f ) ) * 0.5f;

    It shortens the whole frustum though,
     
    Ben-BearFish likes this.
  27. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    @hpjohn That seemed to do the trick. Thanks.

    Currently, to make it seem like the camera is rotating around the Y-Axis I rotate every object in the scene (minus the camera and the lookat target) around the Y-Axis, but to me this seems a bit 'hacky'. Do you have any thoughts for alternatives?
     
  28. hpjohn

    hpjohn

    Joined:
    Aug 14, 2012
    Posts:
    1,920
    if you put the camera and the lookTarget as children of another object, you can just rotate the whole lot together
    As long as their orientations stay the same it should be K
     
  29. Chrarc

    Chrarc

    Joined:
    Nov 20, 2013
    Posts:
    10
    Just want to throw in a thanks for the thread and camera script.
    Hugely helpful as I'm in the process of writing a multi screen netview environment for a vehicular simulator at work, and this has totally solved the issue of off axis frustum for multiple cameras, and even opened up the possibility of headtracking down the line.

    Also has solved the strangeness of camera H fov not quite matching the frustum boundaries even when rotated appropriately.
     
  30. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    @Chrarc , you have any luck with this projection and shadows working?
     
  31. Chrarc

    Chrarc

    Joined:
    Nov 20, 2013
    Posts:
    10
    hardshadows_netview.jpg softshadows_netview.jpg
    Fortunately yes, it seems to just work in my case.
    I had to increase the [ Edit > Project Settings > Quality > Shadow Distance ] and it seemed to be happy with both soft and hard shadows.
     
  32. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    @Chrarc , the shadows for me seem to be ok, as long as the skewing doesn't happen. When I set the Shadow Distance too far, the shadows become really poor quality.

    Might I ask what value you use for the Shadow Distance?
    Also, what version of Unity are you using and are you using forward or deferred lighting?
    Finally, what lights do you use in the scene?
     
  33. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    @hpjohn , I had a question regarding an assumption I made with the skewing of the camera that I never thought to question until now. I've assumed because the 3d meshes are skewing that I need to also write custom code to make sure the shadows skew as well, but I've started to question does it make sense for the shadows to skew like everything else in the scene, or does Unity's default handling of shadows make more sense from a realism standpoint for the viewer?
     
  34. Chrarc

    Chrarc

    Joined:
    Nov 20, 2013
    Posts:
    10
    Ok, after playing around for a bit, I found shadow distance doesn't make any difference to the quality, it will either draw them or not. It is set to 500 at the moment.
    If the quality of the shadows is the issue, I'm not sure how to solve that one.
    It seems Unity is using the default camera FOV projection matrix to determine what shadows to cull, and the resolution of the shadow mapping.
    You can see this when you run the scene, then move the camera so that the shadow area is outside of the cameras default FOV, it will be culled, and when you change the FOV slider, it will change the "fit" of the resolution of the shadow map.

    Without better knowledge of how to control Unity's shadow mapping, I'm at a bit of a dead end here.
     
  35. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    Has anyone tried this with Unity 5?
     
    Last edited: Mar 12, 2015
  36. Chrarc

    Chrarc

    Joined:
    Nov 20, 2013
    Posts:
    10
    In case you were still wondering, yes this setup does still work in Unity 5.
     
  37. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    Could I ask what version of Unity 5 you are using? It still seems to be broken for me. Could you also post a screencap of your camera/light settings?
     
  38. Chrarc

    Chrarc

    Joined:
    Nov 20, 2013
    Posts:
    10
    5.0.1 currently, and here is the script I'm using on the cameras. hope it helps.
     

    Attached Files:

  39. Ben-BearFish

    Ben-BearFish

    Joined:
    Sep 6, 2011
    Posts:
    1,092
    My bad, I meant could I see your light/camera settings in the Unity inspector. I already have a script like that running my camera. I think the issue I'm having is based on Unity's inspector settings.
     
    ralf_b likes this.
  40. ralf_b

    ralf_b

    Joined:
    Jul 9, 2013
    Posts:
    32
    Hello Ben,

    wondered if you got this working. Because I am trying to implement @Chrarc script as well with Unity 5.3.0f4 and my kinect v2.

    Could you provide a sample scene or a screenshot of your gameobject hierachy?
     
  41. DarknessPhoenix

    DarknessPhoenix

    Joined:
    Aug 17, 2016
    Posts:
    3


    Hi @hpjohn,

    Sorry for the noob question, but what exactly do you mean by creating 4 children for the game object?
    are they the 4 walls we see in the video?
    and who exactly to link them to the 4 corners in the script?
     
  42. VictoriaYoung

    VictoriaYoung

    Joined:
    Feb 27, 2017
    Posts:
    1
    @ralf_b did you get @Chrarc 's code working. mine isn't working too. Really need some help