Search Unity

  1. Unity 2019.1 is now released.
    Dismiss Notice

UNet communication problem: message arrival interval not consistent and burst arrival

Discussion in 'Connected Games' started by jose-araujo, Nov 21, 2018.

  1. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    Hi everyone,

    I built a very simple LLAPI message transmission between a client and a server, where I send just a Vector3 from the server to client. Both server and client application are running in the same machine. Running Windows 10 and Unity 2018.2.15f1.

    I tried sending the messages every update loop(avg transmission interval 16 ms), and at the client I notice that between 40% and 50% of the received messages arrive at the exact same time instant (i.e. you have a burst arrival of the messages as 2 or more messages arrive at the same exact time). To measure this I setup a stopwatch which checks the elapsed time between message arrivals. The stopwatch is implemented inside the message reception callback. This is quite problematic since if you are updating a pose of an object and two pose messages arrive at the same time, then only the last message is used to update the object pose.

    I then tried sending the messages every second update loop (avg transmission interval 33 ms). In this case, at the client there are no messages arriving at the same time, but 20% of the messages arrive with an interval that is larger than 36 ms.

    Has anyone experienced this problem as well? Is there any recommended workaround to this issue? My goal was to send messages at a rate of 60 Hz and receive them at the same rate without a large delay.

    In case anyone wants to try the code, here is the Server and Client scripts.

    Code (CSharp):
    1. using UnityEngine;
    2. using UnityEngine.Networking;
    3.  
    4. public class ServerScript : MonoBehaviour
    5. {
    6.     private const short ClientToServerMsgIdHaptic = 3007;
    7.     public bool isAtStartup = true;
    8.  
    9.     private System.Diagnostics.Stopwatch stopWatch;
    10.     private double lastTimeStamp;
    11.     private double currentTimeStamp;
    12.  
    13.     private float pktCounter0 = 0;
    14.     private float pktCounterHighDelay = 0;
    15.     private float pktCounterHighDelayX = 0;
    16.  
    17.     private float pktCountertotal = 0;
    18.     private double avgtime = 0;
    19.     private double avgtimeAcc = 0;
    20.  
    21.     public class ClientToServerMessage : MessageBase
    22.     {
    23.         public Vector3 posObj;
    24.     }
    25.  
    26.     void Update()
    27.     {
    28.         if (isAtStartup)
    29.         {
    30.             SetupServer();
    31.             Debug.Log("Server is running");
    32.         }
    33.    
    34.         // print the avg arrival time after 1000 packets are received
    35.         if(pktCountertotal == 1000)
    36.         {
    37.             avgtime = avgtimeAcc / pktCountertotal;
    38.             Debug.Log("Average time " + avgtime);
    39.         }
    40.     }
    41.  
    42.     // Create a client and connect to the server port
    43.     public void SetupServer()
    44.     {
    45.         NetworkServer.RegisterHandler(ClientToServerMsgIdHaptic, OnCommandReceive);
    46.         NetworkServer.Listen(4444);
    47.         isAtStartup = false;
    48.         Debug.Log("Server has started");
    49.  
    50.         stopWatch = new System.Diagnostics.Stopwatch();
    51.         stopWatch.Start();
    52.         lastTimeStamp = 0;
    53.         pktCounterHighDelay = 0;
    54.     }
    55.  
    56.     public void OnCommandReceive(NetworkMessage netMsg)
    57.     {
    58.         ClientToServerMessage Message = netMsg.ReadMessage<ClientToServerMessage>();
    59.         transform.position = Message.posObj;
    60.  
    61.         currentTimeStamp = stopWatch.ElapsedMilliseconds - lastTimeStamp;
    62.         lastTimeStamp = stopWatch.ElapsedMilliseconds;
    63.  
    64.         // burst arrival
    65.         if (currentTimeStamp <= 2)
    66.         {
    67.             pktCounter0 = pktCounter0 + 1;
    68.         }
    69.         // delay larger than 36
    70.         if (currentTimeStamp > 36)
    71.         {
    72.             pktCounterHighDelay++;
    73.         }
    74.         // delay larger than 51
    75.         if (currentTimeStamp > 51)
    76.         {
    77.             pktCounterHighDelayX++;
    78.         }
    79.         pktCountertotal = pktCountertotal + 1;
    80.         avgtimeAcc = avgtimeAcc + currentTimeStamp;
    81.         Debug.Log("Time between received messages" + currentTimeStamp.ToString("F2") + " pkttotal0 " + pktCounter0 + " pktotal " + pktCountertotal + " pkts high delay " + pktCounterHighDelay + " pkts high delay X " + pktCounterHighDelayX);
    82.  
    83.     }
    84.  
    85. }
    86.  
    And here is the client

    Code (CSharp):
    1. using System.Collections;
    2. using System.Collections.Generic;
    3. using UnityEngine;
    4.  
    5. using UnityEngine.Networking;
    6. using UnityEngine.Networking.NetworkSystem;
    7.  
    8. public class ClientScript : MonoBehaviour {
    9.  
    10.     // Definitions
    11.     NetworkClient myClient;
    12.     private bool isAtStartup = true;
    13.     private const short ClientToServerMsgId = 3007;
    14.     private bool isConnected = false;
    15.     private float counter = 0;
    16.     private System.Diagnostics.Stopwatch stopWatch;
    17.     private double lastTimeStamp;
    18.     private double currentTimeStamp;
    19.  
    20.     public class ClientToServerMessage : MessageBase
    21.     {
    22.         public Vector3 posObj;
    23.     }
    24.  
    25.     // Create a client and connect to the server port
    26.     public void SetupClient()
    27.     {
    28.         myClient = new NetworkClient();
    29.         myClient.RegisterHandler(MsgType.Connect, OnConnected);
    30.         myClient.Connect("YOUR IP ADDRESS", 4444);
    31.         isAtStartup = false;
    32.         Debug.Log("Setting up client");
    33.  
    34.         stopWatch = new System.Diagnostics.Stopwatch();
    35.         stopWatch.Start();
    36.         lastTimeStamp = 0;
    37.     }
    38.     // Use this for initialization
    39.     void Start () {
    40.         if (isAtStartup) // if at startup start the client
    41.         {
    42.             SetupClient();
    43.         }
    44.     }
    45.  
    46.     // Update is called once per frame
    47.     void Update () {
    48.  
    49.         var x = Input.GetAxis("Horizontal") * Time.deltaTime * 150.0f;
    50.         var z = Input.GetAxis("Vertical") * Time.deltaTime * 3.0f;
    51.  
    52.         transform.Rotate(0, x, 0);
    53.         transform.Translate(0, 0, z);
    54.  
    55.         if (isConnected)
    56.         {
    57.             // to send every update loop: counter = 0
    58.             // to send every second update loop: counter = 1
    59.             if (counter == 0)
    60.             {
    61.                 ClientToServerMessage msg = new ClientToServerMessage();
    62.                 msg.posObj = transform.position;
    63.                 currentTimeStamp = stopWatch.ElapsedMilliseconds - lastTimeStamp;
    64.                 lastTimeStamp = stopWatch.ElapsedMilliseconds;
    65.                 Debug.Log("Time between transmitted messages" + currentTimeStamp.ToString("F2"));
    66.  
    67.                 myClient.Send(ClientToServerMsgId, msg);
    68.                 counter = -1;
    69.             }
    70.             counter = counter + 1;
    71.         }
    72.     }
    73.  
    74.     // Create Connected successfull flag
    75.     public void OnConnected(NetworkMessage netMsg)
    76.     {
    77.         Debug.Log("Connected to server");
    78.         isConnected = true;
    79.     }
    80. }
    81.  
     
    Last edited: Nov 22, 2018
  2. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    Anyone with thoughts on this? :)
     
  3. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    The problem here You do not receive messages persistently, you read them with frame-period (16 or 33 ms) interval (the message which physically fired in time 0, can be read in time 16). So, error in one frame is expected. In pair of this there are two parameters affected send rate - GlobalConfig.ThreadAwakeTimeout should be set to 1, and ConnectionConfig.SendDelay should be set to 0.

    Hope this helps
     
  4. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    Thanks a lot for the help! I will try your suggestions on Monday. But if I understand you correctly, is it then impossible to get a consistent transmission and reception at 60 Hz in Unity if one relies on Unet? I mean, that you can send and receive a packet every 60 Hz, even if each packet may be 1/60 ms delayed (arriving on the next frame)
     
  5. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    thanks for question. See, it is not unity problem it is a problem of you receiving algorithm. If your game is frame based, so every frame senders pushes messages to net and every frame receivers get messages from network. The receiving information defines how next frame will be simulated and rendered. A you minimal simulation step = 13ms you don't need to worry about periods < 13ms (all updates received before frame start will be handled in this frame and doesn't matter if update-1 will come in time = 1 ms and update-2 will come in time = 3ms, the time will be exactly the same = frame No X (frame 171 for ex). It works for fast paced games

    In case rpg or mmo, where you can have much more clients, but clients sends updates quite rare, you can switch to async model and set up receive callback, in this case callback will notify you that "there is something to read" and you can handle receiving messages immediately.

    Another opportunity - set up read callback for reading notification and separate thread for message reading, in this case you will get exactly what you want, immediately after message received callback will notify you that there is new message and you will read them in separate thread.

    Was I clean?
    again shortly
    1. set up read callback
    2. create network worker thread
    3. ask this thread wait on semaphore
    4. In read callback add signal semaphore call (not more as callback will be called from unet worket thread and long operation there can lead to network performance degradation)
    5. When semaphore will change its state to signalled, your theead will awake and call Receive() function till receive Nothing event.

    This procedure will minimise delays.
     
  6. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    Very sorry for the delay on coming back to you on this. I tried your suggestion on the first reply and the behavior did not change. I added the following to both the SetupClient and SetupServer scripts.

    Code (CSharp):
    1. GlobalConfig globalConfig = new GlobalConfig();
    2. globalConfig.ThreadAwakeTimeout = 1;
    3. ConnectionConfig config = new ConnectionConfig();
    4. config.SendDelay = 0;
    Unfortunately I could not understand how to implement your proposal in your latest post. Any chance you could give me some more details about how to implement that in the script I provided in my initial post? But I will try to understand again your suggestion.
     
  7. ioannis1987

    ioannis1987

    Joined:
    Jan 10, 2019
    Posts:
    3

    Hi,

    I'm also having this message burst on my client's side. My server's consistently sends messages to the client at a freq of 60Hz. However the client receives the messages in a non-consistent manner. On average the receive rate on the client side is 60Hz, but it happens quite often that the elapsed time between two consecutive messages is either 0ms or 33ms instead of 16ms (=60Hz). I'm attaching you a png with some detailed plots after some recording to understand what I'm talking about. Could you please elaborate a bit more on how exactly to deal with this issue. My knowledge on semaphores is quite limited. Thanks in advance for your time.
     

    Attached Files:

  8. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    @ioannis1987. Hmm, you send message, to your peer, your peer wll read them with delaying one frame = 16 ms, and then send back,you will read them with the same delaying 16 ms. Am I wrong?
     
  9. ioannis1987

    ioannis1987

    Joined:
    Jan 10, 2019
    Posts:
    3
    @aabramychev. My problem is not the one frame delay. I expect that. My problem is that the time between consecutively received messages on the client side is not consistent. Roughly 50% of the times, the elapsed receive-time between two successive messages is 16ms, which is what I'm aiming for. The other two clusters as you can see from my plots are 0ms (message[k] arrives at the same time as message[k-1]) and 33ms (message[k] arrives two frames later instead of one frame compared to message[k-1]). Is there something deeper in the UNET that while trying to prevent packet losses ends up with this message burst I am describing above?
     
  10. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    @ioannis1987 Can I take a look on the project? (I hope it based on llapi only..:))
     
  11. ioannis1987

    ioannis1987

    Joined:
    Jan 10, 2019
    Posts:
    3
    @aabramychev. Today I have actually tried the simple piece of code that José provided above. Just to see if I was meshing up something. I get exactly the same behavior. Client sends the messages with 60Hz, but the server has the burst problem I was referring to in my previous post. The only thing I've added on top of Jose's project is that I'm just storing the data (msgID, timestamp) in a dictionary and I write them afterwards in a txt file inside the OnDestroy() method. Do you get this behavior if you run Jose's code?
     
  12. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    @ioannis1987, I asking about project because it is not llapi code it is hlapi code, and i do not see any configuration parameter which should be provide is Start() method of custom NetworkManager. https://docs.unity3d.com/Manual/UNetManager.html ...

    I will try (cannot promise) to massage my llapi example on weekend for testing. But your the project attachment is still much appreciated
     
  13. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    @aabramychev I am a bit confused. Isnt my code above LLAPI? Why is it HLAPI? If it is not LLAPI, could you point me to a simple send/receive application which I could test?

    But regardless if my code is LLAPI or HLAPI, would it be possible to get your help to address the burst problem I am having with the above code I posted? I did not understand the solution you had advised me to implement on your Nov 30 reply. Thanks a lot!
     
  14. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
  15. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    Got it @aabramychev! But regarding my HLAPI implementation, is there anything that can be done to remove the burstiness and achieve 60Hz? From your previous replies, I got the impression that you said it was possible, but I could not understand how that is achieved. Thanks!
     
  16. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    @jose-araujo you need to change configuration parameters as I described above. To do this you need to use custom configuration of NetworkManager. As far as i understood you didn't do this (I do not see any networkmanager related code in your attachments) , it is why your changes are not used, and code still uses default parameters. I did send ref above about custom parameters, https://docs.unity3d.com/Manual/UNetManager.html ... Sorry, I m not expert in HLAPI and just do not remeber how exactly you can use custom configuration :( Need to google this :(
     
  17. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    @aabramychev as I wrote above on post #6, I did try to apply the configuration parameters you mentioned as follows, but nothing changed:

    Code (CSharp):
    1. GlobalConfig globalConfig = new GlobalConfig();
    2. globalConfig.ThreadAwakeTimeout = 1;
    3. ConnectionConfig config = new ConnectionConfig();
    4. config.SendDelay = 0;
    Is this correct? If it is, is there anything else you recomend me to try to fix the problems I have? Or could you try in your computer/network to see if you see the same problem as me?

    On my code being HLAPI, could you send me a reference to a simple LLAPI send/receive application? I still have not understood why my code was HLAPI and not LLAPI.

    Thanks a lot for the help again!
     
  18. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    @jose-araujo
    Simple question - where you call function NetworkTransport.Init() ? global config should be passed as parameter to this function. I'm pretty sure that you do not call this function et al. And HLAPI does it behind the scene. After you create global config you should init the network layer with this config. in hlapi you should use custom networkmanager for this as I referenced above. The code represented is correct, but this code do nothing. You create two object, but You should use these objects to initialize other objects... I do not remember how it should be exactly done for hlapi library as i havent' spent a lot of time with hlapi :( Unfortunately it is what I can say right now without further research. Try to ask @larus instead, he is hlapi expert.
     
  19. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    Thanks a lot for the feedback @aabramychev , its really super appreciated. I will try to call the function NetworkTransport.Init() as you recommend. At the moment I tried only to call it in the script I attached to the object. For clarification, the send and receive script are the only scripts I attach to the objects in my scene (I have 1 receive scene with 1 object with the receive script, and 1 send scene with 1 object with the send script), and I do not have any other script attached to the object and do not add any HLAPI network components to these objects.

    @larus , do you have any idea on how we could solve this problem? Please see post #1 for a background on this.

    Thanks a lot for the help again!
     
  20. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    @jose-araujo OK, I did small home works for you:
    Just from https://docs.unity3d.com/Manual/UNetManager.html
    1. To get started, create an empty GameObject
    in your starting Scene,
    2. attach the script with your config changes too the custom networkmanager:
    Code (CSharp):
    1. public class CustomManager : NetworkManager {
    2.  
    3.     // Set custom connection parameters early, so they are not too late to be enforced
    4.  
    5.     void Start()
    6.  
    7.     {
    8.  
    9.         customConfig = true;
    10.         globalConfig.ThreadAwakeTimeout = 1;
    11.  
    12.     }
    13.  
    14. }
    3. add this code to your gameobject from point 1.
    4. for server/client configuration use something like this
    Code (CSharp):
    1. public class Example : MonoBehaviour
    2. {
    3.     void StartServer()
    4.     {
    5.         ConnectionConfig config = new ConnectionConfig();
    6.         config.AddChannel(QosType.ReliableSequenced);
    7.         config.AddChannel(QosType.UnreliableSequenced);
    8.         config.SendDelay = 0;
    9.         NetworkServer.Configure(config, 10);
    10.         NetworkServer.Listen(7070);
    11.     }
    12. }
    Note that channel configuration provided on start...

    Hope it will help
     
  21. jose-araujo

    jose-araujo

    Joined:
    Nov 17, 2016
    Posts:
    34
    @aabramychev Thanks a lot. I tried this and it reduced the bursts to no more than around 20% which is quite some improvement!
    However, I notice that the burst is quite random. If I run the application 10 times, one or two times I get no bursts at all, sometimes I get 10% burst, most of the time I get around 20%. Also, it most often stays quite constant, so if you start having bursts it keeps having bursts, and if you dont have bursts, then you never have bursts.

    Any idea why this would be so random?
     
  22. aabramychev

    aabramychev

    Unity Technologies

    Joined:
    Jul 17, 2012
    Posts:
    574
    @jose-araujo hmm, could you strip out your project and send to me? (you can pm me) I will try to take a look on weekend.
    So I need a project and test bed description (For example client and server on loopback, or remote client for server with latency and jitter, or client locally connected via wifi...)