Question Allocated server had issues (query failure [DOWN]), stopping server and deallocating

Hello, I have Implemented the server query handler system, but still, I'm getting this error and my server shuts down after 3 minutes. I have attached the code below.

Code (CSharp):

1. using System;
2. using System.Text;
3. using System.Threading;
4. using System.Threading.Tasks;
5. using Newtonsoft.Json;
6. using Unity.Services.Matchmaker.Models;
7. using Unity.Services.Multiplay;
8. using UnityEngine.Networking;
9. using Debug = UnityEngine.Debug;
10.  
11. namespace Matchplay.Server
12. {
13.     public class MultiplayAllocationService : IDisposable
14.     {
15.         IMultiplayService m_MultiplayService;
16.         MultiplayEventCallbacks m_Servercallbacks;
17.         IServerQueryHandler m_ServerCheckManager;
18.         IServerEvents m_ServerEvents;
19.         string m_AllocationId;
20.         bool m_LocalServerValuesChanged = false;
21.         CancellationTokenSource m_ServerCheckCancel;
22.  
23.         const string k_PayloadProxyUrl = "http://localhost:8086";
24.  
25.         public MultiplayAllocationService()
26.         {
27.             try
28.             {
29.                 m_MultiplayService = MultiplayService.Instance;
30.                 m_ServerCheckCancel = new CancellationTokenSource();
31.             }
32.             catch (Exception ex)
33.             {
34.                 Debug.LogWarning($"Error creating Multiplay allocation service.\n{ex}");
35.             }
36.         }
37.  
38.         /// <summary>
39.         /// Should be wrapped in a timeout function
40.         /// </summary>
41.         public async Task<MatchmakingResults> SubscribeAndAwaitMatchmakerAllocation()
42.         {
43.             if (m_MultiplayService == null)
44.                 return null;
45.             m_AllocationId = null;
46.             m_Servercallbacks = new MultiplayEventCallbacks();
47.             m_Servercallbacks.Allocate += OnMultiplayAllocation;
48.             m_ServerEvents = await m_MultiplayService.SubscribeToServerEventsAsync(m_Servercallbacks);
49.  
50.             var allocationID = await AwaitAllocationID();
51.             var mmPayload = await GetMatchmakerAllocationPayloadAsync();
52.  
53.             return mmPayload;
54.         }
55.  
56.         //The networked server is our source of truth for what is going on, so we update our multiplay check server with values from there.
57.         public async Task BeginServerCheck()
58.         {
59.             if (m_MultiplayService == null)
60.                 return;
61.             m_ServerCheckManager = await m_MultiplayService.StartServerQueryHandlerAsync((ushort)10,
62.                 "", "", "0", "");
63.  
64. #pragma warning disable 4014
65.             ServerCheckLoop(m_ServerCheckCancel.Token);
66. #pragma warning restore 4014
67.         }
68.  
69.         public void SetServerName(string name)
70.         {
71.             m_ServerCheckManager.ServerName = name;
72.             m_LocalServerValuesChanged = true;
73.         }
74.  
75.         public void SetBuildID(string id)
76.         {
77.             m_ServerCheckManager.BuildId = id;
78.             m_LocalServerValuesChanged = true;
79.         }
80.  
81.         public void SetMaxPlayers(ushort players)
82.         {
83.             m_ServerCheckManager.MaxPlayers = players;
84.         }
85.  
86.         public void SetPlayerCount(ushort count)
87.         {
88.             m_ServerCheckManager.CurrentPlayers = count;
89.             m_LocalServerValuesChanged = true;
90.         }
91.  
92.         public void AddPlayer()
93.         {
94.             m_ServerCheckManager.CurrentPlayers += 1;
95.             m_LocalServerValuesChanged = true;
96.         }
97.  
98.         public void RemovePlayer()
99.         {
100.             m_ServerCheckManager.CurrentPlayers -= 1;
101.             m_LocalServerValuesChanged = true;
102.         }
103.  
104.         public void SetMap(string newMap)
105.         {
106.  
107.             m_ServerCheckManager.Map = newMap;
108.             m_LocalServerValuesChanged = true;
109.         }
110.  
111.         public void SetMode(string mode)
112.         {
113.  
114.             m_ServerCheckManager.GameType = mode;
115.             m_LocalServerValuesChanged = true;
116.         }
117.  
118.         public void UpdateServerIfChanged()
119.         {
120.             if (m_LocalServerValuesChanged)
121.             {
122.                 m_ServerCheckManager.UpdateServerCheck();
123.                 m_LocalServerValuesChanged = false;
124.             }
125.         }
126.  
127.         async Task ServerCheckLoop(CancellationToken cancellationToken)
128.         {
129.             while (!cancellationToken.IsCancellationRequested)
130.             {
131.                 UpdateServerIfChanged();
132.                 await Task.Delay(1000);
133.             }
134.         }
135.  
136.         async Task<string> AwaitAllocationID()
137.         {
138.             var config = m_MultiplayService.ServerConfig;
139.             Debug.Log($"Awaiting Allocation. Server Config is:\n" +
140.                 $"-ServerID: {config.ServerId}\n" +
141.                 $"-AllocationID: {config.AllocationId}\n" +
142.                 $"-Port: {config.Port}\n" +
143.                 $"-QPort: {config.QueryPort}\n" +
144.                 $"-logs: {config.ServerLogDirectory}");
145.  
146.             //Waiting on OnMultiplayAllocation() event (Probably wont ever happen in a matchmaker scenario)
147.             while (string.IsNullOrEmpty(m_AllocationId))
148.             {
149.                 var configID = config.AllocationId;
150.  
151.                 if (!string.IsNullOrEmpty(configID) && string.IsNullOrEmpty(m_AllocationId))
152.                 {
153.                     Debug.Log($"Config had AllocationID: {configID}");
154.                     m_AllocationId = configID;
155.                 }
156.  
157.                 await Task.Delay(100);
158.             }
159.  
160.             return m_AllocationId;
161.         }
162.  
163.         /// <summary>
164.         /// Get the Multiplay Allocation Payload for Matchmaker (using Multiplay SDK)
165.         /// </summary>
166.         /// <returns></returns>
167.         async Task<MatchmakingResults> GetMatchmakerAllocationPayloadAsync()
168.         {
169.             var payloadAllocation = await MultiplayService.Instance.GetPayloadAllocationFromJsonAs<MatchmakingResults>();
170.             var modelAsJson = JsonConvert.SerializeObject(payloadAllocation, Formatting.Indented);
171.             Debug.Log(nameof(GetMatchmakerAllocationPayloadAsync) + ":" + Environment.NewLine + modelAsJson);
172.             return payloadAllocation;
173.         }
174.  
175.         void OnMultiplayAllocation(MultiplayAllocation allocation)
176.         {
177.             Debug.Log($"OnAllocation: {allocation.AllocationId}");
178.             if (string.IsNullOrEmpty(allocation.AllocationId))
179.                 return;
180.             m_AllocationId = allocation.AllocationId;
181.         }
182.  
183.         void OnMultiplayDeAllocation(MultiplayDeallocation deallocation)
184.         {
185.             Debug.Log(
186.                 $"Multiplay Deallocated : ID: {deallocation.AllocationId}\nEvent: {deallocation.EventId}\nServer{deallocation.ServerId}");
187.         }
188.  
189.         void OnMultiplayError(MultiplayError error)
190.         {
191.             Debug.Log($"MultiplayError : {error.Reason}\n{error.Detail}");
192.         }
193.  
194.         public void Dispose()
195.         {
196.             if (m_Servercallbacks != null)
197.             {
198.                 m_Servercallbacks.Allocate -= OnMultiplayAllocation;
199.                 m_Servercallbacks.Deallocate -= OnMultiplayDeAllocation;
200.                 m_Servercallbacks.Error -= OnMultiplayError;
201.             }
202.  
203.             if (m_ServerCheckCancel != null)
204.                 m_ServerCheckCancel.Cancel();
205.  
206.             m_ServerEvents?.UnsubscribeAsync();
207.         }
208.     }
209.  
210.     public static class AllocationPayloadExtensions
211.     {
212.         public static string ToString(this MatchmakingResults payload)
213.         {
214.             StringBuilder payloadDescription = new StringBuilder();
215.             payloadDescription.AppendLine("Matchmaker Allocation Payload:");
216.             payloadDescription.AppendFormat("-QueueName: {0}\n", payload.QueueName);
217.             payloadDescription.AppendFormat("-PoolName: {0}\n", payload.PoolName);
218.             payloadDescription.AppendFormat("-ID: {0}\n", payload.BackfillTicketId);
219.             payloadDescription.AppendFormat("-Teams: {0}\n", payload.MatchProperties.Teams.Count);
220.             payloadDescription.AppendFormat("-Players: {0}\n", payload.MatchProperties.Players.Count);
221.             payloadDescription.AppendFormat("-Region: {0}\n", payload.MatchProperties.Region);
222.             return payloadDescription.ToString();
223.         }
224.     }
225. }

 

Hi. I'm experencing the same problem: the server crashes after 3 minutes:

Code (CSharp):

1. Dec 21, 2022, 3:32 PM GMT+1
2. 6175853
3. Allocated server had issues (query failure [DOWN]), stopping server and deallocating
4. Dec 21, 2022, 3:31 PM GMT+1
5. 6175853
6. Server having issues (query failure [DOWN]) for 120 seconds (Restarted 0 time(s))
7. Dec 21, 2022, 3:30 PM GMT+1
8. 6175853
9. Server having issues (query failure [DOWN]) for 61 seconds (Restarted 0 time(s))
10. Dec 21, 2022, 3:28 PM GMT+1
11. 6175853
12. Performed start using build configuration "protobuild01"

I tested both mono and il2cpp linux builds (dedicated server platform).
I'm using the matchplay code as a base for matchmaker.
If i run the server on localhost it works fine (both windows and linux server builds)
I think my server build doesn't need too much resources:



Log:

Code (CSharp):

1.  

 

Perhaps the problem could be in this function:

Code (CSharp):

1.  public void UpdateServerIfChanged()
2.         {
3.             if (m_LocalServerValuesChanged)
4.             {
5.                 m_ServerCheckManager.UpdateServerCheck();
6.                 m_LocalServerValuesChanged = false;
7.             }
8.         }

We only send a query (SQP) when we have some "changes" like setmap, addplayer or RemovePlayer.
Acording documentation:

Code (CSharp):

1. GSH relies on query protocol responses to determine server instances' health. If your build doesn’t support a query protocol or if you set up your query protocol incorrectly, GSH might determine your server instances are unresponsive.

Maybe if we dont send UpdateServerCheck for 3 minutes, GSH would determine that the server is unresponsive and it kill the instance.

 

It's not the problem, I send the UpdateServerCheck more often, but I got the same result: a crash after 3-4 minutes. In Mono I get more info

Code (CSharp):

1. Server exit: triggering deallocation (exit: 139, signal: terminated)
2.     "memorysetup-gfx-thread-allocator-block-size=16777216"
3.     "memorysetup-cache-allocator-block-size=4194304"
4.     "memorysetup-typetree-allocator-block-size=2097152"
5.     "memorysetup-profiler-bucket-allocator-granularity=16"
6.     "memorysetup-profiler-bucket-allocator-bucket-count=8"
7.     "memorysetup-profiler-bucket-allocator-block-size=4194304"
8.     "memorysetup-profiler-bucket-allocator-block-count=1"
9.     "memorysetup-profiler-allocator-block-size=16777216"
10.     "memorysetup-profiler-editor-allocator-block-size=1048576"
11.     "memorysetup-temp-allocator-size-main=4194304"
12.     "memorysetup-job-temp-allocator-block-size=2097152"
13.     "memorysetup-job-temp-allocator-block-size-background=1048576"
14.     "memorysetup-job-temp-allocator-reduction-small-platforms=262144"
15.     "memorysetup-temp-allocator-size-background-worker=32768"
16.     "memorysetup-temp-allocator-size-job-worker=262144"
17.     "memorysetup-temp-allocator-size-preload-manager=262144"
18.     "memorysetup-temp-allocator-size-nav-mesh-worker=65536"
19.     "memorysetup-temp-allocator-size-audio-worker=65536"
20.     "memorysetup-temp-allocator-size-cloud-worker=32768"
21.     "memorysetup-temp-allocator-size-gfx=262144"
22. Output may be truncated for performance

Code (CSharp):

1. Caught fatal signal - signo:11 code:0 errno:0 addr:(nil)
2. Obtained 13 stack frames.
3. #0  0x007fc2ba505420 in funlockfile
4. #1  0x007fc2ba3dc23f in clock_nanosleep
5. #2  0x007fc2ba3e1ec7 in nanosleep
6. #3  0x007fc2bb2e8481 in std::_Rb_tree<void const*, void const*, std::_Identity<void const*>, std::less<void const*>, std::allocator<void const*> >::_M_erase(std::_Rb_tree_node<void const*>*)
7. #4  0x007fc2bb1ba523 in int* std::_V2::__rotate<int*>(int*, int*, int*, std::random_access_iterator_tag)
8. #5  0x007fc2bb1ba5ea in int* std::_V2::__rotate<int*>(int*, int*, int*, std::random_access_iterator_tag)
9. #6  0x007fc2bb20dff7 in int* std::_V2::__rotate<int*>(int*, int*, int*, std::random_access_iterator_tag)
10. #7  0x007fc2bb2073a7 in int* std::_V2::__rotate<int*>(int*, int*, int*, std::random_access_iterator_tag)
11. #8  0x007fc2bb207362 in int* std::_V2::__rotate<int*>(int*, int*, int*, std::random_access_iterator_tag)
12. #9  0x007fc2bb20764c in int* std::_V2::__rotate<int*>(int*, int*, int*, std::random_access_iterator_tag)
13. #10 0x007fc2bb42e9a7 in PlayerMain(int, char**)
14. #11 0x007fc2ba323083 in __libc_start_main
15. #12 0x0056283de4c029 in (Unknown)
16.  
17. Output may be truncated for performance

 

Did you ever find a solution to your problem? It looks like as I've been learning I've taken a similar path to you and have been encountering the same problem.

 

I dont have any solution. The server crashes after some minutes, I read and checked the documentation lot of times

In my case, it's the same code than matchplay sample: https://github.com/Unity-Technologies/com.unity.services.samples.matchplay

ApplicationController

Code (CSharp):

1. Debug.Log("Initalizing Server....");
2.                 var serverSingleton = Instantiate(m_ServerPrefab);
3.                 await serverSingleton.CreateServer(); //run the init instead of relying on start.
4.  
5.                 var defaultGameInfo = new GameInfo
6.                 {
7.                     gameMode = GameMode.Meditating,
8.                     map = Map.Space,
9.                     gameQueue = GameQueue.Casual
10.                 };
11.  
12.                 await serverSingleton.Manager.StartGameServerAsync(defaultGameInfo);

ServerSingleton : Monobehaviour

Code (CSharp):

1.  public async Task CreateServer()
2.         {
3.             await UnityServices.InitializeAsync();
4.  
5.             m_GameManager = new ServerGameManager(
6.                 ApplicationData.IP(),
7.                 ApplicationData.Port(),
8.                 ApplicationData.QPort(),
9.                 NetworkManager.Singleton);
10.         }

ServerGameManager

Code (CSharp):

1. public async Task StartGameServerAsync(GameInfo startingGameInfo)
2.         {
3.             Debug.Log($"Starting server with:{startingGameInfo}.");
4.  
5.                   ....
6.                     await StartAllocationService(startingGameInfo,
7.                         (ushort)matchmakerPayload.MatchProperties.Players.Count);
8.                  .....
9. }
10.  
11.  async Task StartAllocationService(GameInfo startingGameInfo, ushort playerCount)
12.         {
13.  
14.             await m_MultiplayAllocationService.BeginServerCheck();
15.  
16.             //Create a unique name for the server to show that we are joining the same one
17.  
18.             m_MultiplayAllocationService.SetServerName(m_ServerName);
19.             m_MultiplayAllocationService.SetPlayerCount(playerCount);
20.             m_MultiplayAllocationService.SetMaxPlayers(10);
21.             m_MultiplayAllocationService.SetBuildID("0");
22.             m_MultiplayAllocationService.SetMap(startingGameInfo.map.ToString());
23.             m_MultiplayAllocationService.SetMode(startingGameInfo.gameMode.ToString());
24.  
25.          
26.         }
27.         }

MultiplayAllocationService

Code (CSharp):

1.         public async Task BeginServerCheck()
2.         {
3.             if (m_MultiplayService == null)
4.                 return;
5.  
6.             m_ServerCheckManager = await m_MultiplayService.StartServerQueryHandlerAsync((ushort)10,
7.                 "", "", "0", "");
8.  
9. #pragma warning disable 4014
10.             ServerCheckLoop(m_ServerCheckCancel.Token);
11. #pragma warning restore 4014
12.         }

 

Follow up: The matchplay sample actually didn't work with sqp, and now might be fixed. (https://github.com/Unity-Technologies/com.unity.services.samples.matchplay) Take a look and you may be able to resolve your issues. There was a new commit for this issue about a week ago (2-6-23).

 

Hey guys, I solved the issue by updating the query every second and the server is functioning properly now !!

 

I solved the issue too by implementing from scratch all the features: SQP, backfill matchmaking QoS....

 

Hi, I have similar issue. I have called serverQueryHandler.UpdateServerCheck() in Update but server still crash. Anyone know what's wrong in my code?
This is my code, I called StartServerQuery() from another class.

Code (CSharp):

1.  
2. public async Task StartMultiplay()
3.         {
4.             LogServerConfig();
5.             multiplayEventCallbacks = new MultiplayEventCallbacks();
6.             multiplayEventCallbacks.Allocate += OnAllocate;
7.             multiplayEventCallbacks.Deallocate += OnDeallocate;
8.             multiplayEventCallbacks.Error += OnError;
9.             multiplayEventCallbacks.SubscriptionStateChanged += OnSubscriptionStateChanged;
10.          
11.             await MultiplayService.Instance.SubscribeToServerEventsAsync(multiplayEventCallbacks);
12.  
13.             if (!string.IsNullOrEmpty(MultiplayService.Instance.ServerConfig.AllocationId))
14.             {
15.                 await StartServer();
16.             }
17.         }
18.  
19.         public async Task StartServer()
20.         {
21.             matchmakingResults = await MultiplayService.Instance.GetPayloadAllocationFromJsonAs<MatchmakingResults>();
22.  
23.             string matchId = matchmakingResults.MatchId;
24.             Debug.Log("matchId: " + matchId);
25.         }
26.  
27.         public async Task StartServerQuery(SessionRequest session)
28.         {
29.             serverQueryHandler = await MultiplayService.Instance.StartServerQueryHandlerAsync((ushort)session.MaxPlayers, session.ServerName, session.Map, Application.version, session.CustomProperties["map"]);
30.             Debug.Log("StartServerQueryHandlerAsync");
31.         }
32.  
33.         public async void GameServerReady()
34.         {
35.             await MultiplayService.Instance.ReadyServerForPlayersAsync();
36.             Debug.Log("ReadyServerForPlayersAsync");
37.         }
38.  
39.         public void PlayerCountChanged(ushort newPlayerCount)
40.         {
41.             serverQueryHandler.CurrentPlayers = newPlayerCount;
42.             Debug.Log("ServerQueryHandler CurrentPlayers: " + newPlayerCount);
43.         }
44.  
45.         private void Update()
46.         {
47.             if (serverQueryHandler != null)
48.                 serverQueryHandler.UpdateServerCheck();
49.         }
50.  

 

Fixed for me by adding UpdateServerCheck() in Update loop as well. I guess you don't need to do it that way. But I think at least one initial UpdateServerCheck() call is necessary after query handler init. That should be however stated in the documentation if that's the case

 

On my end, even updating the data every 5 seconds does not prevent the server from getting shut down.

EDIT: Changing that frequency from 5 seconds to 1 made everything work... How come?

 

Maybe they increased the polling on their side? So you have to be faster than their query

 

Having the same issue. Server is deallocated after several minutes. Although some sessions are working normally and not being interrupted. I'm calling UpdateServerCheck every 1 second.

 

What is the Events tab telling you? What are the reasons for the deallocation?

 

This is what I see in the events log:

Server having issues (Query didn't respond or responded incorrectly [NONAME]) for 120 seconds (Restarted 0 time(s))

and then

Allocated server had issues (Query didn't respond or responded incorrectly [NONAME]), stopping server and deallocating

In my log file I see:

SQP server: SQP server started on 0.0.0.0:9010

and also made sure I call UpdateServerCheck every 1 second

 

Thanks danri. The documentation suggests using the "Update" method for the update check calls and the sample uses a separate thread. What is the best practice? I'm not sure how intensive is this call and I want to make sure it won't affect my server performance. My current implementation is using the "Update" method.

In the meanwhile, I'll reduce to 100ms and tell you if it fixes my issue.

 

@spikyworm5 ah sorry I missed the part where you said you update every 1 second. I do update just every Update without a limit. That's why it worked for me I guess

 

Thanks @airburst_studios .I wasn't sure if calling it every update will affect performance so I limited it to every 1 sec. Anyway, I changed it to be called from a thread every 100ms (similar to how it's done in the sample app). I'm testing it right now and I'll update if it solved my issue.

 

@airburst_studios @danri I can confirm that issue is now resolved after moving to separate thread and updating every 100ms! Many thanks for the help.

 

I'm having this issue too. My server shuts down every 2-3 minutes and I've tried everything you guys suggested and it is not working. I'm calling "UpdateServerCheck" 10x per second in async loop. I also tried putting it in monobehaviour Update() where it is updating every frame (30+ fps).

I am completely lost. There is no actual game code yet since I haven't gotten to the integration part. It's just simply creating and joining a simple demo match.

 

I figured out the issue. Apparently it requires a name to not be empty (which is visible on the error message). I thought it was displaying the name of the server instead of it being an error message.

Been pulling my hair out for a week over this. The error message should be more clear. Not sure why a server even requires a name.

 

Thanks a lot, I would've never thought of this.

 

I've been encountering the same problem for about 3-4 days. First I set the Update Rate to 1 second but it did the same problem. Then I changed it to 100 milliseconds and tested it now, it's fixed. Thanks guys !