As Unity is very .Net based could it adopt the F# and Visual Basic Languages?

Pretty crazy how much slower Mono is, will try built too

 

wtf: in built I get

For avg: 26740; Foreach avg: 6669.6

 

Dont know what Mono was doign that first round, but here is a more sane result

For avg: 6455.5; Foreach avg: 6373.2

 

Ok I guess I'll eat my words then. Hopefully I'm right about heap/stack and structs at least.

I was just going by dotnetperls, I suppose they are wrong.

Run a million ticks just out of curiosity.

I don't to go off topic here too much anyway. I have a tendency to blabber so.. there you go.

According your test i don't think it would matter what you use unless you were on a 286dx from 1985 lol.

 

Iteration over structs and operate on them vs calling a collection of class references is much cheaper yes (in favor of structs). This is exactly why ECS is so much faster

Also foreach does not alocate, I moved the bench to update and rewrote it so it did not alllocate, and the actual foreach does not allocate



Word of caution though, if you foreach over a IEnumerable<T> the first access to the IEnumerable will allocate



edit: Casting the IENumerable back to a int[] before iterating it does not allocate, Mono moves in mysterious ways

 

Honestly, you shouldn't speak so freely when you're so ignorant. If you don't use a closure this has zero allocation.


How do I get zero allocation with a linq statement? I use list pools and I manually release.

Code (csharp):

1.  
2.   public class AllocationTest : MonoBehaviour{
3.     private List< int > m_RandomNumberList;
4.     private int m_Average;
5.     public int DivisibleCheck = 3;
6.     [UnityMessage]
7.     private void Awake(){
8.       m_RandomNumberList = new List< int >();
9.       for( int i = 0 ; i < 100 ; i++ ) m_RandomNumberList.Add( Dice.Between( 0, 1000 ) );
10.     }
11.     [UnityMessage]
12.     private void Update(){
13.       DoNonClosure();
14.       DoClosure();
15.       LinqNonClosure();
16.     }
17.     private void LinqNonClosure(){
18.       Profiler.BeginSample( "linq closure" );
19.       IEnumerable< int > list = m_RandomNumberList.Where( i => i % DivisibleCheck == 1 );
20.       m_Average = 0;
21.       foreach( int n in list ) m_Average += n;
22.       m_Average = m_Average / list.Count();
23.       Profiler.EndSample();
24.     }
25.     private void DoClosure(){
26.       Profiler.BeginSample( "Getting divisible by x (closure)" );
27.       PooledList< int > list = m_RandomNumberList.GetWhere( i => i % DivisibleCheck == 1 );
28.       m_Average = 0;
29.       for( int i = 0 ; i < list.Count ; i++ ) m_Average += list[i];
30.       m_Average = m_Average / list.Count;
31.       list.Release();
32.       Profiler.EndSample();
33.     }
34.     private void DoNonClosure(){
35.       Profiler.BeginSample( "Getting divisible by 3 (non closure)" );
36.       PooledList< int > list = m_RandomNumberList.GetWhere( i => i % 3 == 1 );
37.       m_Average = 0;
38.       for( int i = 0 ; i < list.Count ; i++ ) m_Average += list[i];
39.       m_Average = m_Average / list.Count;
40.       list.Release();
41.       Profiler.EndSample();
42.     }
43.   }
44.  

Note that there are ZERO allocations when not using a closure.

Read more carefully please.

 

Its a bit dangerous though if you use fluent syntax, like

Code (CSharp):

1. var result = source.Where(s => s.ID == 1)
2.    .Select(s => s.Property);
3.  
4. result.Release();

The first list will not be released

 

Absolutely true. It requires some discipline to use. OTOH, I can use it in places where I do care about performance (but not mega critical path). It's a tradeoff.

The main example for usage - is that I can use these in Update() methods if they're relatively simple and don't require a closure. In general, avoiding closures is the biggest problem.

 

btw - if anyone wants my listpool implementation:

Spoiler

Code (csharp):

1.  
2. using System.Collections.Generic;
3. public class PooledList<T> : List<T>{
4.   public void Release(){ ListPool.Release( this ); }
5.  
6.   /// <summary>
7.   /// Shadow Reverse because it has massive allocations which bugs the F*** out of me.
8.   /// </summary>
9.   public new void Reverse(){
10.     int i = 0;
11.     int j = Count-1;
12.     while (i < j){
13.       var t = this[i];
14.       this[i] = this[j];
15.       this[j] = t;
16.       i++;
17.       j--;
18.     }
19.   }
20. }
21. public static class ListPool{
22.  
23.   public static int GetCount<T>(){
24.     return PoolOfLists< T >.m_Available.Count;
25.   }
26.   public static PooledList< T > Get< T >(){
27.     return PoolOfLists< T >.Get();
28.   }
29.   public static void Release< T >( PooledList< T > items ){
30.     PoolOfLists< T >.Release( items );
31.   }
32.   private static class PoolOfLists< T >{
33.     public static readonly Queue< PooledList< T > > m_Available = new Queue< PooledList< T > >();
34.     public static PooledList< T > Get(){
35.       PooledList< T > list;
36.       if( m_Available.Count > 0 ){ list = m_Available.Dequeue(); }
37.       else{ list = new PooledList< T >(); }
38.       return list;
39.     }
40.     public static void Release( PooledList< T > list ){
41.       // note that Queue.Contains will allocate - this does not.
42.       foreach( var l in m_Available ) if( l == list ){
43. #if UNITY_EDITOR
44.         Log.Error("Releasing a list to pool twice");
45. #endif
46.         return;
47.       }
48.    
49.       list.Clear();
50.       m_Available.Enqueue( list );
51.     }
52.   }
53. }
54.  

I originally got the idea from unity's listpool imp, but I made a subclass to help keep track when lists are pooled or not to prevent misuse.

edit: added spoiler tag

 

What if Unity used the Roslyn/Jobs/Burst Compiler technology combined with a LINQ syntax for ECS could that provide an easier syntax for ECS?

So instead of lots of boilerplate code, we could be writing short ECS systems in a few lines of code.

 

Isn't that the long term goal? We're only seeing the boilerplate we do because it's literally a preview.

 

Moral of the thread so far, always profile before you speak.

 

You could chain the release. But it works against how immutable Linq works by design, and could also lead developers into darkness

for example

Code (CSharp):

1. var r1 = source.Where(foo => foo.Id == 1);
2. var r2 = r1.Take(1);
3.  
4. r2.Release();

Would have the unexpected side effect of r1 being released too. You could have a ReleaseAll method and a Release method. The Release method crashes if it detects more than one chained list to release.

 

Does it really matter? Mono is only used by the editor, most platforms are on IL2CPP now...

 

Not by default and not all libraries etc can be compiled to it

 

Well obviously

 

Thus Mono performance is still important? .NET CLR/compiler is about 5 times faster in my tests above

 

Nobody compiles to Mono anymore if the platform supports IL2CPP, unless there is a case of a 3rd party plugin that doesn't support it or something. You would be silly to still compile to Mono.

Most major platforms support IL2CPP now so my point is that Mono is not really that relevant anymore, except for the build targets that don't support it yet.

 

I wonder what the success rate of running projects under IL2CPP is. PLus if IL2CPP was a silver bullet Unity would default to it

 

So what are the current drawbacks to IL2CPP, anyone know?

 

No JIT compiler optimizations, though I do not know how well that works on Mono

 

Yep. I've just trained myself to avoid fluent syntax when using these. Sometimes I will even use optional {}'s to scope usage.

Code (csharp):

1. var o = source.GetWhere( ... );
2. {
3.   ... stuff
4. }
5. o.Release();

All in all, it's a minor annoyance for a minor performance gain. But it does really make a difference.

Performance is such a weird thing, we're trained to not premature optimize or micro optimize, but when you write tons of code that doesn't concern itself with performance it can really be hard to optimize later - since there are tiny problems scattered everywhere.

Like, if you need to reduce allocation because GC is firing too much, and you look at profiler and there are tiny allocations everywhere what do you do? Same thing with general performance. It's much, much harder to optimize code when there aren't glaringly obvious problem spots and the costs are simply distributed all across a code base.

Like quarter millisecond on it's own is nothing, but 8 components using a quarter milli is 2ms in time cost. It all adds up.

 

As long as you streamline the continues methods you are usually fine. And distinct the continues methods from the user actions methods, if it's a MP game it can also be good to know it the method is local host only or can trigger from other player. For example in our game we have alot of detailed stuff that can only happen twice per frame in extreme worst case (two hands in VR) and never from a remote client, so o(2) is worst. As long as you make sure your flow in continues methods (updates, coroutines etc) are well otpimize you can take the luxury to use some syntax sugar in the the other methods

 

My point is - what do you do when you have like 40 classes each using .25ms per frame. Optimizing that kind of situation is difficult because your attention needs to be so scattered.

Let's say that on average you can save like 30% of the time for each element. That brings your total cost down from 10ms to 7ms, but it's difficult to scan through all that code and look for inefficiency.

I think most of the time, when we hit those situations and the code ripe for optimization isn't obvious, we just kinda give up on optimization. I know I've been guilty of that.

 

In game dev I think you need to think about per frame allocation and frame time right away. It's sub optimization we want to avoid, like the dude earlier that didn't use var becuse he thought they degraded performance and stuff like that.

We have a peformance run before each sprint ends. That sometimes catches a few things. 95 procent of the time in third party libraries

Edit: this sprint is soon one year (Advanced AI) so it will be fun to see how much the performance have degraded

 

This is what I get when I try to build our game with il2cpp

 

No.

And it would be a major drawback if they did. You can't track multiple languages and hold the best practices across the board.
Also almost no one speaks F# https://www.tiobe.com/tiobe-index/ so it would not do any good.

The guys answered this once somewhere, I can't find it right now, but they said they're trying to avoid any syntax which could cause performance-hit if people start to use the given syntax out of the ECS/Jobs scope.
Which means LINQ probably out of luck here, since using LINQ will cause you big problems if you just start to use it left and right. But will see, too many people are complaining about the keystrokes nowadays (apparently coding in ASM is out of the fashion too ).

I personally don't like the LINQ at all and I avoid it for any cost because it's unreadable and you have a hard time to estimate what is really happening if you invoke one. I like the clear code which shows what is happening, how many times it's happening and what is the cost of the given code.

 

How can you know the cost of code that is converted to IL then IL2CPP then compiled to native via a CPP compiler with the option of the Burst compiler kicking in somewhere and Jobs or Unity API overheads added to the code you write.

I think you're saying you like code that is easy for you to read and grasp (because you are used to the syntax) as unless you are a coding genius who traces the entire build process and decompiles each stage there is no way of knowing what is happening from the code you write to what happens on your CPU!

LINQ has unusual syntax and is renowned for expanding out to GC unfriendly code that can cause memory problems or performance issues you have no control over.

What if Unity developed a LINQ compile path for it's ECS/Jobs/Burst compiler technology that was guaranteed to produce performant memory-safe code with a fraction of the boilerplate needed for the OOP ECS API?

 

Thank you for knowing what I'm thinking without asking me what I'm thinking. You're probably right, my 20 years professional software engineering experience and more than 30 years of programming experience mean nothing.
BTW, it is not true that you have no way of knowing what's happening with your code. You can check the intermediate code any time. So you can have pretty good idea what translates to what when it comes to simple commands.
It is extremely hard to learn when it comes to abstract things like LINQ because of the number of commands you will end up with when the underlying code gets included after LINQ has been resolved.
It's not impossible, but it's extremely hard to learn on a meaningful way.

Otherwise I advise you to read https://jacksondunstan.com/ He's doing very great work to investigate what translates what when you compile C# to the intermediate language.

I exactly said just this. You have no way knowing at first glance that what you will end up with. See my rambling above too.

I addressed it above. It is a possibility, but it would create another very bad situation when you advised to use a pattern to write very performant code in a way which in another scope is very non-performant.
(Writing ECS system code would be very convenient and very performant, but when you use LINQ in general, it's a disaster for your game)

 

My point here is that LINQ brings functional programming features to C# and ECS being data-driven would be better suited to a more functional approach IMHO be that LINQ or F#.

It seems a lot of the verbose boilerplate is trying to fit a functional system into an OOP language.

Hopefully, by looking at other alternatives and options like using LINQ and Rosyln compiler technologies Unity can make ECS an easier to use syntax.

Another approach would be a visual ECS programming system, then all of the boilerplate could be dropped and we would have a way to set up the data and logic and operator nodes that process the data.

It's just applying the KISS principle to the task at hand, writing good ECS code. Do we need all of the complex OOP, boilerplate and attributes everywhere?

Could we even use inheritance and subclasses of Systems or would they just fight over their data that matches their filters?

 

Again, why?

 

I want to know why you want to change how ECS is programmed when how ECS is currently entirely prototype / subject to lots of changes?

 

Yes, they would. But as we already established before, introducing a new language is crazy and next to impossible. And it would be very inefficient and wouldn't serve anybody, not even you. But certainly not Unity's average userbase, since as I stated above, in the world, very few people speak F#.

LINQ is a possibility but with great danger. You can bet, if they do that and they don't remove LINQ in general, you will end up with a forum which will be full of the complaints from everyone that "oh, I saw we define very good and quick code like this, I tried to use the same in my Update method and everything is tanked".

I think either way, it's better to have a very long and verbose code instead of a risky one. Because you can help it, there are templates and shortcuts in the IDEs, you can develop intermediate code to help it.

And at the end, Unity probably will find a proper way to reduce the verbose nature of this without pulling off any of these crazy things. You just have to be patient a little bit, given that the current code is in preview, nowhere near even beta.

 

Thats not clear code, clear code is when its easy to understand what the code is ment to achieve, not how its implemented.

 

A long and verbose code IS a risky one. Throw in 10 or more programmers writing long and verbose code in the same project and you have a unmaintainable mess really fast.

 

In my daytime job I think the same, since I'm a business software developer (working on huge code base with hundreds of people for a giant company). But it's not true when you're working on a game's code base with a couple of people and the performance of the code becomes more important than the readability.

 

Depends on the complexity of the software too, not only the number of people involved. Sure if you are one or two people writing something trivial then it could work. I would write clean code anyway, as long as it does not affect performance.

 

Of course, I'm not talking about using lazy LINQ in the main menu or things like that (although I really don't like the philosophy of it, so I probably would find another way regardless), I'm talking about the game code.
Yes, some people would call premature optimization, but if you get used to it, it becomes second nature.
As soon as you work on some action-type game or with high frame rate, it's guaranteed that you will refactor your LINQ from any Update loop. So after a while you won't write in it at the first place.
And you can mitigate these kind of things with simple functions/methods and aggressive inlining.

It's like my very first attempt to write C64 demo in BASIC. Then I learned ASM instead and handled the IRQs properly. Use your tools properly.

 

The thing is this is not a small team or a big team issue it's a game development community, where surely the KISS principle should be paramount to the technology developer.

With the Roslyn compiler then Unity could adapt C# or F# or VB to use a clean easy to use syntax for ECS systems.

I assume you also think the current Unity ECS API is growing in complexity and getting loaded down with boilerplate code for systems that just need to match a data profile and run a process on that data.

If you think about ECS in an abstract way you could just say you are running a method that matches a data filter/pattern. The question is could the API be simplified down to the level of...

Code (CSharp):

1. class MySystem: ECSystem {
2.  
3. Execute(DataProfile data) {
4. ... process data
5. }
6.  
7. }

 

I personally hate when I cant write Linq becasue of performance reasons. Like this method that runs every frame and checks if the VR hands are hovering a item.

Code (CSharp):

1.         private void TryPickupClosest()
2.         {
3.             if (!CanInteractWithWorld) return;
4.  
5.             NVRInteractable closest = null;
6.             float closestDistance = float.MaxValue;
7.        
8.             foreach (var hovering in currentlyHoveringOver)
9.             {
10.                 if (hovering.Key == null || hovering.Key.IsInUseByOthers)
11.                     continue;
12.          
13.                 float distance = (this.Model.PickupPoint.position - hovering.Key.ColliderParent.transform.position).sqrMagnitude;
14.  
15.                 //Ugly I know, but OrderBy prio is expensive
16.                 if (closest != null && hovering.Key.PickupPriority > closest.PickupPriority)
17.                 {
18.                     closest = null;
19.                     closestDistance = float.MaxValue;
20.                 }
21.  
22.                 if ((closest == null || hovering.Key.PickupPriority >= closest.PickupPriority && distance < closestDistance) && GetPickupCommandState(hovering.Key.ItemType) == ButtonAction.PressDown)
23.                 {
24.                     closestDistance = distance;
25.                     closest = hovering.Key;
26.                 }
27.             }
28.      
29.             if (closest != null)
30.             {          
31.                 BeginInteraction(closest.GetCustomPickup(this, true));
32.             }
33.         }

Just imagine if I could use Linq

Code (CSharp):

1.         private void TryPickupClosestLinq()
2.         {
3.             if (!CanInteractWithWorld) return;
4.  
5.             var closest = currentlyHoveringOver
6.                 .OrderByDescending(h => h.Key.PickupPriority)
7.                 .ThenBy(h => (this.Model.PickupPoint.position - h.Key.ColliderParent.transform.position).sqrMagnitude)
8.                 .FirstOrDefault(h => GetPickupCommandState(h.Key.ItemType) == ButtonAction.PressDown)?.Key;
9.  
10.             if(closest != null)
11.                 BeginInteraction(closest.GetCustomPickup(this, true));
12.         }

 

Brrr, I'll have nightmares.

Joking aside I know why you like it, and I know why in enterprise people like the builder pattern (these have the same roots in terms of readability). I personally don't share the enthusiasm over nuance things like this, I read the upper code more easily than the below one, probably because I started where and when I started. Natural language means nothing to me in terms of coding.

BTW, probably Unity can use the Builder pattern to define ECS and Job-related stuff. It's a decent candidate without redefine the language and risking that people will be confused over performance. Also it can be compressed like LINQ and can be structured even on the same way.

 

Fluent syntax, I use it all the time for my own APIs too.

 

That's the thing about LINQ verses classical approach: the LINQ version takes most programmers longer to do. Programming performance isn't dictated by how much you type, but how much thinking there is to do.

There is no logic making ECS accept LINQ syntax.

https://www.zdnet.com/article/developers-despair-half-your-time-is-wasted-on-bad-code/

In this case, developers would quickly and efficiently kill their projects stone dead with LINQ-ECS. Even in this very thread, it was pointed out how dangerous LINQ can be if you're not very experienced in it. So nah, LINQ will stay LINQ and Unity will gently massage C#. Everyone wins.

The only reason LINQ was suggested in this thread is because a) arowx is arowx and b) Anders is elitist (but that's OK he has earned that in his day job, however it makes him a minority around the diverse parts of game dev so have to bear that in mind).

 

Today I rewrote something that before my rewrite took 4 hours to complete, and after rewrite it took 8 seconds. Felt pretty elite

 

LINQ certainly takes me longer to write for common tasks... but that's because I've got loads of practice with the "old" way and barely any practice using LINQ. I can imagine getting pretty darn quick at the LINQ way if I used it a bunch, and if I were writing different software I can imagine using it a bunch.

 

I find LINQ great for some smaller things. Min/MaxBy for example is super useful. I have however, found that large and complex LINQ queries are just hell to debug, and have ended up rewriting some queries procedurally to debug them (at which point I might as well just stick with the procedural version).

 

Since we talk functional languages i guess we can continue talking about Linq since its a functional language more or less inside C#. Though you can write mutable code in Linq, something I think you should avoid, but some times its necessary, example when mapping a EF ORM result to something only existing in memory not database.Like this example Im sitting with right now at day work

Code (CSharp):

1.  
2. return new ListPremiumOrdersResponse
3. {
4.     Orders = query.ToList().Where(info =>
5.     {
6.         //Mutation happens here
7.         info.dto.PremiumBenefitSupplier = GetPremiumBenefitSupplier(info.PremiumType);
8.         return info.ExistingPremiumBenefitSupplierIds.All(id => info.dto.PremiumBenefitSupplier.Id != id);
9.     })
10.     .Select(info => info.dto)
11.     .ToList()
12. };
13.  

Anyway I cant understand how people can think Linq is harder than using vanilla collections, I mean its practically plain english

Code (CSharp):

1. var closest = currentlyHoveringOver
2.     .OrderByDescending(h => h.Key.PickupPriority) //First order by this
3.     .ThenBy(h => (this.Model.PickupPoint.position - h.Key.ColliderParent.transform.position).sqrMagnitude) //If they happen to be same then order by this
4.     .FirstOrDefault(h => GetPickupCommandState(h.Key.ItemType) == ButtonAction.PressDown)?.Key; //Take the first item honoring the predicate

But it can get a bit harder, here is a left join with Linq expressions

Code (CSharp):

1. leftSide.SelectMany(l => rigthSide.Where(r => r.Id == l.Id).DefaultIfEmpty(), (l, r) => new { l, r })
2.    .Select(lr => new { lr.l, HasRightSide = ll.r != null })
3.    .ToList();

 

We've noticed.

It's not that it's truly harder. It's that experience plays a massive role in how quickly and easily someone is able to read and write code for one approach versus the other approach. Having spent years working with collections I'm much more easily able to work with collections than I am with LINQ which I have only dabbled once or twice with.

 

Foreach was slower at some point in time. Nowadays, they're about the same speed, unless, it's using non-cached iterator. Like SortedList's one.

There's a proper list on the net, what allocates and do not allocates GC when using iterators with for (for the current~ version of Unity).

Safe are List, Dictionary, Arrays, HashSet. Maybe something else I forgot. In terms of the iteration performance for / foreach they're the same. (Not counting HashSet for the for, it's drastically slower due to .ElementAt() but that's a Linq anyway).

On the Linq topic - never use Linq, if you want performance and you don't have a heap allocation viewer plugin installed. Otherwise - use with caution.

Linq is a sign, that your code architecture is messed up. Multiple times I saw things that can be done 10 times easier, and duck-taped together with Linq, causing even more performance problems than they should in the first place.

+1 for verbose and simple code.

 

I use sorted list when I want to remove or add without the risk for a o(n) scenario. But you illitterate over them a bit different, this is actual example from our game

Code (CSharp):

1.  public abstract class PollingPool<T> where T : Component
2.     {
3.         private readonly T prefab;
4.  
5.         private readonly Queue<T> pool = new Queue<T>();
6.         private readonly LinkedList<T> inuse = new LinkedList<T>();
7.         private readonly Queue<LinkedListNode<T>> nodePool = new Queue<LinkedListNode<T>>();
8.  
9.         private int lastCheckFrame = -1;
10.  
11.         protected PollingPool(T prefab)
12.         {
13.             this.prefab = prefab;
14.         }
15.  
16.         private void CheckInUse()
17.         {
18.             var node = inuse.First;
19.             while (node != null)
20.             {
21.                 var current = node;
22.                 node = node.Next;
23.  
24.                 if (!IsActive(current.Value))
25.                 {
26.                     current.Value.gameObject.SetActive(false);
27.                     pool.Enqueue(current.Value);
28.                     inuse.Remove(current);
29.                     nodePool.Enqueue(current);
30.                 }
31.             }
32.         }
33.  
34.         protected T Get()
35.         {
36.             T item;
37.  
38.             if (lastCheckFrame != Time.frameCount)
39.             {
40.                 lastCheckFrame = Time.frameCount;
41.                 CheckInUse();
42.             }
43.  
44.             if (pool.Count == 0)
45.                 item = GameObject.Instantiate(prefab);
46.             else
47.                 item = pool.Dequeue();
48.  
49.             if (nodePool.Count == 0)
50.                 inuse.AddLast(item);
51.             else
52.             {
53.                 var node = nodePool.Dequeue();
54.                 node.Value = item;
55.                 inuse.AddLast(node);
56.             }
57.        
58.             item.gameObject.SetActive(true);
59.  
60.             return item;
61.         }
62.  
63.         protected abstract bool IsActive(T component);
64.     }
65. }

And the itterating part

Code (CSharp):

1. private void CheckInUse()
2.         {
3.             var node = inuse.First;
4.             while (node != null)
5.             {
6.                 var current = node;
7.                 node = node.Next;
8.  
9.                 if (!IsActive(current.Value))
10.                 {
11.                     current.Value.gameObject.SetActive(false);
12.                     pool.Enqueue(current.Value);
13.                     inuse.Remove(current);
14.                     nodePool.Enqueue(current);
15.                 }
16.             }
17.         }

When using a LinkedList its important to use the methods that use LinkedListNode<T> instead of T since the latter will allocate since it will drop node references.