Search Unity

  1. Megacity Metro Demo now available. Download now.
    Dismiss Notice
  2. Unity support for visionOS is now available. Learn more in our blog post.
    Dismiss Notice

NativeList Error

Discussion in 'Entity Component System' started by Neriad, Oct 9, 2019.

  1. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    Hey everyone,

    Last time I had this error I decided to use NativeArrays as it was working ok, but now working on something else and getting the same error I would like to understand.

    Code (CSharp):
    1. public List<Area> GetCameraAreasJob (Vector3 _camPoint00, Vector3 _camPoint01, Vector3 _camPoint11)
    2.     {
    3.         if (!init)
    4.             return null;
    5.  
    6.         cameraAreas.Clear();
    7.         _camPoint00.x = _camPoint11.x;
    8.  
    9.         float3 camPoint00 = _camPoint00;
    10.         float3 camPoint11 = _camPoint11;
    11.         float3 areaOffset = this.areaOffset;
    12.         NativeArray<float3> areaPoints00 = new NativeArray<float3>(allAreas.Length, Allocator.TempJob);
    13.         NativeArray<float3> areaPoints11 = new NativeArray<float3>(allAreas.Length, Allocator.TempJob);
    14.         NativeArray<float3> areaCenterPoints = new NativeArray<float3>(allAreas.Length, Allocator.TempJob);
    15.         NativeList<int> xAreas = new NativeList<int>(allAreas.Length, Allocator.TempJob);
    16.         NativeList<int> finalAreas = new NativeList<int>(allAreas.Length, Allocator.TempJob);
    17.  
    18.         for (int i = 0; i < allAreas.Length; i++)
    19.         {
    20.             areaPoints00[i] = allAreas[i].Point00;
    21.             areaPoints11[i] = allAreas[i].Point11;
    22.             areaCenterPoints[i] = allAreas[i].CenterPoint;
    23.         }
    24.  
    25.         GetCameraAreas getCameraAreasJob = new GetCameraAreas
    26.         {
    27.             camPoint00 = camPoint00,
    28.             camPoint11 = camPoint11,
    29.             areaOffset = areaOffset,
    30.             areaPoints00 = areaPoints00,
    31.             areaPoints11 = areaPoints11,
    32.             areaCenterPoints = areaCenterPoints,
    33.  
    34.             xAreas = xAreas,
    35.             finalAreas = finalAreas,
    36.         };
    37.         JobHandle job = getCameraAreasJob.Schedule(allAreas.Length, allAreas.Length);
    38.         job.Complete();
    39.  
    40.         for (int i = 0; i < finalAreas.Length; i++)
    41.             cameraAreas.Add(allAreas[finalAreas[i]]);
    42.      
    43.         areaPoints00.Dispose();
    44.         areaPoints11.Dispose();
    45.         areaCenterPoints.Dispose();
    46.         xAreas.Dispose();
    47.         finalAreas.Dispose();
    48.  
    49.         UpdateTextNumberColors();
    50.  
    51.         return cameraAreas;
    52.     }
    53.  
    54.  
    55. [BurstCompile]
    56. public struct GetCameraAreas : IJobParallelFor
    57. {
    58.     public float3 camPoint00, camPoint11;
    59.     public float3 areaOffset;
    60.     public NativeArray<float3> areaPoints00, areaPoints11, areaCenterPoints;
    61.     // Creates error
    62.     public NativeList<int> xAreas, finalAreas;
    63.  
    64.  
    65.  
    66.     public void Execute (int _index)
    67.     {
    68.         if ((areaPoints00[_index].x > camPoint00.x && areaPoints00[_index].x < camPoint11.x) || (areaPoints11[_index].x > camPoint00.x && areaPoints11[_index].x < camPoint11.x) || (camPoint00.x > areaCenterPoints[_index].x - areaOffset.x && camPoint00.x < areaCenterPoints[_index].x + areaOffset.x) || (camPoint11.x > areaCenterPoints[_index].x - areaOffset.x && camPoint11.x < areaCenterPoints[_index].x + areaOffset.x))
    69.             // Creates error
    70.             xAreas.Add(_index);
    71.  
    72.         if ((areaPoints00[_index].z > camPoint00.z && areaPoints00[_index].z < camPoint11.z) || (areaPoints11[_index].z > camPoint00.z && areaPoints11[_index].z < camPoint11.z) || (camPoint00.z > areaCenterPoints[_index].z - areaOffset.z && camPoint00.z < areaCenterPoints[_index].z + areaOffset.z) || (camPoint11.z > areaCenterPoints[_index].z - areaOffset.z && camPoint11.z < areaCenterPoints[_index].z + areaOffset.z))
    73.             if (xAreas.Contains(_index))
    74.                 // Creates error
    75.                 finalAreas.Add(_index);
    76.     }
    77. }
    Why is this actually not usable ? Native lists do not have the same behaviour as classic lists ? I have commented the parts that are supposedly wrong. The error I get is "InvalidOperationException: GetCameraAreas.xAreas is not declared [ReadOnly] in a IJobParallelFor job. The container does not support parallel writing. Please use a more suitable container type."

    Thanks :)
     
  2. sschoener

    sschoener

    Joined:
    Aug 18, 2014
    Posts:
    73
    The problem here is that you are using a parallel job to write to a container (NativeList in this case). You cannot just write to a single container from multiple jobs: The NativeList is not thread safe. You can get around this by using a NativeQueue like this:
    Code (csharp):
    1. NativeQueue<int> queue = new NativeQueue<int>(Allocator.TempJob);
    2. NativeQueue<int>.ParallelWriter queueWriter = queue.AsParallelWriter();
    3. job.xAreas = queueWriter;
    In your job you can then use Enqueue on the parallel writer to write to the queue from (potentially) multiple threads.
     
  3. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    I'm not sure I understand. Native arrays are also containers right ? And it works. Then what's the point of having native lists ?
     
  4. sschoener

    sschoener

    Joined:
    Aug 18, 2014
    Posts:
    73
    In your example, you are only reading from the NativeArrays you are using, but not writing to them. Parallel reading is no problem, but parallel writing is. If you have a NativeArray in an IJobParallelFor job and you are writing to that NativeArray the safety system will only allow you to write to the current index (the parameter of the Execute function), so you could potentially have a safe write to a NativeArray. With a NativeList however, there is no such mechanism and you can never safely add to it in parallel. Does that help? :)
     
  5. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    Oh okay, I think I start to understand, thanks ! :)
     
  6. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    So, I have managed to make it work, but as a BIG SURPRISE, the job implementation is actually slower (doubles ms needed) than the regular implementation. Usually, when I jobified some code parts, it used to be way better, as supposed.

    Regular :
    Code (CSharp):
    1.     List<Area> cameraAreas = new List<Area>();
    2.     public List<Area> GetCameraAreas (Vector3 _camPoint00, Vector3 _camPoint01, Vector3 _camPoint11)
    3.     {
    4.         if (!init)
    5.             return null;
    6.  
    7.         cameraAreas.Clear();
    8.         _camPoint00.x = _camPoint01.x;
    9.  
    10.         bool xTrue = false;
    11.         for (int i = 0; i < allAreas.Length; i++)
    12.         {
    13.             xTrue = false;
    14.             if ((allAreas[i].Point00.x > _camPoint00.x && allAreas[i].Point00.x < _camPoint11.x) || (allAreas[i].Point11.x > _camPoint00.x && allAreas[i].Point11.x < _camPoint11.x) || (_camPoint00.x > allAreas[i].CenterPoint.x - areaOffset.x && _camPoint00.x < allAreas[i].CenterPoint.x + areaOffset.x) || (_camPoint11.x > allAreas[i].CenterPoint.x - areaOffset.x && _camPoint11.x < allAreas[i].CenterPoint.x + areaOffset.x))
    15.                 xTrue = true;
    16.  
    17.             if ((allAreas[i].Point00.z > _camPoint00.z && allAreas[i].Point00.z < _camPoint11.z) || (allAreas[i].Point11.z > _camPoint00.z && allAreas[i].Point11.z < _camPoint11.z) || (_camPoint00.z > allAreas[i].CenterPoint.z - areaOffset.z && _camPoint00.z < allAreas[i].CenterPoint.z + areaOffset.z) || (_camPoint11.z > allAreas[i].CenterPoint.z - areaOffset.z && _camPoint11.z < allAreas[i].CenterPoint.z + areaOffset.z))
    18.                 if (xTrue)
    19.                      cameraAreas.Add(allAreas[i]);
    20.         }
    21.  
    22.         UpdateTextNumberColors();
    23.  
    24.         return cameraAreas;
    25.     }
    Jobified:
    Code (CSharp):
    1. public List<Area> GetCameraAreasJob (Vector3 _camPoint00, Vector3 _camPoint01, Vector3 _camPoint11)
    2.     {
    3.         if (!init)
    4.             return null;
    5.  
    6.         cameraAreas.Clear();
    7.         _camPoint00.x = _camPoint01.x;
    8.  
    9.         float3 camPoint00 = _camPoint00;
    10.         float3 camPoint11 = _camPoint11;
    11.         float3 areaOffset = this.areaOffset;
    12.         NativeArray<float3> areaPoints00 = new NativeArray<float3>(allAreas.Length, Allocator.TempJob);
    13.         NativeArray<float3> areaPoints11 = new NativeArray<float3>(allAreas.Length, Allocator.TempJob);
    14.         NativeArray<float3> areaCenterPoints = new NativeArray<float3>(allAreas.Length, Allocator.TempJob);
    15.         NativeArray<int> finalAreas = new NativeArray<int>(allAreas.Length, Allocator.TempJob);
    16.  
    17.         for (int i = 0; i < allAreas.Length; i++)
    18.         {
    19.             areaPoints00[i] = allAreas[i].Point00;
    20.             areaPoints11[i] = allAreas[i].Point11;
    21.             areaCenterPoints[i] = allAreas[i].CenterPoint;
    22.         }
    23.  
    24.         GetCameraAreas getCameraAreasJob = new GetCameraAreas
    25.         {
    26.             camPoint00 = camPoint00,
    27.             camPoint11 = camPoint11,
    28.             areaOffset = areaOffset,
    29.             areaPoints00 = areaPoints00,
    30.             areaPoints11 = areaPoints11,
    31.             areaCenterPoints = areaCenterPoints,
    32.  
    33.             xTrue = false,
    34.             finalAreas = finalAreas,
    35.         };
    36.         JobHandle job = getCameraAreasJob.Schedule(allAreas.Length, allAreas.Length);
    37.         job.Complete();
    38.  
    39.         for (int i = 0; i < finalAreas.Length; i++)
    40.             if (finalAreas[i] == 1)
    41.                 cameraAreas.Add(allAreas[i]);
    42.    
    43.         areaPoints00.Dispose();
    44.         areaPoints11.Dispose();
    45.         areaCenterPoints.Dispose();
    46.         finalAreas.Dispose();
    47.  
    48.         UpdateTextNumberColors();
    49.  
    50.         return cameraAreas;
    51.     }
    52.  
    53. [BurstCompile]
    54. public struct GetCameraAreas : IJobParallelFor
    55. {
    56.     public float3 camPoint00, camPoint11;
    57.     public float3 areaOffset;
    58.     public NativeArray<float3> areaPoints00, areaPoints11, areaCenterPoints;
    59.     public bool xTrue;
    60.     public NativeArray<int> finalAreas;
    61.  
    62.  
    63.  
    64.     public void Execute (int _index)
    65.     {
    66.         xTrue = false;
    67.         if ((areaPoints00[_index].x > camPoint00.x && areaPoints00[_index].x < camPoint11.x) || (areaPoints11[_index].x > camPoint00.x && areaPoints11[_index].x < camPoint11.x) || (camPoint00.x > areaCenterPoints[_index].x - areaOffset.x && camPoint00.x < areaCenterPoints[_index].x + areaOffset.x) || (camPoint11.x > areaCenterPoints[_index].x - areaOffset.x && camPoint11.x < areaCenterPoints[_index].x + areaOffset.x))
    68.             xTrue = true;
    69.    
    70.         if ((areaPoints00[_index].z > camPoint00.z && areaPoints00[_index].z < camPoint11.z) || (areaPoints11[_index].z > camPoint00.z && areaPoints11[_index].z < camPoint11.z) || (camPoint00.z > areaCenterPoints[_index].z - areaOffset.z && camPoint00.z < areaCenterPoints[_index].z + areaOffset.z) || (camPoint11.z > areaCenterPoints[_index].z - areaOffset.z && camPoint11.z < areaCenterPoints[_index].z + areaOffset.z))
    71.             if (xTrue)
    72.                 finalAreas[_index] = 1;
    73.     }
    74. }
    Is it because whats in the job isnt really complicated calculations and the time spent is mostly on assigning values to the nativearrays ?
    I have tested with an array of 100 and an array of 40000.
    Regular takes 0.03 ms with 100 and 3.5 ms with 40000.
    Jobified takes 0.09 ms with 100 and 7.5 ms with 40000.

    EDIT : Maybe I can try having those array with Allocator.Persistent as the data never changes at runtime ? But this would require keeping a lot of data stored ?
     
    Last edited: Oct 9, 2019
  7. julian-moschuering

    julian-moschuering

    Joined:
    Apr 15, 2014
    Posts:
    529
    1. Get rid of your loop were you check each element. Use NativeQueue<int>.ParallelWriter.
    2. Try IJobParallelForBatch.
     
  8. eizenhorn

    eizenhorn

    Joined:
    Oct 17, 2016
    Posts:
    2,683
    You tested it in build or editor? You should profile build.
     
  9. julian-moschuering

    julian-moschuering

    Joined:
    Apr 15, 2014
    Posts:
    529
    Also it is an option to just use a IJob with [BurstCompile] and a NativeList and then call Run instead of Schedule to get burst compiled code.
     
  10. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    So, I did set the 3 arrays to persistent, and get them out to global variables, and with 40000 I reduced the ms count to 1.5 ms instead of 7.5 ms. So I guess it's cool, but I thought I would gain more actually.

    @julian-moschuering
    I was actually writing that I don't understand what you were talking about, but I did when writing ahah. I'm gonna try removing that loop that checks if == 1 by using the queue. Ill check the documentation about IJobParallelForBatch after that.
    Meanwhile, I really did not understand your last post with Run and Schedule.

    @eizenhorn
    Nope not yet, but everytime I tried jobifying some code, it was already way better inside the editor, but in my case it was worse.
     
  11. eizenhorn

    eizenhorn

    Joined:
    Oct 17, 2016
    Posts:
    2,683
    When you Run job it's not spend time on schedulig and just runs right here right now like some method (in simple words)
     
  12. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    Oh, okay. It works but seems that nothing has improved with calling it to Run instead of Schedule.
     
  13. julian-moschuering

    julian-moschuering

    Joined:
    Apr 15, 2014
    Posts:
    529
    Run doesn't usw any additional threads. If you make it a simple IJob (single threaded) and use NativeList as you initially intended to, you basically do the same as you do in your no-job example only that the BurstCompiler is used. When the actual work done isn't much this might be a good solution.
     
    Last edited: Oct 9, 2019
  14. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    Thank you guys, it has improved a lot, it feels like jobs now :p
    Last thing, I do get the warning "Internal: JobTempAlloc has allocations that are more than 4 frames old - this is not allowed and likely a leak" multiple times per frame. How do I correct that (besides unticking the "Show Warnings" button ahah) ?

    Here is the improved version of the code (I removed some things to make it shorter but should be understandable) :

    Code (CSharp):
    1. [CreateAssetMenu(menuName = "Variables/Systems/AreaSplitter", order = 0)]
    2. public class AreaSplitter : ScriptableObject
    3. {
    4.     NativeArray<float3> areaPoints00;
    5.     NativeArray<float3> areaPoints11;
    6.     NativeArray<float3> areaCenterPoints;
    7.  
    8.  
    9.     // Used to fill in values to the 3 above arrays
    10.     public void CreateAreas ()
    11.     {
    12.         areaPoints00 = new NativeArray<float3>(TotalAreaNumber, Allocator.Persistent);
    13.         areaPoints11 = new NativeArray<float3>(TotalAreaNumber, Allocator.Persistent);
    14.         areaCenterPoints = new NativeArray<float3>(TotalAreaNumber, Allocator.Persistent);
    15.  
    16.         Vector3 point00 = Vector3.zero, point11 = Vector3.zero;
    17.  
    18.         for (int i = 0; i < areaNumber; i++)
    19.         {
    20.             point00.z = areaSize.z * i - floorSize.z / 2;
    21.             point11.z = areaSize.z * i + areaSize.z - floorSize.z / 2;
    22.             for (int j = 0; j < areaNumber; j++)
    23.             {
    24.                 point00.x = areaSize.x * j - floorSize.x / 2;
    25.                 point11.x = areaSize.x * j + areaSize.x - floorSize.x / 2;
    26.                 allAreas[i * areaNumber + j] = new Area(i * areaNumber + j, point00, point11);
    27.                 areaPoints00[i * areaNumber + j] = point00;
    28.                 areaPoints11[i * areaNumber + j] = point11;
    29.                 areaCenterPoints[i * areaNumber + j] = (point00 + point11) / 2;
    30.             }
    31.         }
    32.  
    33.         InstantiateNumbers();
    34.     }
    35.  
    36.     public void Free ()
    37.     {
    38.         areaPoints00.Dispose();
    39.         areaPoints11.Dispose();
    40.         areaCenterPoints.Dispose();
    41.     }
    42.  
    43.     public List<Area> GetCameraAreasJob (Vector3 _camPoint00, Vector3 _camPoint01, Vector3 _camPoint11)
    44.     {
    45.         if (!init)
    46.             return null;
    47.  
    48.         cameraAreas.Clear();
    49.         _camPoint00.x = _camPoint01.x;
    50.  
    51.         float3 camPoint00 = _camPoint00;
    52.         float3 camPoint11 = _camPoint11;
    53.         float3 areaOffset = this.areaOffset;
    54.         NativeQueue<int> queue = new NativeQueue<int>(Allocator.TempJob);
    55.         NativeQueue<int>.ParallelWriter queueWriter = queue.AsParallelWriter();
    56.  
    57.         GetCameraAreas getCameraAreasJob = new GetCameraAreas
    58.         {
    59.             camPoint00 = camPoint00,
    60.             camPoint11 = camPoint11,
    61.             areaOffset = areaOffset,
    62.             areaPoints00 = areaPoints00,
    63.             areaPoints11 = areaPoints11,
    64.             areaCenterPoints = areaCenterPoints,
    65.  
    66.             xTrue = false,
    67.             finalAreas = queueWriter,
    68.         };
    69.         getCameraAreasJob.Run(allAreas.Length);
    70.  
    71.         while (queue.Count > 0)
    72.             cameraAreas.Add(allAreas[queue.Dequeue()]);
    73.  
    74.         queue.Dispose();
    75.         UpdateTextNumberColors();
    76.  
    77.         return cameraAreas;
    78.     }
    79. }
    80.  
    81.  
    82. [BurstCompile]
    83. public struct GetCameraAreas : IJobParallelFor
    84. {
    85.     public float3 camPoint00, camPoint11;
    86.     public float3 areaOffset;
    87.     public NativeArray<float3> areaPoints00, areaPoints11, areaCenterPoints;
    88.     public bool xTrue;
    89.     public NativeQueue<int>.ParallelWriter finalAreas;
    90.  
    91.  
    92.  
    93.     public void Execute (int _index)
    94.     {
    95.         xTrue = false;
    96.         if ((areaPoints00[_index].x > camPoint00.x && areaPoints00[_index].x < camPoint11.x) || (areaPoints11[_index].x > camPoint00.x && areaPoints11[_index].x < camPoint11.x) || (camPoint00.x > areaCenterPoints[_index].x - areaOffset.x && camPoint00.x < areaCenterPoints[_index].x + areaOffset.x) || (camPoint11.x > areaCenterPoints[_index].x - areaOffset.x && camPoint11.x < areaCenterPoints[_index].x + areaOffset.x))
    97.             xTrue = true;
    98.  
    99.         if ((areaPoints00[_index].z > camPoint00.z && areaPoints00[_index].z < camPoint11.z) || (areaPoints11[_index].z > camPoint00.z && areaPoints11[_index].z < camPoint11.z) || (camPoint00.z > areaCenterPoints[_index].z - areaOffset.z && camPoint00.z < areaCenterPoints[_index].z + areaOffset.z) || (camPoint11.z > areaCenterPoints[_index].z - areaOffset.z && camPoint11.z < areaCenterPoints[_index].z + areaOffset.z))
    100.             if (xTrue)
    101.                 finalAreas.Enqueue(_index);
    102.     }
    103. }
     
  15. Neriad

    Neriad

    Joined:
    Feb 12, 2016
    Posts:
    125
    Well restarting Unity actually solved the issue.