Search Unity

  1. Unity 2020.1 has been released.
    Dismiss Notice
  2. Good news ✨ We have more Unite Now videos available for you to watch on-demand! Come check them out and ask our experts any questions!
    Dismiss Notice

Creating an Enum for NavMesh Areas

Discussion in 'Navigation' started by tarahugger, Dec 15, 2018.

  1. tarahugger

    tarahugger

    Joined:
    Jul 18, 2014
    Posts:
    129
    I've thrown together something that i wanted to share, a way to just have an enum for areas and have it display properly in the inspectors as a flags drop-down without any hassle. Enjoy.

    Source repo:
    https://github.com/imxzjv/UnityNavMeshAreas

    Code (CSharp):
    1. using System.Collections;
    2. using System.Collections.Generic;
    3. using UnityEngine;
    4. using UnityEngine.AI;
    5. using System;
    6. using System.Linq;
    7. using System.IO;
    8. using System.Text;
    9. using System.Runtime.CompilerServices;
    10.  
    11. #if UNITY_EDITOR
    12. using UnityEditor;
    13. #endif
    14.  
    15. namespace UnityEngine.AI
    16. {
    17.     #region Auto-Generated Content
    18.  
    19.         [Flags]
    20.         public enum NavMeshAreas
    21.         {
    22.             None = 0,              
    23.             Walkable = 1, NotWalkable = 2, Jump = 4, Climb = 8, Blocked = 16, Hole = 32, Edge = 64, Fall = 128, New1 = 256, Stuff = 512,
    24.             All = ~0,
    25.         }
    26.        
    27.     #endregion
    28.  
    29. #if UNITY_EDITOR
    30.  
    31.     /// <summary>
    32.     /// Auto-updates the <see cref="NavMeshAreas"/> enum in this file if it has changed, when scripts are compiled or assets saved.
    33.     /// </summary>
    34.     public static class NavMeshAreasGenerator
    35.     {
    36.         private const string EnumValuesToken = "#EnumValues";
    37.         private const string HashSettingsKey = "NavMeshAreasHash";
    38.  
    39.         private static void Update([CallerFilePath] string executingFilePath = "")
    40.         {
    41.             var areaNames = GameObjectUtility.GetNavMeshAreaNames();
    42.             var lastHash = EditorPrefs.GetInt(HashSettingsKey);
    43.             var newHash = GetAreaHash(areaNames);
    44.  
    45.             if (newHash != lastHash)
    46.             {
    47.                 Debug.Log($"{nameof(NavMeshAreas)} have changed, updating enum: '{executingFilePath}'");
    48.                 GenerateFile(areaNames, newHash, executingFilePath);
    49.             }
    50.         }
    51.  
    52.         private static int GetAreaHash(string[] areaNames)
    53.         {
    54.             var input = areaNames.Aggregate((a, b) => a + b);
    55.             var hash = 0;
    56.             foreach (var t in input)
    57.                 hash = (hash << 5) + hash + t;
    58.             return hash;
    59.         }
    60.  
    61.         private static void GenerateFile(string[] areaNames = default, int hash = 0, string outputPath = null)
    62.         {
    63.             if (areaNames == null)
    64.                 areaNames = GameObjectUtility.GetNavMeshAreaNames();
    65.  
    66.             if (hash == 0)
    67.                 hash = GetAreaHash(areaNames);
    68.  
    69.             var values = GetAreaEnumValuesAsText(ref areaNames);
    70.             var newEnumText = ContentTemplate.Replace(EnumValuesToken, values);
    71.             var output = ReplaceEnumInFile(nameof(NavMeshAreas), File.ReadAllLines(outputPath), newEnumText);
    72.  
    73.             CreateScriptAssetWithContent(outputPath, string.Concat(output));
    74.             EditorPrefs.SetInt(HashSettingsKey, hash);
    75.             AssetDatabase.Refresh();
    76.         }
    77.  
    78.         private static string GetAreaEnumValuesAsText(ref string[] areaNames)
    79.         {
    80.             var increment = 0;
    81.             var output = new StringBuilder();
    82.             var seenKeys = new HashSet<string>();
    83.  
    84.             foreach (var name in areaNames)
    85.             {
    86.                 var enumKey = string.Concat(name.Where(char.IsLetterOrDigit));
    87.                 var value = 1 << NavMesh.GetAreaFromName(name);
    88.  
    89.                 output.Append(seenKeys.Contains(name)
    90.                     ? $"{(enumKey + increment++)} = {value}, "
    91.                     : $"{enumKey} = {value}, ");
    92.  
    93.                 seenKeys.Add(enumKey);
    94.             }
    95.             return output.ToString();
    96.         }
    97.  
    98.         private static readonly string ContentTemplate =
    99.         $@"        public enum {nameof(NavMeshAreas)}
    100.        {{
    101.            None = 0,              
    102.            {EnumValuesToken}
    103.            All = ~0,
    104.        }}
    105.        ";
    106.  
    107.         private static string ReplaceEnumInFile(string enumName, string[] fileLines, string newEnum)
    108.         {
    109.             int enumStartLine = 0, enumEndLine = 0;
    110.             var result = new StringBuilder();
    111.             for (int i = 0; i < fileLines.Length; i++)
    112.             {
    113.                 string line = fileLines[i];
    114.                 if (line.Trim().StartsWith("public enum " + enumName))
    115.                 {
    116.                     enumStartLine = i;
    117.                     break;
    118.                 }
    119.                 result.AppendLine(line);
    120.             }
    121.             if (enumStartLine > 0)
    122.             {
    123.                 for (int i = enumStartLine + 1; i < fileLines.Length; i++)
    124.                 {
    125.                     if (fileLines[i].Contains("}"))
    126.                     {
    127.                         enumEndLine = i;
    128.                         break;
    129.                     }
    130.                 }
    131.                 result.Append(newEnum);
    132.                 for (int i = enumEndLine + 1; i < fileLines.Length; i++)
    133.                 {
    134.                     result.AppendLine(fileLines[i]);
    135.                 }
    136.             }
    137.             return result.ToString();
    138.         }
    139.  
    140.         /// <summary>
    141.         /// Create a new script asset.
    142.         /// UnityEditor.ProjectWindowUtil.CreateScriptAssetWithContent (2019.1)
    143.         /// </summary>
    144.         /// <param name="pathName">the path to where the new file should be created</param>
    145.         /// <param name="templateContent">the text to put inside</param>
    146.         /// <returns></returns>
    147.         private static UnityEngine.Object CreateScriptAssetWithContent(string pathName, string templateContent)
    148.         {
    149.             templateContent = SetLineEndings(templateContent, EditorSettings.lineEndingsForNewScripts);
    150.             string fullPath = Path.GetFullPath(pathName);
    151.             System.Text.UTF8Encoding encoding = new System.Text.UTF8Encoding(true);
    152.             File.WriteAllText(fullPath, templateContent, encoding);
    153.             AssetDatabase.ImportAsset(pathName);
    154.             return AssetDatabase.LoadAssetAtPath(pathName, typeof(UnityEngine.Object));
    155.         }
    156.  
    157.         /// <summary>
    158.         /// Ensure correct OS specific line endings for saving file content.
    159.         /// UnityEditor.ProjectWindowUtil.SetLineEndings (2019.1)
    160.         /// </summary>
    161.         /// <param name="content">a string to have line endings checked</param>
    162.         /// <param name="lineEndingsMode">the type of line endings to use</param>
    163.         /// <returns>a cleaned string</returns>
    164.         private static string SetLineEndings(string content, LineEndingsMode lineEndingsMode)
    165.         {
    166.             string replacement;
    167.             switch (lineEndingsMode)
    168.             {
    169.                 case LineEndingsMode.OSNative:
    170.                     replacement = Application.platform == RuntimePlatform.WindowsEditor ? "\r\n" : "\n";
    171.                     break;
    172.                 case LineEndingsMode.Unix:
    173.                     replacement = "\n";
    174.                     break;
    175.                 case LineEndingsMode.Windows:
    176.                     replacement = "\r\n";
    177.                     break;
    178.                 default:
    179.                     replacement = "\n";
    180.                     break;
    181.             }
    182.             content = System.Text.RegularExpressions.Regex.Replace(content, "\\r\\n?|\\n", replacement);
    183.             return content;
    184.         }
    185.  
    186.  
    187.         /// <summary>
    188.         /// Hook that runs the enum generator whenever assets are saved.
    189.         /// </summary>
    190.         private class UpdateOnAssetModification : UnityEditor.AssetModificationProcessor
    191.         {
    192.             public static string[] OnWillSaveAssets(string[] paths)
    193.             {
    194.                 Update();
    195.                 return paths;
    196.             }
    197.         }
    198.  
    199.         /// <summary>
    200.         /// Hook that runs the enum generator whenever scripts are compiled.
    201.         /// </summary>
    202.         [UnityEditor.Callbacks.DidReloadScripts]
    203.         private static void UpdateOnScriptCompile()
    204.         {
    205.             Update();
    206.         }
    207.  
    208.         /// <summary>
    209.         /// Enables manually running the enum generator from the menus.
    210.         /// </summary>
    211.         [MenuItem("Tools/Update NavMeshAreas")]
    212.         private static void UpdateOnMenuCommand()
    213.         {
    214.             UpdateOnScriptCompile();
    215.         }
    216.  
    217.     }
    218.  
    219.     /// <summary>
    220.     /// Flags enum dropdown GUI for selecting <see cref="NavMeshAreas"/> properties in the inspector
    221.     /// </summary>
    222.     [CustomPropertyDrawer(typeof(NavMeshAreas))]
    223.     public class NavMeshAreasDrawer : PropertyDrawer
    224.     {
    225.         public override void OnGUI(Rect position, SerializedProperty property, GUIContent label)
    226.         {
    227.             label = EditorGUI.BeginProperty(position, label, property);
    228.             var oldValue = (Enum)fieldInfo.GetValue(property.serializedObject.targetObject);
    229.             var newValue = EditorGUI.EnumFlagsField(position, label, oldValue);
    230.             if (!newValue.Equals(oldValue))
    231.             {
    232.                 property.intValue = (int)Convert.ChangeType(newValue, fieldInfo.FieldType);
    233.             }
    234.             EditorGUI.EndProperty();
    235.         }
    236.     }
    237.  
    238. #endif
    239.  
    240.     /// <summary>
    241.     /// A helper for flag operations with NavMeshAreas
    242.     /// </summary>
    243.     public struct AreaMask
    244.     {
    245.         private readonly int _value;
    246.  
    247.         public int Value => _value;
    248.         public NavMeshAreas Enum => (NavMeshAreas)_value;
    249.  
    250.         public AreaMask(int value)
    251.         {
    252.             _value = value;
    253.         }
    254.  
    255.         public AreaMask(NavMeshAreas areas)
    256.         {
    257.             _value = (int)areas;
    258.         }
    259.  
    260.         public static implicit operator AreaMask(int value) => new AreaMask(value);
    261.         public static implicit operator AreaMask(string name) => new AreaMask(1 << NavMesh.GetAreaFromName(name));
    262.         public static implicit operator AreaMask(NavMeshAreas areas) => new AreaMask((int)areas);
    263.         public static implicit operator NavMeshAreas(AreaMask flag) => (NavMeshAreas)flag._value;
    264.         public static implicit operator int(AreaMask flag) => flag._value;
    265.  
    266.         public static bool operator ==(AreaMask a, int b) => a._value.Equals(b);
    267.         public static bool operator !=(AreaMask a, int b) => !a._value.Equals(b);
    268.         public static int operator +(AreaMask a, AreaMask b) => a.Add(b._value);
    269.         public static int operator -(AreaMask a, AreaMask b) => a.Remove(b._value);
    270.         public static int operator |(AreaMask a, AreaMask b) => a.Add(b._value);
    271.         public static int operator ~(AreaMask a) => ~a._value;
    272.         public static int operator +(int a, AreaMask b) => a |= b._value;
    273.         public static int operator -(int a, AreaMask b) => a &= ~b._value;
    274.         public static int operator |(int a, AreaMask b) => a |= b._value;
    275.         public static int operator +(AreaMask a, int b) => a.Add(b);
    276.         public static int operator -(AreaMask a, int b) => a.Remove(b);
    277.         public static int operator |(AreaMask a, int b) => a.Add(b);
    278.  
    279.         public bool HasFlag(AreaMask flag) => (_value & flag._value) == flag;
    280.         public bool HasFlag(int value) => (_value & value) == value;
    281.         public AreaMask Add(AreaMask flag) => _value | flag._value;
    282.         public AreaMask Remove(AreaMask flag) => _value & ~flag._value;
    283.         public AreaMask Add(NavMeshAreas flags) => _value | (int)flags;
    284.         public AreaMask Remove(NavMeshAreas flags) => _value & ~(int)flags;
    285.  
    286.         public bool Equals(AreaMask other) => _value == other._value;
    287.         public override string ToString() => ((NavMeshAreas)_value).ToString();
    288.         public override int GetHashCode() => _value;
    289.         public override bool Equals(object obj)
    290.             => !ReferenceEquals(null, obj) && (obj is AreaMask other && Equals(other));
    291.     }
    292.  
    293. }
    294.  
    295.  
    296.  
     
    Seith and florianhanke like this.
  2. Paul-Swanson

    Paul-Swanson

    Joined:
    Jan 22, 2014
    Posts:
    289
    Iv got ask...and I'm not being sarcastic here. Iv seen bitshifting in a few assets I own...
    I struggle to see the point of doing that entirely. What benefit does it actually have over just setting it in the editor or a simple layer = blah
     
  3. tarahugger

    tarahugger

    Joined:
    Jul 18, 2014
    Posts:
    129
    Flags are useful if you want/need to store lots of information in as little memory space as possible. And its also useful for having a nice friendly flags enum to pick options from and make inspector drop-downs from versus the alternatives. Since Unity decided to use it for layers/areas etc we're pretty much stuck with following suit if we want to use the functionality.

    And despite the mockery from Ne0mega :( a lot of people learning unity have little to no experience with how it works (understandably) and questions and difficulties constantly surface related to it.
     
    Paul-Swanson likes this.
  4. Paul-Swanson

    Paul-Swanson

    Joined:
    Jan 22, 2014
    Posts:
    289
    Until i saw it recently like this: Terrain = 1 << 13 i had no idea what this even was. Thx for clearing that up you two.
    I had no idea you could do pickable enums with this.
     
  5. Ne0mega

    Ne0mega

    Joined:
    Feb 18, 2018
    Posts:
    331
    i wasnt mocking you. i really am impressed. retune your sarcasm detector. i also really am a noob, and have yet to actually implement my own system that uses bitshifting. ive just reasearched and toyed with, and thought about it a lot.
     
  6. tarahugger

    tarahugger

    Joined:
    Jul 18, 2014
    Posts:
    129
    My apologies, i completely read that wrong then :(
     
  7. unity-marcus

    unity-marcus

    Joined:
    Apr 16, 2019
    Posts:
    4
    Cause I just bumped into this nice example, I was wondering how the Unity UI does it, decompiled it and adjusted it for an easy use:

    Code (CSharp):
    1. [CustomPropertyDrawer(typeof(NavMeshMaskAttribute))]
    2.     public class NavMeshMaskDrawer : UnityEditor.PropertyDrawer
    3.     {
    4.         // Draw the property inside the given rect
    5.         public override void OnGUI(Rect position, SerializedProperty serializedProperty, GUIContent label)
    6.         {
    7.             EditorGUI.BeginChangeCheck();
    8.             string[] navMeshAreaNames = GameObjectUtility.GetNavMeshAreaNames();
    9.             int navMeshArea = serializedProperty.intValue;
    10.             int selectedIndex = -1;
    11.             for (int i = 0; i < navMeshAreaNames.Length; i++)
    12.             {
    13.                 if (GameObjectUtility.GetNavMeshAreaFromName(navMeshAreaNames[i]) == navMeshArea)
    14.                 {
    15.                     selectedIndex = i;
    16.                     break;
    17.                 }
    18.             }
    19.             int num = EditorGUI.Popup(position, "Navigation Area", selectedIndex, navMeshAreaNames);
    20.             if (EditorGUI.EndChangeCheck())
    21.             {
    22.                 int navMeshAreaFromName = GameObjectUtility.GetNavMeshAreaFromName(navMeshAreaNames[num]);
    23.                 serializedProperty.intValue = navMeshAreaFromName;
    24.             }
    25.         }
    26.     }
    27.  
    28.     public class NavMeshMaskAttribute : UnityEngine.PropertyAttribute { }
    29.  
    How to use it:
    Code (CSharp):
    1. [NavMeshMask]
    2. public int NavMeshMask;
     
  8. Rahd

    Rahd

    Joined:
    May 30, 2014
    Posts:
    315
    Thank you Like A lot !!
    I Was enraged about the way unity process the mask layers !!
    Trying to make Some of my AI enemies jump, and some Can Climb, some can swim, and others can walk on fire or ice ... going to each and every agent and set it by hand is dumb, and monsters have Stamina, so when it's too low, No jump for you Monster,
    Now I just exclude the Jump area, without doing a lot of work, thank you !!
     
unityunity