Search Unity

WebGL with streaming option - like WebPlayer

Discussion in 'WebGL' started by De-Panther, Apr 5, 2015.

  1. De-Panther

    De-Panther

    Joined:
    Dec 27, 2009
    Posts:
    306
    Hey,

    While we worked on a WebGL port for our game, we needed an option to stream levels like in the WebPlayer.
    @jonas echterhoff wrote a basic script that did that on the beta version of Unity5.
    We updated it, and added the option to check download progress.

    Instead of Application.CanStreamedLevelBeLoaded() and Application.GetStreamProgressForLevel()
    We now have
    WebGLLevelStreaming.CanStreamedLevelBeLoaded()
    WebGLLevelStreaming.GetStreamProgressForLevel()

    Use "firstStreamedLevelWithResources" like in the "First Streamed Level" option of the WebPlayer
    http://docs.unity3d.com/Manual/class-PlayerSettingsWeb.html

    So, the code for WebGLLevelStreaming.cs
    Code (CSharp):
    1. using UnityEngine;
    2. #if UNITY_EDITOR
    3. using UnityEditor;
    4. using UnityEditor.Callbacks;
    5. #endif
    6. using System.Collections;
    7. using System.Collections.Generic;
    8. using System.IO;
    9. using System.Diagnostics;
    10. using System.Linq;
    11. using System.Runtime.InteropServices;
    12.  
    13.  
    14. public class WebGLLevelStreaming {
    15.  
    16. #if UNITY_EDITOR && UNITY_WEBGL
    17.     private const string kOutputDataExtension = ".data";
    18.     private const string kOutputFileLoaderFileName = "fileloader.js";
    19.     private const string kResourcesDirName = "Resources";
    20.     private const string kResourcesFileName = "unity_default_resources";
    21.     private const string kResourcesExtraFileName = "unity_builtin_extra";
    22.  
    23.     public static string buildToolsDir
    24.     {
    25.         get {
    26.             // Find WebGL build tools
    27.             var webGLBuildTools = Path.Combine("WebGLSupport", "BuildTools");
    28.             var playbackEngines = Path.Combine(EditorApplication.applicationContentsPath, "PlaybackEngines");
    29.             var path = Path.Combine(playbackEngines, webGLBuildTools);
    30.             if (Directory.Exists(path))
    31.                 return path;
    32.             else
    33.                 return Path.Combine(Path.Combine(EditorApplication.applicationPath, "../../"), webGLBuildTools);
    34.         }
    35.     }
    36.  
    37.     public static string emscriptenDir
    38.     {
    39.         get { return buildToolsDir + "/Emscripten"; }
    40.     }
    41.  
    42.     static string packager
    43.     {
    44.         get { return emscriptenDir + "/" + "tools/file_packager.py"; }
    45.     }
    46.  
    47.     public static string pythonExecutable
    48.     {
    49.         get
    50.         {
    51.             if (Application.platform == RuntimePlatform.WindowsEditor)
    52.                 return "\"" +  buildToolsDir + "/" + "Emscripten_Win/python/2.7.5.3_64bit/python.exe\"";
    53.             return "python";
    54.         }
    55.     }
    56.  
    57.     public static int firstStreamedLevelWithResources = 1;
    58.  
    59.     // Run emscripten file packager to pack a .data file
    60.     private static bool RunPackager (string filename, string stagingAreaData, IEnumerable<string> filesToShip)
    61.     {
    62.         var argumentsForPacker = "\"" + packager + "\"" ;
    63.         argumentsForPacker += string.Format(" \"{0}\"", filename + kOutputDataExtension);
    64.         argumentsForPacker += " --no-heap-copy";
    65.         argumentsForPacker += " --js-output=\"" + filename + ".loader.js" + "\"";
    66.  
    67.         if (PlayerSettings.GetPropertyBool("dataCaching", BuildTargetGroup.WebGL))
    68.             argumentsForPacker += " --use-preload-cache";
    69.  
    70.         argumentsForPacker += " --preload";
    71.         argumentsForPacker += filesToShip.Aggregate("", (current, file) => current + (" \"" + file + "\""));
    72.  
    73.         var processStartInfo = new ProcessStartInfo(pythonExecutable)
    74.         {
    75.             Arguments = argumentsForPacker,
    76.             WorkingDirectory = stagingAreaData,
    77.             UseShellExecute = false,
    78.         };
    79.  
    80.         var p = Process.Start(processStartInfo);
    81.         p.WaitForExit();
    82.         if (p.ExitCode == 0)
    83.             return true;
    84.  
    85.         throw new System.Exception("Failed running " + processStartInfo.FileName + " " + processStartInfo.Arguments);
    86.     }
    87.    
    88.     private static bool PackageData (string filename, string stagingAreaData, IEnumerable<string> _filesToShip)
    89.     {
    90.         var filesToShip = new HashSet<string>(_filesToShip.Select(o => Path.GetFileName(o)));
    91.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
    92.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
    93.  
    94.         if (firstStreamedLevelWithResources < 0)
    95.             firstStreamedLevelWithResources = 0;
    96.         if (firstStreamedLevelWithResources >= Application.levelCount)
    97.             firstStreamedLevelWithResources = Application.levelCount-1;
    98.         var loaderJSStart = "var StreamProgressForLevelArray = [];\nStreamProgressForLevelArray[0]=1;\n";
    99.         var loaderJS = "";
    100.         var loaderJS0 = "";
    101.  
    102.         int packageIndex = 0;
    103.         // Generate a data file for each streaming level
    104.         while (true)
    105.         {
    106.             // find all the files for this level
    107.             var fileNames = new List<string>();
    108.             int levelIndex = packageIndex;
    109.             fileNames.Add("sharedassets" + levelIndex + ".resource");
    110.             fileNames.Add("sharedassets" + levelIndex + ".assets");
    111.             if(packageIndex>0)
    112.             {
    113.                 fileNames.Add("level" + (levelIndex-1) + "");
    114.             }else{
    115.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
    116.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
    117.                 fileNames.Add("mainData");
    118.             }
    119.  
    120.             if(levelIndex==firstStreamedLevelWithResources)
    121.             {
    122.                 fileNames.Add("resources.assets");
    123.                 fileNames.Add("resources.resource");
    124.             }
    125.  
    126.             var files = new List<string>();
    127.             foreach (var f in fileNames)
    128.             {
    129.                 if (filesToShip.Contains(f))
    130.                 {
    131.                     filesToShip.Remove(f);
    132.                     files.Add(f);
    133.                 }
    134.             }
    135.             if (files.Count == 0)
    136.                 break;
    137.  
    138.             var packageFile = filename + "." + packageIndex;
    139.             if (!RunPackager(packageFile, stagingAreaData, files))
    140.                 return false;
    141.  
    142.             // Extract loader script for this data file, and put it's content into a JavaScript function
    143.             if(packageIndex>0)
    144.             {
    145.                 var loaderText = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
    146.                 loaderText = loaderText.Substring(loaderText.IndexOf("\n(function() {"));
    147.                 loaderText = loaderText.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
    148.                                                 "StreamProgressForLevelArray["+packageIndex+"]=event.loaded/size;");
    149.                 loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText + "}\n";
    150.                 loaderJSStart +="StreamProgressForLevelArray[" + packageIndex + "]=0;\n";
    151.             }else{
    152.                 loaderJS0 = File.ReadAllText(Path.Combine(stagingAreaData, filename + ".0.loader.js"));
    153.                 loaderJS0 = loaderJS0.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
    154.                                               "if (Module['setStatus']) Module['setStatus']('Downloading data... (' + event.loaded + '/' + size + ')');");
    155.             }
    156.             packageIndex++;
    157.         }
    158.  
    159.         if(filesToShip.Count>0)
    160.         {
    161.             var packageFile = filename + "." + packageIndex;
    162.             if (!RunPackager(packageFile, stagingAreaData, filesToShip))
    163.                 return false;
    164.             var loaderText2 = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
    165.             loaderText2 = loaderText2.Substring(loaderText2.IndexOf("\n(function() {"));
    166.             loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText2 + "}\n";
    167.             packageIndex++;
    168.         }
    169.  
    170.         loaderJS = loaderJSStart + loaderJS + loaderJS0;
    171.  
    172.         for (int j=0; j < packageIndex-1; j++)
    173.         {
    174.             // Patch loader scripts to invoke the function to start loading the next data file when it's done
    175.             var line = "Module['removeRunDependency']('datafile_" + filename + "." + j + ".data');";
    176.             // Call function to load next data file in a callback to avoid calling it while clearing pre-run dependencies.
    177.             loaderJS = loaderJS.Replace (line, line + "\nStreamProgressForLevelArray[" + j + "]=1;\nwindow.setTimeout(DownloadDataForPackage" + (j+1) + @",1);");
    178.         }
    179.  
    180.         // write out loader script for all data files
    181.         File.WriteAllText(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), loaderJS);
    182.         return true;
    183.     }
    184.  
    185.     // Generate gzip compressed versions of files
    186.     private static void CompressFilesInOutputDirectory (string dir, string outputDir)
    187.     {
    188.         var filesToCompress = Directory.GetFiles(dir).Where(f => f.EndsWith(kOutputFileLoaderFileName) || f.EndsWith(".data"));
    189.         foreach (var file in filesToCompress)
    190.         {
    191.             var processName = "7za";
    192.             if (Application.platform == RuntimePlatform.WindowsEditor)
    193.                 processName = "7z.exe";
    194.             var processStartInfo = new ProcessStartInfo(EditorApplication.applicationContentsPath+"/Tools/"+processName)
    195.             {
    196.                 Arguments = "a -tgzip \""+Path.Combine(outputDir, Path.GetFileName(file))+"gz\" \""+file+"\"",
    197.                 UseShellExecute = false,
    198.                 CreateNoWindow = true
    199.             };
    200.  
    201.             var p = Process.Start(processStartInfo);
    202.             p.WaitForExit();
    203.             if (p.ExitCode != 0)
    204.                 throw new System.Exception("Failed running " + processStartInfo.FileName + " " + processStartInfo.Arguments);
    205.         }
    206.     }
    207.  
    208.     [PostProcessBuild]
    209.     public static void OnPostprocessBuild(BuildTarget target, string pathToBuiltProject) {
    210.         if (target != BuildTarget.WebGL)
    211.             return;
    212.         var stagingAreaData = Path.Combine(Path.Combine("Temp", "StagingArea"), "Data");
    213.         var filename = Path.GetFileName(pathToBuiltProject);
    214.  
    215.         // Find all files to package
    216.         IEnumerable<string> filesToShip = Directory.GetFiles(stagingAreaData, "*.resource");
    217.         filesToShip = filesToShip.Concat(Directory.GetFiles(stagingAreaData, "*.assets"));
    218.         filesToShip = filesToShip.Concat(Directory.GetFiles(stagingAreaData, "mainData"));
    219.         filesToShip = filesToShip.Concat(Directory.GetFiles(stagingAreaData, "level*"));
    220.  
    221.         // put files into packages
    222.         PackageData (filename, stagingAreaData, filesToShip);
    223.  
    224.         // delete old data file
    225.         File.Delete (Path.Combine(pathToBuiltProject, Path.Combine("Release", filename+".data")));
    226.  
    227.         // copy new data files into build
    228.         File.Copy(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), Path.Combine(Path.Combine(pathToBuiltProject, "Release"),kOutputFileLoaderFileName), true);
    229.         foreach (var f in Directory.GetFiles(stagingAreaData, "*.*.data"))
    230.             File.Copy(f, Path.Combine(Path.Combine(pathToBuiltProject, "Release"),Path.GetFileName(f)), true);
    231.  
    232.         // compress new data files
    233.         CompressFilesInOutputDirectory (Path.Combine(pathToBuiltProject, "Release"), Path.Combine(pathToBuiltProject, "Compressed"));
    234.  
    235.         // delete old compressed data file
    236.         File.Delete (Path.Combine(pathToBuiltProject, Path.Combine("Compressed", filename+".datagz")));
    237.     }
    238. #endif
    239.  
    240. #if UNITY_WEBGL
    241.     [DllImport("__Internal")]
    242.     private static extern float GetStreamProgressForLevelFromWeb(int levelIndex);
    243. #endif
    244.  
    245.     static public bool CanStreamedLevelBeLoaded (int levelIndex)
    246.     {
    247. #if UNITY_WEBGL && !UNITY_EDITOR
    248.         if (levelIndex == 0) // file = mainData
    249.             return true;
    250.         return File.Exists("level"+(levelIndex-1));
    251. #else
    252.         return Application.CanStreamedLevelBeLoaded(levelIndex);
    253. #endif  
    254.     }
    255.  
    256.     static public float GetStreamProgressForLevel (int levelIndex)
    257.     {
    258.         #if UNITY_WEBGL && !UNITY_EDITOR
    259.         if (levelIndex == 0) // file = mainData
    260.             return 1;
    261.         return GetStreamProgressForLevelFromWeb(levelIndex);
    262.         #else
    263.         return Application.GetStreamProgressForLevel(levelIndex);
    264.         #endif  
    265.     }
    266. }
    And code for the JS file, WebGLLevelStreamingJS.jslib
    Code (JavaScript):
    1. var WebGLLevelStreamingJS = {
    2.     GetStreamProgressForLevelFromWeb: function(levelIndex)
    3.     {
    4.         return StreamProgressForLevelArray[levelIndex];
    5.     }
    6. };
    7.  
    8. mergeInto(LibraryManager.library, WebGLLevelStreamingJS);
    **IMPORTANT**
    Tested on Unity5.0.0f4.
    Unity still working on the WebGL build process, so this script may not work on versions newer than 5.0.0f4.

    Known Issues:
    GetStreamProgressForLevel() return 0 if loading from cache.
    Some other issues with the progress that we need to check, but weren't critical for us.

    You all are welcome to improve the script :)
     
    Last edited: Apr 5, 2015
    mgear and tomekkie2 like this.
  2. sokki

    sokki

    Joined:
    Jan 31, 2015
    Posts:
    164
    srry for being off-topic on your problem, De-Panther, but could you mention what is your game about? Type, genre, online features, etc?
     
  3. slmoloch

    slmoloch

    Joined:
    Feb 11, 2015
    Posts:
    5
    Hey,

    The script is very useful, thanks!
    For those who is not aware where is is coming from here is the link to the presentation.

    De-Panther, the presentation also contains very interesting slide #25, that says how you dealt with textures and resources. Could you please share details?

    Is that possible to share build script?

     
  4. skeleton-king

    skeleton-king

    Joined:
    Nov 10, 2014
    Posts:
    63
    Can any1 tell me how to make this work??
    I Used the 1st script and it did split files.
    But what to do with 2nd script.

    When i tested not all objects in level where loaded.
     
  5. De-Panther

    De-Panther

    Joined:
    Dec 27, 2009
    Posts:
    306
    @slmoloch I can't share those, as they are specific for the client's project.
    But maybe @liortal can explain how it works - he worked on the build script.
    I just remember that it's something about searching on the scene files for specific textures and replace them with URL to download.
    And about removing resources and scenes - we move them to the "Editor" folder while the build proccess is running, and move them back when it finished.

    @skeleton king , just put the "WebGLLevelStreamingJS.jslib" file under Assets/Plugins/WebGL/
     
  6. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    We have actually created a build infrastructure (see the menu item in the screenshot) that builds the game.

    This takes into account a customized build properties file that has a few pieces of data:
    1. Excluded scenes - this is a collection of scenes (dragged from the project window) that will not be included in the final output. The build process calls BuildPipeline.BuildPlayer; this method accepts an array of scenes, so we can easily pass in the scenes we want to build.
    2. Excluded resources - this is a collection of resources (files / folders under a Resources folder) that also will be excluded from the build. These can be stuff that is unused in the game, so there's no point of including them in the final game build. Since there's no way to "exclude" resources, we do a little trick - we just move these into a folder that is not under "Resources" and then move them back when the build is done.
    3. Remove texture usage - for some textures that were too large, we actually incorporated a small trick - we don't include them in the game, but at runtime we download them from a server. This is done automatically by the build process - we have a data file that describes which textures we'd like to "patch" in this way. The build process looks for all of these textures, and removes them (replacing them with a special component that knows how to fetch them at runtime).
    At this point, the build process was created for this project in our company (Moon Active), i don't think we can share it.

    I have taken the concepts from the design we used for this project and i plan to create a new project (open source?) of build tools for Unity. Watch this space in the future: https://github.com/liortal53/UnityBuildTools
     
    Last edited: Jun 6, 2015
  7. slmoloch

    slmoloch

    Joined:
    Feb 11, 2015
    Posts:
    5
    Nice, thanks for the prompt reply! I totally understand that you can not share the script, but could you point me to the direction on where to find the sample of component that downloads the texture?
     
  8. slmoloch

    slmoloch

    Joined:
    Feb 11, 2015
    Posts:
    5
    Looking forward for updates to that repository. When do you expect to add something there?
     
  9. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    The component that downloads textures is a super basic one.

    It has a public URL field that is set from the editor, and in its Awake method it downloads the texture at the given URL, creates a Sprite object and assigns it to the SpriteRenderer.
     
    Last edited: Jun 10, 2015
  10. jonmalave_

    jonmalave_

    Joined:
    Aug 19, 2012
    Posts:
    1,670
    Very Cool! I need this for my project, but I'm currently using Unity 5.0.2. Any idea if this will work there?

    Also has the "GetStreamProgressForLevel() return 0 if loading from cache." error been resolved? If not does it actually cause a problem in the WebGL build for the player to move forward with loading the game or level even if it was cached?

    I think this should be an officially added feature as with all the Memory Allocation issues WebGL has currently, something like this is critical for us to build and deploy WebGL games in the current state.
     
  11. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    Should be fine, although we haven't tested it with that particular version yet.
     
    De-Panther likes this.
  12. De-Panther

    De-Panther

    Joined:
    Dec 27, 2009
    Posts:
    306
    Thanks @Kuroato
    GetStreamProgressForLevel() should return 1 now if loading from cache, but I suggest you should also check it yourself.

    Please reply if it work or didn't work
     
  13. jonas-echterhoff

    jonas-echterhoff

    Unity Technologies

    Joined:
    Aug 18, 2005
    Posts:
    1,633
    De-Panther and jonmalave_ like this.
  14. jonmalave_

    jonmalave_

    Joined:
    Aug 19, 2012
    Posts:
    1,670
    Cool Thanks! Good to know!
     
  15. Gaurav-Gulati

    Gaurav-Gulati

    Joined:
    Sep 22, 2014
    Posts:
    52
    Hi De-Panther,
    When i compiled your WebGLLevelStreaming.cs script from unity 5.1,i get huge python errors.

    Failed running python "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emcc" @"/Users/apple/Downloads/GameFolder/Game/Assets/../Temp/emcc_arguments.resp"

    stdout:
    stderr:
    warning: unresolved symbol: glFlushMappedBufferRange
    warning: unresolved symbol: glGetInternalformativ
    warning: unresolved symbol: tcflush
    warning: unresolved symbol: pthread_create
    warning: unresolved symbol: _ZN4FMOD13DSPConnection6setMixEf
    warning: unresolved symbol: glUnmapBuffer
    warning: unresolved symbol: glGetStringi
    warning: unresolved symbol: glProgramBinary
    warning: unresolved symbol: glMapBufferRange
    warning: unresolved symbol: glGetProgramBinary
    warning: unresolved symbol: glCopyBufferSubData
    Traceback (most recent call last):
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1675, in <module>
    _main(environ=os.environ)
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1663, in _main
    temp_files.run_and_clean(lambda: main(
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/tools/tempfiles.py", line 39, in run_and_clean
    return func()
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1671, in <lambda>
    DEBUG_CACHE=DEBUG_CACHE,
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 1558, in main
    jcache=jcache, temp_files=temp_files, DEBUG=DEBUG, DEBUG_CACHE=DEBUG_CACHE)
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emscripten.py", line 924, in emscript_fast
    %s''' % (staticbump, global_initializers, mem_init)) # XXX wrong size calculation!
    UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 308695: ordinal not in range(128)
    Traceback (most recent call last):
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/emcc", line 1323, in <module>
    final = shared.Building.emscripten(final, append_ext=False, extra_args=extra_args)
    File "/Applications/Unity5.1/Unity.app/Contents/PlaybackEngines/WebGLSupport/BuildTools/Emscripten/tools/shared.py", line 1535, in emscripten
    assert os.path.exists(filename + '.o.js'), 'Emscripten failed to generate .js'
    AssertionError: Emscripten failed to generate .js

    UnityEditor.HostView:OnGUI()


    Can you help me on this?
     
  16. De-Panther

    De-Panther

    Joined:
    Dec 27, 2009
    Posts:
    306
  17. Madsen

    Madsen

    Joined:
    Nov 29, 2010
    Posts:
    15
    The asset store script is not working with Unity 5.1.2 - it's working with 5.1.1 - but with the new version I get the error:
    Uncaught RangeError: Maximum call stack size exceded

    @jonas echterhoff It would be cool if you could check that for me.
     
  18. dagon

    dagon

    Joined:
    Jan 4, 2013
    Posts:
    19
    Very helpfull.Thanks!
     
  19. slmoloch

    slmoloch

    Joined:
    Feb 11, 2015
    Posts:
    5
    Starting from 5.1.2 there is a another file that you need to add to the first scene - global-metadata.dat. Just add two lines of code:

    filesToShip.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));

    and

    fileNames.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));
     
    De-Panther likes this.
  20. AnonDreamer

    AnonDreamer

    Joined:
    Oct 28, 2014
    Posts:
    30
    Hi, i'm having a bit of an issue, i copied paste the de-panther script and added the 2 lines above, set the "firstStreamedLevelWithResources " to 0 which is my preloader scene, but when i launch the game, only the "webGL.data" file is being downloaded.

    The "webGL.0.data", "webGL.1.data" etc... files aren't downloaded so i guess i must have missed something :s. Is there anything else to add, or maybe call a function somewhere ?

    I tried to use the version of @jonas echterhoff and added the 2 lines above, but i get the following errors:
    - run() called, but dependencies remain, so not running
    - Not implemented: Class::GetClassBitmap (mutilple times)
    - Not implemented: Il2CppTypeCompare::compare

    I noticed that using his version correctly download the "webGL.0.data", "webGL.1.data" files, but generate errors when loading is complete :s.

    Any idea how to fix this ? thx for reading :)
     
  21. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    Hey @AnonDreamer

    1. What Unity version are you on ?
    2. What is the output you get after building your project with either scripts? does the script split up the data files properly? I don't think you should have the webgl.data file at all. It should be split into multiple files eventually after running the script.
    3. are you getting the errors you posted (eg: GetClassbitmap) when not using the streaming script ?

    -Lior
     
  22. Rick Lee

    Rick Lee

    Joined:
    Sep 20, 2013
    Posts:
    1
    Hi, I encountered similar problem as AnonDreamer in which the script works fine on 5.1.1p1, but got the following error when I built it on 5.1.2.

    From browser java console:

    Not implemented: Class::GetClassBitmap (1xx times)

    Cannot enlarge memory arrays. Either (1) compile with -s TOTAL_MEMORY=X with X higher than the current value 536870912, (2) compile with ALLOW_MEMORY_GROWTH which adjusts the size at runtime but prevents some optimizations, or (3) set Module.TOTAL_MEMORY before the program runs.

    .
    .
    .



    Thanks & Regards,
    Rick
     
  23. AnonDreamer

    AnonDreamer

    Joined:
    Oct 28, 2014
    Posts:
    30
    @liortal
    1 - I'm currently using Unity 5.1.2f1, building fast development webGL builds.
    2 - When using the @jonas echterhoff version, i get webGL.0.data, webGL.1.data, webGL.2.data files, the files gets downloaded but in the javascript console, i get multiple "Not implemented: Class::GetClassBitmap" errors and "Not implemented: Il2CppTypeCompare::compare" errors, i have no error in the unity console after the build is done.

    When using the de-panther version, i have one error after the build is finished : DirectoryNotFoundException: Could not find a part of the path "C:/Users/Admin/Desktop/builds_test/webGL\Release\webGL.data", but the streaming seems to works fine when i make a release build instead of a development one. (i only have one "webGL.data" file if i make a development build)
    Also what does the WebGLLevelStreamingJS.jslib script do ?


    3 - If i don't use either streaming scripts, i don't get any error on the output and the builds works just fine.

    So it seems the de-panther script works for me but only if i don't use the development build.

    @Rick Lee the "cannot enlarge memory arrays" can be solved if you either increase the "TOTAL_MEMORY" value in the index.html file or you do it by going to "player settings" in the build window, then Publishing Settings and you modify the WebGL Memory Size value.
    If your browser displays an error like "cannot assign the required memory", try to make a fast build instead of a slow one. I kept getting this error in chrome (even with 256mo) unless i made "fast" builds.
    For the "Not implemented: Class::GetClassBitmap ", i don't have a clue how to solve it :\
     
    Last edited: Aug 18, 2015
  24. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    What line of code exactly throws the error your mentioned ? (e.g: directory not found?)

    Also, i believe the script is hard-wired to use Release builds only (not sure about the original script by @jonas echterhoff)

    You can see that the script is inserting the string "Release" in multiple locations. I *THINK* it shouldn't be too difficult to make some changes so it'll support development builds (unless there's something internal that prevents dev builds from using streaming).

    Lastly, the WebGLLevelStreaming.jslib file is a file containing javascript code that gets included into the final JS code of your game.

    You can see that the WebGLStreaming class calls into it in the GetStreamProgressForLevel method, to check up on the progress for loading a level.
     
  25. Madsen

    Madsen

    Joined:
    Nov 29, 2010
    Posts:
    15
    The Version 5.1.1 was working fine. But with 5.1.2 still not working. We use the original Script from the Asset Store.
    We included the 2 lines here:

    Code (CSharp):
    1.  
    2.      filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
    3.      filesToShip.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));
    4.  
    5.      fileNames.Add("level" + (levelIndex-1) + "");
    6.      fileNames.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));
    7.  

    But still get errors in Browser console and the build doesn't load. Error in Chrome is:
    Not implemented: Class::FromIl2CppType

    Any idea?
     
    Last edited: Aug 19, 2015
  26. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    Is there any stack you are getting with this error? also is that a dev or release build? (i assume release since the script doesn't handle dev builds.. ?)
     
  27. AnonDreamer

    AnonDreamer

    Joined:
    Oct 28, 2014
    Posts:
    30
    @Madsen i didn't make the same changes as you did, here's the de-panther file with the 2 lines of code added:

    Code (CSharp):
    1.  
    2. private static bool PackageData (string filename, string stagingAreaData, IEnumerable<string> _filesToShip)
    3.     {
    4.         var filesToShip = new HashSet<string>(_filesToShip.Select(o => Path.GetFileName(o)));
    5.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
    6.         filesToShip.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
    7.         filesToShip.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat")); //Modification for 5.1.2 version
    8.  
    9.         if (firstStreamedLevelWithResources < 0)
    10.             firstStreamedLevelWithResources = 0;
    11.         if (firstStreamedLevelWithResources >= Application.levelCount)
    12.             firstStreamedLevelWithResources = Application.levelCount-1;
    13.         var loaderJSStart = "var StreamProgressForLevelArray = [];\nStreamProgressForLevelArray[0]=1;\n";
    14.         var loaderJS = "";
    15.         var loaderJS0 = "";
    16.  
    17.         int packageIndex = 0;
    18.         // Generate a data file for each streaming level
    19.         while (true)
    20.         {
    21.             // find all the files for this level
    22.             var fileNames = new List<string>();
    23.             int levelIndex = packageIndex;
    24.             fileNames.Add("sharedassets" + levelIndex + ".resource");
    25.             fileNames.Add("sharedassets" + levelIndex + ".assets");
    26.             fileNames.Add(Path.Combine(@"Il2CppData\Metadata", "global-metadata.dat"));//Modification for 5.1.2 version
    27.  
    28.             if(packageIndex>0)
    29.             {
    30.                 fileNames.Add("level" + (levelIndex-1) + "");
    31.             }else{
    32.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesFileName));
    33.                 fileNames.Add(Path.Combine(kResourcesDirName, kResourcesExtraFileName));
    34.                 fileNames.Add("mainData");
    35.             }
    36.  
    37.             if(levelIndex==firstStreamedLevelWithResources)
    38.             {
    39.                 fileNames.Add("resources.assets");
    40.                 fileNames.Add("resources.resource");
    41.             }
    42.  
    43.             var files = new List<string>();
    44.             foreach (var f in fileNames)
    45.             {
    46.                 if (filesToShip.Contains(f))
    47.                 {
    48.                     filesToShip.Remove(f);
    49.                     files.Add(f);
    50.                 }
    51.             }
    52.             if (files.Count == 0)
    53.                 break;
    54.  
    55.             var packageFile = filename + "." + packageIndex;
    56.             if (!RunPackager(packageFile, stagingAreaData, files))
    57.                 return false;
    58.  
    59.             // Extract loader script for this data file, and put it's content into a JavaScript function
    60.             if(packageIndex>0)
    61.             {
    62.                 var loaderText = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
    63.                 loaderText = loaderText.Substring(loaderText.IndexOf("\n(function() {"));
    64.                 loaderText = loaderText.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
    65.                                                 "StreamProgressForLevelArray["+packageIndex+"]=event.loaded/size;");
    66.                 loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText + "}\n";
    67.                 loaderJSStart +="StreamProgressForLevelArray[" + packageIndex + "]=0;\n";
    68.             }else{
    69.                 loaderJS0 = File.ReadAllText(Path.Combine(stagingAreaData, filename + ".0.loader.js"));
    70.                 loaderJS0 = loaderJS0.Replace("if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');",
    71.                                               "if (Module['setStatus']) Module['setStatus']('Downloading data... (' + event.loaded + '/' + size + ')');");
    72.             }
    73.             packageIndex++;
    74.         }
    75.  
    76.         if(filesToShip.Count>0)
    77.         {
    78.             var packageFile = filename + "." + packageIndex;
    79.             if (!RunPackager(packageFile, stagingAreaData, filesToShip))
    80.                 return false;
    81.             var loaderText2 = File.ReadAllText(Path.Combine(stagingAreaData, packageFile + ".loader.js"));
    82.             loaderText2 = loaderText2.Substring(loaderText2.IndexOf("\n(function() {"));
    83.             loaderJS += "function DownloadDataForPackage" + packageIndex + "(){\n" + loaderText2 + "}\n";
    84.             packageIndex++;
    85.         }
    86.  
    87.         loaderJS = loaderJSStart + loaderJS + loaderJS0;
    88.  
    89.         for (int j=0; j < packageIndex-1; j++)
    90.         {
    91.             // Patch loader scripts to invoke the function to start loading the next data file when it's done
    92.             var line = "Module['removeRunDependency']('datafile_" + filename + "." + j + ".data');";
    93.             // Call function to load next data file in a callback to avoid calling it while clearing pre-run dependencies.
    94.             loaderJS = loaderJS.Replace (line, line + "\nStreamProgressForLevelArray[" + j + "]=1;\nwindow.setTimeout(DownloadDataForPackage" + (j+1) + @",1);");
    95.         }
    96.  
    97.         // write out loader script for all data files
    98.         File.WriteAllText(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), loaderJS);
    99.         return true;
    100.     }
    With this i don't get the "Not implemented: Class::FromIl2CppType" error in chrome as i do with the jonas version.

    @liortal indeed i didn't notice at first it was coded for a release build, i just have to change the folder for each version, the directory error was because of that:
    Code (CSharp):
    1. File.Delete (Path.Combine(pathToBuiltProject, Path.Combine("Release", filename+".data")));
    And thanks for the explanations, i wasn't sure it was necessary since the main code was in csharp and this part in js ^^;
     
    De-Panther likes this.
  28. Madsen

    Madsen

    Joined:
    Nov 29, 2010
    Posts:
    15
    Thanks AnonDreamer, you motivated me to give the improved version a try, and this one works good with 5.1.2p1 .
    So better use this one and not the one from asset store.
    To make it work with both development builds and release builds just change the code like it is in jonas version:
    Code (CSharp):
    1. var outputFolder = "Release";
    2.         if (EditorUserBuildSettings.development)
    3.             outputFolder = "Development";
    4.      
    5.         // delete old data file
    6.         File.Delete (Path.Combine(pathToBuiltProject, Path.Combine(outputFolder, filename+".data")));
    7.      
    8.         // copy new data files into build
    9.         File.Copy(Path.Combine(stagingAreaData, kOutputFileLoaderFileName), Path.Combine(Path.Combine(pathToBuiltProject, outputFolder),kOutputFileLoaderFileName), true);
    10.         foreach (var f in Directory.GetFiles(stagingAreaData, "*.*.data"))
    11.             File.Copy(f, Path.Combine(Path.Combine(pathToBuiltProject, outputFolder),Path.GetFileName(f)), true);
    12.      
    13.    
    14. // compress new data files
    15.      if (!EditorUserBuildSettings.development) {
    16.        CompressFilesInOutputDirectory (Path.Combine (pathToBuiltProject, outputFolder), Path.Combine (pathToBuiltProject, "Compressed"));
    17.      }
    18.  
    Thanks again - hope we don't meet again at the same topic with Unity 5.1.3 :)
     
    De-Panther and liortal like this.
  29. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    Good to see it works fine now :)
     
  30. Clas

    Clas

    Joined:
    Mar 27, 2013
    Posts:
    6
    Hi,

    I would also love to get this to work.

    I tried De-panthers code (+jslib), and added AnonDreamers changes, but can not make this work on Unity 5.2 or 5.1.3. Did I miss something?

    The project compiles without warnings, but when loading the game in the browser I get a bunch of errors, mostly seems like a lot of methods, types etc. can not be found.

    Ex:
    Unable to find type [UnityEngine.dll]UnityEngine.MonoBehaviour
    Unable to find type [UnityEngine.dll]UnityEngine.Component
    Unable to find type [UnityEngine.dll]UnityEngine.ScriptableObject
    Unable to find type [UnityEngine.dll]UnityEngine.Vector2

    + 50 more similar warnings...

    And then a couple of these warnings:
    The referenced script on this Behaviour is missing!
    A script behaviour has a different serialization layout when loading. (Read 32 bytes but expected 68 bytes)
    Did you #ifdef UNITY_EDITOR a section of your serialized properties in any of your scripts?


    Then the split up .data files are loaded from cache...but afterwards, this warning:
    warning: a problem occurred in builtin C++ name demangling; build with -s DEMANGLE_SUPPORT=1 to link in libcxxabi demangling index.html:34:7
    Invoking error handler due to uncaught exception: abort(133) at jsStackTrace@file:///........game.js


    Thankful for any help :)
     
  31. liortal

    liortal

    Joined:
    Oct 17, 2012
    Posts:
    3,367
    Did anyone try this script with the latest version of Unity that's out there (5.2.x) ? does it work out of the box ?
     
  32. OminStyle

    OminStyle

    Joined:
    Mar 18, 2013
    Posts:
    15
    @liortal I tried it but it doesn't work. Sorry I couldn't give you more information. I had to downgrade it because I had limited time.
     
  33. AnonDreamer

    AnonDreamer

    Joined:
    Oct 28, 2014
    Posts:
    30
    i haven't tried yet, i'm still on 5.1.2f1 :\
     
  34. welch89

    welch89

    Joined:
    Mar 7, 2013
    Posts:
    1
    I tried the script in the asset store on Unity 5.2.1, but it reports about memory issues or an uncaught range error (trying different memory allocations).
    It would be nice to see an official update on the asset store or the integration of this feature in the editor.
     
    Stenyin, Jonathan459 and OminStyle like this.
  35. monark

    monark

    Joined:
    May 2, 2008
    Posts:
    1,528
    The asset store now says this:

    Does that mean this technique is impossible to do now or that it has been builtin properly?

    I'm finding I need to set huge memory sizes 1.5Gb even for data files that are only 5Mb compressed, seems odd that we need so much memory to run even small games.
     
  36. pete

    pete

    Joined:
    Jul 21, 2005
    Posts:
    1,648
    I'd like to know that too, monark, or if anyone got this working in 5.3.x (we're in 5.3.1)? Not hitting those memory sizes in ours but initial load time is too long. Player runs great once it loads. We had level streaming working in 5.1.3 but it doesn't really make sense to go back.

    One thing I noticed looking at OnPostprocessBuild where filesToShip is declared. I don't see any .resource, .asset, mainData or level files in my staging area data directory. I'd guess something with the whole process changed in 5.3. UnityLoader.js is new too I think. There used to be UnityConfig and fileloader js files. I'd assume UnityLoader handles all that now? Don't think that one is too human readable iirc.

    Also, for anyone who hasn't noticed, if you're on a Mac the WebGLSupport/BuildTools moved. It used to be inside the app Contents folder. In 5.3.1 it's just next to the app in PlaybackEngines (probably the new installer). Use EditorApplication.applicationPath instead of the contents path and then strip the /Unity.app off the end of the string. It's in buildToolsDir {get} at the top of the class. Not sure where they are on Windows.
     
  37. Marco-Trivellato

    Marco-Trivellato

    Unity Technologies

    Joined:
    Jul 9, 2013
    Posts:
    1,653
    No, we did not implemented it.
    What we recommend is to organize your project in such way that your build only includes the minimum amount of assets and that you download everything else on-demand via asset bundles.

    The WebGL Memory Size determines how much memory is needed by your memory content. Even if your data file is small, your scenes might be a lot more memory. The best way to find out the memory requirements is to profile your build with the Unity memory profiler. 1.5gb seems quite a lot and it will likely fail in 32-bit browsers.
     
    Last edited: Jan 26, 2016
  38. Marco-Trivellato

    Marco-Trivellato

    Unity Technologies

    Joined:
    Jul 9, 2013
    Posts:
    1,653
    We did make some changes in 5.3 to simplify the generated build as well as handle the server compression (and as you noticed the installer changed too). That meant that we had to deprecate the asset store package for the asset streaming. We understand that initial download size is a challenge for many users but at the moment asset bundles is the way to go.
     
  39. pete

    pete

    Joined:
    Jul 21, 2005
    Posts:
    1,648
    Thanks for the reply, Marco. I had a feeling asset bundles would be the recommendation. Not sure how practical they are for us for level streaming. They aren't quite the same as loading a scene. I guess I'll be taking a look!
     
  40. zyndata

    zyndata

    Joined:
    Feb 15, 2015
    Posts:
    4
    Hi.
    I'm using De-Panther script with unity 5.0.4 but I cant make it work.
    My game contains 7 scenes. I put De-Panter C# script to Assets/
    JS script to Assets/Plugins/WebGL
    I build game as WebGL with player settings: Stripping Level (use microlib), optimized mesh data(on), enable exeptions (none), optimization level (slow), development build (off).
    I'm using unity 5.0.4, my webgl build goes to xampp/htdocs/project folder and I open it via latest chrome http://localhost/project/

    My realase folder contains 7 .data files (i think it is good because i have 7 scenes) and one project.js (100mb)


    When i run game i see following errors in my chrome console:
    errors with enable exeptions (off)
    Code (CSharp):
    1. (index):33 run() called, but dependencies remain, so not runningModule.printErr @ (index):33run @ detektyw.js:2892820(anonymous function) @ detektyw.js:2892931
    2. (index):33 pre-main prep time: 669 ms
    3.  
    4. detektyw.js:2892804 Uncaught 14518992 - Exception catching is disabled, this exception cannot be caught. Compile with -s DISABLE_EXCEPTION_CATCHING=0 or DISABLE_EXCEPTION_CATCHING=2 to catch.

    and errors with enable exeptions (on)
    Code (CSharp):
    1.  
    2. (index):33 run() called, but dependencies remain, so not runningModule.printErr @ (index):33run @ detektyw.js:2892820(anonymous function) @ detektyw.js:2892931
    3. (index):33 pre-main prep time: 669 ms
    4.  
    5. detektyw.js:4087481 Uncaught abort() at Error
    6.     at jsStackTrace (http://localhost/detektyw/Release/detektyw.js:992:13)
    7.     at stackTrace (http://localhost/detektyw/Release/detektyw.js:1009:22)
    8.     at abort (http://localhost/detektyw/Release/detektyw.js:4087587:25)
    9.     at enlargeMemory (http://localhost/detektyw/Release/detektyw.js:1028:3)
    10.     at Function.Runtime.dynamicAlloc [as alloc] (http://localhost/detektyw/Release/detektyw.js:408:161)
    11.     at _sbrk (http://localhost/detektyw/Release/detektyw.js:13521:28)
    12.     at _GC_expand_hp_inner (http://localhost/detektyw/Release/detektyw.js:23436:11)
    13.     at _GC_collect_or_expand (http://localhost/detektyw/Release/detektyw.js:18782:9)
    14.     at _GC_alloc_large (http://localhost/detektyw/Release/detektyw.js:18355:12)
    15.     at _GC_generic_malloc (http://localhost/detektyw/Release/detektyw.js:18987:11)
    16. If this abort() is unexpected, build with -s ASSERTIONS=1 which can give more information.
    Can someone explain me what I'm doing wrong? How to set my 0 scene was loaded first?
    How do you switch scenes? Do you check in update if next scene is already loaded?
    Do i need to edit my index or other files inside build folder to make it work?





     
    Last edited: Apr 6, 2016
  41. De-Panther

    De-Panther

    Joined:
    Dec 27, 2009
    Posts:
    306
    you can check with "CanStreamedLevelBeLoaded" to check if you can load the level.
    And you can set "firstStreamedLevelWithResources" to decide the first scenes that should download together on start
     
  42. De-Panther

    De-Panther

    Joined:
    Dec 27, 2009
    Posts:
    306
    We tested it with Unity 5.3.4 and 5.3.5. It seems that there's no way to re-pack the levels for streaming anymore.

    In the past, the engine didn't delete the "sharedassets" files in the build process. But currently the editor pack those files to it's own format, and delete them before the "PostProcessBuild" is called.

    Maybe @jonas-echterhoff will have another idea. But we started to use AssetBundles for some of the stuff, although it's not ideal if we want to share Sprite Atlases between some of the scenes.
     
    unity-werkstatt likes this.
  43. CarlAmbrose

    CarlAmbrose

    Joined:
    Mar 23, 2020
    Posts:
    1
    It seems exactly what i need for my wegbl project, but i dont know how to implement this thing in my project.
    please can anyone help me with this.
     
unityunity