Search Unity

  1. Welcome to the Unity Forums! Please take the time to read our Code of Conduct to familiarize yourself with the forum rules and how to post constructively.
  2. We have updated the language to the Editor Terms based on feedback from our employees and community. Learn more.
    Dismiss Notice
  3. Join us on November 16th, 2023, between 1 pm and 9 pm CET for Ask the Experts Online on Discord and on Unity Discussions.
    Dismiss Notice
New Forum User Notice Update to the Unity Editor Software Terms Unite 2023 - Ask the Experts Online

Byte Streaming DataBase (Easy to use)

Discussion in 'Community Learning & Teaching' started by IsGreen, Aug 2, 2015.

  1. IsGreen

    IsGreen

    Joined:
    Jan 17, 2014
    Posts:
    206
    Download link.

    In relation with old post UniqueID, I created a byte stream database quite easy to use it.

    I can store in a file any type of values, included serialized classes.

    The database organizes data from a unique GameObject ID --> Name property --> property value.

    That is, a same GameObject (UniqueID) can be associated with several properties or values.

    HOW TO USE:

    To create:

    a) Empty database: you can create empty database with new operator, that is, DBByte database = new DBByte("nameDataBase");
    b) Load datase: you can load an existing database through DBByte database = DBByte.Load("nameDataBase");

    To Get/Set values:

    a) Get values with generic method GetField<T>(ID,"nameProperty", value);
    b) Set values with other generic method SetField<T>(ID,"nameProperty", value);

    Code (CSharp):
    1. using System;
    2. using System.IO;
    3. using System.Collections.Generic;
    4. using System.Runtime.Serialization.Formatters.Binary;
    5.  
    6. public class DBByte {
    7.  
    8.     const byte keyLength = 20;
    9.  
    10.     public class Index{
    11.  
    12.         public int size;    //Byte size
    13.         public long pos;     //Place in data file
    14.  
    15.         public Index(){}
    16.         public Index(int size, long pos){
    17.  
    18.             this.size = size;
    19.             this.pos = pos;
    20.  
    21.         }
    22.  
    23.     }
    24.  
    25.     Dictionary<uint,Dictionary<string,Index>> index;
    26.  
    27.     string idxFile, datFile;
    28.  
    29.     static byte[] Segment(byte[] array, long offSet, long length){
    30.  
    31.         byte[] temp = new byte[length];
    32.         for(long i=0; i<length; i++) temp[i] = array[offSet + i];
    33.         return temp;
    34.  
    35.     }
    36.  
    37.     public static DBByte Load(string nameDB){
    38.  
    39.         string idxFile = UnityEngine.Application.dataPath + "/" + nameDB + ".idx";
    40.         string datFile = UnityEngine.Application.dataPath + "/" + nameDB + ".dat";
    41.  
    42.         if(File.Exists(idxFile) && File.Exists(datFile)){
    43.  
    44.             FileStream fsDat = new FileStream(datFile, FileMode.Open);
    45.             FileStream fsIdx = new FileStream(idxFile, FileMode.Open);
    46.  
    47.             if(fsDat.Length == 0 || fsIdx.Length == 0){
    48.  
    49.  
    50.                 fsDat.Close();
    51.                 fsIdx.Close();
    52.                 UnityEngine.Debug.Log("Empty data file");
    53.                 return new DBByte(nameDB);
    54.  
    55.             }
    56.  
    57.             if(fsIdx.Length % (keyLength + 16) != 0){
    58.  
    59.                 UnityEngine.Debug.Log("Idx file corrupted");
    60.                 fsDat.Close();
    61.                 fsIdx.Close();
    62.                 return new DBByte(nameDB);
    63.  
    64.             }
    65.  
    66.             DBByte loadDBByte = new DBByte();
    67.  
    68.             loadDBByte.index = new Dictionary<uint, Dictionary<string,Index>>();
    69.  
    70.             byte[] idxBytes = new byte[fsIdx.Length];
    71.             fsIdx.Read(idxBytes, 0, (int)fsIdx.Length);
    72.             long count = 0;
    73.  
    74.             while(count < idxBytes.LongLength){
    75.  
    76.                 /*
    77.                 (ID) : 4 bytes
    78.                 (tempIndex.size) : 4 bytes;
    79.                 (tempIndex.pos) : 8 bytes;
    80.                 (field.PadRight(keyLength)) : keyLength bytes;
    81.  
    82.                 Total: 16 + keyLength bytes
    83.                 */
    84.  
    85.                 uint ID = BitConverter.ToUInt32(Segment(idxBytes,count,4),0);
    86.                 count += 4;
    87.                 int size = BitConverter.ToInt32(Segment(idxBytes,count,4),0);
    88.                 count += 4;
    89.                 long pos = BitConverter.ToInt64(Segment(idxBytes,count,8),0);
    90.                 count += 8;
    91.  
    92.                 string field = ByteToString(Segment(idxBytes, count, keyLength));
    93.                 count += keyLength;
    94.  
    95.                 if(!loadDBByte.index.ContainsKey(ID)){
    96.  
    97.                     loadDBByte.index.Add(ID, new Dictionary<string, Index>());
    98.  
    99.                 }
    100.  
    101.                 loadDBByte.index[ID].Add(field, new Index(size,pos));
    102.  
    103.             }
    104.  
    105.             fsDat.Close();
    106.             fsIdx.Close();
    107.  
    108.             loadDBByte.idxFile = idxFile;
    109.             loadDBByte.datFile = datFile;
    110.  
    111.             return loadDBByte;
    112.  
    113.         }
    114.  
    115.         return default(DBByte);
    116.  
    117.     }
    118.  
    119.     private DBByte(){}
    120.  
    121.     public DBByte(string nameDB){
    122.  
    123.         this.index = new Dictionary<uint, Dictionary<string,Index>>();
    124.         this.idxFile = UnityEngine.Application.dataPath + "/" + nameDB + ".idx"; //Index Data
    125.         this.datFile = UnityEngine.Application.dataPath + "/" + nameDB + ".dat"; //Byte Data
    126.  
    127.         FileStream fs = File.Create(this.idxFile);
    128.         fs.Close();
    129.  
    130.         fs = File.Create(this.datFile);
    131.         fs.Close();
    132.  
    133.     }
    134.  
    135.     public T GetField<T>(uint ID, string field){
    136.  
    137.         FileStream fsDat = new FileStream(this.datFile, FileMode.Open);
    138.  
    139.         field = field.PadRight(keyLength);
    140.  
    141.         if(this.index.ContainsKey(ID)){
    142.  
    143.             if(this.index[ID].ContainsKey(field)){
    144.  
    145.                 Index tempIndex = this.index[ID][field];
    146.                 byte[] array = new byte[tempIndex.size];
    147.  
    148.                 fsDat.Seek(tempIndex.pos, SeekOrigin.Begin);
    149.                 fsDat.Read(array, 0, tempIndex.size);
    150.  
    151.                 MemoryStream ms = new MemoryStream();
    152.                 BinaryFormatter bf = new BinaryFormatter();
    153.                 ms.Write(array, 0, tempIndex.size);
    154.                 ms.Seek(0, SeekOrigin.Begin);
    155.  
    156.                 T temp = (T)bf.Deserialize(ms);
    157.  
    158.                 bf = null;
    159.                 tempIndex = null;
    160.                 ms.Close();
    161.                 fsDat.Close();
    162.  
    163.                 return temp;
    164.  
    165.             } else UnityEngine.Debug.Log("Field " + field + " doesn't exist");
    166.  
    167.         } else UnityEngine.Debug.Log("ID " + ID + " doesn't exist");
    168.  
    169.         fsDat.Close();
    170.  
    171.         return default(T);
    172.  
    173.     }
    174.  
    175.     public bool SetField<T>(uint ID, string field, T value){
    176.  
    177.         field = field.PadRight(keyLength);
    178.  
    179.         if(this.index.ContainsKey(ID)){
    180.  
    181.             Dictionary<string,Index> tempDictionary = this.index[ID];
    182.  
    183.             if(tempDictionary.ContainsKey(field)){
    184.  
    185.                 FileStream fsDat = new FileStream(this.datFile, FileMode.Open);
    186.  
    187.                 //Convert to binary
    188.  
    189.                 BinaryFormatter bf = new BinaryFormatter();
    190.                 MemoryStream ms = new MemoryStream();
    191.                 bf.Serialize(ms,value);
    192.                 byte[] array = ms.ToArray();
    193.  
    194.                 Index tempIndex = tempDictionary[field];
    195.  
    196.                 if(array.Length == tempIndex.size){
    197.  
    198.                     fsDat.Seek(tempIndex.pos, SeekOrigin.Begin);
    199.                     fsDat.Write(array, 0, array.Length);
    200.  
    201.                 } else {
    202.  
    203.                     UnityEngine.Debug.Log("Length type value is different to original");
    204.                     return false;
    205.  
    206.                 }
    207.  
    208.                 bf=null;
    209.                 array=null;
    210.                 tempIndex=null;
    211.                 ms.Close();
    212.                 fsDat.Close();
    213.  
    214.             } else this.AddField<T>(ID, field, value);
    215.  
    216.         } else {
    217.  
    218.             this.index.Add(ID, new Dictionary<string, Index>());
    219.             this.AddField<T>(ID, field, value);
    220.  
    221.         }
    222.  
    223.         return true;
    224.  
    225.     }
    226.  
    227.     static string ByteToString(byte[] bytes){
    228.  
    229.         string temp = "";
    230.         for(int i=0; i<bytes.Length; i++){
    231.  
    232.             temp += ((char)bytes[i]).ToString();
    233.  
    234.         }
    235.  
    236.         return temp;
    237.  
    238.     }
    239.  
    240.     static byte[] StringToByte(string text){
    241.  
    242.         char[] characters = text.ToCharArray();
    243.         byte[] temp = new byte[text.Length];
    244.  
    245.         for(int i=0; i<text.Length; i++){
    246.  
    247.             temp[i] = (byte)characters[i];
    248.  
    249.         }
    250.  
    251.         return temp;
    252.  
    253.     }
    254.  
    255.     void AddField<T>(uint ID, string field, T value){
    256.  
    257.         FileStream fsDat = new FileStream(this.datFile, FileMode.Append);
    258.         FileStream fsIdx = new FileStream(this.idxFile, FileMode.Append);
    259.         BinaryWriter idx = new BinaryWriter(fsIdx);
    260.    
    261.         BinaryFormatter bf = new BinaryFormatter();
    262.         MemoryStream ms = new MemoryStream();
    263.         bf.Serialize(ms,value);
    264.         byte[] array = ms.ToArray();
    265.  
    266.         Index tempIndex = new Index(array.Length, fsDat.Length);
    267.  
    268.         fsDat.Write(array,0,array.Length);
    269.  
    270.         idx.Write(ID);
    271.         idx.Write(tempIndex.size);
    272.         idx.Write(tempIndex.pos);
    273.         idx.Write(StringToByte(field));
    274.  
    275.         this.index[ID].Add(field, tempIndex);
    276.    
    277.         ms.Close();
    278.         idx.Close();
    279.         fsDat.Close();
    280.         fsIdx.Close();
    281.  
    282.     }
    283.  
    284. }

    First test, where a new database is created:

    Code (CSharp):
    1. using UnityEngine;
    2. using System.Collections;
    3.  
    4. public class TestDB : MonoBehaviour {
    5.  
    6.     void Start () {    
    7.  
    8.         DBByte db = new DBByte("TestDB");
    9.  
    10.         db.SetField<float>(0,"value1",1.1f);
    11.         db.SetField<float>(0,"value2",2.1f);
    12.         db.SetField<bool>(0,"value3",true);
    13.         db.SetField<float>(0,"value4",4.1f);
    14.         db.SetField<string>(0,"value5","Text".PadRight(50));
    15.         db.SetField<float>(0,"value6",6.1f);
    16.         db.SetField<int>(0,"value7",-7000);
    17.  
    18.         TestClass testClass = new TestClass();
    19.  
    20.         db.SetField<TestClass>(0,"value8", testClass);
    21.  
    22.         Debug.Log("value 1: "+db.GetField<float>(0,"value1"));
    23.         Debug.Log("value 2: "+db.GetField<float>(0,"value2"));
    24.         Debug.Log("value 3: "+db.GetField<bool>(0,"value3"));
    25.         Debug.Log("value 4: "+db.GetField<float>(0,"value4"));
    26.         Debug.Log("value 5: "+db.GetField<string>(0,"value5"));
    27.         Debug.Log("value 6: "+db.GetField<float>(0,"value6"));
    28.         Debug.Log("value 7: "+db.GetField<int>(0,"value7"));
    29.  
    30.         TestClass tempTestClass = db.GetField<TestClass>(0,"value8");
    31.  
    32.         Debug.Log("value 8: " + tempTestClass.intValue+" - " + tempTestClass.bValue);
    33.  
    34.         testClass.intValue = 11111;
    35.         testClass.bValue = !testClass.bValue;
    36.  
    37.         db.SetField<TestClass>(0,"value8", testClass);
    38.  
    39.         db.SetField<float>(0,"value2",12.2f);
    40.         db.SetField<bool>(0,"value3",false);
    41.         db.SetField<float>(0,"value4",14.2f);
    42.         db.SetField<string>(0,"value5","Second Text".PadRight(50));
    43.         db.SetField<float>(0,"value6",16.2f);
    44.         db.SetField<int>(0,"value7",15000);
    45.  
    46.         Debug.Log("-------------------------");
    47.  
    48.         Debug.Log("value 1: "+db.GetField<float>(0,"value1"));
    49.         Debug.Log("value 2: "+db.GetField<float>(0,"value2"));
    50.         Debug.Log("value 3: "+db.GetField<bool>(0,"value3"));
    51.         Debug.Log("value 4: "+db.GetField<float>(0,"value4"));
    52.         Debug.Log("value 5: "+db.GetField<string>(0,"value5"));
    53.         Debug.Log("value 6: "+db.GetField<float>(0,"value6"));
    54.         Debug.Log("value 7: "+db.GetField<int>(0,"value7"));
    55.  
    56.         tempTestClass = db.GetField<TestClass>(0,"value8");
    57.    
    58.         Debug.Log("value 8: " + tempTestClass.intValue+" - " + tempTestClass.bValue);
    59.  
    60.     }
    61.  
    62. }
    63.  
    64. [System.Serializable]
    65. public class TestClass{
    66.  
    67.     public int intValue = 1;
    68.     public bool bValue = true;
    69.  
    70. }

    Second test, where a database is loaded.

    Code (CSharp):
    1. using UnityEngine;
    2. using System.Collections;
    3.  
    4. public class LoadDB : MonoBehaviour {
    5.  
    6.     void Start () {
    7.  
    8.         DBByte db = DBByte.Load("TestDB");
    9.  
    10.         Debug.Log("value 1: " + db.GetField<float> (0,"value1"));
    11.         Debug.Log("value 2: " + db.GetField<float> (0,"value2"));
    12.         Debug.Log("value 3: " + db.GetField<bool>  (0,"value3"));
    13.         Debug.Log("value 4: " + db.GetField<float> (0,"value4"));
    14.         Debug.Log("value 5: " + db.GetField<string>(0,"value5"));
    15.         Debug.Log("value 6: " + db.GetField<float> (0,"value6"));
    16.         Debug.Log("value 7: " + db.GetField<int>   (0,"value7"));
    17.  
    18.         TestClass tempTestClass = db.GetField<TestClass>(0,"value8");
    19.  
    20.         Debug.Log("value 8: " + tempTestClass.intValue + " - " + tempTestClass.bValue);
    21.  
    22.     }
    23.  
    24. }
     
    Last edited: Aug 2, 2015
    IsGreen, Aug 2, 2015
    #1

  2. supamigit

    supamigit

    Joined:
    Dec 2, 2012
    Posts:
    22
    Does this work for Textures 2???...
     
    supamigit, Aug 6, 2015
    #2

  3. IsGreen

    IsGreen

    Joined:
    Jan 17, 2014
    Posts:
    206
    You can use Texture methods GetPixels or GetPixels32, that return an array.
     
    IsGreen, Aug 6, 2015
    #3
    supamigit likes this.