[email protected]

## [1.19.0] - 2021-05-20 - Replaced our use of BinaryFormatter with new "USerialize" - USerializer performance in synthetic tests is about 40x faster than BinaryFormatter - Real world project tests have seen about 1.8x improvement of cold cache build times, and about 6x improvement of warm cache build times. - Fixed a case where internal type hash was not being cached correctly causing constant cold cache hits. - Fixed a case where previous build results could influence a new build incorrectly by including the wrong dlls. - Fixed a case where multiple scenes in the same asset bundle could generate invalid or incorrect dependencies and not load all necessary data. - Minor fix for native tep profiling results to separate the event name from the event context and to properly string escape the context. - Added the DisableVisibleSubAssetRepresentations build parameter.
needle-mirror · May 20, 2021 · 0ff6bb0 · 0ff6bb0
1 parent 0830550
commit 0ff6bb0
Show file tree

Hide file tree

Showing 31 changed files with 2,953 additions and 61 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,21 +4,31 @@ All notable changes to this package will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [1.18.0] - 2020-04-08
+## [1.19.0] - 2021-05-20
+- Replaced our use of BinaryFormatter with new "USerialize"
+	- USerializer performance in synthetic tests is about 40x faster than BinaryFormatter
+	- Real world project tests have seen about 1.8x improvement of cold cache build times, and about 6x improvement of warm cache build times.
+- Fixed a case where internal type hash was not being cached correctly causing constant cold cache hits.
+- Fixed a case where previous build results could influence a new build incorrectly by including the wrong dlls.
+- Fixed a case where multiple scenes in the same asset bundle could generate invalid or incorrect dependencies and not load all necessary data.
+- Minor fix for native tep profiling results to separate the event name from the event context and to properly string escape the context.
+- Added the DisableVisibleSubAssetRepresentations build parameter. 
+
+## [1.18.0] - 2021-04-08
 - Added an option to build MonoScripts into their own bundle reducing duplication and potential loading errors on certain project setups.
 - Added a type remap in Link.xml generation for UnityEditor.MonoScript to the correct runtime type.
 - Added an option to build bundles using Non-Recursive Dependency calculation methods. 
   - This approach helps reduce asset bundle rebuilds and runtime memory consumption.
 
-## [1.17.0] - 2020-03-03
+## [1.17.0] - 2021-03-03
 - Added [VersionedCallback] attribute for flagging build impacting changes to IProcessScene, IProcessSceneWithReport, IPreprocessShaders, and IPreprocessComputeShaders callbacks.
 - Fixed an IndexOutOfRange exception thrown by the GenerateSubAssetPathMaps build task.
 - Added faster code paths for common hashing operations.
 - 2019.4+ added additional threading usage for saving BuildCache data.
 - Fixed an edge case where SerializeReference types used across assemblies were being code stripped incorrectly.
 - Fixed a false positive cache hit when changing Player Setting's Graphics APIs .
 
-## [1.16.1] - 2020-01-27
+## [1.16.1] - 2021-01-27
 - Handling of communication error with cache server. Build will now continue, using the local cache only.
 - Regression fix for index out of range error on Unity 2018.4
 

diff --git a/Editor/Interfaces/IBuildLog.cs b/Editor/Interfaces/IBuildLog.cs
@@ -70,6 +70,7 @@ internal struct DeferredEvent
         public DeferredEventType Type;
         public double Time;
         public string Name;
+        public string Context;
     }
 
     internal interface IDeferredBuildLogger
@@ -116,7 +117,15 @@ public void Dispose()
                 return;
 
             IDeferredBuildLogger dLog = (IDeferredBuildLogger)m_Logger;
-            IEnumerable<DeferredEvent> dEvents = events.Select(i => new DeferredEvent() { Level = LogLevel.Verbose, Name = i.Name, Time = (double)i.TimeMicroseconds / (double)1000, Type = BuildLoggerExternsions.ConvertToDeferredType(i.Type) });
+            IEnumerable<DeferredEvent> dEvents = events.Select(i =>
+            {
+                var e = new DeferredEvent();
+                e.Level = LogLevel.Verbose;
+                BuildLoggerExternsions.ConvertNativeEventName(i.Name, out e.Name, out e.Context);
+                e.Time = (double)i.TimeMicroseconds / (double)1000;
+                e.Type = BuildLoggerExternsions.ConvertToDeferredType(i.Type);
+                return e;
+            });
             dLog.HandleDeferredEventStream(dEvents);
         }
     }
@@ -175,6 +184,28 @@ internal static DeferredEventType ConvertToDeferredType(ProfileEventType type)
             if (type == ProfileEventType.Info) return DeferredEventType.Info;
             throw new Exception("Unknown type");
         }
+
+        const string k_WriteFile = "Write file:";
+        const string k_WriteObject = "Write object - ";
+
+        internal static void ConvertNativeEventName(string nativeName, out string eventName, out string eventContext)
+        {
+            eventName = nativeName;
+            eventContext = "";
+            if (nativeName.StartsWith(k_WriteFile, StringComparison.Ordinal))
+            {
+                eventName = "Write File";
+                eventContext = nativeName.Substring(k_WriteFile.Length);
+            }
+            else if (nativeName.StartsWith(k_WriteObject, StringComparison.Ordinal))
+            {
+                eventName = "Write Object";
+                eventContext = nativeName.Substring(k_WriteObject.Length);
+            }
+
+            if (eventContext.Any(c => c == '"'))
+                eventContext = eventContext.Replace("\"", "\\\"");
+        }
 #endif
     }
 }
diff --git a/Editor/Interfaces/IBuildParameters.cs b/Editor/Interfaces/IBuildParameters.cs
@@ -120,5 +120,11 @@ public interface IBundleBuildParameters : IBuildParameters
         /// Packs assets in bundles contiguously based on the ordering of the source asset which results in improved asset loading times.
         /// </summary>
         bool ContiguousBundles { get; set; }
+
+        /// <summary>
+        /// Assume sub Assets have no visible asset representations (are not visible in the Project view) which results in improved build times.
+        /// Sub Assets in the built bundles cannot be accessed by AssetBundle.LoadAsset&lt;T&gt or AssetBundle.LoadAllAssets&lt;T&gt.
+        /// </summary>
+        bool DisableVisibleSubAssetRepresentations { get; set; }
     }
 }
diff --git a/Editor/Shared/BuildParameters.cs b/Editor/Shared/BuildParameters.cs
@@ -182,5 +182,8 @@ public BundleBuildParameters(BuildTarget target, BuildTargetGroup group, string
 
         /// <inheritdoc />
         public bool ContiguousBundles { get; set; }
+
+        /// <inheritdoc />
+        public bool DisableVisibleSubAssetRepresentations { get; set; }
     }
 }
diff --git a/Editor/Tasks/BuildPlayerScripts.cs b/Editor/Tasks/BuildPlayerScripts.cs
@@ -31,6 +31,13 @@ public ReturnCode Run()
                 return ReturnCode.SuccessNotRun;
             }
 
+            // We need to ensure the directory is empty so prior results or other artifacts in this directory do not influence the build result
+            if (Directory.Exists(m_Parameters.ScriptOutputFolder))
+            {
+                Directory.Delete(m_Parameters.ScriptOutputFolder, true);
+                Directory.CreateDirectory(m_Parameters.ScriptOutputFolder);
+            }
+
             m_Results.ScriptResults = PlayerBuildInterface.CompilePlayerScripts(m_Parameters.GetScriptCompilationSettings(), m_Parameters.ScriptOutputFolder);
             m_Parameters.ScriptInfo = m_Results.ScriptResults.typeDB;
             BuildCacheUtility.SetTypeDB(m_Parameters.ScriptInfo);

diff --git a/Editor/Tasks/CalculateAssetDependencyData.cs b/Editor/Tasks/CalculateAssetDependencyData.cs
@@ -29,7 +29,7 @@ public class CalculateAssetDependencyData : IBuildTask
 
 #pragma warning disable 649
         [InjectContext(ContextUsage.In)]
-        IBuildParameters m_Parameters;
+        IBundleBuildParameters m_Parameters;
 
         [InjectContext(ContextUsage.In)]
         IBuildContent m_Content;
@@ -76,7 +76,7 @@ internal struct AssetOutput
             public BuildUsageTagSet usageTags;
             public SpriteImporterData spriteData;
             public ExtendedAssetData extendedData;
-            public List<KeyValuePair<ObjectIdentifier, System.Type[]>> objectTypes;
+            public List<ObjectTypes> objectTypes;
         }
 
         internal struct TaskOutput
@@ -98,7 +98,7 @@ static CachedInfo GetCachedInfo(IBuildCache cache, GUID asset, AssetLoadInfo ass
             info.Asset = GetAssetCacheEntry(cache, asset, NonRecursiveDependencies);
 
             var uniqueTypes = new HashSet<System.Type>();
-            var objectTypes = new List<KeyValuePair<ObjectIdentifier, System.Type[]>>();
+            var objectTypes = new List<ObjectTypes>();
             var dependencies = new HashSet<CacheEntry>();
             ExtensionMethods.ExtractCommonCacheData(cache, assetInfo.includedObjects, assetInfo.referencedObjects, uniqueTypes, objectTypes, dependencies);
             info.Dependencies = dependencies.ToArray();
@@ -142,7 +142,7 @@ public ReturnCode Run()
                         m_SpriteData.ImporterData.Add(o.asset, o.spriteData);
                     }
 
-                    if (o.extendedData != null)
+                    if (!m_Parameters.DisableVisibleSubAssetRepresentations && o.extendedData != null)
                     {
                         if (m_ExtendedAssetData == null)
                             m_ExtendedAssetData = new BuildExtendedAssetData();
@@ -234,7 +234,7 @@ static internal ReturnCode RunInternal(TaskInput input, out TaskOutput output)
                         assetResult.usageTags = cachedInfo[i].Data[1] as BuildUsageTagSet;
                         assetResult.spriteData = cachedInfo[i].Data[2] as SpriteImporterData;
                         assetResult.extendedData = cachedInfo[i].Data[3] as ExtendedAssetData;
-                        assetResult.objectTypes = cachedInfo[i].Data[4] as List<KeyValuePair<ObjectIdentifier, System.Type[]>>;
+                        assetResult.objectTypes = cachedInfo[i].Data[4] as List<ObjectTypes>;
                         output.AssetResults[i] = assetResult;
                         output.CachedAssetCount++;
                         continue;

diff --git a/Editor/Tasks/CalculateCustomDependencyData.cs b/Editor/Tasks/CalculateCustomDependencyData.cs
@@ -86,7 +86,7 @@ CachedInfo GetCachedInfo(CacheEntry entry, AssetLoadInfo assetInfo, BuildUsageTa
             info.Asset = entry;
 
             var uniqueTypes = new HashSet<Type>();
-            var objectTypes = new List<KeyValuePair<ObjectIdentifier, Type[]>>();
+            var objectTypes = new List<ObjectTypes>();
             var dependencies = new HashSet<CacheEntry>();
             ExtensionMethods.ExtractCommonCacheData(m_Cache, assetInfo.includedObjects, assetInfo.referencedObjects, uniqueTypes, objectTypes, dependencies);
             info.Dependencies = dependencies.ToArray();
@@ -110,7 +110,7 @@ bool LoadCachedData(string path, out AssetLoadInfo assetInfo, out BuildUsageTagS
             {
                 assetInfo = (AssetLoadInfo)cachedInfo.Data[0];
                 buildUsage = (BuildUsageTagSet)cachedInfo.Data[1];
-                var objectTypes = (List<KeyValuePair<ObjectIdentifier, Type[]>>)cachedInfo.Data[2];
+                var objectTypes = (List<ObjectTypes>)cachedInfo.Data[2];
                 BuildCacheUtility.SetTypeForObjects(objectTypes);
             }
             else

diff --git a/Editor/Tasks/CalculateSceneDependencyData.cs b/Editor/Tasks/CalculateSceneDependencyData.cs
@@ -58,7 +58,7 @@ CachedInfo GetCachedInfo(GUID scene, IEnumerable<ObjectIdentifier> references, S
 #else
             var uniqueTypes = new HashSet<System.Type>();
 #endif
-            var objectTypes = new List<KeyValuePair<ObjectIdentifier, System.Type[]>>();
+            var objectTypes = new List<ObjectTypes>();
             var dependencies = new HashSet<CacheEntry>(prefabEntries);
             ExtensionMethods.ExtractCommonCacheData(m_Cache, null, references, uniqueTypes, objectTypes, dependencies);
             info.Dependencies = dependencies.ToArray();
@@ -102,7 +102,7 @@ public ReturnCode Run()
                     sceneInfo = (SceneDependencyInfo)cachedInfo[i].Data[0];
                     usageTags = cachedInfo[i].Data[1] as BuildUsageTagSet;
                     prefabDependency = (Hash128)cachedInfo[i].Data[2];
-                    var objectTypes = cachedInfo[i].Data[3] as List<KeyValuePair<ObjectIdentifier, System.Type[]>>;
+                    var objectTypes = cachedInfo[i].Data[3] as List<ObjectTypes>;
                     if (objectTypes != null)
                         BuildCacheUtility.SetTypeForObjects(objectTypes);
                 }

diff --git a/Editor/Tasks/GenerateBundlePacking.cs b/Editor/Tasks/GenerateBundlePacking.cs
@@ -109,6 +109,7 @@ void PackSceneBundle(string bundleName, List<GUID> includedScenes, Dictionary<GU
 
             string firstFileName = "";
             HashSet<ObjectIdentifier> previousSceneObjects = new HashSet<ObjectIdentifier>();
+            HashSet<GUID> previousSceneAssets = new HashSet<GUID>();
             List<string> sceneInternalNames = new List<string>();
             foreach (var scene in includedScenes)
             {
@@ -122,8 +123,9 @@ void PackSceneBundle(string bundleName, List<GUID> includedScenes, Dictionary<GU
 
                 var references = new List<ObjectIdentifier>();
                 references.AddRange(sceneInfo.referencedObjects);
-                assetToReferences[scene] = FilterReferencesForAsset(m_DependencyData, scene, references, previousSceneObjects);
+                assetToReferences[scene] = FilterReferencesForAsset(m_DependencyData, scene, references, previousSceneObjects, previousSceneAssets);
                 previousSceneObjects.UnionWith(references);
+                previousSceneAssets.UnionWith(assetToReferences[scene]);
 
                 m_WriteData.FileToObjects.Add(internalName, references);
                 m_WriteData.FileToBundle.Add(internalName, bundleName);
@@ -136,9 +138,10 @@ void PackSceneBundle(string bundleName, List<GUID> includedScenes, Dictionary<GU
             }
         }
 
-        internal static List<GUID> FilterReferencesForAsset(IDependencyData dependencyData, GUID asset, List<ObjectIdentifier> references, HashSet<ObjectIdentifier> previousSceneObjects = null)
+        internal static List<GUID> FilterReferencesForAsset(IDependencyData dependencyData, GUID asset, List<ObjectIdentifier> references, HashSet<ObjectIdentifier> previousSceneObjects = null, HashSet<GUID> previousSceneReferences = null)
         {
             var referencedAssets = new HashSet<AssetLoadInfo>();
+            var referencedAssetsGuids = new List<GUID>(referencedAssets.Count);
             var referencesPruned = new List<ObjectIdentifier>(references.Count);
             // Remove Default Resources and Includes for Assets assigned to Bundles
             foreach (ObjectIdentifier reference in references)
@@ -147,15 +150,15 @@ internal static List<GUID> FilterReferencesForAsset(IDependencyData dependencyDa
                     continue;
                 if (dependencyData.AssetInfo.TryGetValue(reference.guid, out AssetLoadInfo referenceInfo))
                 {
-                    referencedAssets.Add(referenceInfo);
+                    if (referencedAssets.Add(referenceInfo))
+                        referencedAssetsGuids.Add(referenceInfo.asset);
                     continue;
                 }
                 referencesPruned.Add(reference);
             }
             references.Clear();
             references.AddRange(referencesPruned);
 
-            var referencedAssetsGuids = new List<GUID>(referencedAssets.Count);
             // Remove References also included by non-circular Referenced Assets
             // Remove References also included by circular Referenced Assets if Asset's GUID is higher than Referenced Asset's GUID
             foreach (AssetLoadInfo referencedAsset in referencedAssets)
@@ -164,7 +167,22 @@ internal static List<GUID> FilterReferencesForAsset(IDependencyData dependencyDa
                 bool circularRef = refObjectIdLookup.Select(x => x.guid).Contains(asset);
                 if (!circularRef || (circularRef && asset > referencedAsset.asset || asset == referencedAsset.asset))
                     references.RemoveAll(refObjectIdLookup.Contains);
-                referencedAssetsGuids.Add(referencedAsset.asset);
+            }
+
+            // Special path for scenes, they can reference the same assets previously references
+            if (!previousSceneReferences.IsNullOrEmpty())
+            {
+                foreach (GUID reference in previousSceneReferences)
+                {
+                    if (!dependencyData.AssetInfo.TryGetValue(reference, out AssetLoadInfo referencedAsset))
+                        continue;
+
+                    var refObjectIdLookup = new HashSet<ObjectIdentifier>(referencedAsset.referencedObjects);
+                    // NOTE: It's impossible for an asset to depend on a scene, thus no need for circular reference checks
+                    // So just remove and add a dependency on the asset if there is a need to depend upon it.
+                    if (references.RemoveAll(refObjectIdLookup.Contains) > 0)
+                        referencedAssetsGuids.Add(referencedAsset.asset);
+                }
             }
 
             // Special path for scenes, they can use data from previous sharedAssets in the same bundle