commit fc84dc5f7eb2ca5b505b6b950658c863cb244179 Author: StarBeats <977663818@qq.com> Date: Wed Oct 22 10:20:34 2025 +0800 init repo diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a8c110a --- /dev/null +++ b/.gitignore @@ -0,0 +1,58 @@ +## A streamlined .gitignore for modern .NET projects +## including temporary files, build results, and +## files generated by popular .NET tools. If you are +## developing with Visual Studio, the VS .gitignore +## https://github.com/github/gitignore/blob/main/VisualStudio.gitignore +## has more thorough IDE-specific entries. +## +## Get latest from https://github.com/github/gitignore/blob/main/Dotnet.gitignore + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +.vs +.vscode +.idea + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg + +# Others +~$* +*~ +CodeCoverage/ + +# MSBuild Binary and Structured Log +*.binlog + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml \ No newline at end of file diff --git a/AssetDependencyGraph.cs b/AssetDependencyGraph.cs new file mode 100644 index 0000000..e8a4c99 --- /dev/null +++ b/AssetDependencyGraph.cs @@ -0,0 +1,1031 @@ +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; +using UnityEditor; +using UnityEditor.Experimental.GraphView; +using UnityEditor.UIElements; +using UnityEngine; +using UnityEngine.UIElements; +using Object = UnityEngine.Object; + +namespace AssetDependencyGraph +{ + public class AssetGraphView : GraphView + { + public AssetGraphView() + { + SetupZoom(ContentZoomer.DefaultMinScale, ContentZoomer.DefaultMaxScale); + + this.AddManipulator(new ContentDragger()); + this.AddManipulator(new SelectionDragger()); + this.AddManipulator(new RectangleSelector()); + this.AddManipulator(new FreehandSelector()); + + VisualElement background = new VisualElement + { + style = + { + backgroundColor = new Color(0.17f, 0.17f, 0.17f, 1f) + } + }; + Insert(0, background); + + background.StretchToParentSize(); + } + } + + public class AssetGroup + { + public AssetNode AssetNode; + public Group GroupNode = new Group(); + public Node MainGraphNode = new Node(); + public Rect MainGraphNodeLastPosition = new Rect(); + public List AssetGraphNodes = new List(); + public List AssetGraphConnections = new List(); + public List DependenciesForPlacement = new List(); + } + + + public class AssetDependencyGraph : EditorWindow + { + private const float NodeWidth = 300.0f; + private static System.Text.RegularExpressions.Regex isGuid = new("^[\\da-f]{32}$"); + + private Dictionary type2Toogle = new(); + (string assetType, bool show)[] assetTypeHidenTogleItems = new[] { + ("Executable" , true), ("UnityAssembly", true), ("SourceFile", true), + ("MakeFile", true), ("DatFile", true), ("AudioClip", true), + ("VideoClip", true), ("Texture", false), ("Shader", false), + ("ComputeShader", false), ("ShaderHeader", false), ("Binary", true), + ("TextFile", true), ("Excel", true), ("UnknowFileType", false), + }; + + private Toggle AlignmentToggle; + + private GraphView graphView; + + private readonly List selectedObjects = new List(); + private readonly List assetGroups = new List(); + + private readonly Dictionary fullPathNodeLookup = new Dictionary(); + + private static Dictionary assetId2NodeDic; + private static Dictionary path2NodeIdDic; + + static AssetNode FindAssetNode(string path) + { + if (path2NodeIdDic.TryGetValue(path, out var id)) + { + return assetId2NodeDic[id]; + } + + return null; + } + + [MenuItem("Window/资源引用分析")] + public static void CreateTestGraphViewWindow() + { + var window = GetWindow(true); + window.titleContent = new GUIContent("Asset Dependency Graph"); + } + + public void OnEnable() + { + CreateGraph(); + } + + public void OnDisable() + { + rootVisualElement.Remove(graphView); + } + + public static bool IsGuid(string str) + { + if (str.Length == 32) + { + return isGuid.IsMatch(str); + } + return false; + } + + void ResolveDependency() + { + var dependencyGraph = Path.Combine(Application.dataPath.Replace("Assets", ""), "Library", "dependencyGraph.json"); + if (!File.Exists(dependencyGraph)) + { + EditorUtility.DisplayDialog("提示", "需要解析引用关系", "确认"); + return; + } + var js = File.ReadAllText(dependencyGraph); + assetId2NodeDic = JsonSerializer.Deserialize>(js, options: new JsonSerializerOptions() + { + IncludeFields = true, + Converters = { new AssetIdentifyJsonConverter(), new AssetNodeJsonConverter() } + }); + path2NodeIdDic = new(); + foreach (var node in assetId2NodeDic) + { + path2NodeIdDic[node.Key.Path] = node.Key; + } + } + + void CreateGraph() + { + graphView = new AssetGraphView + { + name = "Asset Dependency Graph", + }; + + VisualElement toolbar = CreateToolbar(); + VisualElement toolbar2 = CreateFilterbar(); + + rootVisualElement.Add(toolbar); + rootVisualElement.Add(toolbar2); + rootVisualElement.Add(graphView); + graphView.StretchToParentSize(); + toolbar.BringToFront(); + toolbar2.BringToFront(); + } + + VisualElement CreateToolbar() + { + var toolbar = new VisualElement + { + style = + { + flexDirection = FlexDirection.Row, + flexGrow = 0, + backgroundColor = new Color(0.25f, 0.25f, 0.25f, 0.75f) + } + }; + + var options = new VisualElement + { + style = { alignContent = Align.Center } + }; + + toolbar.Add(options); + toolbar.Add(new Button(ExploreAsset) + { + text = "Explore Asset", + }); + toolbar.Add(new Button(ClearGraph) + { + text = "Clear" + }); + toolbar.Add(new Button(ResetGroups) + { + text = "Reset Groups" + }); + toolbar.Add(new Button(ResetAllNodes) + { + text = "Reset Nodes" + }); + toolbar.Add(new Button(() => + { + Task.Run(() => + { + System.Diagnostics.Process.Start(startInfo: new() + { + FileName = $"{Application.dataPath}/../Tools/UnityDependencyAnalyzer.exe", + Arguments = $"{Application.dataPath.Replace("Assets", "")} \" \" \" \" localhost ", + UseShellExecute = true, + }) + .WaitForExit(); + + ResolveDependency(); + }); + }) + { + text = "Analyze Reference" + }); + + var ts = new ToolbarSearchField(); + ts.RegisterValueChangedCallback(x => + { + if (string.IsNullOrEmpty(x.newValue)) + { + graphView.FrameAll(); + return; + } + + graphView.ClearSelection(); + graphView.graphElements.ToList().ForEach(y => + { + if (y is Node node && y.title.IndexOf(x.newValue, System.StringComparison.OrdinalIgnoreCase) >= 0) + { + graphView.AddToSelection(node); + } + }); + + graphView.FrameSelection(); + }); + toolbar.Add(ts); + + AlignmentToggle = new Toggle(); + AlignmentToggle.text = "Horizontal Layout"; + AlignmentToggle.value = true; + AlignmentToggle.RegisterValueChangedCallback(x => + { + ResetAllNodes(); + }); + toolbar.Add(AlignmentToggle); + + return toolbar; + } + + VisualElement CreateFilterbar() + { + var toolbar = new VisualElement + { + style = + { + flexDirection = FlexDirection.Row, + flexGrow = 0, + backgroundColor = new Color(0.25f, 0.25f, 0.25f, 0.75f) + } + }; + + var options = new VisualElement + { + style = { alignContent = Align.Center } + }; + + toolbar.Add(options); + + toolbar.Add(new Label("Filters: ")); + foreach (var pair in assetTypeHidenTogleItems) + { + var assetTypeTogle = new Toggle(); + assetTypeTogle.text = "Hide " + pair.assetType; + assetTypeTogle.value = pair.show; + assetTypeTogle.RegisterValueChangedCallback(x => + { + FilterAssetGroups(); + }); + toolbar.Add(assetTypeTogle); + type2Toogle[pair.assetType] = assetTypeTogle; + } + + return toolbar; + } + + private void ExploreAsset() + { + Object[] objs = Selection.objects; + if(path2NodeIdDic == null || path2NodeIdDic.Count == 0) + { + ResolveDependency(); + } + foreach (var obj in objs) + { + //Prevent readding same object + if (selectedObjects.Contains(obj)) + { + Debug.Log("Object already loaded"); + return; + } + selectedObjects.Add(obj); + + AssetGroup AssetGroup = new AssetGroup(); + AssetGroup.AssetNode = FindAssetNode(AssetDatabase.GetAssetPath(obj).Replace('\\', '/').ToLowerInvariant()); + assetGroups.Add(AssetGroup); + + // assetPath will be empty if obj is null or isn't an asset (a scene object) + if (obj == null) + return; + + AssetGroup.GroupNode = new Group { title = obj.name }; + + PopulateGroup(AssetGroup, obj, new Rect(10, graphView.contentRect.height / 2, 0, 0)); + } + + } + + void PopulateGroup(AssetGroup AssetGroup, Object obj, Rect position) + { + if (obj == null) + { + obj = AssetDatabase.LoadMainAssetAtPath(AssetGroup.AssetNode.Self.Path); + + if (obj == null) + { + Debug.Log("Object doesn't exist anymore"); + return; + } + } + + AssetGroup.MainGraphNode = CreateNode(AssetGroup, AssetGroup.AssetNode, obj, true); + AssetGroup.MainGraphNode.userData = 0; + AssetGroup.MainGraphNode.SetPosition(position); + + if (!graphView.Contains(AssetGroup.GroupNode)) + { + graphView.AddElement(AssetGroup.GroupNode); + } + + graphView.AddElement(AssetGroup.MainGraphNode); + + AssetGroup.GroupNode.AddElement(AssetGroup.MainGraphNode); + + CreateDependencyNodes(AssetGroup, AssetGroup.AssetNode, AssetGroup.MainGraphNode, AssetGroup.GroupNode, 1); + CreateDependentNodes(AssetGroup, AssetGroup.AssetNode, AssetGroup.MainGraphNode, AssetGroup.GroupNode, -1); + + AssetGroup.AssetGraphNodes.Add(AssetGroup.MainGraphNode); + AssetGroup.GroupNode.capabilities &= ~Capabilities.Deletable; + + AssetGroup.GroupNode.Focus(); + + AssetGroup.MainGraphNode.RegisterCallback( + UpdateGroupDependencyNodePlacement, AssetGroup + ); + } + + //Recreate the groups but use the already created groups instead of new ones + void FilterAssetGroups() + { + + //first collect the main node's position and then clear the graph + foreach (var AssetGroup in assetGroups) + { + AssetGroup.MainGraphNodeLastPosition = AssetGroup.MainGraphNode.GetPosition(); + } + + fullPathNodeLookup.Clear(); + + foreach (var AssetGroup in assetGroups) + { + //clear the nodes and dependencies after getting the position of the main node + CleanGroup(AssetGroup); + + PopulateGroup(AssetGroup, null, AssetGroup.MainGraphNodeLastPosition); + } + } + + void CleanGroup(AssetGroup assetGroup) + { + if (assetGroup.AssetGraphConnections.Count > 0) + { + foreach (var edge in assetGroup.AssetGraphConnections) + { + graphView.RemoveElement(edge); + } + } + assetGroup.AssetGraphConnections.Clear(); + + foreach (var node in assetGroup.AssetGraphNodes) + { + graphView.RemoveElement(node); + } + assetGroup.AssetGraphNodes.Clear(); + + assetGroup.DependenciesForPlacement.Clear(); + } + + private void CreateDependencyNodes(AssetGroup assetGroup, AssetNode asssetNode, Node selfGraphNode, Group groupGraphNode, int depth) + { + foreach (var dependAssetId in asssetNode.DependencySet) + { + AssetNode dependAssetNode = FindAssetNode(dependAssetId.Path); + if(dependAssetNode == null) + { + continue; + } + var typeName = dependAssetNode.AssetType; + //filter out selected asset types + if (FilterType(typeName)) + { + continue; + } + + var fullPath = dependAssetId.Path; + Debug.Log(fullPath); + if (IsGuid(fullPath)) + { + fullPath = AssetDatabase.GUIDToAssetPath(fullPath); + Debug.Log(fullPath); + } + var obj = AssetDatabase.LoadMainAssetAtPath(fullPath); + if(obj == null) + { + continue; + } + Node dependGraphNode = CreateNode(assetGroup, dependAssetNode, obj, false); + + if (!assetGroup.AssetGraphNodes.Contains(dependGraphNode)) + { + dependGraphNode.userData = depth; + } + + //CreateDependencyNodes(assetGroup, dependAssetNode, dependGraphNode, groupGraphNode, depth + 1); + + //if the node doesnt exists yet, put it in the group + if (!graphView.Contains(dependGraphNode)) + { + graphView.AddElement(dependGraphNode); + + assetGroup.DependenciesForPlacement.Add(dependGraphNode); + groupGraphNode.AddElement(dependGraphNode); + } + else + { + //TODO: if it already exists, put it in a separate group for shared assets + //Check if the dependencyNode is in the same group or not + //if it's a different group move it to a new shared group + /* + if (SharedToggle.value) { + if (!assetGroup.m_AssetNodes.Contains(dependencyNode)) { + if (assetGroup.SharedGroup == null) { + assetGroup.SharedGroup = new AssetGroup(); + + AssetGroups.Add(assetGroup.SharedGroup); + assetGroup.SharedGroup.assetPath = assetGroup.assetPath; + + assetGroup.SharedGroup.groupNode = new Group { title = "Shared Group" }; + + assetGroup.SharedGroup.mainNode = dependencyNode; + assetGroup.SharedGroup.mainNode.userData = 0; + } + + if (!m_GraphView.Contains(assetGroup.SharedGroup.groupNode)) { + m_GraphView.AddElement(assetGroup.SharedGroup.groupNode); + } + + //add the node to the group and remove it from the previous group + assetGroup.m_AssetNodes.Remove(dependencyNode); + //assetGroup.groupNode.RemoveElement(dependencyNode); + assetGroup.m_DependenciesForPlacement.Remove(dependencyNode); + + assetGroup.SharedGroup.m_DependenciesForPlacement.Add(dependencyNode); + + if (!assetGroup.SharedGroup.groupNode.ContainsElement(dependencyNode)) { + assetGroup.SharedGroup.groupNode.AddElement(dependencyNode); + } + + assetGroup.SharedGroup.m_AssetNodes.Add(dependencyNode); + } + }*/ + } + + Edge edge = CreateEdge(dependGraphNode, selfGraphNode); + + assetGroup.AssetGraphConnections.Add(edge); + assetGroup.AssetGraphNodes.Add(dependGraphNode); + } + + } + + private void CreateDependentNodes(AssetGroup assetGroup, AssetNode asssetNode, Node selfGraphNode, Group groupGraphNode, int depth) + { + foreach (var dependAssetId in asssetNode.DependentSet) + { + AssetNode dependAssetNode = FindAssetNode(dependAssetId.Path); + if (dependAssetNode == null) + { + continue; + } + + var typeName = dependAssetNode.AssetType; + //filter out selected asset types + if (FilterType(typeName)) + { + continue; + } + var fullPath = dependAssetId.Path; + Debug.Log(fullPath); + if (IsGuid(fullPath)) + { + fullPath = AssetDatabase.GUIDToAssetPath(fullPath); + Debug.Log(fullPath); + } + var obj = AssetDatabase.LoadMainAssetAtPath(fullPath); + if (obj == null) + { + continue; + } + Node dependentGraphNode = CreateNode(assetGroup, dependAssetNode, obj, false); + + if (!assetGroup.AssetGraphNodes.Contains(dependentGraphNode)) + { + dependentGraphNode.userData = depth; + } + + //CreateDependencyNodes(assetGroup, dependAssetNode, dependGraphNode, groupGraphNode, depth - 1); + + //if the node doesnt exists yet, put it in the group + if (!graphView.Contains(dependentGraphNode)) + { + graphView.AddElement(dependentGraphNode); + + assetGroup.DependenciesForPlacement.Add(dependentGraphNode); + groupGraphNode.AddElement(dependentGraphNode); + } + else + { + //TODO: if it already exists, put it in a separate group for shared assets + //Check if the dependencyNode is in the same group or not + //if it's a different group move it to a new shared group + /* + if (SharedToggle.value) { + if (!assetGroup.m_AssetNodes.Contains(dependencyNode)) { + if (assetGroup.SharedGroup == null) { + assetGroup.SharedGroup = new AssetGroup(); + + AssetGroups.Add(assetGroup.SharedGroup); + assetGroup.SharedGroup.assetPath = assetGroup.assetPath; + + assetGroup.SharedGroup.groupNode = new Group { title = "Shared Group" }; + + assetGroup.SharedGroup.mainNode = dependencyNode; + assetGroup.SharedGroup.mainNode.userData = 0; + } + + if (!m_GraphView.Contains(assetGroup.SharedGroup.groupNode)) { + m_GraphView.AddElement(assetGroup.SharedGroup.groupNode); + } + + //add the node to the group and remove it from the previous group + assetGroup.m_AssetNodes.Remove(dependencyNode); + //assetGroup.groupNode.RemoveElement(dependencyNode); + assetGroup.m_DependenciesForPlacement.Remove(dependencyNode); + + assetGroup.SharedGroup.m_DependenciesForPlacement.Add(dependencyNode); + + if (!assetGroup.SharedGroup.groupNode.ContainsElement(dependencyNode)) { + assetGroup.SharedGroup.groupNode.AddElement(dependencyNode); + } + + assetGroup.SharedGroup.m_AssetNodes.Add(dependencyNode); + } + }*/ + } + + Edge edge = CreateEdge(selfGraphNode, dependentGraphNode); + + assetGroup.AssetGraphConnections.Add(edge); + assetGroup.AssetGraphNodes.Add(dependentGraphNode); + } + } + + Edge CreateEdge(Node dependencyNode, Node parentNode) + { + Edge edge = new Edge + { + input = dependencyNode.inputContainer[0] as Port, + output = parentNode.outputContainer[0] as Port, + }; + edge.input?.Connect(edge); + edge.output?.Connect(edge); + + dependencyNode.RefreshPorts(); + + graphView.AddElement(edge); + + edge.capabilities &= ~Capabilities.Deletable; + + return edge; + } + + private Node CreateNode(AssetGroup assetGroup, AssetNode assetNode, Object obj, bool isMainNode) + { + Node resultNode; + string fullPath = assetNode.Self.Path; + + if (fullPathNodeLookup.TryGetValue(fullPath, out resultNode)) + { + //----not sure what this is, the more dependencies the further removed on the chart? + //int currentDepth = (int)resultNode.userData; + //resultNode.userData = currentDepth + 1; + return resultNode; + } + + if (AssetDatabase.TryGetGUIDAndLocalFileIdentifier(obj, out var assetGuid, out long _)) + { + var objNode = new Node + { + title = obj.name, + style = + { + width = NodeWidth + } + }; + + objNode.extensionContainer.style.backgroundColor = new Color(0.24f, 0.24f, 0.24f, 0.8f); + + #region Select button + objNode.titleContainer.Add(new Button(() => + { + Selection.activeObject = obj; + EditorGUIUtility.PingObject(obj); + }) + { + style = + { + height = 16.0f, + alignSelf = Align.Center, + alignItems = Align.Center + }, + text = "Select" + }); + objNode.titleContainer.Add(new Button(() => + { + if(assetNode.AssetType == "Folder") + { + } + else + { + bool hasRef = false; + foreach (var item in assetNode.DependentSet) + { + if (item.AssetType != "Folder") + { + hasRef = true; + break; + } + } + if(!hasRef) + { + if (EditorUtility.DisplayDialog("提示", "当前 asset 没有引用,是否直接删除", "确认删除", "取消")) + { + AssetDatabase.DeleteAsset(AssetDatabase.GetAssetPath(obj)); + } + } + else + { + EditorUtility.DisplayDialog("提示", "当前 asset 有引用,请先处理引用关系", "确认"); + } + } + }) + { + style = + { + height = 16.0f, + alignSelf = Align.Center, + alignItems = Align.Center + }, + text = "Delete" + }); + #endregion + + #region Padding + var infoContainer = new VisualElement + { + style = + { + paddingBottom = 4.0f, + paddingTop = 4.0f, + paddingLeft = 4.0f, + paddingRight = 4.0f + } + }; + #endregion + + #region Asset Path, removed to improve visibility with large amount of assets + // infoContainer.Add(new Label { + // text = assetPath, + //#if UNITY_2019_1_OR_NEWER + // style = { whiteSpace = WhiteSpace.Normal } + //#else + // style = { wordWrap = true } + //#endif + // }); + #endregion + + #region Asset type + var typeName = assetNode.AssetType; + + var typeLabel = new Label + { + text = $"Type: {typeName}", + }; + infoContainer.Add(typeLabel); + + objNode.extensionContainer.Add(infoContainer); + #endregion + + var typeContainer = new VisualElement + { + style = + { + paddingBottom = 4.0f, + paddingTop = 4.0f, + paddingLeft = 4.0f, + paddingRight = 4.0f, + backgroundColor = GetColorByAssetType(typeName) + } + }; + + objNode.extensionContainer.Add(typeContainer); + + #region Node Icon, replaced with color + //Texture assetTexture = AssetPreview.GetAssetPreview(obj); + //if (!assetTexture) + // assetTexture = AssetPreview.GetMiniThumbnail(obj); + + //if (assetTexture) + //{ + // AddDivider(objNode); + + // objNode.extensionContainer.Add(new Image + // { + // image = assetTexture, + // scaleMode = ScaleMode.ScaleToFit, + // style = + // { + // paddingBottom = 4.0f, + // paddingTop = 4.0f, + // paddingLeft = 4.0f, + // paddingRight = 4.0f + // } + // }); + //} + #endregion + + // Ports + var dependentAmount = 0; + foreach (var item in assetNode.DependentSet) + { + if (item.AssetType != "Folder") + { + ++dependentAmount; + } + } + if (assetNode.DependentSet.Count > 0) + { + Port port = objNode.InstantiatePort(Orientation.Horizontal, Direction.Input, Port.Capacity.Single, typeof(Object)); + port.Add(new Button(() => + { + CreateDependentNodes(assetGroup, assetNode, fullPathNodeLookup[fullPath], assetGroup.GroupNode, (int)fullPathNodeLookup[fullPath].userData - 1); + EditorApplication.delayCall += () => ResetAllNodes(); + }) + { + style = + { + height = 16.0f, + alignSelf = Align.Center, + alignItems = Align.Center + }, + text = "展开" + }); + port.portName = dependentAmount + "个引用"; + objNode.inputContainer.Add(port); + } + + var dependencyAmount = assetNode.DependencySet.Count; + if (dependencyAmount > 0) + { + Port port = objNode.InstantiatePort(Orientation.Horizontal, Direction.Output, Port.Capacity.Single, typeof(Object)); + port.Add(new Button(() => + { + CreateDependencyNodes(assetGroup, assetNode, fullPathNodeLookup[fullPath], assetGroup.GroupNode, (int)fullPathNodeLookup[fullPath].userData + 1); + EditorApplication.delayCall += () => ResetAllNodes(); + + }) + { + style = + { + height = 16.0f, + alignSelf = Align.Center, + alignItems = Align.FlexEnd + }, + text = "展开" + }); + port.portName = dependencyAmount + "个依赖"; + objNode.outputContainer.Add(port); + objNode.RefreshPorts(); + } + + resultNode = objNode; + + resultNode.RefreshExpandedState(); + resultNode.RefreshPorts(); + resultNode.capabilities &= ~Capabilities.Deletable; + resultNode.capabilities |= Capabilities.Collapsible; + } + fullPathNodeLookup[fullPath] = resultNode; + return resultNode; + } + + bool FilterType(string type) + { + if (type2Toogle.TryGetValue(type, out var result)) + { + return result.value; + } + + return false; + } + + + StyleColor GetColorByAssetType(string typeName) + { + switch (typeName) + { + case "MonoScript": + return Color.black; + case "Material": + return new Color(0.1f, 0.5f, 0.1f); //green + case "Texture2D": + return new Color(0.5f, 0.1f, 0.1f); //red + case "RenderTexture": + return new Color(0.8f, 0.1f, 0.1f); //red + case "Shader": + return new Color(0.1f, 0.1f, 0.5f); //dark blue + case "ComputeShader": + return new Color(0.1f, 0.1f, 0.5f); //dark blue + case "GameObject": + return new Color(0f, 0.8f, 0.7f); //light blue + case "AnimationClip": + return new Color(1, 0.7f, 1); //pink + case "AnimatorController": + return new Color(1, 0.7f, 0.8f); //pink + case "AudioClip": + return new Color(1, 0.8f, 0); //orange + case "AudioMixerController": + return new Color(1, 0.8f, 0); //orange + case "Font": + return new Color(0.9f, 1, 0.9f); //light green + case "TMP_FontAsset": + return new Color(0.9f, 1, 0.9f); //light green + case "Mesh": + return new Color(0.5f, 0, 0.5f); //purple + case "TerrainLayer": + return new Color(0.5f, 0.8f, 0f); //green + case "Folder": + return Color.yellow; + default: + break; + } + + return CustomColor(typeName); + } + + //Add custom assets here + StyleColor CustomColor(string assetType) + { + switch (assetType) + { + case "GearObject": + return new Color(0.9f, 0, 0.9f); //pink + case "TalentObject": + return new Color(0.9f, 0, 0.9f); // + case "AbilityInfo": + return new Color(0.9f, 0, 0.9f); // + case "HealthSO": + return new Color(0.9f, 0, 0.9f); // + default: + break; + } + + //standard color + return new Color(0.24f, 0.24f, 0.24f, 0.8f); + } + + private static void AddDivider(Node objNode) + { + var divider = new VisualElement { name = "divider" }; + divider.AddToClassList("horizontal"); + objNode.extensionContainer.Add(divider); + } + + private void ClearGraph() + { + selectedObjects.Clear(); + + foreach (var assetGroup in assetGroups) + { + EmptyGroup(assetGroup); + } + + fullPathNodeLookup.Clear(); + + assetGroups.Clear(); + } + + void EmptyGroup(AssetGroup assetGroup) + { + if (assetGroup.AssetGraphConnections.Count > 0) + { + foreach (var edge in assetGroup.AssetGraphConnections) + { + graphView.RemoveElement(edge); + } + } + assetGroup.AssetGraphConnections.Clear(); + + foreach (var node in assetGroup.AssetGraphNodes) + { + graphView.RemoveElement(node); + } + assetGroup.AssetGraphNodes.Clear(); + + assetGroup.DependenciesForPlacement.Clear(); + + graphView.RemoveElement(assetGroup.GroupNode); + + assetGroup.GroupNode = null; + } + + private void UpdateGroupDependencyNodePlacement(GeometryChangedEvent e, AssetGroup assetGroup) + { + assetGroup.MainGraphNode.UnregisterCallback( + UpdateGroupDependencyNodePlacement + ); + + ResetNodes(assetGroup); + } + + void ResetAllNodes() + { + foreach (var assetGroup in assetGroups) + { + ResetNodes(assetGroup); + } + } + + //Reset the node positions of the given group + void ResetNodes(AssetGroup assetGroup) + { + // The current y offset in per depth + var depthOffset = new Dictionary(); + + foreach (var node in assetGroup.DependenciesForPlacement) + { + int depth = (int)node.userData; + + if (!depthOffset.ContainsKey(depth)) + depthOffset.Add(depth, 0.0f); + + if (AlignmentToggle.value) + { + depthOffset[depth] += node.layout.height; + } + else + { + depthOffset[depth] += node.layout.width; + } + } + + // Move half of the node into negative y space so they're on either size of the main node in y axis + var depths = new List(depthOffset.Keys); + foreach (int depth in depths) + { + if (depth == 0) + continue; + + float offset = depthOffset[depth]; + depthOffset[depth] = (0f - offset / 2.0f); + } + + Rect mainNodeRect = assetGroup.MainGraphNode.GetPosition(); + + foreach (var node in assetGroup.DependenciesForPlacement) + { + int depth = (int)node.userData; + if (AlignmentToggle.value) + { + node.SetPosition(new Rect(mainNodeRect.x + node.layout.width * 1.5f * depth, mainNodeRect.y + depthOffset[depth], 0, 0)); + } + else + { + node.SetPosition(new Rect(mainNodeRect.x + depthOffset[depth], mainNodeRect.y + node.layout.height * 1.5f * depth, 0, 0)); + } + + if (AlignmentToggle.value) + { + depthOffset[depth] += node.layout.height; + } + else + { + depthOffset[depth] += node.layout.width; + } + } + } + + //fix the position of the groups so they dont overlap + void ResetGroups() + { + float y = 0; + float x = 0; + + foreach (var assetGroup in assetGroups) + { + if (AlignmentToggle.value) + { + Rect pos = assetGroup.GroupNode.GetPosition(); + pos.x = x; + assetGroup.GroupNode.SetPosition(pos); + x += assetGroup.GroupNode.GetPosition().width; + } + else + { + Rect pos = assetGroup.GroupNode.GetPosition(); + pos.y = y; + assetGroup.GroupNode.SetPosition(pos); + y += assetGroup.GroupNode.GetPosition().height; + } + } + } + } +} diff --git a/UnityDependencyAnalyzer.sln b/UnityDependencyAnalyzer.sln new file mode 100644 index 0000000..03b3bdc --- /dev/null +++ b/UnityDependencyAnalyzer.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35527.113 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnityFileDumper", "UnityFileDumper\UnityFileDumper.csproj", "{AA81D2D2-75E6-483A-A682-61A8F9A26023}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnityDependencyAnalyzer", "UnityDependencyAnalyzer\UnityDependencyAnalyzer.csproj", "{FC9CD587-C3AF-4892-9270-94BCC7E7C3D3}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {AA81D2D2-75E6-483A-A682-61A8F9A26023}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AA81D2D2-75E6-483A-A682-61A8F9A26023}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AA81D2D2-75E6-483A-A682-61A8F9A26023}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AA81D2D2-75E6-483A-A682-61A8F9A26023}.Release|Any CPU.Build.0 = Release|Any CPU + {FC9CD587-C3AF-4892-9270-94BCC7E7C3D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FC9CD587-C3AF-4892-9270-94BCC7E7C3D3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FC9CD587-C3AF-4892-9270-94BCC7E7C3D3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FC9CD587-C3AF-4892-9270-94BCC7E7C3D3}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/UnityDependencyAnalyzer/AssetDefine.cs b/UnityDependencyAnalyzer/AssetDefine.cs new file mode 100644 index 0000000..fb5fd5a --- /dev/null +++ b/UnityDependencyAnalyzer/AssetDefine.cs @@ -0,0 +1,318 @@ +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AssetDependencyGraph +{ + [BsonIgnoreExtraElements] + public class AssetIdentify + { + public string Path = null!; + public string AssetType = null!; + [AllowNull] + public string Guid; + [AllowNull] + public string Md5; + } + + public sealed class AssetIdentifyJsonConverter : JsonConverter + { + static JsonSerializerOptions serializerOptions = new JsonSerializerOptions() { IncludeFields = true }; + + public override AssetIdentify? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return JsonSerializer.Deserialize(reader.GetString()!, serializerOptions); + } + + public override void Write(Utf8JsonWriter writer, AssetIdentify value, JsonSerializerOptions options) + { + writer.WriteStringValue(JsonSerializer.Serialize(value, serializerOptions)); + } + + public override AssetIdentify ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return Read(ref reader, typeToConvert, serializerOptions)!; + } + + public override void WriteAsPropertyName(Utf8JsonWriter writer, [DisallowNull] AssetIdentify value, JsonSerializerOptions options) + { + writer.WritePropertyName(JsonSerializer.Serialize(value, serializerOptions)); + } + } + + [BsonIgnoreExtraElements] + public class AssetNode + { + public AssetIdentify Self=null!; + public string AssetType=null!; + [JsonIgnore] + public ConcurrentBag Dependencies = new(); + [JsonIgnore] + public ConcurrentBag Dependent = new(); + + [AllowNull] + public HashSet DependencySet; + [AllowNull] + public HashSet DependentSet; + } + + public sealed class AssetNodeJsonConverter : JsonConverter + { + static JsonSerializerOptions serializerOptions = new JsonSerializerOptions() + { + IncludeFields = true, + Converters = { new AssetIdentifyJsonConverter() } + }; + + public override AssetNode? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return JsonSerializer.Deserialize(reader.GetString()!, serializerOptions); + } + + public override void Write(Utf8JsonWriter writer, AssetNode value, JsonSerializerOptions options) + { + writer.WriteStringValue(JsonSerializer.Serialize(value, serializerOptions)); + } + } + + [BsonIgnoreExtraElements] + public sealed class FolderNode : AssetNode + { + } + + [BsonIgnoreExtraElements] + public sealed class PackageNode : AssetNode + { + } + + public class AssetDependencyGraphDB + { + MongoClient client; + IMongoCollection FolderNodes; + IMongoCollection PackageNodes; + IMongoCollection AssetNodes; + Dictionary findCacheDic = new(); + + public AssetDependencyGraphDB(string user, string passwd, string ip) + { + MongoClientSettings settings; + if(string.IsNullOrWhiteSpace(user) && !string.IsNullOrEmpty(ip)) + { + settings = MongoClientSettings.FromUrl(new MongoUrl($"mongodb://{ip}:27017/")); + } + else + { + settings = MongoClientSettings.FromUrl(new MongoUrl($"mongodb://{user}:{passwd}@{ip}:27017/")); + } + + settings.ConnectTimeout = TimeSpan.FromSeconds(5); + settings.MinConnectionPoolSize = 1; + settings.MaxConnectionPoolSize = 25; + client = new MongoClient(settings); + var db = client.GetDatabase("assetgraph"); + FolderNodes = db.GetCollection("folder_nodes"); + PackageNodes = db.GetCollection("package_nodes"); + AssetNodes = db.GetCollection("asset_nodes"); + } + + public void Clean() + { + client.DropDatabase("assetgraph"); + var db = client.GetDatabase("assetgraph"); + FolderNodes = db.GetCollection("folder_nodes"); + PackageNodes = db.GetCollection("package_nodes"); + AssetNodes = db.GetCollection("asset_nodes"); + } + + public void Insert(T node) where T : AssetNode + { + switch (node) + { + case FolderNode folderNode: + { + FolderNodes.InsertOne(folderNode); + break; + } + case PackageNode packageNode: + { + PackageNodes.InsertOne(packageNode); + break; + } + case AssetNode assetNode: + { + AssetNodes.InsertOne(assetNode); + break; + } + default: + break; + } + } + + public void UpdateOrInsert(T node) where T : AssetNode + { + switch (node) + { + case FolderNode folderNode: + { + var filter = Builders.Filter.And( + Builders.Filter.Eq(fn=>fn.Self.Path,node.Self.Path) + ); + var found = FolderNodes.Find(filter); + if (found == null || found.CountDocuments() == 0) + { + FolderNodes.InsertOne(folderNode); + } + else + { + var result = FolderNodes.UpdateOne(filter, Builders.Update.Combine( + Builders.Update.Set(fn => fn.Self, folderNode.Self), + Builders.Update.Set(fn => fn.AssetType, folderNode.AssetType), + Builders.Update.Set(fn => fn.Dependencies, folderNode.Dependencies), + Builders.Update.Set(fn => fn.Dependent, folderNode.Dependent) + )); + } + + break; + } + case PackageNode packageNode: + { + var filter = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, node.Self.Path) + ); + var found = PackageNodes.Find(filter); + if (found == null || found.CountDocuments() == 0) + { + PackageNodes.InsertOne(packageNode); + } + else + { + var result = PackageNodes.UpdateOne(filter, Builders.Update.Combine( + Builders.Update.Set(fn => fn.Self, packageNode.Self), + Builders.Update.Set(fn => fn.AssetType, packageNode.AssetType), + Builders.Update.Set(fn => fn.Dependencies, packageNode.Dependencies), + Builders.Update.Set(fn => fn.Dependent, packageNode.Dependent) + )); + } + break; + } + case AssetNode assetNode: + { + var filter = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, node.Self.Path) + ); + var found = AssetNodes.Find(filter); + if (found == null || found.CountDocuments() == 0) + { + AssetNodes.InsertOne(assetNode); + } + else + { + var result = AssetNodes.UpdateOne(filter, Builders.Update.Combine( + Builders.Update.Set(fn => fn.Self, assetNode.Self), + Builders.Update.Set(fn => fn.AssetType, assetNode.AssetType), + Builders.Update.Set(fn => fn.Dependencies, assetNode.Dependencies), + Builders.Update.Set(fn => fn.Dependent, assetNode.Dependent) + )); + } + break; + } + default: + break; + } + } + + public void Delete(T node) where T : AssetNode + { + switch (node) + { + case FolderNode folderNode: + { + var filter = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, node.Self.Path) + ); + var found = FolderNodes.Find(filter); + if (found != null && found.CountDocuments() == 0) + { + // TODO: del ref dep + FolderNodes.DeleteOne(filter); + } + break; + } + case PackageNode packageNode: + { + var filter = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, node.Self.Path) + ); + var found = PackageNodes.Find(filter); + if (found != null && found.CountDocuments() == 0) + { + // TODO: del ref dep + PackageNodes.DeleteOne(filter); + } + break; + } + case AssetNode assetNode: + { + var filter = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, node.Self.Path) + ); + var found = AssetNodes.Find(filter); + if (found != null && found.CountDocuments() == 0) + { + // TODO: del ref dep + AssetNodes.DeleteOne(filter); + } + break; + } + default: + break; + } + } + + public AssetNode Find(string path) + { + if(findCacheDic.TryGetValue(path, out var assetNode)) + { + return assetNode; + } + + var filter = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, path) + ); + var found = AssetNodes.Find(filter); + if (found != null && found.CountDocuments() != 0) + { + assetNode = found.First(); + findCacheDic[path] = assetNode; + return assetNode; + } + + var filter1 = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, path) + ); + var found1 = PackageNodes.Find(filter1); + if (found1 != null && found1.CountDocuments() != 0) + { + assetNode = found1.First(); + findCacheDic[path] = assetNode; + return assetNode; + } + + var filter2 = Builders.Filter.And( + Builders.Filter.Eq(fn => fn.Self.Path, path) + ); + var found2 = FolderNodes.Find(filter2); + if (found2 != null && found2.CountDocuments() != 0) + { + assetNode = found2.First(); + findCacheDic[path] = assetNode; + return assetNode; + } + + return null!; + } + } +} diff --git a/UnityDependencyAnalyzer/DependencyAnalysis.cs b/UnityDependencyAnalyzer/DependencyAnalysis.cs new file mode 100644 index 0000000..8713e33 --- /dev/null +++ b/UnityDependencyAnalyzer/DependencyAnalysis.cs @@ -0,0 +1,557 @@ +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Bson.Serialization.Options; +using System; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace AssetDependencyGraph +{ + public static class FileExtensionHelper + { + public static string GetTypeByExtension(string ext) + { + switch (ext.ToLowerInvariant()) + { + case ".a": + case ".dll": + case ".so": + case ".exe": + case ".dynlib": + return "Executable"; + case ".asmdef": + case ".asmref": + return "UnityAssembly"; + case ".cs": + case ".lua": + case ".js": + case ".ts": + case ".java": + case ".h": + case ".cpp": + case ".cxx": + case ".mm": + case ".py": + case ".bat": + case ".jar": + case ".arr": + case ".jslib": + return "SourceFile"; + case ".gradle": + return "MakeFile"; + case ".dat": + case ".data": + return "DatFile"; + case ".mp3": + case ".ogg": + case ".wav": + return "AudioClip"; + case ".mp4": + case ".webm": + return "VideoClip"; + case ".mat": + return "Material"; + case ".rendertexture": + case ".dds": + case ".exr": + case ".hdr": + case ".png": + case ".jpg": + case ".gif": + case ".psd": + case ".bmp": + case ".tiff": + case ".tga": + case ".gradient": + case ".spriteatlas": + return "Texture"; + case ".obj": + case ".fbx": + case ".mesh": + return "Mesh"; + case ".shader": + case ".surfshader": + case ".shadergraph": + return "Shader"; + case ".compute": + return "ComputeShader"; + case ".hlsl": + case ".cginc": + case ".shadersubgraph": + return "ShaderHeader"; + case ".otf": + case ".ttf": + return "Font"; + case ".byte": + case ".bytes": + case ".bin": + return "Binary"; + case ".txt": + case ".md": + case ".chm": + case ".yml": + case ".url": + case ".json": + case ".json5": + case ".xml": + case ".uxml": + case ".nson": + case ".config": + case ".pdf": + return "TextFile"; + case ".xlsx": + case ".xls": + return "Excel"; + case ".unity": + case ".scene": + return "Scene"; + case ".prefab": + return "Prefab"; + default: + return "UnknowFileType"; + } + } + + + public static bool IsPackage(string ext) + { + switch (ext.ToLowerInvariant()) + { + case ".prefab": + case ".unity": + return true; + default: + return false; + } + } + + public static bool NeedAnalyzeDepend(string ext) + { + switch (ext.ToLowerInvariant()) + { + case ".prefab": + case ".unity": + case ".asset": + case ".mat": + return true; + default: + return false; + } + } + public static bool Exclude(string path) => path.EndsWith(".meta") + || path.EndsWith(".unitypackage") + || path.EndsWith(".preset") + || path.EndsWith(".backup") + || path.EndsWith(".tmp") + || path.EndsWith(".editor") + || path.EndsWith(".zip") + || path.EndsWith(".scenetemplate"); + } + + public interface IDependencyAnalysis + { + void Analyze(string path, Dictionary> result); + } + + public class FolderDependencyAnalysis : IDependencyAnalysis + { + public void Analyze(string path, Dictionary> result) + { + if (!result.TryGetValue(path, out var list)) + { + result[path] = list = new(); + } + + foreach (string file in Directory.EnumerateFiles(path)) + { + if (FileExtensionHelper.Exclude(file)) + { + continue; + } + + var p = file.ToUniversalPath().ToUnityRelatePath(); + list.Add(p); + } + + foreach (string directory in Directory.EnumerateDirectories(path)) + { + var p = directory.ToUniversalPath().ToUnityRelatePath(); + list.Add(p); + } + } + } + + public class UnityDependencyAnalysis2 : IDependencyAnalysis + { + Regex guidRegex = new Regex("guid:\\s?([\\da-f]+)"); + List GetDepGuidByFile(string path) + { + List result; + try + { + result = UnityFileApi.DependencyTool.GetDependencies(path); + } + catch (NotSupportedException) + { + var str = File.ReadAllText(path); + result = new(); + var matches = guidRegex.Matches(str); + for (int i = 0; i < matches.Count; i++) + { + var guid = matches[i].Groups[1].Value; + if (!result.Contains(guid)) + { + result.Add(guid); + } + } + } + + return result; + } + + public void Analyze(string path, Dictionary> result) + { + if (!result.TryGetValue(path, out var list)) + { + result[path] = list = new(); + } + + var ext = Path.GetExtension(path); + if (FileExtensionHelper.NeedAnalyzeDepend(ext)) + { + var dependencies = GetDepGuidByFile(path); + for (int i = 0; i < dependencies.Count; i++) + { + var dep = dependencies[i]; + list.Add(dep); + } + } + } + } + + internal class DependencyAnalyzer + { + private Dictionary, IDependencyAnalysis> dependencyAnalysisDic = new(); + private Dictionary> path2Dependences = new(); + + private JsonSerializerOptions options = new JsonSerializerOptions { IncludeFields = true }; + [BsonDictionaryOptions(Representation = DictionaryRepresentation.ArrayOfArrays)] + private ConcurrentDictionary assetIdentify2AssetNodeDic = new(); + private ConcurrentDictionary path2Id = new(); + private List<(string path, bool isDir)> allPath = new(); + private UnityLmdb unityLmdb; + private static Regex isGuid = new Regex("^[\\da-f]{32}$"); + + public DependencyAnalyzer() + { + unityLmdb = new UnityLmdb(); + dependencyAnalysisDic.Add(new Predicate<(string path, bool isDir)>(pi => !pi.isDir), new UnityDependencyAnalysis2()); + dependencyAnalysisDic.Add(new Predicate<(string path, bool isDir)>(pi => pi.isDir), new FolderDependencyAnalysis()); + } + + private void Visivt(string path) + { + path = path.ToUniversalPath(); + if (FileExtensionHelper.Exclude(path)) + { + return; + } + allPath.Add((path, Directory.Exists(path))); + } + + public static bool IsGuid(string str) + { + if (str.Length == 32) + { + return isGuid.IsMatch(str); + } + return false; + } + + public (AssetIdentify id, AssetNode node) GetOrCreateFolderNode(string path) + { + if (!path2Id.TryGetValue(path, out var k)) + { + if (k == null) + { + k = new AssetIdentify() + { + Path = path, + AssetType = "Folder", + Guid = null, + Md5 = null + }; + assetIdentify2AssetNodeDic[k] = new FolderNode() + { + Self = k, + AssetType = "Folder", + }; + } + path2Id[path] = k; + } + return (k, assetIdentify2AssetNodeDic[k]); + } + + public (AssetIdentify id, AssetNode node) GetOrCreateAssetNode(string path) + { + if (!path2Id.TryGetValue(path, out var k)) + { + if (k == null) + { + var ext = Path.GetExtension(path); + k = new AssetIdentify() + { + Path = path, + Guid = null, + AssetType = FileExtensionHelper.GetTypeByExtension(ext) + //Md5 = Utils.Md5(path) + }; + if (FileExtensionHelper.IsPackage(ext)) + { + assetIdentify2AssetNodeDic[k] = new PackageNode() + { + Self = k, + AssetType = k.AssetType, + }; + } + else + { + assetIdentify2AssetNodeDic[k] = new AssetNode() + { + Self = k, + AssetType = k.AssetType, + }; + } + path2Id[path] = k; + } + } + + return (k, assetIdentify2AssetNodeDic[k]); + } + + public void ResolveGuidDatabase() + { + unityLmdb.ResolveGuidPath(); + } + + public void AnalyzeMainProcess(string rootFolder, int processCnt = 8) + { + Stopwatch sw = Stopwatch.StartNew(); + sw.Start(); + Utils.TraverseDirectory(rootFolder, Visivt, -1); + sw.Stop(); + Console.WriteLine($"遍历目录耗时:{sw.ElapsedMilliseconds / 1000f}s"); + + sw.Restart(); + var itemCnt = allPath.Count / processCnt; + List subProcessArgs = new(); + List resultJsonPaths = new(); + var projectPath = Environment.GetCommandLineArgs()[1]; + for (int i = 0; i < processCnt; i++) + { + int r = (itemCnt * (i + 1)); + if (r >= allPath.Count) + { + r = allPath.Count; + } + + var s = JsonSerializer.Serialize(allPath[(i * itemCnt)..r], options); + var jsonPath = Path.Combine(Path.GetTempPath(), $"path{i}.json"); + var resulJsonPath = Path.Combine(Path.GetTempPath(), $"result{i}.json"); + resultJsonPaths.Add(resulJsonPath); + subProcessArgs.Add($"-reference {projectPath} SubProcess {jsonPath} {resulJsonPath}"); + File.WriteAllText(jsonPath, s); + } + + Task[] subProcessTask = new Task[subProcessArgs.Count]; + var exe = Environment.GetCommandLineArgs()[0]; + if (exe.EndsWith(".dll")) + { + exe = exe.Replace(".dll", ".exe"); + } + + for (int i = 0; i < subProcessArgs.Count; i++) + { + int index = i; + subProcessTask[i] = Task.Factory.StartNew(() => + { + Process p = new Process(); + p.StartInfo = new ProcessStartInfo() + { + FileName = exe, + Arguments = subProcessArgs[index], + UseShellExecute = true, + }; + p.Start(); + p.WaitForExit(); + if (p.ExitCode != 0) + { + Console.WriteLine("Sub Process Error."); + } + }); + } + + Stopwatch sw1 = Stopwatch.StartNew(); + sw1.Start(); + ResolveGuidDatabase(); + sw1.Stop(); + Console.WriteLine($"加载数据库耗时:{sw1.ElapsedMilliseconds / 1000f}s"); + + Task.WaitAll(subProcessTask); + List>> subProcessResults = new(); + foreach (var item in resultJsonPaths) + { + var s = File.ReadAllText(item); + subProcessResults.Add(JsonSerializer.Deserialize>>(s, options)!); + } + sw.Stop(); + Console.WriteLine($"分析引用耗时:{sw.ElapsedMilliseconds / 1000f}s"); + sw.Restart(); + Parallel.ForEach(subProcessResults, arg => ResolveSubProcessResult(arg)); + sw.Stop(); + Console.WriteLine($"合并数据耗时:{sw.ElapsedMilliseconds / 1000f}s"); + sw.Restart(); + foreach (var item in assetIdentify2AssetNodeDic) + { + item.Value.DependencySet = item.Value.Dependencies.ToHashSet(); + item.Value.DependentSet = item.Value.Dependent.ToHashSet(); + } + + string js = JsonSerializer.Serialize(assetIdentify2AssetNodeDic, options: new() + { + IncludeFields = true, + Converters = { new AssetIdentifyJsonConverter(), new AssetNodeJsonConverter() } + }); + + + File.WriteAllText(Path.Combine(UnityLmdb.ProjPath, "Library", "dependencyGraph.json"), js); + + //AssetDependencyGraphDB db = new AssetDependencyGraphDB(Environment.GetCommandLineArgs()[2], Environment.GetCommandLineArgs()[3], Environment.GetCommandLineArgs()[4]); + //sw.Restart(); + //db.Clean(); + //Parallel.ForEach(assetIdentify2AssetNodeDic, item => + //{ + // db.Insert(item.Value); + //}); + sw.Stop(); + Console.WriteLine($"更新数据库:{sw.ElapsedMilliseconds / 1000f}s"); + } + + private void ResolveSubProcessResult(Dictionary> subProcessResult) + { + Parallel.ForEach(subProcessResult, item => + { + var relPath = item.Key.ToLowerInvariant().ToUnityRelatePath(); + var fullPath = relPath.ToUnityFullPath(); + if (File.Exists(fullPath)) + { + var selfNode = GetOrCreateAssetNode(relPath); + selfNode.id.Guid = unityLmdb.GetGuidByPath(relPath); + foreach (var dep in item.Value) + { + var depPath = dep; + if (IsGuid(dep)) + { + depPath = unityLmdb.GetPathByGuid(dep.ToLowerInvariant()); + if (string.IsNullOrEmpty(depPath)) + { + depPath = dep; + } + } + depPath = depPath.ToLowerInvariant(); + var depNode = GetOrCreateAssetNode(depPath); + depNode.node.Dependent.Add(selfNode.id); + selfNode.node.Dependencies.Add(depNode.id); + } + } + else + { + var selfNode = GetOrCreateFolderNode(relPath); + selfNode.id.Guid = unityLmdb.GetGuidByPath(relPath); + foreach (var dep in item.Value) + { + var depPath = dep.ToLowerInvariant().ToUnityRelatePath(); + fullPath = depPath.ToUnityFullPath(); + (AssetIdentify id, AssetNode node) depNode; + + if (File.Exists(fullPath)) + { + depNode = GetOrCreateAssetNode(depPath); + } + else + { + depNode = GetOrCreateFolderNode(depPath); + } + + depNode.node.Dependent.Add(selfNode.id); + selfNode.node.Dependencies.Add(depNode.id); + } + } + }); + } + + public void AnalyzeSubProcess(string pathFile, string resultFilePath) + { + var s = File.ReadAllText(pathFile); + allPath = JsonSerializer.Deserialize>(s, options)!; + if (allPath != null) + { + foreach (var item in allPath) + { + foreach (var item1 in dependencyAnalysisDic) + { + if (item1.Key(item)) + { + item1.Value.Analyze(item.path, path2Dependences); + } + } + } + + var json = JsonSerializer.Serialize(path2Dependences, options); + File.WriteAllText(resultFilePath, json); + } + } + + + public void Analyze(string rootFolder) + { + Stopwatch sw = Stopwatch.StartNew(); + sw.Start(); + Utils.TraverseDirectory(rootFolder, Visivt, -1); + foreach (var item in allPath) + { + foreach (var item1 in dependencyAnalysisDic) + { + if (item1.Key(item)) + { + item1.Value.Analyze(item.path, path2Dependences); + } + } + } + + //Parallel.ForEach(allPath, (pi) => + //{ + // foreach (var item in dependencyAnalysisDic) + // { + // if (item.Key(pi)) + // { + // item.Value.Analyze(pi.path, path2Dependences); + // } + // } + //}); + + sw.Stop(); + Console.WriteLine($"分析引用耗时:{sw.ElapsedMilliseconds / 1000f}s"); + //AssetDependencyGraphDB db = new AssetDependencyGraphDB("", "", "localhost"); + //sw.Restart(); + //db.Clean(); + //Parallel.ForEach(assetIdentify2AssetNodeDic, item => + //{ + // db.UpdateOrInsert(item.Value); + //}); + //sw.Stop(); + //Console.WriteLine($"更新数据库:{sw.ElapsedMilliseconds / 1000f}s"); + } + } +} diff --git a/UnityDependencyAnalyzer/Libs/AWSSDK.Core.dll b/UnityDependencyAnalyzer/Libs/AWSSDK.Core.dll new file mode 100644 index 0000000..53b57f8 Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/AWSSDK.Core.dll differ diff --git a/UnityDependencyAnalyzer/Libs/Crc32.NET.dll b/UnityDependencyAnalyzer/Libs/Crc32.NET.dll new file mode 100644 index 0000000..21213ed Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/Crc32.NET.dll differ diff --git a/UnityDependencyAnalyzer/Libs/DnsClient.dll b/UnityDependencyAnalyzer/Libs/DnsClient.dll new file mode 100644 index 0000000..323f3e4 Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/DnsClient.dll differ diff --git a/UnityDependencyAnalyzer/Libs/LightningDB.dll b/UnityDependencyAnalyzer/Libs/LightningDB.dll new file mode 100644 index 0000000..361c5fa Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/LightningDB.dll differ diff --git a/UnityDependencyAnalyzer/Libs/Microsoft.Extensions.Logging.Abstractions.dll b/UnityDependencyAnalyzer/Libs/Microsoft.Extensions.Logging.Abstractions.dll new file mode 100644 index 0000000..1580585 Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/Microsoft.Extensions.Logging.Abstractions.dll differ diff --git a/UnityDependencyAnalyzer/Libs/MongoDB.Bson.dll b/UnityDependencyAnalyzer/Libs/MongoDB.Bson.dll new file mode 100644 index 0000000..067a7cb Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/MongoDB.Bson.dll differ diff --git a/UnityDependencyAnalyzer/Libs/MongoDB.Driver.Core.dll b/UnityDependencyAnalyzer/Libs/MongoDB.Driver.Core.dll new file mode 100644 index 0000000..cea23cc Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/MongoDB.Driver.Core.dll differ diff --git a/UnityDependencyAnalyzer/Libs/MongoDB.Driver.dll b/UnityDependencyAnalyzer/Libs/MongoDB.Driver.dll new file mode 100644 index 0000000..ad9affc Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/MongoDB.Driver.dll differ diff --git a/UnityDependencyAnalyzer/Libs/MongoDB.Libmongocrypt.dll b/UnityDependencyAnalyzer/Libs/MongoDB.Libmongocrypt.dll new file mode 100644 index 0000000..f72b181 Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/MongoDB.Libmongocrypt.dll differ diff --git a/UnityDependencyAnalyzer/Libs/SharpCompress.dll b/UnityDependencyAnalyzer/Libs/SharpCompress.dll new file mode 100644 index 0000000..e71466e Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/SharpCompress.dll differ diff --git a/UnityDependencyAnalyzer/Libs/Snappier.dll b/UnityDependencyAnalyzer/Libs/Snappier.dll new file mode 100644 index 0000000..c4e4aba Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/Snappier.dll differ diff --git a/UnityDependencyAnalyzer/Libs/System.Runtime.CompilerServices.Unsafe.dll b/UnityDependencyAnalyzer/Libs/System.Runtime.CompilerServices.Unsafe.dll new file mode 100644 index 0000000..de9e124 Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/System.Runtime.CompilerServices.Unsafe.dll differ diff --git a/UnityDependencyAnalyzer/Libs/UnityFileSystemApi.dll b/UnityDependencyAnalyzer/Libs/UnityFileSystemApi.dll new file mode 100644 index 0000000..fa1460d Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/UnityFileSystemApi.dll differ diff --git a/UnityDependencyAnalyzer/Libs/ZstdSharp.dll b/UnityDependencyAnalyzer/Libs/ZstdSharp.dll new file mode 100644 index 0000000..6eac618 Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/ZstdSharp.dll differ diff --git a/UnityDependencyAnalyzer/Libs/lmdb.dll b/UnityDependencyAnalyzer/Libs/lmdb.dll new file mode 100644 index 0000000..c9f25bd Binary files /dev/null and b/UnityDependencyAnalyzer/Libs/lmdb.dll differ diff --git a/UnityDependencyAnalyzer/Program.cs b/UnityDependencyAnalyzer/Program.cs new file mode 100644 index 0000000..5b3e610 --- /dev/null +++ b/UnityDependencyAnalyzer/Program.cs @@ -0,0 +1,51 @@ +using AssetDependencyGraph; +using System.Text.Json; + +switch (Environment.GetCommandLineArgs()[1]) +{ + case "-dump": + { + UnityLmdb unityLmdb = new(); + var projDir = Environment.GetCommandLineArgs()[2].TrimEnd('/').TrimEnd('\\'); + unityLmdb.ResolveGuidPathByDBPath($"{projDir}/Library/SourceAssetDB"); + var js = unityLmdb.ResultToJson(); + File.WriteAllText($"{projDir}/Library/SourceAssetDB.json", js); + break; + } + case "-verify": + { + Console.WriteLine("Start"); + UnityLmdb unityLmdb = new(); + var projDir = Environment.GetCommandLineArgs()[2].TrimEnd('/').TrimEnd('\\'); + Directory.SetCurrentDirectory(projDir); + unityLmdb.ResolveGuidPathByDBPath($"{projDir}/Library/SourceAssetDB"); + var res = unityLmdb.VerifyGUID(); + if (res.Count > 0) + { + var js = JsonSerializer.Serialize(res, new JsonSerializerOptions { IncludeFields = true }); + Console.WriteLine("Has Error."); + + File.WriteAllText($"{projDir}/verify-result.json", js); + } + Console.WriteLine("End"); + break; + } + case "-reference": + { + UnityLmdb.ProjPath = Environment.GetCommandLineArgs()[2]; + Utils.DataPath = Path.Combine(UnityLmdb.ProjPath, "Assets").ToUniversalPath(); + + if (Environment.GetCommandLineArgs().Length > 3 && Environment.GetCommandLineArgs()[3].Equals("SubProcess")) + { + new DependencyAnalyzer().AnalyzeSubProcess(Environment.GetCommandLineArgs()[4], Environment.GetCommandLineArgs()[5]); + } + else + { + new DependencyAnalyzer().AnalyzeMainProcess(Utils.DataPath, 10); + } + break; + } + default: + break; +} + diff --git a/UnityDependencyAnalyzer/Properties/PublishProfiles/FolderProfile.pubxml b/UnityDependencyAnalyzer/Properties/PublishProfiles/FolderProfile.pubxml new file mode 100644 index 0000000..3594540 --- /dev/null +++ b/UnityDependencyAnalyzer/Properties/PublishProfiles/FolderProfile.pubxml @@ -0,0 +1,19 @@ + + + + + Release + Any CPU + bin\Release\net8.0\publish\win-x64\ + FileSystem + <_TargetId>Folder + net8.0 + win-x64 + true + true + true + false + + \ No newline at end of file diff --git a/UnityDependencyAnalyzer/Properties/PublishProfiles/FolderProfile.pubxml.user b/UnityDependencyAnalyzer/Properties/PublishProfiles/FolderProfile.pubxml.user new file mode 100644 index 0000000..c5dc233 --- /dev/null +++ b/UnityDependencyAnalyzer/Properties/PublishProfiles/FolderProfile.pubxml.user @@ -0,0 +1,10 @@ + + + + + True|2025-07-08T03:14:03.4969097Z||;True|2025-07-07T16:09:23.5193447+08:00||;True|2025-07-07T16:07:31.9772710+08:00||;True|2025-07-07T16:07:24.6544267+08:00||;True|2025-07-07T16:06:58.2999030+08:00||;True|2025-07-07T16:06:47.5176281+08:00||;True|2025-07-07T16:06:30.9937062+08:00||;True|2025-07-07T16:06:16.1827410+08:00||;True|2025-07-04T20:51:36.2342905+08:00||;True|2025-07-04T20:49:41.4585526+08:00||;True|2025-07-04T20:46:06.1627373+08:00||;True|2025-07-04T20:39:45.9777563+08:00||;True|2025-07-04T20:38:51.7922210+08:00||;True|2025-07-04T20:38:30.7653415+08:00||;True|2025-07-04T20:37:48.1335374+08:00||;True|2025-07-04T20:37:32.2007568+08:00||;True|2025-07-04T20:37:03.6443318+08:00||;True|2025-07-04T20:35:04.7675245+08:00||;True|2025-04-16T20:37:48.8637838+08:00||;True|2025-04-16T20:23:11.1448355+08:00||;True|2025-04-16T16:49:35.4476343+08:00||;True|2025-04-16T16:36:22.8513231+08:00||;True|2025-04-16T12:23:59.6108463+08:00||;True|2025-04-16T12:08:50.9715020+08:00||;True|2025-04-16T11:57:10.7843966+08:00||;False|2025-04-16T11:02:10.7479206+08:00||;False|2025-04-16T10:59:42.9239923+08:00||;False|2025-04-16T10:54:01.4031170+08:00||;True|2025-04-16T10:29:13.7528358+08:00||;False|2025-04-16T10:27:03.3522077+08:00||;False|2025-04-16T10:26:07.1734998+08:00||; + + + \ No newline at end of file diff --git a/UnityDependencyAnalyzer/Properties/launchSettings.json b/UnityDependencyAnalyzer/Properties/launchSettings.json new file mode 100644 index 0000000..f56dbae --- /dev/null +++ b/UnityDependencyAnalyzer/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "UnityDependencyAnalyzer": { + "commandName": "Project", + "commandLineArgs": "G:/G_android \" \" \" \" localhost" + } + } +} \ No newline at end of file diff --git a/UnityDependencyAnalyzer/UnityDependencyAnalyzer.csproj b/UnityDependencyAnalyzer/UnityDependencyAnalyzer.csproj new file mode 100644 index 0000000..a60f65a --- /dev/null +++ b/UnityDependencyAnalyzer/UnityDependencyAnalyzer.csproj @@ -0,0 +1,54 @@ + + + + Exe + net8.0 + enable + enable + False + + + + + + + + + Libs\AWSSDK.Core.dll + + + Libs\Crc32.NET.dll + + + Libs\DnsClient.dll + + + Libs\LightningDB.dll + + + Libs\Microsoft.Extensions.Logging.Abstractions.dll + + + Libs\MongoDB.Bson.dll + + + Libs\MongoDB.Driver.dll + + + Libs\MongoDB.Driver.Core.dll + + + Libs\MongoDB.Libmongocrypt.dll + + + Libs\SharpCompress.dll + + + Libs\Snappier.dll + + + Libs\ZstdSharp.dll + + + + diff --git a/UnityDependencyAnalyzer/UnityDependencyAnalyzer.csproj.user b/UnityDependencyAnalyzer/UnityDependencyAnalyzer.csproj.user new file mode 100644 index 0000000..6365cfd --- /dev/null +++ b/UnityDependencyAnalyzer/UnityDependencyAnalyzer.csproj.user @@ -0,0 +1,6 @@ + + + + <_LastSelectedProfileId>C:\Users\xinyt\source\repos\UnityFileDumper\UnityDependencyAnalyzer\Properties\PublishProfiles\FolderProfile.pubxml + + \ No newline at end of file diff --git a/UnityDependencyAnalyzer/UnityLmdb.cs b/UnityDependencyAnalyzer/UnityLmdb.cs new file mode 100644 index 0000000..2a452bc --- /dev/null +++ b/UnityDependencyAnalyzer/UnityLmdb.cs @@ -0,0 +1,164 @@ +using LightningDB; +using System.Collections.Concurrent; +using System.Text; +using System.Text.Json; + +namespace AssetDependencyGraph +{ + public sealed class UnityLmdb + { + private Dictionary guid2Path = new(); + private Dictionary path2Guid = new(); + public static string ProjPath = null!; + + private string dbFilePath = null!; + + public static byte[] Guid2LmdbKey(string guid) + { + var inputByteArray = new byte[guid.Length / 2]; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < guid.Length; i += 2) + { + sb.Append(guid[i + 1]); + sb.Append(guid[i]); + } + guid = sb.ToString(); + for (var x = 0; x < inputByteArray.Length; x++) + { + inputByteArray[x] = (byte)Convert.ToInt32(guid.Substring(x * 2, 2), 16); + } + + return inputByteArray; + } + + public static string LmdbKey2Guid(byte[] bytes) + { + StringBuilder ret = new StringBuilder(); + for (var i = 0; i < bytes.Length; i++) + { + ret.AppendFormat("{0:x2}", bytes[i]); + if (ret.Length == 32) + { + break; + } + } + for (int i = 0; i < ret.Length; i += 2) + { + var c = ret[i]; + ret[i] = ret[i + 1]; + ret[i + 1] = c; + } + var hex = ret.ToString(); + return hex; + } + + public void ResolveGuidPath() + { + var sourceDbPath = Path.Combine(ProjPath, "Library", "SourceAssetDB"); + var dbPath = dbFilePath = Path.Combine(ProjPath, "Library", "SourceAssetDB1"); + File.Copy(sourceDbPath, dbPath, true); + using var env = new LightningEnvironment(dbPath, configuration: new() + { + MaxDatabases = 64, + MaxReaders = 64, + }); + env.Open(EnvironmentOpenFlags.NoSubDir | EnvironmentOpenFlags.ReadOnly); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using (var db = tx.OpenDatabase("GuidToPath", closeOnDispose: true)) + using (var cursor = tx.CreateCursor(db)) + { + foreach (var item in cursor.AsEnumerable()) + { + guid2Path[LmdbKey2Guid(item.Item1.AsSpan().ToArray())] = Encoding.UTF8.GetString(item.Item2.AsSpan()).ToLowerInvariant().Trim('\0'); + } + } + + using (var db = tx.OpenDatabase("PathToGuid", closeOnDispose: true)) + using (var cursor = tx.CreateCursor(db)) + { + foreach (var item in cursor.AsEnumerable()) + { + path2Guid[Encoding.UTF8.GetString(item.Item1.AsSpan()).ToLowerInvariant().Trim('\0')] = LmdbKey2Guid(item.Item2.AsSpan().ToArray()); + } + } + } + + + public void ResolveGuidPathByDBPath(string dbPath) + { + dbFilePath = dbPath; + using var env = new LightningEnvironment(dbPath, configuration: new() + { + MaxDatabases = 64, + MaxReaders = 64, + }); + env.Open(EnvironmentOpenFlags.NoSubDir | EnvironmentOpenFlags.ReadOnly); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using (var db = tx.OpenDatabase("GuidToPath", closeOnDispose: true)) + using (var cursor = tx.CreateCursor(db)) + { + foreach (var item in cursor.AsEnumerable()) + { + guid2Path[LmdbKey2Guid(item.Item1.AsSpan().ToArray())] = Encoding.UTF8.GetString(item.Item2.AsSpan()).ToLowerInvariant().Trim('\0'); + } + } + + using (var db = tx.OpenDatabase("PathToGuid", closeOnDispose: true)) + using (var cursor = tx.CreateCursor(db)) + { + foreach (var item in cursor.AsEnumerable()) + { + path2Guid[Encoding.UTF8.GetString(item.Item1.AsSpan()).ToLowerInvariant().Trim('\0')] = LmdbKey2Guid(item.Item2.AsSpan().ToArray()); + } + } + } + + public ConcurrentBag VerifyGUID() + { + ConcurrentBag result = new (); + Parallel.ForEach(path2Guid, (item) => + { + var f = item.Key + ".meta"; + if (File.Exists(f)) + { + var ftext = File.ReadAllText(f); + if (!ftext.Contains(item.Value)) + { + result.Add(item.Key); + } + } + }); + + return result; + } + + public string ResultToJson() + { + return JsonSerializer.Serialize(path2Guid, new JsonSerializerOptions { IncludeFields = true }); + } + + public string GetGuidByPath(string path) + { + if (path2Guid.ContainsKey(path)) + { + return path2Guid[path]; + } + else + { + return null!; + } + } + + public string GetPathByGuid(string guid) + { + if (guid2Path.ContainsKey(guid)) + { + return guid2Path[guid]; + } + else + { + return null!; + } + } + } +} diff --git a/UnityDependencyAnalyzer/Utils.cs b/UnityDependencyAnalyzer/Utils.cs new file mode 100644 index 0000000..fded6de --- /dev/null +++ b/UnityDependencyAnalyzer/Utils.cs @@ -0,0 +1,94 @@ +using System.Text; + +namespace AssetDependencyGraph +{ + public static class Utils + { + public static string DataPath = null!; + public static string DataPathLow = null!; + + public static string Md5(string filename) + { + try + { + FileStream fs = new FileStream(filename, FileMode.Open); +#pragma warning disable SYSLIB0021 // 类型或成员已过时 + System.Security.Cryptography.MD5 md5 = new System.Security.Cryptography.MD5CryptoServiceProvider(); +#pragma warning restore SYSLIB0021 // 类型或成员已过时 + byte[] retVal = md5.ComputeHash(fs); + fs.Close(); + return BitConverter.ToString(retVal).ToLower().Replace("-", ""); + } + catch + { + throw; + } + } + + public static void TraverseDirectory(string path, Action action, int depth = 1) + { + if(depth == 0) + { + return; + } + + foreach (string file in Directory.EnumerateFiles(path)) + { + action.Invoke(file); + } + + foreach (string directory in Directory.EnumerateDirectories(path)) + { + action.Invoke(directory); + TraverseDirectory(directory, action, --depth); + } + } + + public static string ToUniversalPath(this string path) + { + return path.Replace("\\", "/"); + } + + public static string ToUnityRelatePath(this string path) + { + DataPathLow ??= DataPath.ToLowerInvariant(); + + if (path.StartsWith(DataPathLow.Replace("assets", "")) && !path.StartsWith(DataPathLow + "/assets")) + { + return path.Replace(DataPathLow.Replace("assets", ""), ""); + } + return path.Replace(DataPathLow, "assets"); + } + + public static string ToUnityFullPath(this string path) + { + if(path.StartsWith("packages")) + { + var fullPath = (DataPath.Replace("Assets", "") + path); + if (!File.Exists(fullPath) && Directory.Exists(fullPath)) + { + fullPath = (DataPath.Replace("Assets", "Library/PackageCache") + path); + } + + if (!File.Exists(fullPath) && Directory.Exists(fullPath)) + { + Console.WriteLine($"ToUnityFullPath failure:{path}"); + } + + return fullPath; + } + + return Path.Combine(DataPath.Replace("Assets", "") , path); + } + + public static string ByteString(this byte[] bytes) + { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < bytes.Length; i++) + { + stringBuilder.Append(Convert.ToString(bytes[i], 2) ); + } + return stringBuilder.ToString(); + } + } +} diff --git a/UnityFileDumper/DependencyTool.cs b/UnityFileDumper/DependencyTool.cs new file mode 100644 index 0000000..d5f9543 --- /dev/null +++ b/UnityFileDumper/DependencyTool.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +namespace UnityFileApi +{ + public static class DependencyTool + { + static DependencyTool() + { + UnityFileSystem.Init(); + } + + public static List GetDependencies(string path) + { + List dependencies = new List(); + try + { + using var archive = UnityFileSystem.MountArchive(path, "/"); + foreach (var node in archive.Nodes) + { + Console.WriteLine($"Processing {node.Path} {node.Size} {node.Flags}"); + + if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile)) + { + using (var serializedFile = UnityFileSystem.OpenSerializedFile(path)) + { + foreach (var extRef in serializedFile.ExternalReferences) + { + dependencies.Add(extRef.Guid); + } + } + } + } + return dependencies; + } + catch (NotSupportedException) + { + // Try as SerializedFile + using (var serializedFile = UnityFileSystem.OpenSerializedFile(path)) + { + foreach (var extRef in serializedFile.ExternalReferences) + { + dependencies.Add(extRef.Guid); + } + } + return dependencies; + } + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/DllWrapper.cs b/UnityFileDumper/DllWrapper.cs new file mode 100644 index 0000000..7707fca --- /dev/null +++ b/UnityFileDumper/DllWrapper.cs @@ -0,0 +1,253 @@ +using System; +using System.Runtime.InteropServices; +using System.Text; +namespace UnityFileApi +{ + public class UnityArchiveHandle : SafeHandle + { + public UnityArchiveHandle() : base(IntPtr.Zero, true) + { + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + return DllWrapper.UnmountArchive(handle) == ReturnCode.Success; + } + } + + public class UnityFileHandle : SafeHandle + { + public UnityFileHandle() : base(IntPtr.Zero, true) + { + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + return DllWrapper.CloseFile(handle) == ReturnCode.Success; + } + } + + public class SerializedFileHandle : SafeHandle + { + public SerializedFileHandle() : base(IntPtr.Zero, true) + { + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + return DllWrapper.CloseSerializedFile(handle) == ReturnCode.Success; + } + } + + public class TypeTreeHandle : SafeHandle + { + public TypeTreeHandle() : base(IntPtr.Zero, true) + { + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + return true; + } + + internal IntPtr Handle => handle; + } + + public enum ReturnCode + { + Success, + AlreadyInitialized, + NotInitialized, + FileNotFound, + FileFormatError, + InvalidArgument, + HigherSerializedFileVersion, + DestinationBufferTooSmall, + InvalidObjectId, + UnknownError, + FileError, + ErrorCreatingArchiveFile, + ErrorAddingFileToArchive, + TypeNotFound, + } + + [Flags] + public enum ArchiveNodeFlags + { + None = 0, + Directory = 1 << 0, + Deleted = 1 << 1, + SerializedFile = 1 << 2, + } + + public enum CompressionType + { + None, + Lzma, + Lz4, + Lz4HC, + }; + + public enum SeekOrigin + { + Begin, + Current, + End, + } + + public enum ExternalReferenceType + { + NonAssetType, + DeprecatedCachedAssetType, + SerializedAssetType, + MetaAssetType, + } + + [StructLayout(LayoutKind.Sequential)] + public struct ObjectInfo + { + public readonly long Id; + public readonly long Offset; + public readonly long Size; + public readonly int TypeId; + } + [Flags] + public enum TypeTreeFlags + { + None = 0, + IsArray = 1 << 0, + IsManagedReference = 1 << 1, + IsManagedReferenceRegistry = 1 << 2, + IsArrayOfRefs = 1 << 3, + } + + [Flags] + public enum TypeTreeMetaFlags + { + None = 0, + AlignBytes = 1 << 14, + AnyChildUsesAlignBytes = 1 << 15, + } + + public static class DllWrapper + { + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_Init")] + public static extern ReturnCode Init(); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_Cleanup")] + public static extern ReturnCode Cleanup(); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_MountArchive")] + public static extern ReturnCode MountArchive([MarshalAs(UnmanagedType.LPStr)] string path, [MarshalAs(UnmanagedType.LPStr)] string mountPoint, out UnityArchiveHandle handle); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_UnmountArchive")] + public static extern ReturnCode UnmountArchive(IntPtr handle); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetArchiveNodeCount")] + public static extern ReturnCode GetArchiveNodeCount(UnityArchiveHandle handle, out int count); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetArchiveNode")] + public static extern ReturnCode GetArchiveNode(UnityArchiveHandle handle, int nodeIndex, StringBuilder path, int pathLen, out long size, out ArchiveNodeFlags flags); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_CreateArchive")] + public static extern ReturnCode CreateArchive([MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.LPStr)] string[] sourceFiles, + [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.LPStr)] string[] aliases, bool[] isSerializedFile, int count, + [MarshalAs(UnmanagedType.LPStr)] string archiveFile, CompressionType compression, out int crc); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_OpenFile")] + public static extern ReturnCode OpenFile([MarshalAs(UnmanagedType.LPStr)] string path, out UnityFileHandle handle); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, EntryPoint = "UFS_ReadFile")] + public static extern ReturnCode ReadFile(UnityFileHandle handle, long size, + [MarshalAs(UnmanagedType.LPArray)] byte[] buffer, out long actualSize); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_SeekFile")] + public static extern ReturnCode SeekFile(UnityFileHandle handle, long offset, SeekOrigin origin, out long newPosition); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetFileSize")] + public static extern ReturnCode GetFileSize(UnityFileHandle handle, out long size); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_CloseFile")] + public static extern ReturnCode CloseFile(IntPtr handle); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_OpenSerializedFile")] + public static extern ReturnCode OpenSerializedFile([MarshalAs(UnmanagedType.LPStr)] string path, out SerializedFileHandle handle); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_CloseSerializedFile")] + public static extern ReturnCode CloseSerializedFile(IntPtr handle); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetExternalReferenceCount")] + public static extern ReturnCode GetExternalReferenceCount(SerializedFileHandle handle, out int count); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetExternalReference")] + public static extern ReturnCode GetExternalReference(SerializedFileHandle handle, int index, StringBuilder path, int pathLen, StringBuilder guid, out ExternalReferenceType type); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetObjectCount")] + public static extern ReturnCode GetObjectCount(SerializedFileHandle handle, out int count); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetObjectInfo")] + public static extern ReturnCode GetObjectInfo(SerializedFileHandle handle, [In, Out] ObjectInfo[] objectData, int len); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetTypeTree")] + public static extern ReturnCode GetTypeTree(SerializedFileHandle handle, long objectId, out TypeTreeHandle typeTree); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetRefTypeTypeTree")] + public static extern ReturnCode GetRefTypeTypeTree(SerializedFileHandle handle, [MarshalAs(UnmanagedType.LPStr)] string className, + [MarshalAs(UnmanagedType.LPStr)] string namespaceName, [MarshalAs(UnmanagedType.LPStr)] string assemblyName, out TypeTreeHandle typeTree); + + [DllImport("UnityFileSystemApi", + CallingConvention = CallingConvention.Cdecl, + EntryPoint = "UFS_GetTypeTreeNodeInfo")] + public static extern ReturnCode GetTypeTreeNodeInfo(TypeTreeHandle handle, int node, StringBuilder type, int typeLen, + StringBuilder name, int nameLen, out int offset, out int size, [MarshalAs(UnmanagedType.U4)] out TypeTreeFlags flags, + [MarshalAs(UnmanagedType.U4)] out TypeTreeMetaFlags metaFlags, out int firstChildNode, + out int nextNode); + } +} \ No newline at end of file diff --git a/UnityFileDumper/SerializedFile.cs b/UnityFileDumper/SerializedFile.cs new file mode 100644 index 0000000..7cecf99 --- /dev/null +++ b/UnityFileDumper/SerializedFile.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Text; +namespace UnityFileApi +{ + public struct ExternalReference + { + public string Path; + public string Guid; + public ExternalReferenceType Type; + } + + public class SerializedFile : IDisposable + { + Lazy> m_ExternalReferences; + Lazy m_Objects; + + Dictionary m_TypeTreeCache = new Dictionary(); + + internal SerializedFileHandle m_Handle; + + public IReadOnlyList ExternalReferences => m_ExternalReferences.Value.AsReadOnly(); + public IReadOnlyList Objects => Array.AsReadOnly(m_Objects.Value); + + internal SerializedFile() + { + m_ExternalReferences = new Lazy>(GetExternalReferences); + m_Objects = new Lazy(GetObjects); + } + + public TypeTreeNode GetTypeTreeRoot(long objectId) + { + var r = DllWrapper.GetTypeTree(m_Handle, objectId, out var typeTreeHandle); + UnityFileSystem.HandleErrors(r); + + if (m_TypeTreeCache.TryGetValue(typeTreeHandle.Handle, out var node)) + { + return node; + } + + node = new TypeTreeNode(typeTreeHandle, 0); + m_TypeTreeCache.Add(typeTreeHandle.Handle, node); + + return node; + } + + public TypeTreeNode GetRefTypeTypeTreeRoot(string className, string namespaceName, string assemblyName) + { + var r = DllWrapper.GetRefTypeTypeTree(m_Handle, className, namespaceName, assemblyName, out var typeTreeHandle); + UnityFileSystem.HandleErrors(r); + + if (m_TypeTreeCache.TryGetValue(typeTreeHandle.Handle, out var node)) + { + return node; + } + + node = new TypeTreeNode(typeTreeHandle, 0); + m_TypeTreeCache.Add(typeTreeHandle.Handle, node); + + return node; + } + + private List GetExternalReferences() + { + var r = DllWrapper.GetExternalReferenceCount(m_Handle, out var count); + UnityFileSystem.HandleErrors(r); + + var externalReferences = new List(count); + var path = new StringBuilder(512); + var guid = new StringBuilder(32); + + for (var i = 0; i < count; ++i) + { + DllWrapper.GetExternalReference(m_Handle, i, path, path.Capacity, guid, out var externalReferenceType); + UnityFileSystem.HandleErrors(r); + + externalReferences.Add(new ExternalReference() { Path = path.ToString(), Guid = guid.ToString(), Type = externalReferenceType }); + } + + return externalReferences; + } + + private ObjectInfo[] GetObjects() + { + var r = DllWrapper.GetObjectCount(m_Handle, out var count); + UnityFileSystem.HandleErrors(r); + + if (count == 0) + return null; + + var objs = new ObjectInfo[count]; + DllWrapper.GetObjectInfo(m_Handle, objs, count); + UnityFileSystem.HandleErrors(r); + + return objs; + } + + public void Dispose() + { + if (m_Handle != null && !m_Handle.IsInvalid) + { + m_Handle.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/TextDumperTool.cs b/UnityFileDumper/TextDumperTool.cs new file mode 100644 index 0000000..9bcea30 --- /dev/null +++ b/UnityFileDumper/TextDumperTool.cs @@ -0,0 +1,432 @@ +using System; +using System.IO; +using System.Text; +namespace UnityFileApi +{ + public class TextDumperTool + { + StringBuilder m_StringBuilder = new StringBuilder(1024); + bool m_SkipLargeArrays; + UnityFileReader m_Reader; + SerializedFile m_SerializedFile; + StreamWriter m_Writer; + + public int Dump(string path, string outputPath, bool skipLargeArrays) + { + m_SkipLargeArrays = skipLargeArrays; + + try + { + try + { + using var archive = UnityFileSystem.MountArchive(path, "/"); + foreach (var node in archive.Nodes) + { + Console.WriteLine($"Processing {node.Path} {node.Size} {node.Flags}"); + + if (node.Flags.HasFlag(ArchiveNodeFlags.SerializedFile)) + { + using (m_Writer = new StreamWriter(Path.Combine(outputPath, Path.GetFileName(node.Path) + ".txt"), false)) + { + OutputSerializedFile("/" + node.Path); + } + } + } + } + catch (NotSupportedException) + { + // Try as SerializedFile + using (m_Writer = new StreamWriter(Path.Combine(outputPath, Path.GetFileName(path) + ".txt"), false)) + { + OutputSerializedFile(path); + } + } + } + catch (Exception e) + { + Console.WriteLine("Error!"); + Console.Write($"{e.GetType()}: "); + Console.WriteLine(e.Message); + Console.WriteLine(e.StackTrace); + return 1; + } + + return 0; + } + + void RecursiveDump(TypeTreeNode node, ref long offset, int level, int arrayIndex = -1) + { + bool skipChildren = false; + + if (!node.IsArray) + { + m_StringBuilder.Append(' ', level * 2); + + if (level != 0) + { + m_StringBuilder.Append(node.Name); + if (arrayIndex >= 0) + { + m_StringBuilder.Append('['); + m_StringBuilder.Append(arrayIndex); + m_StringBuilder.Append(']'); + } + m_StringBuilder.Append(' '); + m_StringBuilder.Append('('); + m_StringBuilder.Append(node.Type); + m_StringBuilder.Append(')'); + } + else + { + m_StringBuilder.Append(node.Type); + } + + // Basic data type. + if (node.IsBasicType) + { + m_StringBuilder.Append(' '); + m_StringBuilder.Append(ReadValue(node, offset)); + + offset += node.Size; + } + else if (node.Type == "string") + { + var stringSize = m_Reader.ReadInt32(offset); + + m_StringBuilder.Append(' '); + m_StringBuilder.Append(m_Reader.ReadString(offset + 4, stringSize)); + + offset += stringSize + 4; + + // Skip child nodes as they were already processed here. + skipChildren = true; + } + + m_Writer.WriteLine(m_StringBuilder); + m_StringBuilder.Clear(); + + if (node.IsManagedReferenceRegistry) + { + DumpManagedReferenceRegistry(node, ref offset, level + 1); + + // Skip child nodes as they were already processed here. + skipChildren = true; + } + } + else + { + DumpArray(node, ref offset, level); + + // Skip child nodes as they were already processed here. + skipChildren = true; + } + + if (!skipChildren) + { + foreach (var child in node.Children) + { + RecursiveDump(child, ref offset, level + 1); + } + } + + if ( + ((int)node.MetaFlags & (int)TypeTreeMetaFlags.AlignBytes) != 0 || + ((int)node.MetaFlags & (int)TypeTreeMetaFlags.AnyChildUsesAlignBytes) != 0 + ) + { + offset = (offset + 3) & ~(3); + } + } + + void DumpArray(TypeTreeNode node, ref long offset, int level) + { + // First child contains array size. + var sizeNode = node.Children[0]; + // Second child contains array type information. + var dataNode = node.Children[1]; + + if (sizeNode.Size != 4 || !sizeNode.IsLeaf) + throw new Exception("Invalid array size"); + + var arraySize = m_Reader.ReadInt32(offset); + offset += 4; + + m_StringBuilder.Append(' ', level * 2); + m_StringBuilder.Append("Array"); + m_StringBuilder.Append('<'); + m_StringBuilder.Append(dataNode.Type); + m_StringBuilder.Append(">["); + m_StringBuilder.Append(arraySize); + m_StringBuilder.Append(']'); + + m_Writer.WriteLine(m_StringBuilder); + m_StringBuilder.Clear(); + + if (arraySize > 0) + { + if (dataNode.IsBasicType) + { + m_StringBuilder.Append(' ', (level + 1) * 2); + + if (arraySize > 256 && m_SkipLargeArrays) + { + m_StringBuilder.Append(""); + offset += dataNode.Size * arraySize; + } + else + { + var array = ReadBasicTypeArray(dataNode, offset, arraySize); + offset += dataNode.Size * arraySize; + + m_StringBuilder.Append(array.GetValue(0)); + for (int i = 1; i < arraySize; ++i) + { + m_StringBuilder.Append(", "); + m_StringBuilder.Append(array.GetValue(i)); + } + } + + m_Writer.WriteLine(m_StringBuilder); + m_StringBuilder.Clear(); + } + else + { + ++level; + + for (int i = 0; i < arraySize; ++i) + { + RecursiveDump(dataNode, ref offset, level, i); + } + } + } + } + + void DumpManagedReferenceRegistry(TypeTreeNode node, ref long offset, int level) + { + if (node.Children.Count < 2) + throw new Exception("Invalid ManagedReferenceRegistry"); + + // First child is version number. + var version = m_Reader.ReadInt32(offset); + RecursiveDump(node.Children[0], ref offset, level); + + TypeTreeNode refTypeNode; + TypeTreeNode refObjData; + + if (version == 1) + { + // Second child is the ReferencedObject. + var refObjNode = node.Children[1]; + // And its children are the referenced type and data nodes. + refTypeNode = refObjNode.Children[0]; + refObjData = refObjNode.Children[1]; + + int i = 0; + + while (DumpManagedReferenceData(refTypeNode, refObjData, ref offset, level, i++)) + { } + } + else if (version == 2) + { + // Second child is the RefIds vector. + var refIdsVectorNode = node.Children[1]; + + if (refIdsVectorNode.Children.Count < 1 || refIdsVectorNode.Name != "RefIds") + throw new Exception("Invalid ManagedReferenceRegistry RefIds vector"); + + var refIdsArrayNode = refIdsVectorNode.Children[0]; + + if (refIdsArrayNode.Children.Count != 2 || !refIdsArrayNode.IsArray) + throw new Exception("Invalid ManagedReferenceRegistry RefIds array"); + + // First child is the array size. + int arraySize = m_Reader.ReadInt32(offset); + offset += 4; + + // Second child is the ReferencedObject. + var refObjNode = refIdsArrayNode.Children[1]; + + for (int i = 0; i < arraySize; ++i) + { + // First child is the rid. + long rid = m_Reader.ReadInt64(offset); + offset += 8; + + // And the next children are the referenced type and data nodes. + refTypeNode = refObjNode.Children[1]; + refObjData = refObjNode.Children[2]; + DumpManagedReferenceData(refTypeNode, refObjData, ref offset, level, rid); + } + } + else + { + throw new Exception($"Unsupported ManagedReferenceRegistry version {version}"); + } + } + + bool DumpManagedReferenceData(TypeTreeNode refTypeNode, TypeTreeNode referencedTypeDataNode, ref long offset, int level, long id) + { + if (refTypeNode.Children.Count < 3) + throw new Exception("Invalid ReferencedManagedType"); + + m_StringBuilder.Append(' ', level * 2); + m_StringBuilder.Append($"rid("); + m_StringBuilder.Append(id); + m_StringBuilder.Append(") ReferencedObject"); + + m_Writer.WriteLine(m_StringBuilder); + m_StringBuilder.Clear(); + + ++level; + + var refTypeOffset = offset; + var stringSize = m_Reader.ReadInt32(offset); + var className = m_Reader.ReadString(offset + 4, stringSize); + offset += stringSize + 4; + offset = (offset + 3) & ~(3); + + stringSize = m_Reader.ReadInt32(offset); + var namespaceName = m_Reader.ReadString(offset + 4, stringSize); + offset += stringSize + 4; + offset = (offset + 3) & ~(3); + + stringSize = m_Reader.ReadInt32(offset); + var assemblyName = m_Reader.ReadString(offset + 4, stringSize); + offset += stringSize + 4; + offset = (offset + 3) & ~(3); + + if (className == "Terminus" && namespaceName == "UnityEngine.DMAT" && assemblyName == "FAKE_ASM") + return false; + + // Not the most efficient way, but it simplifies the code. + RecursiveDump(refTypeNode, ref refTypeOffset, level); + + m_StringBuilder.Append(' ', level * 2); + m_StringBuilder.Append(referencedTypeDataNode.Name); + m_StringBuilder.Append(' '); + m_StringBuilder.Append(referencedTypeDataNode.Type); + m_StringBuilder.Append(' '); + + m_Writer.WriteLine(m_StringBuilder); + m_StringBuilder.Clear(); + + if (id == -1 || id == -2) + { + m_StringBuilder.Append(' ', level * 2); + m_StringBuilder.Append(id == -1 ? " unknown" : " null"); + + m_Writer.WriteLine(m_StringBuilder); + m_StringBuilder.Clear(); + + return true; + } + + var refTypeRoot = m_SerializedFile.GetRefTypeTypeTreeRoot(className, namespaceName, assemblyName); + + // Dump the ReferencedObject using its own TypeTree, but skip the root. + foreach (var child in refTypeRoot.Children) + { + RecursiveDump(child, ref offset, level + 1); + } + + return true; + } + + void OutputSerializedFile(string path) + { + using (m_Reader = new UnityFileReader(path, 64 * 1024 * 1024)) + using (m_SerializedFile = UnityFileSystem.OpenSerializedFile(path)) + { + var i = 1; + + m_Writer.WriteLine("External References"); + foreach (var extRef in m_SerializedFile.ExternalReferences) + { + m_Writer.WriteLine($"path({i}): \"{extRef.Path}\" GUID: {extRef.Guid} Type: {(int)extRef.Type}"); + ++i; + } + m_Writer.WriteLine(); + + foreach (var obj in m_SerializedFile.Objects) + { + var root = m_SerializedFile.GetTypeTreeRoot(obj.Id); + var offset = obj.Offset; + + m_Writer.Write($"ID: {obj.Id} (ClassID: {obj.TypeId}) "); + RecursiveDump(root, ref offset, 0); + m_Writer.WriteLine(); + } + } + } + + string ReadValue(TypeTreeNode node, long offset) + { + switch (Type.GetTypeCode(node.CSharpType)) + { + case TypeCode.Int32: + return m_Reader.ReadInt32(offset).ToString(); + + case TypeCode.UInt32: + return m_Reader.ReadUInt32(offset).ToString(); + + case TypeCode.Single: + return m_Reader.ReadFloat(offset).ToString(); + + case TypeCode.Double: + return m_Reader.ReadDouble(offset).ToString(); + + case TypeCode.Int16: + return m_Reader.ReadInt16(offset).ToString(); + + case TypeCode.UInt16: + return m_Reader.ReadUInt16(offset).ToString(); + + case TypeCode.Int64: + return m_Reader.ReadInt64(offset).ToString(); + + case TypeCode.UInt64: + return m_Reader.ReadUInt64(offset).ToString(); + + case TypeCode.SByte: + return m_Reader.ReadUInt8(offset).ToString(); + + case TypeCode.Byte: + case TypeCode.Char: + return m_Reader.ReadUInt8(offset).ToString(); + + case TypeCode.Boolean: + return (m_Reader.ReadUInt8(offset) != 0).ToString(); + + default: + throw new Exception($"Can't get value of {node.Type} type"); + } + } + + Array ReadBasicTypeArray(TypeTreeNode node, long offset, int arraySize) + { + // Special case for boolean arrays. + if (node.CSharpType == typeof(bool)) + { + var tmpArray = new byte[arraySize]; + var boolArray = new bool[arraySize]; + + m_Reader.ReadArray(offset, arraySize * node.Size, tmpArray); + + for (int i = 0; i < arraySize; ++i) + { + boolArray[i] = tmpArray[i] != 0; + } + + return boolArray; + } + else + { + var array = Array.CreateInstance(node.CSharpType, arraySize); + + m_Reader.ReadArray(offset, arraySize * node.Size, array); + + return array; + } + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/TypeTreeNode.cs b/UnityFileDumper/TypeTreeNode.cs new file mode 100644 index 0000000..9b97b07 --- /dev/null +++ b/UnityFileDumper/TypeTreeNode.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace UnityFileApi +{ + // A TypeTreeNode represents how a property of a serialized object was written to disk. + // See the TextDumper library for an example. + public class TypeTreeNode + { + int m_FirstChildNodeIndex; + int m_NextNodeIndex; + TypeTreeHandle m_Handle; + Lazy> m_Children; + Lazy m_CSharpType; + Lazy m_hasConstantSize; + + // The type of the property (basic type like int or float or class names for objects) + public readonly string Type; + // The name of the property (e.g. m_IndexBuffer for a Mesh or m_Width for a Texture) + public readonly string Name; + // The size of the property (for basic types only, otherwise -1) + public readonly int Size; + // The offset of the property (mostly useless). + public readonly int Offset; + // Flags used for different things (e.g. field is an array, data alignment, etc.) + public readonly TypeTreeFlags Flags; + public readonly TypeTreeMetaFlags MetaFlags; + + // Child nodes container. + public List Children => m_Children.Value; + + // True if the field has no child. + public bool IsLeaf => m_FirstChildNodeIndex == 0; + + // True if the field is a basic type. (int, float, char, etc.) + public bool IsBasicType => IsLeaf && Size > 0; + + // True if the field is an array. + public bool IsArray => ((int)Flags & (int)TypeTreeFlags.IsArray) != 0; + + // True if the field is a ManagedReferenceRegistry + public bool IsManagedReferenceRegistry => ((int)Flags & (int)TypeTreeFlags.IsManagedReferenceRegistry) != 0; + + // C# type corresponding to the node type + public Type CSharpType => m_CSharpType.Value; + + // True if the node has a constant size (it contains no array or other containers with variable size). + public bool HasConstantSize => m_hasConstantSize.Value; + + [ThreadStatic] + static StringBuilder s_NodeType; + [ThreadStatic] + static StringBuilder s_NodeName; + + // Properties are required to initialize the ThreadStatic members. + static StringBuilder NodeTypeBuilder + { + get + { + if (s_NodeType == null) + { + s_NodeType = new StringBuilder(512); + } + return s_NodeType; + } + } + + static StringBuilder NodeNameBuilder + { + get + { + if (s_NodeName == null) + { + s_NodeName = new StringBuilder(512); + } + return s_NodeName; + } + } + + internal TypeTreeNode(TypeTreeHandle typeTreeHandle, int nodeIndex) + { + m_Handle = typeTreeHandle; + + var r = DllWrapper.GetTypeTreeNodeInfo(m_Handle, nodeIndex, NodeTypeBuilder, NodeTypeBuilder.Capacity, NodeNameBuilder, NodeNameBuilder.Capacity, out Offset, out Size, out Flags, out MetaFlags, out m_FirstChildNodeIndex, out m_NextNodeIndex); + UnityFileSystem.HandleErrors(r); + + Type = NodeTypeBuilder.ToString(); + Name = NodeNameBuilder.ToString(); + + m_Children = new Lazy>(GetChildren); + m_CSharpType = new Lazy(GetCSharpType); + m_hasConstantSize = new Lazy(GetHasConstantSize); + } + + internal List GetChildren() + { + var children = new List(); + var current = m_FirstChildNodeIndex; + + while (current != 0) + { + var child = new TypeTreeNode(m_Handle, current); + children.Add(child); + current = child.m_NextNodeIndex; + } + + return children; + } + + bool GetHasConstantSize() + { + if (IsArray || CSharpType == typeof(string)) + return false; + + foreach (var child in Children) + { + if (!child.HasConstantSize) + return false; + } + + return true; + } + + Type GetCSharpType() + { + switch (Type) + { + case "int": + case "SInt32": + case "TypePtr": + return typeof(int); + + case "unsigned int": + case "UInt32": + return typeof(uint); + + case "float": + return typeof(float); + + case "double": + return typeof(double); + + case "SInt16": + return typeof(short); + + case "UInt16": + return typeof(ushort); + + case "SInt64": + return typeof(long); + + case "FileSize": + case "UInt64": + return typeof(ulong); + + case "SInt8": + return typeof(sbyte); + + case "UInt8": + case "char": + return typeof(byte); + + case "bool": + return typeof(bool); + + case "string": + return typeof(string); + + default: + { + if (Size == 8) + { + return typeof(long); + } + else if (Size == 4) + { + return typeof(int); + } + else if (Size == 2) + { + return typeof(short); + } + else if (Size == 1) + { + return typeof(sbyte); + } + + return typeof(object); + } + } + + throw new Exception($"Unknown type {Type}"); + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/UnityArchive.cs b/UnityFileDumper/UnityArchive.cs new file mode 100644 index 0000000..6a201b6 --- /dev/null +++ b/UnityFileDumper/UnityArchive.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Text; +namespace UnityFileApi +{ + + // An archive node is a file in an archive. + public struct ArchiveNode + { + public string Path; + public long Size; + public ArchiveNodeFlags Flags; + } + + // Class used to open a Unity archive file (such as an AssetBundle). + public class UnityArchive : IDisposable + { + internal UnityArchiveHandle m_Handle; + Lazy> m_Nodes; + + public IReadOnlyList Nodes => m_Nodes.Value.AsReadOnly(); + + internal UnityArchive() + { + m_Nodes = new Lazy>(() => GetArchiveNodes()); + } + + List GetArchiveNodes() + { + var r = DllWrapper.GetArchiveNodeCount(m_Handle, out var count); + UnityFileSystem.HandleErrors(r); + + if (count == 0) + return null; + + var nodes = new List(count); + var path = new StringBuilder(512); + + for (var i = 0; i < count; ++i) + { + DllWrapper.GetArchiveNode(m_Handle, i, path, path.Capacity, out var size, out var flags); + UnityFileSystem.HandleErrors(r); + + nodes.Add(new ArchiveNode() { Path = path.ToString(), Size = size, Flags = flags }); + } + + return nodes; + } + + public void Dispose() + { + if (m_Handle != null && !m_Handle.IsInvalid) + { + m_Handle.Dispose(); + m_Nodes = new Lazy>(() => GetArchiveNodes()); + } + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/UnityFile.cs b/UnityFileDumper/UnityFile.cs new file mode 100644 index 0000000..9426f05 --- /dev/null +++ b/UnityFileDumper/UnityFile.cs @@ -0,0 +1,46 @@ +using System; +namespace UnityFileApi +{ + // Use this class to read data from a Unity file. + public class UnityFile : IDisposable + { + internal UnityFile() + { + } + + internal UnityFileHandle m_Handle; + + public long Read(long size, byte[] buffer) + { + var r = DllWrapper.ReadFile(m_Handle, size, buffer, out var actualSize); + UnityFileSystem.HandleErrors(r); + + return actualSize; + } + + public long Seek(long offset, SeekOrigin origin = SeekOrigin.Begin) + { + var r = DllWrapper.SeekFile(m_Handle, offset, origin, out var newPosition); + UnityFileSystem.HandleErrors(r); + + return newPosition; + } + public long GetSize() + { + // This could be a property but as it may throw an exception, it's probably better as a method. + + var r = DllWrapper.GetFileSize(m_Handle, out var size); + UnityFileSystem.HandleErrors(r); + + return size; + } + + public void Dispose() + { + if (m_Handle != null && !m_Handle.IsInvalid) + { + m_Handle.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/UnityFileDumper.csproj b/UnityFileDumper/UnityFileDumper.csproj new file mode 100644 index 0000000..7a56e4f --- /dev/null +++ b/UnityFileDumper/UnityFileDumper.csproj @@ -0,0 +1,11 @@ + + + + netstandard2.1 + + + + + + + diff --git a/UnityFileDumper/UnityFileReader.cs b/UnityFileDumper/UnityFileReader.cs new file mode 100644 index 0000000..189db52 --- /dev/null +++ b/UnityFileDumper/UnityFileReader.cs @@ -0,0 +1,139 @@ +using System; +using System.IO; +using System.Text; +using Force.Crc32; +namespace UnityFileApi +{ + // This class can be used to read typed data from a UnityFile. Is uses a buffer for better performance. + public class UnityFileReader : IDisposable + { + UnityFile m_File; + byte[] m_Buffer; + long m_BufferStartInFile; + long m_BufferEndInFile; + + public long Length { get; } + + public UnityFileReader(string path, int bufferSize) + { + m_Buffer = new byte[bufferSize]; + m_BufferStartInFile = 0; + m_BufferEndInFile = 0; + + m_File = UnityFileSystem.OpenFile(path); + Length = m_File.GetSize(); + } + + int GetBufferOffset(long fileOffset, int count) + { + // Should we update the buffer? + if (fileOffset < m_BufferStartInFile || fileOffset + count > m_BufferEndInFile) + { + if (count > m_Buffer.Length) + throw new IOException("Requested size is larger than cache size"); + + m_BufferStartInFile = m_File.Seek(fileOffset); + + if (m_BufferStartInFile != fileOffset) + throw new IOException("Invalid file offset"); + + m_BufferEndInFile = m_File.Read(m_Buffer.Length, m_Buffer); + m_BufferEndInFile += m_BufferStartInFile; + } + + return (int)(fileOffset - m_BufferStartInFile); + } + + public void ReadArray(long fileOffset, int size, Array dest) + { + var offset = GetBufferOffset(fileOffset, size); + Buffer.BlockCopy(m_Buffer, offset, dest, 0, size); + } + + public string ReadString(long fileOffset, int size) + { + var offset = GetBufferOffset(fileOffset, size); + return Encoding.Default.GetString(m_Buffer, offset, size); + } + + public float ReadFloat(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 4); + return BitConverter.ToSingle(m_Buffer, offset); + } + + public double ReadDouble(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 8); + return BitConverter.ToDouble(m_Buffer, offset); + } + + public long ReadInt64(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 8); + return BitConverter.ToInt64(m_Buffer, offset); + } + + public ulong ReadUInt64(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 8); + return BitConverter.ToUInt64(m_Buffer, offset); + } + + public int ReadInt32(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 4); + return BitConverter.ToInt32(m_Buffer, offset); + } + + public uint ReadUInt32(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 4); + return BitConverter.ToUInt32(m_Buffer, offset); + } + + public short ReadInt16(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 2); + return BitConverter.ToInt16(m_Buffer, offset); + } + + public ushort ReadUInt16(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 2); + return BitConverter.ToUInt16(m_Buffer, offset); + } + + public sbyte ReadInt8(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 1); + return (sbyte)m_Buffer[offset]; + } + + public byte ReadUInt8(long fileOffset) + { + var offset = GetBufferOffset(fileOffset, 1); + return m_Buffer[offset]; + } + + public uint ComputeCRC(long fileOffset, int size, uint crc32 = 0) + { + var readSize = size > m_Buffer.Length ? m_Buffer.Length : size; + var readBytes = 0; + + while (readBytes < size) + { + var offset = GetBufferOffset(fileOffset, readSize); + crc32 = Crc32Algorithm.Append(crc32, m_Buffer, offset, readSize); + readBytes += readSize; + } + + return crc32; + } + + public void Dispose() + { + m_File.Dispose(); + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/UnityFileSystem.cs b/UnityFileDumper/UnityFileSystem.cs new file mode 100644 index 0000000..cd5a126 --- /dev/null +++ b/UnityFileDumper/UnityFileSystem.cs @@ -0,0 +1,93 @@ +using System; +using System.IO; +namespace UnityFileApi +{ + // This is the main entry point. Provides methods to mount archives and open files. + public static class UnityFileSystem + { + public static void Init() + { + // Initialize the native library. + var r = DllWrapper.Init(); + + if (r != ReturnCode.Success && r != ReturnCode.AlreadyInitialized) + { + HandleErrors(r); + } + } + + public static void Cleanup() + { + // Uninitialize the native library. + var r = DllWrapper.Cleanup(); + + if (r != ReturnCode.Success && r != ReturnCode.NotInitialized) + { + HandleErrors(r); + } + } + + public static UnityArchive MountArchive(string path, string mountPoint) + { + var r = DllWrapper.MountArchive(path, mountPoint, out var handle); + HandleErrors(r, path); + + return new UnityArchive() { m_Handle = handle }; + } + + public static UnityFile OpenFile(string path) + { + var r = DllWrapper.OpenFile(path, out var handle); + UnityFileSystem.HandleErrors(r, path); + + return new UnityFile() { m_Handle = handle }; + } + + public static SerializedFile OpenSerializedFile(string path) + { + var r = DllWrapper.OpenSerializedFile(path, out var handle); + UnityFileSystem.HandleErrors(r, path); + + return new SerializedFile() { m_Handle = handle }; + } + + internal static void HandleErrors(ReturnCode returnCode, string filename = "") + { + switch (returnCode) + { + case ReturnCode.AlreadyInitialized: + throw new InvalidOperationException("UnityFileSystem is already initialized."); + + case ReturnCode.NotInitialized: + throw new InvalidOperationException("UnityFileSystem is not initialized."); + + case ReturnCode.FileNotFound: + throw new FileNotFoundException("File not found.", filename); + + case ReturnCode.FileFormatError: + throw new NotSupportedException($"Invalid file format reading {filename}."); + + case ReturnCode.InvalidArgument: + throw new ArgumentException(); + + case ReturnCode.HigherSerializedFileVersion: + throw new NotSupportedException("SerializedFile version not supported."); + + case ReturnCode.DestinationBufferTooSmall: + throw new ArgumentException("Destination buffer too small."); + + case ReturnCode.InvalidObjectId: + throw new ArgumentException("Invalid object id."); + + case ReturnCode.UnknownError: + throw new Exception("Unknown error."); + + case ReturnCode.FileError: + throw new IOException("File operation error."); + + case ReturnCode.TypeNotFound: + throw new ArgumentException("Type not found."); + } + } + } +} \ No newline at end of file diff --git a/UnityFileDumper/UnityFileSystemApi.dll b/UnityFileDumper/UnityFileSystemApi.dll new file mode 100644 index 0000000..fa1460d Binary files /dev/null and b/UnityFileDumper/UnityFileSystemApi.dll differ