[#99] Resolve ContentDirectory refs when content files lack the .cf extension

ContentLayout reference resolution keyed content files as <hash>.cf, but the
analyzed file was keyed by its actual on-disk name. Builds whose archive entries
omit the extension (just <hash>) never matched, so references fell through to the
.cfid placeholders in the external reference table, producing phantom .cfid rows
in serialized_files (NULL archive) and a flood of dangling_refs.

Normalize content-file names to the bare content hash (strip a trailing .cf) at
every site where the name becomes a serialized-file id key, so a content file
resolves to the same id whether or not the extension is present on disk.

Author: SkowronskiAndrew

Analyzer/SQLite/Writers/ContentLayoutSQLWriter.cs

@@ -189,9 +189,11 @@ public void WriteContentLayout(string filename, ContentLayout layout)
         }
 
         // Fills the shared dependency map that resolves the external references of the analyzed
-        // .cf files (see ContentFileDependencyMap): for each content file, the dependency indexes
-        // become the target filenames, in order. Built-in dependencies (no content hash) map to
-        // null so the reference falls back to the external table path.
+        // content files (see ContentFileDependencyMap): for each content file, the dependency
+        // indexes become the target names, in order. Built-in dependencies (no content hash) map to
+        // null so the reference falls back to the external table path. Keys and resolved names use
+        // the normalized content-hash name so they match the analyzed files regardless of whether
+        // those carry the ".cf" extension on disk.
         private void PopulateContentFileDependencies(ContentLayout layout)
         {
             var hashByIndex = (layout.SerializedFiles ?? []).ToDictionary(f => f.Index, f => f.ContentHash);
@@ -203,21 +205,24 @@ private void PopulateContentFileDependencies(ContentLayout layout)
 
                 var resolved = (file.SerializedFileDependencies ?? [])
                     .Select(i => hashByIndex.TryGetValue(i, out var hash) && !string.IsNullOrEmpty(hash)
-                        ? hash + ".cf"
+                        ? ContentFileDependencyMap.NormalizeFileName(hash)
                         : null)
                     .ToArray();
 
-                m_ContentFileDependencies.Add(file.ContentHash + ".cf", resolved);
+                m_ContentFileDependencies.Add(ContentFileDependencyMap.NormalizeFileName(file.ContentHash), resolved);
             }
         }
 
         // Fills in the serialized_file column, linking each layout entry to its serialized_files
-        // row. Called after all files are processed. Ids come from the shared IdProvider, so
-        // entries whose .cf file was analyzed link to the row the analyze pass wrote (loose or
-        // inside an archive, with its archive column intact). For files never encountered on the
-        // input (a layout-only analyze, or a subset of a build) a placeholder serialized_files
-        // row is written - name only, archive NULL, no objects - so the link is always valid,
-        // mirroring what the dangling-refs finalize does for referenced-but-unanalyzed files.
+        // row. Called after all files are processed. Ids come from the shared IdProvider (keyed by
+        // the normalized content hash), so entries whose content file was analyzed link to the row
+        // the analyze pass wrote (loose or inside an archive, with its archive column intact),
+        // regardless of whether that file carried the ".cf" extension on disk. For files never
+        // encountered on the input (a layout-only analyze, or a subset of a build) a placeholder
+        // serialized_files row is written - name only, archive NULL, no objects - so the link is
+        // always valid, mirroring what the dangling-refs finalize does for referenced-but-unanalyzed
+        // files. The placeholder's name uses the canonical "<hash>.cf" form since the file was not
+        // seen on disk.
         public void LinkSerializedFiles()
         {
             var existingIds = new HashSet<int>();
@@ -246,8 +251,8 @@ public void LinkSerializedFiles()
             {
                 foreach (var file in m_ImportedFiles)
                 {
-                    var fileName = file.ContentHash + ".cf";
-                    var id = m_SerializedFileIdProvider.GetId(fileName);
+                    var id = m_SerializedFileIdProvider.GetId(
+                        ContentFileDependencyMap.NormalizeFileName(file.ContentHash));
 
                     update.Parameters["@id"].Value = id;
                     update.Parameters["@file_index"].Value = file.Index;
@@ -257,7 +262,7 @@ public void LinkSerializedFiles()
                     {
                         addStubRow.SetValue("id", id);
                         addStubRow.SetValue("archive", null);
-                        addStubRow.SetValue("name", fileName);
+                        addStubRow.SetValue("name", file.ContentHash + ".cf");
                         addStubRow.ExecuteNonQuery();
                     }
                 }

Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs

@@ -31,8 +31,9 @@ public class SerializedFileSQLiteWriter : IDisposable
     private bool m_SkipCrc;
 
     // Global id assignment shared across every serialized file in the database.
-    // m_SerializedFileIdProvider maps a serialized file (by lowercased file name) to its
-    // serialized_files row id; it is owned by AnalyzerTool because the ContentLayout import
+    // m_SerializedFileIdProvider maps a serialized file (by file name, matched case-sensitively as
+    // it appears on disk / in the archive, with any ".cf" content-file extension normalized away)
+    // to its serialized_files row id; it is owned by AnalyzerTool because the ContentLayout import
     // assigns ids through the same provider. m_ObjectIdProvider maps a (serialized file id,
     // pathId) pair to its objects row id. See ObjectIdProvider for how cross-file references
     // are resolved.
@@ -185,7 +186,8 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
         using var sf = UnityFileSystem.OpenSerializedFile(fullPath);
         using var reader = new UnityFileReader(fullPath, 64 * 1024 * 1024);
         using var pptrReader = new PPtrAndCrcProcessor(sf, reader, containingFolder, m_SkipCrc, AddReference);
-        int serializedFileId = m_SerializedFileIdProvider.GetId(Path.GetFileName(fullPath));
+        int serializedFileId = m_SerializedFileIdProvider.GetId(
+            ContentFileDependencyMap.NormalizeFileName(Path.GetFileName(fullPath)));
         int sceneId = -1;
 
         // Two SerializedFiles with the same name map to the same id (the provider deduplicates by
@@ -278,7 +280,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
             // dependency list, matched by position; built-in dependencies (null entries) keep the
             // external table path.
             var resolvedDependencies = m_ContentFileDependencies.GetDependencies(
-                Path.GetFileName(fullPath));
+                ContentFileDependencyMap.NormalizeFileName(Path.GetFileName(fullPath)));
 
             if (resolvedDependencies != null && resolvedDependencies.Length != sf.ExternalReferences.Count)
             {

Analyzer/Util/ContentFileDependencyMap.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 
 namespace UnityDataTools.Analyzer.Util;
@@ -6,10 +7,10 @@ namespace UnityDataTools.Analyzer.Util;
 // In ContentDirectory builds the external reference table inside a SerializedFile holds symbolic
 // .cfid placeholders instead of real filenames; the actual target of a reference is determined
 // positionally through the layout's dependency list (see Documentation/contentdirectory-format.md).
-// This map holds, for each content file (keyed by its "<contenthash>.cf" filename), the resolved
-// dependency filenames in external-reference-table order. An entry is null where the
-// dependency is a built-in file (which has no content hash); such references fall back to the
-// path from the external reference table.
+// This map holds, for each content file (keyed by its normalized content-hash name, see
+// NormalizeFileName), the resolved dependency names in external-reference-table order. An entry is
+// null where the dependency is a built-in file (which has no content hash); such references fall
+// back to the path from the external reference table.
 public class ContentFileDependencyMap
 {
     private Dictionary<string, string[]> m_Dependencies = new();
@@ -19,10 +20,21 @@ public void Add(string fileName, string[] resolvedDependencies)
         m_Dependencies[fileName] = resolvedDependencies;
     }
 
-    // Returns the resolved dependency filenames for the given filename, or null if the file is
-    // not covered by an imported ContentLayout.
+    // Returns the resolved dependency names for the given (normalized) content-file name, or null
+    // if the file is not covered by an imported ContentLayout.
     public string[] GetDependencies(string fileName)
     {
         return m_Dependencies.TryGetValue(fileName, out var dependencies) ? dependencies : null;
     }
+
+    // Content files are named by their content hash and may or may not carry the ".cf" extension:
+    // loose builds and archived builds vary, and the extension is informational (see
+    // Documentation/contentdirectory-format.md). Strip it so a content file resolves to the same
+    // serialized-file id whether or not the extension is present on disk. Non-content files never
+    // end in ".cf", so this is a no-op for them. The name's case is preserved, matching how
+    // serialized-file names are keyed case-sensitively elsewhere.
+    public static string NormalizeFileName(string fileName)
+    {
+        return fileName.EndsWith(".cf", StringComparison.OrdinalIgnoreCase) ? fileName[..^3] : fileName;
+    }
 }

Documentation/contentdirectory-format.md

@@ -65,9 +65,10 @@ A Unity object references data inside a `.resS`/`.resource` file by a content-ad
 form `cah:/<hash>`. This, and the way it differs from AssetBundle and Player builds, is covered in
 [References to resource files](#references-to-resource-files) below.
 
-The extensions are informational. The loading system identifies content by its hash (through the
-`cah:/` scheme), not by file extension, so the extension is not required to resolve content. Extensions
-are present both for loose files and for entries packed inside an archive.
+The extensions are informational. The build manifest, content layout and loading system as a whole identifies content by its hash, not by file extension, and the extension is not required to resolve content. 
+
+Currently extensions are present both for loose files and for entries packed inside an archive, but
+it is best to avoid assumption that the extensions are always present.
 
 ### The build manifest