Improve trace reconciliation performance (#15725)

bendikberg · mjkkirschner · web-flow · commit dec6452a7ed4 · 2025-02-14T15:58:59.000-05:00
Co-authored-by: mjk.kirschner &lt;508936+mjkkirschner@users.noreply.github.com&gt;
diff --git a/src/DynamoCore/Engine/EngineController.cs b/src/DynamoCore/Engine/EngineController.cs
@@ -534,14 +534,14 @@ internal void ReconcileTraceDataAndNotify()
             var callsiteToOrphanMap = new Dictionary<Guid, List<string>>();
             foreach (var cs in liveRunnerServices.RuntimeCore.RuntimeData.CallsiteCache.Values)
             {
-                var orphanedSerializables = cs.GetOrphanedSerializables().ToList();
-                if (callsiteToOrphanMap.ContainsKey(cs.CallSiteID))
+                var orphanedSerializables = cs.GetOrphanedSerializables();
+                if (callsiteToOrphanMap.TryGetValue(cs.CallSiteID, out var serializablesForCallsite))
                 {
-                    callsiteToOrphanMap[cs.CallSiteID].AddRange(orphanedSerializables);
+                    serializablesForCallsite.AddRange(orphanedSerializables);
                 }
                 else
                 {
-                    callsiteToOrphanMap.Add(cs.CallSiteID, orphanedSerializables);
+                    callsiteToOrphanMap.Add(cs.CallSiteID, orphanedSerializables.ToList());
                 }
             }
 
diff --git a/src/DynamoCore/Graph/Workspaces/HomeWorkspaceModel.cs b/src/DynamoCore/Graph/Workspaces/HomeWorkspaceModel.cs
@@ -880,7 +880,7 @@ private void OnPreviewGraphCompleted(AsyncTask asyncTask)
         /// trace data but do not exist in the current CallSite data.
         /// </summary>
         /// <returns></returns>
-        internal IList<string> GetOrphanedSerializablesAndClearHistoricalTraceData()
+        internal List<string> GetOrphanedSerializablesAndClearHistoricalTraceData()
         {
             var orphans = new List<string>();
 
@@ -891,13 +891,14 @@ internal IList<string> GetOrphanedSerializablesAndClearHistoricalTraceData()
             // then add the serializables for that guid to the list of
             // orphans.
 
+            var nodeLookup = Nodes.Select(n => n.GUID).ToHashSet();
             foreach (var nodeData in historicalTraceData)
             {
                 var nodeGuid = nodeData.Key;
 
-                if (Nodes.All(n => n.GUID != nodeGuid))
+                if (!nodeLookup.Contains(nodeGuid))
                 {
-                    orphans.AddRange(nodeData.Value.SelectMany(CallSite.GetAllSerializablesFromSingleRunTraceData).ToList());
+                    orphans.AddRange(nodeData.Value.SelectMany(CallSite.GetAllSerializablesFromSingleRunTraceData));
                 }
             }
 
diff --git a/src/DynamoCore/Models/DynamoModel.cs b/src/DynamoCore/Models/DynamoModel.cs
@@ -1276,22 +1276,35 @@ private void EngineController_TraceReconcliationComplete(TraceReconciliationEven
             // This dictionary gets redistributed into a dictionary keyed by the workspace id.
 
             var workspaceOrphanMap = new Dictionary<Guid, List<string>>();
+            var nodeToWorkspaceMap = new Dictionary<Guid, WorkspaceModel>();
 
-            foreach (var ws in Workspaces.OfType<HomeWorkspaceModel>())
+            foreach (var maybeWs in Workspaces)
             {
+                foreach (var node in maybeWs.Nodes)
+                {
+                    nodeToWorkspaceMap[node.GUID] = maybeWs;
+                }
+
+                if (maybeWs is not HomeWorkspaceModel ws)
+                {
+                    continue;
+                }
+
                 // Get the orphaned serializables to this workspace
-                var wsOrphans = ws.GetOrphanedSerializablesAndClearHistoricalTraceData().ToList();
+                var wsOrphans = ws.GetOrphanedSerializablesAndClearHistoricalTraceData();
 
                 if (!wsOrphans.Any())
+                {
                     continue;
+                }
 
-                if (!workspaceOrphanMap.ContainsKey(ws.Guid))
+                if (workspaceOrphanMap.TryGetValue(ws.Guid, out var workspaceOrphans))
                 {
-                    workspaceOrphanMap.Add(ws.Guid, wsOrphans);
+                    workspaceOrphans.AddRange(wsOrphans);
                 }
                 else
                 {
-                    workspaceOrphanMap[ws.Guid].AddRange(wsOrphans);
+                    workspaceOrphanMap.Add(ws.Guid, wsOrphans);
                 }
             }
 
@@ -1303,23 +1316,20 @@ private void EngineController_TraceReconcliationComplete(TraceReconciliationEven
 
                 // TODO: MAGN-7314
                 // Find the owning workspace for a node.
-                var nodeSpace =
-                    Workspaces.FirstOrDefault(
-                        ws =>
-                            ws.Nodes.FirstOrDefault(n => n.GUID == nodeGuid)
-                                != null);
-
-                if (nodeSpace == null) continue;
+                if (!nodeToWorkspaceMap.TryGetValue(nodeGuid, out var nodeSpace))
+                {
+                    continue;
+                }
 
                 // Add the node's orphaned serializables to the workspace
                 // orphan map.
-                if (workspaceOrphanMap.ContainsKey(nodeSpace.Guid))
+                if (workspaceOrphanMap.TryGetValue(nodeSpace.Guid, out var workspaceOrphans))
                 {
-                    workspaceOrphanMap[nodeSpace.Guid].AddRange(kvp.Value);
+                    workspaceOrphans.AddRange(kvp.Value);
                 }
                 else
                 {
-                    workspaceOrphanMap.Add(nodeSpace.Guid, kvp.Value);
+                    workspaceOrphanMap.Add(nodeSpace.Guid, kvp.Value.ToList());
                 }
             }
 
diff --git a/src/Engine/ProtoCore/Lang/CallSite.cs b/src/Engine/ProtoCore/Lang/CallSite.cs
@@ -155,17 +155,24 @@ public bool Contains(string data)
             public List<string> RecursiveGetNestedData()
             {
                 List<string> ret = new List<string>();
+                RecursiveFillWithNestedData(ret);
+                return ret;
+            }
 
+            internal void RecursiveFillWithNestedData(List<string> listToFill)
+            {
                 if (HasData)
-                    ret.Add(Data);
+                {
+                    listToFill.Add(Data);
+                }
 
                 if (HasNestedData)
                 {
                     foreach (SingleRunTraceData srtd in NestedData)
-                        ret.AddRange(srtd.RecursiveGetNestedData());
+                    {
+                        srtd.RecursiveFillWithNestedData(listToFill);
+                    }
                 }
-
-                return ret;
             }
         }
 
@@ -472,13 +479,13 @@ private static string CompressSerializedTraceData(string json)
         /// </summary>
         public IList<string> GetOrphanedSerializables()
         {
-            var result = new List<string>();
-
             if (!beforeFirstRunSerializables.Any())
-                return result;
+            {
+                return new List<string>();
+            }
 
-            var currentSerializables = traceData.SelectMany(td => td.RecursiveGetNestedData());
-            result.AddRange(beforeFirstRunSerializables.Where(hs => !currentSerializables.Contains(hs)).ToList());
+            var currentSerializables = traceData.SelectMany(td => td.RecursiveGetNestedData()).ToHashSet();
+            var result = beforeFirstRunSerializables.Where(hs => !currentSerializables.Contains(hs)).ToList();
 
             // Clear the historical serializable to avoid 
             // them being used again. 
diff --git a/src/NodeServices/TraceSupport.cs b/src/NodeServices/TraceSupport.cs
@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Runtime.Serialization;
 
 namespace DynamoServices
 {
@@ -70,15 +69,9 @@ internal static void ClearAllKnownTLSKeys()
         /// <returns></returns>
         public static string GetTraceData(string key)
         {
-            string data;  
-            if (!LocalStorageSlot.TryGetValue(key, out data))
-            {
-                return null;
-            }
-            else
-            {
+            if (LocalStorageSlot.TryGetValue(key, out string data))
                 return data;
-            }
+            return null;
         }
 
         /// <summary>
@@ -88,14 +81,7 @@ public static string GetTraceData(string key)
         /// <param name="value"></param>
         public static void SetTraceData(string key, string value)
         {
-            if (LocalStorageSlot.ContainsKey(key))
-            {
-                LocalStorageSlot[key] = value;
-            }
-            else
-            {
-                LocalStorageSlot.Add(key, value);
-            }
+            LocalStorageSlot[key] = value;
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -534,14 +534,14 @@ internal void ReconcileTraceDataAndNotify()`
`534`	`534`	`var callsiteToOrphanMap = new Dictionary<Guid, List<string>>();`
`535`	`535`	`foreach (var cs in liveRunnerServices.RuntimeCore.RuntimeData.CallsiteCache.Values)`
`536`	`536`	`{`
`537`		`- var orphanedSerializables = cs.GetOrphanedSerializables().ToList();`
`538`		`- if (callsiteToOrphanMap.ContainsKey(cs.CallSiteID))`
	`537`	`+ var orphanedSerializables = cs.GetOrphanedSerializables();`
	`538`	`+ if (callsiteToOrphanMap.TryGetValue(cs.CallSiteID, out var serializablesForCallsite))`
`539`	`539`	`{`
`540`		`- callsiteToOrphanMap[cs.CallSiteID].AddRange(orphanedSerializables);`
	`540`	`+ serializablesForCallsite.AddRange(orphanedSerializables);`
`541`	`541`	`}`
`542`	`542`	`else`
`543`	`543`	`{`
`544`		`- callsiteToOrphanMap.Add(cs.CallSiteID, orphanedSerializables);`
	`544`	`+ callsiteToOrphanMap.Add(cs.CallSiteID, orphanedSerializables.ToList());`
`545`	`545`	`}`
`546`	`546`	`}`
`547`	`547`
Original file line number	Diff line number	Diff line change
`@@ -880,7 +880,7 @@ private void OnPreviewGraphCompleted(AsyncTask asyncTask)`
`880`	`880`	`/// trace data but do not exist in the current CallSite data.`
`881`	`881`	`/// </summary>`
`882`	`882`	`/// <returns></returns>`
`883`		`- internal IList<string> GetOrphanedSerializablesAndClearHistoricalTraceData()`
	`883`	`+ internal List<string> GetOrphanedSerializablesAndClearHistoricalTraceData()`
`884`	`884`	`{`
`885`	`885`	`var orphans = new List<string>();`
`886`	`886`
`@@ -891,13 +891,14 @@ internal IList<string> GetOrphanedSerializablesAndClearHistoricalTraceData()`
`891`	`891`	`// then add the serializables for that guid to the list of`
`892`	`892`	`// orphans.`
`893`	`893`
	`894`	`+ var nodeLookup = Nodes.Select(n => n.GUID).ToHashSet();`
`894`	`895`	`foreach (var nodeData in historicalTraceData)`
`895`	`896`	`{`
`896`	`897`	`var nodeGuid = nodeData.Key;`
`897`	`898`
`898`		`- if (Nodes.All(n => n.GUID != nodeGuid))`
	`899`	`+ if (!nodeLookup.Contains(nodeGuid))`
`899`	`900`	`{`
`900`		`- orphans.AddRange(nodeData.Value.SelectMany(CallSite.GetAllSerializablesFromSingleRunTraceData).ToList());`
	`901`	`+ orphans.AddRange(nodeData.Value.SelectMany(CallSite.GetAllSerializablesFromSingleRunTraceData));`
`901`	`902`	`}`
`902`	`903`	`}`
`903`	`904`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,5 @@`
`1`	`1`	`using System;`
`2`	`2`	`using System.Collections.Generic;`
`3`		`-using System.Runtime.Serialization;`
`4`	`3`
`5`	`4`	`namespace DynamoServices`
`6`	`5`	`{`
`@@ -70,15 +69,9 @@ internal static void ClearAllKnownTLSKeys()`
`70`	`69`	`/// <returns></returns>`
`71`	`70`	`public static string GetTraceData(string key)`
`72`	`71`	`{`
`73`		`- string data;`
`74`		`- if (!LocalStorageSlot.TryGetValue(key, out data))`
`75`		`- {`
`76`		`- return null;`
`77`		`- }`
`78`		`- else`
`79`		`- {`
	`72`	`+ if (LocalStorageSlot.TryGetValue(key, out string data))`
`80`	`73`	`return data;`
`81`		`- }`
	`74`	`+ return null;`
`82`	`75`	`}`
`83`	`76`
`84`	`77`	`/// <summary>`
`@@ -88,14 +81,7 @@ public static string GetTraceData(string key)`
`88`	`81`	`/// <param name="value"></param>`
`89`	`82`	`public static void SetTraceData(string key, string value)`
`90`	`83`	`{`
`91`		`- if (LocalStorageSlot.ContainsKey(key))`
`92`		`- {`
`93`		`- LocalStorageSlot[key] = value;`
`94`		`- }`
`95`		`- else`
`96`		`- {`
`97`		`- LocalStorageSlot.Add(key, value);`
`98`		`- }`
	`84`	`+ LocalStorageSlot[key] = value;`
`99`	`85`	`}`
`100`	`86`	`}`
`101`	`87`	`}`