[SOT] Update fast path strategy (#73506)

DrRyanHuang · web-flow · commit a8d6f1b8ad1f · 2025-06-21T10:04:10.000+08:00
diff --git a/python/paddle/jit/sot/opcode_translator/executor/executor_cache.py b/python/paddle/jit/sot/opcode_translator/executor/executor_cache.py
@@ -16,6 +16,7 @@
 
 import gc
 import traceback
+from collections import defaultdict
 from typing import TYPE_CHECKING
 
 import paddle
@@ -81,16 +82,14 @@ class OpcodeExecutorCache(metaclass=Singleton):
     translate_count: int
     code_symbolic_inputs: dict[types.CodeType, dict[str, None | dict[int, int]]]
     compile_time_stats: dict[types.CodeType, float]
-    consecutive_cache_hit_count: dict[types.CodeType, int]
-    last_cache_index: dict[types.CodeType, int]
+    consecutive_cache_hit_count: defaultdict[types.CodeType, int]
 
     def __init__(self):
         self.cache = {}
         self.translate_count = 0
         self.code_symbolic_inputs = {}
         self.compile_time_stats = {}
-        self.consecutive_cache_hit_count = {}
-        self.last_cache_index = {}
+        self.consecutive_cache_hit_count = defaultdict(int)
 
     def get_symbolic_inputs(
         self, code: types.CodeType
@@ -188,8 +187,12 @@ def lookup(
         if enable_unsafe_cache_fastpath and (
             self.is_fastpath_threshold_reached(code)
         ):
-            # NOTE: In inference scenarios, cache misses are generally rare, so we can enable this short path.
-            return guarded_fns[self.last_cache_index[code]][0]
+            # NOTE: In inference scenarios, cache misses are generally rare, so we can enable this unsafe short path.
+            log(
+                2,
+                "[Cache] The CACHE_HIT_FASTPATH_THRESHOLD has been reached, so fast path is now enabled\n",
+            )
+            return guarded_fns[0][0]
 
         cache_index = None
         if enable_strict_guard or enable_guard_tree:
@@ -255,11 +258,19 @@ def lookup(
                         cache_index is None or index == cache_index
                     ), f"cache_index({cache_index}) is not equal to index({index})"
 
-                    if self.last_cache_index.get(code, None) == index:
-                        self.consecutive_cache_hit_count[code] += 1
-                    else:
-                        self.last_cache_index[code] = index
-                        self.consecutive_cache_hit_count[code] = 1
+                    if enable_unsafe_cache_fastpath:
+                        if index == 0:
+                            self.consecutive_cache_hit_count[code] += 1
+                        else:
+                            # Move the current hit to the front
+                            # Note: Be cautious when modifying the order of elements in a list during iteration,
+                            # as it can lead to unexpected behavior.
+                            guarded_fns = [
+                                guarded_fns[index],
+                                *guarded_fns[:index],
+                                *guarded_fns[index + 1 :],
+                            ]
+                            self.consecutive_cache_hit_count[code] = 0
 
                     return custom_code
                 else:
