From d949cc7ded605c8444a5de1172b5ab362c17e863 Mon Sep 17 00:00:00 2001 From: Kyle Hoffmeyer Date: Mon, 1 Jun 2026 15:00:00 -0700 Subject: [PATCH 1/3] perf(treeperf): eliminate redundant GPUEventAnalyser instantiation in compute_perf_metrics compute_perf_metrics() previously called loop_and_aggregate_kernels() twice (once unfiltered, once with non_data_mov_filter) and constructed two GPUEventAnalyser instances, each running a full get_gpu_event_lists() sweep-line pass. Profiling showed this accounted for the majority of compute_metrics call volume (9.3M calls on mi355_dsr1fp4_sglang_tp3). Fix: - Add GPUEventAnalyser.compute_busy_time(kernels) static method that computes merged busy time from a pre-built kernel list via a single merge_intervals pass, skipping the full sweep-line. - In compute_perf_metrics, aggregate kernels once then derive the non-data-movement subset by filtering list_kernels in-place rather than walking the tree a second time. Co-Authored-By: Claude Sonnet 4 --- TraceLens/TreePerf/gpu_event_analyser.py | 16 ++++++++++++++++ TraceLens/TreePerf/tree_perf.py | 19 +++++-------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/TraceLens/TreePerf/gpu_event_analyser.py b/TraceLens/TreePerf/gpu_event_analyser.py index cb3c9464e..2b5975602 100644 --- a/TraceLens/TreePerf/gpu_event_analyser.py +++ b/TraceLens/TreePerf/gpu_event_analyser.py @@ -321,6 +321,22 @@ def compute_metrics_dict(dict: dict, micro_idle_thresh_us=None): "total_memcpy_time": total_memcpy_time, } + @staticmethod + def compute_busy_time(kernels): + """Return the merged busy time (µs) for an already-filtered kernel list. + + Cheaper than compute_metrics() when only busy_time is needed — skips + the full get_gpu_event_lists() sweep-line and computes one merge pass. + """ + if not kernels: + return 0 + intervals = [ + (k["ts"], k["t_end"] if "t_end" in k else k["ts"] + k["dur"]) + for k in kernels + ] + merged = GPUEventAnalyser.merge_intervals(intervals) + return sum(end - start for start, end in merged) + def compute_metrics(self, micro_idle_thresh_us=None): """ Compute various metrics from the GPU event data. diff --git a/TraceLens/TreePerf/tree_perf.py b/TraceLens/TreePerf/tree_perf.py index 56fbebb12..882e5256a 100644 --- a/TraceLens/TreePerf/tree_perf.py +++ b/TraceLens/TreePerf/tree_perf.py @@ -391,22 +391,13 @@ def compute_perf_metrics( cpu_op_list = [self.tree.get_UID2event(uid) for uid in cpu_op_uids] _, list_kernelUIDS = self.loop_and_aggregate_kernels(cpu_op_list) list_kernels = [self.tree.events_by_uid[uid] for uid in list_kernelUIDS] - busy_kernel_time = 0 - if len(list_kernels) > 0: - busy_kernel_time = self.GPUEventAnalyser(list_kernels).compute_metrics()[ - "busy_time" - ] - _, list_non_data_mov_kernelUIDs = self.loop_and_aggregate_kernels( - cpu_op_list, filter_func=self.non_data_mov_filter - ) + busy_kernel_time = self.GPUEventAnalyser.compute_busy_time(list_kernels) list_non_data_mov_kernels = [ - self.tree.events_by_uid[uid] for uid in list_non_data_mov_kernelUIDs + k for k in list_kernels if self.non_data_mov_filter(k) ] - busy_non_data_mov_time = 0 - if len(list_non_data_mov_kernels) > 0: - busy_non_data_mov_time = self.GPUEventAnalyser( - list_non_data_mov_kernels - ).compute_metrics()["busy_time"] + busy_non_data_mov_time = self.GPUEventAnalyser.compute_busy_time( + list_non_data_mov_kernels + ) event["kernel_details"] = [ { "name": kernel["name"], From 51cca8d314441498d91c8dea25bff2ab6648b8a3 Mon Sep 17 00:00:00 2001 From: Kyle Hoffmeyer Date: Tue, 2 Jun 2026 10:17:51 -0700 Subject: [PATCH 2/3] updated docstring --- TraceLens/TreePerf/gpu_event_analyser.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/TraceLens/TreePerf/gpu_event_analyser.py b/TraceLens/TreePerf/gpu_event_analyser.py index 2b5975602..86315a3ea 100644 --- a/TraceLens/TreePerf/gpu_event_analyser.py +++ b/TraceLens/TreePerf/gpu_event_analyser.py @@ -323,11 +323,7 @@ def compute_metrics_dict(dict: dict, micro_idle_thresh_us=None): @staticmethod def compute_busy_time(kernels): - """Return the merged busy time (µs) for an already-filtered kernel list. - - Cheaper than compute_metrics() when only busy_time is needed — skips - the full get_gpu_event_lists() sweep-line and computes one merge pass. - """ + """Return the merged busy time (µs) for an already-filtered kernel list.""" if not kernels: return 0 intervals = [ From c459b4f91b1e2556f743109a9379d3fbe9ea0635 Mon Sep 17 00:00:00 2001 From: Kyle Hoffmeyer Date: Fri, 5 Jun 2026 16:47:28 -0700 Subject: [PATCH 3/3] cleaned up --- TraceLens/TreePerf/gpu_event_analyser.py | 22 +++++++++++++--------- TraceLens/TreePerf/tree_perf.py | 11 ++++++----- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/TraceLens/TreePerf/gpu_event_analyser.py b/TraceLens/TreePerf/gpu_event_analyser.py index 86315a3ea..5a6352b06 100644 --- a/TraceLens/TreePerf/gpu_event_analyser.py +++ b/TraceLens/TreePerf/gpu_event_analyser.py @@ -226,9 +226,10 @@ def verify_dict_gpu_event_lists(dict_gpu_event_lists): @staticmethod def compute_metrics_dict(dict: dict, micro_idle_thresh_us=None): - dict_intervals = {} - for key, events in dict.items(): - dict_intervals[key] = [(event["ts"], event["t_end"]) for event in events] + dict_intervals = { + key: GPUEventAnalyser.kernels_to_intervals(events) + for key, events in dict.items() + } # Merge intervals within each category. comp_union = GPUEventAnalyser.merge_intervals(dict_intervals["computation"]) @@ -322,14 +323,17 @@ def compute_metrics_dict(dict: dict, micro_idle_thresh_us=None): } @staticmethod - def compute_busy_time(kernels): - """Return the merged busy time (µs) for an already-filtered kernel list.""" - if not kernels: - return 0 - intervals = [ - (k["ts"], k["t_end"] if "t_end" in k else k["ts"] + k["dur"]) + def kernels_to_intervals(kernels, filter_func=None): + """Extract (ts, t_end) tuples from a kernel list, with an optional filter.""" + return [ + (k["ts"], k["t_end"]) for k in kernels + if filter_func is None or filter_func(k) ] + + @staticmethod + def compute_busy_time(intervals): + """Return the merged busy time (µs) for a list of (ts, t_end) tuples.""" merged = GPUEventAnalyser.merge_intervals(intervals) return sum(end - start for start, end in merged) diff --git a/TraceLens/TreePerf/tree_perf.py b/TraceLens/TreePerf/tree_perf.py index 499dbf4eb..9760a6b17 100644 --- a/TraceLens/TreePerf/tree_perf.py +++ b/TraceLens/TreePerf/tree_perf.py @@ -391,12 +391,13 @@ def compute_perf_metrics( cpu_op_list = [self.tree.get_UID2event(uid) for uid in cpu_op_uids] _, list_kernelUIDS = self.loop_and_aggregate_kernels(cpu_op_list) list_kernels = [self.tree.events_by_uid[uid] for uid in list_kernelUIDS] - busy_kernel_time = self.GPUEventAnalyser.compute_busy_time(list_kernels) - list_non_data_mov_kernels = [ - k for k in list_kernels if self.non_data_mov_filter(k) - ] + kernel_intervals = self.GPUEventAnalyser.kernels_to_intervals(list_kernels) + busy_kernel_time = self.GPUEventAnalyser.compute_busy_time(kernel_intervals) + non_data_mov_intervals = self.GPUEventAnalyser.kernels_to_intervals( + list_kernels, self.non_data_mov_filter + ) busy_non_data_mov_time = self.GPUEventAnalyser.compute_busy_time( - list_non_data_mov_kernels + non_data_mov_intervals ) event["kernel_details"] = [ {