diff --git a/TraceLens/Agent/Analysis/.cursor/skills/analysis-orchestrator.md b/TraceLens/Agent/Analysis/.cursor/skills/analysis-orchestrator.md index d9ecc941..924b0bef 100644 --- a/TraceLens/Agent/Analysis/.cursor/skills/analysis-orchestrator.md +++ b/TraceLens/Agent/Analysis/.cursor/skills/analysis-orchestrator.md @@ -93,9 +93,9 @@ Use vendor-agnostic terminology throughout such as GPU kernels, collective commu - If **Inference (vLLM/SGLang)** is selected, ask **Execution Mode** → ``: 1. **Eager mode** (`` = `eager`) — only the trace file is needed 2. **Graph replay + capture** (`` = `graph_capture`) — also requires a capture folder path - - If **Graph replay + capture**, ask for **Capture Folder Path** → ``: + - If **Graph replay + capture**, ask for **Capture Folder Path** → ``: - Ask: "Please provide the full path to the graph capture traces folder" - - **Unsupported combination:** If `` = `graph_capture` **and** `` = `comparative`, stop immediately. Inform the user: "Graph replay + capture mode is not yet supported for comparative analysis. Please provide eager mode traces instead." Do not misinterpret as two standalone analyses. Do **not** proceed to Step 1 or beyond. + - If **Graph replay + capture** and **comparative**, ask for **Trace2 Capture Folder Path** → `` 5. **Environment Setup** - Ask: "Are you running locally or on a cluster?" @@ -178,14 +178,23 @@ All commands below append `` and ``, resolved by `/perf_report_trace1.xlsx --output_csvs_dir /perf_report_trace1_csvs` | | `comparative` trace2 | `--profile_json_path --output_xlsx_path /perf_report_trace2.xlsx --output_csvs_dir /perf_report_trace2_csvs` | -**``** — extension flags: +**``** — comparison flags: | scope | value | |-------|-------| | `standalone` | none | | `comparative` trace1 | `--comparison_json_path ` | +| `comparative` trace1 if `` provided | `--comparison_json_path --comparison_capture_folder ` | | `comparative` trace2 | none | +**``** — graph capture flags: + +| scope | value | +|-------|-------| +| `standalone` if `` provided | `--capture_folder ` | +| `comparative` trace1 if `` provided | `--capture_folder ` | +| `comparative` trace2 if `` provided | `--capture_folder ` | + **``** — user extension file: | condition | value | @@ -229,7 +238,6 @@ All commands below append `` and ``, resolved by ` TraceLens_generate_perf_report_pytorch_inference \ --profile_json_path \ - --capture_folder \ --gpu_arch_json_path \ --group_by_parent_module \ --enable_pseudo_ops \ @@ -237,6 +245,7 @@ All commands below append `` and ``, resolved by ` \ \ + \ ``` diff --git a/TraceLens/Agent/Analysis/README.md b/TraceLens/Agent/Analysis/README.md index cdc096a0..ecce88b5 100644 --- a/TraceLens/Agent/Analysis/README.md +++ b/TraceLens/Agent/Analysis/README.md @@ -19,7 +19,7 @@ The TraceLens Agent is an agentic performance analysis tool that generates actio | | Standalone | Comparative | |---|---|---| | **Eager** | ✅ | ✅ | -| **Graph + Capture** | ✅ | ❌ | +| **Graph + Capture** | ✅ | ✅ | | **Graph** | ❌ | ❌ | --- @@ -76,14 +76,22 @@ Roofline analysis compares each measured kernel against your GPU's max-achievabl ### To run via Cursor chat: 1. **In a Cursor chat with Claude Opus 4.7 High, invoke one of:** - - Standalone (single trace): + - Standalone (single eager trace): ``` "Follow the analysis orchestrator installed with TraceLens and run the full agentic analysis workflow on " ``` - - Comparative (two traces): + - Standalone (single graph replay trace with capture trace directory): + ``` + "Follow the analysis orchestrator installed with TraceLens and run the full agentic analysis workflow on with capture folder " + ``` + - Comparative (two eager traces): ``` "Follow the analysis orchestrator installed with TraceLens and run the full agentic analysis workflow on and " ``` + - Comparative (two graph replay traces with capture trace directories): + ``` + "Follow the analysis orchestrator installed with TraceLens and run the full agentic analysis workflow on with capture folder and with capture folder " + ``` **NOTE**: Always pass **baseline** trace as trace1 @@ -320,7 +328,7 @@ It queries user inputs, runs TraceLens to pre-compute trace data, and invokes sy The orchestrator and all 13 sub-agents currently run on **`claude-opus-4-7-high`**, declared in each agent file's front matter under `.cursor/agents/`. The full set: `cpu-idle-analyzer`, `multi-kernel-analyzer`, `kernel-fusion-analyzer`, `model-identification-agent`, `gemm-analyzer`, `sdpa-analyzer`, `elementwise-analyzer`, `reduce-analyzer`, `triton-analyzer`, `moe-analyzer`, `norm-analyzer`, `convolution-analyzer`, `generic-op-analyzer`. -### Supported Standalone Analysis Modes +### Supported Analysis Modes The orchestrator supports two analysis modes, selected during Step 0: diff --git a/TraceLens/Agent/Analysis/category_analyses/analysis_utils.py b/TraceLens/Agent/Analysis/category_analyses/analysis_utils.py index 15764dd3..812cccc9 100644 --- a/TraceLens/Agent/Analysis/category_analyses/analysis_utils.py +++ b/TraceLens/Agent/Analysis/category_analyses/analysis_utils.py @@ -408,9 +408,12 @@ def calculate_efficiency( result["resolved_peak_maf"] = round(peak_maf, 2) if peak_maf else None result["resolved_peak_hbm_bw"] = round(peak_hbm_bw, 2) if peak_hbm_bw else None - if flops_byte: - balance_point = peak_maf / peak_hbm_bw - result["bound_type"] = "compute" if flops_byte > balance_point else "memory" + roofline_bound = row.get("Roofline Bound") + if isinstance(roofline_bound, str): + if roofline_bound == "COMPUTE_BOUND": + result["bound_type"] = "compute" + elif roofline_bound == "MEMORY_BOUND": + result["bound_type"] = "memory" if comparison_scope == "comparative": comparative_efficiency(result, row) diff --git a/agent_evals/Analysis/analysis_tests/combined_traces_comparative.csv b/agent_evals/Analysis/analysis_tests/combined_traces_comparative.csv index aeb3add1..647b0a1d 100644 --- a/agent_evals/Analysis/analysis_tests/combined_traces_comparative.csv +++ b/agent_evals/Analysis/analysis_tests/combined_traces_comparative.csv @@ -1,10 +1,11 @@ -id,sub_category,trace1_path,trace2_path,reference_dir,platform,platform2 -test01_moe_fused_vs_unfused,moe,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test01_moe_fused_vs_unfused/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test01_moe_fused_vs_unfused/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test01_moe_fused_vs_unfused/analysis_output_ref,MI300X,H100 -test02_attention_both_unfused,attention,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test02_attention_both_unfused/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test02_attention_both_unfused/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test02_attention_both_unfused/analysis_output_ref,MI300X,H100 -test03_attention_fused_vs_unfused,attention,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test03_attention_fused_vs_unfused/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test03_attention_fused_vs_unfused/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test03_attention_fused_vs_unfused/analysis_output_ref,MI300X,H100 -test04_gemm_mm_vs_linear,gemm,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test04_gemm_mm_vs_linear/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test04_gemm_mm_vs_linear/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test04_gemm_mm_vs_linear/analysis_output_ref,MI300X,H100 -test05_gemm_identical_runtime_roofline,gemm,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test05_gemm_identical_runtime_roofline/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test05_gemm_identical_runtime_roofline/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test05_gemm_identical_runtime_roofline/analysis_output_ref,MI300X,H100 -test06_conv_gap_exceeds_roofline,conv,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test06_conv_gap_exceeds_roofline/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test06_conv_gap_exceeds_roofline/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test06_conv_gap_exceeds_roofline/analysis_output_ref,MI300X,H100 -bert_small,full_model,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/bert_small/gaunernst_bert-small-uncased__1016001_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/bert_small/gaunernst_bert-small-uncased__1016001_h100.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/bert_small/analysis_output_ref,MI300X,H100 -llama3_2,full_model,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/llama3_2/llama3_2_mi300.pt.trace_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/llama3_2/llama3_2_h100.pt.trace_h100.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/llama3_2/analysis_output_ref,MI300X,H100 -timesformer,full_model,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/timesformer/facebook_timesformer-base-finetuned-k400__1016002_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/timesformer/facebook_timesformer-base-finetuned-k400__1016002_h100.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/timesformer/analysis_output_ref,MI300X,H100 \ No newline at end of file +id,sub_category,trace1_path,trace2_path,reference_dir,platform,platform2,capture_folder1,capture_folder2 +test01_moe_fused_vs_unfused,moe,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test01_moe_fused_vs_unfused/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test01_moe_fused_vs_unfused/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test01_moe_fused_vs_unfused/analysis_output_ref,MI300X,H100,, +test02_attention_both_unfused,attention,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test02_attention_both_unfused/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test02_attention_both_unfused/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test02_attention_both_unfused/analysis_output_ref,MI300X,H100,, +test03_attention_fused_vs_unfused,attention,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test03_attention_fused_vs_unfused/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test03_attention_fused_vs_unfused/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test03_attention_fused_vs_unfused/analysis_output_ref,MI300X,H100,, +test04_gemm_mm_vs_linear,gemm,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test04_gemm_mm_vs_linear/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test04_gemm_mm_vs_linear/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test04_gemm_mm_vs_linear/analysis_output_ref,MI300X,H100,, +test05_gemm_identical_runtime_roofline,gemm,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test05_gemm_identical_runtime_roofline/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test05_gemm_identical_runtime_roofline/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test05_gemm_identical_runtime_roofline/analysis_output_ref,MI300X,H100,, +test06_conv_gap_exceeds_roofline,conv,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test06_conv_gap_exceeds_roofline/trace1.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test06_conv_gap_exceeds_roofline/trace2.json,agent_evals/Analysis/analysis_tests/unit_tests_comparative/test06_conv_gap_exceeds_roofline/analysis_output_ref,MI300X,H100,, +bert_small,full_model,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/bert_small/gaunernst_bert-small-uncased__1016001_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/bert_small/gaunernst_bert-small-uncased__1016001_h100.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/bert_small/analysis_output_ref,MI300X,H100,, +llama3_2,full_model,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/llama3_2/llama3_2_mi300.pt.trace_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/llama3_2/llama3_2_h100.pt.trace_h100.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/llama3_2/analysis_output_ref,MI300X,H100,, +timesformer,full_model,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/timesformer/facebook_timesformer-base-finetuned-k400__1016002_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/timesformer/facebook_timesformer-base-finetuned-k400__1016002_h100.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/timesformer/analysis_output_ref,MI300X,H100,, +gptoss_graph_capture,full_model,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/gptoss_graph_capture/vllm_v21_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/gptoss_graph_capture/vllm_v19_mi300.json.gz,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/gptoss_graph_capture/analysis_output_ref,MI300X,MI300X,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/gptoss_graph_capture/capture_folder_v21,agent_evals/Analysis/analysis_tests/e2e_tests_comparative/gptoss_graph_capture/capture_folder_v19 \ No newline at end of file diff --git a/agent_evals/Analysis/analysis_tests/e2e_tests_comparative.tar.gz b/agent_evals/Analysis/analysis_tests/e2e_tests_comparative.tar.gz index 86ec07f6..8243b24c 100644 Binary files a/agent_evals/Analysis/analysis_tests/e2e_tests_comparative.tar.gz and b/agent_evals/Analysis/analysis_tests/e2e_tests_comparative.tar.gz differ diff --git a/agent_evals/Analysis/eval_scripts/generate_ref.sh b/agent_evals/Analysis/eval_scripts/generate_ref.sh index b8e27b74..6f7f25b9 100755 --- a/agent_evals/Analysis/eval_scripts/generate_ref.sh +++ b/agent_evals/Analysis/eval_scripts/generate_ref.sh @@ -69,7 +69,7 @@ log_status() { # --------------------------------------------------------------------------- generate_single_ref() { - local id="$1" trace1_path="$2" trace2_path="$3" reference_dir="$4" platform="$5" platform2="$6" + local id="$1" trace1_path="$2" trace2_path="$3" reference_dir="$4" platform="$5" platform2="$6" capture_folder1="${7:-}" capture_folder2="${8:-}" local tag="[$id]" local REF_DIR="$REPO_ROOT/$reference_dir" @@ -100,8 +100,13 @@ generate_single_ref() { ( cd "$ANALYSIS_DIR" || exit if [[ "$COMPARISON_SCOPE" == "comparative" ]]; then + local capture_suffix="" + [[ -n "$capture_folder1" ]] && capture_suffix+=" capture folder for trace1 $REPO_ROOT/$capture_folder1" + [[ -n "$capture_folder2" ]] && capture_suffix+=" capture folder for trace2 $REPO_ROOT/$capture_folder2" + local analysis_mode="default" + [[ -n "$capture_folder1" || -n "$capture_folder2" ]] && analysis_mode="inference" agent --model claude-opus-4-8-thinking-medium --print --force --trust --output-format stream-json \ - "Follow the analysis orchestrator installed with the TraceLens pip package (look under TraceLens/Agent/Analysis/.cursor/skills/ in the package installation directory) and run the full agentic analysis workflow on $trace1_path and $trace2_path with platform $platform (trace1) and $platform2 (trace2), analysis mode default, $NODE_LABEL, $RUNTIME_LABEL, output to $OUTPUT_DIR" + "Follow the analysis orchestrator installed with the TraceLens pip package (look under TraceLens/Agent/Analysis/.cursor/skills/ in the package installation directory) and run the full agentic analysis workflow on $trace1_path and $trace2_path${capture_suffix} with platform $platform (trace1) and $platform2 (trace2), analysis mode $analysis_mode, $NODE_LABEL, $RUNTIME_LABEL, output to $OUTPUT_DIR" else agent --model claude-opus-4-8-thinking-medium --print --force --trust --output-format stream-json \ "Follow the analysis orchestrator installed with the TraceLens pip package (look under TraceLens/Agent/Analysis/.cursor/skills/ in the package installation directory) and run the full agentic analysis workflow on $trace1_path with platform $platform, analysis mode default, $NODE_LABEL, $RUNTIME_LABEL, output to $OUTPUT_DIR" @@ -202,14 +207,14 @@ should_run_id() { setup_semaphore if [[ "$COMPARISON_SCOPE" == "comparative" ]]; then - # comparative CSV: id,sub_category,trace1_path,trace2_path,reference_dir,platform,platform2 - while IFS=, read -r id sub_category trace1_path trace2_path reference_dir platform platform2 <&3; do + # comparative CSV: id,sub_category,trace1_path,trace2_path,reference_dir,platform,platform2,capture_folder1,capture_folder2 + while IFS=, read -r id sub_category trace1_path trace2_path reference_dir platform platform2 capture_folder1 capture_folder2 <&3; do [[ -z "$id" ]] && continue should_run_id "$id" || continue read -u4 # acquire semaphore slot ( - generate_single_ref "$id" "$trace1_path" "$trace2_path" "$reference_dir" "$platform" "$platform2" || true + generate_single_ref "$id" "$trace1_path" "$trace2_path" "$reference_dir" "$platform" "$platform2" "${capture_folder1:-}" "${capture_folder2:-}" || true sleep "$SLEEP_BETWEEN" echo >&4 # release semaphore slot ) & diff --git a/agent_evals/Analysis/eval_scripts/run_repeatability_parallel.sh b/agent_evals/Analysis/eval_scripts/run_repeatability_parallel.sh index 9b8e8125..ab746617 100644 --- a/agent_evals/Analysis/eval_scripts/run_repeatability_parallel.sh +++ b/agent_evals/Analysis/eval_scripts/run_repeatability_parallel.sh @@ -81,7 +81,7 @@ expand_archive() { # --------------------------------------------------------------------------- run_single_job() { - local id="$1" repeat="$2" trace1_path="$3" trace2_path="$4" reference_dir="$5" platform="$6" platform2="$7" + local id="$1" repeat="$2" trace1_path="$3" trace2_path="$4" reference_dir="$5" platform="$6" platform2="$7" capture_folder1="${8:-}" capture_folder2="${9:-}" local tag="[$id|run_$repeat]" local CASE_RESULTS="$RESULTS_ROOT/$id/run_${repeat}" @@ -102,8 +102,13 @@ run_single_job() { ( cd "$ANALYSIS_DIR" || exit if [[ "$COMPARISON_SCOPE" == "comparative" ]]; then + local capture_suffix="" + [[ -n "$capture_folder1" ]] && capture_suffix+=" capture folder for trace1 $REPO_ROOT/$capture_folder1" + [[ -n "$capture_folder2" ]] && capture_suffix+=" capture folder for trace2 $REPO_ROOT/$capture_folder2" + local analysis_mode="default" + [[ -n "$capture_folder1" || -n "$capture_folder2" ]] && analysis_mode="inference" timeout 1800 agent --model claude-opus-4-8-thinking-medium --print --force --trust --output-format stream-json \ - "Follow the analysis orchestrator installed with the TraceLens pip package (look under TraceLens/Agent/Analysis/.cursor/skills/ in the package installation directory) and run the full agentic analysis workflow on $trace1_path and $trace2_path with platform $platform (trace1) and $platform2 (trace2), analysis mode default, $NODE_LABEL, $RUNTIME_LABEL, output to $OUTPUT_DIR" + "Follow the analysis orchestrator installed with the TraceLens pip package (look under TraceLens/Agent/Analysis/.cursor/skills/ in the package installation directory) and run the full agentic analysis workflow on $trace1_path and $trace2_path${capture_suffix} with platform $platform (trace1) and $platform2 (trace2), analysis mode $analysis_mode, $NODE_LABEL, $RUNTIME_LABEL, output to $OUTPUT_DIR" else timeout 1800 agent --model claude-opus-4-8-thinking-medium --print --force --trust --output-format stream-json \ "Follow the analysis orchestrator installed with the TraceLens pip package (look under TraceLens/Agent/Analysis/.cursor/skills/ in the package installation directory) and run the full agentic analysis workflow on $trace1_path with platform $platform, $NODE_LABEL, $RUNTIME_LABEL, output to $OUTPUT_DIR" @@ -217,7 +222,7 @@ echo "=========================================" echo "" _spawn_jobs() { - local id="$1" trace1_path="$2" trace2_path="$3" reference_dir="$4" platform="$5" platform2="$6" + local id="$1" trace1_path="$2" trace2_path="$3" reference_dir="$4" platform="$5" platform2="$6" capture_folder1="${7:-}" capture_folder2="${8:-}" if [[ -n "$TEST_IDS" ]]; then case " $TEST_IDS " in @@ -229,7 +234,7 @@ _spawn_jobs() { for ((i = 0; i < NUM_REPEATS; i++)); do read -r -u4 # acquire semaphore slot ( - run_single_job "$id" "$i" "$trace1_path" "$trace2_path" "$reference_dir" "$platform" "$platform2" || true + run_single_job "$id" "$i" "$trace1_path" "$trace2_path" "$reference_dir" "$platform" "$platform2" "${capture_folder1:-}" "${capture_folder2:-}" || true echo >&4 # release semaphore slot sleep 2 # stagger agent startup to avoid ~/.cursor/cli-config.json rename race ) & @@ -240,10 +245,10 @@ _spawn_jobs() { setup_semaphore if [[ "$COMPARISON_SCOPE" == "comparative" ]]; then - # comparative CSV: id,sub_category,trace1_path,trace2_path,reference_dir,platform,platform2 - while IFS=, read -r id sub_category trace1_path trace2_path reference_dir platform platform2 <&3; do + # comparative CSV: id,sub_category,trace1_path,trace2_path,reference_dir,platform,platform2,capture_folder1,capture_folder2 + while IFS=, read -r id sub_category trace1_path trace2_path reference_dir platform platform2 capture_folder1 capture_folder2 <&3; do [[ -z "$id" ]] && continue - _spawn_jobs "$id" "$trace1_path" "$trace2_path" "$reference_dir" "$platform" "$platform2" + _spawn_jobs "$id" "$trace1_path" "$trace2_path" "$reference_dir" "$platform" "$platform2" "${capture_folder1:-}" "${capture_folder2:-}" done 3< <(tail -n +2 "$TEST_TRACES_CSV"; echo) else # standalone CSV: id,sub_category,trace_path,reference_dir,platform diff --git a/agent_evals/Analysis/eval_utils/workflow_scripted_evals.py b/agent_evals/Analysis/eval_utils/workflow_scripted_evals.py index 4b1fdce9..61c88634 100644 --- a/agent_evals/Analysis/eval_utils/workflow_scripted_evals.py +++ b/agent_evals/Analysis/eval_utils/workflow_scripted_evals.py @@ -767,7 +767,7 @@ def _check_issue_template(output_dir, comparison_scope="standalone"): if not rows: all_sections = (compute_section or "") + (system_section or "") intentionally_empty = ( - "No compute kernel optimization opportunities identified" in all_sections + "No compute kernel optimization opportunities" in all_sections or "No system-level bottlenecks detected" in all_sections ) rows.append( @@ -994,14 +994,17 @@ def _check_marker_p_items(output_dir, comparison_scope=None): p_items = _extract_p_items(compute_section) if not p_items: + intentionally_empty = ( + "No compute kernel optimization opportunities" in compute_section + ) return [ _make_marker_row( "marker_eval_2", "P-item markers (kind=p_item)", - "FAIL", - "No P-items found in Compute Kernel Optimizations", - "template", - "Ensure report contains P-items", + "PASS" if intentionally_empty else "FAIL", + "" if intentionally_empty else "No P-items found in Compute Kernel Optimizations", + "" if intentionally_empty else "template", + "" if intentionally_empty else "Add compute P-items or clearly state that no optimization opportunities were identified", ) ] @@ -1088,14 +1091,17 @@ def _check_marker_detail_estimates(output_dir, comparison_scope=None): matches = list(_DETAIL_P_HEADER_FALLBACK_RE.finditer(compute_subsection)) if not matches: + intentionally_empty = ( + "No compute kernel optimization opportunities" in compute_subsection + ) return [ _make_marker_row( "marker_eval_3", "Detail estimate markers (kind=detail_estimate)", - "FAIL", - "No P-item headers found in Detailed Analysis", - "template", - "Ensure Detailed Analysis contains P-item sections", + "PASS" if intentionally_empty else "FAIL", + "" if intentionally_empty else "No P-item headers found in Detailed Analysis", + "" if intentionally_empty else "template", + "" if intentionally_empty else "Add compute P-items to Detailed Analysis or clearly state that no optimization opportunities were identified", ) ]