From 20ebb38b9ac2979dcb983265905407505cded5fa Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Wed, 13 May 2026 00:27:11 +0000
Subject: [PATCH 01/10] Enhancement: Add rocprofv2 trace support for AMD GPUs

- runner.py: Add SB_ENABLE_ROCPROF/SB_ROCPROF_TRACE_DIR env vars to enable
  rocprofv2 profiling (--hip-trace --kernel-trace --plugin json) in local,
  torch.distributed, and mpi modes
- pytorch_base.py: Extend GPU guard to support ROCm (torch.version.hip)
  so PyTorch profiler works on AMD GPUs
---
 .../model_benchmarks/pytorch_base.py          |  4 +-
 superbench/runner/runner.py                   | 49 ++++++++++++++-----
 2 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/superbench/benchmarks/model_benchmarks/pytorch_base.py b/superbench/benchmarks/model_benchmarks/pytorch_base.py
index de06b35d0..f2428f2c6 100644
--- a/superbench/benchmarks/model_benchmarks/pytorch_base.py
+++ b/superbench/benchmarks/model_benchmarks/pytorch_base.py
@@ -591,8 +591,8 @@ def _benchmark(self):
         Run the benchmark then handle post-run model log save/compare.
         Set SB_ENABLE_PYTORCH_PROFILER='1' to enable profiling.
         """
-        # Check if this is a Nvidia GPU
-        if not (torch.cuda.is_available() and torch.version.cuda is not None):
+        # Check if this is a Nvidia or AMD GPU
+        if not (torch.cuda.is_available() and (torch.version.cuda is not None or torch.version.hip is not None)):
             ok = super()._benchmark()
             self._post_run_model_log()
             return ok
diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index a5ac13cbb..5c588bb96 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -135,12 +135,23 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
         enable_nsys = os.environ.get('SB_ENABLE_NSYS', '') == '1'
         trace_dir = os.environ.get('SB_NSYS_TRACE_DIR', self._sb_output_dir)
 
+        # Enable rocprofv2 profiling based on environment variable
+        enable_rocprof = os.environ.get('SB_ENABLE_ROCPROF', '') == '1'
+        rocprof_trace_dir = os.environ.get('SB_ROCPROF_TRACE_DIR', self._sb_output_dir)
+
         mode_command = exec_command
         if mode.name == 'local':
-            trace_command = (
-                f'nsys profile --output {trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
-                f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
-            ) if enable_nsys and mode.proc_rank == 0 else ''
+            trace_command = ''
+            if enable_nsys and mode.proc_rank == 0:
+                trace_command = (
+                    f'nsys profile --output {trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                    f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
+                )
+            elif enable_rocprof and mode.proc_rank == 0:
+                trace_command = (
+                    f'rocprofv2 --hip-trace --kernel-trace --plugin json '
+                    f'-d {rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                )
             # Build the command parts, only including trace if it's not empty
             command_parts = []
             prefix = mode.prefix.format(proc_rank=mode.proc_rank, proc_num=mode.proc_num)
@@ -159,10 +170,17 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 '--nnodes=$NNODES --node_rank=$NODE_RANK --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT '
             )
 
-            nsys_prefix = (
-                f'nsys profile --output {trace_dir}/{benchmark_name}_traces '
-                f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
-            ) if enable_nsys else ''
+            nsys_prefix = ''
+            if enable_nsys:
+                nsys_prefix = (
+                    f'nsys profile --output {trace_dir}/{benchmark_name}_traces '
+                    f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
+                )
+            elif enable_rocprof:
+                nsys_prefix = (
+                    f'rocprofv2 --hip-trace --kernel-trace --plugin json '
+                    f'-d {rocprof_trace_dir}/{benchmark_name}_traces '
+                )
 
             mode_command = (
                 f'{nsys_prefix}'
@@ -172,10 +190,17 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 f' superbench.benchmarks.{benchmark_name}.parameters.distributed_backend=nccl'
             )
         elif mode.name == 'mpi':
-            trace_command = (
-                f'nsys profile --output {trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
-                f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
-            ) if enable_nsys else ''
+            trace_command = ''
+            if enable_nsys:
+                trace_command = (
+                    f'nsys profile --output {trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                    f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
+                )
+            elif enable_rocprof:
+                trace_command = (
+                    f'rocprofv2 --hip-trace --kernel-trace --plugin json '
+                    f'-d {rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                )
             mode_command = (
                 '{trace} '
                 'mpirun '    # use default OpenMPI in image

From 9b1dd82a66b809da10d45e28e344571fa6dab083 Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Tue, 19 May 2026 18:46:06 +0000
Subject: [PATCH 02/10] Use shlex.quote() for trace output paths in runner.py

Address PR review: wrap all interpolated path/name segments in
shlex.quote() to prevent command injection or broken commands when
paths contain whitespace or shell metacharacters. Applied to both
nsys and rocprofv2 trace commands across all three execution modes
(local, torch.distributed, mpi).
---
 superbench/runner/runner.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index 5c588bb96..1432408cc 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -7,6 +7,7 @@
 import sys
 import json
 import random
+import shlex
 import signal
 from pathlib import Path
 from pprint import pformat
@@ -143,14 +144,16 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
         if mode.name == 'local':
             trace_command = ''
             if enable_nsys and mode.proc_rank == 0:
+                trace_output = shlex.quote(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
-                    f'nsys profile --output {trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                    f'nsys profile --output {trace_output} '
                     f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
                 )
             elif enable_rocprof and mode.proc_rank == 0:
+                trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                    f'-d {trace_output} '
                 )
             # Build the command parts, only including trace if it's not empty
             command_parts = []
@@ -172,14 +175,16 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
 
             nsys_prefix = ''
             if enable_nsys:
+                trace_output = shlex.quote(f'{trace_dir}/{benchmark_name}_traces')
                 nsys_prefix = (
-                    f'nsys profile --output {trace_dir}/{benchmark_name}_traces '
+                    f'nsys profile --output {trace_output} '
                     f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
                 )
             elif enable_rocprof:
+                trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_traces')
                 nsys_prefix = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {rocprof_trace_dir}/{benchmark_name}_traces '
+                    f'-d {trace_output} '
                 )
 
             mode_command = (
@@ -192,14 +197,16 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
         elif mode.name == 'mpi':
             trace_command = ''
             if enable_nsys:
+                trace_output = shlex.quote(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
-                    f'nsys profile --output {trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                    f'nsys profile --output {trace_output} '
                     f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
                 )
             elif enable_rocprof:
+                trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces '
+                    f'-d {trace_output} '
                 )
             mode_command = (
                 '{trace} '

From a9485efb300ed3efdca7d2124bac3e47d0042d3e Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Tue, 19 May 2026 18:56:09 +0000
Subject: [PATCH 03/10] Rename nsys_prefix to trace_prefix for tool-agnostic
 naming

Address PR review: the variable holds either an nsys or rocprofv2
prefix, so rename to trace_prefix to avoid implying Nsight-only
behavior.
---
 superbench/runner/runner.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index 1432408cc..c0b2345e5 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -173,22 +173,22 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 '--nnodes=$NNODES --node_rank=$NODE_RANK --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT '
             )
 
-            nsys_prefix = ''
+            trace_prefix = ''
             if enable_nsys:
                 trace_output = shlex.quote(f'{trace_dir}/{benchmark_name}_traces')
-                nsys_prefix = (
+                trace_prefix = (
                     f'nsys profile --output {trace_output} '
                     f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
                 )
             elif enable_rocprof:
                 trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_traces')
-                nsys_prefix = (
+                trace_prefix = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
                     f'-d {trace_output} '
                 )
 
             mode_command = (
-                f'{nsys_prefix}'
+                f'{trace_prefix}'
                 f'torchrun'
                 f' --no_python --nproc_per_node={mode.proc_num} {torch_dist_params}{exec_command}'
                 f' superbench.benchmarks.{benchmark_name}.parameters.distributed_impl=ddp'

From 8a22f3e6d2bbce5ff4d06d9190eacb6bc0996268 Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Tue, 7 Jul 2026 18:19:20 +0000
Subject: [PATCH 04/10] Use getattr for safe torch.version.cuda/hip access in
 pytorch_base.py

Address PR review: torch.version.hip (and .cuda) may not exist on all
PyTorch builds/versions, raising AttributeError. Use getattr with a
None default to make the check compatible across all PyTorch builds.
---
 superbench/benchmarks/model_benchmarks/pytorch_base.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/superbench/benchmarks/model_benchmarks/pytorch_base.py b/superbench/benchmarks/model_benchmarks/pytorch_base.py
index f2428f2c6..96d738364 100644
--- a/superbench/benchmarks/model_benchmarks/pytorch_base.py
+++ b/superbench/benchmarks/model_benchmarks/pytorch_base.py
@@ -592,7 +592,9 @@ def _benchmark(self):
         Set SB_ENABLE_PYTORCH_PROFILER='1' to enable profiling.
         """
         # Check if this is a Nvidia or AMD GPU
-        if not (torch.cuda.is_available() and (torch.version.cuda is not None or torch.version.hip is not None)):
+        if not (torch.cuda.is_available() and (
+            getattr(torch.version, 'cuda', None) is not None or getattr(torch.version, 'hip', None) is not None
+        )):
             ok = super()._benchmark()
             self._post_run_model_log()
             return ok

From a3519510a4d5bde731cf0476a2d6416258b37f29 Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Tue, 7 Jul 2026 18:25:54 +0000
Subject: [PATCH 05/10] Add -- separator after rocprofv2 options in runner.py

Address PR review: add the conventional -- end-of-options separator
after rocprofv2 flags so that benchmark command arguments starting
with - are not misinterpreted as rocprofv2 options. Applied to all
three execution modes (local, torch.distributed, mpi).
---
 superbench/runner/runner.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index c0b2345e5..605a68fd5 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -153,7 +153,7 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {trace_output} '
+                    f'-d {trace_output} -- '
                 )
             # Build the command parts, only including trace if it's not empty
             command_parts = []
@@ -184,7 +184,7 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_traces')
                 trace_prefix = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {trace_output} '
+                    f'-d {trace_output} -- '
                 )
 
             mode_command = (
@@ -206,7 +206,7 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {trace_output} '
+                    f'-d {trace_output} -- '
                 )
             mode_command = (
                 '{trace} '

From 04c55b09a1724311b6b82d7afd57ea66170c7801 Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Wed, 22 Jul 2026 21:25:19 +0000
Subject: [PATCH 06/10] Escape single quotes so trace paths survive bash -lc
 wrapper

Address PR review: shlex.quote() wraps values with spaces or shell
metacharacters in single quotes, which would terminate the outer
bash -lc '...' quoting used to dispatch commands. Add a helper that
runs shlex.quote() and then replaces each ' with '\'' so the quoted
value is safe to embed inside the outer single-quoted command
string. Applied to all six nsys/rocprofv2 trace output paths.
---
 superbench/runner/runner.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index 605a68fd5..3ee920ef9 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -26,6 +26,17 @@
 AnsibleClient = LazyImport('superbench.runner.ansible', 'AnsibleClient')
 
 
+def _quote_for_bash_lc(value):
+    """Quote a value so it is safe both for the shell and for embedding inside an outer bash -lc '...' string.
+
+    ``shlex.quote`` wraps values containing whitespace or shell metacharacters in single quotes. When the
+    resulting command is later interpolated into ``bash -lc '{command}'``, those single quotes would
+    terminate the outer single-quoted context. Escape any single quotes as ``'\\''`` so the value survives
+    both quoting layers.
+    """
+    return shlex.quote(value).replace("'", "'\\''")
+
+
 class SuperBenchRunner():
     """SuperBench runner class."""
     def __init__(self, sb_config, docker_config, ansible_config, sb_output_dir):
@@ -144,13 +155,13 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
         if mode.name == 'local':
             trace_command = ''
             if enable_nsys and mode.proc_rank == 0:
-                trace_output = shlex.quote(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
+                trace_output = _quote_for_bash_lc(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'nsys profile --output {trace_output} '
                     f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
                 )
             elif enable_rocprof and mode.proc_rank == 0:
-                trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
+                trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
                     f'-d {trace_output} -- '
@@ -175,13 +186,13 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
 
             trace_prefix = ''
             if enable_nsys:
-                trace_output = shlex.quote(f'{trace_dir}/{benchmark_name}_traces')
+                trace_output = _quote_for_bash_lc(f'{trace_dir}/{benchmark_name}_traces')
                 trace_prefix = (
                     f'nsys profile --output {trace_output} '
                     f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
                 )
             elif enable_rocprof:
-                trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_traces')
+                trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}_traces')
                 trace_prefix = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
                     f'-d {trace_output} -- '
@@ -197,13 +208,13 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
         elif mode.name == 'mpi':
             trace_command = ''
             if enable_nsys:
-                trace_output = shlex.quote(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
+                trace_output = _quote_for_bash_lc(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'nsys profile --output {trace_output} '
                     f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
                 )
             elif enable_rocprof:
-                trace_output = shlex.quote(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
+                trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
                 trace_command = (
                     f'rocprofv2 --hip-trace --kernel-trace --plugin json '
                     f'-d {trace_output} -- '

From 5cca6cca5f2cf2d34c9172937740c5c04ff51c73 Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Wed, 22 Jul 2026 21:31:52 +0000
Subject: [PATCH 07/10] Warn when both SB_ENABLE_NSYS and SB_ENABLE_ROCPROF are
 set

Address PR review: previously SB_ENABLE_ROCPROF was silently ignored
whenever SB_ENABLE_NSYS was also set, producing no traces and no
diagnostic. Log a warning documenting that nsys takes precedence and
explicitly disable rocprofv2 in that case.
---
 superbench/runner/runner.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index 3ee920ef9..040ea5124 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -151,6 +151,14 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
         enable_rocprof = os.environ.get('SB_ENABLE_ROCPROF', '') == '1'
         rocprof_trace_dir = os.environ.get('SB_ROCPROF_TRACE_DIR', self._sb_output_dir)
 
+        # SB_ENABLE_NSYS and SB_ENABLE_ROCPROF are mutually exclusive; nsys takes precedence when both are set.
+        if enable_nsys and enable_rocprof:
+            logger.warning(
+                'Both SB_ENABLE_NSYS and SB_ENABLE_ROCPROF are set; using nsys and ignoring rocprofv2. '
+                'Unset SB_ENABLE_NSYS to enable rocprofv2 tracing on AMD hardware.'
+            )
+            enable_rocprof = False
+
         mode_command = exec_command
         if mode.name == 'local':
             trace_command = ''

From 497abd92171433e01c16f1418e8a2f86b2c728bb Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Wed, 22 Jul 2026 21:41:49 +0000
Subject: [PATCH 08/10] Document that torch profiler CUDA APIs alias to HIP on
 ROCm builds

Address PR review: clarify that ProfilerActivity.CUDA and event.cuda_time
are aliases that cover HIP kernels on ROCm builds of PyTorch, so the
same profiler code path works for both NVIDIA and AMD GPUs. Verified
end-to-end on MI300x with ROCm 6.3.4.
---
 superbench/benchmarks/model_benchmarks/pytorch_base.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/superbench/benchmarks/model_benchmarks/pytorch_base.py b/superbench/benchmarks/model_benchmarks/pytorch_base.py
index 96d738364..47d692736 100644
--- a/superbench/benchmarks/model_benchmarks/pytorch_base.py
+++ b/superbench/benchmarks/model_benchmarks/pytorch_base.py
@@ -622,6 +622,9 @@ def _benchmark(self):
             local_rank = self._local_rank
 
         diag_agent_prof = profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=True)
+        # Note: on ROCm builds of PyTorch, ProfilerActivity.CUDA and event.cuda_time are aliases that cover
+        # HIP kernels; the same code path therefore works for both NVIDIA and AMD GPUs (verified on MI300x,
+        # ROCm 6.3.4).
         dump_file_dir = os.environ.get('SB_TORCH_PROFILER_TRACE_DIR', '.')
         diag_agent_dump_file_path = f'{dump_file_dir}/torch-profiler-sb-{self._name}-{local_rank}.json'
         diag_agent_prof.__enter__()

From da4635c60f745f8142e3f8181f096bc140bed841 Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Wed, 22 Jul 2026 22:13:18 +0000
Subject: [PATCH 09/10] Factor duplicated trace-command construction into a
 shared helper

Address PR review: the nsys/rocprofv2 selection + path quoting logic
was copy-pasted across all three mode branches (local, torch.distributed,
mpi). Extract __build_trace_command() so future changes to profiler
flags, quoting, or additional tools live in a single place.
---
 superbench/runner/runner.py | 74 ++++++++++++++++++-------------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index 040ea5124..e11a377a2 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -124,6 +124,34 @@ def __get_enabled_benchmarks(self):
                 return list(self._sb_config.superbench.enable)
         return [k for k, v in self._sb_benchmarks.items() if 'enable' in v and v.enable]
 
+    def __build_trace_command(self, benchmark_name, suffix, enable_nsys, enable_rocprof, trace_dir, rocprof_trace_dir):
+        """Build the profiler prefix (nsys or rocprofv2) to prepend to an execution command.
+
+        Args:
+            benchmark_name (str): Benchmark name, used in the output filename.
+            suffix (str): Suffix to append to the output filename (e.g. rank id or empty string).
+            enable_nsys (bool): Whether nsys profiling is enabled.
+            enable_rocprof (bool): Whether rocprofv2 profiling is enabled.
+            trace_dir (str): Output directory for nsys traces.
+            rocprof_trace_dir (str): Output directory for rocprofv2 traces.
+
+        Return:
+            str: Profiler command prefix with a trailing space, or an empty string if no profiler is enabled.
+        """
+        if enable_nsys:
+            trace_output = _quote_for_bash_lc(f'{trace_dir}/{benchmark_name}{suffix}_traces')
+            return (
+                f'nsys profile --output {trace_output} '
+                f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
+            )
+        if enable_rocprof:
+            trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}{suffix}_traces')
+            return (
+                f'rocprofv2 --hip-trace --kernel-trace --plugin json '
+                f'-d {trace_output} -- '
+            )
+        return ''
+
     def __get_mode_command(self, benchmark_name, mode, timeout=None):
         """Get runner command for given mode.
 
@@ -162,17 +190,9 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
         mode_command = exec_command
         if mode.name == 'local':
             trace_command = ''
-            if enable_nsys and mode.proc_rank == 0:
-                trace_output = _quote_for_bash_lc(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
-                trace_command = (
-                    f'nsys profile --output {trace_output} '
-                    f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
-                )
-            elif enable_rocprof and mode.proc_rank == 0:
-                trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
-                trace_command = (
-                    f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {trace_output} -- '
+            if mode.proc_rank == 0:
+                trace_command = self.__build_trace_command(
+                    benchmark_name, f'_{mode.proc_rank}', enable_nsys, enable_rocprof, trace_dir, rocprof_trace_dir
                 )
             # Build the command parts, only including trace if it's not empty
             command_parts = []
@@ -192,19 +212,9 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 '--nnodes=$NNODES --node_rank=$NODE_RANK --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT '
             )
 
-            trace_prefix = ''
-            if enable_nsys:
-                trace_output = _quote_for_bash_lc(f'{trace_dir}/{benchmark_name}_traces')
-                trace_prefix = (
-                    f'nsys profile --output {trace_output} '
-                    f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
-                )
-            elif enable_rocprof:
-                trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}_traces')
-                trace_prefix = (
-                    f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {trace_output} -- '
-                )
+            trace_prefix = self.__build_trace_command(
+                benchmark_name, '', enable_nsys, enable_rocprof, trace_dir, rocprof_trace_dir
+            )
 
             mode_command = (
                 f'{trace_prefix}'
@@ -214,19 +224,9 @@ def __get_mode_command(self, benchmark_name, mode, timeout=None):
                 f' superbench.benchmarks.{benchmark_name}.parameters.distributed_backend=nccl'
             )
         elif mode.name == 'mpi':
-            trace_command = ''
-            if enable_nsys:
-                trace_output = _quote_for_bash_lc(f'{trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
-                trace_command = (
-                    f'nsys profile --output {trace_output} '
-                    f'--backtrace none --sample none --force-overwrite true --cpuctxsw none --trace cuda,nvtx '
-                )
-            elif enable_rocprof:
-                trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}_{mode.proc_rank}_traces')
-                trace_command = (
-                    f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                    f'-d {trace_output} -- '
-                )
+            trace_command = self.__build_trace_command(
+                benchmark_name, f'_{mode.proc_rank}', enable_nsys, enable_rocprof, trace_dir, rocprof_trace_dir
+            )
             mode_command = (
                 '{trace} '
                 'mpirun '    # use default OpenMPI in image

From b4b5e54a68f7f31a3ddcf944c19b8919c5ee8a0d Mon Sep 17 00:00:00 2001
From: Shenghsun Cho <choshenghsun@microsoft.com>
Date: Wed, 22 Jul 2026 22:16:07 +0000
Subject: [PATCH 10/10] Fix yapf formatting and flake8 D301 in runner.py /
 pytorch_base.py

Apply yapf column-limit reflows and mark the _quote_for_bash_lc
docstring as raw (D301) since it contains backslash escapes.
---
 superbench/benchmarks/model_benchmarks/pytorch_base.py | 7 ++++---
 superbench/runner/runner.py                            | 9 +++------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/superbench/benchmarks/model_benchmarks/pytorch_base.py b/superbench/benchmarks/model_benchmarks/pytorch_base.py
index 47d692736..7810b982d 100644
--- a/superbench/benchmarks/model_benchmarks/pytorch_base.py
+++ b/superbench/benchmarks/model_benchmarks/pytorch_base.py
@@ -592,9 +592,10 @@ def _benchmark(self):
         Set SB_ENABLE_PYTORCH_PROFILER='1' to enable profiling.
         """
         # Check if this is a Nvidia or AMD GPU
-        if not (torch.cuda.is_available() and (
-            getattr(torch.version, 'cuda', None) is not None or getattr(torch.version, 'hip', None) is not None
-        )):
+        if not (
+            torch.cuda.is_available() and
+            (getattr(torch.version, 'cuda', None) is not None or getattr(torch.version, 'hip', None) is not None)
+        ):
             ok = super()._benchmark()
             self._post_run_model_log()
             return ok
diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py
index e11a377a2..7a91abae6 100644
--- a/superbench/runner/runner.py
+++ b/superbench/runner/runner.py
@@ -27,11 +27,11 @@
 
 
 def _quote_for_bash_lc(value):
-    """Quote a value so it is safe both for the shell and for embedding inside an outer bash -lc '...' string.
+    r"""Quote a value so it is safe both for the shell and for embedding inside an outer bash -lc '...' string.
 
     ``shlex.quote`` wraps values containing whitespace or shell metacharacters in single quotes. When the
     resulting command is later interpolated into ``bash -lc '{command}'``, those single quotes would
-    terminate the outer single-quoted context. Escape any single quotes as ``'\\''`` so the value survives
+    terminate the outer single-quoted context. Escape any single quotes as ``'\''`` so the value survives
     both quoting layers.
     """
     return shlex.quote(value).replace("'", "'\\''")
@@ -146,10 +146,7 @@ def __build_trace_command(self, benchmark_name, suffix, enable_nsys, enable_rocp
             )
         if enable_rocprof:
             trace_output = _quote_for_bash_lc(f'{rocprof_trace_dir}/{benchmark_name}{suffix}_traces')
-            return (
-                f'rocprofv2 --hip-trace --kernel-trace --plugin json '
-                f'-d {trace_output} -- '
-            )
+            return (f'rocprofv2 --hip-trace --kernel-trace --plugin json ' f'-d {trace_output} -- ')
         return ''
 
     def __get_mode_command(self, benchmark_name, mode, timeout=None):