diff --git a/test/registered/ascend/basic_function/model_tokenizer/test_npu_model_tokenizer.py b/test/registered/ascend/basic_function/model_tokenizer/test_npu_model_tokenizer.py index c1aa3dd82442..85ee09afd8f6 100644 --- a/test/registered/ascend/basic_function/model_tokenizer/test_npu_model_tokenizer.py +++ b/test/registered/ascend/basic_function/model_tokenizer/test_npu_model_tokenizer.py @@ -9,7 +9,10 @@ import requests from sglang.srt.utils import kill_process_tree -from sglang.test.ascend.test_ascend_utils import LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH +from sglang.test.ascend.test_ascend_utils import ( + LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH, + QWEN3_4B_GGUF_Q4_K_M_WEIGHTS_PATH, +) from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -279,5 +282,52 @@ def test_model_skip_tokenizer_request(self): self.assertIn("output_ids", response.text) +class TestNpuLoadFormatGguf(CustomTestCase): + """Testcase: verify --load-format=gguf explicitly loads GGUF model and inference succeeds + + [Test Category] Parameter + [Test Target] --load-format=gguf + """ + + @classmethod + def setUpClass(cls): + cls.model = QWEN3_4B_GGUF_Q4_K_M_WEIGHTS_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--mem-fraction-static", + "0.8", + "--attention-backend", + "ascend", + "--disable-cuda-graph", + "--load-format", + "gguf", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_load_format_gguf(self): + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertIn("Paris", response.text) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/ascend/basic_function/msprobe/test_npu_msprobe_dump_config.py b/test/registered/ascend/basic_function/msprobe/test_npu_msprobe_dump_config.py index 0b3aff67ceda..d115dc0da41c 100644 --- a/test/registered/ascend/basic_function/msprobe/test_npu_msprobe_dump_config.py +++ b/test/registered/ascend/basic_function/msprobe/test_npu_msprobe_dump_config.py @@ -90,21 +90,6 @@ def test_msprobe_dump_config_eager_mode(self): self.assertEqual(response.status_code, 200) self.assertIn("Paris", response.text) - self.err_log_file.seek(0) - err_log = self.err_log_file.read() - self.assertIn( - "When msProbe is enabled", - err_log, - "Expected stderr to contain 'When msProbe is enabled', proving " - "--msprobe-dump-config was parsed and cuda graph + warmup were disabled", - ) - self.assertNotIn( - "Please install msprobe", - err_log, - "Expected stderr NOT to contain 'Please install msprobe', proving " - "mindstudio-probe is installed and PrecisionDebugger was created", - ) - # msprobe writes dump.json into per-step subdirectories # (e.g. step31/dump.json), not at the root of the dump dir. dump_files = [] diff --git a/test/registered/ascend/basic_function/optimization_debug_options/test_npu_cuda_graph_bs.py b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_cuda_graph_bs.py index 8898ca1cf88b..ba59f05f1d16 100644 --- a/test/registered/ascend/basic_function/optimization_debug_options/test_npu_cuda_graph_bs.py +++ b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_cuda_graph_bs.py @@ -3,12 +3,12 @@ import subprocess import tempfile import unittest -from typing import List, Optional -from urllib.parse import urlparse from sglang.bench_serving import run_benchmark from sglang.srt.utils import kill_process_tree -from sglang.test.ascend.test_ascend_utils import LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH +from sglang.test.ascend.test_ascend_utils import ( + QWEN2_5_7B_INSTRUCT_WEIGHTS_PATH, +) from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -18,41 +18,65 @@ ) from sglang.utils import wait_for_http_ready -register_npu_ci(est_time=1200, suite="full-2-npu-a3", nightly=True) +register_npu_ci(est_time=600, suite="full-1-npu-a3", nightly=True) -MODEL = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH +MODEL = QWEN2_5_7B_INSTRUCT_WEIGHTS_PATH _LAUNCH_TIMEOUT = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH -_BS_LOG_RE = re.compile(r"Capture cuda graph bs \[([^\]]+)\]") -_MEM_LOG_RE = re.compile(r"mem usage=([\d.]+) GB") +_DECODE_RE = re.compile(r"Capture target decode.*begin.*bs=\[([^\]]+)\]") +_PREFILL_RE = re.compile(r"Capture target prefill.*begin.*num_tokens=\[([^\]]+)\]") -def _pd_ports(): - p = urlparse(DEFAULT_URL_FOR_TEST) - host = p.hostname - bp = str(p.port) - return { - "host": host, - "lb": bp, - "prefill": str(int(bp) + 100), - "decode": str(int(bp) + 200), - "bootstrap": str(int(bp) + 500), - } +def _read_log(path): + """Read log file content""" + with open(path, encoding="utf-8", errors="replace") as f: + return f.read() -def _pd_transport_args(): - # NPU uses ascend transfer backend (no RDMA/IB devices needed). - return ["--disaggregation-transfer-backend", "ascend"] +def _parse_cg_capture(log_text: str): + """Parse (decode batch size list, prefill batch size list) from CG capture logs + Distinguish log lines via ``target decode`` / ``target prefill`` markers. + """ + decode_bs = None + prefill_bs = None + for line in log_text.splitlines(): + if m := _DECODE_RE.search(line): + decode_bs = [int(x.strip()) for x in m.group(1).split(",")] + print(f"[CG parse] decode | {line.strip()}") + if m := _PREFILL_RE.search(line): + prefill_bs = [int(x.strip()) for x in m.group(1).split(",")] + print(f"[CG parse] prefill | {line.strip()}") + if decode_bs is None and prefill_bs is None: + print("[CG parse] WARNING: No CG start log entry matched") + return decode_bs, prefill_bs -def _launch_pd_server(url, *, mode, bootstrap_port, extra_args, base_gpu_id="0"): - """Launch one PD server (prefill or decode), capturing stderr to a temp file. - Returns (process, stderr_file_path). +def _run_bench(base_url): + """Run benchmark workload""" + bench_args = get_benchmark_args( + base_url=base_url, + backend="sglang", + dataset_name="random", + tokenizer=MODEL, + num_prompts=10, + random_input_len=256, + random_output_len=32, + request_rate=float("inf"), + ) + bench_args.warmup_requests = 0 + return run_benchmark(bench_args) + + +def _launch_server(*, extra_args=None): + """Launch server, redirect stderr to temporary log file + + Returns (process handle, temp log file path) """ + url = DEFAULT_URL_FOR_TEST _, host, port = url.split(":") host = host[2:] - err_fd, err_path = tempfile.mkstemp(suffix=".log", prefix=f"pd_{mode}_") + err_fd, err_path = tempfile.mkstemp(suffix=".log", prefix="cg_bs_") os.close(err_fd) cmd = [ @@ -70,199 +94,97 @@ def _launch_pd_server(url, *, mode, bootstrap_port, extra_args, base_gpu_id="0") "ascend", "--mem-fraction-static", "0.8", - "--disaggregation-mode", - mode, - "--disaggregation-bootstrap-port", - bootstrap_port, - "--base-gpu-id", - base_gpu_id, "--tp", "1", - *extra_args, - *_pd_transport_args(), + *(extra_args or []), ] - env = { - **os.environ, - "ASCEND_MF_STORE_URL": "tcp://127.0.0.1:26666", - } with open(err_path, "w") as err_file: proc = subprocess.Popen( cmd, stdout=subprocess.DEVNULL, stderr=err_file, text=True, - env=env, ) wait_for_http_ready(url + "/health", timeout=_LAUNCH_TIMEOUT, process=proc) return proc, err_path -def _launch_router(prefill_url, decode_url, host, lb_port): - cmd = [ - "python3", - "-m", - "sglang_router.launch_router", - "--pd-disaggregation", - "--prefill", - prefill_url, - "--decode", - decode_url, - "--host", - host, - "--port", - lb_port, - ] - proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - lb_url = f"http://{host}:{lb_port}" - wait_for_http_ready(lb_url + "/health", timeout=_LAUNCH_TIMEOUT, process=proc) - return proc, lb_url - - -def _launch_pd(*, prefill_args=None, decode_args=None): - """Launch full PD stack. Returns (prefill_proc, decode_proc, lb_proc, lb_url, - prefill_err_path, decode_err_path). - """ - ports = _pd_ports() - prefill_url = f"http://{ports['host']}:{ports['prefill']}" - decode_url = f"http://{ports['host']}:{ports['decode']}" - - os.environ["MC_TCP_ENABLE_CONNECTION_POOL"] = "true" - - pp, pe = _launch_pd_server( - prefill_url, - mode="prefill", - bootstrap_port=ports["bootstrap"], - extra_args=prefill_args or [], - base_gpu_id="0", - ) - dp, de = _launch_pd_server( - decode_url, - mode="decode", - bootstrap_port=ports["bootstrap"], - extra_args=decode_args or [], - base_gpu_id="1", - ) - lp, lb_url = _launch_router(prefill_url, decode_url, ports["host"], ports["lb"]) - return pp, dp, lp, lb_url, pe, de - - -def _cleanup_pd(pp, dp, lp, pe, de): - for proc in (lp, dp, pp): - if proc: - kill_process_tree(proc.pid) - for path in (pe, de): - try: - os.remove(path) - except OSError: - pass - - -def _read_log(path): - with open(path, encoding="utf-8", errors="replace") as f: - return f.read() - - -def _parse_capture_bs(log_text: str) -> Optional[List[int]]: - for line in log_text.splitlines(): - m = _BS_LOG_RE.search(line) - if m: - return [int(x.strip()) for x in m.group(1).split(",")] - return None - - -def _parse_graph_memory_gb(log_text: str) -> Optional[float]: - for line in log_text.splitlines(): - m = _MEM_LOG_RE.search(line) - if m: - return float(m.group(1)) - return None - - -def _has_graph_begin(log_text: str) -> bool: - return any("graph begin" in line for line in log_text.splitlines()) +class TestCudaGraphBs(CustomTestCase): + """Test case: Verify CUDA Graph batch size parameters for each phase in non-PD mode - -def _run_bench(base_url): - bench_args = get_benchmark_args( - base_url=base_url, - backend="sglang", - dataset_name="random", - tokenizer=MODEL, - num_prompts=10, - random_input_len=256, - random_output_len=32, - request_rate=float("inf"), - ) - bench_args.warmup_requests = 0 - return run_benchmark(bench_args) - - -class TestCudaGraphBsPD(CustomTestCase): - """Testcase: verify per-phase CUDA-graph BS parameters in PD disaggregation. - - PD hook (pd_disaggregation_hook.py:77) force-disables CG on the prefill - server, so --cuda-graph-max-bs-decode / --cuda-graph-bs-decode are parsed - but never trigger graph capture on the prefill side. The decode server - captures CUDA graphs normally and respects the parameters. - - All tests launch a full PD stack (prefill + decode + LB), send traffic - through the LB with bench_serving, and verify CG behaviour on each side - independently. - - [Test Category] Parameter - [Test Target] --cuda-graph-max-bs-decode; --cuda-graph-max-bs-prefill; - --cuda-graph-bs-decode; --cuda-graph-bs-prefill; - --disaggregation-mode + [Test Category] Parameter Test + [Test Target] --cuda-graph-max-bs-decode; --cuda-graph-bs-decode; + --cuda-graph-max-bs-prefill; --cuda-graph-bs-prefill """ - # max_bs only, bs auto-generated on decode side + # ---- Only set max batch size, auto-generate batch sizes for both phases ---- def test_max_bs_auto_generates_bs(self): - pp, dp, lp, lb_url, pe, de = _launch_pd( - decode_args=["--cuda-graph-max-bs-decode", "8"], + proc, err_path = _launch_server( + extra_args=[ + "--cuda-graph-max-bs-decode", + "8", + "--cuda-graph-backend-prefill", + "tc_piecewise", + "--cuda-graph-max-bs-prefill", + "256", + ], ) try: - res = _run_bench(lb_url) - self.assertEqual(res["completed"], 10) - - prefill_log = _read_log(pe) - decode_log = _read_log(de) + log_text = _read_log(err_path) finally: - _cleanup_pd(pp, dp, lp, pe, de) - - # Prefill: CG disabled by PD hook - self.assertFalse( - _has_graph_begin(prefill_log), "Prefill CG must be disabled by PD hook" + kill_process_tree(proc.pid) + try: + os.remove(err_path) + except OSError: + pass + + decode_bs, prefill_bs = _parse_cg_capture(log_text) + self.assertIsNotNone( + decode_bs, "Log should contain decode phase CG capture info" ) - # Decode: bs auto-generated, all ≤ 8 - decode_bs = _parse_capture_bs(decode_log) - self.assertIsNotNone(decode_bs, "Expected capture bs in decode log") self.assertEqual(max(decode_bs), 8) self.assertTrue(all(b <= 8 for b in decode_bs)) - # explicit bs only, max_bs derived on decode side - def test_explicit_bs_derives_max_bs(self): - pp, dp, lp, lb_url, pe, de = _launch_pd( - decode_args=["--cuda-graph-bs-decode", "1", "2", "4", "8"], + self.assertIsNotNone( + prefill_bs, "Log should contain prefill phase CG capture info" + ) + self.assertEqual(max(prefill_bs), 256) + self.assertTrue(all(b <= 256 for b in prefill_bs)) + + # ---- Explicitly specify batch size lists for both phases ---- + def test_explicit_bs_used_exactly(self): + proc, err_path = _launch_server( + extra_args=[ + "--cuda-graph-bs-decode", + "1", + "2", + "4", + "8", + "--cuda-graph-backend-prefill", + "tc_piecewise", + "--cuda-graph-bs-prefill", + "64", + "128", + "256", + ], ) try: - res = _run_bench(lb_url) - self.assertEqual(res["completed"], 10) - decode_log = _read_log(de) - prefill_log = _read_log(pe) + log_text = _read_log(err_path) finally: - _cleanup_pd(pp, dp, lp, pe, de) + kill_process_tree(proc.pid) + try: + os.remove(err_path) + except OSError: + pass - self.assertFalse(_has_graph_begin(prefill_log)) - decode_bs = _parse_capture_bs(decode_log) + decode_bs, prefill_bs = _parse_cg_capture(log_text) self.assertEqual(decode_bs, [1, 2, 4, 8]) - mem = _parse_graph_memory_gb(decode_log) - self.assertIsNotNone(mem) - self.assertGreater(mem, 0) - - # both max_bs and bs set, max_bs silently overwritten - def test_max_bs_overwritten_when_bs_set(self): - pp, dp, lp, lb_url, pe, de = _launch_pd( - decode_args=[ + self.assertEqual(prefill_bs, [64, 128, 256]) + + # ---- Decode only: Explicit batch size overrides max batch size argument ---- + def test_decode_max_bs_overwritten_when_bs_set(self): + proc, err_path = _launch_server( + extra_args=[ "--cuda-graph-max-bs-decode", "4", "--cuda-graph-bs-decode", @@ -272,93 +194,78 @@ def test_max_bs_overwritten_when_bs_set(self): ], ) try: - res = _run_bench(lb_url) - self.assertEqual(res["completed"], 10) - decode_log = _read_log(de) - prefill_log = _read_log(pe) + log_text = _read_log(err_path) finally: - _cleanup_pd(pp, dp, lp, pe, de) + kill_process_tree(proc.pid) + try: + os.remove(err_path) + except OSError: + pass - self.assertFalse(_has_graph_begin(prefill_log)) - decode_bs = _parse_capture_bs(decode_log) + decode_bs, _ = _parse_cg_capture(log_text) self.assertEqual(decode_bs, [1, 2, 8]) - self.assertEqual(max(decode_bs), 8, "max_bs should be 8 (overwritten), not 4") + self.assertEqual( + max(decode_bs), 8, "Max batch size shall be overridden to 8 instead of 4" + ) - # disable cuda graph padding, sequential bs generated - def test_disable_padding_sequential_bs(self): - pp, dp, lp, lb_url, pe, de = _launch_pd( - decode_args=[ + # ---- Decode only: Sequential batch sizes generated when padding disabled ---- + def test_decode_disable_padding_sequential_bs(self): + proc, err_path = _launch_server( + extra_args=[ "--cuda-graph-max-bs-decode", "8", "--disable-cuda-graph-padding", ], ) try: - res = _run_bench(lb_url) - self.assertEqual(res["completed"], 10) - decode_log = _read_log(de) - prefill_log = _read_log(pe) + log_text = _read_log(err_path) finally: - _cleanup_pd(pp, dp, lp, pe, de) + kill_process_tree(proc.pid) + try: + os.remove(err_path) + except OSError: + pass - self.assertFalse(_has_graph_begin(prefill_log)) - decode_bs = _parse_capture_bs(decode_log) + decode_bs, _ = _parse_cg_capture(log_text) self.assertEqual(decode_bs, list(range(1, 9))) - # cuda graph disabled, no graph capture, serving works - def test_disable_cuda_graph_serving_works(self): - pp, dp, lp, lb_url, pe, de = _launch_pd( - decode_args=["--cuda-graph-max-bs-decode", "8", "--disable-cuda-graph"], - ) - try: - res = _run_bench(lb_url) - self.assertEqual(res["completed"], 10) - decode_log = _read_log(de) - prefill_log = _read_log(pe) - finally: - _cleanup_pd(pp, dp, lp, pe, de) - - self.assertFalse( - _has_graph_begin(prefill_log), "Prefill CG must be disabled by PD hook" - ) - self.assertFalse( - _has_graph_begin(decode_log), - "Decode CG must be disabled by --disable-cuda-graph", - ) - - # prefill CG disabled + decode CG behaviour verified by tests above - - # TTFT comparison with different max_bs values + # ---- TTFT performance comparison under different max batch size settings ---- def test_max_bs_ttft_comparison(self): - # max_bs=1 - pp1, dp1, lp1, lb1, pe1, de1 = _launch_pd( - decode_args=["--cuda-graph-max-bs-decode", "1"], + proc1, err1 = _launch_server( + extra_args=["--cuda-graph-max-bs-decode", "1"], ) try: - r1 = _run_bench(lb1) + r1 = _run_bench(DEFAULT_URL_FOR_TEST) self.assertEqual(r1["completed"], 10) finally: - _cleanup_pd(pp1, dp1, lp1, pe1, de1) - - # max_bs=8 - pp8, dp8, lp8, lb8, pe8, de8 = _launch_pd( - decode_args=["--cuda-graph-max-bs-decode", "8"], + kill_process_tree(proc1.pid) + try: + os.remove(err1) + except OSError: + pass + + proc8, err8 = _launch_server( + extra_args=["--cuda-graph-max-bs-decode", "8"], ) try: - r8 = _run_bench(lb8) + r8 = _run_bench(DEFAULT_URL_FOR_TEST) self.assertEqual(r8["completed"], 10) finally: - _cleanup_pd(pp8, dp8, lp8, pe8, de8) + kill_process_tree(proc8.pid) + try: + os.remove(err8) + except OSError: + pass t1, t8 = r1["mean_ttft_ms"], r8["mean_ttft_ms"] p1, p8 = r1["p99_ttft_ms"], r8["p99_ttft_ms"] print( - f"\n=== TTFT comparison (PD mode): max_bs=1 vs max_bs=8 ===\n" + f"\n=== TTFT Comparison: max_bs=1 vs max_bs=8 ===\n" f" Mean TTFT: {t1:.1f} ms (max_bs=1) vs {t8:.1f} ms (max_bs=8)\n" f" P99 TTFT: {p1:.1f} ms (max_bs=1) vs {p8:.1f} ms (max_bs=8)" ) - self.assertGreater(t1, 0) - self.assertGreater(t8, 0) + # Larger max_bs enables request batching, reducing average TTFT. + self.assertGreater(t1, t8) if __name__ == "__main__": diff --git a/test/registered/ascend/basic_function/optimization_debug_options/test_npu_embedding_interpolation.py b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_embedding_interpolation.py index f2582f6235eb..642897027951 100644 --- a/test/registered/ascend/basic_function/optimization_debug_options/test_npu_embedding_interpolation.py +++ b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_embedding_interpolation.py @@ -5,7 +5,10 @@ import requests from sglang.srt.utils import kill_process_tree -from sglang.test.ascend.test_ascend_utils import QWEN3_VL_4B_INSTRUCT_WEIGHTS_PATH +from sglang.test.ascend.test_ascend_utils import ( + IMAGES_MAN_PATH, + QWEN3_VL_4B_INSTRUCT_WEIGHTS_PATH, +) from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -13,14 +16,15 @@ CustomTestCase, popen_launch_server, ) -from sglang.test.vlm_utils import IMAGE_MAN_IRONING_URL + +IMAGE_MAN_IRONING_URL = IMAGES_MAN_PATH register_npu_ci(est_time=600, suite="full-1-npu-a3", nightly=True) class TestPreciseEmbeddingInterpolation(CustomTestCase): """Testcase: verify --enable-precise-embedding-interpolation changes ViT - position-embedding interpolation on Qwen3-VL, producing different outputs + position-embedding interpolation on Qwen3-VL, producing different logprobs for the same image at temperature=0 [Test Category] Parameter @@ -85,15 +89,19 @@ def _image_request(self, base_url): }, ], "temperature": 0, - # Limit output to keep inference predictable; - # a single-sentence description is well under 128 tokens. "max_tokens": 128, + "logprobs": True, + "top_logprobs": 5, }, ) + @staticmethod + def _extract_logprobs(resp): + """Extract logprob of the top-1 token at each generation position.""" + content = resp.json()["choices"][0]["logprobs"]["content"] + return [entry["top_logprobs"][0]["logprob"] for entry in content] + def test_precise_embedding_interpolation_contrastive(self): - # Port for the second launch (first uses DEFAULT_URL_FOR_TEST). - # Sequential launch/teardown on the same port risks TIME_WAIT. alt_url = "http://127.0.0.1:23001" # ---- Launch WITH --enable-precise-embedding-interpolation ---- @@ -110,12 +118,10 @@ def test_precise_embedding_interpolation_contrastive(self): f"— check whether {IMAGE_MAN_IRONING_URL} is reachable", ) text_enabled = resp_enabled.json()["choices"][0]["message"]["content"] + logprobs_enabled = self._extract_logprobs(resp_enabled) finally: self._cleanup() - # Brief pause so the OS releases the first port before binding - # the second one. Together with using a different port this - # eliminates any TIME_WAIT race. time.sleep(2) # ---- Launch WITHOUT the flag (default: False) ---- @@ -129,6 +135,7 @@ def test_precise_embedding_interpolation_contrastive(self): f"— check whether {IMAGE_MAN_IRONING_URL} is reachable", ) text_default = resp_default.json()["choices"][0]["message"]["content"] + logprobs_default = self._extract_logprobs(resp_default) finally: self._cleanup() @@ -144,18 +151,29 @@ def test_precise_embedding_interpolation_contrastive(self): f"Expected vehicle-related word in: {text}", ) - # Core assertion: outputs differ, proving the flag changes interpolation. - # If this fails with identical outputs it may indicate the ViT graph path - # is not active (silent fallback to eager, which hardcodes align_corners=True - # regardless of the flag) or the flag is no longer read at model init. - self.assertNotEqual( - text_enabled, - text_default, - "Outputs should differ because --enable-precise-embedding-interpolation " + # Core assertion: logprobs differ, proving the flag changes interpolation. + # Comparing logprobs is more sensitive than comparing decoded text — + # the interpolation difference may shift token probabilities without + # changing the argmax (top-1) token at temperature=0. + self.assertEqual( + len(logprobs_enabled), + len(logprobs_default), + "Token counts should match since both runs use the same image and max_tokens", + ) + + diffs = [ + i + for i, (lp_en, lp_def) in enumerate(zip(logprobs_enabled, logprobs_default)) + if abs(lp_en - lp_def) > 1e-6 + ] + + self.assertTrue( + len(diffs) > 0, + "Logprobs should differ because --enable-precise-embedding-interpolation " "changes _get_interpolation_indices (align_corners=True vs False). " - "Identical outputs may mean the ViT graph path is not active " - "(SGLANG_VIT_ENABLE_CUDA_GRAPH fallback to eager) or the flag is " - "not being read at model init time.", + "If no logprob differences are found, the flag may not be active on this " + "backend (fallback to eager, which hardcodes torch.linspace) or the flag " + "is not being read at model init time.", ) diff --git a/test/registered/ascend/basic_function/optimization_debug_options/test_npu_full_decode_graph_gsm8k.py b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_full_decode_graph_gsm8k.py new file mode 100644 index 000000000000..b25d7fcdc3d4 --- /dev/null +++ b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_full_decode_graph_gsm8k.py @@ -0,0 +1,181 @@ +"""Full decode CUDA-graph capture accuracy test on NPU. + +Exercises the ``--cuda-graph-backend-decode full`` path on +Kimi-K2.6-W4A8 with modelslim quantization to verify that full decode +graph capture does not degrade accuracy on NPU. +""" + +import os +import unittest +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Dict, List + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.test_ascend_utils import KIMI_K2_6_W4A8_MODEL_PATH +from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_npu_ci(est_time=3600, suite="full-16-npu-a3", nightly=True) + +MODEL_PATH = KIMI_K2_6_W4A8_MODEL_PATH +SERVER_LAUNCH_TIMEOUT = 3600 +GSM8K_NUM_QUESTIONS = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) +ACCURACY_THRESHOLD = 0.9121 + + +@dataclass +class CaptureConfig: + """A prefill cuda-graph capture backend variant to validate.""" + + variant: str + # Extra server args that select the prefill capture backend. + capture_args: List[str] + env_vars: Dict[str, str] = field(default_factory=dict) + + +# Common args: TP16, ascend backend, modelslim quantization, 8192 chunked prefill. +COMMON_ARGS: List[str] = [ + "--tensor-parallel-size", + "16", + "--trust-remote-code", + "--attention-backend", + "ascend", + "--quantization", + "modelslim", + "--mem-fraction-static", + "0.765", + "--disable-radix-cache", + "--prefill-attention-backend", + "ascend", + "--decode-attention-backend", + "ascend", + "--kv-cache-dtype", + "auto", + "--max-running-requests", + "1024", + "--chunked-prefill-size", + "8192", + "--max-prefill-tokens", + "8192", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--cuda-graph-bs-decode", + "1", + "2", + "4", + "8", + "--enable-dp-attention", + "--dp-size", + "2", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", +] + + +def get_capture_configs() -> List[CaptureConfig]: + return [ + # Full decode graph capture. + CaptureConfig( + variant="bcg", + capture_args=[ + "--cuda-graph-backend-prefill", + "disabled", + "--cuda-graph-backend-decode", + "full", + ], + env_vars={ + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", + "HCCL_BUFFSIZE": "1200", + "HCCL_OP_EXPANSION_MODE": "AIV", + }, + ), + ] + + +class TestNpuFullDecodeGraphGsm8k(CustomTestCase): + """Testcase: Validate full decode CUDA-graph accuracy on NPU. + + [Test Category] Parameter + [Test Target] --cuda-graph-backend-decode + """ + + @classmethod + def setUpClass(cls): + cls.model = MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.configs = get_capture_configs() + + def _run_variant(self, config: CaptureConfig) -> float: + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(COMMON_ARGS) + list(config.capture_args) + process = popen_launch_server( + self.model, + self.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env=env, + ) + try: + requests.get(self.base_url + "/flush_cache") + args = SimpleNamespace( + model=self.model, + eval_name="gsm8k", + api="completion", + num_shots=8, + num_examples=GSM8K_NUM_QUESTIONS, + num_threads=128, + max_tokens=512, + base_url=self.base_url, + ) + metrics = run_eval(args) + print(f"[{config.variant}] {metrics=}") + return metrics["score"] + finally: + kill_process_tree(process.pid) + + def test_full_decode_graph_gsm8k(self): + summary = "### Kimi-K2.6-W4A8 full decode graph (NPU, TP16)\n\n" + summary += "| Capture backend | Accuracy | Threshold | Status |\n" + summary += "| --------------- | -------- | --------- | ------ |\n" + + failures = [] + for config in self.configs: + with self.subTest(variant=config.variant): + acc = self._run_variant(config) + passed = acc >= ACCURACY_THRESHOLD + status = "PASS" if passed else "FAIL" + summary += ( + f"| {config.variant} | {acc:.3f} | " + f"{ACCURACY_THRESHOLD} | {status} |\n" + ) + if not passed: + failures.append((config.variant, acc)) + + if is_in_ci(): + write_github_step_summary(summary) + + self.assertEqual( + failures, + [], + f"Full decode graph accuracy below {ACCURACY_THRESHOLD}: {failures}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/basic_function/optimization_debug_options/test_npu_no_extra_forked_npu_context.py b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_no_extra_forked_npu_context.py new file mode 100644 index 000000000000..d5f20c51c7f5 --- /dev/null +++ b/test/registered/ascend/basic_function/optimization_debug_options/test_npu_no_extra_forked_npu_context.py @@ -0,0 +1,152 @@ +import subprocess +import time +import unittest + +import psutil + +from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.test_ascend_utils import LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH +from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_npu_ci(est_time=200, suite="full-2-npu-a3", nightly=True) + + +class TestTPServerNPUProcesses(CustomTestCase): + """Testcase: Verify TP server does not create extra NPU processes beyond TP workers. + + [Test Category] Parameter + [Test Target] --cuda-graph-backend-decode; --cuda-graph-backend-prefill + """ + + tp_size = 2 + + @classmethod + def setUpClass(cls): + cls.model = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + str(cls.tp_size), + "--attention-backend", + "ascend", + "--mem-fraction-static", + "0.70", + "--cuda-graph-backend-decode", + "disabled", + "--cuda-graph-backend-prefill", + "disabled", + ], + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + kill_process_tree(cls.process.pid) + + def test_tp_server_has_only_worker_npu_processes(self): + rows = self._wait_for_server_npu_processes() + # On NPU the launcher parent may acquire a lightweight context + # (e.g. HCCL init) — filter it out, only count TP worker processes. + worker_pids = {row["pid"] for row in rows if row["pid"] != self.process.pid} + + self.assertEqual( + len(worker_pids), + self.tp_size, + f"TP={self.tp_size} server should have exactly {self.tp_size} " + f"NPU worker processes, got {len(worker_pids)}: {self._format_rows(rows)}", + ) + + def _wait_for_server_npu_processes(self): + deadline = time.monotonic() + 60 + stable_since = None + last_rows = [] + + while time.monotonic() < deadline: + tree_pids = self._server_process_tree_pids() + rows = [ + row for row in self._query_npu_processes() if row["pid"] in tree_pids + ] + last_rows = rows + + if len({row["pid"] for row in rows}) >= self.tp_size: + if stable_since is None: + stable_since = time.monotonic() + elif time.monotonic() - stable_since >= 3: + return rows + else: + stable_since = None + + time.sleep(0.5) + + self.fail( + f"Timed out waiting for TP={self.tp_size} NPU worker processes. " + f"Last observed rows: {self._format_rows(last_rows)}" + ) + + def _server_process_tree_pids(self): + pids = {self.process.pid} + try: + parent = psutil.Process(self.process.pid) + pids.update(child.pid for child in parent.children(recursive=True)) + except psutil.NoSuchProcess: + pass + return pids + + def _query_npu_processes(self): + result = subprocess.run( + ["npu-smi", "info"], + check=True, + capture_output=True, + text=True, + ) + + rows = [] + pid_col = None # column index of "Process id" header + for line in result.stdout.splitlines(): + if "|" not in line: + continue + parts = [p.strip() for p in line.split("|")] + + # Locate the process table by its "Process id" column header. + if pid_col is None: + for i, part in enumerate(parts): + if part == "Process id": + pid_col = i + break + continue # skip the header row itself + + # Reached the next section header ("NPU" row) → stop. + if any(parts) and parts[1].startswith("NPU"): + break + # Separator line or empty data row → skip. + if "+" in line or not any(parts): + continue + # No data in PID column → skip. + if not parts[pid_col].isdigit(): + continue + + pid = int(parts[pid_col]) + # Safety guard: only accept real OS-level PIDs. + if psutil.pid_exists(pid): + rows.append({"pid": pid}) + + return rows + + def _format_rows(self, rows): + if not rows: + return "[]" + return "[" + ", ".join(str(row) for row in rows) + "]" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/basic_function/runtime_options/test_npu_disaggregation_pp.py b/test/registered/ascend/basic_function/runtime_options/test_npu_disaggregation_pp.py new file mode 100644 index 000000000000..01a9806f4a43 --- /dev/null +++ b/test/registered/ascend/basic_function/runtime_options/test_npu_disaggregation_pp.py @@ -0,0 +1,223 @@ +import os +import time +import unittest +from types import SimpleNamespace + +from sglang.test.ascend.disaggregation_utils import TestDisaggregationBase +from sglang.test.ascend.test_ascend_utils import LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH +from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_pd_server, +) + +register_npu_ci(est_time=400, suite="full-16-npu-a3", nightly=True) + + +class TestDisaggregationPrefillPPAccuracy(TestDisaggregationBase): + """Test Case: Verify the accuracy of base model when only prefill enables PP parallelism in PD disaggregation scenario + + [Test Category] Parameter + [Test Target] --pp-size + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH + os.environ["ASCEND_MF_STORE_URL"] = "tcp://127.0.0.1:24666" + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) + + cls.launch_lb() + os.environ["OPENAI_API_KEY"] = "sk-123456" + os.environ["OPENAI_API_BASE"] = f"http://{cls.base_host}:{cls.lb_port}/v1" + + @classmethod + def tearDownClass(cls): + os.environ.pop("ASCEND_MF_STORE_URL") + os.environ.pop("OPENAI_API_KEY", None) + os.environ.pop("OPENAI_API_BASE", None) + super().tearDownClass() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp-size", + "1", + "--pp-size", + "4", + "--disable-overlap-schedule", + "--attention-backend", + "ascend", + "--disaggregation-transfer-backend", + "ascend", + ] + prefill_args += cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp-size", + "1", + "--base-gpu-id", + "8", + "--attention-backend", + "ascend", + "--disaggregation-transfer-backend", + "ascend", + "--disable-overlap-schedule", + "--disable-cuda-graph", + ] + decode_args += cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + self.assertGreater(metrics["score"], 0.24) + # Wait a little bit so that the memory check happens. + time.sleep(5) + + +class TestDisaggregationDecodePPAccuracy(TestDisaggregationBase): + """Test Case: Verify the accuracy of base model when both prefill and decode enable PP parallelism in PD disaggregation scenario + + [Test Category] Parameter + [Test Target] --pp-size; --pp-async-batch-depth; --pp-max-micro-batch-size + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH + os.environ["ASCEND_MF_STORE_URL"] = "tcp://127.0.0.1:24666" + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) + cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) + + cls.launch_lb() + os.environ["OPENAI_API_KEY"] = "sk-123456" + os.environ["OPENAI_API_BASE"] = f"http://{cls.base_host}:{cls.lb_port}/v1" + + @classmethod + def tearDownClass(cls): + os.environ.pop("ASCEND_MF_STORE_URL") + os.environ.pop("OPENAI_API_KEY", None) + os.environ.pop("OPENAI_API_BASE", None) + super().tearDownClass() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp-size", + "2", + "--pp-size", + "4", + "--pp-async-batch-depth", + "2", + "--pp-max-micro-batch-size", + "2", + "--disable-overlap-schedule", + "--attention-backend", + "ascend", + "--disaggregation-transfer-backend", + "ascend", + ] + prefill_args += cls.rdma_devices + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp-size", + "2", + "--pp-size", + "4", + "--base-gpu-id", + "8", + "--attention-backend", + "ascend", + "--disaggregation-transfer-backend", + "ascend", + "--disable-overlap-schedule", + "--disable-cuda-graph", + ] + decode_args += cls.rdma_devices + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + self.assertGreater(metrics["score"], 0.24) + # Wait a little bit so that the memory check happens. + time.sleep(5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/basic_function/runtime_options/test_npu_pp_single_node.py b/test/registered/ascend/basic_function/runtime_options/test_npu_pp_single_node.py new file mode 100644 index 000000000000..95c89001f7ec --- /dev/null +++ b/test/registered/ascend/basic_function/runtime_options/test_npu_pp_single_node.py @@ -0,0 +1,250 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.bench_one_batch_server import BenchArgs as OneBatchBenchArgs +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree +from sglang.test.ascend.test_ascend_utils import ( + DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH, + LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_bench_one_batch_server, +) + +register_npu_ci(est_time=10800, suite="full-16-npu-a3", nightly=True) + + +class TestPPAccuracy(unittest.TestCase): + """Test Case: Verify the accuracy of LLM models under TP+PP hybrid parallelism + + [Test Category] Parameter + [Test Target] --pp-size; --tp-size + """ + + @classmethod + def setUpClass(cls): + cls.model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH + cls.base_url = "http://127.0.0.1:23333" + cls.process = popen_launch_server( + LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "2", + "--pp-size", + "4", + "--chunked-prefill-size", + "256", + "--attention-backend", + "ascend", + "--mem-fraction-static", + "0.8", + "--disable-cuda-graph", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + self.assertGreater(metrics["score"], 0.74) + # Wait a little bit so that the memory check happens. + time.sleep(4) + + def test_logprob(self): + # Test the format correctness of logprob returned under TP+PP hybrid parallelism + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "logprob_start_len": 0, + }, + ) + response_json = response.json() + input_token_logprobs = response_json["meta_info"]["input_token_logprobs"] + output_token_logprobs = response_json["meta_info"]["output_token_logprobs"] + output_top_logprobs = response_json["meta_info"]["output_top_logprobs"] + + assert len(input_token_logprobs) == 6 + assert len(output_token_logprobs) == 16 + assert len(output_top_logprobs) == 16 + + +class TestDPAttentionDP2PP2(CustomTestCase): + """Test Case: Verify the accuracy of MLA models under TP+DP+PP hybrid parallelism + + [Test Category] Parameter + [Test Target] --pp-size; --tp-size; --dp + """ + + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--pp-size", + "4", + "--enable-dp-attention", + "--dp", + "2", + "--attention-backend", + "ascend", + "--mem-fraction-static", + "0.8", + "--max-running-requests", + "32", + "--context-length", + "16384", + "--disable-cuda-graph", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.8) + + +class TestPPMixedChunk(CustomTestCase): + """Test Case: Verify the accuracy of base model when PP + mixed chunk are both enabled + + [Test Category] Parameter + [Test Target] --pp-size; --enable-mixed-chunk + """ + + @classmethod + def setUpClass(cls): + cls.model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH + cls.base_url = "http://127.0.0.1:23338" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "2", + "--pp-size", + "4", + "--chunked-prefill-size", + "256", + "--enable-mixed-chunk", + "--attention-backend", + "ascend", + "--mem-fraction-static", + "0.8", + "--disable-cuda-graph", + ], + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process"): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(f"{metrics=}") + + self.assertGreater(metrics["score"], 0.74) + time.sleep(4) + + +class TestFixedBugs(unittest.TestCase): + """Test Case: Verify normal inference under small batch size scenario with PP+chunked-prefill enabled + [Test Category] Parameter + [Test Target] --pp-size; --chunked-prefill-size + """ + + def test_chunked_prefill_with_small_bs(self): + model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH + server_args = ServerArgs(model_path=model) + bench_args = OneBatchBenchArgs( + batch_size=(1,), + input_len=(1,), + output_len=(1,), + base_url=DEFAULT_URL_FOR_TEST, + ) + other_server_args = [ + "--tp-size", + "2", + "--pp-size", + "4", + "--chunked-prefill-size", + "256", + "--max-running-requests", + "2", + "--attention-backend", + "ascend", + "--mem-fraction-static", + "0.8", + "--disable-cuda-graph", + ] + run_bench_one_batch_server( + model, + DEFAULT_URL_FOR_TEST, + server_args, + bench_args, + other_server_args, + ) + + +if __name__ == "__main__": + unittest.main()