From 13908d26d0e7325b6cdbac89735543fa578f7a7a Mon Sep 17 00:00:00 2001 From: jschen069 <3563624058@qq.com> Date: Mon, 17 Aug 2026 20:24:38 +0800 Subject: [PATCH 01/25] fix the logic for displaying steady-state test results (#464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 修改初版代码 * 添加修改逻辑 * 添加测试用例 --- .../stable_perf_metric_calculator.py | 121 ++++++++---------- .../test_stable_perf_metric_calculator.py | 69 +++++++++- 2 files changed, 116 insertions(+), 74 deletions(-) diff --git a/ais_bench/benchmark/calculators/stable_perf_metric_calculator.py b/ais_bench/benchmark/calculators/stable_perf_metric_calculator.py index c8cc1ce0..22255211 100644 --- a/ais_bench/benchmark/calculators/stable_perf_metric_calculator.py +++ b/ais_bench/benchmark/calculators/stable_perf_metric_calculator.py @@ -10,18 +10,18 @@ from ais_bench.benchmark.utils.logging.error_codes import CALC_CODES from ais_bench.benchmark.utils.logging.exceptions import AISBenchDataContentError -WAVE_OFFSET = 0.02 -INTERVAL_OFFSET = 0.001 @PERF_METRIC_CALCULATORS.register_module() class StablePerfMetricCalculator(BasePerfMetricCalculator): """ Performance metric calculator for stable stage analysis. - This calculator focuses on analyzing the stable phase of benchmark execution, - where the system operates at maximum concurrency with minimal fluctuations. - It identifies and analyzes the stable period to provide more accurate - performance metrics. + This calculator identifies the stable phase as the interval from the second + request that reaches max_concurrency (the first is treated as a warm-up) to + the last time concurrency is at that level. All requests starting within this + interval are included regardless of concurrency fluctuations, providing a + robust steady-state performance measurement without requiring manual + parameter tuning. Args: stats_list (list, optional): List of statistics to calculate @@ -63,6 +63,12 @@ def _get_requests_id(self, perf_details: dict) -> list: """ Identify requests that belong to the stable stage. + The stable stage is defined as the interval from the second request + that reaches max_concurrency (the first is treated as a warm-up) to + the last moment concurrency is at max_concurrency. All requests with + start_time within this closed interval are included, regardless of + concurrency fluctuations. + Args: perf_details (dict): Performance details dictionary @@ -70,11 +76,12 @@ def _get_requests_id(self, perf_details: dict) -> list: list: List of request IDs in the stable stage Raises: - RuntimeError: If no stable stage can be identified + AISBenchDataContentError: If no stable stage can be identified """ # Calculate the minimum start time as the baseline min_start_time = min(perf_details["start_time"]) - time_point_concurrency = [0] * 2 * len(perf_details["id"]) + + # Build sorted list of start/end events for all requests request_time_sections = [] for id in range(len(perf_details["id"])): request_time_sections.append( @@ -92,84 +99,56 @@ def _get_requests_id(self, perf_details: dict) -> list: } ) sorted_time_sections = sorted(request_time_sections, key=lambda x: x["time"]) - id_lists = [] + self.logger.info("Starting stable stage calculation...") - requested = 0 + first_max_time = None + last_max_time = None + concurrency = 0 + max_start_count = 0 + progress_bar = tqdm( total=len(sorted_time_sections), desc="Calculating stable stage", unit=" req", ) - for i, section in enumerate(sorted_time_sections): + + for section in sorted_time_sections: if section["attr"] == "start": - time_point_concurrency[i] = time_point_concurrency[i - 1] + 1 - requested += 1 + concurrency += 1 else: - time_point_concurrency[i] = time_point_concurrency[i - 1] - 1 - if ( - section["attr"] == "start" - and time_point_concurrency[i] == self.max_concurrency - ): - id_lists.append(section["id"]) - if len(id_lists) == 2: - self.stage_section[0] = section["time"] # total start time - elif ( - section["attr"] == "start" - and time_point_concurrency[i] - >= int(self.max_concurrency * (1 - WAVE_OFFSET)) - and len(id_lists) > 2 - ): - id_lists.append(section["id"]) - elif requested == len(perf_details["id"]) and section["attr"] == "end": - self.stage_section[1] = section["time"] - progress_bar.update(len(sorted_time_sections) - progress_bar.n) - break - elif ( - len(id_lists) > 1 - and section["attr"] == "end" - and time_point_concurrency[i] - < int(self.max_concurrency * (1 - WAVE_OFFSET)) - ): - # Check if there's a start event within INTERVAL_OFFSET that recovers concurrency - # Skip consecutive end events and look ahead for start events - should_exit_stable = True - current_time = section["time"] - current_concurrency = time_point_concurrency[i] - - # Look ahead to find the next start event within INTERVAL_OFFSET - for j in range(i + 1, len(sorted_time_sections)): - next_section = sorted_time_sections[j] - time_interval = next_section["time"] - current_time - - # If time interval exceeds INTERVAL_OFFSET, exit stable stage - if time_interval > INTERVAL_OFFSET: - break - - # Update concurrency based on the event type - if next_section["attr"] == "end": - current_concurrency -= 1 - else: # start event - current_concurrency += 1 - # If concurrency recovers to threshold, don't exit stable stage - if current_concurrency >= int( - self.max_concurrency * (1 - WAVE_OFFSET) - ): - should_exit_stable = False - break - - if should_exit_stable: - self.stage_section[1] = section["time"] - progress_bar.update(len(sorted_time_sections) - progress_bar.n) - break + concurrency -= 1 + + if concurrency == self.max_concurrency: + if section["attr"] == "start": + max_start_count += 1 + if max_start_count == 2: + # 第 2 个达到 max 的请求才是稳定阶段起点，忽略首个达到 max 的请求 + first_max_time = section["time"] + last_max_time = section["time"] + progress_bar.update(1) progress_bar.close() - if len(id_lists) > 0: - id_lists.pop(0) # ignore first request that reached max concurrency + + if first_max_time is None: + raise AISBenchDataContentError( + CALC_CODES.CAN_NOT_FIND_STABLE_STAGE, + "Can not find a stable stage from performance results! Please check the conccurency plot.", + ) + + self.stage_section = [first_max_time, last_max_time] + + # Collect all requests whose start_time falls within the stable interval + id_lists = [ + id for id in range(len(perf_details["id"])) + if first_max_time <= perf_details["start_time"][id] <= last_max_time + ] + if len(id_lists) == 0: raise AISBenchDataContentError( CALC_CODES.CAN_NOT_FIND_STABLE_STAGE, "Can not find a stable stage from performance results! Please check the conccurency plot.", ) + # Convert to relative time based on minimum start time relative_start_time = self.stage_section[0] - min_start_time relative_end_time = self.stage_section[1] - min_start_time diff --git a/tests/UT/calculators/test_stable_perf_metric_calculator.py b/tests/UT/calculators/test_stable_perf_metric_calculator.py index cf8c3542..ab785237 100644 --- a/tests/UT/calculators/test_stable_perf_metric_calculator.py +++ b/tests/UT/calculators/test_stable_perf_metric_calculator.py @@ -169,7 +169,68 @@ def test_get_requests_id_concurrency_drop(self): calculator.stage_section = [2.0, 4.0] self.assertTrue(calculator.stage_section[0] > 0) self.assertTrue(calculator.stage_section[1] > calculator.stage_section[0]) - + + def test_get_requests_id_second_max_concurrency_as_start(self): + # 稳定阶段起点应为第 2 个达到 max_concurrency 的请求，而非第 1 个 + calculator = StablePerfMetricCalculator() + calculator.max_concurrency = 2 + calculator.logger = self.mock_logger + calculator.stage_section = [0, 0] + + # req0/req1 爬坡：req1 启动时并发达到 2（第 1 个达到 max） + # req2 启动时并发再次达到 2（第 2 个达到 max）-> 稳定阶段起点 + # req3 启动时并发第三次达到 2（最后一次达到 max）-> 稳定阶段终点 + perf_details = { + "id": [0, 1, 2, 3], + "start_time": [1.0, 2.0, 4.0, 6.0], + "end_time": [3.5, 5.5, 7.0, 9.0], + "success": [True, True, True, True] + } + + result = calculator._get_requests_id(perf_details) + + # 起点 = req2 的 start(4.0)，终点 = 最后一次达到 max 的 req3 的 start(6.0) + self.assertEqual(calculator.stage_section, [4.0, 6.0]) + # 第 1 个达到 max 的 req1 被排除；req2、req3 纳入 + self.assertEqual(result, [2, 3]) + + def test_get_requests_id_single_max_reaching_raises(self): + # 并发只达到 max 一次（不存在第 2 个达到 max 的请求）时应抛出异常 + calculator = StablePerfMetricCalculator() + calculator.max_concurrency = 2 + calculator.logger = self.mock_logger + calculator.stage_section = [0, 0] + + perf_details = { + "id": [0, 1], + "start_time": [1.0, 2.0], + "end_time": [3.0, 4.0], + "success": [True, True] + } + + with self.assertRaises(AISBenchDataContentError): + calculator._get_requests_id(perf_details) + + def test_get_requests_id_excludes_requests_after_last_max(self): + # 稳定阶段终点为最后一次达到 max 的时间，之后启动的请求应被排除 + calculator = StablePerfMetricCalculator() + calculator.max_concurrency = 2 + calculator.logger = self.mock_logger + calculator.stage_section = [0, 0] + + perf_details = { + "id": [0, 1, 2, 3, 4], + "start_time": [1.0, 2.0, 3.5, 5.0, 8.0], + "end_time": [3.0, 4.0, 6.0, 7.0, 9.0], + "success": [True, True, True, True, True] + } + + result = calculator._get_requests_id(perf_details) + + self.assertEqual(calculator.stage_section, [3.5, 5.0]) + # req2(3.5)、req3(5.0) 落在区间内；req4(8.0) 在终点之后被排除 + self.assertEqual(result, [2, 3]) + def test_process_result(self): # 测试处理结果方法 calculator = StablePerfMetricCalculator() @@ -300,11 +361,13 @@ def test_convert_result_integration(self): def test_edge_case_empty_stable_stage(self): # 测试边缘情况 - 稳定阶段只有少量请求 + # 单个请求达到 max_concurrency=1 时，作为首个达到 max 的请求会被忽略（pop）， + # 不存在第 2 个达到 max 的请求，因此应抛出异常 calculator = StablePerfMetricCalculator() calculator.max_concurrency = 1 calculator.logger = self.mock_logger calculator.stage_section = [0, 0] # 初始化必要的属性 - + # 准备测试数据 - 只有一个请求 perf_details = { "id": [0], @@ -312,7 +375,7 @@ def test_edge_case_empty_stable_stage(self): "end_time": [2.0], "success": [True] } - + # 验证抛出异常 with self.assertRaises(AISBenchDataContentError): calculator._get_requests_id(perf_details) From 24416660b2d0825badff17487c3150d2e8f08609 Mon Sep 17 00:00:00 2001 From: jschen069 <3563624058@qq.com> Date: Wed, 19 Aug 2026 15:43:51 +0800 Subject: [PATCH 02/25] delete redundent pkg (#461) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加ttft的bug * 修改判断逻辑 * 删除冗余依赖 --- requirements/runtime.txt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 94028faa..751f4a72 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -7,12 +7,10 @@ evaluate>=0.3.0 func_timeout fuzzywuzzy gradio-client -h5py httpx immutabledict importlib-metadata jieba -json5 jsonlines mmengine-lite nltk>=3.7 @@ -22,14 +20,11 @@ opencv-python-headless orjson pandas plotly -prettytable protobuf pyext==0.5 python-Levenshtein -rank_bm25==0.2.2 rapidfuzz requests>=2.31.0 -retrying rich rouge rouge_chinese @@ -40,14 +35,12 @@ scipy seaborn tabulate tiktoken -timeout_decorator tokenizers torch tqdm>=4.64.1 transformers tree-sitter==0.21.3 tree_sitter_languages>=1.10.2 -typer Pillow==11.2.1 janus emoji From 2cb058b15d5a5048317681a6d71f47c53546bff4 Mon Sep 17 00:00:00 2001 From: Bo Lee Date: Wed, 19 Aug 2026 17:14:35 +0800 Subject: [PATCH 03/25] =?UTF-8?q?[Feature]AISBench=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=A4=A7=E6=A8=A1=E5=9E=8B=E5=93=8D=E5=BA=94=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=20(#468)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add msProbe response anomaly detection for service inference * feat: support configurable msProbe model files and auto-generation - Move model-specific msProbe settings into model-level response_anomaly config - Use ILLDetector with configurable config.yaml/mtype_config.json/token2category paths - Add ais_bench-gen-response-anomaly-config tool wrapping the official msProbe generator - Auto-generate model configs from model_path when custom paths are absent - Make status board, resume inheritance and unsupported-mode gating robust * fix: harden response anomaly streaming accumulation and cleanup - Replace same-length non-prefix snapshots instead of dropping them - Reuse ResponseAnomalyCoordinator.STATUS_FILE_NAME in all call sites - Keep gen_model_config tools dir on failure and add tests - Remove redundant model_name fallback and simplify status file read * fix: enforce anomaly logprobs config and keep payload aligned - Force logprobs/top_logprobs when response anomaly is enabled and validate - Extend monitor progress total for auxiliary tasks - Handle mixed full-token/current-topk stream chunks and ignore misaligned ones * fix: protect mtype config on failure, use generated config.yaml, and match id+uuid on resume - Run the official msProbe generator inside an isolated temp directory so a partial failure never clobbers the target mtype_config.json; merge generated files into the target only on success - Overwrite None-valued msprobe_config_path (set by ConfigManager) with the auto-generated path so user-tuned config.yaml thresholds take effect - Match inherited results on id+uuid instead of id alone so a re-inferred response is not assigned a stale anomaly result - Add tests for all three fixes * fix: preserve duplicate stream tokens and cache detector per model - Change the full-snapshot branch from >= to > so that a single-token chunk equal to the current state (two consecutive identical tokens in an incremental stream) falls through to the incremental-append branch instead of being silently dropped as a no-op snapshot - Cache the per-model anomaly config and ILLDetector so that a model with multiple datasets only generates its msProbe config and loads token2category once instead of once per dataset - Add test_stream_consecutive_identical_tokens_are_not_dropped * fix: warn on empty predictions and trace dropped misaligned chunks - Log a debug message when a misaligned streaming response chunk is dropped so payload misalignment is not silently ignored - Warn per model/dataset group when predictions are missing, and warn once when no service model/dataset groups are configured at all, instead of silently finishing with zero analyzed cases - Hoist the result directory mkdir out of the per-prediction loop - Add end-to-end tests for the _detect loop, the empty-prediction warning, and misaligned chunk logging * test: lock _normalize_name behavior to match msProbe official The normalized model name is written by AISBench and looked up by msProbe using the same split+join algorithm, so consecutive separators produce double dashes on both sides (foo__bar -> foo--bar). Lock this behavior with tests so a future cleanup cannot silently break the msProbe sync. * fix: create msProbe generator output directories * docs: add response anomaly model config template * fix: parse vLLM response anomaly logprobs * test: add response anomaly fault proxy * Revert "test: add response anomaly fault proxy" This reverts commit 0855249184335516ba1f1b70b2174b9bc8211e23. * feat: archive anomaly payloads as compressed jsonl * perf: stream response anomaly payload detection * feat: add response anomaly task logs * fix: make response anomaly top logprobs internal * feat: add response anomaly payload retention option * perf: optimize response anomaly payload resume * fix: restore inherited anomaly payloads on resume * fix: deduplicate all-mode payload backfill on resume Avoid writing payloads that are already retained in the seeded archive when resuming in all-retention mode, and stop recording transient payload shard references in detection results. * docs: add response anomaly detection user guide Document the new msProbe response anomaly detection feature in install, mode, models, and cli_args docs (zh + en). Cover: - Optional dependencies (mindstudio-probe + zstandard) via requirements/response_anomaly.txt or response_anomaly extra - Mode support: all/infer/infer_judge only; perf/viz/Agent unsupported - Service model-level response_anomaly config fields - Full CLI args and retention semantics mirrored from zh to en * fix: restore indentation after rebase conflict resolution The manual merge of the spec-decode wrapper and the response anomaly startup in Infer.do_work dropped the method-level indentation of the spec decode snapshot call during rebase conflict resolution. * fix: guard optional zstandard dependency and degrade staging gracefully - tests: use pytest.importorskip('zstandard') so the UT modules skip instead of failing collection when the response_anomaly extra is not installed (CI installs api.txt + extra.txt only) - base_handler: a response anomaly payload staging failure no longer aborts inference/eval; warn once, disable staging, and keep payloads inline in predictions - add regression test for graceful staging degradation * feat: add response_anomaly config to vllm_api_stream_chat Mirror the response_anomaly template block from vllm_api_general_chat into the streaming chat config. VLLMCustomAPIChat already injects return_token_ids request params and base_api handles streaming payload accumulation, so no code change is required. * feat: harden response anomaly config per expert review - Bind detection to the inference stage: log when the coordinator starts right after the runner finishes, and pin the ordering with tests - Enforce msProbe model resources: model_path falls back to the model 'path' (tokenizer dir); missing resources now fail fast with fix guidance instead of silently falling back to msProbe built-ins, and auto-generated config paths are logged - Reject unsupported model classes (only VLLMCustomAPIChat backends can return token ids + top-k logprobs) at config init - Expose per-task anomaly report and print result/payload/log paths whenever anomalies are detected * fix: separate anomaly status from eval board and fail fast on bad msprobe paths - TasksMonitor only reads the response anomaly status file when opted in; eval/judge boards no longer mix the detection task into their table, and the dedicated anomaly board keeps rendering it - Validate that explicitly configured msprobe resources (config/mtype/ token2category) and model_path exist at config time; missing files now abort startup with the offending paths instead of failing every case at runtime - Drop placeholder msprobe paths from the vllm chat config templates so the auto-generation fallback (model path -> tokenizer dir) is the documented default - ResponseAnomalyWait now also warns when cases ended up failed/ unavailable, pointing at the task log for the root cause * fix: keep msprobe path keys in vllm chat templates with empty values Restore msprobe_config_path/msprobe_mtype_path/msprobe_token2category_dir keys in the response_anomaly template blocks so the config surface stays discoverable; empty-string values keep them inert (treated as unset by the existence validation) while placeholder '/path/to/...' values are gone. * feat: generate msprobe resources into configured locations When model_path (local tokenizer) is provided, explicitly configured msprobe_config_path / msprobe_mtype_path / msprobe_token2category_dir now act as generation outputs: the generated resources are placed there, reusing existing files untouched. An empty msprobe_config_path keeps the msProbe built-in default. Without model_path the configured paths must already exist (unchanged). * fix: always print the dedicated anomaly status board The dedicated board only started while detection was still running, but detection usually finishes before the workflow reaches ResponseAnomalyWait (it runs in parallel with eval and is much faster). Combined with evaluation boards no longer rendering detection status, the final detection table disappeared from the console entirely. Start the board whenever a detection status file exists so the final table is always printed; keep skipping it when no status was ever written to avoid the board waiting forever on tasks that never started. * feat: bind response anomaly detection serially to the inference stage Detection now starts and completes inside the Infer worker: the dedicated status board renders right after the inference board and before any evaluation board starts, so the detection table finally appears between infer and eval as requested. ResponseAnomalyWait and its WORK_FLOW entries are removed (no longer needed), and the design doc is updated accordingly. Trade-off: detection no longer overlaps with eval; total runtime grows by the detection duration. * test: guard staging test against missing zstandard test_write_to_json_stages_payload_separately exercises the zstd staging writer, which imports zstandard lazily. In environments without the response_anomaly extra the graceful staging degradation keeps payloads inline, failing the assertNotIn. Skip the test there, matching the guard already used by the other response anomaly test modules. * refactor: simplify response anomaly config initialization * refactor: simplify response anomaly support validation * refactor: centralize anomaly payload storage config * refactor: remove obsolete anomaly concurrency support * refactor: run anomaly status board without extra process * test: isolate stale anomaly status cleanup * refactor: split response anomaly detection flow * docs: move anomaly setup out of model templates --- ais_bench/benchmark/cli/argument_parser.py | 18 +- ais_bench/benchmark/cli/config_manager.py | 446 +++++- ais_bench/benchmark/cli/workers.py | 117 +- .../benchmark/models/api_models/base_api.py | 174 ++- .../models/api_models/vllm_custom_api_chat.py | 5 + .../icl_inferencer/icl_base_inferencer.py | 8 +- .../icl_inferencer/icl_gen_inferencer.py | 11 +- .../icl_inferencer/icl_lmm_gen_inferencer.py | 10 +- .../icl_multiturn_inferencer.py | 10 +- .../output_handler/base_handler.py | 74 +- .../gen_inferencer_output_handler.py | 2 + ais_bench/benchmark/runners/base.py | 68 +- ais_bench/benchmark/utils/config/build.py | 1 + ais_bench/benchmark/utils/response_anomaly.py | 1370 +++++++++++++++++ .../benchmark/utils/response_anomaly_jsonl.py | 233 +++ ais_bench/tools/response_anomaly/__init__.py | 1 + .../response_anomaly/gen_model_config.py | 181 +++ .../base_tutorials/all_params/cli_args.md | 61 +- .../base_tutorials/all_params/mode.md | 3 + .../base_tutorials/all_params/models.md | 14 +- docs/source_en/get_started/install.md | 13 + .../base_tutorials/all_params/cli_args.md | 61 +- .../base_tutorials/all_params/mode.md | 5 + .../base_tutorials/all_params/models.md | 14 +- ...41\345\235\227\350\256\276\350\256\241.md" | 372 +++++ docs/source_zh_cn/get_started/install.md | 13 + requirements/response_anomaly.txt | 4 + setup.py | 6 + tests/UT/cli/test_argument_parser.py | 21 +- tests/UT/cli/test_config_manager.py | 416 ++++- tests/UT/cli/test_workers.py | 166 +- .../test_response_anomaly_payload.py | 307 ++++ .../api_models/test_vllm_custom_api_chat.py | 19 +- .../output_handler/test_base_handler.py | 110 +- tests/UT/runners/test_base.py | 70 +- .../response_anomaly/test_gen_model_config.py | 146 ++ tests/UT/utils/test_response_anomaly.py | 1253 +++++++++++++++ tests/UT/utils/test_response_anomaly_jsonl.py | 91 ++ 38 files changed, 5859 insertions(+), 35 deletions(-) create mode 100644 ais_bench/benchmark/utils/response_anomaly.py create mode 100644 ais_bench/benchmark/utils/response_anomaly_jsonl.py create mode 100644 ais_bench/tools/response_anomaly/__init__.py create mode 100644 ais_bench/tools/response_anomaly/gen_model_config.py create mode 100644 "docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" create mode 100644 requirements/response_anomaly.txt create mode 100644 tests/UT/models/api_models/test_response_anomaly_payload.py create mode 100644 tests/UT/tools/response_anomaly/test_gen_model_config.py create mode 100644 tests/UT/utils/test_response_anomaly.py create mode 100644 tests/UT/utils/test_response_anomaly_jsonl.py diff --git a/ais_bench/benchmark/cli/argument_parser.py b/ais_bench/benchmark/cli/argument_parser.py index c7d535bb..687d502a 100644 --- a/ais_bench/benchmark/cli/argument_parser.py +++ b/ais_bench/benchmark/cli/argument_parser.py @@ -122,6 +122,22 @@ def _base_parser(self): type=validate_num_warmups, default=1 ) + parser.add_argument( + '--response-anomaly', + action=argparse.BooleanOptionalAction, + default=None, + help='Enable or disable msProbe response anomaly detection. ' + 'The command-line value overrides response_anomaly.enabled in the config file. ' + 'Only supported in all/infer/infer_judge modes; perf and Agent modes are unsupported.' + ) + parser.add_argument( + '--response-anomaly-payload-retention', + choices=('all', 'anomalies', 'none'), + default=None, + help='Select which response anomaly payloads to retain after detection. ' + 'The command-line value overrides response_anomaly.payload_retention ' + 'in the config file. Defaults to anomalies when neither is set.' + ) def _accuracy_parser(self): """These args are all for the accuracy evaluation.""" @@ -176,5 +192,3 @@ def _custom_dataset_parser(self): type=str, choices=['gen']) - - diff --git a/ais_bench/benchmark/cli/config_manager.py b/ais_bench/benchmark/cli/config_manager.py index a0425fcc..ce6ea148 100644 --- a/ais_bench/benchmark/cli/config_manager.py +++ b/ais_bench/benchmark/cli/config_manager.py @@ -10,6 +10,20 @@ from ais_bench.benchmark.utils.config.run import try_fill_in_custom_cfgs from ais_bench.benchmark.utils.logging.exceptions import CommandError, AISBenchConfigError from ais_bench.benchmark.cli.utils import fill_model_path_if_datasets_need, fill_test_range_use_num_prompts, recur_convert_config_type +from ais_bench.benchmark.utils.response_anomaly import ResponseAnomalyCoordinator + +RESPONSE_ANOMALY_TOP_LOGPROBS = 20 + +# Backends allowed to enable response anomaly detection. New backends that can +# return token ids + top-k logprobs should subclass VLLMCustomAPIChat (then +# this check passes automatically) or be added to the name tuple below when +# configured by class-name string. +RESPONSE_ANOMALY_SUPPORTED_MODEL_NAMES = ( + 'VLLMCustomAPIChat', + 'VLLMCustomAPIChatStream', + 'VllmMultiturnAPIChatStream', +) + class CustomConfigChecker: MODEL_REQUIRED_FIELDS = ['abbr'] @@ -98,12 +112,429 @@ def search_configs_location(self): def load_config(self, workflow): self.cfg = self._get_config_from_arg() + self._init_response_anomaly_config() self._update_and_init_work_dir() + self._inject_response_anomaly_payload_storage() self._fill_dataset_configs() self._update_cfg_of_workflow(workflow) self._dump_and_reload_config() return self.cfg + def _init_response_anomaly_config(self): + """Normalize the optional response anomaly detection configuration.""" + global_cfg, configured_top_logprobs = ( + self._normalize_response_anomaly_global_config() + ) + self.cfg['response_anomaly'] = global_cfg + if not global_cfg['enabled']: + return + + self._validate_response_anomaly_top_logprobs(configured_top_logprobs) + self._validate_response_anomaly_support() + self._validate_response_anomaly_payload_config(global_cfg) + + service_models = self._get_response_anomaly_service_models() + if service_models is None: + return + self._warn_shared_response_anomaly_model_name(global_cfg, service_models) + for model_cfg in service_models: + self._init_response_anomaly_model(model_cfg, global_cfg) + + def _inject_response_anomaly_payload_storage(self): + """Add runtime payload storage settings to supported model configs.""" + anomaly_cfg = self.cfg.get('response_anomaly') or {} + if not anomaly_cfg.get('enabled', False): + return + models = self.cfg.get('models') + if not isinstance(models, list): + return + storage_cfg = dict(anomaly_cfg.get('payload_storage') or {}) + for model_cfg in models: + if model_cfg.get('attr', 'service') != 'service': + continue + model_cfg['response_anomaly_payload_storage'] = { + 'work_dir': self.cfg['work_dir'], + 'model_abbr': model_cfg['abbr'], + **storage_cfg, + } + + def _normalize_response_anomaly_global_config(self): + """Apply CLI overrides and defaults to the global anomaly config.""" + raw_anomaly_cfg = self.cfg.get('response_anomaly') or {} + global_cfg = dict(raw_anomaly_cfg) if isinstance(raw_anomaly_cfg, dict) else {} + cli_enabled = getattr(self.args, 'response_anomaly', None) + if isinstance(cli_enabled, bool): + global_cfg['enabled'] = cli_enabled + global_cfg.setdefault('enabled', False) + configured_top_logprobs = global_cfg.pop('top_logprobs', None) + global_cfg.setdefault('msprobe_config_path', None) + cli_payload_retention = getattr( + self.args, 'response_anomaly_payload_retention', None + ) + if isinstance(cli_payload_retention, str): + global_cfg['payload_retention'] = cli_payload_retention + global_cfg.setdefault('payload_retention', 'anomalies') + payload_storage = dict(global_cfg.get('payload_storage') or {}) + payload_storage.setdefault('format', 'jsonl') + payload_storage.setdefault('compression', 'zstd') + payload_storage.setdefault('compression_level', 3) + payload_storage.setdefault('rows_per_shard', 2000) + global_cfg['payload_storage'] = payload_storage + return global_cfg, configured_top_logprobs + + @staticmethod + def _validate_response_anomaly_top_logprobs(configured_top_logprobs): + """Reject attempts to override the detector's fixed top-k value.""" + if ( + configured_top_logprobs is not None + and ( + not isinstance(configured_top_logprobs, int) + or isinstance(configured_top_logprobs, bool) + or configured_top_logprobs != RESPONSE_ANOMALY_TOP_LOGPROBS + ) + ): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly.top_logprobs is fixed at " + f"{RESPONSE_ANOMALY_TOP_LOGPROBS} and cannot be configured.", + ) + + @staticmethod + def _validate_response_anomaly_payload_config(global_cfg): + """Validate payload retention and compressed storage settings.""" + if global_cfg['payload_retention'] not in ('all', 'anomalies', 'none'): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly.payload_retention must be one of " + "'all', 'anomalies' or 'none'.", + ) + payload_storage = global_cfg['payload_storage'] + if payload_storage['format'] != 'jsonl': + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly.payload_storage.format must be 'jsonl'.", + ) + if payload_storage['compression'] != 'zstd': + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly.payload_storage.compression must be 'zstd'.", + ) + compression_level = payload_storage['compression_level'] + rows_per_shard = payload_storage['rows_per_shard'] + if ( + not isinstance(compression_level, int) + or isinstance(compression_level, bool) + or not 1 <= compression_level <= 22 + ): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly.payload_storage.compression_level must be " + "an integer between 1 and 22.", + ) + if ( + not isinstance(rows_per_shard, int) + or isinstance(rows_per_shard, bool) + or rows_per_shard <= 0 + ): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly.payload_storage.rows_per_shard must be a " + "positive integer.", + ) + + def _get_response_anomaly_service_models(self): + """Return configured service models, preserving absent-model behavior.""" + models = self.cfg.get('models') + if not isinstance(models, list): + return None + service_models = [ + model_cfg + for model_cfg in models + if model_cfg.get('attr', 'service') == 'service' + ] + if not service_models: + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly is enabled but no service model is configured. " + "Response anomaly detection requires service models (attr='service').", + ) + return service_models + + def _warn_shared_response_anomaly_model_name( + self, global_cfg, service_models + ): + """Warn when one global model name may be applied to multiple models.""" + if ( + len(service_models) > 1 + and global_cfg.get('model_name') + and any( + 'model_name' not in (model_cfg.get('response_anomaly') or {}) + for model_cfg in service_models + ) + ): + self.logger.warning( + "response_anomaly.model_name is configured globally while multiple " + "service models are present; prefer setting model_name inside each " + "model's response_anomaly config." + ) + + def _init_response_anomaly_model(self, model_cfg, global_cfg): + """Build and validate one model's anomaly detection configuration.""" + model_anomaly_cfg = self._merge_response_anomaly_model_config( + model_cfg, global_cfg + ) + self._resolve_response_anomaly_model_path(model_cfg, model_anomaly_cfg) + self._validate_response_anomaly_model_resources( + model_cfg, model_anomaly_cfg + ) + self._validate_response_anomaly_resource_paths( + model_cfg, model_anomaly_cfg + ) + model_cfg['response_anomaly'] = model_anomaly_cfg + self._inject_response_anomaly_request_config(model_cfg) + + def _merge_response_anomaly_model_config(self, model_cfg, global_cfg): + """Merge global model resources with model-level overrides.""" + model_anomaly_cfg = dict(model_cfg.get('response_anomaly') or {}) + configured_top_logprobs = model_anomaly_cfg.pop('top_logprobs', None) + self._validate_response_anomaly_top_logprobs(configured_top_logprobs) + model_anomaly_cfg.setdefault( + 'model_name', + global_cfg.get('model_name') or model_cfg.get('abbr'), + ) + for key in ( + 'model_path', + 'msprobe_config_path', + 'msprobe_mtype_path', + 'msprobe_token2category_dir', + ): + if key not in model_anomaly_cfg: + model_anomaly_cfg[key] = global_cfg.get(key) + return model_anomaly_cfg + + @staticmethod + def _resolve_response_anomaly_model_path(model_cfg, model_anomaly_cfg): + """Use the model tokenizer path when no explicit model path is set.""" + if model_anomaly_cfg.get('model_path'): + return + model_path = str(model_cfg.get('path') or '').strip() + if not model_path: + return + if not osp.isdir(model_path): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response_anomaly is enabled for model " + f"'{model_cfg.get('abbr', '')}' but its 'path' field " + f"points to a non-existent directory: {model_path}. " + "Fix the model 'path' or configure " + "response_anomaly.model_path / msprobe paths.", + ) + model_anomaly_cfg['model_path'] = model_path + + @staticmethod + def _validate_response_anomaly_model_resources(model_cfg, model_anomaly_cfg): + """Require either a tokenizer path or explicit model resources.""" + if model_anomaly_cfg.get('model_path') or ( + model_anomaly_cfg.get('msprobe_mtype_path') + and model_anomaly_cfg.get('msprobe_token2category_dir') + ): + return + missing = [ + "response_anomaly.model_path is not set and the model " + "'path' field (tokenizer directory) is empty" + ] + if not model_anomaly_cfg.get('msprobe_mtype_path'): + missing.append("response_anomaly.msprobe_mtype_path is not set") + if not model_anomaly_cfg.get('msprobe_token2category_dir'): + missing.append( + "response_anomaly.msprobe_token2category_dir is not set" + ) + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response_anomaly is enabled for model " + f"'{model_cfg.get('abbr', '')}' but no msProbe model " + "resources are available: the token2category vocabulary is " + "model-specific and cannot fall back to msProbe built-in " + "defaults. Missing:\n" + + "\n".join(f" - {item}" for item in missing) + + "\nProvide one of the following:\n" + " 1) set the model 'path' to the local tokenizer directory " + "so the msProbe config files and token2category vocabulary " + "are auto-generated;\n" + " 2) set response_anomaly.model_path to the local " + "model/tokenizer directory;\n" + " 3) generate them manually with " + "`ais_bench-gen-response-anomaly-config --model-path ` " + "and set msprobe_mtype_path together with " + "msprobe_token2category_dir.", + ) + + @staticmethod + def _validate_response_anomaly_resource_paths(model_cfg, model_anomaly_cfg): + """Validate paths that will not be generated from a local model.""" + invalid_paths = [] + model_path = model_anomaly_cfg.get('model_path') + if model_path and not osp.isdir(model_path): + invalid_paths.append(f"model_path={model_path} (directory not found)") + if not model_path: + path_checks = ( + ('msprobe_config_path', osp.isfile, 'file'), + ('msprobe_mtype_path', osp.isfile, 'file'), + ('msprobe_token2category_dir', osp.isdir, 'directory'), + ) + for key, exists, path_type in path_checks: + path = model_anomaly_cfg.get(key) + if path and not exists(path): + invalid_paths.append(f"{key}={path} ({path_type} not found)") + if not invalid_paths: + return + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response_anomaly is enabled for model " + f"'{model_cfg.get('abbr', '')}' but some configured msProbe " + "resources do not exist:\n" + + "\n".join(f" - {item}" for item in invalid_paths) + + "\nCheck that the paths are mounted/copied to this machine, " + "or re-generate them with " + "`ais_bench-gen-response-anomaly-config --model-path `.", + ) + + @staticmethod + def _inject_response_anomaly_request_config(model_cfg): + """Force the service response fields required by anomaly detection.""" + generation_kwargs = model_cfg.setdefault('generation_kwargs', {}) + if not isinstance(generation_kwargs, dict): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + "response_anomaly is enabled but " + f"model '{model_cfg.get('abbr', '')}' has invalid " + "generation_kwargs; expected a dict.", + ) + generation_kwargs['logprobs'] = True + generation_kwargs['top_logprobs'] = RESPONSE_ANOMALY_TOP_LOGPROBS + generation_kwargs['response_anomaly_enabled'] = True + + @staticmethod + def _is_supported_response_anomaly_model(model_cfg: dict) -> bool: + """Only chat backends returning token ids + top-k logprobs are allowed.""" + model_type = model_cfg.get('type') + if isinstance(model_type, str): + return model_type in RESPONSE_ANOMALY_SUPPORTED_MODEL_NAMES + if not isinstance(model_type, type): + return False + from ais_bench.benchmark.models.api_models.vllm_custom_api_chat import ( + VLLMCustomAPIChat, + ) + return issubclass(model_type, VLLMCustomAPIChat) + + def _validate_response_anomaly_support(self): + """Reject modes/links that are intentionally unsupported.""" + self._validate_response_anomaly_mode() + self._validate_response_anomaly_infer_task() + self._validate_response_anomaly_models() + self._validate_response_anomaly_datasets() + + def _validate_response_anomaly_mode(self): + """Allow response anomaly detection only in inference workflows.""" + mode = getattr(self.args, 'mode', 'all') + if isinstance(mode, str) and mode not in ('all', 'infer', 'infer_judge'): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response anomaly detection is not supported in mode " + f"'{mode}'; supported modes are 'all', 'infer' and " + "'infer_judge'.", + ) + + def _validate_response_anomaly_infer_task(self): + """Reject custom inference tasks that bypass the supported pipeline.""" + infer_cfg = self.cfg.get('infer') + if not isinstance(infer_cfg, dict): + return + task_type = (infer_cfg.get('runner') or {}).get('task', {}).get('type') + task_name = self._cfg_type_name(task_type) + if task_name and task_name not in ('OpenICLInferTask', 'OpenICLApiInferTask'): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response anomaly detection is not supported for infer task " + f"'{task_name}' (Agent/custom tasks are not supported).", + ) + + def _validate_response_anomaly_models(self): + """Validate every configured model against the supported backends.""" + models = self.cfg.get('models') + if not isinstance(models, list): + return + for model_cfg in models: + if isinstance(model_cfg, dict): + self._validate_response_anomaly_model_support(model_cfg) + + def _validate_response_anomaly_model_support(self, model_cfg): + """Reject Agent and service backends without anomaly payload support.""" + agent_keys = ('agent', 'agent_name', 'llm_agent', 'llm_user') + if any(key in model_cfg for key in agent_keys): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response anomaly detection is not supported for Agent " + f"models (model abbr='{model_cfg.get('abbr', '')}').", + ) + if model_cfg.get('attr', 'service') != 'service': + return + if self._is_supported_response_anomaly_model(model_cfg): + return + model_type = model_cfg.get('type') + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response anomaly detection is not supported for model " + f"type '{getattr(model_type, '__name__', model_type)}' " + f"(model abbr='{model_cfg.get('abbr', '')}'): only " + "VLLMCustomAPIChat backends return the required token " + "ids and top-k logprobs. Use the vllm_api_general_chat / " + "vllm_api_stream_chat / vllm_api_stream_chat_multiturn " + "model configs instead.", + ) + + def _validate_response_anomaly_datasets(self): + """Validate every dataset against unsupported Agent-style links.""" + datasets = self.cfg.get('datasets') + if not isinstance(datasets, list): + return + for dataset_cfg in datasets: + if isinstance(dataset_cfg, dict): + self._validate_response_anomaly_dataset_support(dataset_cfg) + + def _validate_response_anomaly_dataset_support(self, dataset_cfg): + """Reject datasets whose inferencer uses an Agent-style protocol.""" + infer_cfg = dataset_cfg.get('infer_cfg') or {} + inferencer = infer_cfg.get('inferencer') or {} + inferencer_name = self._cfg_type_name(inferencer.get('type')) + dataset_name = self._cfg_type_name(dataset_cfg.get('type')) + haystack = f"{inferencer_name} {dataset_name}".lower() + unsupported_markers = ( + 'swebench', + 'bfcl', + 'agent', + 'function_call', + 'tool_call', + 'harbor', + 'tau2', + ) + if any(marker in haystack for marker in unsupported_markers): + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response anomaly detection is not supported for Agent/custom " + f"evaluation (inferencer='{inferencer_name}', " + f"dataset='{dataset_name}').", + ) + + @staticmethod + def _cfg_type_name(value) -> str: + """Return the short class name of a config type value.""" + if value is None: + return '' + if isinstance(value, type): + return value.__name__ + return str(value).rsplit('.', 1)[-1] + def _fill_dataset_configs(self): for dataset_cfg in self.cfg["datasets"]: if dataset_cfg.get("infer_cfg", None) is None: @@ -320,6 +751,19 @@ def _update_and_init_work_dir(self): self.logger.info(f'Current exp folder: {current_workdir}') os.makedirs(osp.join(self.cfg.work_dir, 'configs'), exist_ok=True) + # Remove a response anomaly status left by a previous interrupted run so + # stale state never blocks or misleads a new run's task board. + stale_anomaly_status = osp.join( + self.cfg.work_dir, + 'status_tmp', + ResponseAnomalyCoordinator.STATUS_FILE_NAME, + ) + try: + if os.path.isfile(stale_anomaly_status): + os.remove(stale_anomaly_status) + except OSError: + # Best-effort cleanup; a concurrent process may have removed it. + pass def _update_cfg_of_workflow(self, workflow): for work in workflow: @@ -341,4 +785,4 @@ def _dump_and_reload_config(self): try: self.cfg = Config.fromfile(output_config_path, format_python_code=False) except BaseException as e: - raise AISBenchConfigError(TMAN_CODES.INVAILD_SYNTAX_IN_CFG_CONTENT, f'Config file {output_config_path} contain invaild syntax: {e}') \ No newline at end of file + raise AISBenchConfigError(TMAN_CODES.INVAILD_SYNTAX_IN_CFG_CONTENT, f'Config file {output_config_path} contain invaild syntax: {e}') diff --git a/ais_bench/benchmark/cli/workers.py b/ais_bench/benchmark/cli/workers.py index 1dbd4e3f..afe9a212 100644 --- a/ais_bench/benchmark/cli/workers.py +++ b/ais_bench/benchmark/cli/workers.py @@ -17,13 +17,17 @@ from ais_bench.benchmark.utils.logging.exceptions import PredictionInvalidException from ais_bench.benchmark.utils.logging.error_codes import TMAN_CODES from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners import LocalRunner +from ais_bench.benchmark.runners import LocalRunner, TasksMonitor from ais_bench.benchmark.tasks import OpenICLEvalTask, OpenICLApiInferTask, OpenICLInferTask from ais_bench.benchmark.tasks.base import EmptyTask from ais_bench.benchmark.summarizers import DefaultSummarizer, DefaultPerfSummarizer from ais_bench.benchmark.calculators import DefaultPerfMetricCalculator from ais_bench.benchmark.cli.utils import clear_repeat_tasks from ais_bench.benchmark.utils.file.file import load_jsonl, dump_jsonl +from ais_bench.benchmark.utils.response_anomaly import ( + ANOMALY_RESULT_NAMES, + ResponseAnomalyCoordinator, +) logger = AISLogger() @@ -45,6 +49,19 @@ class _SpecDecodeContext: entries: dict[str, _URLSnapshotEntry] = field(default_factory=dict) +def _run_response_anomaly_monitor( + task_names: list, work_dir: str, is_debug: bool +) -> None: + """Run a dedicated status board for the response anomaly task.""" + tasks_monitor = TasksMonitor( + task_names, + work_dir, + is_debug, + include_anomaly_status=True, + ) + tasks_monitor.launch_state_board() + + class BaseWorker(ABC): def __init__(self, args) -> None: self.args = args @@ -125,11 +142,43 @@ def do_work(self, cfg: ConfigDict): spec_ctx = self._spec_decode_before_snapshot(cfg) + if cfg.get('response_anomaly', {}).get('enabled', False): + # Remove a stale status left by a previous interrupted run so the + # inference board does not wait on an outdated ResponseAnomaly state. + stale_status = osp.join( + cfg['work_dir'], + 'status_tmp', + ResponseAnomalyCoordinator.STATUS_FILE_NAME, + ) + try: + if os.path.isfile(stale_status): + os.remove(stale_status) + except OSError as exc: + logger.warning( + "Failed to remove stale response anomaly status file %s: %s", + stale_status, + exc, + ) + runner = RUNNERS.build(cfg.infer.runner) runner(tasks) self._spec_decode_finalize(cfg, spec_ctx) + if cfg.get('response_anomaly', {}).get('enabled', False): + logger.info( + "Inference finished; starting response anomaly detection " + "(bound to the inference stage)..." + ) + self.response_anomaly_coordinator.start(cfg) + # Detection runs serially inside the inference stage: wait for + # it to finish (its status board prints between the inference + # board and any evaluation board) before the workflow continues. + _finalize_response_anomaly_detection( + self.response_anomaly_coordinator, + cfg['work_dir'], + cfg.get('cli_args', {}).get('debug', False), + ) logger.info("Inference tasks completed.") def _merge_datasets(self, tasks): @@ -655,6 +704,70 @@ def _output_spec_decode_results(cfg: ConfigDict) -> None: print(format_spec_decode_na(url, result.get("error"))) +def _finalize_response_anomaly_detection( + coordinator, work_dir: str, is_debug: bool +) -> None: + """Wait for detection to finish and print its status board and summary. + + Called from the Infer worker so detection is serially bound to the + inference stage: the dedicated board renders right after the inference + board and before evaluation starts. + """ + # The dedicated board is the only place detection status is rendered now + # that evaluation boards stay separate. Start it whenever detection has + # produced a status (it may have already finished for tiny datasets), so + # the final table is always printed. Skip it when no status was ever + # written to avoid the board waiting forever on tasks that never started. + anomaly_status_file = osp.join( + work_dir, + 'status_tmp', + ResponseAnomalyCoordinator.STATUS_FILE_NAME, + ) + if coordinator.is_running or osp.isfile(anomaly_status_file): + _run_response_anomaly_monitor( + coordinator.task_names, + work_dir, + is_debug, + ) + coordinator.join() + TasksMonitor.rm_tmp_files(work_dir) + if coordinator.summary: + logger.info( + "Response anomaly detection completed: %s", + coordinator.summary, + ) + for task_name, info in coordinator.anomaly_report.items(): + counts = info.get("counts", {}) + anomalies = { + name: count + for name, count in counts.items() + if name in ANOMALY_RESULT_NAMES and count + } + if anomalies: + logger.warning( + "Response anomalies detected for %s: %s", + task_name, + anomalies, + ) + logger.warning(" detection results: %s", info.get("result_file")) + if info.get("payload_dir"): + logger.warning(" payload archive: %s", info["payload_dir"]) + logger.warning(" task log: %s", info.get("task_log")) + undetected = { + name: count + for name, count in counts.items() + if name in ("failed", "unavailable") and count + } + if undetected: + logger.warning( + "Response anomaly detection did not complete for %s: %s. " + "Check the task log for the root cause.", + task_name, + undetected, + ) + logger.warning(" task log: %s", info.get("task_log")) + + WORK_FLOW = dict( all=[Infer, JudgeInfer, Eval, AccViz], infer=[Infer], @@ -671,8 +784,10 @@ class WorkFlowExecutor: def __init__(self, cfg, workflow) -> None: self.cfg = cfg self.workflow = workflow + self.response_anomaly_coordinator = ResponseAnomalyCoordinator() def execute(self) -> None: for worker in self.workflow: + worker.response_anomaly_coordinator = self.response_anomaly_coordinator cfg = copy.deepcopy(self.cfg) worker.do_work(cfg) diff --git a/ais_bench/benchmark/models/api_models/base_api.py b/ais_bench/benchmark/models/api_models/base_api.py index db5eef7f..b5a3fa77 100644 --- a/ais_bench/benchmark/models/api_models/base_api.py +++ b/ais_bench/benchmark/models/api_models/base_api.py @@ -104,7 +104,12 @@ def __init__( self.url = url self.enable_ssl = enable_ssl self.template_parser = APITemplateParser(self.meta_template) - self.generation_kwargs = generation_kwargs + self.generation_kwargs = dict(generation_kwargs or {}) + # Injected by ConfigManager when response anomaly detection is enabled. + # Popped here so it never reaches the service request body. + self.response_anomaly_enabled = bool( + self.generation_kwargs.pop('response_anomaly_enabled', False) + ) self.verbose = verbose self.session = None self.base_url = self._get_base_url() @@ -226,6 +231,171 @@ async def parse_stream_response(self, data, output): f"{self.__class__.__name__} should be implemented if stream is True", ) + @staticmethod + def _extract_vllm_token_id(item): + """Extract a token id from a vLLM OpenAI-style logprob item.""" + if not isinstance(item, dict): + return None + if 'token_id' in item: + value = item['token_id'] + try: + return int(value) if not isinstance(value, bool) else None + except (TypeError, ValueError): + return None + + value = item.get('token') + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str) and value.startswith('token_id:'): + try: + return int(value.removeprefix('token_id:')) + except ValueError: + return None + return None + + @classmethod + def _extract_vllm_openai_logprobs(cls, candidate: dict, tokens): + """Convert choices[0].logprobs.content into msProbe's top-k maps.""" + logprobs = candidate.get('logprobs') + content = logprobs.get('content') if isinstance(logprobs, dict) else None + if not isinstance(content, list) or not content: + return tokens, None + + sampled_token_ids = [] + topk_logprobs = [] + for token_item in content: + if not isinstance(token_item, dict): + return tokens, None + sampled_token_id = cls._extract_vllm_token_id(token_item) + sampled_token_ids.append(sampled_token_id) + + topk_items = token_item.get('top_logprobs') + if not isinstance(topk_items, list): + return tokens, None + token_logprobs = {} + for topk_item in topk_items: + token_id = cls._extract_vllm_token_id(topk_item) + if token_id is None or 'logprob' not in topk_item: + return tokens, None + token_logprobs[token_id] = topk_item['logprob'] + + # Keep the sampled token even when the server omits it from top-k. + if ( + sampled_token_id is not None + and 'logprob' in token_item + and sampled_token_id not in token_logprobs + ): + token_logprobs[sampled_token_id] = token_item['logprob'] + if not token_logprobs: + return tokens, None + topk_logprobs.append(token_logprobs) + + if not isinstance(tokens, list) or len(tokens) != len(topk_logprobs): + if any(token_id is None for token_id in sampled_token_ids): + return tokens, None + tokens = sampled_token_ids + return tokens, topk_logprobs + + @classmethod + def _extract_response_anomaly_payload(cls, data: dict): + """Extract service-provided token ids and top-k logprobs.""" + candidate = data + choices = data.get('choices') if isinstance(data, dict) else None + if isinstance(choices, list) and choices: + candidate = choices[0] + if not isinstance(candidate, dict): + return None, None + tokens = candidate.get('token_ids', candidate.get('tokens')) + topk_logprobs = candidate.get('topk_logprobs') + if tokens is None and isinstance(data, dict): + tokens = data.get('token_ids', data.get('tokens')) + if topk_logprobs is None and isinstance(data, dict): + topk_logprobs = data.get('topk_logprobs') + if topk_logprobs is None: + tokens, topk_logprobs = cls._extract_vllm_openai_logprobs( + candidate, tokens + ) + return tokens, topk_logprobs + + def _record_response_anomaly_payload(self, data: dict, output: Output) -> None: + """Preserve service-provided token ids and top-k logprobs for msProbe. + + Compatible services expose these fields either at the response root or + in the first choice. Responses without token ids are intentionally not + synthesized: msProbe requires the model vocabulary ids. + """ + if not self.response_anomaly_enabled: + return + tokens, topk_logprobs = self._extract_response_anomaly_payload(data) + if isinstance(tokens, list) and isinstance(topk_logprobs, list): + output.extra_details_data['response_anomaly_payload'] = { + 'tokens': tokens, + 'topk_logprobs': topk_logprobs, + } + + def _accumulate_response_anomaly_payload( + self, data: dict, output: Output + ) -> None: + """Accumulate per-chunk token ids/logprobs for streaming responses. + + Supports both incremental chunks (one new token per chunk) and + full-snapshot chunks (the service resends the complete list). Some + services send the full token list but only the current token's top-k + logprobs; those are handled as incremental appends. + """ + if not self.response_anomaly_enabled: + return + tokens, topk_logprobs = self._extract_response_anomaly_payload(data) + if not isinstance(tokens, list) or not isinstance(topk_logprobs, list): + return + + current = output.extra_details_data.get('response_anomaly_payload') + cur_tokens = (current or {}).get('tokens') or [] + + # Mixed format: full token list so far + topk logprobs for the newly + # generated token. Treat it as an incremental append of the last token + # so the accumulated payload stays aligned. + if ( + len(topk_logprobs) == 1 + and len(tokens) > 1 + and len(tokens) == len(cur_tokens) + 1 + and list(tokens[:-1]) == list(cur_tokens) + ): + tokens = tokens[-1:] + elif len(tokens) != len(topk_logprobs): + # A misaligned chunk cannot be merged without corrupting the + # accumulated payload; drop it but leave a trace for debugging. + self.logger.debug( + "Dropping misaligned response anomaly chunk: " + "%d token ids vs %d topk logprobs", + len(tokens), + len(topk_logprobs), + ) + return + + if current is None: + current = {'tokens': [], 'topk_logprobs': []} + output.extra_details_data['response_anomaly_payload'] = current + + # Full snapshot: the incoming list is strictly longer than what we + # have and its prefix matches. Using ">" (not ">=") ensures that a + # single-token chunk equal to the current state (e.g. two consecutive + # identical tokens in an incremental stream) falls through to the + # incremental-append branch instead of being treated as a no-op + # snapshot that drops the duplicate token. + if len(tokens) > len(cur_tokens) and list(tokens[:len(cur_tokens)]) == list(cur_tokens): + current['tokens'] = list(tokens) + current['topk_logprobs'] = list(topk_logprobs) + # Incremental stream: one new token per chunk. + elif len(tokens) == 1 and cur_tokens: + current['tokens'].append(tokens[0]) + current['topk_logprobs'].append(topk_logprobs[0]) + # Full snapshot whose prefix differs or whose length matches (e.g. the + # service restarted and re-sent a complete list of the same length). + elif len(tokens) >= len(cur_tokens): + current['tokens'] = list(tokens) + current['topk_logprobs'] = list(topk_logprobs) + async def generate( self, input_data: PromptType, @@ -306,6 +476,7 @@ async def stream_infer(self, request_body: dict, output: Output): f"Unexpected response format. Please check 'error_info' in ***_failed.jsonl for more information.", ) await self.parse_stream_response(data, output) + self._accumulate_response_anomaly_payload(data, output) output.success = True else: output.error_info = response.reason @@ -329,6 +500,7 @@ async def text_infer(self, request_body, output: Output): f"Unexpected response format. Please check ***_details.jsonl for more information.", ) await self.parse_text_response(data, output) + self._record_response_anomaly_payload(data, output) output.success = True else: output.error_info = response.reason diff --git a/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py b/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py index 03413794..23a88fad 100644 --- a/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py +++ b/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py @@ -157,6 +157,11 @@ async def get_request_body( messages.append(msg) output.input = messages generation_kwargs = self.generation_kwargs.copy() + if self.response_anomaly_enabled: + # vLLM returns OpenAI-style logprobs with token text by default. + # Token ids are required to convert them into msProbe input. + generation_kwargs['return_token_ids'] = True + generation_kwargs['return_tokens_as_token_ids'] = True generation_kwargs.update({"max_tokens": max_out_len}) # Multi-LoRA: override model field with the resolved LoRA adapter name. lora_model_name = self._resolve_lora_model_name(output) diff --git a/ais_bench/benchmark/openicl/icl_inferencer/icl_base_inferencer.py b/ais_bench/benchmark/openicl/icl_inferencer/icl_base_inferencer.py index de1cf739..9c7906ee 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/icl_base_inferencer.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/icl_base_inferencer.py @@ -7,6 +7,7 @@ from collections import defaultdict import json +import copy from mmengine.dist import is_main_process @@ -43,7 +44,10 @@ def __init__( ) -> None: # basic parameters normalization self.logger = AISLogger() - self.model_cfg = model_cfg + self.model_cfg = copy.deepcopy(model_cfg) + self.response_anomaly_payload_storage = self.model_cfg.pop( + "response_anomaly_payload_storage", None + ) self.batch_size = int(batch_size) if batch_size else 1 if self.batch_size < 1 or self.batch_size > MAX_BATCH_SIZE: @@ -57,7 +61,7 @@ def __init__( self.logger.debug(f"Output JSON file path: {self.output_json_filepath}") # construct model and output handler (if needed, can be changed to lazy build) - self.model: BaseModel = build_model_from_cfg(model_cfg) # type: ignore + self.model: BaseModel = build_model_from_cfg(self.model_cfg) # type: ignore self.output_handler = BaseInferencerOutputHandler() # identify whether the current process is the main process (avoid covering the method with boolean) diff --git a/ais_bench/benchmark/openicl/icl_inferencer/icl_gen_inferencer.py b/ais_bench/benchmark/openicl/icl_inferencer/icl_gen_inferencer.py index e570bcc3..1e0bd13c 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/icl_gen_inferencer.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/icl_gen_inferencer.py @@ -54,8 +54,13 @@ def __init__( self.stopping_criteria = list(stopping_criteria) if stopping_criteria else [] self.gen_field_replace_token = gen_field_replace_token or "" - self.output_handler = GenInferencerOutputHandler(perf_mode=self.perf_mode, - save_every=self.save_every) + self.output_handler = GenInferencerOutputHandler( + perf_mode=self.perf_mode, + save_every=self.save_every, + response_anomaly_payload_storage=( + self.response_anomaly_payload_storage + ), + ) async def do_request( self, data: dict, token_bucket: BoundedSemaphore, session: aiohttp.ClientSession @@ -170,4 +175,4 @@ def get_data_list( for index, timestamp in enumerate(timestamps): if isinstance(timestamp, (int, float)): data_list[index]["timestamp"] = timestamp / 1000 # ms to s - return data_list \ No newline at end of file + return data_list diff --git a/ais_bench/benchmark/openicl/icl_inferencer/icl_lmm_gen_inferencer.py b/ais_bench/benchmark/openicl/icl_inferencer/icl_lmm_gen_inferencer.py index 35a1c227..a5c0b63b 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/icl_lmm_gen_inferencer.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/icl_lmm_gen_inferencer.py @@ -32,7 +32,13 @@ def __init__( **kwargs, ) - self.output_handler = LMMGenInferencerOutputHandler(perf_mode=self.perf_mode, save_every=self.save_every) + self.output_handler = LMMGenInferencerOutputHandler( + perf_mode=self.perf_mode, + save_every=self.save_every, + response_anomaly_payload_storage=( + self.response_anomaly_payload_storage + ), + ) def inference( self, @@ -69,4 +75,4 @@ def batch_inference( ): self.output_handler.report_cache_info_sync( index, input, output, data_abbr, gold - ) \ No newline at end of file + ) diff --git a/ais_bench/benchmark/openicl/icl_inferencer/icl_multiturn_inferencer.py b/ais_bench/benchmark/openicl/icl_inferencer/icl_multiturn_inferencer.py index ea7b10ea..b93e4ac3 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/icl_multiturn_inferencer.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/icl_multiturn_inferencer.py @@ -58,7 +58,13 @@ def __init__( self.stopping_criteria = list(stopping_criteria) if stopping_criteria else [] self.gen_field_replace_token = gen_field_replace_token or "" - self.output_handler = GenInferencerOutputHandler(perf_mode=self.perf_mode, save_every=self.save_every) + self.output_handler = GenInferencerOutputHandler( + perf_mode=self.perf_mode, + save_every=self.save_every, + response_anomaly_payload_storage=( + self.response_anomaly_payload_storage + ), + ) self.infer_mode = infer_mode self.logger.info(f"Multiturn Inferencer infer with mode: {self.infer_mode}") @@ -245,4 +251,4 @@ def get_data_list( max_out_len if max_out_len else self.model.max_out_len ) - return data_list \ No newline at end of file + return data_list diff --git a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py index 3f305541..78d7388b 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py @@ -39,7 +39,12 @@ class BaseInferencerOutputHandler: all_success (bool): Flag indicating if all operations were successful """ - def __init__(self, perf_mode: bool = False, save_every: int = 100) -> None: + def __init__( + self, + perf_mode: bool = False, + save_every: int = 100, + response_anomaly_payload_storage: Optional[dict] = None, + ) -> None: """ Initialize the base inferencer output handler. @@ -55,6 +60,9 @@ def __init__(self, perf_mode: bool = False, save_every: int = 100) -> None: self.perf_mode = perf_mode self.all_success = True self.save_every = save_every + self.response_anomaly_payload_storage = response_anomaly_payload_storage + self._response_anomaly_staging_writer = None + self._response_anomaly_staging_error = None @abstractmethod def get_prediction_result( @@ -156,9 +164,16 @@ def write_to_json(self, save_dir: str, perf_mode: bool) -> None: file_path = Path(save_dir) try: + # Response anomaly payload staging is an auxiliary feature: a + # previous staging failure must not abort prediction writing. # Ensure directory exists Path(save_dir).mkdir(parents=True, exist_ok=True) + for results_dict in self.results_dict.values(): + for result in results_dict.values(): + self._stage_response_anomaly_payload(result) + self._close_response_anomaly_staging_writer() + for data_abbr, results_dict in self.results_dict.items(): if not results_dict: continue @@ -382,6 +397,8 @@ def run_cache_consumer( } json_data.update(result_data) + if not perf_mode and result_data.get("success", True): + self._stage_response_anomaly_payload(json_data) if perf_mode: json_data["db_name"] = db_name self.results_dict[data_abbr][uid] = json_data @@ -410,6 +427,8 @@ def run_cache_consumer( cache_data = [] except Exception as e: + if self._response_anomaly_staging_error is not None: + continue # Continue processing other items self.logger.debug(f"Failed to process item {item}: {str(e)}") continue @@ -419,6 +438,14 @@ def run_cache_consumer( f.writelines(cache_data) f.flush() + try: + self._close_response_anomaly_staging_writer() + except Exception as exc: + self._response_anomaly_staging_error = exc + self.logger.error( + "Failed to finalize response anomaly payload staging: %s", exc + ) + # Handle database file based on performance mode conn.commit() conn.close() @@ -465,3 +492,48 @@ def stop_cache_consumer(self) -> None: raise AISBenchRuntimeError(ICLI_CODES.UNKNOWN_ERROR, f"Failed to send stop signal to cache consumer: {str(e)}") self.logger.debug("Stop signal sent to cache consumer") + def _stage_response_anomaly_payload(self, json_data: dict) -> None: + runtime = self.response_anomaly_payload_storage + payload = json_data.get("response_anomaly_payload") + if not runtime or not isinstance(payload, dict): + return + if self._response_anomaly_staging_error is not None: + # Staging already failed; keep the payload inline in predictions + # and stop retrying so the auxiliary feature never aborts the + # inference/eval pipeline. + return + from ais_bench.benchmark.utils.response_anomaly_jsonl import ( + ResponseAnomalyStagingWriter, + ) + + if self._response_anomaly_staging_writer is None: + self._response_anomaly_staging_writer = ( + ResponseAnomalyStagingWriter(runtime) + ) + try: + self._response_anomaly_staging_writer.write(json_data) + except Exception as exc: + self._response_anomaly_staging_writer = None + self._response_anomaly_staging_error = exc + self.logger.warning( + "Response anomaly payload staging failed and is disabled; " + "payloads will stay inline in predictions: %s", exc + ) + return + json_data.pop("response_anomaly_payload", None) + + def _close_response_anomaly_staging_writer(self) -> None: + writer, self._response_anomaly_staging_writer = ( + self._response_anomaly_staging_writer, + None, + ) + if writer is not None: + try: + writer.close() + except Exception as exc: + self._response_anomaly_staging_error = exc + self.logger.warning( + "Failed to finalize response anomaly payload staging: %s", + exc, + ) + raise diff --git a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py index 5beeffcb..3cae69e4 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py @@ -68,6 +68,8 @@ def get_prediction_result( else output ), } + if isinstance(output, Output) and output.extra_details_data.get('response_anomaly_payload'): + result_data['response_anomaly_payload'] = output.extra_details_data['response_anomaly_payload'] if gold: result_data["gold"] = gold diff --git a/ais_bench/benchmark/runners/base.py b/ais_bench/benchmark/runners/base.py index 5494815b..1487bb01 100644 --- a/ais_bench/benchmark/runners/base.py +++ b/ais_bench/benchmark/runners/base.py @@ -1,5 +1,6 @@ import os import time +import json import psutil import shutil from tqdm import tqdm @@ -13,6 +14,7 @@ from ais_bench.benchmark.utils.logging.logger import AISLogger from ais_bench.benchmark.utils.file import read_and_clear_statuses +from ais_bench.benchmark.utils.response_anomaly import ResponseAnomalyCoordinator def create_progress_bar(finished_count=0, total_count=1000, description="", length=30): @@ -41,9 +43,15 @@ def __init__(self, output_path: str, is_debug: bool = False, refresh_interval:float = 0.3, + include_anomaly_status: bool = False, ): self.logger = AISLogger() self.output_path = output_path + self.task_names = list(task_names) + # Only boards that explicitly opt in (the dedicated response anomaly + # board) render the auxiliary detection status; evaluation boards stay + # separate from the detection task. + self.include_anomaly_status = include_anomaly_status self.tmp_file_path = os.path.join(self.output_path, "status_tmp") self.tmp_file_name_list = [f"tmp_{task_name.replace('/', '_')}.json" for task_name in task_names] if not os.path.exists(self.tmp_file_path): @@ -74,11 +82,27 @@ def is_running_in_background(self): @staticmethod def rm_tmp_files(work_dir: str): - """ - Remove temporary files - """ - if os.path.exists(os.path.join(work_dir, "status_tmp")): - shutil.rmtree(os.path.join(work_dir, "status_tmp")) + """Remove task status files after the monitored stage finishes.""" + tmp_path = os.path.join(work_dir, "status_tmp") + if not os.path.exists(tmp_path): + return + logger = AISLogger() + for name in os.listdir(tmp_path): + path = os.path.join(tmp_path, name) + try: + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) + except OSError as exc: + logger.warning("Failed to remove task status path %s: %s", path, exc) + try: + if not os.listdir(tmp_path): + shutil.rmtree(tmp_path) + except OSError as exc: + logger.warning( + "Failed to remove task status directory %s: %s", tmp_path, exc + ) def launch_state_board(self): if self.is_debug: @@ -94,11 +118,11 @@ def launch_state_board(self): def _is_all_task_done(self): unfinished_tasks = [] - for task_name, state in self.tasks_state_map.items(): + for task_name in self.task_names: + state = self.tasks_state_map[task_name] status = state.get("status") if status not in ("finish", "error", "killed"): unfinished_tasks.append((task_name, status)) - if unfinished_tasks: return False @@ -107,7 +131,30 @@ def _is_all_task_done(self): def _refresh_task_state(self): start_time = time.time() - statuses = read_and_clear_statuses(self.tmp_file_path, self.tmp_file_name_list) + # The dedicated anomaly board reads the atomically replaced status file + # without clearing it. Other stage boards do not opt in. + anomaly_status_file_name = ResponseAnomalyCoordinator.STATUS_FILE_NAME + anomaly_statuses = [] + if self.include_anomaly_status: + anomaly_status_file = os.path.join( + self.tmp_file_path, anomaly_status_file_name + ) + try: + with open(anomaly_status_file, "r", encoding="utf-8") as file: + anomaly_statuses = json.load(file) + except (json.JSONDecodeError, OSError) as exc: + self.logger.debug( + "Failed to read response anomaly status: %s", exc + ) + statuses = read_and_clear_statuses( + self.tmp_file_path, + [ + name + for name in self.tmp_file_name_list + if name != anomaly_status_file_name + ], + ) + statuses.extend(anomaly_statuses) if len(statuses) == 0: # check whether process exist @@ -127,6 +174,11 @@ def _refresh_task_state(self): for status in statuses: # get status information from queue task_name = status['task_name'] + if task_name not in self.tasks_state_map: + self.logger.debug( + "Ignore status for unregistered task: %s", task_name + ) + continue if not self.tasks_state_map[task_name].get('start_time'): self.tasks_state_map[task_name]['start_time'] = time.time() self.tasks_state_map[task_name]['status'] = "start" diff --git a/ais_bench/benchmark/utils/config/build.py b/ais_bench/benchmark/utils/config/build.py index 782402bc..6ef40fce 100644 --- a/ais_bench/benchmark/utils/config/build.py +++ b/ais_bench/benchmark/utils/config/build.py @@ -137,6 +137,7 @@ def build_model_from_cfg(model_cfg: ConfigDict): model_cfg.pop("batch_size", None) model_cfg.pop("abbr", None) model_cfg.pop("attr", None) + model_cfg.pop("response_anomaly", None) model_cfg.pop("summarizer_abbr", None) model_cfg.pop("pred_postprocessor", None) model_cfg.pop("min_out_len", None) diff --git a/ais_bench/benchmark/utils/response_anomaly.py b/ais_bench/benchmark/utils/response_anomaly.py new file mode 100644 index 00000000..01162c53 --- /dev/null +++ b/ais_bench/benchmark/utils/response_anomaly.py @@ -0,0 +1,1370 @@ +"""msProbe response anomaly detection for completed AISBench predictions.""" + +import json +import logging +import os +import shutil +import threading +import time +import uuid +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from ais_bench.benchmark.utils.logging import AISLogger +from ais_bench.benchmark.utils.results import safe_write + + +_ANOMALY_TYPE_NAMES = { + 0: "normal", + 1: "rare_character", + 2: "garbled", + 3: "repetition", + 4: "nan_value", +} + +# anomaly_type_name values that indicate a detected response anomaly +# (as opposed to non-detection statuses such as skipped/failed/unavailable). +ANOMALY_RESULT_NAMES = frozenset( + {"rare_character", "garbled", "repetition", "nan_value", "unknown"} +) + + +class _ThreadLogFilter(logging.Filter): + def __init__(self, thread_id: int) -> None: + super().__init__() + self.thread_id = thread_id + + def filter(self, record: logging.LogRecord) -> bool: + return record.thread == self.thread_id + + +@dataclass +class _DetectionTask: + model_abbr: str + dataset_abbr: str + model_cfg: Dict[str, Any] + prediction_file: Path + predictions: List[Dict[str, Any]] + + +@dataclass +class _GroupProgress: + total: int + completed: int = 0 + counts: Counter[str] = field(default_factory=Counter) + + +@dataclass +class _PayloadState: + retention: str + storage_cfg: Dict[str, Any] + payload_dir: Path + source_dir: Path + staging_dir: Path + archive_is_current: bool + writer: Any = None + retained_keys: set[str] = field(default_factory=set) + + +@dataclass +class _DetectionContext: + task_name: str + task_log_path: str + progress: _GroupProgress + started_at: float + anomaly_cfg: Dict[str, Any] + detector: Any + init_error: Any + result_file: Path + prediction_keys: set[str] + inherited: Dict[str, Dict[str, Any]] + payload: _PayloadState + + +@dataclass +class _ActiveResources: + log_handler: Optional[logging.FileHandler] = None + payload_build_dir: Optional[Path] = None + payload_writers: List[Any] = field(default_factory=list) + + +class ResponseAnomalyCoordinator: + """Run detection serially in Infer while a status board refreshes.""" + + STATUS_TASK_NAME = "ResponseAnomaly" + STATUS_FILE_NAME = "tmp_ResponseAnomaly.json" + + def __init__(self) -> None: + self.logger = AISLogger() + self._thread: Optional[threading.Thread] = None + self._summary: Dict[str, int] = {} + self._task_names: List[str] = [] + self._task_statuses: Dict[str, Dict[str, Any]] = {} + self._anomaly_report: Dict[str, Dict[str, Any]] = {} + + @property + def is_running(self) -> bool: + return bool(self._thread and self._thread.is_alive()) + + @property + def summary(self) -> Dict[str, int]: + return dict(self._summary) + + @property + def anomaly_report(self) -> Dict[str, Dict[str, Any]]: + """Per-task anomaly counts and on-disk locations for user guidance.""" + return {name: dict(info) for name, info in self._anomaly_report.items()} + + @property + def task_names(self) -> List[str]: + return list(self._task_names or [self.STATUS_TASK_NAME]) + + @classmethod + def task_name(cls, model_abbr: str, dataset_abbr: str) -> str: + return f"{cls.STATUS_TASK_NAME}/{model_abbr}/{dataset_abbr}" + + @staticmethod + def task_log_path(model_abbr: str, dataset_abbr: str) -> str: + return ( + Path("logs") + .joinpath("response_anomaly", model_abbr, f"{dataset_abbr}.out") + .as_posix() + ) + + @classmethod + def task_names_from_cfg(cls, cfg: Dict[str, Any]) -> List[str]: + names = [ + cls.task_name(model["abbr"], dataset["abbr"]) + for model in cfg.get("models", []) + if model.get("attr", "service") == "service" + for dataset in cfg.get("datasets", []) + ] + return names or [cls.STATUS_TASK_NAME] + + def start(self, cfg: Dict[str, Any]) -> None: + if self.is_running: + return + self._summary = {} + self._task_names = self.task_names_from_cfg(cfg) + self._task_statuses = {} + self._thread = threading.Thread( + target=self._detect, + args=(cfg,), + name="response-anomaly", + daemon=False, + ) + self._thread.start() + + def join(self) -> None: + if self._thread: + self._thread.join() + + def _open_task_log( + self, work_dir: str, model_abbr: str, dataset_abbr: str + ) -> logging.FileHandler: + log_file = Path(work_dir) / self.task_log_path(model_abbr, dataset_abbr) + log_file.parent.mkdir(parents=True, exist_ok=True) + handler = logging.FileHandler(log_file, mode="w", encoding="utf-8") + handler.setFormatter(self.logger.formatter) + handler.addFilter(_ThreadLogFilter(threading.get_ident())) + self.logger.logger.addHandler(handler) + return handler + + def _close_task_log(self, handler: Optional[logging.FileHandler]) -> None: + if handler is None: + return + handler.flush() + self.logger.logger.removeHandler(handler) + handler.close() + + def _detect(self, cfg: Dict[str, Any]) -> None: + work_dir = cfg["work_dir"] + status_dir = Path(work_dir) / "status_tmp" + status_file = status_dir / self.STATUS_FILE_NAME + counts: Counter[str] = Counter() + resources = _ActiveResources() + try: + task_groups = self._build_detection_tasks(cfg, work_dir) + if not self._initialize_detection_tasks( + task_groups, status_file, work_dir + ): + return + + model_name_warned = False + # Cache per-model config and detector so that a model with multiple + # datasets only generates its msProbe config and initializes the + # ILLDetector once (token2category loading is expensive). + detector_cache: Dict[str, tuple] = {} + for task in task_groups: + model_name_warned = self._detect_task( + task, + cfg, + work_dir, + status_file, + counts, + detector_cache, + resources, + model_name_warned, + ) + + self._summary = dict(counts) + except Exception as exc: + self._handle_detection_failure(status_file, counts, exc) + finally: + self._cleanup_detection_resources( + resources.log_handler, + resources.payload_build_dir, + resources.payload_writers, + ) + + def _build_detection_tasks( + self, cfg: Dict[str, Any], work_dir: str + ) -> List[_DetectionTask]: + """Build model/dataset tasks and load their prediction records.""" + tasks = [] + for model in cfg.get("models", []): + if model.get("attr", "service") != "service": + continue + for dataset in cfg.get("datasets", []): + model_abbr = model["abbr"] + dataset_abbr = dataset["abbr"] + prediction_file = ( + Path(work_dir) + / "predictions" + / model_abbr + / f"{dataset_abbr}.jsonl" + ) + tasks.append( + _DetectionTask( + model_abbr=model_abbr, + dataset_abbr=dataset_abbr, + model_cfg=model, + prediction_file=prediction_file, + predictions=self._read_jsonl(prediction_file), + ) + ) + return tasks + + def _initialize_detection_tasks( + self, + tasks: List[_DetectionTask], + status_file: Path, + work_dir: str, + ) -> bool: + """Initialize status entries and report whether work is available.""" + self._task_names = [ + self.task_name(task.model_abbr, task.dataset_abbr) for task in tasks + ] or [self.STATUS_TASK_NAME] + self._task_statuses = {} + if not tasks: + self.logger.warning( + "Response anomaly detection has no service model/dataset " + "groups to analyze under %s.", + Path(work_dir) / "predictions", + ) + self._post_status( + status_file, + 0, + 0, + Counter(), + "response anomaly finished", + "finish", + ) + return False + for task in tasks: + self._post_status( + status_file, + 0, + len(task.predictions), + Counter(), + "waiting for response anomaly detection", + "start", + self.task_name(task.model_abbr, task.dataset_abbr), + self.task_log_path(task.model_abbr, task.dataset_abbr), + ) + return True + + def _detect_task( + self, + task: _DetectionTask, + cfg: Dict[str, Any], + work_dir: str, + status_file: Path, + counts: Counter[str], + detector_cache: Dict[str, tuple], + resources: _ActiveResources, + model_name_warned: bool, + ) -> bool: + """Detect anomalies and publish results for one model/dataset pair.""" + context, model_name_warned = self._prepare_detection_context( + task, + cfg, + work_dir, + status_file, + counts, + detector_cache, + resources, + model_name_warned, + ) + self.logger.info( + "Response anomaly detecting %s/%s from %s", + task.model_abbr, + task.dataset_abbr, + context.payload.source_dir, + ) + self.logger.info( + "Start detecting %d response anomaly payloads", + max(0, context.progress.total - context.progress.completed), + ) + self._post_status( + status_file, + context.progress.completed, + context.progress.total, + context.progress.counts, + f"streaming response anomaly payloads for {task.dataset_abbr}", + task_name=context.task_name, + task_log_path=context.task_log_path, + ) + detected_keys, shard_rows, context.progress.completed = ( + self._process_staged_payloads( + context.payload, + context.prediction_keys, + context.inherited, + context.anomaly_cfg, + context.detector, + context.init_error, + context.result_file, + status_file, + context.task_name, + context.task_log_path, + context.progress.total, + context.progress.completed, + context.progress.counts, + counts, + ) + ) + context.progress.completed = self._process_prediction_payloads( + task, + context.payload, + context.inherited, + detected_keys, + context.anomaly_cfg, + context.detector, + context.init_error, + context.result_file, + context.progress.completed, + context.progress.counts, + counts, + shard_rows, + resources.payload_writers, + ) + self._finalize_group_payloads( + task, + context.payload, + shard_rows, + status_file, + context.task_name, + context.task_log_path, + context.progress.total, + context.progress.completed, + context.progress.counts, + resources.payload_writers, + ) + resources.payload_build_dir = None + self._finish_detection_task( + context.task_name, + context.task_log_path, + work_dir, + context.result_file, + context.payload.payload_dir, + context.progress.counts, + context.progress.completed, + context.progress.total, + status_file, + context.started_at, + ) + self._close_task_log(resources.log_handler) + resources.log_handler = None + return model_name_warned + + def _prepare_detection_context( + self, + task: _DetectionTask, + cfg: Dict[str, Any], + work_dir: str, + status_file: Path, + counts: Counter[str], + detector_cache: Dict[str, tuple], + resources: _ActiveResources, + model_name_warned: bool, + ) -> tuple[_DetectionContext, bool]: + """Prepare detector, resume state, and payload storage for one task.""" + task_name = self.task_name(task.model_abbr, task.dataset_abbr) + task_log_path = self.task_log_path(task.model_abbr, task.dataset_abbr) + progress = _GroupProgress(total=len(task.predictions)) + started_at = time.perf_counter() + resources.log_handler = self._open_task_log( + work_dir, task.model_abbr, task.dataset_abbr + ) + self.logger.info("Task [%s]", task_name) + self.logger.info("Found %d predictions", progress.total) + if not task.predictions: + self.logger.warning( + "No predictions found for model '%s' dataset '%s'; " + "response anomaly detection will skip this group.", + task.model_abbr, + task.dataset_abbr, + ) + + anomaly_cfg, detector, init_error = self._get_model_detector( + task, + cfg["response_anomaly"], + work_dir, + status_file, + progress.completed, + progress.total, + progress.counts, + task_name, + task_log_path, + detector_cache, + ) + result_file = ( + Path(work_dir) + / "response_anomaly" + / task.model_abbr + / f"{task.dataset_abbr}.jsonl" + ) + prediction_keys = { + f"{item.get('id')}:{item.get('uuid')}" for item in task.predictions + } + inherited = self._load_inherited_results(result_file, prediction_keys) + inherited_names = [ + item.get("anomaly_type_name", "unknown") + for item in inherited.values() + ] + progress.completed += len(inherited) + progress.counts.update(inherited_names) + counts.update(inherited_names) + if inherited: + self.logger.info( + "Found %d completed response anomaly results in cache", + len(inherited), + ) + if not model_name_warned and not anomaly_cfg.get("model_name"): + self.logger.warning( + "response_anomaly.model_name is not set; falling back to model " + "abbr '%s'. msProbe model matching may be degraded.", + task.model_cfg.get("abbr"), + ) + model_name_warned = True + + result_file.parent.mkdir(parents=True, exist_ok=True) + payload = self._prepare_payload_state( + task, + cfg["response_anomaly"], + work_dir, + prediction_keys, + inherited, + resources.payload_writers, + ) + resources.payload_build_dir = payload.staging_dir + context = _DetectionContext( + task_name=task_name, + task_log_path=task_log_path, + progress=progress, + started_at=started_at, + anomaly_cfg=anomaly_cfg, + detector=detector, + init_error=init_error, + result_file=result_file, + prediction_keys=prediction_keys, + inherited=inherited, + payload=payload, + ) + return context, model_name_warned + + def _get_model_detector( + self, + task: _DetectionTask, + global_cfg: Dict[str, Any], + work_dir: str, + status_file: Path, + completed: int, + total: int, + counts: Counter[str], + task_name: str, + task_log_path: str, + detector_cache: Dict[str, tuple], + ) -> tuple: + """Return a cached detector or prepare one for the task model.""" + if task.model_abbr in detector_cache: + self.logger.info( + "Reuse response anomaly detector for model [%s]", + task.model_abbr, + ) + return detector_cache[task.model_abbr] + + anomaly_cfg = self._merge_model_anomaly_config(task.model_cfg, global_cfg) + try: + self.logger.info( + "Preparing response anomaly config for model [%s]", + task.model_abbr, + ) + self._post_status( + status_file, + completed, + total, + counts, + f"preparing response anomaly config for {task.model_abbr}", + task_name=task_name, + task_log_path=task_log_path, + ) + anomaly_cfg = self._prepare_model_config( + task.model_abbr, anomaly_cfg, work_dir + ) + self.logger.info( + "Loading response anomaly detector for model [%s]", + task.model_abbr, + ) + self._post_status( + status_file, + completed, + total, + counts, + f"loading response anomaly detector for {task.model_abbr}", + task_name=task_name, + task_log_path=task_log_path, + ) + detector, init_error = self._build_detector(anomaly_cfg) + if detector is not None: + self._cache_detector_token_categories(detector) + self.logger.info( + "Response anomaly detector initialized for model [%s]", + task.model_abbr, + ) + elif init_error: + self.logger.warning( + "Response anomaly detector is %s: %s", + init_error[0], + init_error[1], + ) + except Exception as exc: + self.logger.logger.error( + "Failed to prepare response anomaly detection for model %s: %s", + task.model_abbr, + exc, + ) + detector = None + init_error = ( + "failed", + f"Failed to prepare msProbe configuration: {exc}", + ) + detector_cache[task.model_abbr] = (anomaly_cfg, detector, init_error) + return detector_cache[task.model_abbr] + + def _prepare_payload_state( + self, + task: _DetectionTask, + anomaly_cfg: Dict[str, Any], + work_dir: str, + prediction_keys: set[str], + inherited: Dict[str, Dict[str, Any]], + active_writers: List[Any], + ) -> _PayloadState: + """Validate the existing archive and prepare the next payload build.""" + retention = anomaly_cfg.get("payload_retention", "anomalies") + storage_cfg = anomaly_cfg.get("payload_storage", {}) + payload_dir = ( + Path(work_dir) + / "response_anomaly" + / task.model_abbr + / "payload" + / task.dataset_abbr + ) + source_dir = ( + Path(work_dir) + / "response_anomaly" + / task.model_abbr + / "payload_staging" + / task.dataset_abbr + ) + staging_dir = payload_dir.with_name( + f".{task.dataset_abbr}.payload-build-{uuid.uuid4().hex[:8]}" + ) + self._cleanup_stale_payload_build_dirs(payload_dir) + if payload_dir.exists(): + manifest_path = payload_dir / "payload_manifest.json" + if not manifest_path.exists(): + raise RuntimeError( + "Existing response anomaly payload archive has no " + "manifest. Use a new work directory." + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("payload_retention", "all") != retention: + raise RuntimeError( + "Cannot change response anomaly payload_retention while " + "reusing an existing payload archive. Use a new work directory." + ) + pending_keys = prediction_keys.difference(inherited) + archive_is_current = ( + not pending_keys + and not source_dir.exists() + and ( + payload_dir.exists() + if retention != "none" + else not payload_dir.exists() + ) + ) + if archive_is_current: + self.logger.info( + "No new response anomaly payloads for %s/%s; " + "keeping the existing payload archive unchanged", + task.model_abbr, + task.dataset_abbr, + ) + elif payload_dir.exists() and retention == "all": + self._seed_payload_directory(payload_dir, source_dir) + + state = _PayloadState( + retention=retention, + storage_cfg=storage_cfg, + payload_dir=payload_dir, + source_dir=source_dir, + staging_dir=staging_dir, + archive_is_current=archive_is_current, + ) + if retention != "anomalies" or archive_is_current: + return state + + from ais_bench.benchmark.utils.response_anomaly_jsonl import ( + ResponseAnomalyJsonlWriter, + iter_jsonl_zstd_records, + ) + + if payload_dir.exists(): + state.retained_keys = { + f"{record.get('id')}:{record.get('uuid')}" + for record in iter_jsonl_zstd_records(payload_dir) + } + self._seed_payload_directory( + payload_dir, staging_dir, include_manifest=True + ) + state.writer = ResponseAnomalyJsonlWriter( + staging_dir, + compression_level=storage_cfg.get("compression_level", 3), + rows_per_shard=storage_cfg.get("rows_per_shard", 2000), + ) + active_writers.append(state.writer) + return state + + def _process_staged_payloads( + self, + payload: _PayloadState, + prediction_keys: set[str], + inherited: Dict[str, Dict[str, Any]], + anomaly_cfg: Dict[str, Any], + detector, + init_error, + result_file: Path, + status_file: Path, + task_name: str, + task_log_path: str, + total: int, + completed: int, + group_counts: Counter[str], + counts: Counter[str], + ) -> tuple: + """Detect payloads from compressed staging shards.""" + from ais_bench.benchmark.utils.response_anomaly_jsonl import ( + iter_jsonl_zstd_records, + ) + + result_batch = {} + detected_keys = set(inherited) + last_status_time = time.monotonic() + shard_rows: Counter[str] = Counter() + for record in iter_jsonl_zstd_records(payload.source_dir): + shard_rows[record["payload_shard"]] += 1 + case_key = f"{record.get('id')}:{record.get('uuid')}" + if payload.retention == "all": + payload.retained_keys.add(case_key) + if case_key not in prediction_keys: + continue + if case_key in inherited: + self._write_retained_payload( + payload.writer, + payload.retention, + inherited[case_key], + record, + case_key, + payload.retained_keys, + ) + continue + if case_key in detected_keys: + continue + result = self._detect_case(record, anomaly_cfg, detector, init_error) + result_batch[case_key] = result + self._write_retained_payload( + payload.writer, + payload.retention, + result, + record, + case_key, + payload.retained_keys, + ) + detected_keys.add(case_key) + completed += 1 + result_name = result["anomaly_type_name"] + group_counts[result_name] += 1 + counts[result_name] += 1 + now = time.monotonic() + if len(result_batch) >= 100: + safe_write(result_batch, result_file) + result_batch = {} + if now - last_status_time >= 1.0: + self._post_status( + status_file, + completed, + total, + group_counts, + "response anomaly detecting", + task_name=task_name, + task_log_path=task_log_path, + ) + last_status_time = now + if result_batch: + safe_write(result_batch, result_file) + return detected_keys, shard_rows, completed + + def _process_prediction_payloads( + self, + task: _DetectionTask, + payload: _PayloadState, + inherited: Dict[str, Dict[str, Any]], + detected_keys: set[str], + anomaly_cfg: Dict[str, Any], + detector, + init_error, + result_file: Path, + completed: int, + group_counts: Counter[str], + counts: Counter[str], + shard_rows: Counter[str], + active_writers: List[Any], + ) -> int: + """Process legacy payloads still embedded in prediction records.""" + legacy_writer = None + for prediction in task.predictions: + case_key = f"{prediction.get('id')}:{prediction.get('uuid')}" + if case_key in inherited: + result = inherited[case_key] + is_inherited = True + else: + if case_key in detected_keys: + continue + result = self._detect_case( + prediction, anomaly_cfg, detector, init_error + ) + safe_write({case_key: result}, result_file) + is_inherited = False + self._write_retained_payload( + payload.writer, + payload.retention, + result, + prediction, + case_key, + payload.retained_keys, + ) + if ( + payload.retention == "all" + and not payload.archive_is_current + and case_key not in payload.retained_keys + and isinstance(prediction.get("response_anomaly_payload"), dict) + ): + from ais_bench.benchmark.utils.response_anomaly_jsonl import ( + ResponseAnomalyJsonlWriter, + ) + + if legacy_writer is None: + legacy_writer = ResponseAnomalyJsonlWriter( + payload.source_dir, + payload.storage_cfg.get("compression_level", 3), + payload.storage_cfg.get("rows_per_shard", 2000), + ) + active_writers.append(legacy_writer) + legacy_writer.write(prediction) + payload.retained_keys.add(case_key) + if is_inherited: + continue + completed += 1 + result_name = result["anomaly_type_name"] + group_counts[result_name] += 1 + counts[result_name] += 1 + if legacy_writer is not None: + manifest = legacy_writer.close(write_manifest=False) + active_writers.remove(legacy_writer) + shard_rows.update( + {shard["file"]: shard["rows"] for shard in manifest["shards"]} + ) + return completed + + def _finalize_group_payloads( + self, + task: _DetectionTask, + payload: _PayloadState, + shard_rows: Counter[str], + status_file: Path, + task_name: str, + task_log_path: str, + total: int, + completed: int, + counts: Counter[str], + active_writers: List[Any], + ) -> None: + """Publish the retained archive and remove temporary payloads.""" + self._post_status( + status_file, + completed, + total, + counts, + f"finalizing response anomaly payloads for {task.dataset_abbr}", + task_name=task_name, + task_log_path=task_log_path, + ) + if payload.archive_is_current: + pass + elif payload.retention == "all": + from ais_bench.benchmark.utils.response_anomaly_jsonl import ( + build_jsonl_zstd_manifest, + ) + + build_jsonl_zstd_manifest( + payload.source_dir, + payload.storage_cfg.get("compression_level", 3), + payload.retention, + dict(shard_rows), + ) + self._replace_payload_archive( + payload.source_dir, payload.payload_dir + ) + elif payload.writer is not None: + manifest = payload.writer.close(payload.retention) + active_writers.remove(payload.writer) + if manifest["total_rows"] or not payload.payload_dir.exists(): + self._replace_payload_archive( + payload.staging_dir, payload.payload_dir + ) + else: + shutil.rmtree(payload.staging_dir) + elif payload.payload_dir.exists(): + shutil.rmtree(payload.payload_dir) + if payload.source_dir.exists(): + shutil.rmtree(payload.source_dir) + if any( + "response_anomaly_payload" in prediction + for prediction in task.predictions + ): + self._strip_payloads_from_predictions( + task.prediction_file, task.predictions + ) + + def _finish_detection_task( + self, + task_name: str, + task_log_path: str, + work_dir: str, + result_file: Path, + payload_dir: Path, + counts: Counter[str], + completed: int, + total: int, + status_file: Path, + started_at: float, + ) -> None: + """Record task outputs, timing, and the final monitor state.""" + self._anomaly_report[task_name] = { + "counts": dict(counts), + "result_file": str(result_file), + "payload_dir": str(payload_dir) if payload_dir.exists() else None, + "task_log": str(Path(work_dir) / task_log_path), + } + self.logger.info("Response anomaly detection completed: %s", dict(counts)) + self.logger.info( + "Response anomaly task time elapsed: %.2fs", + time.perf_counter() - started_at, + ) + self.logger.info("Task state is finish, exit loop") + self._post_status( + status_file, + completed, + total, + counts, + "response anomaly finished", + "finish", + task_name, + task_log_path, + ) + + def _handle_detection_failure( + self, status_file: Path, counts: Counter[str], exc: Exception + ) -> None: + """Mark every unfinished task as failed without masking the cause.""" + self.logger.logger.error("Response anomaly detection failed: %s", exc) + self._summary = dict(counts) + for task_name in self.task_names: + state = self._task_statuses.get(task_name, {}) + if state.get("status") in ("finish", "error"): + continue + self._post_status( + status_file, + state.get("finish_count", 0), + state.get("total_count", 0), + Counter(state.get("other_kwargs", {})), + f"response anomaly failed: {exc}", + "error", + task_name, + state.get("task_log_path"), + ) + + def _cleanup_detection_resources( + self, + log_handler: Optional[logging.FileHandler], + payload_build_dir: Optional[Path], + payload_writers: List[Any], + ) -> None: + """Best-effort cleanup for resources left by an interrupted task.""" + for writer in payload_writers: + try: + writer.close(write_manifest=False) + except Exception as exc: + self.logger.warning( + "Failed to close response anomaly payload writer: %s", exc + ) + if payload_build_dir is not None and payload_build_dir.exists(): + self._remove_payload_build_dir(payload_build_dir) + self._close_task_log(log_handler) + + def _load_inherited_results( + self, result_file: Path, prediction_keys: Iterable[str] + ) -> Dict[str, Dict[str, Any]]: + """Return previously completed results whose id+uuid still exist in predictions. + + Matching on id+uuid ensures that a re-inferred response (different + uuid) is not incorrectly assigned a stale anomaly result. + Non-final statuses (skipped/unavailable/failed) are intentionally not + inherited so they can be retried on resume. + """ + existing_by_key: Dict[str, Dict[str, Any]] = {} + for item in self._read_jsonl(result_file): + key = f"{item.get('id')}:{item.get('uuid')}" + existing_by_key[key] = item + return { + key: item + for key, item in existing_by_key.items() + if key in prediction_keys and item.get("detection_status") == "completed" + } + + @staticmethod + def _should_retain_payload( + retention: str, result: Dict[str, Any] + ) -> bool: + if retention == "all": + return True + if retention == "none": + return False + return bool(result.get("is_anomaly")) or result.get( + "detection_status" + ) in ("failed", "unavailable") + + @classmethod + def _write_retained_payload( + cls, + payload_writer, + retention: str, + result: Dict[str, Any], + record: Dict[str, Any], + case_key: str, + retained_payload_keys: set, + ) -> None: + if ( + payload_writer is None + or case_key in retained_payload_keys + or not isinstance(record.get("response_anomaly_payload"), dict) + or not cls._should_retain_payload(retention, result) + ): + return + payload_writer.write(record) + retained_payload_keys.add(case_key) + + @staticmethod + def _replace_payload_archive(staging_dir: Path, payload_dir: Path) -> None: + payload_dir.parent.mkdir(parents=True, exist_ok=True) + backup_dir = payload_dir.with_name(payload_dir.name + ".old") + if backup_dir.exists(): + shutil.rmtree(backup_dir) + if payload_dir.exists(): + os.replace(str(payload_dir), str(backup_dir)) + try: + os.replace(str(staging_dir), str(payload_dir)) + except Exception: + if backup_dir.exists() and not payload_dir.exists(): + os.replace(str(backup_dir), str(payload_dir)) + raise + if backup_dir.exists(): + shutil.rmtree(backup_dir) + + def _cleanup_stale_payload_build_dirs(self, payload_dir: Path) -> None: + """Remove unpublished payload archives left by interrupted detection.""" + parent = payload_dir.parent + if not parent.exists(): + return + prefix = f".{payload_dir.name}.payload-build-" + for candidate in parent.iterdir(): + if candidate.is_dir() and candidate.name.startswith(prefix): + self._remove_payload_build_dir(candidate) + + def _remove_payload_build_dir(self, directory: Path) -> None: + try: + shutil.rmtree(directory) + except FileNotFoundError: + return + except OSError as exc: + self.logger.warning( + "Failed to clean response anomaly payload build directory %s: %s", + directory, + exc, + ) + + @staticmethod + def _seed_payload_directory( + source_dir: Path, + destination_dir: Path, + include_manifest: bool = False, + ) -> None: + """Seed a payload build with hard links, falling back to copies.""" + destination_dir.mkdir(parents=True, exist_ok=True) + for source in source_dir.glob("part-*.jsonl.zst"): + destination = destination_dir / source.name + if destination.exists(): + destination.unlink() + try: + os.link(source, destination) + except OSError: + shutil.copy2(source, destination) + destination_manifest = destination_dir / "payload_manifest.json" + if include_manifest: + shutil.copy2( + source_dir / "payload_manifest.json", + destination_manifest, + ) + elif destination_manifest.exists(): + destination_manifest.unlink() + + @staticmethod + def _strip_payloads_from_predictions( + prediction_file: Path, predictions: List[Dict[str, Any]] + ) -> None: + if not prediction_file.exists(): + return + tmp_file = prediction_file.with_name(prediction_file.name + ".tmp") + with tmp_file.open("w", encoding="utf-8") as file: + for prediction in predictions: + prediction.pop("response_anomaly_payload", None) + file.write(json.dumps(prediction, ensure_ascii=False) + "\n") + os.replace(str(tmp_file), str(prediction_file)) + + @staticmethod + def _merge_model_anomaly_config( + model_cfg: Dict[str, Any], global_cfg: Dict[str, Any] + ) -> Dict[str, Any]: + """Merge global response_anomaly config with the model-level overrides.""" + merged = dict(global_cfg) + model_cfg_anomaly = dict(model_cfg.get("response_anomaly") or {}) + for key, value in model_cfg_anomaly.items(): + if value is not None: + merged[key] = value + return merged + + def _prepare_model_config( + self, + model_abbr: str, + anomaly_cfg: Dict[str, Any], + work_dir: str, + ) -> Dict[str, Any]: + """Auto-generate msProbe model files when a local model path is given.""" + model_path = anomaly_cfg.get("model_path") + if not model_path: + return anomaly_cfg + + has_mtype = bool(anomaly_cfg.get("msprobe_mtype_path")) + has_tk2cat = bool(anomaly_cfg.get("msprobe_token2category_dir")) + if ( + has_mtype + and has_tk2cat + and Path(anomaly_cfg["msprobe_mtype_path"]).is_file() + and Path(anomaly_cfg["msprobe_token2category_dir"]).is_dir() + ): + return anomaly_cfg + if has_mtype != has_tk2cat: + raise RuntimeError( + "response_anomaly.msprobe_mtype_path and " + "response_anomaly.msprobe_token2category_dir must be configured " + "together; either provide both or rely on model_path " + "auto-generation." + ) + + from ais_bench.tools.response_anomaly.gen_model_config import ( + generate_model_config, + ) + + output_dir = Path(work_dir) / "response_anomaly_config" / model_abbr + generated = generate_model_config( + model_path=str(model_path), + model_name=anomaly_cfg.get("model_name"), + output_dir=str(output_dir), + ) + merged = dict(anomaly_cfg) + for key, value in generated.items(): + # Overwrite None/empty values (e.g. msprobe_config_path set to + # None by ConfigManager) so auto-generated paths take effect. + if not merged.get(key): + merged[key] = value + # Explicitly configured locations act as generation outputs: place the + # generated resources there, reusing existing files untouched. An empty + # msprobe_config_path keeps the msProbe built-in default (config.yaml + # is model-agnostic and does not need to be generated). + if not anomaly_cfg.get("msprobe_config_path"): + merged.pop("msprobe_config_path", None) + elif merged.get("msprobe_config_path") != generated["msprobe_config_path"]: + merged["msprobe_config_path"] = self._place_generated_resource( + Path(generated["msprobe_config_path"]), + merged["msprobe_config_path"], + is_dir=False, + ) + if merged.get("msprobe_mtype_path") != generated["msprobe_mtype_path"]: + merged["msprobe_mtype_path"] = self._place_generated_resource( + Path(generated["msprobe_mtype_path"]), + merged["msprobe_mtype_path"], + is_dir=False, + ) + if ( + merged.get("msprobe_token2category_dir") + != generated["msprobe_token2category_dir"] + ): + merged["msprobe_token2category_dir"] = self._place_generated_resource( + Path(generated["msprobe_token2category_dir"]), + merged["msprobe_token2category_dir"], + is_dir=True, + ) + self.logger.info( + "Auto-generated msProbe model config for [%s]:\n" + " config: %s\n" + " mtype_config: %s\n" + " token2category: %s", + model_abbr, + merged.get("msprobe_config_path"), + merged.get("msprobe_mtype_path"), + merged.get("msprobe_token2category_dir"), + ) + return merged + + @staticmethod + def _place_generated_resource( + source: Path, target: str, is_dir: bool + ) -> str: + """Copy a generated msProbe resource to its configured location. + + Existing targets are reused untouched; missing parent directories are + created as needed. Returns the configured target path. + """ + target_path = Path(target) + if not target_path.exists(): + if is_dir: + shutil.copytree(source, target_path) + else: + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target_path) + return str(target_path) + + @staticmethod + def _build_detector(anomaly_cfg: Dict[str, Any]): + """Create one msProbe ILLDetector with the configured file paths.""" + try: + import msprobe.response_anomaly as response_anomaly_pkg + from msprobe.response_anomaly.detector import ILLDetector + except ImportError: + return None, ( + "unavailable", + "mindstudio-probe is required for response anomaly detection. " + "Install the AISBench response_anomaly extra.", + ) + + base = Path(response_anomaly_pkg.__file__).resolve().parent + config_path = anomaly_cfg.get("msprobe_config_path") or str( + base / "configs" / "config.yaml" + ) + mtype_path = anomaly_cfg.get("msprobe_mtype_path") or str( + base / "configs" / "mtype_config.json" + ) + tk2cat_path = anomaly_cfg.get("msprobe_token2category_dir") or str( + base / "token2category" + ) + try: + detector = ILLDetector(config_path, mtype_path, tk2cat_path) + except Exception as exc: + return None, ( + "failed", + f"Failed to initialize msProbe detector: {exc}", + ) + return detector, None + + @staticmethod + def _cache_detector_token_categories(detector) -> None: + """Cache msProbe token-category maps instead of loading them per case.""" + get_tk2cat = getattr(detector, "get_tk2cat", None) + if not callable(get_tk2cat): + return + cache = {} + + def cached_get_tk2cat(eos_token, model_config=None): + try: + model_key = json.dumps( + model_config, ensure_ascii=False, sort_keys=True + ) + except (TypeError, ValueError): + model_key = repr(model_config) + key = (int(eos_token), model_key) + if key not in cache: + cache[key] = get_tk2cat(eos_token, model_config) + return cache[key] + + detector.get_tk2cat = cached_get_tk2cat + + def _detect_case( + self, + prediction: Dict[str, Any], + anomaly_cfg: Dict[str, Any], + detector=None, + init_error=None, + ) -> Dict[str, Any]: + result = { + "id": prediction.get("id"), + "uuid": prediction.get("uuid"), + "is_anomaly": False, + "anomaly_type": 0, + "anomaly_type_name": "normal", + } + payload = prediction.get("response_anomaly_payload") + if not isinstance(payload, dict): + result["detection_status"] = "skipped" + result["reason"] = "Response does not contain token ids and top-k logprobs." + result["anomaly_type_name"] = "skipped" + return result + + tokens = payload.get("tokens") + topk_logprobs = payload.get("topk_logprobs") + if ( + not isinstance(tokens, list) + or not isinstance(topk_logprobs, list) + or len(tokens) == 0 + or len(tokens) != len(topk_logprobs) + or any(not isinstance(item, dict) or not item for item in topk_logprobs) + ): + result["detection_status"] = "skipped" + result["reason"] = ( + "tokens and topk_logprobs must be non-empty lists of equal length " + "with non-empty per-token logprob maps." + ) + result["anomaly_type_name"] = "skipped" + return result + + if init_error is not None: + status, reason = init_error + result.update( + detection_status=status, + reason=reason, + anomaly_type_name=status, + ) + return result + + try: + topk_logprobs = self._normalize_logprobs(topk_logprobs) + tokens = [int(token) for token in tokens] + model_name = anomaly_cfg.get("model_name") + is_anomaly, anomaly_type = detector.run( + [topk_logprobs], [tokens], [model_name] + )[0] + anomaly_type = int(anomaly_type) + result.update( + is_anomaly=bool(is_anomaly), + anomaly_type=anomaly_type, + anomaly_type_name=_ANOMALY_TYPE_NAMES.get(anomaly_type, "unknown"), + detection_status="completed", + ) + except Exception as exc: + result.update( + detection_status="failed", + reason=f"{type(exc).__name__}: {exc}", + anomaly_type_name="failed", + ) + return result + + @staticmethod + def _normalize_logprobs(items: Iterable[Dict[Any, Any]]) -> list[Dict[int, float]]: + return [ + {int(token_id): float(logprob) for token_id, logprob in item.items()} + for item in items + ] + + def _post_status( + self, + status_file: Path, + completed: int, + total: int, + counts: Counter[str], + description: str, + status: str = "response anomaly", + task_name: Optional[str] = None, + task_log_path: Optional[str] = None, + ) -> None: + """Atomically write the latest status. + + The status file is replaced instead of appended, so the coordinator + writer and TasksMonitor readers never observe partial JSON. + """ + status_file.parent.mkdir(parents=True, exist_ok=True) + task_name = task_name or self.STATUS_TASK_NAME + state = { + "task_name": task_name, + "process_id": os.getpid(), + "finish_count": completed, + "total_count": total, + "progress_description": description, + "status": status, + "other_kwargs": dict(counts), + } + if task_log_path: + state["task_log_path"] = task_log_path + self._task_statuses[task_name] = state + payload = list(self._task_statuses.values()) + tmp_file = status_file.with_name(status_file.name + ".tmp") + tmp_file.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + os.replace(str(tmp_file), str(status_file)) + + def _read_jsonl(self, path: Path) -> List[Dict[str, Any]]: + if not path.exists(): + return [] + records = [] + with path.open(encoding="utf-8") as file: + for line_no, line in enumerate(file, 1): + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError as exc: + self.logger.warning( + "Skip malformed line %s:%s: %s", path, line_no, exc + ) + return records diff --git a/ais_bench/benchmark/utils/response_anomaly_jsonl.py b/ais_bench/benchmark/utils/response_anomaly_jsonl.py new file mode 100644 index 00000000..0ec13a92 --- /dev/null +++ b/ais_bench/benchmark/utils/response_anomaly_jsonl.py @@ -0,0 +1,233 @@ +"""Zstandard-compressed JSONL storage for response anomaly payloads.""" + +import hashlib +import io +import json +import os +import uuid +from pathlib import Path +from typing import Any, Dict, Iterator, Optional, Tuple + + +def _load_zstandard(): + try: + import zstandard + except ImportError as exc: + raise RuntimeError( + "zstandard is required for compressed response anomaly payloads. " + "Install the AISBench response_anomaly extra." + ) from exc + return zstandard + + +class ResponseAnomalyJsonlWriter: + """Write payload records to bounded, atomically published JSONL.ZST shards.""" + + def __init__(self, directory: Path, compression_level: int, rows_per_shard: int): + self.directory = Path(directory) + self.compression_level = compression_level + self.rows_per_shard = rows_per_shard + self.session_id = uuid.uuid4().hex[:8] + self.shard_index = 0 + self.shard_rows = 0 + self.total_rows = 0 + self._raw_file = None + self._stream = None + self._inprogress_path: Optional[Path] = None + self._shards = [] + manifest_path = self.directory / "payload_manifest.json" + if manifest_path.exists(): + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + self._shards = list(manifest.get("shards", [])) + self.total_rows = int(manifest.get("total_rows", 0)) + + def write(self, record: Dict[str, Any]) -> None: + payload = record.get("response_anomaly_payload") + if not isinstance(payload, dict): + return + if self._stream is None: + self._open_shard() + line = json.dumps( + { + "data_abbr": record.get("data_abbr"), + "id": record.get("id"), + "uuid": record.get("uuid"), + "response_anomaly_payload": payload, + }, + ensure_ascii=False, + separators=(",", ":"), + ) + "\n" + self._stream.write(line.encode("utf-8")) + self.shard_rows += 1 + self.total_rows += 1 + if self.shard_rows >= self.rows_per_shard: + self._close_shard() + + def close( + self, + payload_retention: Optional[str] = None, + write_manifest: bool = True, + ) -> Dict[str, Any]: + if self._stream is not None: + self._close_shard() + manifest = { + "format": "jsonl", + "compression": "zstd", + "compression_level": self.compression_level, + "total_rows": self.total_rows, + "shards": self._shards, + } + if payload_retention is not None: + manifest["payload_retention"] = payload_retention + if write_manifest: + self.directory.mkdir(parents=True, exist_ok=True) + path = self.directory / "payload_manifest.json" + tmp_path = path.with_name(path.name + ".tmp") + tmp_path.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8" + ) + os.replace(str(tmp_path), str(path)) + return manifest + + def _open_shard(self) -> None: + zstandard = _load_zstandard() + self.directory.mkdir(parents=True, exist_ok=True) + name = ( + f"part-p{os.getpid()}-{self.session_id}-" + f"{self.shard_index:05d}.jsonl.zst" + ) + self._inprogress_path = self.directory / f"{name}.inprogress" + self._raw_file = self._inprogress_path.open("wb") + compressor = zstandard.ZstdCompressor(level=self.compression_level) + self._stream = compressor.stream_writer(self._raw_file, closefd=False) + self.shard_rows = 0 + + def _close_shard(self) -> None: + self._stream.flush(_load_zstandard().FLUSH_FRAME) + self._stream.close() + self._raw_file.flush() + os.fsync(self._raw_file.fileno()) + self._raw_file.close() + final_path = self._inprogress_path.with_suffix("") + os.replace(str(self._inprogress_path), str(final_path)) + self._shards.append( + { + "file": final_path.name, + "rows": self.shard_rows, + "size_bytes": final_path.stat().st_size, + "sha256": f"sha256:{_sha256_file(final_path)}", + } + ) + self.shard_index += 1 + self._raw_file = None + self._stream = None + self._inprogress_path = None + + +class ResponseAnomalyStagingWriter: + """Route inference payloads to per-dataset compressed staging shards.""" + + def __init__(self, runtime: Dict[str, Any]) -> None: + self.root = ( + Path(runtime["work_dir"]) + / "response_anomaly" + / str(runtime["model_abbr"]) + / "payload_staging" + ) + self.compression_level = int(runtime.get("compression_level", 3)) + self.rows_per_shard = int(runtime.get("rows_per_shard", 2000)) + self._writers: Dict[str, ResponseAnomalyJsonlWriter] = {} + + def write(self, record: Dict[str, Any]) -> None: + data_abbr = str(record.get("data_abbr", "")) + writer = self._writers.get(data_abbr) + if writer is None: + writer = ResponseAnomalyJsonlWriter( + self.root / data_abbr, + self.compression_level, + self.rows_per_shard, + ) + self._writers[data_abbr] = writer + writer.write(record) + + def close(self) -> None: + for writer in self._writers.values(): + writer.close(write_manifest=False) + self._writers.clear() + + +def iter_jsonl_zstd_records(directory: Path) -> Iterator[Dict[str, Any]]: + """Stream records from all completed JSONL.ZST shards in a directory.""" + for shard in sorted(Path(directory).glob("part-*.jsonl.zst")): + for line_no, record in _iter_jsonl_zstd_shard(shard): + record["payload_shard"] = shard.name + record["payload_row"] = line_no - 1 + yield record + + +def build_jsonl_zstd_manifest( + directory: Path, + compression_level: int, + payload_retention: str, + shard_rows: Optional[Dict[str, int]] = None, +) -> Dict[str, Any]: + """Build and atomically write a manifest for existing staging shards.""" + shards = [] + total_rows = 0 + for shard in sorted(Path(directory).glob("part-*.jsonl.zst")): + rows = ( + shard_rows[shard.name] + if shard_rows is not None and shard.name in shard_rows + else sum(1 for _ in _iter_jsonl_zstd_shard(shard)) + ) + total_rows += rows + shards.append( + { + "file": shard.name, + "rows": rows, + "size_bytes": shard.stat().st_size, + "sha256": f"sha256:{_sha256_file(shard)}", + } + ) + manifest = { + "format": "jsonl", + "compression": "zstd", + "compression_level": compression_level, + "payload_retention": payload_retention, + "total_rows": total_rows, + "shards": shards, + } + directory.mkdir(parents=True, exist_ok=True) + path = directory / "payload_manifest.json" + tmp_path = path.with_name(path.name + ".tmp") + tmp_path.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8" + ) + os.replace(str(tmp_path), str(path)) + return manifest + + +def _iter_jsonl_zstd_shard(path: Path) -> Iterator[Tuple[int, Dict[str, Any]]]: + """Yield parsed records with line numbers from one compressed shard.""" + zstandard = _load_zstandard() + with Path(path).open("rb") as raw_file: + with zstandard.ZstdDecompressor().stream_reader(raw_file) as reader: + text_reader = io.TextIOWrapper(reader, encoding="utf-8") + for line_no, line in enumerate(text_reader, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Malformed compressed payload {path}:{line_no}: {exc}" + ) from exc + yield line_no, record + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/ais_bench/tools/response_anomaly/__init__.py b/ais_bench/tools/response_anomaly/__init__.py new file mode 100644 index 00000000..279a0621 --- /dev/null +++ b/ais_bench/tools/response_anomaly/__init__.py @@ -0,0 +1 @@ +"""AISBench helpers for preparing msProbe response anomaly model configs.""" diff --git a/ais_bench/tools/response_anomaly/gen_model_config.py b/ais_bench/tools/response_anomaly/gen_model_config.py new file mode 100644 index 00000000..669b4542 --- /dev/null +++ b/ais_bench/tools/response_anomaly/gen_model_config.py @@ -0,0 +1,181 @@ +"""Generate msProbe response anomaly model configs from a local model. + +msProbe's official ``gen_model_config.py`` always writes into the installed +msProbe package (``response_anomaly/configs`` and ``response_anomaly/token2category``) +and overwrites ``mtype_config.json`` on every run. This wrapper runs the +official script inside ``/tools`` so its cwd-relative outputs land +in ``/configs`` and ``/token2category`` instead, merges +new model entries into an existing ``mtype_config.json``, and copies the default +algorithm-threshold ``config.yaml`` when it is not already present. +""" + +import argparse +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Dict, Optional + + +def _normalize_name(name: str) -> str: + """Keep in sync with msProbe's model-name normalization rules.""" + return "-".join(re.split(r"\.|-|_", name.lower())) + + +def _msprobe_response_anomaly_dir() -> Path: + try: + import msprobe.response_anomaly as response_anomaly + except ImportError as exc: + raise RuntimeError( + "mindstudio-probe is required to generate response anomaly model " + "configs. Install the AISBench response_anomaly extra first." + ) from exc + return Path(response_anomaly.__file__).resolve().parent + + +def _official_script_path() -> Path: + script = _msprobe_response_anomaly_dir() / "tools" / "gen_model_config.py" + if not script.exists(): + raise RuntimeError( + f"msProbe response anomaly generator not found at {script}. " + "Please reinstall the pinned mindstudio-probe version." + ) + return script + + +def generate_model_config( + model_path: str, + model_name: Optional[str] = None, + output_dir: Optional[str] = None, +) -> Dict[str, str]: + """Generate msProbe model files into a user-owned directory. + + Args: + model_path: Local model/tokenizer directory. + model_name: msProbe model name; defaults to the directory basename and + is normalized by msProbe (lowercase, ``-_.`` -> ``-``). + output_dir: Destination directory. Generated layout: + ``/configs/config.yaml``, + ``/configs/mtype_config.json``, + ``/token2category/_.json``. + + Returns: + Dict with ``msprobe_config_path``, ``msprobe_mtype_path`` and + ``msprobe_token2category_dir``. + """ + output_dir = Path(output_dir or "msprobe_configs").resolve() + script = _official_script_path() + effective_model_name = _normalize_name( + model_name or Path(model_path).name + ) + + configs_dir = output_dir / "configs" + token2category_dir = output_dir / "token2category" + configs_dir.mkdir(parents=True, exist_ok=True) + token2category_dir.mkdir(parents=True, exist_ok=True) + + mtype_path = configs_dir / "mtype_config.json" + existing_mtype = {} + if mtype_path.exists(): + existing_mtype = json.loads(mtype_path.read_text(encoding="utf-8")) + + # Run the official script inside an isolated temp directory so a partial + # failure (e.g. mtype_config.json written but token2category generation + # fails) never clobbers the target files. The script derives its output + # paths from the current working directory (".."), so cwd is set to + # /tools and outputs land in /configs and + # /token2category. + gen_dir = output_dir / f"_gen_tmp_{effective_model_name}" + gen_tools_dir = gen_dir / "tools" + gen_configs_dir = gen_dir / "configs" + gen_token2category_dir = gen_dir / "token2category" + gen_tools_dir.mkdir(parents=True, exist_ok=True) + gen_configs_dir.mkdir(parents=True, exist_ok=True) + gen_token2category_dir.mkdir(parents=True, exist_ok=True) + command = [sys.executable, str(script), "--model-path", str(model_path)] + if model_name: + command += ["--model-name", model_name] + try: + proc = subprocess.run( + command, + cwd=str(gen_tools_dir), + text=True, + capture_output=True, + ) + except Exception as exc: + raise RuntimeError( + f"msProbe gen_model_config failed to start: {exc}. " + f"Inspection files are kept at {gen_dir}." + ) from exc + + if proc.returncode != 0: + raise RuntimeError( + "msProbe gen_model_config failed " + f"(return code {proc.returncode}): {proc.stderr or proc.stdout}. " + f"Inspection files are kept at {gen_dir}." + ) + + # Success: merge generated files from the temp directory into the target. + # The target mtype_config.json is written with a merge of old and new + # entries so existing models are preserved. + gen_mtype_path = gen_dir / "configs" / "mtype_config.json" + generated_mtype = {} + if gen_mtype_path.exists(): + generated_mtype = json.loads(gen_mtype_path.read_text(encoding="utf-8")) + merged_mtype = {**existing_mtype, **generated_mtype} + mtype_path.write_text( + json.dumps(merged_mtype, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + gen_token2cat_dir = gen_dir / "token2category" + if gen_token2cat_dir.exists(): + for item in gen_token2cat_dir.iterdir(): + if item.is_file(): + shutil.copy2(item, token2category_dir / item.name) + + shutil.rmtree(gen_dir, ignore_errors=True) + + config_yaml = configs_dir / "config.yaml" + if not config_yaml.exists(): + default_yaml = _msprobe_response_anomaly_dir() / "configs" / "config.yaml" + shutil.copy2(default_yaml, config_yaml) + + return { + "model_name": effective_model_name, + "msprobe_config_path": str(config_yaml), + "msprobe_mtype_path": str(mtype_path), + "msprobe_token2category_dir": str(token2category_dir), + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate msProbe response anomaly configs for a local model." + ) + parser.add_argument("--model-path", required=True, help="Local model directory.") + parser.add_argument( + "--model-name", + default=None, + help="msProbe model name; defaults to the directory basename.", + ) + parser.add_argument( + "--output-dir", + default="msprobe_configs", + help="Output directory for configs/ and token2category/.", + ) + args = parser.parse_args() + + generated = generate_model_config( + model_path=args.model_path, + model_name=args.model_name, + output_dir=args.output_dir, + ) + for key, path in generated.items(): + print(f"{key}={path}") + + +if __name__ == "__main__": + main() diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index ab70c4ad..c3868853 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -36,6 +36,8 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | | `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | +| `--response-anomaly` / `--no-response-anomaly` | Enables or disables msProbe response anomaly detection. The command-line value overrides `response_anomaly.enabled` in the config file. Detection runs in a thread in parallel with Eval; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | +| `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | # ### Accuracy Evaluation Parameters @@ -67,4 +69,61 @@ The currently supported parameter configurations are as follows: | `WORKERS_NUM` | Number of processes used for sending requests. The default value is 0, which means automatic allocation based on the maximum number of concurrent requests configured by the user. (Invalid when the command-line parameter `--debug` is specified; single-core execution is used for sending requests, which limits concurrency capabilities.) | [0, number of CPU cores] | | `MAX_CHUNK_SIZE` | Maximum cache size for a single chunk returned by the streaming inference model backend. The default value is 65535 bytes (64KB). | `(0, 16777216]` (Unit: Byte) | | `REQUEST_TIME_OUT` | Timeout period for the client to wait for a response after sending a request. The default value is None, meaning infinite waiting (always waiting for the model to return results). | `None` or `>0` (Unit: seconds) | -| `LOG_LEVEL` | Log level, optional values: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Default value: `INFO`. | `[DEBUG, INFO, WARNING, ERROR, CRITICAL]` | \ No newline at end of file +| `LOG_LEVEL` | Log level, optional values: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Default value: `INFO`. | `[DEBUG, INFO, WARNING, ERROR, CRITICAL]` | + +## Response Anomaly Detection Configuration + +Response anomaly detection currently supports only the vLLM Chat API model configurations `vllm_api_general_chat`, `vllm_api_stream_chat`, and `vllm_api_stream_chat_multiturn`. Other model backends are not supported yet. + +Add a `response_anomaly` entry to the top-level config file to enable detection; it can also be overridden with `--response-anomaly`: + +```python +response_anomaly = dict( + enabled=True, +) +``` + +Model-specific msProbe configuration goes into the model config: + +```python +models = [ + dict( + abbr='qwen3-30b', + attr='service', + response_anomaly=dict( + model_name="", # Model name, for example Qwen3-30B-A3B + model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs + msprobe_mtype_path='/path/to/mtype_config.json', + msprobe_token2category_dir='/path/to/token2category/', + ), + ), +] +``` + +When `msprobe_mtype_path` / `msprobe_token2category_dir` are not provided, the default files inside the msProbe package are used; when `model_path` is configured, configs are auto-generated into `/response_anomaly_config//`. They can also be generated manually: + +```bash +ais_bench-gen-response-anomaly-config \ + --model-path /home/Qwen3-30B-A3B \ + --model-name Qwen3-30B-A3B \ + --output-dir ./msprobe_configs +``` + +When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the service inference requests; the value is constrained by the detection algorithm and cannot be configured externally. During inference, the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start. After inference finishes, the detection thread streams and decompresses the staging data and calls msProbe; detection results are written to `response_anomaly//.jsonl`. Each Case contains `is_anomaly`, `anomaly_type` (0: normal, 1: rare character, 2: garbled, 3: repetition, 4: NaN value) and `detection_status`. After detection, the staging data is retained or cleaned according to `payload_retention`. The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. + +```python +response_anomaly = dict( + enabled=True, + payload_retention='anomalies', # all | anomalies | none + payload_storage=dict( + format='jsonl', + compression='zstd', + compression_level=3, + rows_per_shard=2000, + ), +) +``` + +`all` keeps every payload and atomically promotes the staging data to the official archive after detection without re-compressing; `anomalies` keeps only detected anomalies plus detection-failed/unavailable Cases; `none` keeps no payload. All three modes keep the standalone detection results. `--reuse` must keep the retention policy of the original work directory. Results are written to disk in batches, and the status is refreshed at most once per second; msProbe token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. + +Detection runs through msProbe's `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)`; all three file paths can be configured in AISBench. Install the optional dependencies first: `pip install 'ais-bench-benchmark[response_anomaly]'`. During installation, pip downloads and builds the pinned msProbe source from GitCode, so the environment needs Git and network access. The service response must contain `token_ids` (or `tokens`) and `topk_logprobs`; Cases missing these fields are recorded with a `skipped` status. `model_name` must be consistent with msProbe's `mtype_config.json` and the token-category mapping. When using `--reuse`, existing detection results are inherited by Case id, and completed Cases are not re-detected. diff --git a/docs/source_en/base_tutorials/all_params/mode.md b/docs/source_en/base_tutorials/all_params/mode.md index b53cc3f3..750b6aff 100644 --- a/docs/source_en/base_tutorials/all_params/mode.md +++ b/docs/source_en/base_tutorials/all_params/mode.md @@ -106,6 +106,9 @@ outputs/default/ └── ... ``` +### Response Anomaly Detection Mode Support (Optional) + +msProbe response anomaly detection (`--response-anomaly`) is only supported in the `all`, `infer`, and `infer_judge` generation chains: the detection thread starts after inference finishes and runs in parallel with the Judge / Eval / Summary workflow, and the workflow waits for detection to complete before exiting. The `perf` and `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do **not** support this feature; enabling it raises an explicit error during config initialization. This feature requires the service model to return token ids and top-k logprobs; see [Response Anomaly Detection Configuration](./cli_args.md#response-anomaly-detection-configuration) for details. ## Performance Evaluation Scenarios ### Perf Mode diff --git a/docs/source_en/base_tutorials/all_params/models.md b/docs/source_en/base_tutorials/all_params/models.md index 010eec62..cf3679d7 100644 --- a/docs/source_en/base_tutorials/all_params/models.md +++ b/docs/source_en/base_tutorials/all_params/models.md @@ -31,6 +31,9 @@ The model configurations corresponding to different service-oriented backends ar ### Parameter Description for Service-Oriented Inference Backend Configuration The configuration file for the service-oriented inference backend is configured using Python syntax, as shown in the example below: + +The common model templates do not preconfigure `response_anomaly`. Add the `response_anomaly` block shown below only when response anomaly detection is needed: locate the target model in the `models` list of the model configuration file and add it inside that model's `dict`, at the same level as fields such as `generation_kwargs` and `pred_postprocessor`. It is not required when anomaly detection is disabled. + ```python from ais_bench.benchmark.models import VLLMCustomAPI @@ -55,6 +58,12 @@ models = [ generation_kwargs = dict( # Model inference parameters, configured with reference to the VLLM documentation; the AISBench evaluation tool does not process these parameters and attaches them to the sent requests temperature = 0.01, ignore_eos=False, + ), + response_anomaly = dict( # Optional; model-level config for msProbe response anomaly detection + model_name="", # Model name, for example Qwen3-30B-A3B + model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs + msprobe_mtype_path='/path/to/mtype_config.json', + msprobe_token2category_dir='/path/to/token2category/', ) ) ] @@ -86,15 +95,18 @@ The description of configurable parameters for the service-oriented inference ba | `generation_kwargs` | Dict | Configuration of inference generation parameters, depending on the specific service-oriented backend and interface type. Note: Currently, multi-sampling parameters such as `best_of` and `n` are not supported, but multiple independent inferences can be performed using the `num_return_sequences` parameter (for details, refer to 🔗 [the role of `num_return_sequences` in the Text Generation Documentation](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences\(int,)) | | `returns_tool_calls` | Bool | Controls the extraction method of function call information. When set to `True`, the system extracts function call information from the `tool_calls` field of the API response; when set to `False`, the system parses function call information from the `content` field | | `pred_postprocessor` | Dict | Post-processing configuration for model output results. It is used to format, clean, or convert the original model output to meet the requirements of specific evaluation tasks | +| `response_anomaly` | Dict | Optional; model-level config for msProbe response anomaly detection, including `model_name` (must match the name in msProbe's mtype_config.json), `model_path` (local model directory, optional, used to auto-generate configs), `msprobe_mtype_path`, and `msprobe_token2category_dir`. When mtype/token-category paths are not provided, the default files inside the msProbe package are used | **Precautions**: +- Response anomaly detection currently supports only the vLLM Chat API model configurations `vllm_api_general_chat`, `vllm_api_stream_chat`, and `vllm_api_stream_chat_multiturn`. Other model backends are not supported yet. - `request_rate` is affected by hardware performance. You can increase 📚 [WORKERS_NUM](./cli_args.md#configuration-constant-file-parameters) to improve concurrency capability. - The function of `request_rate` may be overwritten by the `traffic_cfg` item. For specific reasons, refer to 🔗 [Parameter Interpretation Section in the Description of Request Rate (RPS) Distribution Control and Visualization](../../advanced_tutorials/rps_distribution.md#parameter-interpretation). - When the dataset has timestamps and **use_timestamp** is True in the model config, requests are scheduled by timestamp and **request_rate** and **traffic_cfg** are ignored. - Setting `batch_size` too large may result in high CPU usage. Please configure it reasonably based on hardware conditions. - The default service address used by the service-oriented inference evaluation API is `localhost:8080`. In actual use, you need to modify it to the IP and port of the service-oriented backend according to the actual deployment. - When using an IPv6 literal (such as `::1` or `2001:db8::1`) as `host_ip`, the tool will automatically wrap it in brackets in the generated URL (for example, `http://[2001:db8::1]:8080/`), so you do not need to manually add brackets in the configuration. +- When response anomaly detection (`response_anomaly`) is enabled, the service must return token ids and top-k logprobs; AISBench automatically adds `logprobs=True` and a fixed `top_logprobs=20` to the inference requests, and Cases whose responses lack these fields are marked as `skipped` in the detection results. See [Response Anomaly Detection Configuration](./cli_args.md#response-anomaly-detection-configuration) for details. ### Multi-LoRA Routing @@ -235,4 +247,4 @@ The description of configurable parameters for the vllm offline inference local | `sample_kwargs` | Dict | LLM sample params, refer 🔗 [sample params](https://docs.vllm.ai/en/v0.6.5/dev/sampling_params.html) | | `vision_kwargs` | Dict | multi-modal input params，refer 🔗 [multi-modal vllm offline inference](https://docs.vllm.ai/en/v0.7.3/getting_started/examples/vision_language.html) | | `max_out_len` | Int | Maximum number of output tokens generated by inference. | -| `batch_size` | Int | Batch size for inference requests. Valid range: (0, 64000] | \ No newline at end of file +| `batch_size` | Int | Batch size for inference requests. Valid range: (0, 64000] | diff --git a/docs/source_en/get_started/install.md b/docs/source_en/get_started/install.md index c6be65b9..b33fd33c 100644 --- a/docs/source_en/get_started/install.md +++ b/docs/source_en/get_started/install.md @@ -31,6 +31,19 @@ pip3 install -r requirements/api.txt pip3 install -r requirements/extra.txt ``` +⚙️ Response Anomaly Detection Support (Optional) + +If you need to use msProbe response anomaly detection (`--response-anomaly`), install the additional dependencies: +```shell +pip3 install -r requirements/response_anomaly.txt +``` +Or install them via the extra: +```shell +pip3 install 'ais-bench-benchmark[response_anomaly]' +``` + +**Note**: These dependencies include building the `mindstudio-probe` source code from a pinned commit on GitCode, so the installation environment needs Git and network access. Without them, the AISBench main workflow is not affected; the affected Cases are marked as `unavailable` in the detection results. + ⚙️ Huggingface Multi-modal Model / vLLM Multi-modal Offline Inference Support (Optional) ```shell diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index c5970401..a251e301 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -35,6 +35,8 @@ ais_bench [OPTIONS] | `--num-prompts` | 指定数据集测评条数（按照数据集顺序选取），需传入正整数，超过数据集条数或默认情况下表示对全量数据集进行测评。 | `--num-prompts 500` | | `--max-num-workers` | 并行任务数，范围 `[1, CPU 核数]`，默认 `1`。在指定`--debug`时配置无效，所有任务串行执行。注意：性能测评场景下，并发数过高可能会导致不同进程出现资源抢占，导致测试结果失真。 | `--max-num-workers 2` | |`--num-warmups`|发送请求前预热次数，按照数据集顺序选取数据进行测试，大概num-warmups大于数据集条数时，会循环发送数据集中数据。默认 `1`；若设为0，则不预热。如果warmup阶段所有请求失败，后续推理任务将不会执行。| `--num-warmups 10` | +| `--response-anomaly` / `--no-response-anomaly` | 开启或关闭 msProbe 推理响应异常检测。命令行配置优先于配置文件中的 `response_anomaly.enabled`。检测在线程中与 Eval 并行运行；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。 | `--response-anomaly` | +| `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。命令行配置优先于配置文件，默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | ### 精度测评参数 仅在模式为 `all、infer、eval` 或 `viz` 时有效。 @@ -53,6 +55,63 @@ ais_bench [OPTIONS] ## 配置常量文件参数 +## 推理响应异常检测配置 + +当前响应异常检测仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 + +在总配置文件中增加 `response_anomaly` 可启用检测，也可通过 `--response-anomaly` 覆盖： + +```python +response_anomaly = dict( + enabled=True, +) +``` + +模型相关的 msProbe 配置放在模型配置中： + +```python +models = [ + dict( + abbr='qwen3-30b', + attr='service', + response_anomaly=dict( + model_name="", # 填写模型名称，如 Qwen3-30B-A3B + model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 + msprobe_mtype_path='/path/to/mtype_config.json', + msprobe_token2category_dir='/path/to/token2category/', + ), + ), +] +``` + +未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时回退到 msProbe 包内默认文件；配置了 `model_path` 时会自动生成到 `/response_anomaly_config/<模型 abbr>/`。也可手动生成： + +```bash +ais_bench-gen-response-anomaly-config \ + --model-path /home/Qwen3-30B-A3B \ + --model-name Qwen3-30B-A3B \ + --output-dir ./msprobe_configs +``` + +启用后，AISBench 会在服务推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`；该值由检测算法约束，不支持外部配置。推理阶段将完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果。推理结束后，检测线程流式解压 staging 数据并调用 msProbe，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`；每个 Case 包含 `is_anomaly`、`anomaly_type`（0：正常，1：生僻字，2：乱码，3：重复，4：NaN Value）和 `detection_status`。检测完成后按 `payload_retention` 保留或清理 staging。状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 + +```python +response_anomaly = dict( + enabled=True, + payload_retention='anomalies', # all | anomalies | none + payload_storage=dict( + format='jsonl', + compression='zstd', + compression_level=3, + rows_per_shard=2000, + ), +) +``` + +`all` 保存全部 payload，检测后直接将 staging 原子转为正式归档，不会二次压缩；`anomalies` 只保存已检出异常以及检测失败/不可用 Case；`none` 不保存 payload。三种模式都保留独立检测结果。`--reuse` 必须沿用原工作目录的保留策略。检测结果按批写盘，状态最多每秒刷新一次；msProbe token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 + +检测通过 msProbe 的 `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)` 完成，三个文件路径均可由 AISBench 配置。请先安装 AISBench 的可选依赖：`pip install 'ais-bench-benchmark[response_anomaly]'`。安装过程中 pip 会从 GitCode 下载并构建已固定提交的 msProbe 源码，因此安装环境需要 Git 和网络访问。服务响应必须包含 `token_ids`（或 `tokens`）和 `topk_logprobs`；缺少这些字段的 Case 会以 `skipped` 状态落盘。`model_name` 需与 msProbe 的 `mtype_config.json` 配置以及 token 分类映射保持一致。使用 `--reuse` 时，已有检测结果按 Case id 继承，已完成 Case 不会重复检测。 + 部分全局常量不区分任务类型，推荐保持默认；如需自定义，可编辑常量文件：[`global_consts.py`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/global_consts.py)配置。 当前支持的参数配置如下： | 参数名| 说明| 取值范围 / 要求 | @@ -62,4 +121,4 @@ ais_bench [OPTIONS] | `REQUEST_TIME_OUT` | Client 端请求发送后等待返回的超时时间。默认为 None，即无限等待，始终等待模型返回结果。 | `None` 或 `>0`（单位：秒）| |`LOG_LEVEL`|日志级别，可选：`DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`。默认 `INFO`。|`[DEBUG, INFO, WARNING, ERROR, CRITICAL]`| | `PRESSURE_TIME`| 压测持续时间，仅在指定 `--pressure` 模式时生效。单位为秒。(该参数将在未来版本中废弃，请使用 `--pressure-time` 参数代替)| `[1, 86400]`（即 1 秒 至 24 小时） | -| `CONNECTION_ADD_RATE`| 并发线程创建速率。表示每秒新增的并发线程数，直至达到最大并发限制。仅在指定 `--pressure` 模式时生效。(该参数将在未来版本中废弃，请在模型配置文件中修改 `request_rate` 参数代替) | `> 0.1`（单位：线程数 / 秒） | \ No newline at end of file +| `CONNECTION_ADD_RATE`| 并发线程创建速率。表示每秒新增的并发线程数，直至达到最大并发限制。仅在指定 `--pressure` 模式时生效。(该参数将在未来版本中废弃，请在模型配置文件中修改 `request_rate` 参数代替) | `> 0.1`（单位：线程数 / 秒） | diff --git a/docs/source_zh_cn/base_tutorials/all_params/mode.md b/docs/source_zh_cn/base_tutorials/all_params/mode.md index ab0aad45..d9266a05 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/mode.md +++ b/docs/source_zh_cn/base_tutorials/all_params/mode.md @@ -104,6 +104,11 @@ outputs/default/ │ └── summary/ # 新增汇总报告（viz 输出） └── ... ``` + +### 响应异常检测模式支持（可选） + +msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测线程在推理完成后启动，与 Judge / Eval / 汇总流程并行执行，工作流退出前会等待检测完成。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，具体配置请参考 [推理响应异常检测配置](./cli_args.md#推理响应异常检测配置)。 + ## 性能评测场景 ### perf模式 diff --git a/docs/source_zh_cn/base_tutorials/all_params/models.md b/docs/source_zh_cn/base_tutorials/all_params/models.md index 6f917179..c9e22895 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/models.md +++ b/docs/source_zh_cn/base_tutorials/all_params/models.md @@ -27,6 +27,9 @@ AISBench Benchmark 支持多种服务化推理后端，包括 vLLM、SGLang、Tr ### 服务化推理后端配置参数说明 服务化推理后端配置文件采用Python语法格式配置，示例如下： + +通用模型配置模板不预置 `response_anomaly`。仅在需要响应异常检测时，在实际模型配置文件的 `models` 列表中找到目标模型，将下例中的 `response_anomaly` 添加到该模型的 `dict` 内；它与 `generation_kwargs`、`pred_postprocessor` 等模型字段同级。不开启异常检测时无需添加。 + ```python from ais_bench.benchmark.models import VLLMCustomAPI @@ -51,6 +54,12 @@ models = [ generation_kwargs = dict( # 模型推理参数，参考VLLM文档配置，AISBench评测工具不做处理，在发送的请求中附带 temperature = 0.01, ignore_eos=False, + ), + response_anomaly = dict( # 可选，msProbe 推理响应异常检测的模型级配置 + model_name="", # 填写模型名称，如 Qwen3-30B-A3B + model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 + msprobe_mtype_path='/path/to/mtype_config.json', + msprobe_token2category_dir='/path/to/token2category/', ) ) ] @@ -81,11 +90,14 @@ models = [ | `generation_kwargs` | Dict | 推理生成参数配置，依赖具体的服务化后端和接口类型。注意：当前不支持 `best_of` 和 `n` 等多次采样参数，但支持通过`num_return_sequences`参数进行多次独立推理(具体请参考🔗[Text Generation 文档](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences)中`num_return_sequences`的作用) | | `returns_tool_calls` | Bool | 控制函数调用信息的提取方式。当设置为True时，系统从API响应的`tool_calls`字段中提取函数调用信息；当设置为False时，系统从`content`字段中解析函数调用信息 | | `pred_postprocessor` | Dict | 模型输出结果的后处理配置。用于对原始模型输出进行格式化、清理或转换，以满足特定评估任务的要求 | +| `response_anomaly` | Dict | 可选，msProbe 推理响应异常检测的模型级配置，包含 `model_name`（与 msProbe 的 mtype_config.json 名称一致）、`model_path`（本地模型目录，用于自动生成配置，可选）、`msprobe_mtype_path`、`msprobe_token2category_dir`。未提供 mtype/token 分类路径时回退到 msProbe 包内默认文件 | **注意事项：** +- 响应异常检测当前仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 - `request_rate` 受硬件性能影响，可通过增加 📚 [WORKERS_NUM](./cli_args.md#配置常量文件参数) 提高并发能力。 - `request_rate` 功能可能被`traffic_cfg`项覆盖，具体原因请参考 🔗 [请求速率(RPS)分布控制及可视化说明中的参数解读章节](../../advanced_tutorials/rps_distribution.md#参数解读)。 - 当数据集含 timestamp 且模型配置中 **use_timestamp** 为 True 时，请求按 timestamp 发送，**request_rate** 与 **traffic_cfg** 将被忽略。 +- 使用响应异常检测（`response_anomaly`）时，服务端必须返回 token id 与 top-k logprobs；启用检测后 AISBench 会自动在推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`，服务响应缺少这些字段的 Case 检测结果标记为 `skipped`。详细配置请参考 [推理响应异常检测配置](./cli_args.md#推理响应异常检测配置)。 - `batch_size` 设置过大可能导致 CPU 占用过高，请根据硬件条件合理配置。 - 服务化推理评测 API 默认使用的服务地址为 `localhost:8080`。实际使用时需根据实际部署修改为服务化后端的 IP 和端口。 - 当使用 IPv6 字面量（如 `::1`、`2001:db8::1`）作为 `host_ip` 时，工具会在生成的访问 URL 中自动为其添加方括号（例如 `http://[2001:db8::1]:8080/`），无需在配置中手动编写方括号。 @@ -225,4 +237,4 @@ models = [ | `sample_kwargs` | Dict | 模型推理采样参数，参考 🔗 [sample parameter配置](https://docs.vllm.ai/en/v0.6.5/dev/sampling_params.html) | | `vision_kwargs` | Dict | 多模态输入参数，参考 🔗 [多模态推理举例](https://docs.vllm.ai/en/v0.7.3/getting_started/examples/vision_language.html) | | `max_out_len` | Int | 推理生成的最大输出 Token 数量 | -| `batch_size` | Int | 推理请求的批处理大小，合法范围：(0, 64000] | \ No newline at end of file +| `batch_size` | Int | 推理请求的批处理大小，合法范围：(0, 64000] | diff --git "a/docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" "b/docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" new file mode 100644 index 00000000..5892e599 --- /dev/null +++ "b/docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" @@ -0,0 +1,372 @@ +# AISBench 推理响应异常检测模块设计 + +## 1. 概述 + +### 1.1 背景 + +大模型推理服务可能出现生僻字、乱码、重复输出以及 logprob 为 `NaN` 或 `Inf` 等输出异常。AISBench 在完成服务模型推理后，基于推理响应中的 token id 与 top-k logprobs 调用 msProbe 的 Response Anomaly 能力，对每个 Case 进行异常检测。 + +本模块不实现或改写检测算法；异常判定统一由已安装的官方 `mindstudio-probe` 包提供的 `msprobe.response_anomaly.detector.ILLDetector` 完成。 + +### 1.2 目标 + +1. 通过总配置与命令行开关启用或关闭响应异常检测。 +2. 推理完成后以后台线程执行检测，并与后续 Eval、汇总阶段并行。 +3. 输出 Case 级异常状态、异常类型与检测执行状态。 +4. 将检测明细落盘，并在任务状态面板显示检测进度及类型统计。 +5. 在 `--reuse` 中断续推场景继承已有异常检测结果和统计数量。 +6. 将 msProbe 作为可选依赖；未安装时不影响普通 AISBench 推理与评测。 + +### 1.3 非目标 + +- 不在 AISBench 内部复制、修改或维护 msProbe 的异常检测算法。 +- 不在 AISBench 内部复制或维护 token 分类生成算法；AISBench 仅提供包装工具调用 msProbe 官方生成器，并将产物输出到用户指定目录。 +- 不将异常 Case 自动改写为推理失败，也不改变原有评测指标；异常信息是独立审计结果。 +- 当前仅覆盖通过 `BaseAPIModel` 的服务模型生成链路。 +- 不支持性能模式（`perf`）与 Agent 测评链路（SWE-bench / SWE-bench Pro / BFCL / agent_example 等）；在这些场景启用会直接报错，避免静默空转或改变 Agent 请求参数。 + +## 2. 依赖与前置条件 + +### 2.1 软件依赖 + +响应异常检测通过可选 extra 引入官方包： + +```bash +pip install 'ais-bench-benchmark[response_anomaly]' +``` + +依赖定义位于 [requirements/response_anomaly.txt](../../../requirements/response_anomaly.txt)。安装 `response_anomaly` extra 时，pip 会从 GitCode 下载并构建固定提交 `3de412d71d6566a62c28b9131f9969930628d87f` 的官方 msProbe 源码。AISBench 正常安装不强制安装该依赖；安装环境需要 Git 与 GitCode 网络访问。 + +### 2.2 服务响应要求 + +当前仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 + +服务端必须在响应中返回： + +- 生成 token 序列：`token_ids` 或 `tokens` +- 每个生成 token 对应的 top-k logprobs：`topk_logprobs` + +AISBench 启用功能时会向服务请求参数补充： + +```python +logprobs=True +top_logprobs=20 +``` + +`top_logprobs` 由检测算法固定为 `20`，不作为外部配置项开放。 + +服务适配器将上述字段提取为 `response_anomaly_payload`，输出处理器立即将其分流到独立的 ZSTD 压缩 JSONL staging，prediction 不保存完整 payload。检测线程流式解压 staging；服务未提供必要字段时，Case 仍正常评测，但检测结果记录为 `skipped`。 + +### 2.3 msProbe 模型配置要求 + +msProbe 检测依赖三个文件： + +| 文件 | 作用 | +| --- | --- | +| `config.yaml` | 检测算法阈值配置。 | +| `mtype_config.json` | 模型名与 BOS/EOS token id 映射，用于交叉验证模型。 | +| `token2category/<模型名>_<词表大小>.json` | token id 到字符类别映射，用于生僻字和乱码检测。 | + +三个文件均可通过 AISBench 配置指定路径；未指定时回退到 msProbe 安装包内默认文件。对于 msProbe 未内置的模型，可使用 AISBench 提供的包装工具调用 msProbe 官方 `gen_model_config.py` 生成： + +```bash +ais_bench-gen-response-anomaly-config \ + --model-path /home/Qwen3-30B-A3B \ + --model-name Qwen3-30B-A3B \ + --output-dir ./msprobe_configs +``` + +产物布局： + +```text +./msprobe_configs/ +├── configs/ +│ ├── config.yaml +│ └── mtype_config.json +└── token2category/ + └── qwen3-30b-a3b_151643.json +``` + +`mtype_config.json` 支持多模型合并，多次运行不会互相覆盖；`config.yaml` 已存在时不会被覆盖，便于用户手工调阈值。 + +## 3. 总体设计 + +### 3.1 模块关系 + +```mermaid +flowchart LR + CLI[CLI / 总配置] --> CM[ConfigManager] + CM --> API[BaseAPIModel] + API --> PRED[轻量预测 JSONL] + API --> PAYLOAD[独立 JSONL.ZST staging] + PAYLOAD --> COORD[ResponseAnomalyCoordinator] + COORD --> MSP[msprobe.ILLDetector] + MSP --> RESULT[异常结果 JSONL] + COORD --> STATUS[状态文件] + STATUS --> BOARD[任务状态面板] + PRED --> EVAL[Eval] + EVAL --> SUMMARY[评测汇总] +``` + +### 3.2 执行时序 + +```mermaid +sequenceDiagram + participant I as Infer + participant P as Predictions JSONL + participant C as ResponseAnomalyCoordinator + participant M as msProbe + participant E as Eval / Summary + participant R as Response Anomaly JSONL + + I->>P: 写入轻量推理 Case + I->>C: token/logprobs 写入独立 JSONL.ZST staging + I->>C: 推理阶段结束后启动后台线程 + par 异常检测 + C->>M: ILLDetector.run(topk_logprobs, tokens, model_configs) + M-->>C: [is_ill, ill_type] + C->>R: 写入 Case 检测结果 + and 正常评测 + I->>E: 进入 Judge / Eval / Summary + end + C-->>E: 工作流收尾前 join +``` + +### 3.3 关键模块 + +| 模块 | 文件 | 职责 | +| --- | --- | --- | +| CLI 开关 | [ais_bench/benchmark/cli/argument_parser.py](../../../ais_bench/benchmark/cli/argument_parser.py) | 提供 `--response-anomaly` 和 `--no-response-anomaly`。 | +| 配置归一化 | [ais_bench/benchmark/cli/config_manager.py](../../../ais_bench/benchmark/cli/config_manager.py) | 合并 CLI / 总配置，注入服务端 logprobs 请求参数。 | +| 工作流协调 | [ais_bench/benchmark/cli/workers.py](../../../ais_bench/benchmark/cli/workers.py) | 推理结束后启动检测线程；工作流结束前等待线程完成。 | +| 响应采集 | [ais_bench/benchmark/models/api_models/base_api.py](../../../ais_bench/benchmark/models/api_models/base_api.py) | 从流式或非流式服务响应中提取 token 与 top-k logprobs。 | +| Case 分流 | [ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py](../../../ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py) | 将 `response_anomaly_payload` 写入独立 JSONL.ZST staging，并从 prediction Case 移除。 | +| 检测协调器 | [ais_bench/benchmark/utils/response_anomaly.py](../../../ais_bench/benchmark/utils/response_anomaly.py) | 读预测、合并模型级配置、按需生成 msProbe 配置、初始化检测器、落盘、恢复与状态上报。 | +| 配置生成工具 | [ais_bench/tools/response_anomaly/gen_model_config.py](../../../ais_bench/tools/response_anomaly/gen_model_config.py) | 包装 msProbe 官方生成器，输出到用户目录并合并 mtype 配置。 | +| 面板展示 | [ais_bench/benchmark/runners/base.py](../../../ais_bench/benchmark/runners/base.py) | 动态读取 `ResponseAnomaly` 状态并展示。 | + +## 4. 配置设计 + +### 4.1 总配置与模型级配置 + +运行级开关与公共参数放在全局 `response_anomaly`，模型相关配置放在各模型的 `response_anomaly` 中： + +异常检测配置不会预置在通用模型配置模板中。需要启用该功能时，请在实际使用的模型配置文件中找到 `models` 列表里的目标模型，并在该模型的 `dict` 内添加 `response_anomaly`；它与 `generation_kwargs`、`pred_postprocessor` 等模型字段同级。未启用异常检测时无需添加。 + +```python +response_anomaly = dict( + enabled=True, + payload_retention='anomalies', # all | anomalies | none + payload_storage=dict( + format='jsonl', + compression='zstd', + compression_level=3, + rows_per_shard=2000, + ), + msprobe_config_path='/path/to/config.yaml', # 可选，算法阈值配置 +) + +models = [ + dict( + abbr='qwen3-30b', + attr='service', + response_anomaly=dict( + model_name="", # 填写模型名称，如 Qwen3-30B-A3B + model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 + msprobe_mtype_path='/path/to/mtype_config.json', + msprobe_token2category_dir='/path/to/token2category/', + ), + ), +] +``` + +| 配置项 | 层级 | 类型 | 默认值 | 说明 | +| --- | --- | --- | --- | --- | +| `enabled` | 全局 | bool | `False` | 是否启动异常检测。 | +| `payload_retention` | 全局 | str | `anomalies` | `all` 保存全部 payload；`anomalies` 仅保存异常及检测失败/不可用 payload；`none` 不保存 payload。 | +| `payload_storage.compression_level` | 全局 | int | `3` | ZSTD 压缩级别，范围 1-22。 | +| `payload_storage.rows_per_shard` | 全局 | int | `2000` | 每个 `.jsonl.zst` 分片的最大 Case 数。 | +| `model_name` | 模型 | str | 模型 `abbr` | msProbe 模型名称，应与其 `mtype_config.json` 及 token 分类映射一致。 | +| `model_path` | 模型 | str | 无 | 本地模型目录；配置后且未指定 mtype/token2category 时自动生成。 | +| `msprobe_config_path` | 全局/模型 | str | msProbe 包内默认 | 算法阈值 `config.yaml` 路径。 | +| `msprobe_mtype_path` | 模型 | str | msProbe 包内默认 | `mtype_config.json` 路径。 | +| `msprobe_token2category_dir` | 模型 | str | msProbe 包内默认 | `token2category/` 目录路径。 | + +当 `model_path` 已配置且未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时，AISBench 在检测启动前自动调用配置生成工具，输出到 `/response_anomaly_config/<模型 abbr>/`。 + +### 4.2 命令行优先级 + +- `--response-anomaly`：强制启用。 +- `--no-response-anomaly`：强制关闭。 +- `--response-anomaly-payload-retention {all,anomalies,none}`：覆盖 payload 保存模式，但不隐式开启异常检测。 +- 未传命令行参数：采用 `response_anomaly.enabled`。 + +命令行优先级高于总配置；未指定 payload 保存模式时默认使用 `anomalies`。 + +## 5. 数据与接口设计 + +### 5.1 msProbe 调用接口 + +```python +from msprobe.response_anomaly.detector import ILLDetector + +detector = ILLDetector( + config_path, + mtype_path, + tk2cat_path, +) +result = detector.run([topk_logprobs], [tokens], [model_name]) +``` + +AISBench 直接使用 `ILLDetector`，以便传入用户配置的三个文件路径；每个模型组只初始化一次检测器，避免逐 Case 重复加载配置。输入、输出与 msProbe 保持一致： + +| 参数 | 类型 | 说明 | +| --- | --- | --- | +| `topk_logprobs` | `List[List[Dict[int, float]]]` | 每个请求中每个 token 的候选 token id 与 logprob。 | +| `tokens` | `List[List[int]]` | 每个请求的生成 token id 序列。 | +| `model_configs` | `List[Any]` | 每个请求对应的模型名称或 msProbe 支持的模型配置。 | +| 返回值 | `List[List[Any]]` | 格式为 `[[is_ill, ill_type], ...]`。 | + +异常类型约定：`0` 正常、`1` 生僻字、`2` 乱码、`3` 重复、`4` NaN Value。 + +### 5.2 预测 Case 扩展字段 + +当服务返回必要信息时，原始预测 JSONL 会增加： + +```json +{ + "id": 12, + "uuid": "...", + "success": true, + "prediction": "...", + "response_anomaly_payload": { + "tokens": [151643, 123, 456], + "topk_logprobs": [ + {"151643": -0.01, "12": -5.2}, + {"123": -0.12, "88": -3.5} + ] + } +} +``` + +该字段仅作为检测输入保留，Eval 不读取该字段，因此不改变原有指标计算。 + +### 5.3 异常结果 Schema + +异常检测结果写入： + +```text +/response_anomaly//.jsonl +``` + +每行对应一个 Case： + +```json +{ + "id": 12, + "uuid": "...", + "is_anomaly": true, + "anomaly_type": 3, + "anomaly_type_name": "repetition", + "detection_status": "completed" +} +``` + +`detection_status` 的取值： + +| 状态 | 含义 | +| --- | --- | +| `completed` | 已调用 msProbe 并得到检测结果。 | +| `skipped` | 推理响应未携带 token 或 top-k logprobs。 | +| `unavailable` | 未安装 `mindstudio-probe`。 | +| `failed` | 调用或输入转换发生异常，`reason` 保存错误摘要。 | + +## 6. 并发、状态与恢复设计 + +### 6.1 并发策略 + +检测线程由 `ResponseAnomalyCoordinator` 管理： + +1. Infer runner 全部任务完成后启动一个后台线程。 +2. 线程逐个读取预测 Case 并调用 msProbe。 +3. Infer worker 在启动检测线程后串行等待其完成：专属检测状态面板在推理面板之后、评测面板之前渲染，检测结束并打屏最终状态后才进入 Eval / JudgeInfer / AccViz。 + +检测不参与模型请求链路，不增加单 Case 推理请求的同步等待时间；检测串行绑定在推理阶段内完成，保证 Infer 阶段退出时检测结果与 payload 归档均已落盘。 + +同一个 `work_dir` 在任一时刻只允许一个 AISBench 任务读写。协调器的线程状态检查仅防止单进程内重复启动，不提供跨进程互斥；并行执行任务时必须使用不同的 `work_dir`，`--reuse` 也应在原任务退出后串行执行。 + +### 6.2 状态面板 + +协调器将状态写入： + +```text +/status_tmp/tmp_ResponseAnomaly.json +``` + +状态字段包括： + +- `finish_count`：已处理 Case 数量。 +- `total_count`：预测文件中的 Case 总数。 +- `progress_description`：检测中或检测完成。 +- `other_kwargs`：按 `anomaly_type_name` 聚合的数量。 + +检测状态由独立的专属面板渲染（评测面板不再混入检测行，保证两类状态分开放置）：`ResponseAnomaly` 状态文件使用原子替换写入，普通 runner 清理临时目录时会保留该文件；检测启动后 Infer worker 会拉起一个独立监控进程展示检测进度，检测完成后打出最终状态表并统一清理状态目录。 + +### 6.3 中断续推 + +检测开始前，协调器读取已存在的异常结果 JSONL： + +1. 以 `id` 建立已处理 Case 集合。 +2. 已存在的结果不重复调用 msProbe。 +3. 将既有结果的异常类型累加到实时统计。 +4. 仅处理预测 JSONL 中未记录的 Case。 + +因此，结合 `--reuse` 继续执行时，已完成 Case 的异常数量和检测结果均被继承。 + +当 `payload_retention='anomalies'` 且没有需要保留的 Case 时，仍会发布一个只包含空 manifest 的 payload 目录。该目录表示归档流程已经成功完成，并用于后续无操作续跑判断，不属于残留文件。 + +## 7. 异常处理 + +| 场景 | 行为 | +| --- | --- | +| 未安装 msProbe | Case 结果写为 `unavailable`，普通推理与 Eval 不失败。 | +| 没有 token/logprobs | Case 结果写为 `skipped`。 | +| msProbe 抛出异常 | Case 结果写为 `failed`，保留异常类型与消息到 `reason`。 | +| 单个 Case 检测失败 | 继续处理后续 Case。 | +| 结果文件已存在 | 通过文件锁追加写入，避免并发写入冲突。 | +| 推理结果文件不存在 | 该模型/数据集组合按空输入处理，不产生 Case 检测结果。 | + +## 8. 测试设计 + +### 8.1 单元测试 + +[tests/UT/utils/test_response_anomaly.py](../../../tests/UT/utils/test_response_anomaly.py) 覆盖： + +- `msprobe.response_anomaly.detector.ILLDetector` 的初始化与调用参数，以及自定义三个配置文件路径的传递。 +- 官方接口返回的异常标志及类型向 Case 结果的映射。 +- 缺少 `response_anomaly_payload` 时的 `skipped` 分支。 + +### 8.2 集成测试 + +集成环境应满足： + +1. 安装 `ais-bench-benchmark[response_anomaly]`。 +2. 准备 msProbe 支持的模型名称及 token 分类文件。 +3. 使用能返回 `token_ids` 和 `topk_logprobs` 的兼容推理服务。 +4. 执行 `--mode all --response-anomaly`，校验预测、评测、异常结果与状态统计。 + +### 8.3 回归验收 + +- 功能关闭时：预测与 Eval 行为应与未接入模块前一致。 +- msProbe 不可用时：不影响普通推理与评测，异常结果明确标记不可用。 +- `--reuse` 时：重复执行不应新增相同 `id` 的异常结果。 +- 已知异常样本：msProbe 返回的类型与落盘类型一致。 + +## 9. 安全、兼容性与限制 + +- 当前仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置。 +- 预测 JSONL 中的 token/logprob 可能增加落盘体积，应只在开关启用时请求和保存。 +- 当前服务响应字段兼容根节点或首个 `choices` 节点；新的服务协议需要在 `BaseAPIModel` 扩展提取逻辑。 +- 仅支持 `all` / `infer` / `infer_judge` 普通生成链路；性能模式与 Agent 测评模式不支持。 +- msProbe 依赖已锁定到验证通过的 Git commit；升级时应同步更新依赖声明、模型资源兼容性验证与回归结果。 +- 该模块依赖 msProbe 提供的模型资源。模型未在其 `mtype_config.json` 或 token 分类映射中配置时，部分检测能力可能无法生效。 diff --git a/docs/source_zh_cn/get_started/install.md b/docs/source_zh_cn/get_started/install.md index 0426a22c..a3118b4d 100644 --- a/docs/source_zh_cn/get_started/install.md +++ b/docs/source_zh_cn/get_started/install.md @@ -31,6 +31,19 @@ pip3 install -r requirements/api.txt pip3 install -r requirements/extra.txt ``` +⚙️ 推理响应异常检测支持（可选） + +若需使用 msProbe 推理响应异常检测（`--response-anomaly`），需额外安装相关依赖： +```shell +pip3 install -r requirements/response_anomaly.txt +``` +或通过 extra 方式安装： +```shell +pip3 install 'ais-bench-benchmark[response_anomaly]' +``` + +**注意**：该依赖包含从 GitCode 下载并构建固定提交的 `mindstudio-probe` 源码，安装环境需要 Git 与网络访问。未安装该依赖不影响 AISBench 主流程，相关 Case 的检测结果会标记为 `unavailable`。 + ⚙️ Huggingface多模态模型/vllm多模态离线推理支持（可选） ```shell diff --git a/requirements/response_anomaly.txt b/requirements/response_anomaly.txt new file mode 100644 index 00000000..3acba250 --- /dev/null +++ b/requirements/response_anomaly.txt @@ -0,0 +1,4 @@ +# Install the official msProbe source at a verified commit for reproducible +# response anomaly detection. Requires Git and network access during install. +mindstudio-probe @ git+https://gitcode.com/Ascend/msprobe.git@3de412d71d6566a62c28b9131f9969930628d87f +zstandard>=0.22.0 diff --git a/setup.py b/setup.py index f53c3152..64cec3c7 100644 --- a/setup.py +++ b/setup.py @@ -173,11 +173,15 @@ def do_setup(): 'ocrbench_v2': parse_requirements('requirements/datasets/ocrbench_v2.txt') + parse_requirements('requirements/runtime.txt'), + 'response_anomaly': + parse_requirements('requirements/response_anomaly.txt') + + parse_requirements('requirements/runtime.txt'), 'full': parse_requirements('requirements/extra.txt') + parse_requirements('requirements/api.txt') + parse_requirements('requirements/hf_vl_dependency.txt') + parse_requirements('requirements/datasets/bfcl_dependencies.txt') + + parse_requirements('requirements/response_anomaly.txt') + parse_requirements('requirements/runtime.txt'), }, license='Apache License 2.0', @@ -204,6 +208,8 @@ def do_setup(): entry_points={ 'console_scripts': [ 'ais_bench = ais_bench.benchmark.cli.main:main', + 'ais_bench-gen-response-anomaly-config = ' + 'ais_bench.tools.response_anomaly.gen_model_config:main', ], }, ) diff --git a/tests/UT/cli/test_argument_parser.py b/tests/UT/cli/test_argument_parser.py index 11c98de3..d07c977a 100644 --- a/tests/UT/cli/test_argument_parser.py +++ b/tests/UT/cli/test_argument_parser.py @@ -37,6 +37,25 @@ def test_parse_args_default(self, mock_get_current_time_str): self.assertEqual(args.max_workers_per_gpu, 1) self.assertEqual(args.num_warmups, 1) self.assertIsNone(args.num_prompts) + self.assertIsNone(args.response_anomaly_payload_retention) + + @patch('ais_bench.benchmark.cli.argument_parser.get_current_time_str') + def test_parse_args_response_anomaly_payload_retention( + self, mock_get_current_time_str + ): + mock_get_current_time_str.return_value = "20230516_144254" + for retention in ('all', 'anomalies', 'none'): + sys.argv = [ + 'benchmark.py', + '--response-anomaly-payload-retention', + retention, + ] + + with self.subTest(retention=retention): + args = ArgumentParser().parse_args() + self.assertEqual( + args.response_anomaly_payload_retention, retention + ) @patch('ais_bench.benchmark.cli.argument_parser.get_current_time_str') def test_parse_args_with_config(self, mock_get_current_time_str): @@ -217,4 +236,4 @@ def test_init_method_creates_parser(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/UT/cli/test_config_manager.py b/tests/UT/cli/test_config_manager.py index 04610d9f..aea26c5c 100644 --- a/tests/UT/cli/test_config_manager.py +++ b/tests/UT/cli/test_config_manager.py @@ -5,6 +5,7 @@ import shutil from ais_bench.benchmark.cli.config_manager import CustomConfigChecker, ConfigManager +from ais_bench.benchmark.models import VLLMCustomAPI, VLLMCustomAPIChat from ais_bench.benchmark.utils.logging.exceptions import CommandError, AISBenchConfigError from ais_bench.benchmark.utils.logging.error_codes import TMAN_CODES @@ -174,6 +175,12 @@ def setUp(self): self.args.custom_dataset_infer_method = None self.args.custom_dataset_data_type = None self.args.custom_dataset_meta_path = None + self.args.response_anomaly_payload_retention = None + + # Local tokenizer directory consumed by response anomaly model-path + # fallback tests. + self.tokenizer_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tokenizer_dir) # 创建配置目录结构 os.makedirs(os.path.join(self.args.config_dir, 'models'), exist_ok=True) @@ -686,5 +693,412 @@ def test_load_config(self, mock_fill_dataset_configs, mock_dump_reload, mock_upd mock_dump_reload.assert_called_once() self.assertEqual(result, config_manager.cfg) + def _service_model(self, **overrides): + """A service model that passes the response anomaly whitelist and + model-resource checks (chat backend + tokenizer path fallback).""" + model = { + 'abbr': 'model', + 'attr': 'service', + 'type': VLLMCustomAPIChat, + 'generation_kwargs': {}, + 'path': self.tokenizer_dir, + } + model.update(overrides) + return model + + def test_response_anomaly_rejected_in_perf_mode(self): + """响应异常检测不支持性能模式。""" + self.args.mode = 'perf' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [{'abbr': 'model', 'attr': 'service'}], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError): + config_manager._init_response_anomaly_config() + + def test_response_anomaly_rejected_for_agent_model(self): + """响应异常检测不支持 Agent 模型。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [{'abbr': 'agent-model', 'agent_name': 'x', 'attr': 'service'}], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError): + config_manager._init_response_anomaly_config() + + def test_response_anomaly_injects_request_kwargs(self): + """启用响应异常检测时为 service 模型注入 logprobs 与内部开关。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [self._service_model()], + 'datasets': [{'abbr': 'dataset'}], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + generation_kwargs = config_manager.cfg['models'][0]['generation_kwargs'] + self.assertTrue(generation_kwargs['logprobs']) + self.assertEqual(generation_kwargs['top_logprobs'], 20) + self.assertTrue(generation_kwargs['response_anomaly_enabled']) + self.assertTrue(config_manager.cfg['response_anomaly']['enabled']) + self.assertEqual( + config_manager.cfg['response_anomaly']['payload_retention'], + 'anomalies', + ) + self.assertEqual( + config_manager.cfg['response_anomaly']['payload_storage'], + { + 'format': 'jsonl', + 'compression': 'zstd', + 'compression_level': 3, + 'rows_per_shard': 2000, + }, + ) + + def test_response_anomaly_injects_payload_storage_into_service_models(self): + """work_dir 初始化后，为 service 模型注入 payload 运行时配置。""" + config_manager = ConfigManager(self.args) + service_model = self._service_model(abbr='service-model') + local_model = {'abbr': 'local-model', 'attr': 'local'} + config_manager.cfg = { + 'work_dir': '/test/workdir', + 'response_anomaly': { + 'enabled': True, + 'payload_storage': { + 'format': 'jsonl', + 'compression': 'zstd', + }, + }, + 'models': [service_model, local_model], + } + + config_manager._inject_response_anomaly_payload_storage() + + self.assertEqual( + service_model['response_anomaly_payload_storage'], + { + 'work_dir': '/test/workdir', + 'model_abbr': 'service-model', + 'format': 'jsonl', + 'compression': 'zstd', + }, + ) + self.assertNotIn('response_anomaly_payload_storage', local_model) + + def test_response_anomaly_skips_payload_storage_when_disabled(self): + """异常检测关闭时不向模型配置添加运行时字段。""" + config_manager = ConfigManager(self.args) + model_cfg = self._service_model() + config_manager.cfg = { + 'work_dir': '/test/workdir', + 'response_anomaly': {'enabled': False}, + 'models': [model_cfg], + } + + config_manager._inject_response_anomaly_payload_storage() + + self.assertNotIn('response_anomaly_payload_storage', model_cfg) + + def test_response_anomaly_rejects_invalid_payload_retention(self): + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'response_anomaly': {'payload_retention': 'sometimes'}, + 'models': [self._service_model()], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError): + config_manager._init_response_anomaly_config() + + def test_response_anomaly_cli_payload_retention_overrides_config(self): + self.args.mode = 'all' + self.args.response_anomaly = True + self.args.response_anomaly_payload_retention = 'none' + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'response_anomaly': {'payload_retention': 'all'}, + 'models': [self._service_model()], + 'datasets': [], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + self.assertEqual( + config_manager.cfg['response_anomaly']['payload_retention'], + 'none', + ) + + def test_response_anomaly_rejects_invalid_payload_storage(self): + self.args.mode = 'all' + self.args.response_anomaly = True + for payload_storage in ( + {'format': 'parquet'}, + {'compression': 'gzip'}, + {'compression_level': 0}, + {'compression_level': True}, + {'rows_per_shard': 0}, + {'rows_per_shard': True}, + ): + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'response_anomaly': { + 'payload_storage': payload_storage, + }, + 'models': [self._service_model()], + 'datasets': [], + 'cli_args': {}, + } + + with self.subTest(payload_storage=payload_storage): + with self.assertRaises(AISBenchConfigError): + config_manager._init_response_anomaly_config() + + def test_response_anomaly_overrides_explicit_logprobs_config(self): + """启用异常检测时强制覆盖模型里显式的 logprobs/top_logprobs。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + self._service_model( + generation_kwargs={ + 'logprobs': False, + 'top_logprobs': 5, + }, + ) + ], + 'datasets': [{'abbr': 'dataset'}], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + generation_kwargs = config_manager.cfg['models'][0]['generation_kwargs'] + self.assertIs(generation_kwargs['logprobs'], True) + self.assertEqual(generation_kwargs['top_logprobs'], 20) + self.assertTrue(generation_kwargs['response_anomaly_enabled']) + + def test_response_anomaly_rejects_configurable_top_logprobs(self): + """异常检测使用固定 top_logprobs，不允许外部修改。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'response_anomaly': {'top_logprobs': 30}, + 'models': [self._service_model()], + 'datasets': [{'abbr': 'dataset'}], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError): + config_manager._init_response_anomaly_config() + + def test_response_anomaly_merges_model_level_config(self): + """模型级 response_anomaly 配置覆盖全局配置。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + self._service_model( + response_anomaly={ + 'model_name': 'Custom-Name', + 'model_path': self.tokenizer_dir, + 'top_logprobs': 20, + }, + ) + ], + 'datasets': [{'abbr': 'dataset'}], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + model_anomaly_cfg = config_manager.cfg['models'][0]['response_anomaly'] + self.assertEqual(model_anomaly_cfg['model_name'], 'Custom-Name') + self.assertEqual(model_anomaly_cfg['model_path'], self.tokenizer_dir) + self.assertNotIn('top_logprobs', model_anomaly_cfg) + self.assertEqual( + config_manager.cfg['models'][0]['generation_kwargs']['top_logprobs'], + 20, + ) + + def test_response_anomaly_rejected_in_eval_viz_judge_modes(self): + """单开 eval/viz/judge 模式不支持响应异常检测。""" + for mode in ('eval', 'viz', 'judge'): + self.args.mode = mode + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [self._service_model()], + 'datasets': [], + 'cli_args': {}, + } + with self.subTest(mode=mode): + with self.assertRaises(AISBenchConfigError) as cm: + config_manager._init_response_anomaly_config() + self.assertIn('not supported in mode', str(cm.exception)) + + def test_response_anomaly_rejected_for_unsupported_model_class(self): + """completions 后端（VLLMCustomAPI）无法返回 token id，应被拦截。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + self._service_model(type=VLLMCustomAPI), + ], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError) as cm: + config_manager._init_response_anomaly_config() + self.assertIn('VLLMCustomAPIChat', str(cm.exception)) + + def test_response_anomaly_rejected_for_local_models(self): + """本地模型不在检测范围，白名单检查应跳过（由 service 校验兜底）。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + { + 'abbr': 'local-model', + 'attr': 'local', + 'path': self.tokenizer_dir, + } + ], + 'datasets': [], + 'cli_args': {}, + } + + # 无 service 模型时报错（白名单不应先对 local 模型误报）。 + with self.assertRaises(AISBenchConfigError) as cm: + config_manager._init_response_anomaly_config() + self.assertIn('no service model', str(cm.exception)) + + def test_response_anomaly_model_path_falls_back_to_model_path_field(self): + """response_anomaly.model_path 未配置时回退模型 path 字段（tokenizer 目录）。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [self._service_model()], + 'datasets': [], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + model_anomaly_cfg = config_manager.cfg['models'][0]['response_anomaly'] + self.assertEqual(model_anomaly_cfg['model_path'], self.tokenizer_dir) + + def test_response_anomaly_rejects_invalid_model_path_field(self): + """模型 path 指向不存在目录时应报错并说明原因。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [self._service_model(path='/nonexistent/tokenizer-dir')], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError) as cm: + config_manager._init_response_anomaly_config() + self.assertIn('non-existent directory', str(cm.exception)) + + def test_response_anomaly_rejects_missing_model_resources(self): + """无 model_path 也无 msprobe 三件套时直接报错，不回退 msProbe 默认。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [self._service_model(path='')], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError) as cm: + config_manager._init_response_anomaly_config() + message = str(cm.exception) + self.assertIn('msprobe_mtype_path', message) + self.assertIn('msprobe_token2category_dir', message) + self.assertIn('ais_bench-gen-response-anomaly-config', message) + + def test_response_anomaly_accepts_explicit_msprobe_paths(self): + """显式配置 mtype + token2category（真实存在）时无需 model_path。""" + import os as _os + + mtype_path = _os.path.join(self.tokenizer_dir, 'mtype_config.json') + tk2cat_dir = _os.path.join(self.tokenizer_dir, 'token2category') + open(mtype_path, 'w').close() + _os.makedirs(tk2cat_dir, exist_ok=True) + + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + self._service_model( + path='', + response_anomaly={ + 'msprobe_mtype_path': mtype_path, + 'msprobe_token2category_dir': tk2cat_dir, + }, + ) + ], + 'datasets': [], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + model_anomaly_cfg = config_manager.cfg['models'][0]['response_anomaly'] + self.assertEqual(model_anomaly_cfg['msprobe_mtype_path'], mtype_path) + self.assertIsNone(model_anomaly_cfg['model_path']) + + def test_response_anomaly_rejects_nonexistent_msprobe_paths(self): + """显式配置的 msProbe 路径不存在时启动即报错，而不是运行期全 failed。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + self._service_model( + path='', + response_anomaly={ + 'msprobe_mtype_path': '/workspace/missing/mtype_config.json', + 'msprobe_token2category_dir': '/workspace/missing/token2category', + }, + ) + ], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError) as cm: + config_manager._init_response_anomaly_config() + message = str(cm.exception) + self.assertIn('do not exist', message) + self.assertIn('/workspace/missing/mtype_config.json', message) + self.assertIn('/workspace/missing/token2category', message) + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/UT/cli/test_workers.py b/tests/UT/cli/test_workers.py index 459d536f..d6201273 100644 --- a/tests/UT/cli/test_workers.py +++ b/tests/UT/cli/test_workers.py @@ -17,7 +17,8 @@ AccViz, PerfViz, WorkFlowExecutor, - WORK_FLOW + WORK_FLOW, + _finalize_response_anomaly_detection, ) from ais_bench.benchmark.partitioners import NaivePartitioner from ais_bench.benchmark.runners import LocalRunner @@ -311,6 +312,167 @@ def test_update_tasks_cfg_without_attack(self): # 执行测试 - 不应抛出异常 self.infer_worker._update_tasks_cfg(tasks, cfg) + @patch('ais_bench.benchmark.cli.workers.PARTITIONERS') + @patch('ais_bench.benchmark.cli.workers.RUNNERS') + @patch('ais_bench.benchmark.cli.workers.logger') + def test_do_work_starts_anomaly_detection_after_runner(self, mock_logger, mock_runners, mock_partitioners): + """启用检测时，协调器在 runner 完成后启动并串行等待完成（绑定 infer 阶段）""" + mock_partitioner = MagicMock() + mock_partitioners.build.return_value = mock_partitioner + mock_partitioner.return_value = [] + mock_runner = MagicMock() + mock_runners.build.return_value = mock_runner + + coordinator = MagicMock() + coordinator.is_running = False + coordinator.anomaly_report = {} + coordinator.summary = {'normal': 1} + self.infer_worker.response_anomaly_coordinator = coordinator + + order = [] + mock_runner.side_effect = lambda tasks: order.append('runner') + coordinator.start.side_effect = ( + lambda cfg: order.append('coordinator.start') + ) + coordinator.join.side_effect = lambda: order.append('coordinator.join') + + cfg = MockConfigDict({ + 'infer': {'partitioner': {}, 'runner': {}}, + 'cli_args': MagicMock(merge_ds=False, mode='all'), + 'work_dir': '/test/workdir', + 'response_anomaly': {'enabled': True, 'payload_storage': {}}, + }) + + with patch.object(self.infer_worker, '_update_tasks_cfg'): + self.infer_worker.do_work(cfg) + + coordinator.start.assert_called_once_with(cfg) + coordinator.join.assert_called_once() + assert order == ['runner', 'coordinator.start', 'coordinator.join'] + + @patch('ais_bench.benchmark.cli.workers.TasksMonitor.rm_tmp_files') + @patch('ais_bench.benchmark.cli.workers._run_response_anomaly_monitor') + def test_finalize_anomaly_detection_runs_monitor_in_current_process( + self, mock_monitor, mock_rm_tmp_files + ): + """检测线程运行时，主线程同步展示专用状态看板。""" + coordinator = MagicMock() + coordinator.is_running = True + coordinator.task_names = ['ResponseAnomaly/model/dataset'] + coordinator.summary = {} + coordinator.anomaly_report = {} + order = [] + mock_monitor.side_effect = lambda *args: order.append('monitor') + coordinator.join.side_effect = lambda: order.append('join') + + _finalize_response_anomaly_detection( + coordinator, '/test/workdir', False + ) + + mock_monitor.assert_called_once_with( + coordinator.task_names, '/test/workdir', False + ) + assert order == ['monitor', 'join'] + mock_rm_tmp_files.assert_called_once_with('/test/workdir') + + @patch('ais_bench.benchmark.cli.workers.TasksMonitor.rm_tmp_files') + @patch('ais_bench.benchmark.cli.workers._run_response_anomaly_monitor') + @patch('ais_bench.benchmark.cli.workers.osp.isfile', return_value=False) + def test_finalize_anomaly_detection_without_status_only_joins( + self, mock_isfile, mock_monitor, mock_rm_tmp_files + ): + """检测未产生状态时不启动看板，仍等待线程并完成清理。""" + coordinator = MagicMock() + coordinator.is_running = False + coordinator.summary = {} + coordinator.anomaly_report = {} + + _finalize_response_anomaly_detection( + coordinator, '/test/workdir', False + ) + + mock_monitor.assert_not_called() + coordinator.join.assert_called_once() + mock_rm_tmp_files.assert_called_once_with('/test/workdir') + + @patch('ais_bench.benchmark.cli.workers.PARTITIONERS') + @patch('ais_bench.benchmark.cli.workers.RUNNERS') + @patch('ais_bench.benchmark.cli.workers.logger') + @patch('os.path.isfile', return_value=True) + @patch('os.remove', side_effect=OSError('permission denied')) + def test_do_work_warns_when_stale_anomaly_status_cannot_be_removed( + self, + mock_remove, + mock_isfile, + mock_logger, + mock_runners, + mock_partitioners, + ): + """旧状态清理失败时告警，但不阻断推理和异常检测。""" + mock_partitioner = MagicMock() + mock_partitioners.build.return_value = mock_partitioner + mock_partitioner.return_value = [] + mock_runner = MagicMock() + mock_runners.build.return_value = mock_runner + + coordinator = MagicMock() + coordinator.anomaly_report = {} + coordinator.summary = {'normal': 1} + self.infer_worker.response_anomaly_coordinator = coordinator + cfg = MockConfigDict({ + 'infer': {'partitioner': {}, 'runner': {}}, + 'cli_args': MagicMock(merge_ds=False, mode='all'), + 'work_dir': '/test/workdir', + 'response_anomaly': {'enabled': True}, + }) + + with ( + patch.object(self.infer_worker, '_update_tasks_cfg'), + patch( + 'ais_bench.benchmark.cli.workers._run_response_anomaly_monitor' + ), + patch( + 'ais_bench.benchmark.cli.workers.TasksMonitor.rm_tmp_files' + ), + ): + self.infer_worker.do_work(cfg) + + mock_remove.assert_called_once() + mock_logger.warning.assert_any_call( + "Failed to remove stale response anomaly status file %s: %s", + '/test/workdir/status_tmp/tmp_ResponseAnomaly.json', + mock_remove.side_effect, + ) + mock_runner.assert_called_once_with([]) + coordinator.start.assert_called_once_with(cfg) + coordinator.join.assert_called_once() + + @patch('ais_bench.benchmark.cli.workers.PARTITIONERS') + @patch('ais_bench.benchmark.cli.workers.RUNNERS') + @patch('ais_bench.benchmark.cli.workers.logger') + def test_do_work_skips_anomaly_detection_when_disabled(self, mock_logger, mock_runners, mock_partitioners): + """未启用检测时不启动协调器""" + mock_partitioner = MagicMock() + mock_partitioners.build.return_value = mock_partitioner + mock_partitioner.return_value = [] + mock_runner = MagicMock() + mock_runners.build.return_value = mock_runner + + coordinator = MagicMock() + coordinator.is_running = False + self.infer_worker.response_anomaly_coordinator = coordinator + + cfg = MockConfigDict({ + 'infer': {'partitioner': {}, 'runner': {}}, + 'cli_args': MagicMock(merge_ds=False, mode='all'), + 'work_dir': '/test/workdir', + }) + + with patch.object(self.infer_worker, '_update_tasks_cfg'): + self.infer_worker.do_work(cfg) + + coordinator.start.assert_not_called() + class TestEval: def setup_method(self): @@ -877,4 +1039,4 @@ def test_update_tasks_cfg_with_judge_infer(self): assert 'judge_infer_cfg' not in task['datasets'][0][0] assert task['models'][0]['type'] == 'judge_model_type' - assert task['datasets'][0][0]['type'] == 'judge_dataset' \ No newline at end of file + assert task['datasets'][0][0]['type'] == 'judge_dataset' diff --git a/tests/UT/models/api_models/test_response_anomaly_payload.py b/tests/UT/models/api_models/test_response_anomaly_payload.py new file mode 100644 index 00000000..97c0a4a9 --- /dev/null +++ b/tests/UT/models/api_models/test_response_anomaly_payload.py @@ -0,0 +1,307 @@ +import unittest +from unittest.mock import patch + +from ais_bench.benchmark.models import VLLMCustomAPI +from ais_bench.benchmark.models.api_models import base_api +from ais_bench.benchmark.models.output import Output + + +class TestResponseAnomalyPayload(unittest.TestCase): + def setUp(self): + self._get_service_model_path_patcher = patch.object( + base_api.BaseAPIModel, "_get_service_model_path" + ) + self.mock_get_model_path = self._get_service_model_path_patcher.start() + self.mock_get_model_path.return_value = "mocked-model-path" + self._get_url_patcher = patch.object( + VLLMCustomAPI, + "_get_url", + return_value="http://localhost:8080/v1/completions", + ) + self._get_url_patcher.start() + + def tearDown(self): + self._get_url_patcher.stop() + self._get_service_model_path_patcher.stop() + + def _make_model(self, enabled=True): + generation_kwargs = {"temperature": 0.7} + if enabled: + generation_kwargs["response_anomaly_enabled"] = True + return VLLMCustomAPI( + path="test-model", + model="test-model-name", + generation_kwargs=generation_kwargs, + ) + + def test_enable_flag_is_consumed_and_not_forwarded(self): + model = self._make_model(enabled=True) + + self.assertTrue(model.response_anomaly_enabled) + self.assertNotIn("response_anomaly_enabled", model.generation_kwargs) + + def test_disabled_does_not_capture_payload(self): + model = self._make_model(enabled=False) + output = Output() + + model._record_response_anomaly_payload( + {"token_ids": [1], "topk_logprobs": [{"1": -0.1}]}, output + ) + + self.assertNotIn("response_anomaly_payload", output.extra_details_data) + + def test_records_vllm_openai_logprobs_payload(self): + model = self._make_model(enabled=True) + output = Output() + + model._record_response_anomaly_payload( + { + "choices": [ + { + "token_ids": [10, 11], + "logprobs": { + "content": [ + { + "token": "token_id:10", + "logprob": -0.1, + "top_logprobs": [ + {"token": "token_id:10", "logprob": -0.1}, + {"token": "token_id:12", "logprob": -1.2}, + ], + }, + { + "token": "token_id:11", + "logprob": -0.2, + "top_logprobs": [ + {"token": "token_id:11", "logprob": -0.2}, + {"token": "token_id:13", "logprob": -1.3}, + ], + }, + ] + }, + } + ] + }, + output, + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [10, 11]) + self.assertEqual( + payload["topk_logprobs"], + [{10: -0.1, 12: -1.2}, {11: -0.2, 13: -1.3}], + ) + + def test_vllm_openai_logprobs_can_supply_sampled_token_ids(self): + model = self._make_model(enabled=True) + output = Output() + + model._record_response_anomaly_payload( + { + "choices": [ + { + "logprobs": { + "content": [ + { + "token": "token_id:21", + "logprob": -0.1, + "top_logprobs": [ + {"token": "token_id:21", "logprob": -0.1} + ], + } + ] + } + } + ] + }, + output, + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [21]) + self.assertEqual(payload["topk_logprobs"], [{21: -0.1}]) + + def test_stream_accumulates_vllm_openai_logprobs_chunks(self): + model = self._make_model(enabled=True) + output = Output() + + for token_id, logprob in ((31, -0.1), (32, -0.2)): + model._accumulate_response_anomaly_payload( + { + "choices": [ + { + "token_ids": [token_id], + "logprobs": { + "content": [ + { + "token": f"token_id:{token_id}", + "logprob": logprob, + "top_logprobs": [ + { + "token": f"token_id:{token_id}", + "logprob": logprob, + } + ], + } + ] + }, + } + ] + }, + output, + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [31, 32]) + self.assertEqual(payload["topk_logprobs"], [{31: -0.1}, {32: -0.2}]) + + def test_stream_accumulates_incremental_chunks(self): + model = self._make_model(enabled=True) + output = Output() + + model._accumulate_response_anomaly_payload( + {"token_ids": [10], "topk_logprobs": [{"10": -0.1}]}, output + ) + model._accumulate_response_anomaly_payload( + {"token_ids": [11], "topk_logprobs": [{"11": -0.2}]}, output + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [10, 11]) + self.assertEqual(payload["topk_logprobs"], [{"10": -0.1}, {"11": -0.2}]) + + def test_stream_full_token_ids_with_single_current_topk_appends_incrementally(self): + model = self._make_model(enabled=True) + output = Output() + + model._accumulate_response_anomaly_payload( + {"token_ids": [10], "topk_logprobs": [{"10": -0.1}]}, output + ) + model._accumulate_response_anomaly_payload( + { + "token_ids": [10, 11], + "topk_logprobs": [{"11": -0.2}], + }, + output, + ) + model._accumulate_response_anomaly_payload( + { + "token_ids": [10, 11, 12], + "topk_logprobs": [{"12": -0.3}], + }, + output, + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [10, 11, 12]) + self.assertEqual( + payload["topk_logprobs"], + [{"10": -0.1}, {"11": -0.2}, {"12": -0.3}], + ) + + def test_stream_consecutive_identical_tokens_are_not_dropped(self): + """两个连续相同 token 不应被全量快照覆盖。""" + model = self._make_model(enabled=True) + output = Output() + + model._accumulate_response_anomaly_payload( + {"token_ids": [7], "topk_logprobs": [{"7": -0.1}]}, output + ) + model._accumulate_response_anomaly_payload( + {"token_ids": [7], "topk_logprobs": [{"7": -0.2}]}, output + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [7, 7]) + self.assertEqual(payload["topk_logprobs"], [{"7": -0.1}, {"7": -0.2}]) + + def test_stream_mismatched_snapshot_does_not_corrupt_payload(self): + model = self._make_model(enabled=True) + output = Output() + + model._accumulate_response_anomaly_payload( + { + "token_ids": [10, 11], + "topk_logprobs": [{"10": -0.1}, {"11": -0.2}], + }, + output, + ) + model._accumulate_response_anomaly_payload( + { + "token_ids": [10, 11, 12], + "topk_logprobs": [], + }, + output, + ) + model._accumulate_response_anomaly_payload( + {"token_ids": [12], "topk_logprobs": [{"12": -0.3}]}, + output, + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [10, 11, 12]) + self.assertEqual( + payload["topk_logprobs"], + [{"10": -0.1}, {"11": -0.2}, {"12": -0.3}], + ) + + def test_stream_misaligned_chunk_logs_debug(self): + """长度不匹配的 chunk 应丢弃并留下 debug 日志。""" + model = self._make_model(enabled=True) + output = Output() + + with patch.object(model.logger, "debug") as mock_debug: + model._accumulate_response_anomaly_payload( + { + "token_ids": [10, 11], + "topk_logprobs": [{"10": -0.1}], + }, + output, + ) + + mock_debug.assert_called_once() + self.assertNotIn("response_anomaly_payload", output.extra_details_data) + + def test_stream_same_length_snapshot_with_different_prefix_replaces_previous_state(self): + model = self._make_model(enabled=True) + output = Output() + + model._accumulate_response_anomaly_payload( + { + "token_ids": [10, 11], + "topk_logprobs": [{"10": -0.1}, {"11": -0.2}], + }, + output, + ) + model._accumulate_response_anomaly_payload( + { + "token_ids": [20, 21], + "topk_logprobs": [{"20": -0.3}, {"21": -0.4}], + }, + output, + ) + model._accumulate_response_anomaly_payload( + {"token_ids": [22], "topk_logprobs": [{"22": -0.5}]}, + output, + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [20, 21, 22]) + + def test_stream_snapshot_chunk_replaces_previous_state(self): + model = self._make_model(enabled=True) + output = Output() + + model._accumulate_response_anomaly_payload( + {"token_ids": [10], "topk_logprobs": [{"10": -0.1}]}, output + ) + model._accumulate_response_anomaly_payload( + { + "token_ids": [10, 11], + "topk_logprobs": [{"10": -0.1}, {"11": -0.2}], + }, + output, + ) + + payload = output.extra_details_data["response_anomaly_payload"] + self.assertEqual(payload["tokens"], [10, 11]) diff --git a/tests/UT/models/api_models/test_vllm_custom_api_chat.py b/tests/UT/models/api_models/test_vllm_custom_api_chat.py index 93c3115a..decca28f 100644 --- a/tests/UT/models/api_models/test_vllm_custom_api_chat.py +++ b/tests/UT/models/api_models/test_vllm_custom_api_chat.py @@ -158,6 +158,23 @@ async def test_get_request_body_stream_enabled(self): self.assertTrue(request_body["stream"]) self.assertIn("stream_options", request_body) self.assertEqual(request_body["stream_options"]["include_usage"], True) + + def test_response_anomaly_requests_vllm_token_ids(self): + kwargs = self.default_kwargs.copy() + kwargs["generation_kwargs"] = { + "response_anomaly_enabled": True, + "return_token_ids": False, + "return_tokens_as_token_ids": False, + } + model = VLLMCustomAPIChat(**kwargs) + + request_body = asyncio.run( + model.get_request_body("test prompt", 100, RequestOutput()) + ) + + self.assertTrue(request_body["return_token_ids"]) + self.assertTrue(request_body["return_tokens_as_token_ids"]) + self.assertNotIn("response_anomaly_enabled", request_body) async def test_parse_stream_response(self): """测试parse_stream_response方法""" @@ -536,4 +553,4 @@ def test_request_body_lora_hit_with_promptlist(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/UT/openicl/icl_inferencer/output_handler/test_base_handler.py b/tests/UT/openicl/icl_inferencer/output_handler/test_base_handler.py index a5da84f6..2874a3e5 100644 --- a/tests/UT/openicl/icl_inferencer/output_handler/test_base_handler.py +++ b/tests/UT/openicl/icl_inferencer/output_handler/test_base_handler.py @@ -7,6 +7,9 @@ import numpy as np import functools import shutil +import json + +import pytest from ais_bench.benchmark.openicl.icl_inferencer.output_handler.base_handler import BaseInferencerOutputHandler from ais_bench.benchmark.models.output import Output @@ -52,6 +55,112 @@ def test_write_to_json_success(self): file_path = os.path.join(tmpdir, "test.jsonl") self.assertTrue(os.path.exists(file_path)) + def test_write_to_json_stages_payload_separately(self): + pytest.importorskip("zstandard") + with TempDirectory() as tmpdir: + handler = ConcreteOutputHandler( + response_anomaly_payload_storage={ + "work_dir": tmpdir, + "model_abbr": "modelA", + "compression_level": 3, + "rows_per_shard": 10, + } + ) + handler.results_dict["ds"] = { + "uid1": { + "data_abbr": "ds", + "id": 1, + "uuid": "u1", + "prediction": "ok", + "response_anomaly_payload": { + "tokens": [1], + "topk_logprobs": [{"1": -0.1}], + }, + } + } + + handler.write_to_json(os.path.join(tmpdir, "predictions"), False) + + prediction = json.loads( + open( + os.path.join(tmpdir, "predictions", "ds.jsonl"), + encoding="utf-8", + ).read() + ) + self.assertNotIn("response_anomaly_payload", prediction) + staging = os.path.join( + tmpdir, + "response_anomaly", + "modelA", + "payload_staging", + "ds", + ) + self.assertEqual( + len([name for name in os.listdir(staging) if name.endswith(".zst")]), + 1, + ) + + def test_write_to_json_staging_failure_degrades_gracefully(self): + """staging 失败不应中断 prediction 写入，payload 保留内联且不再重试""" + with TempDirectory() as tmpdir: + handler = ConcreteOutputHandler( + response_anomaly_payload_storage={ + "work_dir": tmpdir, + "model_abbr": "modelA", + "compression_level": 3, + "rows_per_shard": 10, + } + ) + handler.results_dict["ds"] = { + "uid1": { + "data_abbr": "ds", + "id": 1, + "uuid": "u1", + "prediction": "ok", + "response_anomaly_payload": { + "tokens": [1], + "topk_logprobs": [{"1": -0.1}], + }, + }, + "uid2": { + "data_abbr": "ds", + "id": 2, + "uuid": "u2", + "prediction": "ok", + "response_anomaly_payload": { + "tokens": [2], + "topk_logprobs": [{"2": -0.1}], + }, + }, + } + failing_writer = mock.Mock() + failing_writer.write.side_effect = OSError("disk full") + + with mock.patch( + "ais_bench.benchmark.utils.response_anomaly_jsonl." + "ResponseAnomalyStagingWriter", + return_value=failing_writer, + ) as writer_cls: + # 不得抛异常，prediction 必须正常写出 + handler.write_to_json(os.path.join(tmpdir, "predictions"), False) + + self.assertIsNotNone(handler._response_anomaly_staging_error) + # 失败后禁用 staging，不再创建 writer 或重试写入 + writer_cls.assert_called_once() + failing_writer.write.assert_called_once() + predictions = [ + json.loads(line) + for line in open( + os.path.join(tmpdir, "predictions", "ds.jsonl"), + encoding="utf-8", + ) + if line.strip() + ] + self.assertEqual(len(predictions), 2) + # 两个 Case 的 payload 都保留在 prediction 内联字段中 + for prediction in predictions: + self.assertIn("response_anomaly_payload", prediction) + def test_write_to_json_empty_results_dict(self): """测试write_to_json在results_dict为空时不创建文件""" handler = ConcreteOutputHandler() @@ -378,4 +487,3 @@ def test_stop_cache_consumer_failure(self): if __name__ == '__main__': unittest.main() - diff --git a/tests/UT/runners/test_base.py b/tests/UT/runners/test_base.py index c85589c2..39fa4035 100644 --- a/tests/UT/runners/test_base.py +++ b/tests/UT/runners/test_base.py @@ -100,12 +100,36 @@ def test_tasks_monitor_init(self, mock_logger_class, mock_makedirs, mock_exists) mock_makedirs.assert_called_once() @patch('ais_bench.benchmark.runners.base.os.path.exists', return_value=True) + @patch('ais_bench.benchmark.runners.base.os.listdir', return_value=[]) @patch('ais_bench.benchmark.runners.base.shutil.rmtree') - def test_rm_tmp_files(self, mock_rmtree, mock_exists): + def test_rm_tmp_files(self, mock_rmtree, mock_listdir, mock_exists): """Test rm_tmp_files static method.""" TasksMonitor.rm_tmp_files("/tmp/test") mock_rmtree.assert_called_once() + @patch('ais_bench.benchmark.runners.base.os.path.exists', return_value=True) + @patch( + 'ais_bench.benchmark.runners.base.os.listdir', + side_effect=[['tmp_task1.json'], ['tmp_task1.json']], + ) + @patch( + 'ais_bench.benchmark.runners.base.os.remove', + side_effect=OSError('permission denied'), + ) + @patch('ais_bench.benchmark.runners.base.AISLogger') + def test_rm_tmp_files_warns_and_continues_on_remove_error( + self, mock_logger_class, mock_remove, mock_listdir, mock_exists + ): + """Status cleanup failures are visible but do not abort the workflow.""" + TasksMonitor.rm_tmp_files("/tmp/test") + + mock_remove.assert_called_once_with('/tmp/test/status_tmp/tmp_task1.json') + mock_logger_class.return_value.warning.assert_called_once_with( + "Failed to remove task status path %s: %s", + '/tmp/test/status_tmp/tmp_task1.json', + mock_remove.side_effect, + ) + @patch('ais_bench.benchmark.runners.base.os.path.exists', return_value=False) @patch('ais_bench.benchmark.runners.base.os.makedirs') @patch('ais_bench.benchmark.runners.base.AISLogger') @@ -159,6 +183,48 @@ def test_tasks_monitor_refresh_task_state(self, mock_read_statuses, mock_logger_ self.assertEqual(monitor.tasks_state_map['task1']['finish_count'], 10) self.assertEqual(monitor.tasks_state_map['task1']['status'], 'running') + @patch('ais_bench.benchmark.runners.base.os.path.exists', return_value=False) + @patch('ais_bench.benchmark.runners.base.os.makedirs') + @patch('ais_bench.benchmark.runners.base.AISLogger') + @patch('ais_bench.benchmark.runners.base.read_and_clear_statuses') + @patch('ais_bench.benchmark.runners.base.json.load') + @patch('ais_bench.benchmark.runners.base.open', create=True) + def test_tasks_monitor_anomaly_status_is_opt_in( + self, mock_open, mock_json_load, mock_read_statuses, + mock_logger_class, mock_makedirs, mock_exists + ): + """默认（eval/judge 面板）不读取检测状态；专属检测面板 opt-in 读取。""" + mock_read_statuses.return_value = [] + monitor = TasksMonitor( + task_names=self.task_names, + output_path=self.output_path, + is_debug=True + ) + + monitor._refresh_task_state() + + mock_open.assert_not_called() + + anomaly_task_name = 'ResponseAnomaly/model/ds' + opt_in_monitor = TasksMonitor( + task_names=[anomaly_task_name], + output_path=self.output_path, + is_debug=True, + include_anomaly_status=True, + ) + mock_json_load.return_value = [ + { + 'task_name': anomaly_task_name, + 'process_id': 1, + 'finish_count': 1, + 'total_count': 2, + 'status': 'response anomaly', + } + ] + opt_in_monitor._refresh_task_state() + + self.assertIn(anomaly_task_name, opt_in_monitor.tasks_state_map) + @patch('ais_bench.benchmark.runners.base.os.path.exists', return_value=False) @patch('ais_bench.benchmark.runners.base.os.makedirs') @patch('ais_bench.benchmark.runners.base.AISLogger') @@ -229,6 +295,7 @@ def test_tasks_monitor_update_tasks_progress(self, mock_sleep, mock_tqdm, mock_pbar.close.assert_called_once() + class TestBaseRunner(unittest.TestCase): """Tests for BaseRunner class.""" @@ -289,4 +356,3 @@ def test_base_runner_launch_abstract(self, mock_logger_class): if __name__ == "__main__": unittest.main() - diff --git a/tests/UT/tools/response_anomaly/test_gen_model_config.py b/tests/UT/tools/response_anomaly/test_gen_model_config.py new file mode 100644 index 00000000..8986fd19 --- /dev/null +++ b/tests/UT/tools/response_anomaly/test_gen_model_config.py @@ -0,0 +1,146 @@ +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from ais_bench.tools.response_anomaly.gen_model_config import ( + _normalize_name, + generate_model_config, +) + + +class TestGenerateModelConfig(unittest.TestCase): + def test_normalize_name_matches_msprobe_official(self): + """锁定与 msProbe 官方 _normalize_name 完全一致的行为。 + + msProbe 官方 gen_model_config.py 使用相同的 split+join 算法，保留 + 连续分隔符产生的空段（foo__bar -> foo--bar）。AISBench 必须逐字符 + 一致，否则生成的 key 与 msProbe 侧查找 key 不匹配。 + """ + self.assertEqual(_normalize_name("foo__bar"), "foo--bar") + self.assertEqual(_normalize_name("foo..bar"), "foo--bar") + self.assertEqual(_normalize_name("foo_-bar"), "foo--bar") + self.assertEqual(_normalize_name("deepseek--v3"), "deepseek--v3") + self.assertEqual(_normalize_name("Qwen2.5-72B"), "qwen2-5-72b") + self.assertEqual(_normalize_name("MyModel"), "mymodel") + + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_dir.cleanup) + self.root = Path(self.temp_dir.name) + + self.msprobe_dir = self.root / "msprobe" + (self.msprobe_dir / "tools").mkdir(parents=True) + (self.msprobe_dir / "tools" / "gen_model_config.py").write_text( + "# fake", encoding="utf-8" + ) + (self.msprobe_dir / "configs").mkdir(parents=True) + (self.msprobe_dir / "configs" / "config.yaml").write_text( + "window_size: 128", encoding="utf-8" + ) + + self.output_dir = self.root / "msprobe_configs" + (self.output_dir / "configs").mkdir(parents=True) + (self.output_dir / "configs" / "mtype_config.json").write_text( + json.dumps({"old-model": {"eos": 1}}), encoding="utf-8" + ) + + def _patch_msprobe_dir(self): + return mock.patch( + "ais_bench.tools.response_anomaly.gen_model_config." + "_msprobe_response_anomaly_dir", + return_value=self.msprobe_dir, + ) + + def test_generate_model_config_merges_and_copies_defaults(self): + def fake_run(command, cwd, **kwargs): + output_root = Path(cwd).parent + self.assertTrue((output_root / "configs").is_dir()) + self.assertTrue((output_root / "token2category").is_dir()) + (output_root / "configs" / "mtype_config.json").write_text( + json.dumps({"new-model": {"eos": 2}}), encoding="utf-8" + ) + (output_root / "token2category" / "new-model_10.json").write_text( + json.dumps({"0": "other"}), encoding="utf-8" + ) + return subprocess.CompletedProcess(command, 0) + + with self._patch_msprobe_dir(), mock.patch( + "ais_bench.tools.response_anomaly.gen_model_config.subprocess.run", + side_effect=fake_run, + ): + generated = generate_model_config( + model_path="/models/new", + model_name="New-Model", + output_dir=str(self.output_dir), + ) + + mtype = json.loads( + (self.output_dir / "configs" / "mtype_config.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(set(mtype), {"old-model", "new-model"}) + self.assertTrue( + (self.output_dir / "configs" / "config.yaml").exists() + ) + self.assertTrue( + (self.output_dir / "token2category" / "new-model_10.json").exists() + ) + self.assertEqual( + generated["msprobe_config_path"], + str(self.output_dir / "configs" / "config.yaml"), + ) + self.assertEqual(generated["model_name"], "new-model") + self.assertFalse((self.output_dir / "_gen_tmp_new-model").exists()) + + def test_generate_model_config_failure_raises(self): + failed = subprocess.CompletedProcess( + ["python", "gen_model_config.py"], 1, stdout="", stderr="boom" + ) + with self._patch_msprobe_dir(), mock.patch( + "ais_bench.tools.response_anomaly.gen_model_config.subprocess.run", + return_value=failed, + ): + with self.assertRaises(RuntimeError) as cm: + generate_model_config( + model_path="/models/new", + model_name="New-Model", + output_dir=str(self.output_dir), + ) + self.assertIn("boom", str(cm.exception)) + self.assertTrue((self.output_dir / "_gen_tmp_new-model").exists()) + # Old mtype_config.json must not have been clobbered. + mtype = json.loads( + (self.output_dir / "configs" / "mtype_config.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(set(mtype), {"old-model"}) + + def test_generate_model_config_keeps_tools_dir_when_run_raises(self): + with self._patch_msprobe_dir(), mock.patch( + "ais_bench.tools.response_anomaly.gen_model_config.subprocess.run", + side_effect=OSError("cannot run"), + ): + with self.assertRaises(RuntimeError) as cm: + generate_model_config( + model_path="/models/new", + model_name="New-Model", + output_dir=str(self.output_dir), + ) + + self.assertIn("cannot run", str(cm.exception)) + self.assertTrue((self.output_dir / "_gen_tmp_new-model").exists()) + mtype = json.loads( + (self.output_dir / "configs" / "mtype_config.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(set(mtype), {"old-model"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/UT/utils/test_response_anomaly.py b/tests/UT/utils/test_response_anomaly.py new file mode 100644 index 00000000..bcbdcf05 --- /dev/null +++ b/tests/UT/utils/test_response_anomaly.py @@ -0,0 +1,1253 @@ +import json +import sys +import threading +import types +from collections import Counter + +import pytest + +pytest.importorskip("zstandard") + +from ais_bench.benchmark.utils.response_anomaly import ResponseAnomalyCoordinator +from ais_bench.benchmark.utils.response_anomaly_jsonl import ( + ResponseAnomalyJsonlWriter, + iter_jsonl_zstd_records, +) + + +class FakeDetector: + def __init__(self, result): + self.result = result + + def run(self, topk_logprobs, tokens, model_configs): + return [self.result] + + +class TokenDetector: + def run(self, topk_logprobs, tokens, model_configs): + return [[tokens[0][0] == 2, 1 if tokens[0][0] == 2 else 0]] + + +def _payload_record(case_id): + return { + "data_abbr": "ds", + "id": case_id, + "uuid": f"u{case_id}", + "response_anomaly_payload": { + "tokens": [case_id], + "topk_logprobs": [{str(case_id): -0.1}], + }, + } + + +def _completed_anomaly_result(case_id): + return { + "id": case_id, + "uuid": f"u{case_id}", + "is_anomaly": True, + "anomaly_type": 2, + "anomaly_type_name": "garbled", + "detection_status": "completed", + } + + +def _write_jsonl(path, records): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + + +def _anomaly_cfg(tmp_path): + return { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds"}], + "response_anomaly": {"payload_retention": "anomalies"}, + } + + +def test_detect_case_calls_msprobe_detector(): + result = ResponseAnomalyCoordinator()._detect_case( + { + "id": 2, + "uuid": "case-2", + "response_anomaly_payload": { + "tokens": [1, 2], + "topk_logprobs": [{"1": -0.1}, {"2": -0.2}], + }, + }, + {"model_name": "model"}, + FakeDetector([True, 3]), + None, + ) + + assert result["is_anomaly"] is True + assert result["anomaly_type"] == 3 + assert result["anomaly_type_name"] == "repetition" + + +def test_detect_case_skips_missing_token_payload(): + result = ResponseAnomalyCoordinator()._detect_case( + {"id": 2, "uuid": "case-2"}, {}, None, None + ) + + assert result["detection_status"] == "skipped" + assert result["is_anomaly"] is False + + +def test_detect_case_skips_inconsistent_payload(): + result = ResponseAnomalyCoordinator()._detect_case( + { + "id": 3, + "uuid": "case-3", + "response_anomaly_payload": { + "tokens": [1, 2], + "topk_logprobs": [{"1": -0.1}], + }, + }, + {}, + None, + None, + ) + + assert result["detection_status"] == "skipped" + assert "equal length" in result["reason"] + + +def test_detect_case_reports_detector_init_error(): + result = ResponseAnomalyCoordinator()._detect_case( + { + "id": 4, + "uuid": "case-4", + "response_anomaly_payload": { + "tokens": [1], + "topk_logprobs": [{"1": -0.1}], + }, + }, + {}, + None, + ("unavailable", "mindstudio-probe is required"), + ) + + assert result["detection_status"] == "unavailable" + assert result["anomaly_type_name"] == "unavailable" + + +def test_build_detector_reports_missing_msprobe(monkeypatch): + monkeypatch.setitem(sys.modules, "msprobe", None) + + detector, init_error = ResponseAnomalyCoordinator._build_detector({}) + + assert detector is None + assert init_error[0] == "unavailable" + assert "mindstudio-probe" in init_error[1] + + +def test_build_detector_reports_ill_detector_init_failure(monkeypatch): + msprobe_pkg = types.ModuleType("msprobe") + response_anomaly_pkg = types.ModuleType("msprobe.response_anomaly") + response_anomaly_pkg.__file__ = ( + "/fake/msprobe/response_anomaly/__init__.py" + ) + detector_module = types.ModuleType("msprobe.response_anomaly.detector") + + class FailingILLDetector: + def __init__(self, *args, **kwargs): + raise RuntimeError("boom") + + detector_module.ILLDetector = FailingILLDetector + monkeypatch.setitem(sys.modules, "msprobe", msprobe_pkg) + monkeypatch.setitem( + sys.modules, "msprobe.response_anomaly", response_anomaly_pkg + ) + monkeypatch.setitem( + sys.modules, "msprobe.response_anomaly.detector", detector_module + ) + + detector, init_error = ResponseAnomalyCoordinator._build_detector({}) + + assert detector is None + assert init_error[0] == "failed" + assert "boom" in init_error[1] + + +def test_cache_detector_token_categories_loads_each_key_once(): + class Detector: + def __init__(self): + self.calls = 0 + + def get_tk2cat(self, eos_token, model_config=None): + self.calls += 1 + return {"1": "latin"}, 100 + + detector = Detector() + ResponseAnomalyCoordinator._cache_detector_token_categories(detector) + + first = detector.get_tk2cat(2, "model") + second = detector.get_tk2cat(2, "model") + third = detector.get_tk2cat(3, "model") + + assert first == second + assert third == first + assert detector.calls == 2 + + +def test_merge_model_anomaly_config_prefers_model_level(): + merged = ResponseAnomalyCoordinator._merge_model_anomaly_config( + { + "abbr": "qwen", + "response_anomaly": { + "model_name": "Qwen3-30B-A3B", + "msprobe_mtype_path": "/custom/mtype.json", + }, + }, + { + "enabled": True, + "model_name": "global-name", + "msprobe_mtype_path": None, + }, + ) + + assert merged["model_name"] == "Qwen3-30B-A3B" + assert merged["msprobe_mtype_path"] == "/custom/mtype.json" + + +def test_prepare_model_config_auto_generates_when_paths_missing( + tmp_path, monkeypatch +): + generated = { + "msprobe_config_path": str(tmp_path / "config.yaml"), + "msprobe_mtype_path": str(tmp_path / "mtype.json"), + "msprobe_token2category_dir": str(tmp_path / "tk2cat"), + } + monkeypatch.setattr( + "ais_bench.tools.response_anomaly.gen_model_config.generate_model_config", + lambda **kwargs: generated, + ) + + cfg = ResponseAnomalyCoordinator()._prepare_model_config( + "qwen", + {"model_path": "/models/qwen", "model_name": "Qwen3-30B-A3B"}, + str(tmp_path), + ) + + assert cfg["msprobe_mtype_path"] == str(tmp_path / "mtype.json") + assert cfg["msprobe_token2category_dir"] == str(tmp_path / "tk2cat") + assert cfg["model_name"] == "Qwen3-30B-A3B" + + +def test_prepare_model_config_empty_config_path_keeps_builtin_default( + tmp_path, monkeypatch +): + """config.yaml 留空时保持内置默认，不填充生成路径。""" + generated = { + "msprobe_config_path": str(tmp_path / "generated" / "config.yaml"), + "msprobe_mtype_path": str(tmp_path / "generated" / "mtype.json"), + "msprobe_token2category_dir": str(tmp_path / "generated" / "tk2cat"), + } + monkeypatch.setattr( + "ais_bench.tools.response_anomaly.gen_model_config.generate_model_config", + lambda **kwargs: generated, + ) + + cfg = ResponseAnomalyCoordinator()._prepare_model_config( + "qwen", + { + "model_path": "/models/qwen", + "model_name": "Qwen3-30B-A3B", + "msprobe_config_path": None, + }, + str(tmp_path), + ) + + assert cfg.get("msprobe_config_path") is None + assert cfg["msprobe_mtype_path"] == generated["msprobe_mtype_path"] + assert cfg["msprobe_token2category_dir"] == generated[ + "msprobe_token2category_dir" + ] + + +def test_prepare_model_config_places_generated_resources_at_targets( + tmp_path, monkeypatch +): + """显式配置的输出位置缺失时，生成产物被放置到目标路径并复用已存在文件。""" + gen_root = tmp_path / "generated" + target_root = tmp_path / "data" + + def fake_generate(**kwargs): + configs = gen_root / "configs" + tk2cat = gen_root / "token2category" + configs.mkdir(parents=True, exist_ok=True) + tk2cat.mkdir(parents=True, exist_ok=True) + (configs / "config.yaml").write_text("config", encoding="utf-8") + (configs / "mtype_config.json").write_text("mtype", encoding="utf-8") + (tk2cat / "qwen_vocab.json").write_text("vocab", encoding="utf-8") + return { + "msprobe_config_path": str(configs / "config.yaml"), + "msprobe_mtype_path": str(configs / "mtype_config.json"), + "msprobe_token2category_dir": str(tk2cat), + } + + monkeypatch.setattr( + "ais_bench.tools.response_anomaly.gen_model_config.generate_model_config", + fake_generate, + ) + + # 已存在的目标文件不应被覆盖。 + existing_mtype = target_root / "configs" / "mtype_config.json" + existing_mtype.parent.mkdir(parents=True, exist_ok=True) + existing_mtype.write_text("user-mtype", encoding="utf-8") + + cfg = ResponseAnomalyCoordinator()._prepare_model_config( + "qwen", + { + "model_path": "/models/qwen", + "model_name": "Qwen3-30B-A3B", + "msprobe_config_path": str(target_root / "configs" / "config.yaml"), + "msprobe_mtype_path": str(existing_mtype), + "msprobe_token2category_dir": str(target_root / "token2category"), + }, + str(tmp_path), + ) + + assert cfg["msprobe_config_path"] == str( + target_root / "configs" / "config.yaml" + ) + assert (target_root / "configs" / "config.yaml").read_text( + encoding="utf-8" + ) == "config" + # 已存在文件未被覆盖。 + assert cfg["msprobe_mtype_path"] == str(existing_mtype) + assert existing_mtype.read_text(encoding="utf-8") == "user-mtype" + assert cfg["msprobe_token2category_dir"] == str( + target_root / "token2category" + ) + assert (target_root / "token2category" / "qwen_vocab.json").read_text( + encoding="utf-8" + ) == "vocab" + + +def test_prepare_model_config_requires_both_custom_paths(tmp_path): + with pytest.raises(RuntimeError): + ResponseAnomalyCoordinator()._prepare_model_config( + "qwen", + { + "model_path": "/models/qwen", + "msprobe_mtype_path": "/tmp/mtype.json", + }, + str(tmp_path), + ) + + +def test_post_status_is_atomic(tmp_path): + coordinator = ResponseAnomalyCoordinator() + status_file = tmp_path / ResponseAnomalyCoordinator.STATUS_FILE_NAME + + coordinator._post_status( + status_file, + completed=1, + total=2, + counts=Counter({"normal": 1}), + description="detecting", + ) + + data = json.loads(status_file.read_text(encoding="utf-8")) + assert len(data) == 1 + assert data[0]["task_name"] == "ResponseAnomaly" + assert data[0]["finish_count"] == 1 + assert data[0]["other_kwargs"] == {"normal": 1} + assert not status_file.with_name(status_file.name + ".tmp").exists() + + +def test_post_status_keeps_each_model_dataset_task(tmp_path): + coordinator = ResponseAnomalyCoordinator() + status_file = tmp_path / ResponseAnomalyCoordinator.STATUS_FILE_NAME + + for dataset_abbr in ("ds1", "ds2"): + coordinator._post_status( + status_file, + completed=1, + total=2, + counts=Counter({"normal": 1}), + description="detecting", + task_name=coordinator.task_name("modelA", dataset_abbr), + task_log_path=coordinator.task_log_path("modelA", dataset_abbr), + ) + + data = json.loads(status_file.read_text(encoding="utf-8")) + assert [item["task_name"] for item in data] == [ + "ResponseAnomaly/modelA/ds1", + "ResponseAnomaly/modelA/ds2", + ] + assert [item["task_log_path"] for item in data] == [ + "logs/response_anomaly/modelA/ds1.out", + "logs/response_anomaly/modelA/ds2.out", + ] + + +def test_task_log_only_captures_response_anomaly_thread(tmp_path): + coordinator = ResponseAnomalyCoordinator() + handler = coordinator._open_task_log(str(tmp_path), "modelA", "ds") + coordinator.logger.info("response anomaly message") + other_thread = threading.Thread( + target=coordinator.logger.info, + args=("unrelated workflow message",), + ) + other_thread.start() + other_thread.join() + coordinator._close_task_log(handler) + + content = ( + tmp_path + / "logs" + / "response_anomaly" + / "modelA" + / "ds.out" + ).read_text(encoding="utf-8") + assert "response anomaly message" in content + assert "unrelated workflow message" not in content + + +def test_read_jsonl_skips_broken_lines(tmp_path): + path = tmp_path / "pred.jsonl" + path.write_text('{"id": 1}\nnot-json\n{"id": 2}\n', encoding="utf-8") + + records = ResponseAnomalyCoordinator()._read_jsonl(path) + + assert [item["id"] for item in records] == [1, 2] + + +def test_load_inherited_results_only_keeps_completed(tmp_path): + result_file = tmp_path / "result.jsonl" + result_file.write_text( + json.dumps({"id": 1, "uuid": "abc", "detection_status": "completed", "anomaly_type_name": "normal"}) + + "\n" + + json.dumps({"id": 2, "uuid": "def", "detection_status": "skipped", "anomaly_type_name": "skipped"}) + + "\n", + encoding="utf-8", + ) + + inherited = ResponseAnomalyCoordinator()._load_inherited_results( + result_file, {"1:abc"} + ) + + assert set(inherited) == {"1:abc"} + + +def test_load_inherited_results_rejects_different_uuid(tmp_path): + """同 id 不同 uuid 的旧结果不应被继承。""" + result_file = tmp_path / "result.jsonl" + result_file.write_text( + json.dumps({"id": 1, "uuid": "old-uuid", "detection_status": "completed", "anomaly_type_name": "normal"}) + + "\n", + encoding="utf-8", + ) + + inherited = ResponseAnomalyCoordinator()._load_inherited_results( + result_file, {"1:new-uuid"} + ) + + assert len(inherited) == 0 + + +@pytest.mark.parametrize( + ("retention", "result", "expected"), + [ + ("all", {"is_anomaly": False, "detection_status": "completed"}, True), + ("anomalies", {"is_anomaly": True, "detection_status": "completed"}, True), + ("anomalies", {"is_anomaly": False, "detection_status": "failed"}, True), + ("anomalies", {"is_anomaly": False, "detection_status": "unavailable"}, True), + ("anomalies", {"is_anomaly": False, "detection_status": "completed"}, False), + ("anomalies", {"is_anomaly": False, "detection_status": "skipped"}, False), + ("none", {"is_anomaly": True, "detection_status": "completed"}, False), + ], +) +def test_should_retain_payload(retention, result, expected): + assert ( + ResponseAnomalyCoordinator._should_retain_payload(retention, result) + is expected + ) + + +def test_strip_payloads_from_predictions_is_atomic(tmp_path): + prediction_file = tmp_path / "ds.jsonl" + predictions = [ + {"id": 1, "response_anomaly_payload": {"tokens": [1]}}, + {"id": 2, "prediction": "ok"}, + ] + prediction_file.write_text("old", encoding="utf-8") + + ResponseAnomalyCoordinator._strip_payloads_from_predictions( + prediction_file, predictions + ) + + restored = [ + json.loads(line) + for line in prediction_file.read_text(encoding="utf-8").splitlines() + ] + assert restored == [{"id": 1}, {"id": 2, "prediction": "ok"}] + assert not prediction_file.with_name("ds.jsonl.tmp").exists() + + +def test_detect_runs_full_workflow(tmp_path, monkeypatch): + """端到端驱动 _detect 主循环：读预测、逐条检测、写结果、收尾状态。""" + work_dir = tmp_path + prediction_file = work_dir / "predictions" / "modelA" / "ds.jsonl" + prediction_file.parent.mkdir(parents=True) + prediction_file.write_text( + json.dumps( + { + "id": 1, + "uuid": "u1", + "response_anomaly_payload": { + "tokens": [1], + "topk_logprobs": [{"1": -0.1}], + }, + } + ) + + "\n" + + json.dumps( + { + "id": 2, + "uuid": "u2", + "response_anomaly_payload": { + "tokens": [1, 2], + "topk_logprobs": [{"1": -0.1}, {"2": -0.2}], + }, + } + ) + + "\n", + encoding="utf-8", + ) + cfg = { + "work_dir": str(work_dir), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds"}], + "response_anomaly": {}, + } + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, + "_build_detector", + lambda cfg: (FakeDetector([False, 0]), None), + ) + coordinator._detect(cfg) + + result_lines = ( + work_dir / "response_anomaly" / "modelA" / "ds.jsonl" + ).read_text(encoding="utf-8").strip().splitlines() + assert len(result_lines) == 2 + results = [json.loads(line) for line in result_lines] + assert all(item["detection_status"] == "completed" for item in results) + assert coordinator.summary == {"normal": 2} + + status = json.loads( + ( + work_dir / "status_tmp" / ResponseAnomalyCoordinator.STATUS_FILE_NAME + ).read_text(encoding="utf-8") + )[0] + assert status["task_name"] == "ResponseAnomaly/modelA/ds" + assert status["task_log_path"] == ( + "logs/response_anomaly/modelA/ds.out" + ) + assert status["status"] == "finish" + assert status["finish_count"] == 2 + assert status["total_count"] == 2 + log_file = work_dir / status["task_log_path"] + assert log_file.exists() + log_content = log_file.read_text(encoding="utf-8") + assert "Task [ResponseAnomaly/modelA/ds]" in log_content + assert "Found 2 predictions" in log_content + assert "Response anomaly detection completed: {'normal': 2}" in log_content + + +def test_detect_writes_separate_status_and_log_for_each_dataset( + tmp_path, monkeypatch +): + for dataset_abbr in ("ds1", "ds2"): + prediction_file = ( + tmp_path + / "predictions" + / "modelA" + / f"{dataset_abbr}.jsonl" + ) + prediction_file.parent.mkdir(parents=True, exist_ok=True) + prediction_file.write_text( + json.dumps( + { + "id": 1, + "uuid": f"{dataset_abbr}-u1", + "response_anomaly_payload": { + "tokens": [1], + "topk_logprobs": [{"1": -0.1}], + }, + } + ) + + "\n", + encoding="utf-8", + ) + cfg = { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds1"}, {"abbr": "ds2"}], + "response_anomaly": {}, + } + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, + "_build_detector", + lambda cfg: (FakeDetector([False, 0]), None), + ) + + coordinator._detect(cfg) + + statuses = json.loads( + ( + tmp_path + / "status_tmp" + / ResponseAnomalyCoordinator.STATUS_FILE_NAME + ).read_text(encoding="utf-8") + ) + assert [item["task_name"] for item in statuses] == [ + "ResponseAnomaly/modelA/ds1", + "ResponseAnomaly/modelA/ds2", + ] + assert all(item["status"] == "finish" for item in statuses) + assert all(item["finish_count"] == 1 for item in statuses) + for dataset_abbr in ("ds1", "ds2"): + log_file = ( + tmp_path + / "logs" + / "response_anomaly" + / "modelA" + / f"{dataset_abbr}.out" + ) + assert log_file.exists() + assert ( + f"Task [ResponseAnomaly/modelA/{dataset_abbr}]" + in log_file.read_text(encoding="utf-8") + ) + + +@pytest.mark.parametrize( + ("retention", "expected_ids"), + [("all", [1, 2]), ("anomalies", [2]), ("none", [])], +) +def test_detect_compresses_selected_payloads_and_strips_predictions( + tmp_path, monkeypatch, retention, expected_ids +): + import zstandard + + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + prediction_file.parent.mkdir(parents=True) + predictions = [ + { + "data_abbr": "ds", + "id": case_id, + "uuid": f"u{case_id}", + "prediction": "ok", + "response_anomaly_payload": { + "tokens": [case_id], + "topk_logprobs": [{str(case_id): -0.1}], + }, + } + for case_id in (1, 2) + ] + prediction_file.write_text( + "".join(json.dumps(item) + "\n" for item in predictions), + encoding="utf-8", + ) + cfg = { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds"}], + "response_anomaly": { + "payload_retention": retention, + "payload_storage": { + "compression_level": 3, + "rows_per_shard": 1, + }, + }, + } + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + + coordinator._detect(cfg) + + restored_predictions = [ + json.loads(line) + for line in prediction_file.read_text(encoding="utf-8").splitlines() + ] + assert all( + "response_anomaly_payload" not in item + for item in restored_predictions + ) + payload_dir = ( + tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + ) + if retention == "none": + assert not payload_dir.exists() + return + manifest = json.loads( + (payload_dir / "payload_manifest.json").read_text(encoding="utf-8") + ) + assert manifest["payload_retention"] == retention + assert manifest["total_rows"] == len(expected_ids) + archived_ids = [] + for shard in sorted(payload_dir.glob("*.jsonl.zst")): + with shard.open("rb") as file: + reader = zstandard.ZstdDecompressor().stream_reader(file) + archived_ids.extend( + json.loads(line)["id"] + for line in reader.read().decode("utf-8").splitlines() + ) + assert archived_ids == expected_ids + + +def test_resume_backfills_inherited_anomaly_without_published_archive( + tmp_path, monkeypatch +): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + _write_jsonl( + prediction_file, + [{"data_abbr": "ds", "id": 2, "uuid": "u2", "prediction": "ok"}], + ) + result_file = tmp_path / "response_anomaly" / "modelA" / "ds.jsonl" + _write_jsonl(result_file, [_completed_anomaly_result(2)]) + source_dir = ( + tmp_path + / "response_anomaly" + / "modelA" + / "payload_staging" + / "ds" + ) + source_writer = ResponseAnomalyJsonlWriter(source_dir, 3, 10) + source_writer.write(_payload_record(2)) + source_writer.close(write_manifest=False) + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + monkeypatch.setattr( + coordinator, + "_detect_case", + lambda *args: pytest.fail("inherited cases must not be detected again"), + ) + + coordinator._detect(_anomaly_cfg(tmp_path)) + + payload_dir = tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + assert [ + record["id"] for record in iter_jsonl_zstd_records(payload_dir) + ] == [2] + + +def test_resume_backfills_only_inherited_payloads_missing_from_archive( + tmp_path, monkeypatch +): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + _write_jsonl( + prediction_file, + [ + {"data_abbr": "ds", "id": 1, "uuid": "u1"}, + {"data_abbr": "ds", "id": 2, "uuid": "u2"}, + ], + ) + result_file = tmp_path / "response_anomaly" / "modelA" / "ds.jsonl" + _write_jsonl( + result_file, + [_completed_anomaly_result(1), _completed_anomaly_result(2)], + ) + payload_dir = tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + archive_writer = ResponseAnomalyJsonlWriter(payload_dir, 3, 10) + archive_writer.write(_payload_record(1)) + archive_writer.close("anomalies") + source_dir = ( + tmp_path + / "response_anomaly" + / "modelA" + / "payload_staging" + / "ds" + ) + source_writer = ResponseAnomalyJsonlWriter(source_dir, 3, 10) + source_writer.write(_payload_record(1)) + source_writer.write(_payload_record(2)) + source_writer.close(write_manifest=False) + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + monkeypatch.setattr( + coordinator, + "_detect_case", + lambda *args: pytest.fail("inherited cases must not be detected again"), + ) + + coordinator._detect(_anomaly_cfg(tmp_path)) + + archived_ids = [ + record["id"] for record in iter_jsonl_zstd_records(payload_dir) + ] + assert sorted(archived_ids) == [1, 2] + assert len(archived_ids) == len(set(archived_ids)) + + +def test_resume_backfills_inherited_legacy_prediction_payload( + tmp_path, monkeypatch +): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + prediction = _payload_record(2) + prediction["prediction"] = "ok" + _write_jsonl(prediction_file, [prediction]) + result_file = tmp_path / "response_anomaly" / "modelA" / "ds.jsonl" + _write_jsonl(result_file, [_completed_anomaly_result(2)]) + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + monkeypatch.setattr( + coordinator, + "_detect_case", + lambda *args: pytest.fail("inherited cases must not be detected again"), + ) + + coordinator._detect(_anomaly_cfg(tmp_path)) + + payload_dir = tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + assert [ + record["id"] for record in iter_jsonl_zstd_records(payload_dir) + ] == [2] + + +def test_resume_preserves_inherited_legacy_payload_in_all_mode( + tmp_path, monkeypatch +): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + prediction = _payload_record(2) + prediction["prediction"] = "ok" + _write_jsonl(prediction_file, [prediction]) + result_file = tmp_path / "response_anomaly" / "modelA" / "ds.jsonl" + _write_jsonl(result_file, [_completed_anomaly_result(2)]) + cfg = _anomaly_cfg(tmp_path) + cfg["response_anomaly"]["payload_retention"] = "all" + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + monkeypatch.setattr( + coordinator, + "_detect_case", + lambda *args: pytest.fail("inherited cases must not be detected again"), + ) + + coordinator._detect(cfg) + + payload_dir = tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + assert [ + record["id"] for record in iter_jsonl_zstd_records(payload_dir) + ] == [2] + + +def test_all_mode_deduplicates_seeded_and_inline_payloads( + tmp_path, monkeypatch +): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + inherited_prediction = _payload_record(1) + inherited_prediction["prediction"] = "old" + _write_jsonl( + prediction_file, + [ + inherited_prediction, + {"data_abbr": "ds", "id": 2, "uuid": "u2", "prediction": "new"}, + ], + ) + result_file = tmp_path / "response_anomaly" / "modelA" / "ds.jsonl" + _write_jsonl(result_file, [_completed_anomaly_result(1)]) + payload_dir = tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + archive_writer = ResponseAnomalyJsonlWriter(payload_dir, 3, 10) + archive_writer.write(_payload_record(1)) + archive_writer.close("all") + source_dir = ( + tmp_path + / "response_anomaly" + / "modelA" + / "payload_staging" + / "ds" + ) + source_writer = ResponseAnomalyJsonlWriter(source_dir, 3, 10) + source_writer.write(_payload_record(2)) + source_writer.close(write_manifest=False) + cfg = _anomaly_cfg(tmp_path) + cfg["response_anomaly"]["payload_retention"] = "all" + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + + coordinator._detect(cfg) + + archived_ids = [ + record["id"] for record in iter_jsonl_zstd_records(payload_dir) + ] + manifest = json.loads( + (payload_dir / "payload_manifest.json").read_text(encoding="utf-8") + ) + assert sorted(archived_ids) == [1, 2] + assert len(archived_ids) == len(set(archived_ids)) + assert manifest["total_rows"] == 2 + + +def test_detection_results_do_not_reference_transient_payload_locations( + tmp_path, monkeypatch +): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + _write_jsonl( + prediction_file, + [{"data_abbr": "ds", "id": 2, "uuid": "u2"}], + ) + source_dir = ( + tmp_path + / "response_anomaly" + / "modelA" + / "payload_staging" + / "ds" + ) + source_writer = ResponseAnomalyJsonlWriter(source_dir, 3, 10) + source_writer.write(_payload_record(2)) + source_writer.close(write_manifest=False) + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + coordinator._detect(_anomaly_cfg(tmp_path)) + + result_file = tmp_path / "response_anomaly" / "modelA" / "ds.jsonl" + result = json.loads(result_file.read_text(encoding="utf-8")) + assert "payload_shard" not in result + assert "payload_row" not in result + + +def test_detect_exposes_anomaly_report_with_locations(tmp_path, monkeypatch): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + _write_jsonl( + prediction_file, + [ + {"data_abbr": "ds", "id": 1, "uuid": "u1", "prediction": "ok"}, + {"data_abbr": "ds", "id": 2, "uuid": "u2", "prediction": "bad"}, + ], + ) + source_dir = ( + tmp_path + / "response_anomaly" + / "modelA" + / "payload_staging" + / "ds" + ) + source_writer = ResponseAnomalyJsonlWriter(source_dir, 3, 10) + source_writer.write(_payload_record(1)) + source_writer.write(_payload_record(2)) + source_writer.close(write_manifest=False) + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + coordinator._detect(_anomaly_cfg(tmp_path)) + + report = coordinator.anomaly_report + assert set(report) == {"ResponseAnomaly/modelA/ds"} + info = report["ResponseAnomaly/modelA/ds"] + assert info["counts"] == {"normal": 1, "rare_character": 1} + assert info["result_file"] == str( + tmp_path / "response_anomaly" / "modelA" / "ds.jsonl" + ) + assert info["payload_dir"] == str( + tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + ) + assert info["task_log"] == str( + tmp_path / "logs" / "response_anomaly" / "modelA" / "ds.out" + ) + + +def test_detect_noop_resume_keeps_payload_archive_unchanged( + tmp_path, monkeypatch +): + import ais_bench.benchmark.utils.response_anomaly as anomaly_module + import ais_bench.benchmark.utils.response_anomaly_jsonl as jsonl_module + + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + prediction_file.parent.mkdir(parents=True) + prediction_file.write_text( + json.dumps( + { + "data_abbr": "ds", + "id": 1, + "uuid": "u1", + "prediction": "ok", + "response_anomaly_payload": { + "tokens": [1], + "topk_logprobs": [{"1": -0.1}], + }, + } + ) + + "\n", + encoding="utf-8", + ) + cfg = { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds"}], + "response_anomaly": {"payload_retention": "all"}, + } + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + coordinator._detect(cfg) + + payload_dir = tmp_path / "response_anomaly" / "modelA" / "payload" / "ds" + archive_before = { + path.name: path.read_bytes() for path in payload_dir.iterdir() + } + + def unexpected_copy(*args, **kwargs): + raise AssertionError("no-op resume must not copy the payload archive") + + def unexpected_decode(path): + raise AssertionError("no-op resume must not decode payload shards") + + monkeypatch.setattr(anomaly_module.shutil, "copytree", unexpected_copy) + monkeypatch.setattr(anomaly_module.shutil, "copy2", unexpected_copy) + monkeypatch.setattr( + jsonl_module, "_iter_jsonl_zstd_shard", unexpected_decode + ) + + resumed = ResponseAnomalyCoordinator() + monkeypatch.setattr( + resumed, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + resumed._detect(cfg) + + assert resumed.summary == {"normal": 1} + assert archive_before == { + path.name: path.read_bytes() for path in payload_dir.iterdir() + } + + +def test_detect_legacy_payload_uses_writer_row_counts(tmp_path, monkeypatch): + import ais_bench.benchmark.utils.response_anomaly_jsonl as jsonl_module + + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + prediction_file.parent.mkdir(parents=True) + prediction_file.write_text( + json.dumps( + { + "data_abbr": "ds", + "id": 1, + "uuid": "u1", + "response_anomaly_payload": { + "tokens": [1], + "topk_logprobs": [{"1": -0.1}], + }, + } + ) + + "\n", + encoding="utf-8", + ) + cfg = { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds"}], + "response_anomaly": {"payload_retention": "all"}, + } + + def unexpected_decode(path): + raise AssertionError("new legacy shards already have writer row counts") + + monkeypatch.setattr( + jsonl_module, "_iter_jsonl_zstd_shard", unexpected_decode + ) + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + + coordinator._detect(cfg) + + manifest = json.loads( + ( + tmp_path + / "response_anomaly" + / "modelA" + / "payload" + / "ds" + / "payload_manifest.json" + ).read_text(encoding="utf-8") + ) + assert manifest["total_rows"] == 1 + + +def test_detect_cleans_payload_build_directory_after_failure( + tmp_path, monkeypatch +): + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + prediction_file.parent.mkdir(parents=True) + prediction_file.write_text( + json.dumps( + { + "data_abbr": "ds", + "id": 2, + "uuid": "u2", + "response_anomaly_payload": { + "tokens": [2], + "topk_logprobs": [{"2": -0.1}], + }, + } + ) + + "\n", + encoding="utf-8", + ) + cfg = { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds"}], + "response_anomaly": {"payload_retention": "anomalies"}, + } + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + original_post_status = coordinator._post_status + + def fail_during_finalization(*args, **kwargs): + if len(args) > 4 and str(args[4]).startswith("finalizing"): + raise RuntimeError("injected finalization failure") + return original_post_status(*args, **kwargs) + + monkeypatch.setattr(coordinator, "_post_status", fail_during_finalization) + + coordinator._detect(cfg) + + payload_parent = tmp_path / "response_anomaly" / "modelA" / "payload" + assert not list(payload_parent.glob(".ds.payload-build-*")) + + +def test_cleanup_stale_payload_build_directories(tmp_path): + payload_dir = tmp_path / "payload" / "ds" + payload_dir.parent.mkdir(parents=True) + stale = payload_dir.parent / ".ds.payload-build-deadbeef" + unrelated = payload_dir.parent / ".other.payload-build-deadbeef" + stale.mkdir() + unrelated.mkdir() + + ResponseAnomalyCoordinator()._cleanup_stale_payload_build_dirs(payload_dir) + + assert not stale.exists() + assert unrelated.exists() + + +def test_cleanup_stale_payload_build_failure_warns_and_continues( + tmp_path, monkeypatch +): + payload_dir = tmp_path / "payload" / "ds" + payload_dir.parent.mkdir(parents=True) + stale = payload_dir.parent / ".ds.payload-build-deadbeef" + stale.mkdir() + warnings = [] + coordinator = ResponseAnomalyCoordinator() + + def fail_rmtree(path, *args, **kwargs): + raise PermissionError("denied") + + monkeypatch.setattr( + "ais_bench.benchmark.utils.response_anomaly.shutil.rmtree", + fail_rmtree, + ) + monkeypatch.setattr( + coordinator.logger, + "warning", + lambda message, *args: warnings.append(message % args), + ) + + coordinator._cleanup_stale_payload_build_dirs(payload_dir) + + assert stale.exists() + assert any(str(stale) in warning for warning in warnings) + + +def test_seed_payload_directory_falls_back_to_copy(tmp_path, monkeypatch): + source_dir = tmp_path / "source" + destination_dir = tmp_path / "destination" + source_dir.mkdir() + shard = source_dir / "part-00000.jsonl.zst" + shard.write_bytes(b"payload") + (source_dir / "payload_manifest.json").write_text( + '{"total_rows": 1}', encoding="utf-8" + ) + + def fail_link(*args): + raise OSError("unsupported") + + monkeypatch.setattr( + "ais_bench.benchmark.utils.response_anomaly.os.link", + fail_link, + ) + + ResponseAnomalyCoordinator._seed_payload_directory( + source_dir, destination_dir, include_manifest=True + ) + + assert (destination_dir / shard.name).read_bytes() == b"payload" + assert json.loads( + (destination_dir / "payload_manifest.json").read_text(encoding="utf-8") + ) == {"total_rows": 1} + + ResponseAnomalyCoordinator._seed_payload_directory( + source_dir, destination_dir + ) + assert not (destination_dir / "payload_manifest.json").exists() + + +def test_detect_warns_when_no_predictions_found(tmp_path, monkeypatch): + """没有任何预测样本时应告警而不是静默完成。""" + warnings = [] + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, + "_build_detector", + lambda cfg: (FakeDetector([False, 0]), None), + ) + monkeypatch.setattr( + coordinator.logger, + "warning", + lambda msg, *args: warnings.append(msg), + ) + cfg = { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds"}], + "response_anomaly": {}, + } + + coordinator._detect(cfg) + + assert coordinator.summary == {} + status = json.loads( + ( + tmp_path / "status_tmp" / ResponseAnomalyCoordinator.STATUS_FILE_NAME + ).read_text(encoding="utf-8") + )[0] + assert status["status"] == "finish" + assert status["total_count"] == 0 + assert any("No predictions" in message for message in warnings) diff --git a/tests/UT/utils/test_response_anomaly_jsonl.py b/tests/UT/utils/test_response_anomaly_jsonl.py new file mode 100644 index 00000000..31957da5 --- /dev/null +++ b/tests/UT/utils/test_response_anomaly_jsonl.py @@ -0,0 +1,91 @@ +import json + +import pytest + +zstandard = pytest.importorskip("zstandard") + +from ais_bench.benchmark.utils.response_anomaly_jsonl import ( + ResponseAnomalyJsonlWriter, + ResponseAnomalyStagingWriter, + build_jsonl_zstd_manifest, + iter_jsonl_zstd_records, +) + + +def _record(case_id): + return { + "data_abbr": "ds", + "id": case_id, + "uuid": f"u{case_id}", + "response_anomaly_payload": { + "tokens": [case_id], + "topk_logprobs": [ + { + str(case_id): -0.123456789012345, + str(case_id + 1): -2.987654321098765, + } + ], + }, + } + + +def _read_shard(path): + with path.open("rb") as file: + reader = zstandard.ZstdDecompressor().stream_reader(file) + return [ + json.loads(line) + for line in reader.read().decode("utf-8").splitlines() + ] + + +def test_jsonl_zstd_writer_round_trips_and_shards(tmp_path): + writer = ResponseAnomalyJsonlWriter(tmp_path, 3, 2) + records = [_record(case_id) for case_id in range(3)] + for record in records: + writer.write(record) + + manifest = writer.close() + + shards = sorted(tmp_path.glob("*.jsonl.zst")) + assert len(shards) == 2 + assert manifest["total_rows"] == 3 + assert [item["rows"] for item in manifest["shards"]] == [2, 1] + assert manifest["shards"][0]["sha256"].startswith("sha256:") + restored = [item for shard in shards for item in _read_shard(shard)] + assert restored == records + assert not list(tmp_path.glob("*.inprogress")) + + +def test_staging_writer_routes_datasets_and_streams_records(tmp_path): + writer = ResponseAnomalyStagingWriter( + { + "work_dir": str(tmp_path), + "model_abbr": "modelA", + "compression_level": 3, + "rows_per_shard": 2, + } + ) + first = _record(1) + second = _record(2) + second["data_abbr"] = "ds2" + writer.write(first) + writer.write(second) + writer.close() + + root = tmp_path / "response_anomaly" / "modelA" / "payload_staging" + assert [item["id"] for item in iter_jsonl_zstd_records(root / "ds")] == [1] + assert [item["id"] for item in iter_jsonl_zstd_records(root / "ds2")] == [2] + + +def test_manifest_uses_streaming_row_counts_without_second_decode(tmp_path): + writer = ResponseAnomalyJsonlWriter(tmp_path, 3, 10) + writer.write(_record(1)) + writer.close(write_manifest=False) + shard = next(tmp_path.glob("*.jsonl.zst")) + + manifest = build_jsonl_zstd_manifest( + tmp_path, 3, "all", {shard.name: 1} + ) + + assert manifest["total_rows"] == 1 + assert manifest["shards"][0]["rows"] == 1 From 800cd1507fb5443358376ec1415fba34364f3b28 Mon Sep 17 00:00:00 2001 From: Hanye <1037452625@qq.com> Date: Wed, 19 Aug 2026 17:27:07 +0800 Subject: [PATCH 04/25] [DOCS] Add docs of terminal-bench 2.1 (#476) * add PR check workflow * add PR check workflow * terminal-bench 2.1 doc * terminal-bench 2.1 doc en * unsupport aarch64 --------- Co-authored-by: SJTUyh --- .../extended_benchmark/agent/harbor_bench.md | 65 ++++++++++++++++-- .../extended_benchmark/agent/harbor_bench.md | 68 +++++++++++++++++-- 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/docs/source_en/extended_benchmark/agent/harbor_bench.md b/docs/source_en/extended_benchmark/agent/harbor_bench.md index d2e95308..bf0e6712 100644 --- a/docs/source_en/extended_benchmark/agent/harbor_bench.md +++ b/docs/source_en/extended_benchmark/agent/harbor_bench.md @@ -53,6 +53,40 @@ Ensure deployment of tested inference services following OpenAI chat/completions ```bash pip install harbor==0.6.1 ``` +3. Edit the Harbor docker compose configuration file. + First, run: + ```bash + pip3 show harbor | grep Loca + ``` + Find the site-packages path and locate `harbor/environments/docker/docker-compose-base.yaml` under that path, for example: + ``` + /usr/local/lib/python3.12/dist-packages/harbor/environments/docker/docker-compose-base.yaml + ``` + Configure the file as follows: + ```yaml + services: + main: + network_mode: host # Share host network, required + environment: # Environment variables applied globally in the container + http_proxy: XXXXXXXX # Configure proxy environment variables if the execution environment has no internet access + https_proxy: XXXXXXXX # Configure proxy environment variables if the execution environment has no internet access + no_proxy: XXXXXXXX + volumes: + - type: bind + source: ${HOST_VERIFIER_LOGS_PATH} + target: ${ENV_VERIFIER_LOGS_PATH} + - type: bind + source: ${HOST_AGENT_LOGS_PATH} + target: ${ENV_AGENT_LOGS_PATH} + - type: bind + source: ${HOST_ARTIFACTS_PATH} + target: ${ENV_ARTIFACTS_PATH} + deploy: + resources: + limits: + cpus: ${CPUS} + memory: ${MEMORY} + ``` > ⚠️ Note: Installing Harbor will upgrade the datasets library to version 4.0.0 or higher, which will cause dependency conflicts for the datasets library after installation. This does not affect tests for Terminal-Bench datasets using Harbor. However, if you need to test other datasets, you will need to downgrade the datasets library. #### 2.2 Install Inside a Docker Container @@ -68,6 +102,10 @@ services: network_mode: host # Share host network, required security_opt: # Required when starting the container with Mode B (Socket Passthrough) - seccomp=unconfined + environment: # Environment variables applied globally in the container + http_proxy: XXXXXXXX # Configure proxy environment variables if the execution environment has no internet access + https_proxy: XXXXXXXX # Configure proxy environment variables if the execution environment has no internet access + no_proxy: XXXXXXXX volumes: - type: bind source: ${HOST_VERIFIER_LOGS_PATH} @@ -87,11 +125,13 @@ services: > ⚠️ Note: Installing Harbor will upgrade the datasets library to version 4.0.0 or higher, which will cause dependency conflicts for the datasets library after installation. This does not affect tests for Terminal-Bench datasets using Harbor. However, if you need to test other datasets, you will need to downgrade the datasets library. -### 3. Prepare AISBench-modified Terminal-Bench-2 Dataset and Images +### 3. Prepare AISBench-modified Terminal-Bench Dataset and Images + +#### 3.1 terminal-bench 2 AISBench modified dataset repository: [https://github.com/AISBench/terminal-bench-2](https://github.com/AISBench/terminal-bench-2) -> Note: AISBench only centralized all environment preparation into the Dockerfile without changing the case content, avoiding repeated environment building and dependency installation. +> Note: AISBench did not change the case content. It only centralized all environment preparation (including all dependencies of the terminus-2 agent and verification resources) into the Dockerfile, avoiding repeated environment building and dependency installation on every run. Terminal-Bench-2 pre-packaged images: | Image Name | Download Link | CPU Architecture | Compressed Size | @@ -101,10 +141,23 @@ Terminal-Bench-2 pre-packaged images: > Tip: If you don't want to prepare images for all cases, you can get the terminal-bench-2-offline-mini sampled dataset from [terminal-bench-2-offline-mini](https://modelers.cn/datasets/AISBench/terminal-bench-2-offline-mini). +#### 3.2 terminal-bench 2.1 + +> terminal-bench 2.1 is essentially a bug-fix release of terminal-bench-2.0. The number of cases and case names are completely identical; only the dataset content and image content of certain cases differ. + +AISBench modified dataset repository: [https://github.com/AISBench/terminal-bench-2.1](https://github.com/AISBench/terminal-bench-2.1) + +> Note: AISBench did not modify the content of the official original images of terminal-bench-2.1; only the image tag names have been changed to distinguish them from terminal-bench-2.0. The image names in task.toml within the dataset are also updated accordingly. + +| Image Name | Download Link | CPU Architecture | Compressed Size | +| --------- | ------------ | ---------------- | -------------- | +| `terminal-bench-2.1-images-aarch64.tar` | Not Supported | aarch64 | NA | +| `terminal-bench-2.1-images-x86_64.tar` | [Link](https://aisbench.obs.cn-north-4.myhuaweicloud.com/terminal-bench-2-images/terminal-bench-2.1-images-x86_64.tar) | x86_64 | 38.62 GB | + > ⚠️ Note: -> If you installed AISBench & Harbor dependencies from source, deploy the Terminal-Bench-2 images on the **host machine** by running `docker load -i xxxxxxx.tar`. -> If you started the AISBench container using Mode A (true Docker-in-Docker), deploy the Terminal-Bench-2 images **inside the container** by running `docker load -i xxxxxxx.tar`. -> If you started the AISBench container using Mode B (Socket Passthrough), deploy the Terminal-Bench-2 images on the **host machine** by running `docker load -i xxxxxxx.tar`. +> If you installed AISBench & Harbor dependencies from source, deploy the Terminal-Bench-2/2.1 images on the **host machine** by running `docker load -i xxxxxxx.tar`. +> If you started the AISBench container using Mode A (true Docker-in-Docker), deploy the Terminal-Bench-2/2.1 images **inside the container** by running `docker load -i xxxxxxx.tar`. +> If you started the AISBench container using Mode B (Socket Passthrough), deploy the Terminal-Bench-2/2.1 images on the **host machine** by running `docker load -i xxxxxxx.tar`. ### 4. Configure Custom Configuration File for Harbor Tasks @@ -145,7 +198,7 @@ datasets.append( # ...... n_concurrent_trials=5, # -n/--n-concurrent: Number of concurrent trials # ...... - path="/path/to/terminal-bench-2/", # -p/--path: Local dataset path + path="/path/to/terminal-bench-2/", # -p/--path: Local dataset path, the path to either the terminal-bench-2 or terminal-bench 2.1 dataset # ...... n_tasks=None, # --n-tasks: Maximum number of tasks, None runs all, try setting a few for quick testing # ...... diff --git a/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md b/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md index 113d55f9..85bc1a04 100644 --- a/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md +++ b/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md @@ -52,6 +52,44 @@ ```bash pip install harbor==0.6.1 ``` +3. 编辑harbor中的docker compose配置文件 +首先执行： +```bash +pip3 show harbor | grep Loca +``` +找到site-package路径，在这个路径下找到`harbor/environments/docker/docker-compose-base.yaml`, 例如 + +``` +/usr/local/lib/python3.12/dist-packages/harbor/environments/docker/docker-compose-base.yaml +``` + +在这个文件中配置： + +```yaml +services: + main: + network_mode: host # 共享主机网络，必须配置 + environment: # 在容器中普遍生效的环境变量 + http_proxy: XXXXXXXX # 如果执行环境无法访问互联网，需要配置代理环境变量 + https_proxy: XXXXXXXX # 如果执行环境无法访问互联网，需要配置代理环境变量 + no_proxy: XXXXXXXX + volumes: + - type: bind + source: ${HOST_VERIFIER_LOGS_PATH} + target: ${ENV_VERIFIER_LOGS_PATH} + - type: bind + source: ${HOST_AGENT_LOGS_PATH} + target: ${ENV_AGENT_LOGS_PATH} + - type: bind + source: ${HOST_ARTIFACTS_PATH} + target: ${ENV_ARTIFACTS_PATH} + deploy: + resources: + limits: + cpus: ${CPUS} + memory: ${MEMORY} +``` + > ⚠️注意：安装harbor会将datasets库的版本升级到4.0.0以上的版本，这会导致安装后报datasets库的依赖冲突，对于执行harbor测试terminal-bench相关数据集没有影响，但是如果你需要测试其他数据集，需要降低datasets库的版本。 #### 2.2 在docker容器中安装 @@ -67,6 +105,10 @@ services: network_mode: host # 共享主机网络，必须配置 security_opt: # 模式 B 启动的容器需要配置 - seccomp=unconfined + environment: # 在容器中普遍生效的环境变量 + http_proxy: XXXXXXXX # 如果执行环境无法访问互联网，需要配置代理环境变量 + https_proxy: XXXXXXXX # 如果执行环境无法访问互联网，需要配置代理环境变量 + no_proxy: XXXXXXXX volumes: - type: bind source: ${HOST_VERIFIER_LOGS_PATH} @@ -86,9 +128,10 @@ services: > ⚠️注意：安装harbor会将datasets库的版本升级到4.0.0以上的版本，这会导致安装后报datasets库的依赖冲突，对于执行harbor测试terminal-bench相关数据集没有影响，但是如果你需要测试其他数据集，需要降低datasets库的版本。 -### 3. 准备AISBench修改过的Terminal-Bench-2数据集和对应镜像 +### 3. 准备AISBench修改过的Terminal-Bench数据集和对应镜像 +#### 3.1 terminal-bench 2 AISBench修改的数据集获取链接：https://github.com/AISBench/terminal-bench-2 -> 👉注意: AISBench没有改用例内容，只是将所有环境的准备全部集中到Dockerfile中，避免反复执行还需要反复构建环境和安装依赖 +> 👉注意: AISBench没有改用例内容，只是将所有环境的准备（包括terminus 2这个agent的所有依赖以及验证资源的）全部集中到Dockerfile中，避免反复执行还需要反复构建环境和安装依赖 Terminal-Bench-2 预制打包镜像信息： | 镜像名称 | 获取链接 |cpu架构| 打包压缩包大小 | @@ -98,10 +141,23 @@ Terminal-Bench-2 预制打包镜像信息： > 🌟提示：如果不想准备所有case的镜像，可以从[terminal-bench-2-offline-mini](https://modelers.cn/datasets/AISBench/terminal-bench-2-offline-mini)获取基于terminal-bench-2.0小规模采样的数据集及对应打包镜像 +#### 3.2 terminal-bench 2.1 +> terminal-bench 2.1简单来说就是terminal-bench-2.0的一个bugfix版本，用例数量和用例名称是完全一致的，只有部分case的数据集内容和镜像内容有所差别。 + +AISBench修改的数据集获取链接：https://github.com/AISBench/terminal-bench-2.1 + +> 👉注意：AISBench没有修改terminal-bench-2.1的官方原始镜像的内容，仅修改了镜像的tag名称便于和terminal-bench-2.0进行区分，数据集中的task.toml中镜像名称也做了同步修改。 + +| 镜像名称 | 获取链接 |cpu架构| 打包压缩包大小 | +| -------- | -------- | ------- |-------- | +|`terminal-bench-2.1-images-aarch64.tar`| 暂不支持 | aarch64 | NA | +|`terminal-bench-2.1-images-x86_64.tar`| https://aisbench.obs.cn-north-4.myhuaweicloud.com/terminal-bench-2-images/terminal-bench-2.1-images-x86_64.tar | x86_64 | 38.62GB | + + > ⚠️注意： -> 如果通过源码安装 AISBench 测评工具 & Harbor 依赖这种方式安装依赖的情况下，部署Terminal-Bench-2的镜像需要在**物理机**上执行`docker load -i xxxxxxx.tar` -> 如果通过模式 A（真 docker in docker）启动AISBench容器，部署Terminal-Bench-2的镜像需要在**容器内**上执行`docker load -i xxxxxxx.tar` -> 如果提供给模式 B（Socket 代理）启动AISBench容器，部署Terminal-Bench-2的镜像需要在**物理机**上执行`docker load -i xxxxxxx.tar` +> 如果通过源码安装 AISBench 测评工具 & Harbor 依赖这种方式安装依赖的情况下，部署Terminal-Bench-2/2.1的镜像需要在**物理机**上执行`docker load -i xxxxxxx.tar` +> 如果通过模式 A（真 docker in docker）启动AISBench容器，部署Terminal-Bench-2/2.1的镜像需要在**容器内**上执行`docker load -i xxxxxxx.tar` +> 如果提供给模式 B（Socket 代理）启动AISBench容器，部署Terminal-Bench-2/2.1的镜像需要在**物理机**上执行`docker load -i xxxxxxx.tar` ### 4. 配置 Harbor 任务的自定义配置文件 @@ -143,7 +199,7 @@ for task in sub_tasks: # ...... n_concurrent_trials=5, # -n/--n-concurrent: 并发运行的trial数量 # ...... - path="/path/to/terminal-bench-2/", # -p/--path: 本地数据集路径 + path="/path/to/terminal-bench-2/", # -p/--path: 本地数据集路径，terminal-bench-2或者terminal-bench 2.1数据集的路径 # ...... n_tasks=None, # --n-tasks: 最大任务数量, None默认跑全部，快速入门可以尝试设置几条快速跑通流程 # ...... From 29c363e38e9d1560e6f19eff582f6117943b6a77 Mon Sep 17 00:00:00 2001 From: flame-hu <109938139+flame-hu@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:49:16 +0800 Subject: [PATCH 05/25] [Feature] Added herding_coreset_selector, a tool for compressing large model quantization evaluation sets. (#474) * [Feature] Added herding_coreset_selector, a tool for compressing large model quantization evaluation sets. * [Feature] Added herding_coreset_selector, a tool for compressing large model quantization evaluation sets. * [Feature] Added herding_coreset_selector, a tool for compressing large model quantization evaluation sets. * [Feature] Added herding_coreset_selector, a tool for compressing large model quantization evaluation sets. --------- Co-authored-by: qq_53076697 --- .../herding_coreset_selector/conftest.py | 12 ++ .../test_algorithm.py | 49 +++++ .../herding_coreset_selector/test_cli.py | 96 +++++++++ .../test_eval_datasets.py | 91 +++++++++ .../herding_coreset_selector/test_features.py | 53 +++++ tools/herding_coreset_selector/README.md | 185 ++++++++++++++++++ .../herding/__init__.py | 27 +++ .../herding/__main__.py | 115 +++++++++++ .../herding/algorithm.py | 67 +++++++ .../herding/eval_datasets/__init__.py | 17 ++ .../herding/eval_datasets/aime2025.py | 49 +++++ .../herding/eval_datasets/dataset_base.py | 63 ++++++ .../herding/eval_datasets/gpqa.py | 75 +++++++ .../herding/features.py | 27 +++ 14 files changed, 926 insertions(+) create mode 100644 tests/UT/tools/herding_coreset_selector/conftest.py create mode 100644 tests/UT/tools/herding_coreset_selector/test_algorithm.py create mode 100644 tests/UT/tools/herding_coreset_selector/test_cli.py create mode 100644 tests/UT/tools/herding_coreset_selector/test_eval_datasets.py create mode 100644 tests/UT/tools/herding_coreset_selector/test_features.py create mode 100644 tools/herding_coreset_selector/README.md create mode 100644 tools/herding_coreset_selector/herding/__init__.py create mode 100644 tools/herding_coreset_selector/herding/__main__.py create mode 100644 tools/herding_coreset_selector/herding/algorithm.py create mode 100644 tools/herding_coreset_selector/herding/eval_datasets/__init__.py create mode 100644 tools/herding_coreset_selector/herding/eval_datasets/aime2025.py create mode 100644 tools/herding_coreset_selector/herding/eval_datasets/dataset_base.py create mode 100644 tools/herding_coreset_selector/herding/eval_datasets/gpqa.py create mode 100644 tools/herding_coreset_selector/herding/features.py diff --git a/tests/UT/tools/herding_coreset_selector/conftest.py b/tests/UT/tools/herding_coreset_selector/conftest.py new file mode 100644 index 00000000..e88e1f52 --- /dev/null +++ b/tests/UT/tools/herding_coreset_selector/conftest.py @@ -0,0 +1,12 @@ +"""Test configuration for the standalone herding coreset selector.""" + +import sys +from pathlib import Path + + +TOOL_ROOT = ( + Path(__file__).resolve().parents[4] / "tools" / "herding_coreset_selector" +) + +if str(TOOL_ROOT) not in sys.path: + sys.path.insert(0, str(TOOL_ROOT)) diff --git a/tests/UT/tools/herding_coreset_selector/test_algorithm.py b/tests/UT/tools/herding_coreset_selector/test_algorithm.py new file mode 100644 index 00000000..b03ad84a --- /dev/null +++ b/tests/UT/tools/herding_coreset_selector/test_algorithm.py @@ -0,0 +1,49 @@ +import numpy as np +import torch + +from herding import algorithm + + +def test_kernel_helpers(): + data = np.array([[0.0], [2.0], [4.0]]) + kernel = algorithm._rbf_kernel(data[:2], data[:2], length_scale=2.0) + + np.testing.assert_allclose( + kernel, + [[1.0, np.exp(-0.5)], [np.exp(-0.5), 1.0]], + ) + assert algorithm._median_heuristic(data) == 2.0 + + +def test_features_to_coreset_matrix(): + features = (torch.tensor(row) for row in ([1.0, 2.0], [3.0, 4.0])) + np.testing.assert_array_equal( + algorithm.features_to_coreset_matrix(features), + np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32), + ) + + +def test_coreset_indices_boundary_and_determinism(): + data = np.array([[0.0], [0.5], [2.0], [8.0], [9.0]]) + + assert algorithm.coreset_indices(data, 5) == list(range(5)) + first = algorithm.coreset_indices(data, 3) + assert first == algorithm.coreset_indices(data, 3) + assert len(first) == len(set(first)) == 3 + + +def test_coreset_indices_fallback_and_kernel_batches(monkeypatch): + data = np.ones((5, 2)) + calls = [] + original_kernel = algorithm._rbf_kernel + + def record(left, right, length_scale): + calls.append(left.shape[0]) + return original_kernel(left, right, length_scale) + + monkeypatch.setattr(algorithm, "KERNEL_BATCH", 2) + monkeypatch.setattr(algorithm, "_median_heuristic", lambda _: 0.0) + monkeypatch.setattr(algorithm, "_rbf_kernel", record) + + assert algorithm.coreset_indices(data, 2) == [0, 1] + assert calls[:3] == [2, 2, 1] diff --git a/tests/UT/tools/herding_coreset_selector/test_cli.py b/tests/UT/tools/herding_coreset_selector/test_cli.py new file mode 100644 index 00000000..0449cac3 --- /dev/null +++ b/tests/UT/tools/herding_coreset_selector/test_cli.py @@ -0,0 +1,96 @@ +import argparse +import sys +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest + +import herding +from herding import __main__ as cli +from herding import algorithm, eval_datasets, features + + +@pytest.mark.parametrize("value", ["0.1", "1", "1.0"]) +def test_coreset_ratio_accepts_valid_values(value): + assert cli._coreset_ratio(value) == float(value) + + +@pytest.mark.parametrize("value", ["0", "-0.1", "1.01", "bad"]) +def test_coreset_ratio_rejects_invalid_values(value): + with pytest.raises(argparse.ArgumentTypeError): + cli._coreset_ratio(value) + + +def test_model_name_and_parse_defaults(monkeypatch): + assert cli._model_name("/models/example/") == "example" + with pytest.raises(ValueError, match="Unable to infer"): + cli._model_name("/") + + monkeypatch.setattr( + sys, + "argv", + [ + "herding", + "--eval-dataset", "aime2025", + "--dataset-path", "/datasets/aime2025", + "--model-path", "/models/example", + ], + ) + args = cli.parse_args() + assert args.coreset_ratio == cli.DEFAULT_CORESET_RATIO + assert args.output_dir == cli.DEFAULT_OUTPUT_DIR + + +def test_main_generates_and_saves_coreset(tmp_path, monkeypatch, capsys): + args = SimpleNamespace( + eval_dataset="gpqa", + dataset_path="/datasets/gpqa", + model_path="/models/example", + coreset_ratio=0.4, + output_dir=str(tmp_path), + ) + dataset = Mock() + dataset.load_indices.return_value = [0, 1, 2, 3, 4] + dataset.save_data_by_indices.side_effect = ["/saved/origin", "/saved/coreset"] + get_dataset = Mock(return_value=dataset) + generate = Mock(return_value=[3, 1]) + monkeypatch.setattr(cli, "parse_args", lambda: args) + monkeypatch.setattr(eval_datasets, "get_eval_dataset", get_dataset) + monkeypatch.setattr(herding, "generate_coreset", generate) + + cli.main() + + get_dataset.assert_called_once_with( + "gpqa", + dataset_path="/datasets/gpqa", + output_dir=str(tmp_path / "gpqa" / "herding" / "example"), + ) + assert dataset.save_data_by_indices.call_args_list[0].args == ( + [0, 1, 2, 3, 4], "origin" + ) + generate.assert_called_once_with(2, eval_dataset=dataset, model_path="/models/example") + assert dataset.save_data_by_indices.call_args_list[1].args == ([3, 1], "coreset") + assert "output: /saved/coreset" in capsys.readouterr().out + + +def test_generate_coreset_pipeline(monkeypatch): + dataset = Mock() + dataset.dataset_prompts.return_value = iter(["one", "two"]) + dataset.dataset_size.return_value = 2 + load = Mock(return_value=("model", "tokenizer")) + generate = Mock(return_value=iter(["feature-1", "feature-2"])) + matrix = np.array([[1.0], [2.0]]) + to_matrix = Mock(return_value=matrix) + select = Mock(return_value=[1]) + monkeypatch.setattr(features, "load_model", load) + monkeypatch.setattr(features, "generate_logits", generate) + monkeypatch.setattr(algorithm, "features_to_coreset_matrix", to_matrix) + monkeypatch.setattr(algorithm, "coreset_indices", select) + + assert herding.generate_coreset(1, dataset, "/model") == [1] + load.assert_called_once_with("/model") + generate.assert_called_once() + to_matrix.assert_called_once() + np.testing.assert_array_equal(select.call_args.args[0], matrix) + assert select.call_args.args[1] == 1 diff --git a/tests/UT/tools/herding_coreset_selector/test_eval_datasets.py b/tests/UT/tools/herding_coreset_selector/test_eval_datasets.py new file mode 100644 index 00000000..794457f7 --- /dev/null +++ b/tests/UT/tools/herding_coreset_selector/test_eval_datasets.py @@ -0,0 +1,91 @@ +import csv +import json +from types import SimpleNamespace + +import pytest + +from herding.eval_datasets import dataset_base +from herding.eval_datasets.aime2025 import Aime2025Dataset +from herding.eval_datasets.gpqa import FILENAME as GPQA_FILENAME +from herding.eval_datasets.gpqa import GpqaDataset + + +class DummyDataset(dataset_base.EvalDatasetBase): + def dataset_size(self): + return 3 + + def dataset_prompts(self): + return iter(()) + + def save_data_by_indices(self, indices, outpath): + return indices, outpath + + +def test_base_indices_and_registry(tmp_path): + dataset = DummyDataset(tmp_path, tmp_path / "output") + strategy_dir = tmp_path / "output" / "strategy" + strategy_dir.mkdir(parents=True) + dataset.save_indices([2, 0], strategy_dir) + + assert dataset.load_indices() == [0, 1, 2] + assert dataset.load_indices_from_strategy("strategy") == [2, 0] + assert dataset.load_indices_from_strategy("missing") is None + + name = "unit_test_dataset" + dataset_base.reg_eval_dataset(name)(DummyDataset) + try: + assert isinstance( + dataset_base.get_eval_dataset(name, tmp_path, tmp_path / "out"), + DummyDataset, + ) + with pytest.raises(ValueError, match="Unknown dataset"): + dataset_base.get_eval_dataset("unknown", tmp_path, tmp_path) + finally: + dataset_base.EVAL_DATASETS.pop(name) + + +def test_aime2025_load_prompt_and_save(tmp_path): + source = tmp_path / "source" + source.mkdir() + rows = [ + {"question": "What is 1 + 1?", "answer": "2"}, + {"question": "What is 2 + 2?", "answer": "4"}, + ] + (source / "aime2025.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n\n", + encoding="utf-8", + ) + dataset = Aime2025Dataset(source, tmp_path / "result") + + assert dataset.dataset_size() == 2 + assert "What is 1 + 1?" in next(dataset.dataset_prompts()) + output = dataset.save_data_by_indices([1], "coreset") + saved = (tmp_path / "result" / "coreset" / "aime2025.jsonl").read_text() + assert json.loads(saved) == rows[1] + assert json.loads((tmp_path / "result" / "coreset" / "indices.json").read_text()) == [1] + assert output == str(tmp_path / "result" / "coreset") + + +def test_gpqa_prompt_and_save(tmp_path, monkeypatch): + source = tmp_path / "source" + source.mkdir() + rows = [ + ["question", "A", "B", "C", "D", "answer"], + ["q1", "a1", "b1", "c1", "d1", "A"], + ["q2", "a2", "b2", "c2", "d2", "B"], + ] + with (source / GPQA_FILENAME).open("w", newline="") as output: + csv.writer(output).writerows(rows) + items = [dict(zip(rows[0][:-1], row[:-1])) for row in rows[1:]] + monkeypatch.setattr( + "herding.eval_datasets.gpqa.build_dataset_from_cfg", + lambda _cfg: SimpleNamespace(test=items), + ) + dataset = GpqaDataset(source, tmp_path / "result") + + assert "A) a1" in next(dataset.dataset_prompts()) + dataset.save_data_by_indices([1, 0], "coreset") + with (tmp_path / "result" / "coreset" / GPQA_FILENAME).open() as saved: + assert list(csv.reader(saved)) == [rows[0], rows[2], rows[1]] + indices = tmp_path / "result" / "coreset" / "indices.json" + assert json.loads(indices.read_text()) == [1, 0] diff --git a/tests/UT/tools/herding_coreset_selector/test_features.py b/tests/UT/tools/herding_coreset_selector/test_features.py new file mode 100644 index 00000000..a07a88fe --- /dev/null +++ b/tests/UT/tools/herding_coreset_selector/test_features.py @@ -0,0 +1,53 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import torch + +from herding import features + + +def test_load_model(monkeypatch): + tokenizer = object() + model = Mock() + tokenizer_loader = Mock(return_value=tokenizer) + model_loader = Mock(return_value=model) + monkeypatch.setattr(features.AutoTokenizer, "from_pretrained", tokenizer_loader) + monkeypatch.setattr(features.AutoModelForCausalLM, "from_pretrained", model_loader) + + assert features.load_model("/models/example") == (model, tokenizer) + tokenizer_loader.assert_called_once_with("/models/example") + model_loader.assert_called_once_with("/models/example", device_map="auto") + model.eval.assert_called_once_with() + + +def test_generate_logits_extracts_hidden_state(): + class Inputs: + input_ids = torch.tensor([[1, 2]]) + + def to(self, device): + self.device = device + return self + + inputs = Inputs() + tokenizer = Mock(return_value=inputs) + hidden = torch.tensor([[[1.0, 2.0], [3.0, 4.0]]]) + model = Mock( + device=torch.device("cpu"), + config=SimpleNamespace(num_hidden_layers=1), + ) + model.generate.return_value = SimpleNamespace( + hidden_states=[None, [torch.zeros_like(hidden), hidden]] + ) + + result = list(features.generate_logits(model, tokenizer, ["prompt"])) + + tokenizer.assert_called_once_with("prompt", return_tensors="pt") + model.generate.assert_called_once_with( + inputs.input_ids, + max_new_tokens=2, + do_sample=False, + output_hidden_states=True, + return_dict_in_generate=True, + ) + assert inputs.device == model.device + torch.testing.assert_close(result[0], torch.tensor([3.0, 4.0])) diff --git a/tools/herding_coreset_selector/README.md b/tools/herding_coreset_selector/README.md new file mode 100644 index 00000000..8a8a9c03 --- /dev/null +++ b/tools/herding_coreset_selector/README.md @@ -0,0 +1,185 @@ +# Herding Coreset Selector + +## 简介 + +Herding Coreset Selector 是一个用于评测数据集代表性样本筛选的独立 Coreset 工具。 + +工具使用指定语言模型提取样本 Prompt 的隐藏状态特征，并基于 RBF Kernel 的 Kernel Herding 方法，从完整数据集中选择指定比例的代表性样本。生成结果保持原数据格式，同时保存样本在完整数据集中的索引，便于结果复现和追溯。 + +## 1. 环境准备 + +安装运行依赖： + +```shell +pip install numpy torch transformers tqdm +``` + +数据集适配器会复用 AISBench 中的数据集或 Prompt 组件，因此需要保证当前环境可以导入 `ais_bench`。在 Benchmark 仓库中执行： + +```shell +cd benchmark +pip install -e . +``` + +## 2. 命令行帮助 + +进入工具目录： + +```shell +cd benchmark/tools/herding_coreset_selector +``` + +查看完整参数说明： + +```shell +python -m herding --help +``` + +主要参数： + +| 参数 | 是否必选 | 说明 | +| --- | --- | --- | +| `--eval-dataset` | 是 | 数据集适配器名称，当前支持 `gpqa`、`aime2025` | +| `--dataset-path` | 是 | 原始评测数据所在目录 | +| `--model-path` | 是 | 用于提取隐藏状态特征的本地 Hugging Face 模型路径 | +| `--coreset-ratio` | 否 | Coreset 占完整数据集的比例，范围 `(0, 1]`，默认 `0.2` | +| `--output-dir` | 否 | 结果输出根目录，默认 `./datasets` | + +工具运行时配置通过命令行显式传入，不依赖环境变量。 + +## 3. 运行 Coreset 压缩 + +通用命令： + +```shell +python -m herding \ + --eval-dataset \ + --dataset-path /path/to/datasets/ \ + --model-path /path/to/model \ + --coreset-ratio 0.2 +``` + +工具会依次完成： + +```text +读取数据 + ↓ +构造 Prompt + ↓ +特征模型提取隐藏状态 + ↓ +计算 RBF Kernel + ↓ +Kernel Herding 选择样本 + ↓ +保存 origin 与 coreset +``` + +## 4. 输出目录 + +默认输出结构为： + +```text +datasets/ +└── / + └── herding/ + └── / + ├── origin/ + │ ├── + │ └── indices.json + └── coreset/ + ├── + └── indices.json +``` + +其中 `` 会从 `--model-path` 的最后一级目录名自动获取。 + +- `origin/`：本次筛选对应的完整数据； +- `origin/indices.json`：完整数据对应的原始索引； +- `coreset/`：筛选后的 Coreset； +- `coreset/indices.json`：Coreset 样本在完整数据中的原始索引。 + +## 5. GPQA 压缩示例 + +准备数据： + +```text +benchmark/ais_bench/datasets/gpqa/gpqa_diamond.csv +``` + +执行： + +```shell +cd benchmark/tools/herding_coreset_selector + +python -m herding \ + --eval-dataset gpqa \ + --dataset-path ../../ais_bench/datasets/gpqa \ + --model-path /path/to/Qwen2.5-7B-Instruct \ + --coreset-ratio 0.2 +``` + +如果模型目录名为 `Qwen2.5-7B-Instruct`，则结果写入： + +```text +datasets/gpqa/herding/Qwen2.5-7B-Instruct/ +├── origin/ +│ ├── gpqa_diamond.csv +│ └── indices.json +└── coreset/ + ├── gpqa_diamond.csv + └── indices.json +``` + +例如只保留约 10% 的样本： + +```shell +python -m herding \ + --eval-dataset gpqa \ + --dataset-path ../../ais_bench/datasets/gpqa \ + --model-path /path/to/Qwen2.5-7B-Instruct \ + --coreset-ratio 0.1 +``` + +## 6. AIME2025 压缩示例 + +准备数据： + +```text +benchmark/ais_bench/datasets/aime2025/aime2025.jsonl +``` + +执行： + +```shell +python -m herding \ + --eval-dataset aime2025 \ + --dataset-path ../../ais_bench/datasets/aime2025 \ + --model-path /path/to/Qwen2.5-7B-Instruct \ + --coreset-ratio 0.2 +``` + +## 7. 接入新的数据集 + +在 `herding/eval_datasets/` 中新增适配器，并继承 `EvalDatasetBase`： + +```python +@reg_eval_dataset("my_dataset") +class MyDataset(EvalDatasetBase): + def __init__(self, dataset_path, output_dir): + super().__init__(dataset_path, output_dir) + ... + + def dataset_size(self): + ... + + def dataset_prompts(self): + ... + + def save_data_by_indices(self, indices, outpath): + ... +``` + +然后在 `herding/eval_datasets/__init__.py` 中导入该适配器模块以完成注册，并在 `herding/__main__.py` 的 `SUPPORTED_DATASETS` 中增加对应名称。 + +建议 `save_data_by_indices()` 保持原始数据格式不变，以便压缩结果可以直接用于后续 Benchmark 测评。 diff --git a/tools/herding_coreset_selector/herding/__init__.py b/tools/herding_coreset_selector/herding/__init__.py new file mode 100644 index 00000000..b8a85660 --- /dev/null +++ b/tools/herding_coreset_selector/herding/__init__.py @@ -0,0 +1,27 @@ +import time + + +def generate_coreset(coreset_size, eval_dataset, model_path): + """Generate coreset indices for an initialized evaluation dataset.""" + # Keep package import lightweight so `python -m herding --help` does not + # import torch/transformers/numpy before command-line arguments are parsed. + from tqdm import tqdm + + from .algorithm import coreset_indices, features_to_coreset_matrix + from .features import generate_logits, load_model + + model, tokenizer = load_model(model_path) + + prompts_generator = eval_dataset.dataset_prompts() + logits_generator = generate_logits(model, tokenizer, prompts_generator) + logits_generator = tqdm( + logits_generator, + total=eval_dataset.dataset_size(), + desc="features", + ) + logits_matrix = features_to_coreset_matrix(logits_generator) + + start = time.perf_counter() + indices = coreset_indices(logits_matrix, coreset_size) + print(f" herding: {time.perf_counter() - start:.2f}s") + return indices diff --git a/tools/herding_coreset_selector/herding/__main__.py b/tools/herding_coreset_selector/herding/__main__.py new file mode 100644 index 00000000..890a7ffd --- /dev/null +++ b/tools/herding_coreset_selector/herding/__main__.py @@ -0,0 +1,115 @@ +import argparse +from pathlib import Path + + +DEFAULT_CORESET_RATIO = 0.2 +DEFAULT_OUTPUT_DIR = "./datasets" +CORESET_METHOD = "herding" + + +def _coreset_ratio(value: str) -> float: + """Validate coreset ratio passed from the command line.""" + try: + ratio = float(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("must be a floating-point number") from exc + + if not 0 < ratio <= 1: + raise argparse.ArgumentTypeError("must be in the range (0, 1]") + return ratio + + +def parse_args(): + parser = argparse.ArgumentParser( + description=( + "Select representative evaluation samples with RBF Kernel Herding " + "using hidden-state features extracted from a Hugging Face model." + ) + ) + parser.add_argument( + "--eval-dataset", + required=True, + + help="Evaluation dataset adapter to use.", + ) + parser.add_argument( + "--dataset-path", + required=True, + help=( + "Directory containing the source evaluation dataset " + "(for example, gpqa_diamond.csv for GPQA)." + ), + ) + parser.add_argument( + "--model-path", + required=True, + help="Local Hugging Face model path used to extract hidden-state features.", + ) + parser.add_argument( + "--coreset-ratio", + type=_coreset_ratio, + default=DEFAULT_CORESET_RATIO, + help=( + "Fraction of the original dataset selected for the coreset, in (0, 1]. " + f"Default: {DEFAULT_CORESET_RATIO}." + ), + ) + parser.add_argument( + "--output-dir", + default=DEFAULT_OUTPUT_DIR, + help=( + "Root directory for generated results. The final path is " + "//herding//. " + f"Default: {DEFAULT_OUTPUT_DIR}." + ), + ) + return parser.parse_args() + + +def _model_name(model_path: str) -> str: + normalized = model_path.rstrip("/\\") + name = Path(normalized).name + if not name: + raise ValueError(f"Unable to infer model name from model path: {model_path!r}") + return name + + +def main(): + args = parse_args() + + # Heavy dependencies are imported only after CLI parsing, so + # `python -m herding --help` remains a lightweight, self-contained entry. + from herding import generate_coreset + from herding.eval_datasets import get_eval_dataset + + output_dir = ( + Path(args.output_dir) + / args.eval_dataset + / CORESET_METHOD + / _model_name(args.model_path) + ) + + eval_dataset = get_eval_dataset( + args.eval_dataset, + dataset_path=args.dataset_path, + output_dir=str(output_dir), + ) + + indices = eval_dataset.load_indices() + if not indices: + raise ValueError("The evaluation dataset is empty; cannot generate a coreset.") + + eval_dataset.save_data_by_indices(indices, "origin") + coreset_size = max(1, round(len(indices) * args.coreset_ratio)) + + selected_indices = generate_coreset( + coreset_size, + eval_dataset=eval_dataset, + model_path=args.model_path, + ) + output_path = eval_dataset.save_data_by_indices(selected_indices, "coreset") + print(f"output: {output_path}") + + +if __name__ == "__main__": + main() diff --git a/tools/herding_coreset_selector/herding/algorithm.py b/tools/herding_coreset_selector/herding/algorithm.py new file mode 100644 index 00000000..ce6b7c27 --- /dev/null +++ b/tools/herding_coreset_selector/herding/algorithm.py @@ -0,0 +1,67 @@ +"""Kernel Herding: greedy MMD minimization, RBF kernel.""" + +import numpy as np +import torch +from typing import Iterator + + +BANDWIDTH_SAMPLE_CAP = 1000 +KERNEL_BATCH = 512 +DEFAULT_BANDWIDTH = 1.0 + + +def _rbf_kernel(X, Y, length_scale): + X_sq = np.sum(X ** 2, axis=1, keepdims=True) + Y_sq = np.sum(Y ** 2, axis=1, keepdims=True) + dist_sq = X_sq + Y_sq.T - 2.0 * (X @ Y.T) + return np.exp(-dist_sq / (2.0 * length_scale ** 2)) + + +def _median_heuristic(data): + sq = np.sum(data ** 2, axis=1, keepdims=True) + dist_sq = sq + sq.T - 2.0 * (data @ data.T) + n = data.shape[0] + dists = np.sqrt(dist_sq[np.triu_indices(n, k=1)]) + return float(np.median(dists)) + + +def features_to_coreset_matrix(features_generator: Iterator[torch.Tensor]) -> np.ndarray: + return np.stack([t.cpu().numpy() for t in features_generator], axis=0) + + +def coreset_indices(data, coreset_size, length_scale=None): + n = data.shape[0] + if coreset_size >= n: + return list(range(n)) + + if length_scale is None: + num_samples = min(n, BANDWIDTH_SAMPLE_CAP) + rng = np.random.default_rng(42) + idx = rng.choice(n, num_samples, replace=False) + length_scale = _median_heuristic(data[idx]) + if length_scale <= 0: + length_scale = DEFAULT_BANDWIDTH + + # kernel mean + K_sum = np.zeros(n, dtype=np.float64) + for i in range(0, n, KERNEL_BATCH): + end = min(i + KERNEL_BATCH, n) + K_sum += _rbf_kernel(data[i:end], data, length_scale).sum(axis=0) + K_mean = K_sum / n + + # greedy selection + selected = [] + selected_mask = np.zeros(n, dtype=bool) + K_selected_sum = np.zeros(n, dtype=np.float64) + + for t in range(coreset_size): + score = K_mean - K_selected_sum / (t + 1) + score[selected_mask] = -np.inf + best_idx = int(np.argmax(score)) + selected.append(best_idx) + selected_mask[best_idx] = True + K_selected_sum += _rbf_kernel( + data[best_idx:best_idx + 1], data, length_scale + ).ravel() + + return selected diff --git a/tools/herding_coreset_selector/herding/eval_datasets/__init__.py b/tools/herding_coreset_selector/herding/eval_datasets/__init__.py new file mode 100644 index 00000000..00cadce8 --- /dev/null +++ b/tools/herding_coreset_selector/herding/eval_datasets/__init__.py @@ -0,0 +1,17 @@ +from herding.eval_datasets.dataset_base import ( + EVAL_DATASETS, + EvalDatasetBase, + get_eval_dataset, + reg_eval_dataset, +) + +# Import adapters so their registration decorators are executed. +from herding.eval_datasets import aime2025 as _aime2025 # noqa: F401,E402 +from herding.eval_datasets import gpqa as _gpqa # noqa: F401,E402 + +__all__ = [ + "EVAL_DATASETS", + "EvalDatasetBase", + "get_eval_dataset", + "reg_eval_dataset", +] diff --git a/tools/herding_coreset_selector/herding/eval_datasets/aime2025.py b/tools/herding_coreset_selector/herding/eval_datasets/aime2025.py new file mode 100644 index 00000000..5202c021 --- /dev/null +++ b/tools/herding_coreset_selector/herding/eval_datasets/aime2025.py @@ -0,0 +1,49 @@ +import json +import os + +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.registry import ICL_PROMPT_TEMPLATES + +from herding.eval_datasets.dataset_base import EvalDatasetBase, reg_eval_dataset + + +FILENAME = "aime2025.jsonl" + +prompt_template_cfg = dict( + type=PromptTemplate, + template="{question}\nPlease reason step by step, and put your final answer within \\boxed{}.", +) + +prompt_template = ICL_PROMPT_TEMPLATES.build(prompt_template_cfg) + + +@reg_eval_dataset("aime2025") +class Aime2025Dataset(EvalDatasetBase): + def __init__(self, dataset_path, output_dir): + super().__init__(dataset_path, output_dir) + self.dataset = self._load_data() + + def _load_data(self): + filepath = os.path.join(self.dataset_path, FILENAME) + with open(filepath, "r", encoding="utf-8") as f: + return [json.loads(line) for line in f if line.strip()] + + def dataset_size(self): + return len(self.dataset) + + def dataset_prompts(self): + for item in self.dataset: + yield prompt_template.generate_item(item) + + def save_data_by_indices(self, indices, outpath): + output_dir = os.path.join(self.output_dir, outpath) + os.makedirs(output_dir, exist_ok=True) + + selected_data = [self.dataset[idx] for idx in indices] + output_filepath = os.path.join(output_dir, FILENAME) + with open(output_filepath, "w", encoding="utf-8") as f: + for item in selected_data: + f.write(json.dumps(item, ensure_ascii=False) + "\n") + + self.save_indices(indices, output_dir) + return output_dir diff --git a/tools/herding_coreset_selector/herding/eval_datasets/dataset_base.py b/tools/herding_coreset_selector/herding/eval_datasets/dataset_base.py new file mode 100644 index 00000000..75708e57 --- /dev/null +++ b/tools/herding_coreset_selector/herding/eval_datasets/dataset_base.py @@ -0,0 +1,63 @@ +import json +import os +from abc import ABC, abstractmethod + + +class EvalDatasetBase(ABC): + """Base class for evaluation datasets.""" + + def __init__(self, dataset_path, output_dir): + self.dataset_path = os.path.abspath(os.path.expanduser(dataset_path)) + self.output_dir = os.path.abspath(os.path.expanduser(output_dir)) + + @abstractmethod + def dataset_size(self) -> int: + """Return total number of items in the dataset.""" + + @abstractmethod + def dataset_prompts(self): + """Yield prompt strings one by one.""" + + @abstractmethod + def save_data_by_indices(self, indices, outpath): + """Save selected items into a subdirectory under ``self.output_dir``.""" + + def load_indices(self): + return list(range(self.dataset_size())) + + @staticmethod + def save_indices(indices, outpath): + """Save selected source indices to ``indices.json``.""" + indices_path = os.path.join(outpath, "indices.json") + with open(indices_path, "w", encoding="utf-8") as f: + json.dump(indices, f) + + def load_indices_from_strategy(self, strategy_name): + """Load indices saved under another strategy subdirectory, if present.""" + indices_path = os.path.join(self.output_dir, strategy_name, "indices.json") + if os.path.exists(indices_path): + with open(indices_path, "r", encoding="utf-8") as f: + return json.load(f) + return None + + +EVAL_DATASETS = {} + + +def reg_eval_dataset(dataset_name): + def wrapper(dataset_cls): + EVAL_DATASETS[dataset_name] = dataset_cls + return dataset_cls + + return wrapper + + +def get_eval_dataset(dataset_name, dataset_path, output_dir) -> EvalDatasetBase: + """Create a registered evaluation dataset adapter.""" + dataset_cls = EVAL_DATASETS.get(dataset_name) + if dataset_cls is None: + raise ValueError( + f'Unknown dataset "{dataset_name}". ' + f"Registered: {list(EVAL_DATASETS.keys())}" + ) + return dataset_cls(dataset_path=dataset_path, output_dir=output_dir) diff --git a/tools/herding_coreset_selector/herding/eval_datasets/gpqa.py b/tools/herding_coreset_selector/herding/eval_datasets/gpqa.py new file mode 100644 index 00000000..9d3ccfc7 --- /dev/null +++ b/tools/herding_coreset_selector/herding/eval_datasets/gpqa.py @@ -0,0 +1,75 @@ +import csv +import os + +from ais_bench.benchmark.datasets import GPQADataset +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.registry import ICL_PROMPT_TEMPLATES +from ais_bench.benchmark.utils.config.build import build_dataset_from_cfg + +from herding.eval_datasets.dataset_base import EvalDatasetBase, reg_eval_dataset + + +FILENAME = "gpqa_diamond.csv" + +align_prompt = """ +Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. + +{question} + +A) {A} +B) {B} +C) {C} +D) {D} +""".strip() + +prompt_template_cfg = dict( + type=PromptTemplate, + template=align_prompt, +) + +prompt_template = ICL_PROMPT_TEMPLATES.build(prompt_template_cfg) + + +@reg_eval_dataset("gpqa") +class GpqaDataset(EvalDatasetBase): + def __init__(self, dataset_path, output_dir): + super().__init__(dataset_path, output_dir) + + dataset_cfg = dict( + abbr="GPQA_diamond", + type=GPQADataset, + path=self.dataset_path, + name=FILENAME, + reader_cfg=dict( + input_columns=["question", "A", "B", "C", "D"], + output_column="answer", + ), + ) + self.dataset = build_dataset_from_cfg(dataset_cfg).test + + def dataset_size(self): + return len(self.dataset) + + def dataset_prompts(self): + for item in self.dataset: + yield prompt_template.generate_item(item) + + def save_data_by_indices(self, indices, outpath): + filepath = os.path.join(self.dataset_path, FILENAME) + with open(filepath, newline="", encoding="utf-8") as f: + data = list(csv.reader(f)) + + header = [data[0]] + data = data[1:] + + output_dir = os.path.join(self.output_dir, outpath) + os.makedirs(output_dir, exist_ok=True) + + rearranged_data = [data[idx] for idx in indices] + output_filepath = os.path.join(output_dir, FILENAME) + with open(output_filepath, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerows(header + rearranged_data) + + self.save_indices(indices, output_dir) + return output_dir diff --git a/tools/herding_coreset_selector/herding/features.py b/tools/herding_coreset_selector/herding/features.py new file mode 100644 index 00000000..ad51192b --- /dev/null +++ b/tools/herding_coreset_selector/herding/features.py @@ -0,0 +1,27 @@ +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + + +def load_model(model_path): + """Load the Hugging Face model used for hidden-state feature extraction.""" + tokenizer = AutoTokenizer.from_pretrained(model_path) + model = AutoModelForCausalLM.from_pretrained(model_path, device_map="auto") + model.eval() + return model, tokenizer + + +def generate_logits(model, tokenizer, prompts_generator): + for prompt in prompts_generator: + model_inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + with torch.no_grad(): + outputs = model.generate( + model_inputs.input_ids, + max_new_tokens=2, + do_sample=False, + output_hidden_states=True, + return_dict_in_generate=True, + ) + # first generated step, last layer, last position + last_layer_idx = model.config.num_hidden_layers + first_token_hidden = outputs.hidden_states[1][last_layer_idx][:, -1, :] + yield first_token_hidden.squeeze(0) From 693b16f0353fea11ae2f5ecfc6eb90c523a32028 Mon Sep 17 00:00:00 2001 From: ivanbao9783 Date: Wed, 26 Aug 2026 11:20:54 +0800 Subject: [PATCH 06/25] feat(output): add origin_top_logprobs field in vllm custom api (#458) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(output): add origin_top_logprobs field in vllm custom api * fix: 修复review意见 * feat(UT): add UT * feat(doc): add logprobs collection doc --- .../benchmark/models/api_models/base_api.py | 29 +++ .../models/api_models/vllm_custom_api.py | 28 +++ .../models/api_models/vllm_custom_api_chat.py | 13 ++ ais_bench/benchmark/models/output.py | 5 + .../gen_inferencer_output_handler.py | 6 + .../advanced_tutorials/logprobs_collection.md | 197 ++++++++++++++++++ .../base_tutorials/all_params/models.md | 2 +- .../advanced_tutorials/logprobs_collection.md | 197 ++++++++++++++++++ .../base_tutorials/all_params/models.md | 2 +- tests/UT/models/api_models/test_base_api.py | 21 ++ .../models/api_models/test_vllm_custom_api.py | 93 +++++++++ .../api_models/test_vllm_custom_api_chat.py | 101 +++++++++ tests/UT/models/test_output.py | 24 +++ .../test_gen_inferencer_output_handler.py | 68 +++++- 14 files changed, 779 insertions(+), 7 deletions(-) create mode 100644 docs/source_en/advanced_tutorials/logprobs_collection.md create mode 100644 docs/source_zh_cn/advanced_tutorials/logprobs_collection.md diff --git a/ais_bench/benchmark/models/api_models/base_api.py b/ais_bench/benchmark/models/api_models/base_api.py index b5a3fa77..880d7a86 100644 --- a/ais_bench/benchmark/models/api_models/base_api.py +++ b/ais_bench/benchmark/models/api_models/base_api.py @@ -113,6 +113,11 @@ def __init__( self.verbose = verbose self.session = None self.base_url = self._get_base_url() + if self._logprobs_enabled(): + self.logger.warning( + "logprobs is enabled, which will increase response size and " + "may impact evaluation performance and memory usage" + ) @abstractmethod def _get_url(self) -> str: @@ -219,6 +224,30 @@ async def get_request_body( " to be called in base classes", ) + async def _parse_logprobs(self, choice: dict, output: Output) -> None: + """ + 基类空实现，子类按需 override。vLLM 系列模型 override 此方法 + 以解析 OpenAI 兼容的 logprobs 响应。 + """ + pass + + def _logprobs_enabled(self) -> bool: + """检查 generation_kwargs 是否开启了 logprobs 采集。 + + chat API: logprobs 为 bool，True 表示开启 + completions API: logprobs 为 int，> 0 表示开启 + """ + if not self.generation_kwargs: + return False + val = self.generation_kwargs.get("logprobs") + if val is None: + return False + if isinstance(val, bool): + return val + if isinstance(val, int): + return val > 0 + return False + async def parse_text_response(self, data, output): raise AISBenchNotImplementedError( MODEL_CODES.PARSE_TEXT_RSP_NOT_IMPLEMENTED, diff --git a/ais_bench/benchmark/models/api_models/vllm_custom_api.py b/ais_bench/benchmark/models/api_models/vllm_custom_api.py index 3e4f5083..045ce4db 100644 --- a/ais_bench/benchmark/models/api_models/vllm_custom_api.py +++ b/ais_bench/benchmark/models/api_models/vllm_custom_api.py @@ -142,10 +142,38 @@ async def _parse_usage(self, json_content: dict, output: Output): output.input_tokens = json_content["usage"].get("prompt_tokens", 0) output.output_tokens = json_content["usage"].get("completion_tokens", 0) + async def _parse_logprobs(self, choice: dict, output: Output) -> None: + # completions API 格式：并行数组 {tokens, token_logprobs, top_logprobs} + # 转换为与 chat API 一致的嵌套结构：[{token, logprob, top_logprobs}, ...] + lp = choice.get("logprobs") + if not lp: + if self._logprobs_enabled(): + output.extra_details_data["logprobs_warning"] = ( + "logprobs is enabled in generation_kwargs but missing in response" + ) + return + tokens = lp.get("tokens", []) or [] + token_logprobs = lp.get("token_logprobs", []) or [] + top_logprobs = lp.get("top_logprobs", []) or [] + result = [] + for i in range(len(tokens)): + # token_logprobs 首项可能为 None（predefined token），保留为 None 项以对齐位置 + if token_logprobs[i] is None: + result.append(None) + continue + item = { + "token": tokens[i], + "logprob": token_logprobs[i], + "top_logprobs": top_logprobs[i] if i < len(top_logprobs) else [], + } + result.append(item) + output.origin_logprobs = result + async def parse_text_response(self, api_response: dict, output: Output): generated_text = api_response.get("choices", [{}])[0].get("text", "") output.content = generated_text await self._parse_usage(api_response, output) + await self._parse_logprobs(api_response.get("choices", [{}])[0], output) self.logger.debug(f"Output content: {output.content}") async def parse_stream_response(self, api_response: dict, output: Output): diff --git a/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py b/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py index 23a88fad..30a6a25c 100644 --- a/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py +++ b/ais_bench/benchmark/models/api_models/vllm_custom_api_chat.py @@ -199,12 +199,25 @@ async def parse_stream_response(self, json_content, output): output.reasoning_content += reasoning await self._parse_usage(json_content, output) + async def _parse_logprobs(self, choice: dict, output: Output) -> None: + # chat API 格式：choice.logprobs.content[] + # 直接透传 vLLM 原始结构，每个 item 含 {token, logprob, bytes, top_logprobs} + lp = choice.get("logprobs") + if not lp: + if self._logprobs_enabled(): + output.extra_details_data["logprobs_warning"] = ( + "logprobs is enabled in generation_kwargs but missing in response" + ) + return + output.origin_logprobs = lp.get("content") or [] + async def parse_text_response(self, json_content, output): for item in json_content.get("choices", []): if content:=item["message"].get("content"): output.content += content if reasoning_content:=item["message"].get("reasoning_content") or item["message"].get("reasoning"): output.reasoning_content += reasoning_content + await self._parse_logprobs(item, output) await self._parse_usage(json_content, output) output.update_extra_details_data_from_text_response(json_content) self.logger.debug(f"Output content: {output.content}") diff --git a/ais_bench/benchmark/models/output.py b/ais_bench/benchmark/models/output.py index aaf7379b..ef1d91bd 100644 --- a/ais_bench/benchmark/models/output.py +++ b/ais_bench/benchmark/models/output.py @@ -26,6 +26,9 @@ def __init__(self, perf_mode: bool = False) -> None: # In multi-turn dialogue scenarios, all turns of the same sample share the same uuid. # In pass@k scenarios, the same sample is sampled k times and each run receives a distinct uuid self.turn_id: int = 0 + # 生成 token 的 logprobs 信息（仅配置了 logprobs=True 时才有数据） + # 保留 vLLM 原始结构：[{token, logprob, bytes, top_logprobs: [...]}, ...] + self.origin_logprobs: list = [] @abstractmethod def get_metrics(self) -> dict: @@ -147,6 +150,8 @@ def get_metrics(self) -> dict: def clean_result(res): for key in ["content", "reasoning_content", "perf_mode"]: res.pop(key, None) + if not res.get("origin_logprobs"): + res.pop("origin_logprobs", None) return res self.prediction = self.get_prediction() diff --git a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py index 3cae69e4..58f58d23 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py @@ -71,6 +71,12 @@ def get_prediction_result( if isinstance(output, Output) and output.extra_details_data.get('response_anomaly_payload'): result_data['response_anomaly_payload'] = output.extra_details_data['response_anomaly_payload'] + if isinstance(output, Output): + result_data["input_tokens"] = output.input_tokens + result_data["output_tokens"] = output.output_tokens + if output.origin_logprobs: + result_data["origin_logprobs"] = output.origin_logprobs + if gold: result_data["gold"] = gold return result_data \ No newline at end of file diff --git a/docs/source_en/advanced_tutorials/logprobs_collection.md b/docs/source_en/advanced_tutorials/logprobs_collection.md new file mode 100644 index 00000000..e4952fb9 --- /dev/null +++ b/docs/source_en/advanced_tutorials/logprobs_collection.md @@ -0,0 +1,197 @@ +# Logprobs Collection and Analysis + +## Overview + +In both accuracy evaluation (`--mode accuracy`) and performance evaluation (`--mode perf`) scenarios, AISBench supports enabling **token logprobs collection** via the model configuration's `generation_kwargs`. The token probability information returned by the inference service is persisted to result files for downstream dataset-level post-processing analysis (e.g., avg/min/max/outlier statistics, candidate token distribution analysis). + +The collected logprobs data follows the vLLM API's original response format, where each item contains the token text, log probability, byte sequence, and top candidate token distribution. + +--- + +## Prerequisites + +1. **vLLM-series inference backend** — Currently only `VLLMCustomAPI` (completions API) and `VLLMCustomAPIChat` (chat API) model types are supported. +2. **Non-streaming interface** — Must configure `stream=False`; streaming mode does not yet support logprobs collection. +3. **Inference service supports logprobs** — The server-side vLLM version must support the `logprobs` / `top_logprobs` parameters. + +> ⚠️ **Support scope**: Other API backends (TGI / Triton / Mindie, etc.) and local models (HF, etc.) are not yet supported. Extensions will be added as needed. + +--- + +## Quick Start + +Add the `logprobs` parameter to `generation_kwargs` in the model configuration to enable: + +```python +from ais_bench.benchmark.models import VLLMCustomAPIChat + +models = [ + dict( + type=VLLMCustomAPIChat, + abbr='vllm-chat-logprobs', + host_ip='127.0.0.1', host_port=8080, + stream=False, # must be non-streaming + generation_kwargs=dict( + temperature=0.6, + top_p=0.95, + logprobs=True, # chat API: enable logprobs collection + top_logprobs=5, # optional: return top 5 candidate distribution per token + ), + ), +] +``` + +### Parameter Description + +| Parameter | Applicable API | Type | Description | +|-----------|---------------|------|-------------| +| `logprobs` | chat API | Bool | `True` to enable logprobs collection, `False` to disable | +| `logprobs` | completions API | Int | Number of top candidates to return, range `[0, 20]`; `0` disables, `>0` enables and returns the corresponding number of candidates | +| `top_logprobs` | chat API | Int | Number of top candidates per token, range `[0, 20]`. **Only needs separate configuration under chat API**; completions API specifies this directly via `logprobs` | + +> 💡 **Distinguishing two API parameter semantics**: For chat API, `logprobs` is a boolean switch and `top_logprobs` separately specifies the candidate count; for completions API, `logprobs` itself is an integer that directly specifies the candidate count, requiring no additional parameter. + +--- + +## How It Works + +```mermaid +sequenceDiagram + participant User + participant AISBench + participant Server as vLLM Server + participant Disk as Result File + + User->>AISBench: Configure generation_kwargs.logprobs + AISBench->>AISBench: Print warning at startup about performance impact + AISBench->>Server: Send inference request (with logprobs parameter) + Server-->>AISBench: Return response (with logprobs field) + AISBench->>AISBench: _parse_logprobs parses and writes to output.origin_logprobs + alt Enabled but missing in response + AISBench->>AISBench: Write logprobs_warning to extra_details_data + end + AISBench->>Disk: Persist to result file +``` + +1. **Startup check**: At model instantiation, checks whether `generation_kwargs` has logprobs enabled; if so, prints a warning about performance impact. +2. **Request sending**: The logprobs parameter in `generation_kwargs` is passed through to the inference service request body. +3. **Response parsing**: The `_parse_logprobs` method parses the logprobs field in the vLLM response into a unified nested structure and writes it to `output.origin_logprobs`. +4. **Anomaly alert**: If the user enabled logprobs but the field is missing in the response, an alert message is written to `output.extra_details_data["logprobs_warning"]`, avoiding false positives on normal requests that don't enable logprobs. +5. **Result persistence**: `origin_logprobs` is output with the result file; empty lists are filtered out. + +--- + +## Persisted Data Structure + +### Accuracy Evaluation Scenario + +Persisted file: `outputs//predictions//.jsonl` + +New fields per case: + +| Field | Type | Description | +|-------|------|-------------| +| `input_tokens` | Int | Number of input tokens | +| `output_tokens` | Int | Number of output tokens | +| `origin_logprobs` | List[Dict\|None] | Token logprobs information list; not persisted when disabled | + +### Performance Evaluation Scenario + +Persisted file: `outputs//performance/_details.jsonl` + +Each case carries the `origin_logprobs` field via `get_metrics()`'s `to_dict()`; empty lists are filtered out. + +### origin_logprobs Data Format + +Both **chat API** and **completions API** are unified into the following nested structure: + +```json +[ + { + "token": "Hello", + "logprob": -0.5234, + "bytes": [72, 101, 108, 108, 111], + "top_logprobs": [ + {"token": "Hello", "logprob": -0.5234, "bytes": [72, 101, 108, 108, 111]}, + {"token": "Hi", "logprob": -2.1034, "bytes": [72, 105]} + ] + }, + null, + { + "token": "world", + "logprob": -0.0012, + "bytes": [119, 111, 114, 108, 100], + "top_logprobs": [] + } +] +``` + +| Field | Type | Description | +|-------|------|-------------| +| `token` | String | Token text | +| `logprob` | Float | Log probability of the token | +| `bytes` | List[Int] | UTF-8 byte sequence of the token (only returned by chat API; completions API does not include this field) | +| `top_logprobs` | List[Dict] | Top candidate token distribution, each item contains `token` / `logprob` / `bytes`. Empty array when `top_logprobs` is not configured | + +> ⚠️ **Meaning of `null` entries**: The `logprob` of the first item in the token sequence may be `null` (indicating a predefined token with no probability value). `null` entries are preserved to align token positions for position-indexed analysis. + +### logprobs_warning Field + +When the user enables logprobs but the inference service response is missing the field, a `logprobs_warning` field appears in the result file (under `extra_details_data`): + +```json +{ + "extra_details_data": { + "logprobs_warning": "logprobs is enabled in generation_kwargs but missing in response" + } +} +``` + +**Possible causes**: +- Inference service version does not support the logprobs parameter +- Request parameter was ignored or filtered by the server +- Backend model type is incompatible with logprobs + +> 💡 **Design note**: Missing logprobs does not trigger request retry (unlike `error_info`); it serves only as an alert for the user to check the configuration. + +--- + +## Configuration Scenario Comparison + +| Configuration | chat API persistence effect | completions API persistence effect | +|---------------|----------------------------|-----------------------------------| +| logprobs not configured | No `origin_logprobs` field | No `origin_logprobs` field | +| `logprobs=True` (chat) / `logprobs=1` (completions) | `origin_logprobs` contains token/logprob/bytes; `top_logprobs` is empty array | `origin_logprobs` contains token/logprob; `top_logprobs` is empty array | +| `logprobs=True` + `top_logprobs=5` (chat) / `logprobs=5` (completions) | `origin_logprobs` contains full candidate distribution | `origin_logprobs` contains full candidate distribution | + +--- + +## Performance Impact and Precautions + +> ⚠️ **Important**: Enabling logprobs significantly increases response size, affecting evaluation efficiency and memory usage. + +### Response Size Estimation + +Taking `max_out_len=4096` as an example: + +| Configuration | Estimated single response size | +|---------------|-------------------------------| +| logprobs disabled | A few KB (text + usage only) | +| `logprobs=True` (no top_logprobs) | ~200-400 KB (~50-100B per token) | +| `logprobs=True` + `top_logprobs=20` | ~4-8 MB (~1-2KB per token) | + +### Risk Points + +| Risk | Description | +|------|-------------| +| **Memory peak** | During response parsing, `response.text()` + `json.loads` + `output.origin_logprobs` hold three copies in memory simultaneously | +| **Concurrency amplification** | The worker loop runs `batch_size` concurrent requests, each holding a large response | +| **Disk amplification** | Each case's jsonl writes full logprobs, disk usage grows linearly | +| **No body size limit** | AISBench currently does not constrain response body size in non-streaming scenarios | + +### Recommendations + +1. **Limit `top_logprobs` value** (e.g., ≤5) — this is the primary factor in response bloat +2. **Control the product of `max_out_len` and `batch_size`** to avoid high concurrency memory peaks +3. **Validate with a small dataset first** before running full data, confirming server-side support +4. **Watch startup warnings**: AISBench prints a logprobs performance impact notice at startup diff --git a/docs/source_en/base_tutorials/all_params/models.md b/docs/source_en/base_tutorials/all_params/models.md index cf3679d7..682aa379 100644 --- a/docs/source_en/base_tutorials/all_params/models.md +++ b/docs/source_en/base_tutorials/all_params/models.md @@ -92,7 +92,7 @@ The description of configurable parameters for the service-oriented inference ba | `max_out_len` | Int | Maximum output length of the inference response; the actual length may be limited by the server. | | `batch_size` | Int | Batch size for concurrent requests. Valid range: (0, 64000] | | `trust_remote_code` | Boolean | Whether the tokenizer trusts remote code, default is `False`| -| `generation_kwargs` | Dict | Configuration of inference generation parameters, depending on the specific service-oriented backend and interface type. Note: Currently, multi-sampling parameters such as `best_of` and `n` are not supported, but multiple independent inferences can be performed using the `num_return_sequences` parameter (for details, refer to 🔗 [the role of `num_return_sequences` in the Text Generation Documentation](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences\(int,)) | +| `generation_kwargs` | Dict | Configuration of inference generation parameters, depending on the specific service-oriented backend and interface type. Note: Currently, multi-sampling parameters such as `best_of` and `n` are not supported, but multiple independent inferences can be performed using the `num_return_sequences` parameter (for details, refer to 🔗 [the role of `num_return_sequences` in the Text Generation Documentation](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences\(int,)). Supports configuring `logprobs` / `top_logprobs` parameters to enable token probability information collection; see 🔗[Logprobs Collection and Analysis](../../advanced_tutorials/logprobs_collection.md) | | `returns_tool_calls` | Bool | Controls the extraction method of function call information. When set to `True`, the system extracts function call information from the `tool_calls` field of the API response; when set to `False`, the system parses function call information from the `content` field | | `pred_postprocessor` | Dict | Post-processing configuration for model output results. It is used to format, clean, or convert the original model output to meet the requirements of specific evaluation tasks | | `response_anomaly` | Dict | Optional; model-level config for msProbe response anomaly detection, including `model_name` (must match the name in msProbe's mtype_config.json), `model_path` (local model directory, optional, used to auto-generate configs), `msprobe_mtype_path`, and `msprobe_token2category_dir`. When mtype/token-category paths are not provided, the default files inside the msProbe package are used | diff --git a/docs/source_zh_cn/advanced_tutorials/logprobs_collection.md b/docs/source_zh_cn/advanced_tutorials/logprobs_collection.md new file mode 100644 index 00000000..c7046523 --- /dev/null +++ b/docs/source_zh_cn/advanced_tutorials/logprobs_collection.md @@ -0,0 +1,197 @@ +# Logprobs 采集与分析 + +## 概述 + +在精度评测（`--mode accuracy`）和性能评测（`--mode perf`）场景下，AISBench 支持通过模型配置的 `generation_kwargs` 开启 **token logprobs 采集**，将推理服务返回的 token 概率信息落盘到结果文件中，供后续数据集维度的后处理分析使用（如 avg/min/max/异常值统计、候选 token 分布分析等）。 + +采集的 logprobs 数据为 vLLM API 的原始响应格式，每项包含 token 文本、对数概率、字节序列以及 top 候选 token 分布。 + +--- + +## 前置条件 + +1. **推理后端为 vLLM 系列** — 当前仅支持 `VLLMCustomAPI`（completions API）和 `VLLMCustomAPIChat`（chat API）两种模型类型。 +2. **非流式接口** — 需配置 `stream=False`，流式模式暂不支持 logprobs 采集。 +3. **推理服务支持 logprobs** — 服务端 vLLM 版本需支持 `logprobs` / `top_logprobs` 参数。 + +> ⚠️ **支持范围**：其他 API 后端（TGI / Triton / Mindie 等）和本地模型（HF 等）暂不支持。后续按需扩展。 + +--- + +## 快速使用 + +在模型配置的 `generation_kwargs` 中增加 `logprobs` 参数即可启用： + +```python +from ais_bench.benchmark.models import VLLMCustomAPIChat + +models = [ + dict( + type=VLLMCustomAPIChat, + abbr='vllm-chat-logprobs', + host_ip='127.0.0.1', host_port=8080, + stream=False, # 必须为非流式 + generation_kwargs=dict( + temperature=0.6, + top_p=0.95, + logprobs=True, # chat API: 开启 logprobs 采集 + top_logprobs=5, # 可选：返回每 token 的 top 5 候选分布 + ), + ), +] +``` + +### 参数说明 + +| 参数 | 适用 API | 类型 | 说明 | +|------|---------|------|------| +| `logprobs` | chat API | Bool | `True` 开启 logprobs 采集，`False` 关闭 | +| `logprobs` | completions API | Int | 返回的 top 候选数量，取值范围 `[0, 20]`，`0` 表示关闭，`>0` 表示开启并返回对应数量的候选 | +| `top_logprobs` | chat API | Int | 每 token 返回的 top 候选数量，取值范围 `[0, 20]`。**仅在 chat API 下需要单独配置**，completions API 由 `logprobs` 直接指定 | + +> 💡 **区分两种 API 的参数语义**：chat API 的 `logprobs` 是布尔开关，`top_logprobs` 单独指定候选数；completions API 的 `logprobs` 本身就是整数，直接指定候选数，无需额外参数。 + +--- + +## 工作原理 + +```mermaid +sequenceDiagram + participant User + participant AISBench + participant Server as vLLM Server + participant Disk as 结果文件 + + User->>AISBench: 配置 generation_kwargs.logprobs + AISBench->>AISBench: 启动时打印 warning 提示性能影响 + AISBench->>Server: 发送推理请求（携带 logprobs 参数） + Server-->>AISBench: 返回响应（含 logprobs 字段） + AISBench->>AISBench: _parse_logprobs 解析并写入 output.origin_logprobs + alt 用户开启但响应缺失 + AISBench->>AISBench: 写入 logprobs_warning 到 extra_details_data + end + AISBench->>Disk: 落盘到结果文件 +``` + +1. **启动检查**：模型实例化时检查 `generation_kwargs` 是否开启 logprobs，若开启则打印 warning 提示性能影响。 +2. **请求发送**：`generation_kwargs` 中的 logprobs 参数透传到推理服务请求体。 +3. **响应解析**：`_parse_logprobs` 方法将 vLLM 响应中的 logprobs 字段解析为统一的嵌套结构，写入 `output.origin_logprobs`。 +4. **异常告警**：若用户开启了 logprobs 但响应中缺失该字段，将告警信息写入 `output.extra_details_data["logprobs_warning"]`，避免误伤未开启 logprobs 的正常请求。 +5. **结果落盘**：`origin_logprobs` 随结果文件输出，空列表会被过滤移除。 + +--- + +## 落盘数据结构 + +### 精度评测场景 + +落盘文件：`outputs//predictions//.jsonl` + +每条 case 新增字段： + +| 字段 | 类型 | 说明 | +|------|------|------| +| `input_tokens` | Int | 输入 token 数量 | +| `output_tokens` | Int | 输出 token 数量 | +| `origin_logprobs` | List[Dict\|None] | token logprobs 信息列表，未开启时该字段不落盘 | + +### 性能评测场景 + +落盘文件：`outputs//performance/_details.jsonl` + +每条 case 通过 `get_metrics()` 的 `to_dict()` 带出 `origin_logprobs` 字段，空列表会被过滤移除。 + +### origin_logprobs 数据格式 + +**chat API** 和 **completions API** 均统一为以下嵌套结构： + +```json +[ + { + "token": "Hello", + "logprob": -0.5234, + "bytes": [72, 101, 108, 108, 111], + "top_logprobs": [ + {"token": "Hello", "logprob": -0.5234, "bytes": [72, 101, 108, 108, 111]}, + {"token": "Hi", "logprob": -2.1034, "bytes": [72, 105]} + ] + }, + null, + { + "token": "world", + "logprob": -0.0012, + "bytes": [119, 111, 114, 108, 100], + "top_logprobs": [] + } +] +``` + +| 字段 | 类型 | 说明 | +|------|------|------| +| `token` | String | token 文本 | +| `logprob` | Float | 该 token 的对数概率 | +| `bytes` | List[Int] | token 的 UTF-8 字节序列（仅 chat API 返回，completions API 不含此字段） | +| `top_logprobs` | List[Dict] | top 候选 token 分布，每项含 `token` / `logprob` / `bytes`。未配置 `top_logprobs` 时为空数组 | + +> ⚠️ **`null` 项含义**：token 序列首项的 `logprob` 可能为 `null`（表示 predefined token，无概率值）。保留 `null` 项以对齐 token 位置，便于后续按位置索引分析。 + +### logprobs_warning 字段 + +当用户开启了 logprobs 但推理服务响应中缺失该字段时，结果文件中会出现 `logprobs_warning` 字段（位于 `extra_details_data` 下）： + +```json +{ + "extra_details_data": { + "logprobs_warning": "logprobs is enabled in generation_kwargs but missing in response" + } +} +``` + +**可能原因**： +- 推理服务版本不支持 logprobs 参数 +- 请求参数被服务端忽略或过滤 +- 后端模型类型与 logprobs 不兼容 + +> 💡 **设计说明**：logprobs 缺失不会触发请求重试（不同于 `error_info`），仅作为告警信息提示用户检查配置。 + +--- + +## 配置场景对照 + +| 配置 | chat API 落盘效果 | completions API 落盘效果 | +|------|------------------|------------------------| +| 未配置 logprobs | 无 `origin_logprobs` 字段 | 无 `origin_logprobs` 字段 | +| `logprobs=True`（chat）/ `logprobs=1`（completions） | `origin_logprobs` 含 token/logprob/bytes，`top_logprobs` 为空数组 | `origin_logprobs` 含 token/logprob，`top_logprobs` 为空数组 | +| `logprobs=True` + `top_logprobs=5`（chat）/ `logprobs=5`（completions） | `origin_logprobs` 含完整候选分布 | `origin_logprobs` 含完整候选分布 | + +--- + +## 性能影响与注意事项 + +> ⚠️ **重要提示**：开启 logprobs 会显著增大响应体积，影响评测效率和内存占用。 + +### 响应体积估算 + +以 `max_out_len=4096` 为例： + +| 配置 | 单条 response 估算大小 | +|------|----------------------| +| 不开 logprobs | 几 KB（仅 text + usage） | +| `logprobs=True`（无 top_logprobs） | ~200-400 KB（每 token 约 50-100B） | +| `logprobs=True` + `top_logprobs=20` | ~4-8 MB（每 token 约 1-2KB） | + +### 风险点 + +| 风险 | 说明 | +|------|------| +| **内存峰值** | 响应解析过程中，`response.text()` + `json.loads` + `output.origin_logprobs` 三份副本同时在内存 | +| **并发放大** | worker loop 按 `batch_size` 并发，多请求并行持有大 response | +| **落盘放大** | 每条 case 的 jsonl 都写入完整 logprobs，磁盘占用线性增长 | +| **无 body 大小限制** | 当前 AISBench 在非流式场景下未约束 response body 大小 | + +### 建议 + +1. **限制 `top_logprobs` 取值**（如 ≤5），这是响应膨胀的主要因素 +2. **控制 `max_out_len` 与 `batch_size` 的乘积**，避免并发内存峰值过高 +3. **优先使用小数据集验证**，确认服务端支持后再跑全量数据 +4. **关注启动 warning**：AISBench 会在启动时打印 logprobs 性能影响提示 diff --git a/docs/source_zh_cn/base_tutorials/all_params/models.md b/docs/source_zh_cn/base_tutorials/all_params/models.md index c9e22895..00e749c4 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/models.md +++ b/docs/source_zh_cn/base_tutorials/all_params/models.md @@ -87,7 +87,7 @@ models = [ | `max_out_len` | Int | 推理响应的最大输出长度，实际长度可能受服务端限制。 | | `batch_size` | Int | 请求的并发批处理大小。合法范围：(0, 64000] | | `trust_remote_code` | Boolean | tokenizer是否信任远程代码，默认False; | -| `generation_kwargs` | Dict | 推理生成参数配置，依赖具体的服务化后端和接口类型。注意：当前不支持 `best_of` 和 `n` 等多次采样参数，但支持通过`num_return_sequences`参数进行多次独立推理(具体请参考🔗[Text Generation 文档](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences)中`num_return_sequences`的作用) | +| `generation_kwargs` | Dict | 推理生成参数配置，依赖具体的服务化后端和接口类型。注意：当前不支持 `best_of` 和 `n` 等多次采样参数，但支持通过`num_return_sequences`参数进行多次独立推理(具体请参考🔗[Text Generation 文档](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences)中`num_return_sequences`的作用)。支持配置 `logprobs` / `top_logprobs` 参数开启 token 概率信息采集，详见 🔗[Logprobs 采集与分析](../../advanced_tutorials/logprobs_collection.md) | | `returns_tool_calls` | Bool | 控制函数调用信息的提取方式。当设置为True时，系统从API响应的`tool_calls`字段中提取函数调用信息；当设置为False时，系统从`content`字段中解析函数调用信息 | | `pred_postprocessor` | Dict | 模型输出结果的后处理配置。用于对原始模型输出进行格式化、清理或转换，以满足特定评估任务的要求 | | `response_anomaly` | Dict | 可选，msProbe 推理响应异常检测的模型级配置，包含 `model_name`（与 msProbe 的 mtype_config.json 名称一致）、`model_path`（本地模型目录，用于自动生成配置，可选）、`msprobe_mtype_path`、`msprobe_token2category_dir`。未提供 mtype/token 分类路径时回退到 msProbe 包内默认文件 | diff --git a/tests/UT/models/api_models/test_base_api.py b/tests/UT/models/api_models/test_base_api.py index 83b05d0d..d50237e8 100644 --- a/tests/UT/models/api_models/test_base_api.py +++ b/tests/UT/models/api_models/test_base_api.py @@ -548,5 +548,26 @@ def test_parse_template_with_meta_template(self): self.assertEqual(result[2]["role"], "assistant") +class TestBaseAPIModelParseLogprobs(unittest.TestCase): + def test_parse_logprobs_base_default_no_op(self): + """测试 BaseAPIModel._parse_logprobs 基类空实现不报错且不修改 output""" + import asyncio + from unittest.mock import MagicMock + + # BaseAPIModel 需要初始化参数，用 mock 绕过 + model = MagicMock(spec=BaseAPIModel) + # 绑定真实方法到 mock 对象上 + model._parse_logprobs = BaseAPIModel._parse_logprobs.__get__(model, BaseAPIModel) + + output = Output() + output.origin_logprobs = [] + + choice = {"logprobs": {"tokens": ["A"], "token_logprobs": [-0.5]}} + + asyncio.run(model._parse_logprobs(choice, output)) + + self.assertEqual(output.origin_logprobs, []) + + if __name__ == "__main__": unittest.main() diff --git a/tests/UT/models/api_models/test_vllm_custom_api.py b/tests/UT/models/api_models/test_vllm_custom_api.py index 9f0e6238..d43f34b5 100644 --- a/tests/UT/models/api_models/test_vllm_custom_api.py +++ b/tests/UT/models/api_models/test_vllm_custom_api.py @@ -326,6 +326,99 @@ def test_parse_stream_response_without_content_wrapper(self): def test_parse_stream_response_no_choices_wrapper(self): self.run_async_test(self.test_parse_stream_response_no_choices()) + async def test_parse_logprobs_with_data(self): + """测试_parse_logprobs正确解析logprobs响应""" + model = VLLMCustomAPI(**self.default_kwargs) + output = Output() + + choice = { + "text": "B", + "logprobs": { + "tokens": ["B"], + "token_logprobs": [-0.5], + "text_offset": [0], + "top_logprobs": [{"333": -0.5, "444": -2.1}] + } + } + + await model._parse_logprobs(choice, output) + + # completions API 转换为统一嵌套结构 + self.assertEqual(output.origin_logprobs, [ + {"token": "B", "logprob": -0.5, "top_logprobs": {"333": -0.5, "444": -2.1}} + ]) + + async def test_parse_logprobs_without_logprobs_field(self): + """测试_parse_logprobs在响应无logprobs字段时不报错""" + model = VLLMCustomAPI(**self.default_kwargs) + output = Output() + + choice = {"text": "B"} + + await model._parse_logprobs(choice, output) + + self.assertEqual(output.origin_logprobs, []) + + async def test_parse_logprobs_with_none_token_logprob(self): + """测试_parse_logprobs处理None的token_logprob（保留为None项对齐位置）""" + model = VLLMCustomAPI(**self.default_kwargs) + output = Output() + + choice = { + "text": "AB", + "logprobs": { + "tokens": ["A", "B"], + "token_logprobs": [-0.5, None], + "top_logprobs": [{"333": -0.5}, None] + } + } + + await model._parse_logprobs(choice, output) + + self.assertEqual(output.origin_logprobs, [ + {"token": "A", "logprob": -0.5, "top_logprobs": {"333": -0.5}}, + None, + ]) + + async def test_parse_text_response_with_logprobs(self): + """测试parse_text_response正确调用_parse_logprobs""" + model = VLLMCustomAPI(**self.default_kwargs) + output = Output() + output.content = "" + + response = { + "choices": [{ + "text": "B", + "logprobs": { + "tokens": ["B"], + "token_logprobs": [-0.5], + "top_logprobs": [{"333": -0.5}] + } + }], + "usage": {"prompt_tokens": 10, "completion_tokens": 1} + } + + await model.parse_text_response(response, output) + + self.assertEqual(output.content, "B") + self.assertEqual(output.input_tokens, 10) + self.assertEqual(output.output_tokens, 1) + self.assertEqual(output.origin_logprobs, [ + {"token": "B", "logprob": -0.5, "top_logprobs": {"333": -0.5}} + ]) + + def test_parse_logprobs_with_data_wrapper(self): + self.run_async_test(self.test_parse_logprobs_with_data()) + + def test_parse_logprobs_without_logprobs_field_wrapper(self): + self.run_async_test(self.test_parse_logprobs_without_logprobs_field()) + + def test_parse_logprobs_with_none_token_logprob_wrapper(self): + self.run_async_test(self.test_parse_logprobs_with_none_token_logprob()) + + def test_parse_text_response_with_logprobs_wrapper(self): + self.run_async_test(self.test_parse_text_response_with_logprobs()) + def test_calc_ppl(self): """测试_calc_ppl方法""" model = VLLMCustomAPI(**self.default_kwargs) diff --git a/tests/UT/models/api_models/test_vllm_custom_api_chat.py b/tests/UT/models/api_models/test_vllm_custom_api_chat.py index decca28f..7a7fd8ee 100644 --- a/tests/UT/models/api_models/test_vllm_custom_api_chat.py +++ b/tests/UT/models/api_models/test_vllm_custom_api_chat.py @@ -333,6 +333,107 @@ def test_parse_stream_response_wrapper(self): def test_parse_text_response_wrapper(self): self.run_async_test(self.test_parse_text_response()) + async def test_parse_logprobs_with_data(self): + """测试_parse_logprobs正确解析chat API logprobs响应""" + model = VLLMCustomAPIChat(**self.default_kwargs) + output = RequestOutput() + + content_list = [ + {"token": "B", "logprob": -0.5, "bytes": [66], "top_logprobs": [{"token": "B", "logprob": -0.5}, {"token": "A", "logprob": -2.1}]} + ] + choice = { + "message": {"content": "B"}, + "logprobs": {"content": content_list} + } + + await model._parse_logprobs(choice, output) + + # chat API 直接透传 lp.content，保持 vLLM 原始结构 + self.assertEqual(output.origin_logprobs, content_list) + + async def test_parse_logprobs_without_logprobs_field(self): + """测试_parse_logprobs在响应无logprobs字段时不报错""" + model = VLLMCustomAPIChat(**self.default_kwargs) + output = RequestOutput() + + choice = {"message": {"content": "B"}} + + await model._parse_logprobs(choice, output) + + self.assertEqual(output.origin_logprobs, []) + + async def test_parse_logprobs_enabled_but_missing_warns(self): + """测试开启logprobs但响应缺失时，写入logprobs_warning到extra_details_data""" + kwargs = self.default_kwargs.copy() + kwargs["generation_kwargs"] = {"logprobs": True} + model = VLLMCustomAPIChat(**kwargs) + output = RequestOutput() + + choice = {"message": {"content": "B"}} # 无 logprobs 字段 + + await model._parse_logprobs(choice, output) + + self.assertEqual(output.origin_logprobs, []) + self.assertIn("logprobs_warning", output.extra_details_data) + self.assertIn("logprobs is enabled", output.extra_details_data["logprobs_warning"]) + + async def test_parse_logprobs_with_none_content(self): + """测试_parse_logprobs保留None项（predefined token）""" + model = VLLMCustomAPIChat(**self.default_kwargs) + output = RequestOutput() + + content_list = [ + None, + {"token": "B", "logprob": -0.3, "top_logprobs": [{"token": "B", "logprob": -0.3}]} + ] + choice = { + "message": {"content": "AB"}, + "logprobs": {"content": content_list} + } + + await model._parse_logprobs(choice, output) + + # 直接透传，None 项保留 + self.assertEqual(output.origin_logprobs, content_list) + + async def test_parse_text_response_with_logprobs(self): + """测试parse_text_response正确调用_parse_logprobs""" + model = VLLMCustomAPIChat(**self.default_kwargs) + output = RequestOutput() + + content_list = [ + {"token": "B", "logprob": -0.5, "top_logprobs": [{"token": "B", "logprob": -0.5}]} + ] + response = { + "choices": [{ + "message": {"content": "B"}, + "logprobs": {"content": content_list} + }], + "usage": {"prompt_tokens": 10, "completion_tokens": 1} + } + + await model.parse_text_response(response, output) + + self.assertEqual(output.content, "B") + self.assertEqual(output.input_tokens, 10) + self.assertEqual(output.output_tokens, 1) + self.assertEqual(output.origin_logprobs, content_list) + + def test_parse_logprobs_with_data_wrapper(self): + self.run_async_test(self.test_parse_logprobs_with_data()) + + def test_parse_logprobs_without_logprobs_field_wrapper(self): + self.run_async_test(self.test_parse_logprobs_without_logprobs_field()) + + def test_parse_logprobs_enabled_but_missing_warns_wrapper(self): + self.run_async_test(self.test_parse_logprobs_enabled_but_missing_warns()) + + def test_parse_logprobs_with_none_content_wrapper(self): + self.run_async_test(self.test_parse_logprobs_with_none_content()) + + def test_parse_text_response_with_logprobs_wrapper(self): + self.run_async_test(self.test_parse_text_response_with_logprobs()) + def test_calc_ppl(self): """测试_calc_ppl方法""" model = VLLMCustomAPIChat(**self.default_kwargs) diff --git a/tests/UT/models/test_output.py b/tests/UT/models/test_output.py index 1f294802..894df6d4 100644 --- a/tests/UT/models/test_output.py +++ b/tests/UT/models/test_output.py @@ -30,6 +30,7 @@ def test_output_initialization(): assert output.input is None assert output.uuid == "" assert output.turn_id == 0 + assert output.origin_logprobs == [] output_perf = ConcreteOutput(perf_mode=True) assert output_perf.perf_mode is True @@ -185,6 +186,29 @@ def test_request_output_get_metrics(): assert metrics["output_tokens"] == 20 +def test_request_output_get_metrics_empty_logprobs_removed(): + """测试性能模式下空 origin_logprobs 被 clean_result 移除""" + output = RequestOutput() + output.success = True + output.time_points = [time.perf_counter() - 1, time.perf_counter()] + output.origin_logprobs = [] + + metrics = output.get_metrics() + assert "origin_logprobs" not in metrics + + +def test_request_output_get_metrics_nonempty_logprobs_kept(): + """测试性能模式下非空 origin_logprobs 被 clean_result 保留""" + output = RequestOutput() + output.success = True + output.time_points = [time.perf_counter() - 1, time.perf_counter()] + lp_data = [{"token": "B", "logprob": -0.5, "top_logprobs": []}] + output.origin_logprobs = lp_data + + metrics = output.get_metrics() + assert metrics["origin_logprobs"] == lp_data + + def test_request_output_edge_cases(): """测试RequestOutput类的边缘情况""" output = RequestOutput() diff --git a/tests/UT/openicl/icl_inferencer/output_handler/test_gen_inferencer_output_handler.py b/tests/UT/openicl/icl_inferencer/output_handler/test_gen_inferencer_output_handler.py index b50b5173..aec442c0 100644 --- a/tests/UT/openicl/icl_inferencer/output_handler/test_gen_inferencer_output_handler.py +++ b/tests/UT/openicl/icl_inferencer/output_handler/test_gen_inferencer_output_handler.py @@ -249,16 +249,16 @@ def test_get_result_accuracy_mode_failure_no_error_info(self): """Test get_result accuracy mode failure without error_info (lines 82-90)""" handler = GenInferencerOutputHandler(perf_mode=False) conn = sqlite3.connect(":memory:") - + output = Output() output.success = False output.uuid = "test_uuid" output.get_prediction = mock.Mock(return_value="") - + # Delete error_info if it exists (Output may have it as default attribute) if hasattr(output, "error_info"): delattr(output, "error_info") - + # Mock get_prediction_result to return failed result handler.get_prediction_result = mock.Mock(return_value={ "success": False, @@ -267,15 +267,73 @@ def test_get_result_accuracy_mode_failure_no_error_info(self): "origin_prompt": "input", "gold": "gold" }) - + result = handler.get_result(conn, "data_abbr", "input", output, "gold") self.assertEqual(result["success"], False) self.assertFalse(handler.all_success) # Should not have error_info when it doesn't exist self.assertNotIn("error_info", result) - + conn.close() + def test_get_prediction_result_with_tokens(self): + """测试 get_prediction_result 始终输出 input_tokens 和 output_tokens""" + handler = GenInferencerOutputHandler(perf_mode=False) + + output = Output() + output.success = True + output.uuid = "test_uuid" + output.input_tokens = 100 + output.output_tokens = 50 + output.get_prediction = mock.Mock(return_value="predicted_text") + + result = handler.get_prediction_result(output, "gold", "input", "data_abbr") + + self.assertEqual(result["input_tokens"], 100) + self.assertEqual(result["output_tokens"], 50) + + def test_get_prediction_result_with_logprobs(self): + """测试 get_prediction_result 在有 logprobs 数据时输出""" + handler = GenInferencerOutputHandler(perf_mode=False) + + output = Output() + output.success = True + output.uuid = "test_uuid" + output.input_tokens = 100 + output.output_tokens = 50 + output.origin_logprobs = [{"token": "B", "logprob": -0.5, "top_logprobs": [{"token": "B", "logprob": -0.5}]}] + output.get_prediction = mock.Mock(return_value="predicted_text") + + result = handler.get_prediction_result(output, "gold", "input", "data_abbr") + + self.assertEqual(result["origin_logprobs"], [{"token": "B", "logprob": -0.5, "top_logprobs": [{"token": "B", "logprob": -0.5}]}]) + + def test_get_prediction_result_without_logprobs(self): + """测试 get_prediction_result 在无 logprobs 数据时不输出 logprobs 字段""" + handler = GenInferencerOutputHandler(perf_mode=False) + + output = Output() + output.success = True + output.uuid = "test_uuid" + output.input_tokens = 100 + output.output_tokens = 50 + output.origin_logprobs = [] + output.get_prediction = mock.Mock(return_value="predicted_text") + + result = handler.get_prediction_result(output, "gold", "input", "data_abbr") + + self.assertNotIn("origin_logprobs", result) + + def test_get_prediction_result_string_output_no_tokens(self): + """测试 get_prediction_result 在 string output 时不输出 token 字段""" + handler = GenInferencerOutputHandler(perf_mode=False) + + result = handler.get_prediction_result("predicted_text", "gold", "input", "data_abbr") + + self.assertNotIn("input_tokens", result) + self.assertNotIn("output_tokens", result) + self.assertNotIn("origin_logprobs", result) + if __name__ == '__main__': unittest.main() From 52001d2e7aac360ce7d6e308ccdec76cf2512e4a Mon Sep 17 00:00:00 2001 From: ivanbao9783 Date: Wed, 26 Aug 2026 11:23:30 +0800 Subject: [PATCH 07/25] feat(simulator): add /metrics endpoint for spec-decode exception testing (#472) --- tools/infer_serve_simulator/README.md | 28 ++++++ .../infer_serve_simulator/api/api_config.yaml | 21 ++++- tools/infer_serve_simulator/api/base_api.py | 2 +- .../infer_serve_simulator/api/metrics_api.py | 86 +++++++++++++++++++ tools/infer_serve_simulator/flask_service.py | 14 +++ 5 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 tools/infer_serve_simulator/api/metrics_api.py diff --git a/tools/infer_serve_simulator/README.md b/tools/infer_serve_simulator/README.md index 87d9c0e7..6c42b766 100644 --- a/tools/infer_serve_simulator/README.md +++ b/tools/infer_serve_simulator/README.md @@ -32,8 +32,35 @@ random_dataset: random_content: False # 字符是否随机，不随机默认用'A'构造 min_tokens: 10 # 最小长度 tokens_per_chunk: 2 # mtp场景下的chunk大小 + +# /metrics 端点配置（用于 spec-decode 异常场景测试） +metrics: + mode: static_counters # error_http | no_spec | static_counters + error_code: 503 # mode=error_http 时返回的 HTTP 状态码 + counters: # mode=static_counters 时返回的固定计数器值 + num_drafts: 100 + num_draft_tokens: 500 + num_accepted_tokens: 450 + accepted_per_pos: # 每个 draft position 的采纳 token 数 + 0: 450 + 1: 400 + 2: 350 + 3: 300 + 4: 250 ``` +### /metrics 端点说明 + +模拟器额外提供 `/metrics` 端点（Prometheus 格式），用于配合 AISBench 的 `--spec-decode` 功能测试 spec-decode 指标采集的异常场景。通过 `metrics.mode` 配置项控制端点行为： + +| mode | 行为 | 触发的 spec-decode 异常分支 | +|------|------|---------------------------| +| `error_http` | 返回指定 HTTP 错误码（默认503） | fetcher: `Metrics endpoint returned HTTP {code}` | +| `no_spec` | 返回200但不含 spec_decode 计数器 | snapshot: `No spec decode metrics found on server` | +| `static_counters` | 返回200 + 固定 spec_decode 计数器（before==after → delta=0） | calculator: `No spec decode activity detected during benchmark window` | + +切换模式时修改 `api_config.yaml` 后重启服务即可。 + ## 支持的API列表 |api类型|endpoint子服务|备注| |----|-----|----| @@ -47,6 +74,7 @@ random_dataset: |triton stream|v2/models/qwen/generate_stream|模型名称为`qwen`| |mindie origin text|infer|| |mindie origin stream|infer|| +|prometheus metrics|metrics|用于 spec-decode 异常场景测试，行为由 `metrics.mode` 配置控制| ## 启动服务 ```shell diff --git a/tools/infer_serve_simulator/api/api_config.yaml b/tools/infer_serve_simulator/api/api_config.yaml index a379d95a..a8b57b10 100644 --- a/tools/infer_serve_simulator/api/api_config.yaml +++ b/tools/infer_serve_simulator/api/api_config.yaml @@ -11,4 +11,23 @@ text_latency: random_dataset: random_content: False # 字符是否随机，不随机默认用'A'构造 min_tokens: 10 # 最小长度 - tokens_per_chunk: 2 # mtp场景 \ No newline at end of file + tokens_per_chunk: 2 # mtp场景 + +# /metrics 端点配置（用于 spec-decode 异常场景测试） +# mode 可选值: +# error_http - 返回指定 HTTP 错误码（由 error_code 指定），模拟 /metrics 端点不可达或服务端错误 +# no_spec - 返回 200 但不含 vllm:spec_decode 计数器，模拟服务端未开启投机推理 +# static_counters - 返回 200 + spec_decode 计数器，但值固定不变（before==after → delta=0），模拟评测窗口内无投机活动 +metrics: + mode: static_counters # error_http | no_spec | static_counters + error_code: 503 # mode=error_http 时返回的 HTTP 状态码 + counters: # mode=static_counters 时返回的固定计数器值 + num_drafts: 100 + num_draft_tokens: 500 + num_accepted_tokens: 450 + accepted_per_pos: # 每个 draft position 的采纳 token 数 + 0: 450 + 1: 400 + 2: 350 + 3: 300 + 4: 250 \ No newline at end of file diff --git a/tools/infer_serve_simulator/api/base_api.py b/tools/infer_serve_simulator/api/base_api.py index 091775a4..74c0ded3 100644 --- a/tools/infer_serve_simulator/api/base_api.py +++ b/tools/infer_serve_simulator/api/base_api.py @@ -15,7 +15,7 @@ def load_yaml_config(): config_path = os.path.join(cur_dir, 'api_config.yaml') if not os.path.exists(config_path): raise FileExistsError(f"Can't find api_config.yaml in {cur_dir}") - with open(config_path, 'r') as file: + with open(config_path, 'r', encoding='utf-8') as file: return yaml.safe_load(file) class BaseAPI(ABC): diff --git a/tools/infer_serve_simulator/api/metrics_api.py b/tools/infer_serve_simulator/api/metrics_api.py new file mode 100644 index 00000000..4e069aff --- /dev/null +++ b/tools/infer_serve_simulator/api/metrics_api.py @@ -0,0 +1,86 @@ +from flask import Response, jsonify +from api.base_api import load_yaml_config + + +class MetricsAPI: + """处理 /metrics 端点请求，用于 spec-decode 异常场景测试。 + + 通过 api_config.yaml 的 metrics.mode 配置项控制行为： + - error_http: 返回指定 HTTP 错误码 + - no_spec: 返回 200 但不含 spec_decode 计数器 + - static_counters: 返回 200 + 固定不变的 spec_decode 计数器 + """ + + # Prometheus 文本格式中其他常见的 vllm 计数器（用于 no_spec 模式填充） + _OTHER_VLLM_METRICS = [ + '# HELP vllm:num_requests_total Number of requests processed.', + '# TYPE vllm:num_requests_total counter', + 'vllm:num_requests_total 42.0', + '# HELP vllm:cache_config_info Cache configuration info.', + '# TYPE vllm:cache_config_info gauge', + 'vllm:cache_config_info{block_size="16"} 1.0', + ] + + def __init__(self): + self.config = load_yaml_config() + self.metrics_conf = self.config.get("metrics", {}) + + def handle_metrics(self): + """根据配置返回 /metrics 响应。""" + mode = self.metrics_conf.get("mode", "static_counters") + + if mode == "error_http": + return self._error_http_response() + elif mode == "no_spec": + return self._no_spec_response() + elif mode == "static_counters": + return self._static_counters_response() + else: + # 未知 mode，当作 no_spec 处理 + return self._no_spec_response() + + def _error_http_response(self): + """返回指定 HTTP 错误码，模拟 /metrics 不可达或服务端错误。""" + error_code = int(self.metrics_conf.get("error_code", 503)) + return Response(f"metrics endpoint error: {error_code}", status=error_code) + + def _no_spec_response(self): + """返回 200 + 其他 vllm 计数器，但不含 spec_decode 计数器。 + + 触发 fetcher → snapshot.py 的 parse_spec_decode_metrics 返回 None， + 最终输出 "No spec decode metrics found on server"。 + """ + text = "\n".join(self._OTHER_VLLM_METRICS) + "\n" + return Response(text, status=200, mimetype="text/plain") + + def _static_counters_response(self): + """返回 200 + 固定 spec_decode 计数器，值不随请求变化。 + + before/after 快照拿到相同值 → delta_draft_tokens=0 + → calculator.py 返回 None + → 输出 "No spec decode activity detected during benchmark window" + """ + counters = self.metrics_conf.get("counters", {}) + lines = [ + '# HELP vllm:spec_decode_num_drafts_total Number of draft-and-verify cycles.', + '# TYPE vllm:spec_decode_num_drafts_total counter', + f'vllm:spec_decode_num_drafts_total {counters.get("num_drafts", 100)}.0', + '# HELP vllm:spec_decode_num_draft_tokens_total Number of draft tokens proposed.', + '# TYPE vllm:spec_decode_num_draft_tokens_total counter', + f'vllm:spec_decode_num_draft_tokens_total {counters.get("num_draft_tokens", 500)}.0', + '# HELP vllm:spec_decode_num_accepted_tokens_total Number of accepted tokens.', + '# TYPE vllm:spec_decode_num_accepted_tokens_total counter', + f'vllm:spec_decode_num_accepted_tokens_total {counters.get("num_accepted_tokens", 450)}.0', + ] + + accepted_per_pos = counters.get("accepted_per_pos", {}) + if accepted_per_pos: + lines.append('# HELP vllm:spec_decode_num_accepted_tokens_per_pos_total Accepted tokens per position.') + lines.append('# TYPE vllm:spec_decode_num_accepted_tokens_per_pos_total counter') + for pos in sorted(accepted_per_pos.keys()): + lines.append( + f'vllm:spec_decode_num_accepted_tokens_per_pos_total{{position="{pos}"}} {accepted_per_pos[pos]}.0' + ) + + text = "\n".join(lines) + "\n" + return Response(text, status=200, mimetype="text/plain") diff --git a/tools/infer_serve_simulator/flask_service.py b/tools/infer_serve_simulator/flask_service.py index 2cc575a4..35b1aa24 100644 --- a/tools/infer_serve_simulator/flask_service.py +++ b/tools/infer_serve_simulator/flask_service.py @@ -3,6 +3,7 @@ import time from flask import Flask, jsonify, request, Response from api import OpenAIChatAPI, OpenAIAPI, TGIAPI, TritonAPI, MindIEOriginAPI, MindIEOriginTokenAPI +from api.metrics_api import MetricsAPI import threading app = Flask(__name__) @@ -97,5 +98,18 @@ def mindie_origin_token_api(): # 处理服务的主函数 return api.generate_stream(req_data) +@app.route('/metrics', methods=['GET']) +def metrics_api(): + """Prometheus /metrics 端点，用于 spec-decode 异常场景测试。 + + 行为由 api_config.yaml 的 metrics.mode 配置项控制： + - error_http: 返回指定 HTTP 错误码 + - no_spec: 返回 200 但不含 spec_decode 计数器 + - static_counters: 返回 200 + 固定 spec_decode 计数器 + """ + api = MetricsAPI() + return api.handle_metrics() + + if __name__ == '__main__': # 单进程用python3 直接执行 app.run(host='xx.xx.xx.xx', port=5101, threaded=True) # host为IP，port为端口号 \ No newline at end of file From b1fb0143387f279d47a6156582be87033906b62a Mon Sep 17 00:00:00 2001 From: Hanye <1037452625@qq.com> Date: Wed, 26 Aug 2026 15:14:25 +0800 Subject: [PATCH 08/25] [Docs] Add recommand instruction for custom configs (#349) * add mini dataset doc * review fix * multilingual mini fix * add upload_pip.sh * add testpypi only * add testpypi only * add testpypi only * add testpypi only * add testpypi only * add testpypi only * update install docs * ai fix custom cfg sample * add quick_start custom cfg sample * disable summarizer check * add import * add two column * add two column * fix add two column * add two column v1 * fix add two column v1 * fix add two column v1 * fix add two column v1 * custom cfg doc fix in models.md * custom cfg doc fix in summarizer.md * accuracy_benchmark doc * base tutorials docs add custom_cfg * add en docs * add test_range in docs * add test_range in docs * docs fix * doc fix * doc fix 2 * remove no need summarzier check * delete unused UT * Runner fix * Api fix * fix * doc table fix --------- Co-authored-by: SJTUyh --- README.md | 108 +- README_en.md | 90 +- ais_bench/benchmark/cli/config_manager.py | 1 - .../configs/datasets/ARC_c/README.md | 10 +- .../configs/datasets/ARC_c/README_en.md | 10 +- .../configs/datasets/ARC_e/README.md | 10 +- .../configs/datasets/ARC_e/README_en.md | 10 +- .../configs/datasets/FewCLUE_bustm/README.md | 6 +- .../datasets/FewCLUE_bustm/README_en.md | 6 +- .../configs/datasets/FewCLUE_chid/README.md | 6 +- .../datasets/FewCLUE_chid/README_en.md | 6 +- .../datasets/FewCLUE_cluewsc/README.md | 6 +- .../datasets/FewCLUE_cluewsc/README_en.md | 6 +- .../configs/datasets/FewCLUE_csl/README.md | 6 +- .../configs/datasets/FewCLUE_csl/README_en.md | 6 +- .../datasets/FewCLUE_eprstmt/README.md | 6 +- .../datasets/FewCLUE_eprstmt/README_en.md | 6 +- .../configs/datasets/FewCLUE_tnews/README.md | 6 +- .../datasets/FewCLUE_tnews/README_en.md | 6 +- .../datasets/SuperGLUE_BoolQ/README.md | 12 +- .../datasets/SuperGLUE_BoolQ/README_en.md | 12 +- .../benchmark/configs/datasets/Xsum/README.md | 8 +- .../configs/datasets/Xsum/README_en.md | 8 +- .../configs/datasets/agieval/README.md | 6 +- .../configs/datasets/agieval/README_en.md | 6 +- .../configs/datasets/aime2024/README.md | 8 +- .../configs/datasets/aime2024/README_en.md | 8 +- .../configs/datasets/aime2025/README.md | 8 +- .../configs/datasets/aime2025/README_en.md | 8 +- .../configs/datasets/aime2026/README.md | 8 +- .../configs/datasets/aime2026/README_en.md | 8 +- .../benchmark/configs/datasets/bbh/README.md | 6 +- .../configs/datasets/bbh/README_en.md | 6 +- .../configs/datasets/ceval/README.md | 12 +- .../configs/datasets/ceval/README_en.md | 12 +- .../configs/datasets/cmmlu/README.md | 10 +- .../configs/datasets/cmmlu/README_en.md | 10 +- .../configs/datasets/dapo_math/README.md | 8 +- .../configs/datasets/dapo_math/README_en.md | 8 +- .../benchmark/configs/datasets/demo/README.md | 8 +- .../configs/datasets/demo/README_en.md | 8 +- .../configs/datasets/docvqa/README.md | 6 +- .../configs/datasets/docvqa/README_en.md | 6 +- .../benchmark/configs/datasets/drop/README.md | 8 +- .../configs/datasets/drop/README_en.md | 8 +- .../benchmark/configs/datasets/gpqa/README.md | 10 +- .../configs/datasets/gpqa/README_en.md | 10 +- .../configs/datasets/gsm8k/README.md | 14 +- .../configs/datasets/gsm8k/README_en.md | 14 +- .../configs/datasets/hellaswag/README.md | 10 +- .../configs/datasets/hellaswag/README_en.md | 10 +- .../benchmark/configs/datasets/hle/README.md | 8 +- .../configs/datasets/hle/README_en.md | 8 +- .../configs/datasets/humaneval/README.md | 6 +- .../configs/datasets/humaneval/README_en.md | 6 +- .../configs/datasets/humanevalx/README.md | 6 +- .../configs/datasets/humanevalx/README_en.md | 6 +- .../configs/datasets/ifeval/README.md | 6 +- .../configs/datasets/ifeval/README_en.md | 6 +- .../configs/datasets/infovqa/README.md | 6 +- .../configs/datasets/infovqa/README_en.md | 6 +- .../configs/datasets/lambada/README.md | 8 +- .../configs/datasets/lambada/README_en.md | 8 +- .../configs/datasets/lcsts/README.md | 8 +- .../configs/datasets/lcsts/README_en.md | 8 +- .../configs/datasets/livecodebench/README.md | 10 +- .../datasets/livecodebench/README_en.md | 10 +- .../configs/datasets/longbench/README.md | 8 +- .../configs/datasets/longbench/README_en.md | 8 +- .../configs/datasets/longbenchv2/README.md | 6 +- .../configs/datasets/longbenchv2/README_en.md | 6 +- .../benchmark/configs/datasets/math/README.md | 10 +- .../configs/datasets/math/README_en.md | 10 +- .../configs/datasets/mathvision/README.md | 6 +- .../configs/datasets/mathvision/README_en.md | 6 +- .../benchmark/configs/datasets/mbpp/README.md | 8 +- .../configs/datasets/mbpp/README_en.md | 8 +- .../benchmark/configs/datasets/mgsm/README.md | 8 +- .../configs/datasets/mgsm/README_en.md | 8 +- .../benchmark/configs/datasets/mmlu/README.md | 12 +- .../configs/datasets/mmlu/README_en.md | 12 +- .../configs/datasets/mmlu_pro/README.md | 8 +- .../configs/datasets/mmlu_pro/README_en.md | 8 +- .../benchmark/configs/datasets/mmmu/README.md | 6 +- .../configs/datasets/mmmu/README_en.md | 6 +- .../configs/datasets/mmmu_pro/README.md | 12 +- .../configs/datasets/mmmu_pro/README_en.md | 12 +- .../configs/datasets/mmstar/README.md | 8 +- .../configs/datasets/mmstar/README_en.md | 8 +- .../configs/datasets/mooncake_trace/README.md | 6 +- .../datasets/mooncake_trace/README_en.md | 6 +- .../configs/datasets/mtbench/README.md | 6 +- .../configs/datasets/mtbench/README_en.md | 6 +- .../configs/datasets/needlebench_v2/README.md | 64 +- .../datasets/needlebench_v2/README_en.md | 64 +- .../configs/datasets/ocrbench_v2/README.md | 6 +- .../configs/datasets/ocrbench_v2/README_en.md | 6 +- .../configs/datasets/omnidocbench/README.md | 6 +- .../datasets/omnidocbench/README_en.md | 6 +- .../benchmark/configs/datasets/piqa/README.md | 10 +- .../configs/datasets/piqa/README_en.md | 10 +- .../benchmark/configs/datasets/race/README.md | 14 +- .../configs/datasets/race/README_en.md | 14 +- .../configs/datasets/realworldqa/README.md | 8 +- .../configs/datasets/realworldqa/README_en.md | 8 +- .../configs/datasets/refcoco/README.md | 8 +- .../configs/datasets/refcoco/README_en.md | 8 +- .../configs/datasets/refcoco_plus/README.md | 8 +- .../datasets/refcoco_plus/README_en.md | 8 +- .../configs/datasets/refcocog/README.md | 8 +- .../configs/datasets/refcocog/README_en.md | 8 +- .../configs/datasets/sharegpt/README.md | 6 +- .../configs/datasets/sharegpt/README_en.md | 6 +- .../benchmark/configs/datasets/siqa/README.md | 8 +- .../configs/datasets/siqa/README_en.md | 8 +- .../configs/datasets/textvqa/README.md | 10 +- .../configs/datasets/textvqa/README_en.md | 10 +- .../configs/datasets/triviaqa/README.md | 6 +- .../configs/datasets/triviaqa/README_en.md | 6 +- .../configs/datasets/videobench/README.md | 8 +- .../configs/datasets/videobench/README_en.md | 8 +- .../configs/datasets/videomme/README.md | 8 +- .../configs/datasets/videomme/README_en.md | 8 +- .../configs/datasets/vocalsound/README.md | 8 +- .../configs/datasets/vocalsound/README_en.md | 8 +- .../configs/datasets/winogrande/README.md | 8 +- .../configs/datasets/winogrande/README_en.md | 8 +- .../accuracy_benchmark/ceval_merge_en.py | 23 + .../accuracy_benchmark/ceval_merge_zh_cn.py | 23 + .../accuracy_benchmark/fixed_prompts_en.py | 23 + .../accuracy_benchmark/fixed_prompts_zh_cn.py | 23 + .../inference_re_eval_en.py | 28 + .../inference_re_eval_zh_cn.py | 28 + .../accuracy_benchmark/multi_repeat_en.py | 30 + .../accuracy_benchmark/multi_repeat_zh_cn.py | 30 + .../accuracy_benchmark/multi_task_en.py | 33 + .../multi_task_parallel_en.py | 33 + .../multi_task_parallel_zh_cn.py | 33 + .../multi_task_resume_partial_en.py | 24 + .../multi_task_resume_partial_zh_cn.py | 24 + .../accuracy_benchmark/multi_task_zh_cn.py | 33 + .../accuracy_benchmark/single_task_en.py | 26 + .../accuracy_benchmark/single_task_zh_cn.py | 26 + .../ceval_merge_en.py | 38 + .../ceval_merge_zh_cn.py | 38 + .../inference_re_eval_en.py | 43 + .../inference_re_eval_zh_cn.py | 43 + .../accuracy_benchmark_local/multi_task_en.py | 41 + .../multi_task_zh_cn.py | 41 + .../single_task_en.py | 38 + .../single_task_zh_cn.py | 38 + .../infer_mindie_stream_api_general.py | 8 +- .../api_examples/infer_vllm_api_general.py | 10 +- .../infer_vllm_api_general_chat.py | 8 +- ...nfer_vllm_api_multi_model_multi_dataset.py | 15 + .../api_examples/infer_vllm_api_old.py | 46 - .../infer_vllm_api_stream_chat.py | 10 +- .../infer_vllm_api_with_judge_model.py | 45 + ...llm_api_with_model_dataset_combinations.py | 20 + .../perf_vllm_api_custom_dataset.py | 65 ++ .../api_examples/perf_vllm_api_multiturn.py | 45 + .../perf_vllm_api_rps_distribution.py | 40 + .../perf_vllm_api_stable_stage.py | 35 + .../api_examples/perf_vllm_api_synthetic.py | 48 + .../configs/hf_example/infer_hf_base_model.py | 8 +- .../configs/hf_example/infer_hf_chat_model.py | 8 +- .../infer_hf_multi_model_multi_dataset.py | 46 + .../lmm_example/infer_lmm_multi_dataset.py | 12 + ais_bench/configs/model_api_test_en.py | 36 + ais_bench/configs/model_api_test_zh_cn.py | 36 + .../fixed_prompts_zh_cn.py | 26 + .../multi_task_synthetic_zh_cn.py | 79 ++ .../performance_benchmark/multi_task_zh_cn.py | 33 + .../perf_recalculate_zh_cn.py | 37 + .../performance_fixed_request.py | 35 + .../performance_multi_dataset.py | 31 + .../performance_multi_model.py | 31 + .../performance_multi_rate.py | 27 + .../performance_qwen2_7b_sharegpt.py | 20 + .../performance_re_eval.py | 40 + .../performance_seq_combinations.py | 57 ++ .../performance_synthetic.py | 41 + .../single_task_zh_cn.py | 26 + .../synthetic_gen_string_zh_cn.py | 49 + docs/requirements.txt | 3 +- .../advanced_tutorials/custom_dataset.md | 7 +- .../judge_model_evaluate.md | 12 +- .../multimodal_benchmark.md | 3 + .../advanced_tutorials/multiturn_benchmark.md | 7 +- .../advanced_tutorials/rps_distribution.md | 3 + .../advanced_tutorials/run_custom_config.md | 774 +++++++++++++- .../advanced_tutorials/stable_stage.md | 7 +- .../advanced_tutorials/synthetic_dataset.md | 20 +- .../base_tutorials/all_params/cli_args.md | 15 +- .../base_tutorials/all_params/models.md | 48 +- .../base_tutorials/all_params/summarizer.md | 12 +- .../scenes_intro/accuracy_benchmark.md | 478 ++++++++- .../scenes_intro/accuracy_benchmark_local.md | 197 +++- .../base_tutorials/scenes_intro/home.md | 6 +- .../scenes_intro/performance_benchmark.md | 944 +++++++++++++----- .../best_practices/practice_ascend.md | 2 + .../best_practices/practice_nvidia.md | 3 +- .../replicate_llm_datasets_accuracy.md | 169 ++-- docs/source_en/conf.py | 2 + .../extended_benchmark/agent/harbor_bench.md | 4 +- .../extended_benchmark/agent/swe_bench.md | 2 + .../extended_benchmark/agent/swe_bench_pro.md | 2 + .../extended_benchmark/agent/tau2_bench.md | 8 +- .../lmm_generate/gedit_bench.md | 4 +- .../extended_benchmark/lmm_generate/vbench.md | 8 +- docs/source_en/faqs/error_codes.md | 2 +- docs/source_en/get_started/quick_start.md | 136 ++- docs/source_en/index.rst | 3 +- .../advanced_tutorials/custom_dataset.md | 6 +- .../judge_model_evaluate.md | 7 +- .../multimodal_benchmark.md | 2 + .../advanced_tutorials/multiturn_benchmark.md | 6 + .../advanced_tutorials/rps_distribution.md | 4 + .../advanced_tutorials/run_custom_config.md | 798 ++++++++++++++- .../advanced_tutorials/stable_stage.md | 6 +- .../advanced_tutorials/synthetic_dataset.md | 4 + .../base_tutorials/all_params/cli_args.md | 7 +- .../base_tutorials/all_params/models.md | 44 +- .../base_tutorials/all_params/summarizer.md | 12 +- .../scenes_intro/accuracy_benchmark.md | 455 ++++++++- .../scenes_intro/accuracy_benchmark_local.md | 199 +++- .../base_tutorials/scenes_intro/home.md | 6 +- .../scenes_intro/performance_benchmark.md | 454 ++++++++- .../best_practices/practice_ascend.md | 2 + .../best_practices/practice_nvidia.md | 4 +- .../replicate_llm_datasets_accuracy.md | 3 + docs/source_zh_cn/conf.py | 2 + .../extended_benchmark/agent/harbor_bench.md | 2 + .../extended_benchmark/agent/swe_bench.md | 2 + .../extended_benchmark/agent/swe_bench_pro.md | 2 + .../extended_benchmark/agent/tau2_bench.md | 2 + .../lmm_generate/gedit_bench.md | 4 +- .../extended_benchmark/lmm_generate/vbench.md | 8 +- docs/source_zh_cn/faqs/error_codes.md | 2 +- docs/source_zh_cn/get_started/quick_start.md | 160 ++- docs/source_zh_cn/index.rst | 2 +- plugin_examples/README.md | 8 +- .../config_example/perf_example.py | 8 +- tests/UT/cli/test_config_manager.py | 34 - 244 files changed, 6948 insertions(+), 1328 deletions(-) create mode 100644 ais_bench/configs/accuracy_benchmark/ceval_merge_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_repeat_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_task_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark/single_task_en.py create mode 100644 ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/multi_task_en.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/single_task_en.py create mode 100644 ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py create mode 100644 ais_bench/configs/api_examples/infer_vllm_api_multi_model_multi_dataset.py delete mode 100644 ais_bench/configs/api_examples/infer_vllm_api_old.py create mode 100644 ais_bench/configs/api_examples/infer_vllm_api_with_judge_model.py create mode 100644 ais_bench/configs/api_examples/infer_vllm_api_with_model_dataset_combinations.py create mode 100644 ais_bench/configs/api_examples/perf_vllm_api_custom_dataset.py create mode 100644 ais_bench/configs/api_examples/perf_vllm_api_multiturn.py create mode 100644 ais_bench/configs/api_examples/perf_vllm_api_rps_distribution.py create mode 100644 ais_bench/configs/api_examples/perf_vllm_api_stable_stage.py create mode 100644 ais_bench/configs/api_examples/perf_vllm_api_synthetic.py create mode 100644 ais_bench/configs/hf_example/infer_hf_multi_model_multi_dataset.py create mode 100644 ais_bench/configs/lmm_example/infer_lmm_multi_dataset.py create mode 100644 ais_bench/configs/model_api_test_en.py create mode 100644 ais_bench/configs/model_api_test_zh_cn.py create mode 100644 ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py create mode 100644 ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py create mode 100644 ais_bench/configs/performance_benchmark/multi_task_zh_cn.py create mode 100644 ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py create mode 100644 ais_bench/configs/performance_benchmark/performance_fixed_request.py create mode 100644 ais_bench/configs/performance_benchmark/performance_multi_dataset.py create mode 100644 ais_bench/configs/performance_benchmark/performance_multi_model.py create mode 100644 ais_bench/configs/performance_benchmark/performance_multi_rate.py create mode 100644 ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py create mode 100644 ais_bench/configs/performance_benchmark/performance_re_eval.py create mode 100644 ais_bench/configs/performance_benchmark/performance_seq_combinations.py create mode 100644 ais_bench/configs/performance_benchmark/performance_synthetic.py create mode 100644 ais_bench/configs/performance_benchmark/single_task_zh_cn.py create mode 100644 ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py diff --git a/README.md b/README.md index 451bda82..9ae700a1 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,8 @@ - 性能评测场景使用[自定义数据集](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/advanced_tutorials/custom_dataset.html)，支持按请求粒度指定最大输出长度！🔥🔥🔥 - **\[2025.6.19]** 支持📚[性能评测结果可视化](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/results_intro/performance_visualization.html)，辅助定位推理服务性能瓶颈！🔥🔥🔥 - **\[2025.6.12]** 支持[textvqa](ais_bench/benchmark/configs/datasets/textvqa/README.md)、[videobench](ais_bench/benchmark/configs/datasets/videobench/README.md)和[vocalsound](ais_bench/benchmark/configs/datasets/vocalsound/README.md)等多模态数据集的精度和性能评测！🔥🔥🔥 -- **\[2025.6.6]** AISBench支持稳态性能评测，获取系统真实最佳性能，参考📚 [服务化稳定状态性能测试](doc/users_guide/stable_stage.md)进行快速上手! 🔥🔥🔥 -- **\[2025.5.16]** 支持3W+高并发服务化性能评测，📚 [性能指标](doc/users_guide/performance_metric.md)对齐🔗 [vllm benchmark](https://github.com/vllm-project/vllm/tree/main/benchmarks)，参考📚 [服务化性能测评指南](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/scenes_intro/performance_benchmark.html)了解详情！🔥🔥🔥 +- **\[2025.6.6]** AISBench支持稳态性能评测，获取系统真实最佳性能，参考📚 [服务化稳定状态性能测试](docs/source_zh_cn/advanced_tutorials/stable_stage.md)进行快速上手! 🔥🔥🔥 +- **\[2025.5.16]** 支持3W+高并发服务化性能评测，📚 [性能指标](docs/source_zh_cn/base_tutorials/results_intro/performance_metric.md)对齐🔗 [vllm benchmark](https://github.com/vllm-project/vllm/tree/main/benchmarks)，参考📚 [服务化性能测评指南](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/scenes_intro/performance_benchmark.html)了解详情！🔥🔥🔥 - **\[2025.4.30]** 精度评测支持断点续测和失败用例重测，大幅提高精度评测鲁棒性，参考📚 [中断续测 & 失败用例重测](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/scenes_intro/accuracy_benchmark.html#id10)进行快速上手! 🔥🔥🔥 ## 🌏 简介 @@ -143,15 +143,21 @@ pip3 install -r requirements/datasets/ocrbench_v2.txt ### 📦 安装方式-一键安装（备选） AISBench 也提供了一键安装方式，适用于基于预置配置文件的快速体验和评估场景，请确保安装环境联网。 + - 基本功能的安装命令如下： + ```shell pip3 install ais_bench_benchmark ``` + - 全量功能的安装命令如下： + ```shell pip3 install ais_bench_benchmark[full] ``` +如需进一步配置、使用 CLI 或 Python 脚本发起评测任务，请参考[快速入门指南](#快速入门)。 + ## ❌ 工具卸载 如需卸载 AISBench Benchmark，可执行以下命令： @@ -162,9 +168,85 @@ pip3 uninstall ais_bench_benchmark ## 🚀 快速入门 -### 命令含义 +### 运行命令前置准备 + +- 需要准备支持`v1/chat/completions`子服务的推理服务，可以参考🔗 [VLLM启动OpenAI 兼容服务器](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server)启动推理服务 +- 需要准备gsm8k数据集，可以从🔗 [opencompass + 提供的gsm8k数据集压缩包](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip)下载。将解压后的`gsm8k/`文件夹部署到AISBench评测工具根路径下的`ais_bench/datasets`文件夹下。 + +### 启动测评（两种方式任选其一） + +| ⭐ 推荐：使用自定义配置文件 | 备选：使用命令行参数(原快速入门方式) | +| :------------------ | :------------------------------ | +| 修改一个文件，集中管理所有配置，在任意路径写配置 | 通过 `--models` `--datasets` 参数指定 | +| 一次编写，多次复用 | 每次运行需输入完整命令 | +| 支持 Python 全部语法，灵活扩展 | 仅支持笛卡尔积组合 | + +**⭐ 推荐：使用自定义配置文件** + +AISBench 提供了预置的自定义配置文件 [model\_api\_test\_zh\_cn.py](ais_bench/configs/model_api_test_zh_cn.py)，将常见的推理服务化测试配置（模型选择、服务地址、端口、生成参数等）集中在一个文件中，无需分别查找和修改多个配置文件。该文件本质上是 Python 脚本，支持所有 Python 语法，你可以自由扩展。 + +打开 `ais_bench/configs/model_api_test_zh_cn.py`，根据实际情况修改以下配置（如果是`pip3 install ais_bench_benchmark`方式直接安装工具，可以在任意路径自行创建`model_api_test_zh_cn.py`，将以下配置内容写入该文件）： + +```python +from mmengine.config import read_base + +with read_base(): +# 模型任务，选择其中一个，其他模型任务参考：https://ais-bench-benchmark-rf.readthedocs.io/zh-cn/latest/base_tutorials/all_params/models.html 获取更多模型任务 + # vllm_api_general 是基础模型，仅支持文本生成 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + # vllm_api_general_chat 是对话模型，支持对话 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + # vllm_api_stream_chat 是流式对话模型，支持流式对话 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + # vllm_api_general_stream 是流式模型，支持流式生成 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + +# 数据集任务，参考：https://ais-bench-benchmark-rf.readthedocs.io/zh-cn/latest/get_started/datasets.html 获取更多数据集任务 + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = vllm_api_general_chat + +models[0]["path"] = "" # 指定模型序列化词表文件的绝对路径（精度测试场景一般不需要配置） +models[0]["model"] = "" # 指定服务端加载的模型名称，根据 VLLM 推理服务实际拉取的模型名称配置（配置为空字符串则自动获取） +models[0]["request_rate"] = 0 # 请求发送频率：每 1/request_rate 秒向服务端发送 1 条请求；小于 0.001 时一次性发送所有请求 +models[0]["api_key"] = "" # 自定义 API key，默认为空字符串 +models[0]["host_ip"] = "localhost" # 指定推理服务的 IP +models[0]["host_port"] = 8080 # 指定推理服务的端口 +models[0]["url"] = "" # 自定义访问推理服务的 URL 路径（当基础 URL 不是 http://host_ip:host_port 的组合时需要配置；配置后 host_ip 和 host_port 将被忽略） +models[0]["max_out_len"] = 512 # 推理服务输出的最大 token 数 +models[0]["batch_size"] = 1 # 发送请求的最大并发数 +models[0]["trust_remote_code"] = False # tokenizer 是否信任远程代码，默认为 False +models[0]["generation_kwargs"] = dict( # 模型推理参数，参考 VLLM 文档配置；AISBench 评测工具不做处理，直接附加到发送的请求中 + temperature=0.01, + ignore_eos=False, +) + +# datasets[0]["path"] = ais_bench/datasets/gsm8k # 指定数据集目录的绝对路径（精度测试场景需要配置） + +work_dir = 'outputs/default/' # 指定任务结果和日志的保存工作目录（默认为 outputs/default/） + +``` +> 💡 配置文件中已预置了常用模型类型的导入（`vllm_api_general`、`vllm_api_general_chat`、`vllm_api_stream_chat`、`vllm_api_general_stream`），只需取消/修改注释即可切换。更多自定义配置文件的用法请参考 📚 [自定义配置文件运行AISBench](./docs/source_zh_cn/advanced_tutorials/run_custom_config.md)。 -AISBench命令执行的单个或多个评测任务是由模型任务（单个或多个）、数据集任务（单个或多个）和结果呈现任务（单个）的组合定义的，AISBench的其他命令行则规定了评测任务的场景（精度评测场景、性能评测场景等）。以如下AISBench命令为例： +数据集任务的选取、准备和使用参考如下步骤： +1. 在📚 [开源数据集](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/get_started/datasets.html#id3)内选取数据集任务 +2. 进入数据的 📚 [详细介绍/数据集部署](ais_bench/benchmark/configs/datasets/demo/README.md#数据集部署)准备数据集 +3. 参考📚 [详细介绍/可用数据集任务](ais_bench/benchmark/configs/datasets/demo/README.md#可用数据集任务)选取可用数据集任务，并将对应的任务导入方式（例如`from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets`）复制到自定义配置文件中 + +修改好配置文件后，执行如下命令启动服务化精度评测： + +```bash +ais_bench ais_bench/configs/model_api_test_zh_cn.py +``` + +*** + +**备选：使用命令行参数** + +如果你更习惯使用命令行参数方式，AISBench 同样支持通过 `--models`、`--datasets`、`--summarizer` 参数直接指定任务。以下是与上述自定义配置文件方式**执行效果完全相同**的命令行方式。 + +AISBench命令执行的单个或多个评测任务是由模型任务（单个或多个）、数据集任务（单个或多个）和结果呈现任务（单个）的组合定义的。以如下AISBench命令为例： ```shell ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer example @@ -174,28 +256,18 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch - `--models`指定了模型任务，即`vllm_api_general_chat`模型任务。 - `--datasets`指定了数据集任务，即`demo_gsm8k_gen_4_shot_cot_chat_prompt`数据集任务。 -- `--summarizer`指定了结果呈现任务，即`example`结果呈现任务(不指定`--summarizer`精度评测场景默认使用`example`任务)，一般使用默认，不需要在命令行中指定，后续命令不指定。 +- `--summarizer`指定了结果呈现任务，即`example`结果呈现任务(不指定`--summarizer`精度评测场景默认使用`example`任务)，一般使用默认，不需要在命令行中指定。 多任务测评请参考：📚 精度场景的[多任务测评](./docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md#多任务测评) 和 性能场景的[多任务测评](./docs/source_zh_cn/base_tutorials/scenes_intro/performance_benchmark.md#多任务测评)。 如需自行组合测评任务，实现更灵活的测评方式，可参考：📚 [自定义配置文件运行AISBench](./docs/source_zh_cn/advanced_tutorials/run_custom_config.md#自定义配置文件运行AISBench)。 -### 任务含义查询(可选) - 所选模型任务`vllm_api_general_chat`、数据集任务`demo_gsm8k_gen_4_shot_cot_chat_prompt`和结果呈现任务`example`的具体信息(简介，使用约束等)可以分别从如下链接中查询含义： - `--models`: 📚 [服务化推理后端](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/all_params/models.html#id2) - `--datasets`: 📚 [开源数据集](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/get_started/datasets.html#id3) → 📚 [详细介绍](ais_bench/benchmark/configs/datasets/demo/README.md) - `--summarizer`: 📚 [结果汇总任务](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/all_params/summarizer.html) -### 运行命令前置准备 - -- `--models`: 使用`vllm_api_general_chat`模型任务，需要准备支持`v1/chat/completions`子服务的推理服务，可以参考🔗 [VLLM启动OpenAI 兼容服务器](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server)启动推理服务 -- `--datasets`: 使用`demo_gsm8k_gen_4_shot_cot_chat_prompt`数据集任务，需要准备gsm8k数据集，可以从🔗 [opencompass - 提供的gsm8k数据集压缩包](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip)下载。将解压后的`gsm8k/`文件夹部署到AISBench评测工具根路径下的`ais_bench/datasets`文件夹下。 - -### 任务对应配置文件修改 - 每个模型任务、数据集任务和结果呈现任务都对应一个配置文件，运行命令前需要修改这些配置文件的内容。这些配置文件路径可以通过在原有AISBench命令基础上加上`--search`来查询，例如： ```shell @@ -250,15 +322,13 @@ models = [ ] ``` -### 执行命令 - 修改好配置文件后，执行命令启动服务化精度评测： ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt ``` -#### 查看任务执行细节 +### 查看任务执行细节 执行AISBench命令后，任务管理界面会在命令行实时刷新显示任务执行状态（键盘按"P"键可以暂停/恢复刷新，用于复制看板信息，再按"P"键可以继续刷新）。任务管理界面支持同时监控多个任务的详细执行状态，包括任务名称、进度、时间成本、状态、日志路径、扩展参数等信息，例如： @@ -313,7 +383,7 @@ outputs/default/20250628_151326/logs/infer/vllm-api-general-chat/demo_gsm8k.out > ⚠️ **注意**： 不同评测场景落盘任务执行细节内容不同，具体请参考具体评测场景的指南。 -#### 输出结果 +### 输出结果 因为只有8条数据，会很快跑出结果，结果显示的示例如下 diff --git a/README_en.md b/README_en.md index 6ab270c8..da38148c 100644 --- a/README_en.md +++ b/README_en.md @@ -72,9 +72,9 @@ - **\[2025.6.12\]** Supported accuracy and performance evaluation for multimodal datasets including [textvqa](ais_bench/benchmark/configs/datasets/textvqa/README_en.md), [videobench](ais_bench/benchmark/configs/datasets/videobench/README_en.md), and [vocalsound](ais_bench/benchmark/configs/datasets/vocalsound/README_en.md)! 🔥🔥🔥 -- **\[2025.6.6\]** AISBench supports steady-state performance evaluation to obtain the true optimal performance of the system. Refer to 📚 [Service Deployment Steady-State Performance Test](doc/users_guide/stable_stage.md) to get started quickly! 🔥🔥🔥 +- **\[2025.6.6\]** AISBench supports steady-state performance evaluation to obtain the true optimal performance of the system. Refer to 📚 [Service Deployment Steady-State Performance Test](docs/source_en/advanced_tutorials/stable_stage.md) to get started quickly! 🔥🔥🔥 -- **\[2025.5.16\]** Supported performance evaluation for high concurrency service deployment (up to 30,000+ concurrent requests). 📚 [Performance Metrics](doc/users_guide/performance_metric.md) are aligned with 🔗 [vllm benchmark](https://github.com/vllm-project/vllm/tree/main/benchmarks). See 📚 [Service Deployment Performance Evaluation Guide](https://ais-bench-benchmark.readthedocs.io/en/latest/base_tutorials/scenes_intro/performance_benchmark.html) for details! 🔥🔥🔥 +- **\[2025.5.16\]** Supported performance evaluation for high concurrency service deployment (up to 30,000+ concurrent requests). 📚 [Performance Metrics](docs/source_en/base_tutorials/results_intro/performance_metric.md) are aligned with 🔗 [vllm benchmark](https://github.com/vllm-project/vllm/tree/main/benchmarks). See 📚 [Service Deployment Performance Evaluation Guide](https://ais-bench-benchmark.readthedocs.io/en/latest/base_tutorials/scenes_intro/performance_benchmark.html) for details! 🔥🔥🔥 - **\[2025.4.30\]** Accuracy evaluation supports resuming from breakpoints and re-evaluating failed cases, significantly improving the robustness of accuracy evaluation. Refer to 📚 [Resume from Interruption & Re-evaluate Failed Cases](https://ais-bench-benchmark.readthedocs.io/en/latest/base_tutorials/scenes_intro/accuracy_benchmark.html#id10) to get started quickly! 🔥🔥🔥 @@ -161,16 +161,98 @@ pip3 uninstall ais_bench_benchmark ## 🚀 Quick Start -### Command Meaning -A single or multiple evaluation tasks executed by an AISBench command are defined by a combination of model tasks (single or multiple), dataset tasks (single or multiple), and result presentation tasks (single). Other command-line options of AISBench specify the scenario of the evaluation task (accuracy evaluation scenario, performance evaluation scenario, etc.). Take the following AISBench command as an example: +### Pre-execution Preparation + +- You need to prepare an inference service that supports the `v1/chat/completions` sub-service. Refer to 🔗 [VLLM Start OpenAI-Compatible Server](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server) to start the inference service. +- You need to prepare the GSM8K dataset. You can download it from the 🔗 [GSM8K dataset archive provided by OpenCompass](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip). After decompression, place the `gsm8k/` folder under the `ais_bench/datasets` directory of the AISBench evaluation tool root path. + +### Start Evaluation (Choose one of two methods) + +| ⭐ Recommended: Use Custom Configuration File | Alternative: Use Command-Line Parameters (Original Quick Start) | +| :------------------ | :------------------------------ | +| Modify one file to centrally manage all configurations, write configurations at any path | Specify via `--models` and `--datasets` parameters | +| Write once, reuse many times | Each run requires entering the full command | +| Supports full Python syntax for flexible extension | Only supports Cartesian product combinations | + +**⭐ Recommended: Use Custom Configuration File** + +AISBench provides a preset custom configuration file [model_api_test_en.py](ais_bench/configs/model_api_test_en.py), which centralizes common inference service-deployed test configurations (model selection, service address, port, generation parameters, etc.) in one file, eliminating the need to look up and modify multiple configuration files. The file is essentially a Python script that supports all Python syntax, allowing you to freely extend it. + +Open `ais_bench/configs/model_api_test_en.py` and modify the following configuration according to your actual situation (if you installed the tool via `pip3 install ais_bench_benchmark`, you can create `model_api_test_en.py` at any path and write the following configuration content into that file): + +```python +from mmengine.config import read_base + +with read_base(): +# Model task, select one. For other model tasks, see: https://ais-bench-benchmark-rf.readthedocs.io/en/latest/base_tutorials/all_params/models.html for more model tasks + # vllm_api_general is a base model that only supports text generation + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + # vllm_api_general_chat is a chat model that supports dialogue + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + # vllm_api_stream_chat is a streaming chat model that supports streaming dialogue + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + # vllm_api_general_stream is a streaming model that supports streaming generation + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + +# Dataset task, see: https://ais-bench-benchmark-rf.readthedocs.io/en/latest/get_started/datasets.html for more dataset tasks + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = vllm_api_general_chat + +models[0]["path"] = "" # Specify the absolute path of the model serialized vocabulary file (generally not required for accuracy testing scenarios) +models[0]["model"] = "" # Specify the model name loaded on the server, configured according to the actual model name pulled by the VLLM inference service (configure as an empty string to get it automatically) +models[0]["request_rate"] = 0 # Request sending frequency: send 1 request to the server every 1/request_rate seconds; if less than 0.001, all requests are sent at once +models[0]["api_key"] = "" # Custom API key, default is an empty string +models[0]["host_ip"] = "localhost" # Specify the IP of the inference service +models[0]["host_port"] = 8080 # Specify the port of the inference service +models[0]["url"] = "" # Custom URL path for accessing the inference service (needs to be configured when the base URL is not a combination of http://host_ip:host_port; after configuration, host_ip and host_port will be ignored) +models[0]["max_out_len"] = 512 # Maximum number of tokens output by the inference service +models[0]["batch_size"] = 1 # Maximum concurrency for sending requests +models[0]["trust_remote_code"] = False # Whether the tokenizer trusts remote code, default is False +models[0]["generation_kwargs"] = dict( # Model inference parameters, configured with reference to the VLLM documentation; the AISBench evaluation tool does not process them and attaches them directly to the sent request + temperature=0.01, + ignore_eos=False, +) + +# datasets[0]["path"] = ais_bench/datasets/gsm8k # Specify the absolute path of the dataset directory (required for accuracy testing scenarios) + +work_dir = 'outputs/default/' # Specify the working directory for saving task results and logs (default is outputs/default/) + +``` +> 💡 The configuration file has pre-imported common model types (`vllm_api_general`, `vllm_api_general_chat`, `vllm_api_stream_chat`, `vllm_api_general_stream`). You only need to uncomment/modify the comment to switch. For more usage of custom configuration files, please refer to 📚 [Running AISBench with Custom Configuration File](./docs/source_en/advanced_tutorials/run_custom_config.md). + +For selecting, preparing, and using dataset tasks, refer to the following steps: +1. Select a dataset task in 📚 [Open-Source Datasets](https://ais-bench-benchmark.readthedocs.io/en/latest/get_started/datasets.html#id3) +2. Enter the data's 📚 [Detailed Introduction/Dataset Deployment](ais_bench/benchmark/configs/datasets/demo/README_en.md#dataset-deployment) to prepare the dataset +3. Refer to 📚 [Detailed Introduction/Available Dataset Tasks](ais_bench/benchmark/configs/datasets/demo/README_en.md#available-dataset-tasks) to select an available dataset task, and copy the corresponding task import method (e.g., `from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets`) to the custom configuration file + +After modifying the configuration file, execute the following command to start the service-deployed accuracy evaluation: + +```bash +ais_bench ais_bench/configs/model_api_test_en.py +``` + +*** + +**Alternative: Use Command-Line Parameters** + +If you are more familiar with using command-line parameters, AISBench also supports directly specifying tasks via the `--models`, `--datasets`, and `--summarizer` parameters. The following is the command-line approach that has **exactly the same execution effect** as the above custom configuration file approach. + +A single or multiple evaluation tasks executed by an AISBench command are defined by a combination of model tasks (single or multiple), dataset tasks (single or multiple), and result presentation tasks (single). Take the following AISBench command as an example: + ```shell ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer example ``` + This command does not specify other command-line options, so it defaults to an accuracy evaluation task, where: - `--models` specifies the model task: the `vllm_api_general_chat` model task. - `--datasets` specifies the dataset task: the `demo_gsm8k_gen_4_shot_cot_chat_prompt` dataset task. - `--summarizer` specifies the result presentation task: the `example` result presentation task (if `--summarizer` is not specified, the `example` task is used by default for accuracy evaluation scenarios). It is generally used as default and does not need to be specified in the command line (subsequent commands will omit this option). +For multi-task evaluation, refer to: 📚 [Multi-Task Evaluation in Accuracy Scenarios](./docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md#multi-task-evaluation) and [Multi-Task Evaluation in Performance Scenarios](./docs/source_en/base_tutorials/scenes_intro/performance_benchmark.md#multi-task-evaluation). + +If you need to combine evaluation tasks on your own for more flexible evaluation methods, refer to: 📚 [Running AISBench with Custom Configuration File](./docs/source_en/advanced_tutorials/run_custom_config.md#running-aisbench-with-a-custom-configuration-file). + ### Task Meaning Query (Optional) Detailed information (introduction, usage constraints, etc.) about the selected model task (`vllm_api_general_chat`), dataset task (`demo_gsm8k_gen_4_shot_cot_chat_prompt`), and result presentation task (`example`) can be queried from the following links: diff --git a/ais_bench/benchmark/cli/config_manager.py b/ais_bench/benchmark/cli/config_manager.py index ce6ea148..8e5d2f09 100644 --- a/ais_bench/benchmark/cli/config_manager.py +++ b/ais_bench/benchmark/cli/config_manager.py @@ -37,7 +37,6 @@ def __init__(self, config, file_path): def check(self): self._check_models_config() self._check_datasets_config() - self._check_summarizer_config() def _check_models_config(self): models = self.config.get('models', []) diff --git a/ais_bench/benchmark/configs/datasets/ARC_c/README.md b/ais_bench/benchmark/configs/datasets/ARC_c/README.md index dcff1b00..65b32bcd 100644 --- a/ais_bench/benchmark/configs/datasets/ARC_c/README.md +++ b/ais_bench/benchmark/configs/datasets/ARC_c/README.md @@ -28,8 +28,8 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|ARC_c_gen_0_shot_chat_prompt|ARC Challenge Set数据集生成式任务|accuracy|0-shot|对话格式|[ARC_c_gen_0_shot_chat_prompt.py](ARC_c_gen_0_shot_chat_prompt.py)| -|ARC_c_gen_25_shot_chat_prompt|ARC Challenge Set数据集生成式任务|accuracy|25-shot|对话格式|[ARC_c_gen_25_shot_chat_prompt.py](ARC_c_gen_25_shot_chat_prompt.py)| -|ARC_c_ppl_0_shot_str|ARC Challenge Set数据集PPL任务|accuracy|0-shot|字符串格式|[ARC_c_ppl_0_shot_str.py](ARC_c_ppl_0_shot_str.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|ARC_c_gen_0_shot_chat_prompt|ARC Challenge Set数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.ARC_c.ARC_c_gen_0_shot_chat_prompt import ARC_c_datasets as datasets`|[ARC_c_gen_0_shot_chat_prompt.py](ARC_c_gen_0_shot_chat_prompt.py)| +|ARC_c_gen_25_shot_chat_prompt|ARC Challenge Set数据集生成式任务|accuracy|25-shot|对话格式|`from ais_bench.benchmark.configs.datasets.ARC_c.ARC_c_gen_25_shot_chat_prompt import ARC_c_datasets as datasets`|[ARC_c_gen_25_shot_chat_prompt.py](ARC_c_gen_25_shot_chat_prompt.py)| +|ARC_c_ppl_0_shot_str|ARC Challenge Set数据集PPL任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.ARC_c.ARC_c_ppl_0_shot_str import ARC_c_datasets as datasets`|[ARC_c_ppl_0_shot_str.py](ARC_c_ppl_0_shot_str.py)| diff --git a/ais_bench/benchmark/configs/datasets/ARC_c/README_en.md b/ais_bench/benchmark/configs/datasets/ARC_c/README_en.md index 988c27ce..9920e960 100644 --- a/ais_bench/benchmark/configs/datasets/ARC_c/README_en.md +++ b/ais_bench/benchmark/configs/datasets/ARC_c/README_en.md @@ -28,8 +28,8 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| ARC_c_gen_0_shot_chat_prompt | Generative task for the ARC Challenge Set dataset | Accuracy | 0-shot | Chat format | [ARC_c_gen_0_shot_chat_prompt.py](ARC_c_gen_0_shot_chat_prompt.py) | -| ARC_c_gen_25_shot_chat_prompt | Generative task for the ARC Challenge Set dataset | Accuracy | 25-shot | Chat format | [ARC_c_gen_25_shot_chat_prompt.py](ARC_c_gen_25_shot_chat_prompt.py) | -| ARC_c_ppl_0_shot_str | PPL task for ARC Challenge Set dataset | Accuracy | 0-shot | String format | [ARC_c_ppl_0_shot_str.py](ARC_c_ppl_0_shot_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| ARC_c_gen_0_shot_chat_prompt | Generative task for the ARC Challenge Set dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.ARC_c.ARC_c_gen_0_shot_chat_prompt import ARC_c_datasets as datasets`| [ARC_c_gen_0_shot_chat_prompt.py](ARC_c_gen_0_shot_chat_prompt.py) | +| ARC_c_gen_25_shot_chat_prompt | Generative task for the ARC Challenge Set dataset | Accuracy | 25-shot | Chat format |`from ais_bench.benchmark.configs.datasets.ARC_c.ARC_c_gen_25_shot_chat_prompt import ARC_c_datasets as datasets`| [ARC_c_gen_25_shot_chat_prompt.py](ARC_c_gen_25_shot_chat_prompt.py) | +| ARC_c_ppl_0_shot_str | PPL task for ARC Challenge Set dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.ARC_c.ARC_c_ppl_0_shot_str import ARC_c_datasets as datasets`| [ARC_c_ppl_0_shot_str.py](ARC_c_ppl_0_shot_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/ARC_e/README.md b/ais_bench/benchmark/configs/datasets/ARC_e/README.md index 40ef4e23..0e124373 100644 --- a/ais_bench/benchmark/configs/datasets/ARC_e/README.md +++ b/ais_bench/benchmark/configs/datasets/ARC_e/README.md @@ -27,8 +27,8 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|ARC_e_gen_0_shot_chat_prompt|ARC Easy Set数据集生成式任务|accuracy|0-shot|对话格式|[ARC_e_gen_0_shot_chat_prompt.py](ARC_e_gen_0_shot_chat_prompt.py)| -|ARC_e_gen_25_shot_chat_prompt|ARC Easy Set数据集生成式任务|accuracy|25-shot|对话格式|[ARC_e_gen_25_shot_chat_prompt.py](ARC_e_gen_25_shot_chat_prompt.py)| -|ARC_e_ppl_0_shot_str|ARC Easy Set数据集PPL任务|accuracy|0-shot|字符串模式|[ARC_e_ppl_0_shot_str.py](ARC_e_ppl_0_shot_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|ARC_e_gen_0_shot_chat_prompt|ARC Easy Set数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.ARC_e.ARC_e_gen_0_shot_chat_prompt import ARC_e_datasets as datasets`|[ARC_e_gen_0_shot_chat_prompt.py](ARC_e_gen_0_shot_chat_prompt.py)| +|ARC_e_gen_25_shot_chat_prompt|ARC Easy Set数据集生成式任务|accuracy|25-shot|对话格式|`from ais_bench.benchmark.configs.datasets.ARC_e.ARC_e_gen_25_shot_chat_prompt import ARC_e_datasets as datasets`|[ARC_e_gen_25_shot_chat_prompt.py](ARC_e_gen_25_shot_chat_prompt.py)| +|ARC_e_ppl_0_shot_str|ARC Easy Set数据集PPL任务|accuracy|0-shot|字符串模式|`from ais_bench.benchmark.configs.datasets.ARC_e.ARC_e_ppl_0_shot_str import ARC_e_datasets as datasets`|[ARC_e_ppl_0_shot_str.py](ARC_e_ppl_0_shot_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/ARC_e/README_en.md b/ais_bench/benchmark/configs/datasets/ARC_e/README_en.md index 60671ff9..f0dd01d0 100644 --- a/ais_bench/benchmark/configs/datasets/ARC_e/README_en.md +++ b/ais_bench/benchmark/configs/datasets/ARC_e/README_en.md @@ -27,8 +27,8 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| ARC_e_gen_0_shot_chat_prompt | Generative task for the ARC Easy Set dataset | Accuracy | 0-shot | Chat format | [ARC_e_gen_0_shot_chat_prompt.py](ARC_e_gen_0_shot_chat_prompt.py) | -| ARC_e_gen_25_shot_chat_prompt | Generative task for the ARC Easy Set dataset | Accuracy | 25-shot | Chat format | [ARC_e_gen_25_shot_chat_prompt.py](ARC_e_gen_25_shot_chat_prompt.py) | -| ARC_e_ppl_0_shot_str | PPL task for the ARC Easy Set dataset | Accuracy | 0-shot | String Format | [ARC_e_ppl_0_shot_str.py](ARC_e_ppl_0_shot_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| ARC_e_gen_0_shot_chat_prompt | Generative task for the ARC Easy Set dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.ARC_e.ARC_e_gen_0_shot_chat_prompt import ARC_e_datasets as datasets`| [ARC_e_gen_0_shot_chat_prompt.py](ARC_e_gen_0_shot_chat_prompt.py) | +| ARC_e_gen_25_shot_chat_prompt | Generative task for the ARC Easy Set dataset | Accuracy | 25-shot | Chat format |`from ais_bench.benchmark.configs.datasets.ARC_e.ARC_e_gen_25_shot_chat_prompt import ARC_e_datasets as datasets`| [ARC_e_gen_25_shot_chat_prompt.py](ARC_e_gen_25_shot_chat_prompt.py) | +| ARC_e_ppl_0_shot_str | PPL task for the ARC Easy Set dataset | Accuracy | 0-shot | String Format |`from ais_bench.benchmark.configs.datasets.ARC_e.ARC_e_ppl_0_shot_str import ARC_e_datasets as datasets`| [ARC_e_ppl_0_shot_str.py](ARC_e_ppl_0_shot_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README.md b/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README.md index f9b442f4..91362532 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README.md @@ -39,6 +39,6 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|FewCLUE_bustm_ppl_0_shot_chat|FewCLUE_bustm数据集PPL任务|accuracy|0-shot|对话格式|[FewCLUE_bustm_ppl_0_shot_chat.py](FewCLUE_bustm_ppl_0_shot_chat.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|FewCLUE_bustm_ppl_0_shot_chat|FewCLUE_bustm数据集PPL任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.FewCLUE_bustm.FewCLUE_bustm_ppl_0_shot_chat import bustm_datasets as datasets`|[FewCLUE_bustm_ppl_0_shot_chat.py](FewCLUE_bustm_ppl_0_shot_chat.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README_en.md b/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README_en.md index 32a8821f..813a83ce 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README_en.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_bustm/README_en.md @@ -39,6 +39,6 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| FewCLUE_bustm_ppl_0_shot_chat | PPL task for the FewCLUE_bustm dataset | Accuracy | 0-shot | Chat format | [FewCLUE_bustm_ppl_0_shot_chat.py](FewCLUE_bustm_ppl_0_shot_chat.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| FewCLUE_bustm_ppl_0_shot_chat | PPL task for the FewCLUE_bustm dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.FewCLUE_bustm.FewCLUE_bustm_ppl_0_shot_chat import bustm_datasets as datasets`| [FewCLUE_bustm_ppl_0_shot_chat.py](FewCLUE_bustm_ppl_0_shot_chat.py) | diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README.md b/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README.md index 08ac377b..223550d3 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README.md @@ -39,6 +39,6 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|FewCLUE_chid_ppl_0_shot_str|FewCLUE_chid数据集PPL任务|accuracy|0-shot|字符串格式|[FewCLUE_chid_ppl_0_shot_str.py](FewCLUE_chid_ppl_0_shot_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|FewCLUE_chid_ppl_0_shot_str|FewCLUE_chid数据集PPL任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.FewCLUE_chid.FewCLUE_chid_ppl_0_shot_str import chid_datasets as datasets`|[FewCLUE_chid_ppl_0_shot_str.py](FewCLUE_chid_ppl_0_shot_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README_en.md b/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README_en.md index 975b2aee..7790fc51 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README_en.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_chid/README_en.md @@ -39,6 +39,6 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| FewCLUE_chid_ppl_0_shot_str | PPL task for the FewCLUE_chid dataset | Accuracy | 0-shot | String format | [FewCLUE_chid_ppl_0_shot_str.py](FewCLUE_chid_ppl_0_shot_str.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| FewCLUE_chid_ppl_0_shot_str | PPL task for the FewCLUE_chid dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.FewCLUE_chid.FewCLUE_chid_ppl_0_shot_str import chid_datasets as datasets`| [FewCLUE_chid_ppl_0_shot_str.py](FewCLUE_chid_ppl_0_shot_str.py) | diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README.md b/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README.md index 32c8eb02..f76e22f7 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README.md @@ -41,6 +41,6 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|FewCLUE_cluewsc_ppl_0_shot_chat|FewCLUE_cluewsc数据集PPL任务|accuracy|0-shot|对话格式|[FewCLUE_cluewsc_ppl_0_shot_chat.py](FewCLUE_cluewsc_ppl_0_shot_chat.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|FewCLUE_cluewsc_ppl_0_shot_chat|FewCLUE_cluewsc数据集PPL任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.FewCLUE_cluewsc.FewCLUE_cluewsc_ppl_0_shot_chat import cluewsc_datasets as datasets`|[FewCLUE_cluewsc_ppl_0_shot_chat.py](FewCLUE_cluewsc_ppl_0_shot_chat.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README_en.md b/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README_en.md index 52f41c7a..57b21982 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README_en.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_cluewsc/README_en.md @@ -41,6 +41,6 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| FewCLUE_cluewsc_ppl_0_shot_chat | PPL task for the FewCLUE_cluewsc dataset | Accuracy | 0-shot | Chat format | [FewCLUE_cluewsc_ppl_0_shot_chat.py](FewCLUE_cluewsc_ppl_0_shot_chat.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| FewCLUE_cluewsc_ppl_0_shot_chat | PPL task for the FewCLUE_cluewsc dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.FewCLUE_cluewsc.FewCLUE_cluewsc_ppl_0_shot_chat import cluewsc_datasets as datasets`| [FewCLUE_cluewsc_ppl_0_shot_chat.py](FewCLUE_cluewsc_ppl_0_shot_chat.py) | diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README.md b/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README.md index c1102098..03b957fa 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README.md @@ -39,7 +39,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|FewCLUE_csl_ppl_0_shot_str|FewCLUE_csl数据集PPL任务|accuracy|0-shot|字符串格式|[FewCLUE_csl_ppl_0_shot_str.py](FewCLUE_csl_ppl_0_shot_str.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|FewCLUE_csl_ppl_0_shot_str|FewCLUE_csl数据集PPL任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.FewCLUE_csl.FewCLUE_csl_ppl_0_shot_str import csl_datasets as datasets`|[FewCLUE_csl_ppl_0_shot_str.py](FewCLUE_csl_ppl_0_shot_str.py)| diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README_en.md b/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README_en.md index af1bb9cd..fd22f30f 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README_en.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_csl/README_en.md @@ -39,7 +39,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| FewCLUE_csl_ppl_0_shot_str | PPL task for the FewCLUE_csl dataset | Accuracy | 0-shot | String format | [FewCLUE_csl_ppl_0_shot_str.py](FewCLUE_csl_ppl_0_shot_str.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| FewCLUE_csl_ppl_0_shot_str | PPL task for the FewCLUE_csl dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.FewCLUE_csl.FewCLUE_csl_ppl_0_shot_str import csl_datasets as datasets`| [FewCLUE_csl_ppl_0_shot_str.py](FewCLUE_csl_ppl_0_shot_str.py) | diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README.md b/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README.md index dacc2c88..66f9f609 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README.md @@ -39,7 +39,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|FewCLUE_eprstmt_ppl_0_shot_chat|FewCLUE_eprstmt数据集PPL任务|accuracy|0-shot|对话格式|[FewCLUE_eprstmt_ppl_0_shot_chat.py](FewCLUE_eprstmt_ppl_0_shot_chat.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|FewCLUE_eprstmt_ppl_0_shot_chat|FewCLUE_eprstmt数据集PPL任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.FewCLUE_eprstmt.FewCLUE_eprstmt_ppl_0_shot_chat import eprstmt_datasets as datasets`|[FewCLUE_eprstmt_ppl_0_shot_chat.py](FewCLUE_eprstmt_ppl_0_shot_chat.py)| diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README_en.md b/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README_en.md index 1405475c..62e95061 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README_en.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_eprstmt/README_en.md @@ -39,7 +39,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| FewCLUE_eprstmt_ppl_0_shot_chat | PPL task for the FewCLUE_eprstmt dataset | Accuracy | 0-shot | Chat format | [FewCLUE_eprstmt_ppl_0_shot_chat.py](FewCLUE_eprstmt_ppl_0_shot_chat.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| FewCLUE_eprstmt_ppl_0_shot_chat | PPL task for the FewCLUE_eprstmt dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.FewCLUE_eprstmt.FewCLUE_eprstmt_ppl_0_shot_chat import eprstmt_datasets as datasets`| [FewCLUE_eprstmt_ppl_0_shot_chat.py](FewCLUE_eprstmt_ppl_0_shot_chat.py) | diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README.md b/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README.md index cc3280f6..c8df7a8e 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README.md @@ -39,6 +39,6 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|FewCLUE_tnews_ppl_0_shot_chat|FewCLUE_tnews数据集PPL任务|accuracy|0-shot|对话格式|[FewCLUE_tnews_ppl_0_shot_chat.py](FewCLUE_tnews_ppl_0_shot_chat.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|FewCLUE_tnews_ppl_0_shot_chat|FewCLUE_tnews数据集PPL任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.FewCLUE_tnews.FewCLUE_tnews_ppl_0_shot_chat import tnews_datasets as datasets`|[FewCLUE_tnews_ppl_0_shot_chat.py](FewCLUE_tnews_ppl_0_shot_chat.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README_en.md b/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README_en.md index 8ada0c8b..aad7c344 100644 --- a/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README_en.md +++ b/ais_bench/benchmark/configs/datasets/FewCLUE_tnews/README_en.md @@ -39,7 +39,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| FewCLUE_tnews_ppl_0_shot_chat | PPL task for the FewCLUE_tnews dataset | Accuracy | 0-shot | Chat format | [FewCLUE_tnews_ppl_0_shot_chat.py](FewCLUE_tnews_ppl_0_shot_chat.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| FewCLUE_tnews_ppl_0_shot_chat | PPL task for the FewCLUE_tnews dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.FewCLUE_tnews.FewCLUE_tnews_ppl_0_shot_chat import tnews_datasets as datasets`| [FewCLUE_tnews_ppl_0_shot_chat.py](FewCLUE_tnews_ppl_0_shot_chat.py) | diff --git a/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README.md b/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README.md index 25fb8f59..18eddb3d 100644 --- a/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README.md +++ b/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README.md @@ -23,9 +23,9 @@ rm SuperGLUE.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|SuperGLUE_BoolQ_gen_883d50_str|BoolQ数据集生成式任务|accuracy(naive_average)|0-shot|string|[SuperGLUE_BoolQ_gen_883d50_str.py](SuperGLUE_BoolQ_gen_883d50_str.py)| -|SuperGLUE_BoolQ_gen_0_shot_cot_str|BoolQ数据集生成式任务，prompt带逻辑链|accuracy(naive_average)|0-shot|string|[SuperGLUE_BoolQ_gen_0_shot_cot_str.py](SuperGLUE_BoolQ_gen_0_shot_cot_str.py)| -|SuperGLUE_BoolQ_gen_5_shot_str|BoolQ数据集生成式任务，few-shot|accuracy(naive_average)|5-shot|string|[SuperGLUE_BoolQ_gen_5_shot_str.py](SuperGLUE_BoolQ_gen_5_shot_str.py)| -|SuperGLUE_BoolQ_gen_0_shot_str|BoolQ数据集生成式任务，few-shot|accuracy(naive_average)|5-shot|string|[SuperGLUE_BoolQ_gen_0_shot_str.py](SuperGLUE_BoolQ_gen_0_shot_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|SuperGLUE_BoolQ_gen_0_shot_noncot_str|BoolQ数据集生成式任务|accuracy(naive_average)|0-shot|string|`from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_0_shot_noncot_str import BoolQ_datasets as datasets`|[SuperGLUE_BoolQ_gen_0_shot_noncot_str.py](SuperGLUE_BoolQ_gen_0_shot_noncot_str.py)| +|SuperGLUE_BoolQ_gen_0_shot_cot_str|BoolQ数据集生成式任务，prompt带逻辑链|accuracy(naive_average)|0-shot|string|`from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_0_shot_cot_str import BoolQ_datasets as datasets`|[SuperGLUE_BoolQ_gen_0_shot_cot_str.py](SuperGLUE_BoolQ_gen_0_shot_cot_str.py)| +|SuperGLUE_BoolQ_gen_5_shot_str|BoolQ数据集生成式任务，few-shot|accuracy(naive_average)|5-shot|string|`from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_5_shot_str import BoolQ_datasets as datasets`|[SuperGLUE_BoolQ_gen_5_shot_str.py](SuperGLUE_BoolQ_gen_5_shot_str.py)| +|SuperGLUE_BoolQ_gen_0_shot_str|BoolQ数据集生成式任务，few-shot|accuracy(naive_average)|5-shot|string|`from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_0_shot_str import BoolQ_datasets as datasets`|[SuperGLUE_BoolQ_gen_0_shot_str.py](SuperGLUE_BoolQ_gen_0_shot_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README_en.md b/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README_en.md index 9f0a9b0a..69907276 100644 --- a/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README_en.md +++ b/ais_bench/benchmark/configs/datasets/SuperGLUE_BoolQ/README_en.md @@ -23,12 +23,12 @@ rm SuperGLUE.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| SuperGLUE_BoolQ_gen_883d50_str | Generative task for the BoolQ dataset | Accuracy (naive_average) | 0-shot | String | [SuperGLUE_BoolQ_gen_883d50_str.py](SuperGLUE_BoolQ_gen_883d50_str.py) | -| SuperGLUE_BoolQ_gen_0_shot_cot_str | Generative task for the BoolQ dataset, with a chain-of-thought in the prompt | Accuracy (naive_average) | 0-shot | String | [SuperGLUE_BoolQ_gen_0_shot_cot_str.py](SuperGLUE_BoolQ_gen_0_shot_cot_str.py) | -| SuperGLUE_BoolQ_gen_5_shot_str | Generative task for the BoolQ dataset (few-shot setting) | Accuracy (naive_average) | 5-shot | String | [SuperGLUE_BoolQ_gen_5_shot_str.py](SuperGLUE_BoolQ_gen_5_shot_str.py) | -| SuperGLUE_BoolQ_gen_0_shot_str | Generative task for the BoolQ dataset (note: there is a possible inconsistency between the "Few-Shot" setting and the task name; the "Few-Shot" column shows 5-shot, while the task name indicates 0-shot) | Accuracy (naive_average) | 5-shot | String | [SuperGLUE_BoolQ_gen_0_shot_str.py](SuperGLUE_BoolQ_gen_0_shot_str.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| SuperGLUE_BoolQ_gen_0_shot_noncot_str | Generative task for the BoolQ dataset | Accuracy (naive_average) | 0-shot | String | `from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_0_shot_noncot_str import BoolQ_datasets as datasets` | [SuperGLUE_BoolQ_gen_0_shot_noncot_str.py](SuperGLUE_BoolQ_gen_0_shot_noncot_str.py) | +| SuperGLUE_BoolQ_gen_0_shot_cot_str | Generative task for the BoolQ dataset, with a chain-of-thought in the prompt | Accuracy (naive_average) | 0-shot | String |`from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_0_shot_cot_str import BoolQ_datasets as datasets`| [SuperGLUE_BoolQ_gen_0_shot_cot_str.py](SuperGLUE_BoolQ_gen_0_shot_cot_str.py) | +| SuperGLUE_BoolQ_gen_5_shot_str | Generative task for the BoolQ dataset (few-shot setting) | Accuracy (naive_average) | 5-shot | String |`from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_5_shot_str import BoolQ_datasets as datasets`| [SuperGLUE_BoolQ_gen_5_shot_str.py](SuperGLUE_BoolQ_gen_5_shot_str.py) | +| SuperGLUE_BoolQ_gen_0_shot_str | Generative task for the BoolQ dataset (note: there is a possible inconsistency between the "Few-Shot" setting and the task name; the "Few-Shot" column shows 5-shot, while the task name indicates 0-shot) | Accuracy (naive_average) | 5-shot | String |`from ais_bench.benchmark.configs.datasets.SuperGLUE_BoolQ.SuperGLUE_BoolQ_gen_0_shot_str import BoolQ_datasets as datasets`| [SuperGLUE_BoolQ_gen_0_shot_str.py](SuperGLUE_BoolQ_gen_0_shot_str.py) | ### Note diff --git a/ais_bench/benchmark/configs/datasets/Xsum/README.md b/ais_bench/benchmark/configs/datasets/Xsum/README.md index 475bca44..5c6eae9e 100644 --- a/ais_bench/benchmark/configs/datasets/Xsum/README.md +++ b/ais_bench/benchmark/configs/datasets/Xsum/README.md @@ -27,7 +27,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|Xsum_gen_0_shot_chat|Xsum数据集生成式任务|accuracy|0-shot|对话格式|[Xsum_gen_0_shot_chat.py](Xsum_gen_0_shot_chat.py)| -|Xsum_gen_0_shot_str|Xsum数据集生成式任务|accuracy|0-shot|字符串格式|[Xsum_gen_0_shot_str.py](Xsum_gen_0_shot_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|Xsum_gen_0_shot_chat|Xsum数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.Xsum.Xsum_gen_0_shot_chat import Xsum_datasets as datasets`|[Xsum_gen_0_shot_chat.py](Xsum_gen_0_shot_chat.py)| +|Xsum_gen_0_shot_str|Xsum数据集生成式任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.Xsum.Xsum_gen_0_shot_str import Xsum_datasets as datasets`|[Xsum_gen_0_shot_str.py](Xsum_gen_0_shot_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/Xsum/README_en.md b/ais_bench/benchmark/configs/datasets/Xsum/README_en.md index 1963013f..7c3935d3 100644 --- a/ais_bench/benchmark/configs/datasets/Xsum/README_en.md +++ b/ais_bench/benchmark/configs/datasets/Xsum/README_en.md @@ -27,7 +27,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| Xsum_gen_0_shot_chat | Generative task for the XSum dataset | Accuracy | 0-shot | Chat Format | [Xsum_gen_0_shot_chat.py](Xsum_gen_0_shot_chat.py) | -| Xsum_gen_0_shot_str | Generative task for the XSum dataset | Accuracy | 0-shot | String Format | [Xsum_gen_0_shot_str.py](Xsum_gen_0_shot_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| Xsum_gen_0_shot_chat | Generative task for the XSum dataset | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.Xsum.Xsum_gen_0_shot_chat import Xsum_datasets as datasets`| [Xsum_gen_0_shot_chat.py](Xsum_gen_0_shot_chat.py) | +| Xsum_gen_0_shot_str | Generative task for the XSum dataset | Accuracy | 0-shot | String Format |`from ais_bench.benchmark.configs.datasets.Xsum.Xsum_gen_0_shot_str import Xsum_datasets as datasets`| [Xsum_gen_0_shot_str.py](Xsum_gen_0_shot_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/agieval/README.md b/ais_bench/benchmark/configs/datasets/agieval/README.md index 81c0966f..b7d04a73 100644 --- a/ais_bench/benchmark/configs/datasets/agieval/README.md +++ b/ais_bench/benchmark/configs/datasets/agieval/README.md @@ -45,6 +45,6 @@ rm -r OpenCompassData-core-20240207.zip └── sat-math.jsonl ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|agieval_gen_0_shot_chat_prompt|AGIEval数据集生成式任务，共包含21个子任务|accuracy|0-shot|对话格式|[agieval_gen_0_shot_chat_prompt.py](agieval_gen_0_shot_chat_prompt.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|agieval_gen_0_shot_chat_prompt|AGIEval数据集生成式任务，共包含21个子任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.agieval.agieval_gen_0_shot_chat_prompt import agieval_datasets as datasets`|[agieval_gen_0_shot_chat_prompt.py](agieval_gen_0_shot_chat_prompt.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/agieval/README_en.md b/ais_bench/benchmark/configs/datasets/agieval/README_en.md index 0a6bc98e..beb49f2a 100644 --- a/ais_bench/benchmark/configs/datasets/agieval/README_en.md +++ b/ais_bench/benchmark/configs/datasets/agieval/README_en.md @@ -45,7 +45,7 @@ rm -r OpenCompassData-core-20240207.zip └── sat-math.jsonl ``` ## Available Dataset Tasks -|Task Name|Description|Evaluation Metric|Few-shot|Prompt Format|Corresponding Source Code Configuration File Path| -| --- | --- | --- | --- | --- | --- | -|agieval_gen_0_shot_chat_prompt|AGIEval dataset generative task, containing a total of 21 subtasks|accuracy|0-shot|Chat format|[agieval_gen_0_shot_chat_prompt.py](agieval_gen_0_shot_chat_prompt.py)| +|Task Name|Description|Evaluation Metric|Few-shot|Prompt Format|Import Statement|Corresponding Source Code Configuration File Path| +| --- | --- | --- | --- | --- | --- | --- | +|agieval_gen_0_shot_chat_prompt|AGIEval dataset generative task, containing a total of 21 subtasks|accuracy|0-shot|Chat format|`from ais_bench.benchmark.configs.datasets.agieval.agieval_gen_0_shot_chat_prompt import agieval_datasets as datasets`|[agieval_gen_0_shot_chat_prompt.py](agieval_gen_0_shot_chat_prompt.py)| ``` diff --git a/ais_bench/benchmark/configs/datasets/aime2024/README.md b/ais_bench/benchmark/configs/datasets/aime2024/README.md index 736dbcfb..b63ef3ef 100644 --- a/ais_bench/benchmark/configs/datasets/aime2024/README.md +++ b/ais_bench/benchmark/configs/datasets/aime2024/README.md @@ -24,7 +24,7 @@ rm aime.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|aime2024_gen_0_shot_str|aime2024数据集生成式任务|accuracy(pass@1)|0-shot|字符串格式|[aime2024_gen_0_shot_str.py](aime2024_gen_0_shot_str.py)| -|aime2024_gen_0_shot_chat_prompt|aime2024数据集生成式任务（对齐DeepSeek R1精度测试）|accuracy(pass@1)|0-shot|对话格式|[aime2024_gen_0_shot_chat_prompt.py](aime2024_gen_0_shot_chat_prompt.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|aime2024_gen_0_shot_str|aime2024数据集生成式任务|accuracy(pass@1)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_str import aime2024_datasets as datasets`|[aime2024_gen_0_shot_str.py](aime2024_gen_0_shot_str.py)| +|aime2024_gen_0_shot_chat_prompt|aime2024数据集生成式任务（对齐DeepSeek R1精度测试）|accuracy(pass@1)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets as datasets`|[aime2024_gen_0_shot_chat_prompt.py](aime2024_gen_0_shot_chat_prompt.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/aime2024/README_en.md b/ais_bench/benchmark/configs/datasets/aime2024/README_en.md index 2473c146..6762f9c4 100644 --- a/ais_bench/benchmark/configs/datasets/aime2024/README_en.md +++ b/ais_bench/benchmark/configs/datasets/aime2024/README_en.md @@ -24,7 +24,7 @@ rm aime.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| aime2024_gen_0_shot_str | Generative task for the aime2024 dataset | accuracy (pass@1) | 0-shot | String format | [aime2024_gen_0_shot_str.py](aime2024_gen_0_shot_str.py) | -| aime2024_gen_0_shot_chat_prompt | Generative task for the aime2024 dataset (aligned with DeepSeek R1 accuracy test) | accuracy (pass@1) | 0-shot | Chat format | [aime2024_gen_0_shot_chat_prompt.py](aime2024_gen_0_shot_chat_prompt.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| aime2024_gen_0_shot_str | Generative task for the aime2024 dataset | accuracy (pass@1) | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_str import aime2024_datasets as datasets`| [aime2024_gen_0_shot_str.py](aime2024_gen_0_shot_str.py) | +| aime2024_gen_0_shot_chat_prompt | Generative task for the aime2024 dataset (aligned with DeepSeek R1 accuracy test) | accuracy (pass@1) | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets as datasets`| [aime2024_gen_0_shot_chat_prompt.py](aime2024_gen_0_shot_chat_prompt.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/aime2025/README.md b/ais_bench/benchmark/configs/datasets/aime2025/README.md index 86e7d442..28894b42 100644 --- a/ais_bench/benchmark/configs/datasets/aime2025/README.md +++ b/ais_bench/benchmark/configs/datasets/aime2025/README.md @@ -23,7 +23,7 @@ rm aime2025.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|aime2025_gen|AIME2025|数据集生成式任务|准确率(accuracy)|0-shot|对话格式|aime2025_gen_0_shot_chat_prompt.py| -|aime2025_gen_0_shot_llmjudge|AIME2025|数据集生成式任务|准确率(accuracy)， 裁判模型评价的结果|0-shot|对话格式|aime2025_gen_0_shot_llmjudge.py| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|aime2025_gen_0_shot_chat_prompt|AIME2025 数据集生成式任务|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.aime2025.aime2025_gen_0_shot_chat_prompt import aime2025_datasets as datasets`|[aime2025_gen_0_shot_chat_prompt.py](aime2025_gen_0_shot_chat_prompt.py)| +|aime2025_gen_0_shot_llmjudge|AIME2025 数据集生成式任务|准确率(accuracy)， 裁判模型评价的结果|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.aime2025.aime2025_gen_0_shot_llmjudge import aime2025_datasets as datasets`|[aime2025_gen_0_shot_llmjudge.py](aime2025_gen_0_shot_llmjudge.py)| diff --git a/ais_bench/benchmark/configs/datasets/aime2025/README_en.md b/ais_bench/benchmark/configs/datasets/aime2025/README_en.md index 273dc9fb..42b5fb3e 100644 --- a/ais_bench/benchmark/configs/datasets/aime2025/README_en.md +++ b/ais_bench/benchmark/configs/datasets/aime2025/README_en.md @@ -23,7 +23,7 @@ rm aime2025.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| aime2025_gen | Generative task for the AIME2025 dataset | Accuracy | 0-shot | Chat format | aime2025_gen_0_shot_chat_prompt.py | -| aime2025_gen_0_shot_llmjudge | AIME2025 | Generative task for the AIME2025 dataset | Accuracy evaluated by judge model | 0-shot | Chat format | aime2025_gen_0_shot_llmjudge.py | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| aime2025_gen_0_shot_chat_prompt | Generative task for the AIME2025 dataset | Accuracy | 0-shot | Chat format | `from ais_bench.benchmark.configs.datasets.aime2025.aime2025_gen_0_shot_chat_prompt import aime2025_datasets as datasets` | [aime2025_gen_0_shot_chat_prompt.py](aime2025_gen_0_shot_chat_prompt.py) | +| aime2025_gen_0_shot_llmjudge | Generative task for the AIME2025 dataset | Accuracy evaluated by judge model | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.aime2025.aime2025_gen_0_shot_llmjudge import aime2025_datasets as datasets`| [aime2025_gen_0_shot_llmjudge.py](aime2025_gen_0_shot_llmjudge.py) | diff --git a/ais_bench/benchmark/configs/datasets/aime2026/README.md b/ais_bench/benchmark/configs/datasets/aime2026/README.md index 5ec37f6f..b1a289bd 100644 --- a/ais_bench/benchmark/configs/datasets/aime2026/README.md +++ b/ais_bench/benchmark/configs/datasets/aime2026/README.md @@ -40,7 +40,7 @@ Remember to put your answer inside \boxed{}. ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|aime2026_gen|AIME2026 数据集生成式任务|准确率(accuracy)|0-shot|对话格式|aime2026_gen_0_shot_chat_prompt.py| -|aime2026_gen_0_shot_str|AIME2026 数据集生成式任务|准确率(accuracy)|0-shot|字符串格式|aime2026_gen_0_shot_str.py| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|aime2026_gen_0_shot_chat_prompt|AIME2026 数据集生成式任务|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.aime2026.aime2026_gen_0_shot_chat_prompt import aime2026_datasets as datasets`|[aime2026_gen_0_shot_chat_prompt.py](aime2026_gen_0_shot_chat_prompt.py)| +|aime2026_gen_0_shot_str|AIME2026 数据集生成式任务|准确率(accuracy)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.aime2026.aime2026_gen_0_shot_str import aime2026_datasets as datasets`|[aime2026_gen_0_shot_str.py](aime2026_gen_0_shot_str.py)| diff --git a/ais_bench/benchmark/configs/datasets/aime2026/README_en.md b/ais_bench/benchmark/configs/datasets/aime2026/README_en.md index ab928216..ff126556 100644 --- a/ais_bench/benchmark/configs/datasets/aime2026/README_en.md +++ b/ais_bench/benchmark/configs/datasets/aime2026/README_en.md @@ -40,7 +40,7 @@ Remember to put your answer inside \boxed{}. ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| aime2026_gen | Generative task for the AIME2026 dataset | Accuracy | 0-shot | Chat format | aime2026_gen_0_shot_chat_prompt.py | -| aime2026_gen_0_shot_str | Generative task for the AIME2026 dataset | Accuracy | 0-shot | String format | aime2026_gen_0_shot_str.py | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| aime2026_gen_0_shot_chat_prompt | Generative task for the AIME2026 dataset | Accuracy | 0-shot | Chat format | `from ais_bench.benchmark.configs.datasets.aime2026.aime2026_gen_0_shot_chat_prompt import aime2026_datasets as datasets` | [aime2026_gen_0_shot_chat_prompt.py](aime2026_gen_0_shot_chat_prompt.py) | +| aime2026_gen_0_shot_str | Generative task for the AIME2026 dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.aime2026.aime2026_gen_0_shot_str import aime2026_datasets as datasets`|[aime2026_gen_0_shot_str.py](aime2026_gen_0_shot_str.py) | diff --git a/ais_bench/benchmark/configs/datasets/bbh/README.md b/ais_bench/benchmark/configs/datasets/bbh/README.md index aa1a5726..b708ca29 100644 --- a/ais_bench/benchmark/configs/datasets/bbh/README.md +++ b/ais_bench/benchmark/configs/datasets/bbh/README.md @@ -78,6 +78,6 @@ rm BBH.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|bbh_gen_3_shot_cot_chat|BBH数据集生成式任务|score(accuracy)|3-shot|对话格式|[bbh_gen_3_shot_cot_chat.py](bbh_gen_3_shot_cot_chat.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|bbh_gen_3_shot_cot_chat|BBH数据集生成式任务|score(accuracy)|3-shot|对话格式|`from ais_bench.benchmark.configs.datasets.bbh.bbh_gen_3_shot_cot_chat import bbh_datasets as datasets`|[bbh_gen_3_shot_cot_chat.py](bbh_gen_3_shot_cot_chat.py)| diff --git a/ais_bench/benchmark/configs/datasets/bbh/README_en.md b/ais_bench/benchmark/configs/datasets/bbh/README_en.md index c7cb4a5b..36c4c1a6 100644 --- a/ais_bench/benchmark/configs/datasets/bbh/README_en.md +++ b/ais_bench/benchmark/configs/datasets/bbh/README_en.md @@ -78,6 +78,6 @@ rm BBH.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| bbh_gen_3_shot_cot_chat | Generative task for the BBH dataset | Score (Accuracy) | 3-shot | Chat format | [bbh_gen_3_shot_cot_chat.py](bbh_gen_3_shot_cot_chat.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| bbh_gen_3_shot_cot_chat | Generative task for the BBH dataset | Score (Accuracy) | 3-shot | Chat format |`from ais_bench.benchmark.configs.datasets.bbh.bbh_gen_3_shot_cot_chat import bbh_datasets as datasets`| [bbh_gen_3_shot_cot_chat.py](bbh_gen_3_shot_cot_chat.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/ceval/README.md b/ais_bench/benchmark/configs/datasets/ceval/README.md index 18357b98..baae8651 100644 --- a/ais_bench/benchmark/configs/datasets/ceval/README.md +++ b/ais_bench/benchmark/configs/datasets/ceval/README.md @@ -184,9 +184,9 @@ rm ceval-exam.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|ceval_gen_0_shot_str|C-Eval数据集生成式任务|accuracy|0-shot|字符串格式|[ceval_gen_0_shot_str.py](ceval_gen_0_shot_str.py)| -|ceval_gen_5_shot_str|C-Eval数据集生成式任务|accuracy|5-shot|字符串格式|[ceval_gen_5_shot_str.py](ceval_gen_5_shot_str.py)| -|ceval_gen_0_shot_cot_chat_prompt|C-Eval数据集生成式任务，prompt带逻辑链（对齐DeepSeek R1精度测试）|accuracy|0-shot|对话格式|[ceval_gen_0_shot_cot_chat_prompt.py](ceval_gen_0_shot_cot_chat_prompt.py)| -|ceval_ppl_0_shot_str|C-Eval数据集PPL任务|accuracy|0-shot|字符串格式|[ceval_ppl_0_shot_str.py](ceval_ppl_0_shot_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|ceval_gen_0_shot_str|C-Eval数据集生成式任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_0_shot_str import ceval_datasets as datasets`|[ceval_gen_0_shot_str.py](ceval_gen_0_shot_str.py)| +|ceval_gen_5_shot_str|C-Eval数据集生成式任务|accuracy|5-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets`|[ceval_gen_5_shot_str.py](ceval_gen_5_shot_str.py)| +|ceval_gen_0_shot_cot_chat_prompt|C-Eval数据集生成式任务，prompt带逻辑链（对齐DeepSeek R1精度测试）|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_0_shot_cot_chat_prompt import ceval_datasets as datasets`|[ceval_gen_0_shot_cot_chat_prompt.py](ceval_gen_0_shot_cot_chat_prompt.py)| +|ceval_ppl_0_shot_str|C-Eval数据集PPL任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.ceval.ceval_ppl_0_shot_str import ceval_datasets as datasets`|[ceval_ppl_0_shot_str.py](ceval_ppl_0_shot_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/ceval/README_en.md b/ais_bench/benchmark/configs/datasets/ceval/README_en.md index 181c0f65..733f4c13 100644 --- a/ais_bench/benchmark/configs/datasets/ceval/README_en.md +++ b/ais_bench/benchmark/configs/datasets/ceval/README_en.md @@ -184,9 +184,9 @@ rm ceval-exam.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| ceval_gen_0_shot_str | Generative task for the C-Eval dataset | Accuracy | 0-shot | String format | [ceval_gen_0_shot_str.py](ceval_gen_0_shot_str.py) | -| ceval_gen_5_shot_str | Generative task for the C-Eval dataset | Accuracy | 5-shot | String format | [ceval_gen_5_shot_str.py](ceval_gen_5_shot_str.py) | -| ceval_gen_0_shot_cot_chat_prompt | Generative task for the C-Eval dataset with logical chain in prompt (aligned with DeepSeek R1 accuracy test) | Accuracy | 0-shot | Chat format | [ceval_gen_0_shot_cot_chat_prompt.py](ceval_gen_0_shot_cot_chat_prompt.py) | -| ceval_ppl_0_shot_str | PPL task for the C-Eval dataset | Accuracy | 0-shot | String format | [ceval_ppl_0_shot_str.py](ceval_ppl_0_shot_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| ceval_gen_0_shot_str | Generative task for the C-Eval dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_0_shot_str import ceval_datasets as datasets`| [ceval_gen_0_shot_str.py](ceval_gen_0_shot_str.py) | +| ceval_gen_5_shot_str | Generative task for the C-Eval dataset | Accuracy | 5-shot | String format |`from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets`| [ceval_gen_5_shot_str.py](ceval_gen_5_shot_str.py) | +| ceval_gen_0_shot_cot_chat_prompt | Generative task for the C-Eval dataset with logical chain in prompt (aligned with DeepSeek R1 accuracy test) | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_0_shot_cot_chat_prompt import ceval_datasets as datasets`| [ceval_gen_0_shot_cot_chat_prompt.py](ceval_gen_0_shot_cot_chat_prompt.py) | +| ceval_ppl_0_shot_str | PPL task for the C-Eval dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.ceval.ceval_ppl_0_shot_str import ceval_datasets as datasets`| [ceval_ppl_0_shot_str.py](ceval_ppl_0_shot_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/cmmlu/README.md b/ais_bench/benchmark/configs/datasets/cmmlu/README.md index bb843067..f9651cb8 100644 --- a/ais_bench/benchmark/configs/datasets/cmmlu/README.md +++ b/ais_bench/benchmark/configs/datasets/cmmlu/README.md @@ -157,8 +157,8 @@ rm cmmlu.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|cmmlu_gen_0_shot_cot_chat_prompt|CMMLU数据集生成式任务, prompt带逻辑链|accuracy|0-shot|对话格式|[cmmlu_gen_0_shot_cot_chat_prompt.py](cmmlu_gen_0_shot_cot_chat_prompt.py)| -|cmmlu_gen_5_shot_cot_chat_prompt|CMMLU数据集生成式任务, prompt带逻辑链|accuracy|5-shot|对话格式|[cmmlu_gen_5_shot_cot_chat_prompt.py](cmmlu_gen_5_shot_cot_chat_prompt.py)| -|cmmlu_ppl_0_shot_cot_chat_prompt|CMMLU数据集PPL任务，prompt带逻辑链|accuracy|0-shot|对话格式|[cmmlu_ppl_0_shot_cot_chat_prompt.py](cmmlu_ppl_0_shot_cot_chat_prompt.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|cmmlu_gen_0_shot_cot_chat_prompt|CMMLU数据集生成式任务, prompt带逻辑链|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.cmmlu.cmmlu_gen_0_shot_cot_chat_prompt import cmmlu_datasets as datasets`|[cmmlu_gen_0_shot_cot_chat_prompt.py](cmmlu_gen_0_shot_cot_chat_prompt.py)| +|cmmlu_gen_5_shot_cot_chat_prompt|CMMLU数据集生成式任务, prompt带逻辑链|accuracy|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.cmmlu.cmmlu_gen_5_shot_cot_chat_prompt import cmmlu_datasets as datasets`|[cmmlu_gen_5_shot_cot_chat_prompt.py](cmmlu_gen_5_shot_cot_chat_prompt.py)| +|cmmlu_ppl_0_shot_cot_chat_prompt|CMMLU数据集PPL任务，prompt带逻辑链|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.cmmlu.cmmlu_ppl_0_shot_cot_chat_prompt import cmmlu_datasets as datasets`|[cmmlu_ppl_0_shot_cot_chat_prompt.py](cmmlu_ppl_0_shot_cot_chat_prompt.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/cmmlu/README_en.md b/ais_bench/benchmark/configs/datasets/cmmlu/README_en.md index 4a55905a..214e499b 100644 --- a/ais_bench/benchmark/configs/datasets/cmmlu/README_en.md +++ b/ais_bench/benchmark/configs/datasets/cmmlu/README_en.md @@ -157,8 +157,8 @@ rm cmmlu.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| cmmlu_gen_0_shot_cot_chat_prompt | Generative task for the CMMLU dataset with logical chain in prompt | Accuracy | 0-shot | Chat format | [cmmlu_gen_0_shot_cot_chat_prompt.py](cmmlu_gen_0_shot_cot_chat_prompt.py) | -| cmmlu_gen_5_shot_cot_chat_prompt | Generative task for the CMMLU dataset with logical chain in prompt | Accuracy | 5-shot | Chat format | [cmmlu_gen_5_shot_cot_chat_prompt.py](cmmlu_gen_5_shot_cot_chat_prompt.py) | -| cmmlu_ppl_0_shot_cot_chat_prompt | PPL task for the CMMLU dataset with logical chain in prompt | Accuracy | 0-shot | Chat format | [cmmlu_ppl_0_shot_cot_chat_prompt.py](cmmlu_ppl_0_shot_cot_chat_prompt.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| cmmlu_gen_0_shot_cot_chat_prompt | Generative task for the CMMLU dataset with logical chain in prompt | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.cmmlu.cmmlu_gen_0_shot_cot_chat_prompt import cmmlu_datasets as datasets`| [cmmlu_gen_0_shot_cot_chat_prompt.py](cmmlu_gen_0_shot_cot_chat_prompt.py) | +| cmmlu_gen_5_shot_cot_chat_prompt | Generative task for the CMMLU dataset with logical chain in prompt | Accuracy | 5-shot | Chat format |`from ais_bench.benchmark.configs.datasets.cmmlu.cmmlu_gen_5_shot_cot_chat_prompt import cmmlu_datasets as datasets`| [cmmlu_gen_5_shot_cot_chat_prompt.py](cmmlu_gen_5_shot_cot_chat_prompt.py) | +| cmmlu_ppl_0_shot_cot_chat_prompt | PPL task for the CMMLU dataset with logical chain in prompt | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.cmmlu.cmmlu_ppl_0_shot_cot_chat_prompt import cmmlu_datasets as datasets`| [cmmlu_ppl_0_shot_cot_chat_prompt.py](cmmlu_ppl_0_shot_cot_chat_prompt.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/dapo_math/README.md b/ais_bench/benchmark/configs/datasets/dapo_math/README.md index 891092c1..3085ad56 100644 --- a/ais_bench/benchmark/configs/datasets/dapo_math/README.md +++ b/ais_bench/benchmark/configs/datasets/dapo_math/README.md @@ -33,10 +33,10 @@ rm -rf dapo-math-17k/data ``` ## 可用数据集任务 -| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 对应源码配置文件路径 | -| --- | --- | --- | --- | --- | --- | -| dapo_math_gen_0_shot_str | DAPO-math-17k 数据集生成式任务，使用 Minerva 方式提取答案 | accuracy | 0-shot | 字符串格式 | [dapo_math_gen_0_shot_str.py](dapo_math_gen_0_shot_str.py) | -| dapo_math_gen_0_shot_cot_str | DAPO-math-17k 数据集生成式任务，使用严格 boxed 方式提取答案 | accuracy | 0-shot | 字符串格式 | [dapo_math_gen_0_shot_cot_str.py](dapo_math_gen_0_shot_cot_str.py) | +| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| dapo_math_gen_0_shot_str | DAPO-math-17k 数据集生成式任务，使用 Minerva 方式提取答案 | accuracy | 0-shot | 字符串格式 | `from ais_bench.benchmark.configs.datasets.dapo_math.dapo_math_gen_0_shot_str import dapo_math_datasets as datasets` | [dapo_math_gen_0_shot_str.py](dapo_math_gen_0_shot_str.py) | +| dapo_math_gen_0_shot_cot_str | DAPO-math-17k 数据集生成式任务，使用严格 boxed 方式提取答案 | accuracy | 0-shot | 字符串格式 | `from ais_bench.benchmark.configs.datasets.dapo_math.dapo_math_gen_0_shot_cot_str import dapo_math_datasets as datasets` | [dapo_math_gen_0_shot_cot_str.py](dapo_math_gen_0_shot_cot_str.py) | ## 评估方式说明 数据集支持两种答案提取和评估方式： diff --git a/ais_bench/benchmark/configs/datasets/dapo_math/README_en.md b/ais_bench/benchmark/configs/datasets/dapo_math/README_en.md index 13209768..77599dc1 100644 --- a/ais_bench/benchmark/configs/datasets/dapo_math/README_en.md +++ b/ais_bench/benchmark/configs/datasets/dapo_math/README_en.md @@ -33,10 +33,10 @@ rm -rf dapo-math-17k/data ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| dapo_math_gen_0_shot_str | Generative task for DAPO-math-17k dataset, using Minerva method to extract answers | accuracy | 0-shot | String Format | [dapo_math_gen_0_shot_str.py](dapo_math_gen_0_shot_str.py) | -| dapo_math_gen_0_shot_cot_str | Generative task for DAPO-math-17k dataset, using strict boxed method to extract answers | accuracy | 0-shot | String Format | [dapo_math_gen_0_shot_cot_str.py](dapo_math_gen_0_shot_cot_str.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| dapo_math_gen_0_shot_str | Generative task for DAPO-math-17k dataset, using Minerva method to extract answers | accuracy | 0-shot | String Format |`from ais_bench.benchmark.configs.datasets.dapo_math.dapo_math_gen_0_shot_str import dapo_math_datasets as datasets`| [dapo_math_gen_0_shot_str.py](dapo_math_gen_0_shot_str.py) | +| dapo_math_gen_0_shot_cot_str | Generative task for DAPO-math-17k dataset, using strict boxed method to extract answers | accuracy | 0-shot | String Format |`from ais_bench.benchmark.configs.datasets.dapo_math.dapo_math_gen_0_shot_cot_str import dapo_math_datasets as datasets`| [dapo_math_gen_0_shot_cot_str.py](dapo_math_gen_0_shot_cot_str.py) | ## Evaluation Method Description The dataset supports two answer extraction and evaluation methods: diff --git a/ais_bench/benchmark/configs/datasets/demo/README.md b/ais_bench/benchmark/configs/datasets/demo/README.md index 1b464b8f..ba351340 100644 --- a/ais_bench/benchmark/configs/datasets/demo/README.md +++ b/ais_bench/benchmark/configs/datasets/demo/README.md @@ -24,7 +24,7 @@ rm gsm8k.zip └── train_socratic.jsonl ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|demo_gsm8k_gen_4_shot_cot_chat_prompt|gsm8k数据集生成式任务(只取8条数据)，带逻辑链|accuracy|4-shot|字符串格式|[demo_gsm8k_gen_4_shot_cot_chat_prompt.py](demo_gsm8k_gen_4_shot_cot_chat_prompt.py)| -|demo_gsm8k_gen_0_shot_cot_str_perf|gsm8k数据集生成式任务(只取8条数据)，带逻辑链|性能评测|0-shot|字符串格式|[demo_gsm8k_gen_0_shot_cot_str_perf.py](demo_gsm8k_gen_0_shot_cot_str_perf.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|demo_gsm8k_gen_4_shot_cot_chat_prompt|gsm8k数据集生成式任务(只取8条数据)，带逻辑链|accuracy|4-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets` |[demo_gsm8k_gen_4_shot_cot_chat_prompt.py](demo_gsm8k_gen_4_shot_cot_chat_prompt.py)| +|demo_gsm8k_gen_0_shot_cot_str_perf|gsm8k数据集生成式任务(只取8条数据)，带逻辑链|性能评测|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_0_shot_cot_str_perf import gsm8k_datasets as datasets` |[demo_gsm8k_gen_0_shot_cot_str_perf.py](demo_gsm8k_gen_0_shot_cot_str_perf.py)| diff --git a/ais_bench/benchmark/configs/datasets/demo/README_en.md b/ais_bench/benchmark/configs/datasets/demo/README_en.md index 219d2dda..a6f50681 100644 --- a/ais_bench/benchmark/configs/datasets/demo/README_en.md +++ b/ais_bench/benchmark/configs/datasets/demo/README_en.md @@ -25,10 +25,10 @@ rm gsm8k.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| demo_gsm8k_gen_4_shot_cot_chat_prompt | Generative task for the GSM8K dataset (only 8 entries used) with logical chain | Accuracy | 4-shot | String format | [demo_gsm8k_gen_4_shot_cot_chat_prompt.py](demo_gsm8k_gen_0_shot_cot_str_perf.py) | -| demo_gsm8k_gen_0_shot_cot_str_perf | Generative task for the GSM8K dataset (only 8 entries used) with logical chain | Performance Evaluation | 0-shot | String format | [demo_gsm8k_gen_0_shot_cot_str_perf.py](demo_gsm8k_gen_0_shot_cot_str_perf.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| demo_gsm8k_gen_4_shot_cot_chat_prompt | Generative task for the GSM8K dataset (only 8 entries used) with logical chain | Accuracy | 4-shot | String format | `from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets` | [demo_gsm8k_gen_4_shot_cot_chat_prompt.py](demo_gsm8k_gen_4_shot_cot_chat_prompt.py) | +| demo_gsm8k_gen_0_shot_cot_str_perf | Generative task for the GSM8K dataset (only 8 entries used) with logical chain | Performance Evaluation | 0-shot | String format | `from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_0_shot_cot_str_perf import gsm8k_datasets as datasets` | [demo_gsm8k_gen_0_shot_cot_str_perf.py](demo_gsm8k_gen_0_shot_cot_str_perf.py) | ### Translation Notes diff --git a/ais_bench/benchmark/configs/datasets/docvqa/README.md b/ais_bench/benchmark/configs/datasets/docvqa/README.md index 57c6a99c..332c918a 100644 --- a/ais_bench/benchmark/configs/datasets/docvqa/README.md +++ b/ais_bench/benchmark/configs/datasets/docvqa/README.md @@ -24,6 +24,6 @@ wget https://opencompass.openxlab.space/utils/VLMEval/DocVQA_VAL.tsv ## 可用数据集任务 #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|docvqa_gen|docvqa数据集生成式任务|anls|0-shot|字符串格式|[docvqa_gen.py](docvqa_gen.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|docvqa_gen|docvqa数据集生成式任务|anls|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.docvqa.docvqa_gen import docvqa_datasets as datasets`|[docvqa_gen.py](docvqa_gen.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/docvqa/README_en.md b/ais_bench/benchmark/configs/datasets/docvqa/README_en.md index bd76bfd1..7f455176 100644 --- a/ais_bench/benchmark/configs/datasets/docvqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/docvqa/README_en.md @@ -24,6 +24,6 @@ wget https://opencompass.openxlab.space/utils/VLMEval/DocVQA_VAL.tsv ## Available Dataset Tasks #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -|docvqa_gen|docvqa dataset generative task|anls|0-shot|String format|[docvqa_gen.py](docvqa_gen.py)| +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +|docvqa_gen|docvqa dataset generative task|anls|0-shot|String format|`from ais_bench.benchmark.configs.datasets.docvqa.docvqa_gen import docvqa_datasets as datasets`|[docvqa_gen.py](docvqa_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/drop/README.md b/ais_bench/benchmark/configs/datasets/drop/README.md index 30e69a87..34082549 100644 --- a/ais_bench/benchmark/configs/datasets/drop/README.md +++ b/ais_bench/benchmark/configs/datasets/drop/README.md @@ -22,7 +22,7 @@ rm drop_simple_eval.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|drop_gen_0_shot_str|drop数据集生成式任务|accuracy(pass@1)|0-shot|字符串格式|[drop_gen_0_shot_str.py](drop_gen_0_shot_str.py)| -|drop_gen_3_shot_str|drop数据集生成式任务|accuracy(pass@1)|3-shot|字符串格式|[drop_gen_3_shot_str.py](drop_gen_3_shot_str.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|drop_gen_0_shot_str|drop数据集生成式任务|accuracy(pass@1)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.drop.drop_gen_0_shot_str import drop_datasets as datasets`|[drop_gen_0_shot_str.py](drop_gen_0_shot_str.py)| +|drop_gen_3_shot_str|drop数据集生成式任务|accuracy(pass@1)|3-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.drop.drop_gen_3_shot_str import drop_datasets as datasets`|[drop_gen_3_shot_str.py](drop_gen_3_shot_str.py)| diff --git a/ais_bench/benchmark/configs/datasets/drop/README_en.md b/ais_bench/benchmark/configs/datasets/drop/README_en.md index c5bf7d16..3532edae 100644 --- a/ais_bench/benchmark/configs/datasets/drop/README_en.md +++ b/ais_bench/benchmark/configs/datasets/drop/README_en.md @@ -22,7 +22,7 @@ rm drop_simple_eval.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| drop_gen_0_shot_str | Generative task for the DROP dataset | Accuracy (pass@1) | 0-shot | String format | [drop_gen_0_shot_str.py](drop_gen_0_shot_str.py) | -| drop_gen_3_shot_str | Generative task for the DROP dataset | Accuracy (pass@1) | 3-shot | String format | [drop_gen_3_shot_str.py](drop_gen_3_shot_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| drop_gen_0_shot_str | Generative task for the DROP dataset | Accuracy (pass@1) | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.drop.drop_gen_0_shot_str import drop_datasets as datasets`| [drop_gen_0_shot_str.py](drop_gen_0_shot_str.py) | +| drop_gen_3_shot_str | Generative task for the DROP dataset | Accuracy (pass@1) | 3-shot | String format |`from ais_bench.benchmark.configs.datasets.drop.drop_gen_3_shot_str import drop_datasets as datasets`| [drop_gen_3_shot_str.py](drop_gen_3_shot_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/gpqa/README.md b/ais_bench/benchmark/configs/datasets/gpqa/README.md index b5825611..c28af0ad 100644 --- a/ais_bench/benchmark/configs/datasets/gpqa/README.md +++ b/ais_bench/benchmark/configs/datasets/gpqa/README.md @@ -25,8 +25,8 @@ rm gpqa.zip └── license.txt ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|gpqa_gen_0_shot_str|gpqa数据集生成式任务|accuracy(pass@1)|0-shot|字符串格式|[gpqa_gen_0_shot_str.py](gpqa_gen_0_shot_str.py)| -|gpqa_gen_0_shot_cot_chat_prompt|gpqa数据集生成式任务（对齐DeepSeek R1精度测试）|accuracy(pass@1)|0-shot|对话格式|[gpqa_gen_0_shot_cot_chat_prompt.py](gpqa_gen_0_shot_cot_chat_prompt.py)| -|gpqa_ppl_0_shot_str|gpqa数据集PPL任务|accuracy(pass@1)|0-shot|字符串格式|[gpqa_ppl_0_shot_str.py](gpqa_ppl_0_shot_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|gpqa_gen_0_shot_str|gpqa数据集生成式任务|accuracy(pass@1)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.gpqa.gpqa_gen_0_shot_str import gpqa_datasets as datasets`|[gpqa_gen_0_shot_str.py](gpqa_gen_0_shot_str.py)| +|gpqa_gen_0_shot_cot_chat_prompt|gpqa数据集生成式任务（对齐DeepSeek R1精度测试）|accuracy(pass@1)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.gpqa.gpqa_gen_0_shot_cot_chat_prompt import gpqa_datasets as datasets`|[gpqa_gen_0_shot_cot_chat_prompt.py](gpqa_gen_0_shot_cot_chat_prompt.py)| +|gpqa_ppl_0_shot_str|gpqa数据集PPL任务|accuracy(pass@1)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.gpqa.gpqa_ppl_0_shot_str import gpqa_datasets as datasets`|[gpqa_ppl_0_shot_str.py](gpqa_ppl_0_shot_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/gpqa/README_en.md b/ais_bench/benchmark/configs/datasets/gpqa/README_en.md index 56180a89..5a05f755 100644 --- a/ais_bench/benchmark/configs/datasets/gpqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/gpqa/README_en.md @@ -26,11 +26,11 @@ rm gpqa.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| gpqa_gen_0_shot_str | Generative task for the GPQA dataset | Accuracy (pass@1) | 0-shot | String format | [gpqa_gen_0_shot_str.py](gpqa_gen_0_shot_str.py) | -| gpqa_gen_0_shot_cot_chat_prompt | Generative task for the GPQA dataset (aligned with DeepSeek R1 accuracy test) | Accuracy (pass@1) | 0-shot | Chat format | [gpqa_gen_0_shot_cot_chat_prompt.py](gpqa_gen_0_shot_cot_chat_prompt.py) | -| gpqa_ppl_0_shot_str | PPL task for the GPQA dataset | Accuracy (pass@1) | 0-shot | String format | [gpqa_ppl_0_shot_str.py](gpqa_ppl_0_shot_str.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| gpqa_gen_0_shot_str | Generative task for the GPQA dataset | Accuracy (pass@1) | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.gpqa.gpqa_gen_0_shot_str import gpqa_datasets as datasets`| [gpqa_gen_0_shot_str.py](gpqa_gen_0_shot_str.py) | +| gpqa_gen_0_shot_cot_chat_prompt | Generative task for the GPQA dataset (aligned with DeepSeek R1 accuracy test) | Accuracy (pass@1) | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.gpqa.gpqa_gen_0_shot_cot_chat_prompt import gpqa_datasets as datasets`| [gpqa_gen_0_shot_cot_chat_prompt.py](gpqa_gen_0_shot_cot_chat_prompt.py) | +| gpqa_ppl_0_shot_str | PPL task for the GPQA dataset | Accuracy (pass@1) | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.gpqa.gpqa_ppl_0_shot_str import gpqa_datasets as datasets`| [gpqa_ppl_0_shot_str.py](gpqa_ppl_0_shot_str.py) | ### Translation Notes 1. **Term Consistency**: Technical terms such as "生成式任务" (generative task), "评估指标" (evaluation metric), and "对齐DeepSeek R1精度测试" (aligned with DeepSeek R1 accuracy test) follow standard expressions in AI dataset documentation to ensure clarity for technical users. diff --git a/ais_bench/benchmark/configs/datasets/gsm8k/README.md b/ais_bench/benchmark/configs/datasets/gsm8k/README.md index c6cb9e69..ac5e04e8 100644 --- a/ais_bench/benchmark/configs/datasets/gsm8k/README.md +++ b/ais_bench/benchmark/configs/datasets/gsm8k/README.md @@ -25,10 +25,10 @@ rm gsm8k.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|gsm8k_gen_4_shot_cot_str|gsm8k数据集生成式任务，带逻辑链|accuracy|4-shot|字符串格式|[gsm8k_gen_4_shot_cot_str.py](gsm8k_gen_4_shot_cot_str.py)| -|gsm8k_gen_4_shot_cot_chat_prompt|gsm8k数据集生成式任务，带逻辑链|accuracy|4-shot|对话格式|[gsm8k_gen_4_shot_cot_chat_prompt.py](gsm8k_gen_4_shot_cot_chat_prompt.py)| -|gsm8k_gen_0_shot_cot_str|gsm8k数据集生成式任务|accuracy|0-shot|字符串格式|[gsm8k_gen_0_shot_cot_str.py](gsm8k_gen_0_shot_cot_str.py)| -|gsm8k_gen_0_shot_cot_chat_prompt|gsm8k数据集生成式任务|accuracy|0-shot|对话格式|[gsm8k_gen_0_shot_cot_chat_prompt.py](gsm8k_gen_0_shot_cot_chat_prompt.py)| -|gsm8k_gen_0_shot_cot_str_perf|gsm8k数据集生成式任务（用于性能测评）|性能测评|0-shot|字符串格式|[gsm8k_gen_0_shot_cot_str_perf.py](gsm8k_gen_0_shot_cot_str_perf.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|gsm8k_gen_4_shot_cot_str|gsm8k数据集生成式任务，带逻辑链|accuracy|4-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets as datasets`|[gsm8k_gen_4_shot_cot_str.py](gsm8k_gen_4_shot_cot_str.py)| +|gsm8k_gen_4_shot_cot_chat_prompt|gsm8k数据集生成式任务，带逻辑链|accuracy|4-shot|对话格式|`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets`|[gsm8k_gen_4_shot_cot_chat_prompt.py](gsm8k_gen_4_shot_cot_chat_prompt.py)| +|gsm8k_gen_0_shot_cot_str|gsm8k数据集生成式任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as datasets`|[gsm8k_gen_0_shot_cot_str.py](gsm8k_gen_0_shot_cot_str.py)| +|gsm8k_gen_0_shot_cot_chat_prompt|gsm8k数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_chat_prompt import gsm8k_datasets as datasets`|[gsm8k_gen_0_shot_cot_chat_prompt.py](gsm8k_gen_0_shot_cot_chat_prompt.py)| +|gsm8k_gen_0_shot_cot_str_perf|gsm8k数据集生成式任务（用于性能测评）|性能测评|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str_perf import gsm8k_datasets as datasets`|[gsm8k_gen_0_shot_cot_str_perf.py](gsm8k_gen_0_shot_cot_str_perf.py)| diff --git a/ais_bench/benchmark/configs/datasets/gsm8k/README_en.md b/ais_bench/benchmark/configs/datasets/gsm8k/README_en.md index 4631d7c0..dcbeca5e 100644 --- a/ais_bench/benchmark/configs/datasets/gsm8k/README_en.md +++ b/ais_bench/benchmark/configs/datasets/gsm8k/README_en.md @@ -25,10 +25,10 @@ rm gsm8k.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| gsm8k_gen_4_shot_cot_str | Generative task for the GSM8K dataset with logical chain | Accuracy | 4-shot | String format | [gsm8k_gen_4_shot_cot_str.py](gsm8k_gen_4_shot_cot_str.py) | -| gsm8k_gen_4_shot_cot_chat_prompt | Generative task for the GSM8K dataset with logical chain | Accuracy | 4-shot | Chat format | [gsm8k_gen_4_shot_cot_chat_prompt.py](gsm8k_gen_4_shot_cot_chat_prompt.py) | -| gsm8k_gen_0_shot_cot_str | Generative task for the GSM8K dataset | Accuracy | 0-shot | String format | [gsm8k_gen_0_shot_cot_str.py](gsm8k_gen_0_shot_cot_str.py) | -| gsm8k_gen_0_shot_cot_chat_prompt | Generative task for the GSM8K dataset | Accuracy | 0-shot | Chat format | [gsm8k_gen_0_shot_cot_chat_prompt.py](gsm8k_gen_0_shot_cot_chat_prompt.py) | -| gsm8k_gen_0_shot_cot_str_perf | Generative task for the GSM8K dataset (for performance evaluation) | Performance Evaluation | 0-shot | String format | [gsm8k_gen_0_shot_cot_str_perf.py](gsm8k_gen_0_shot_cot_str_perf.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| gsm8k_gen_4_shot_cot_str | Generative task for the GSM8K dataset with logical chain | Accuracy | 4-shot | String format |`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets as datasets`| [gsm8k_gen_4_shot_cot_str.py](gsm8k_gen_4_shot_cot_str.py) | +| gsm8k_gen_4_shot_cot_chat_prompt | Generative task for the GSM8K dataset with logical chain | Accuracy | 4-shot | Chat format |`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets`| [gsm8k_gen_4_shot_cot_chat_prompt.py](gsm8k_gen_4_shot_cot_chat_prompt.py) | +| gsm8k_gen_0_shot_cot_str | Generative task for the GSM8K dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as datasets`| [gsm8k_gen_0_shot_cot_str.py](gsm8k_gen_0_shot_cot_str.py) | +| gsm8k_gen_0_shot_cot_chat_prompt | Generative task for the GSM8K dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_chat_prompt import gsm8k_datasets as datasets`| [gsm8k_gen_0_shot_cot_chat_prompt.py](gsm8k_gen_0_shot_cot_chat_prompt.py) | +| gsm8k_gen_0_shot_cot_str_perf | Generative task for the GSM8K dataset (for performance evaluation) | Performance Evaluation | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str_perf import gsm8k_datasets as datasets`| [gsm8k_gen_0_shot_cot_str_perf.py](gsm8k_gen_0_shot_cot_str_perf.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/hellaswag/README.md b/ais_bench/benchmark/configs/datasets/hellaswag/README.md index e5590adb..44b047cc 100644 --- a/ais_bench/benchmark/configs/datasets/hellaswag/README.md +++ b/ais_bench/benchmark/configs/datasets/hellaswag/README.md @@ -24,8 +24,8 @@ rm hellaswag.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|hellaswag_gen_0_shot_chat_prompt|hellaswag数据集生成式任务|accuracy|0-shot|对话格式|[hellaswag_gen_0_shot_chat_prompt.py](hellaswag_gen_0_shot_chat_prompt.py)| -|hellaswag_gen_10_shot_chat_prompt|hellaswag数据集生成式任务|accuracy|10-shot|对话格式|[hellaswag_gen_10_shot_chat_prompt.py](hellaswag_gen_10_shot_chat_prompt.py)| -|hellaswag_ppl_0_shot_chat_prompt|hellaswag数据集PPL任务|accuracy|0-shot|对话格式|[hellaswag_ppl_0_shot_chat_prompt.py](hellaswag_ppl_0_shot_chat_prompt.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|hellaswag_gen_0_shot_chat_prompt|hellaswag数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.hellaswag.hellaswag_gen_0_shot_chat_prompt import hellaswag_datasets as datasets`|[hellaswag_gen_0_shot_chat_prompt.py](hellaswag_gen_0_shot_chat_prompt.py)| +|hellaswag_gen_10_shot_chat_prompt|hellaswag数据集生成式任务|accuracy|10-shot|对话格式|`from ais_bench.benchmark.configs.datasets.hellaswag.hellaswag_gen_10_shot_chat_prompt import hellaswag_datasets as datasets`|[hellaswag_gen_10_shot_chat_prompt.py](hellaswag_gen_10_shot_chat_prompt.py)| +|hellaswag_ppl_0_shot_chat_prompt|hellaswag数据集PPL任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.hellaswag.hellaswag_ppl_0_shot_chat_prompt import hellaswag_datasets as datasets`|[hellaswag_ppl_0_shot_chat_prompt.py](hellaswag_ppl_0_shot_chat_prompt.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/hellaswag/README_en.md b/ais_bench/benchmark/configs/datasets/hellaswag/README_en.md index f840d06a..f8a66b49 100644 --- a/ais_bench/benchmark/configs/datasets/hellaswag/README_en.md +++ b/ais_bench/benchmark/configs/datasets/hellaswag/README_en.md @@ -24,8 +24,8 @@ rm hellaswag.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| hellaswag_gen_0_shot_chat_prompt | Generative task for the HellaSwag dataset | Accuracy | 0-shot | Chat format | [hellaswag_gen_0_shot_chat_prompt.py](hellaswag_gen_0_shot_chat_prompt.py) | -| hellaswag_gen_10_shot_chat_prompt | Generative task for the HellaSwag dataset | Accuracy | 10-shot | Chat format | [hellaswag_gen_10_shot_chat_prompt.py](hellaswag_gen_10_shot_chat_prompt.py) | -| hellaswag_ppl_0_shot_chat_prompt | PPL task for the hellaswag dataset | Accuracy | 0-shot | Chat format | [hellaswag_ppl_0_shot_chat_prompt.py](hellaswag_ppl_0_shot_chat_prompt.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| hellaswag_gen_0_shot_chat_prompt | Generative task for the HellaSwag dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.hellaswag.hellaswag_gen_0_shot_chat_prompt import hellaswag_datasets as datasets`| [hellaswag_gen_0_shot_chat_prompt.py](hellaswag_gen_0_shot_chat_prompt.py) | +| hellaswag_gen_10_shot_chat_prompt | Generative task for the HellaSwag dataset | Accuracy | 10-shot | Chat format |`from ais_bench.benchmark.configs.datasets.hellaswag.hellaswag_gen_10_shot_chat_prompt import hellaswag_datasets as datasets`| [hellaswag_gen_10_shot_chat_prompt.py](hellaswag_gen_10_shot_chat_prompt.py) | +| hellaswag_ppl_0_shot_chat_prompt | PPL task for the hellaswag dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.hellaswag.hellaswag_ppl_0_shot_chat_prompt import hellaswag_datasets as datasets`| [hellaswag_ppl_0_shot_chat_prompt.py](hellaswag_ppl_0_shot_chat_prompt.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/hle/README.md b/ais_bench/benchmark/configs/datasets/hle/README.md index 6dab0adc..d204cf49 100644 --- a/ais_bench/benchmark/configs/datasets/hle/README.md +++ b/ais_bench/benchmark/configs/datasets/hle/README.md @@ -6,7 +6,7 @@ HLE（Humanity's Last Exam）是 Center for AI Safety 发布的前沿多模态基准测试数据集，旨在成为最后一个广泛覆盖学科领域的闭卷学术基准测试。该数据集包含 2,500 道高质量题目，涵盖数学、人文科学、自然科学等多个学科领域。题目由领域专家精心设计，确保了题目的专业性和挑战性。HLE 支持纯文本和图像输入，并通过 LLM Judge 评估协议进行自动评分，同时提供置信度校准指标，适合全面评估模型在多学科知识和多模态理解方面的能力。 > 🔗 数据集主页链接: [https://huggingface.co/datasets/cais/hle](https://huggingface.co/datasets/cais/hle) -> +> > 🔗 官方 GitHub 仓库: [https://github.com/centerforaisafety/hle](https://github.com/centerforaisafety/hle) @@ -24,7 +24,7 @@ HLE（Humanity's Last Exam）是 Center for AI Safety 发布的前沿多模态 ## 可用数据集任务 -| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 对应源码配置文件路径 | -| --- | --- | --- | --- | --- | --- | -| hle_llmjudge | HLE 数据集 | 准确率 (accuracy)、置信度校准误差 (calibration_error) | 0-shot | 对话格式 | hle_llmjudge.py | +| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| hle | HLE 数据集 | 准确率 (accuracy)、置信度校准误差 (calibration_error) | 0-shot | 对话格式 | `from ais_bench.benchmark.configs.datasets.hle.hle_llmjudge import hle_datasets as datasets` | [hle_llmjudge.py](hle_llmjudge.py) | diff --git a/ais_bench/benchmark/configs/datasets/hle/README_en.md b/ais_bench/benchmark/configs/datasets/hle/README_en.md index 0bc49608..c34949ae 100644 --- a/ais_bench/benchmark/configs/datasets/hle/README_en.md +++ b/ais_bench/benchmark/configs/datasets/hle/README_en.md @@ -6,7 +6,7 @@ HLE (Humanity's Last Exam) is a frontier multimodal benchmark dataset released by the Center for AI Safety, designed to be the last widely covering closed-book academic benchmark across multiple subject domains. The dataset contains 2,500 high-quality questions covering multiple subject areas including mathematics, humanities, and natural sciences. The questions are carefully designed by domain experts, ensuring professional quality and challenging difficulty. HLE supports both pure text and image inputs, and uses the LLM Judge evaluation protocol for automatic scoring while providing confidence calibration metrics. It is suitable for comprehensive evaluation of models' capabilities in multidisciplinary knowledge and multimodal understanding. > 🔗 Dataset Homepage Link: [https://huggingface.co/datasets/cais/hle](https://huggingface.co/datasets/cais/hle) -> +> > 🔗 Official GitHub Repository: [https://github.com/centerforaisafety/hle](https://github.com/centerforaisafety/hle) @@ -24,7 +24,7 @@ HLE (Humanity's Last Exam) is a frontier multimodal benchmark dataset released b ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| hle_llmjudge | HLE dataset | Accuracy, Calibration Error | 0-shot | Chat format | hle_llmjudge.py | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| hle | HLE dataset | Accuracy, Calibration Error | 0-shot | Chat format | `from ais_bench.benchmark.configs.datasets.hle.hle_llmjudge import hle_datasets as datasets` | [hle_llmjudge.py](hle_llmjudge.py) | diff --git a/ais_bench/benchmark/configs/datasets/humaneval/README.md b/ais_bench/benchmark/configs/datasets/humaneval/README.md index f709d2be..f8944918 100644 --- a/ais_bench/benchmark/configs/datasets/humaneval/README.md +++ b/ais_bench/benchmark/configs/datasets/humaneval/README.md @@ -28,6 +28,6 @@ rm humaneval.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|humaneval_gen_0_shot|humaneval数据集生成式任务|pass@1|0-shot|字符串格式|[humaneval_gen_0_shot.py](humaneval_gen_0_shot.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|humaneval_gen_0_shot|humaneval数据集生成式任务|pass@1|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.humaneval.humaneval_gen_0_shot import humaneval_datasets as datasets`|[humaneval_gen_0_shot.py](humaneval_gen_0_shot.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/humaneval/README_en.md b/ais_bench/benchmark/configs/datasets/humaneval/README_en.md index e6784e03..4e06434f 100644 --- a/ais_bench/benchmark/configs/datasets/humaneval/README_en.md +++ b/ais_bench/benchmark/configs/datasets/humaneval/README_en.md @@ -28,6 +28,6 @@ rm humaneval.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| humaneval_gen_0_shot | Generative task for the HumanEval dataset | pass@1 | 0-shot | String format | [humaneval_gen_0_shot.py](humaneval_gen_0_shot.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| humaneval_gen_0_shot | Generative task for the HumanEval dataset | pass@1 | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.humaneval.humaneval_gen_0_shot import humaneval_datasets as datasets`| [humaneval_gen_0_shot.py](humaneval_gen_0_shot.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/humanevalx/README.md b/ais_bench/benchmark/configs/datasets/humanevalx/README.md index 014f0df4..14ae0ff8 100644 --- a/ais_bench/benchmark/configs/datasets/humanevalx/README.md +++ b/ais_bench/benchmark/configs/datasets/humanevalx/README.md @@ -32,6 +32,6 @@ rm humanevalx.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|humanevalx_gen_0_shot|humanevalx数据集生成式任务|pass@1|0-shot|字符串格式|[humanevalx_gen_0_shot.py](humanevalx_gen_0_shot.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|humanevalx_gen_0_shot|humanevalx数据集生成式任务|pass@1|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.humanevalx.humanevalx_gen_0_shot import humanevalx_datasets as datasets`|[humanevalx_gen_0_shot.py](humanevalx_gen_0_shot.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/humanevalx/README_en.md b/ais_bench/benchmark/configs/datasets/humanevalx/README_en.md index 344c9f35..f5b602e3 100644 --- a/ais_bench/benchmark/configs/datasets/humanevalx/README_en.md +++ b/ais_bench/benchmark/configs/datasets/humanevalx/README_en.md @@ -32,6 +32,6 @@ rm humanevalx.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| humanevalx_gen_0_shot | Generative task for the HumanEvalX dataset | pass@1 | 0-shot | String format | [humanevalx_gen_0_shot.py](humanevalx_gen_0_shot.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| humanevalx_gen_0_shot | Generative task for the HumanEvalX dataset | pass@1 | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.humanevalx.humanevalx_gen_0_shot import humanevalx_datasets as datasets`| [humanevalx_gen_0_shot.py](humanevalx_gen_0_shot.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/ifeval/README.md b/ais_bench/benchmark/configs/datasets/ifeval/README.md index 2bbeb9b0..e9b61641 100644 --- a/ais_bench/benchmark/configs/datasets/ifeval/README.md +++ b/ais_bench/benchmark/configs/datasets/ifeval/README.md @@ -29,6 +29,6 @@ rm ifeval.zip ## 可用数据集任务 ### ifeval_0_shot_gen_str -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|ifeval_0_shot_gen_str|ifeval数据集生成式任务|accuracy|0-shot|字符串格式|[ifeval_0_shot_gen_str.py](ifeval_0_shot_gen_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|ifeval_0_shot_gen_str|ifeval数据集生成式任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.ifeval.ifeval_0_shot_gen_str import ifeval_datasets as datasets`|[ifeval_0_shot_gen_str.py](ifeval_0_shot_gen_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/ifeval/README_en.md b/ais_bench/benchmark/configs/datasets/ifeval/README_en.md index e0e1590d..07fb1967 100644 --- a/ais_bench/benchmark/configs/datasets/ifeval/README_en.md +++ b/ais_bench/benchmark/configs/datasets/ifeval/README_en.md @@ -29,6 +29,6 @@ rm ifeval.zip ## Available Dataset Tasks ### ifeval_0_shot_gen_str -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| ifeval_0_shot_gen_str | Generative task for the IFEval dataset | Accuracy | 0-shot | String format | [ifeval_0_shot_gen_str.py](ifeval_0_shot_gen_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| ifeval_0_shot_gen_str | Generative task for the IFEval dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.ifeval.ifeval_0_shot_gen_str import ifeval_datasets as datasets`| [ifeval_0_shot_gen_str.py](ifeval_0_shot_gen_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/infovqa/README.md b/ais_bench/benchmark/configs/datasets/infovqa/README.md index 323d0b08..19ac9975 100644 --- a/ais_bench/benchmark/configs/datasets/infovqa/README.md +++ b/ais_bench/benchmark/configs/datasets/infovqa/README.md @@ -24,6 +24,6 @@ wget https://opencompass.openxlab.space/utils/VLMEval/InfoVQA_VAL.tsv ## 可用数据集任务 #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|infovqa_gen|infovqa数据集生成式任务|anls|0-shot|字符串格式|[infovqa_gen.py](infovqa_gen.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|infovqa_gen|infovqa数据集生成式任务|anls|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.infovqa.infovqa_gen import infovqa_datasets as datasets`|[infovqa_gen.py](infovqa_gen.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/infovqa/README_en.md b/ais_bench/benchmark/configs/datasets/infovqa/README_en.md index 4574d31c..ea23b02a 100644 --- a/ais_bench/benchmark/configs/datasets/infovqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/infovqa/README_en.md @@ -24,6 +24,6 @@ wget https://opencompass.openxlab.space/utils/VLMEval/InfoVQA_VAL.tsv ## Available Dataset Tasks #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -|infovqa_gen|infovqa dataset generative task|anls|0-shot|String format|[infovqa_gen.py](infovqa_gen.py)| +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +|infovqa_gen|infovqa dataset generative task|anls|0-shot|String format|`from ais_bench.benchmark.configs.datasets.infovqa.infovqa_gen import infovqa_datasets as datasets`|[infovqa_gen.py](infovqa_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/lambada/README.md b/ais_bench/benchmark/configs/datasets/lambada/README.md index e0cee1f2..ae9fc7ef 100644 --- a/ais_bench/benchmark/configs/datasets/lambada/README.md +++ b/ais_bench/benchmark/configs/datasets/lambada/README.md @@ -24,7 +24,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|lambada_gen_0_shot_chat|lambada数据集生成式任务|accuracy|0-shot|对话格式|[lambada_gen_0_shot_chat.py](lambada_gen_0_shot_chat.py)| -|lambada_gen_0_shot_str|lambada数据集生成式任务|accuracy|0-shot|字符串格式|[lambada_gen_0_shot_str.py](lambada_gen_0_shot_str.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|lambada_gen_0_shot_chat|lambada数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.lambada.lambada_gen_0_shot_chat import lambada_datasets as datasets`|[lambada_gen_0_shot_chat.py](lambada_gen_0_shot_chat.py)| +|lambada_gen_0_shot_str|lambada数据集生成式任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.lambada.lambada_gen_0_shot_str import lambada_datasets as datasets`|[lambada_gen_0_shot_str.py](lambada_gen_0_shot_str.py)| diff --git a/ais_bench/benchmark/configs/datasets/lambada/README_en.md b/ais_bench/benchmark/configs/datasets/lambada/README_en.md index f6069a6e..a0f8b671 100644 --- a/ais_bench/benchmark/configs/datasets/lambada/README_en.md +++ b/ais_bench/benchmark/configs/datasets/lambada/README_en.md @@ -25,10 +25,10 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| lambada_gen_0_shot_chat | Generative task for the LAMBADA dataset | Accuracy | 0-shot | Chat format | [lambada_gen_0_shot_chat.py](lambada_gen_0_shot_chat.py) | -| lambada_gen_0_shot_str | Generative task for the LAMBADA dataset | Accuracy | 0-shot | String format | [lambada_gen_0_shot_str.py](lambada_gen_0_shot_str.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| lambada_gen_0_shot_chat | Generative task for the LAMBADA dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.lambada.lambada_gen_0_shot_chat import lambada_datasets as datasets`| [lambada_gen_0_shot_chat.py](lambada_gen_0_shot_chat.py) | +| lambada_gen_0_shot_str | Generative task for the LAMBADA dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.lambada.lambada_gen_0_shot_str import lambada_datasets as datasets`| [lambada_gen_0_shot_str.py](lambada_gen_0_shot_str.py) | ### Translation Notes diff --git a/ais_bench/benchmark/configs/datasets/lcsts/README.md b/ais_bench/benchmark/configs/datasets/lcsts/README.md index a6b476f6..c66296ef 100644 --- a/ais_bench/benchmark/configs/datasets/lcsts/README.md +++ b/ais_bench/benchmark/configs/datasets/lcsts/README.md @@ -25,7 +25,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|lcsts_gen_0_shot_chat|lcsts数据集生成式任务|accuracy|0-shot|对话格式|[lcsts_gen_0_shot_chat.py](lcsts_gen_0_shot_chat.py)| -|lcsts_gen_0_shot_str|lcsts数据集生成式任务|accuracy|0-shot|字符串格式|[lcsts_gen_0_shot_str.py](lcsts_gen_0_shot_str.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|lcsts_gen_0_shot_chat|lcsts数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.lcsts.lcsts_gen_0_shot_chat import lcsts_datasets as datasets`|[lcsts_gen_0_shot_chat.py](lcsts_gen_0_shot_chat.py)| +|lcsts_gen_0_shot_str|lcsts数据集生成式任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.lcsts.lcsts_gen_0_shot_str import lcsts_datasets as datasets`|[lcsts_gen_0_shot_str.py](lcsts_gen_0_shot_str.py)| diff --git a/ais_bench/benchmark/configs/datasets/lcsts/README_en.md b/ais_bench/benchmark/configs/datasets/lcsts/README_en.md index 82bfa706..764a0eca 100644 --- a/ais_bench/benchmark/configs/datasets/lcsts/README_en.md +++ b/ais_bench/benchmark/configs/datasets/lcsts/README_en.md @@ -26,7 +26,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| lcsts_gen_0_shot_chat | Generative task for the LCSTS dataset | Accuracy | 0-shot | Chat format | [lcsts_gen_0_shot_chat.py](lcsts_gen_0_shot_chat.py) | -| lcsts_gen_0_shot_str | Generative task for the LCSTS dataset | Accuracy | 0-shot | String format | [lcsts_gen_0_shot_str.py](lcsts_gen_0_shot_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| lcsts_gen_0_shot_chat | Generative task for the LCSTS dataset | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.lcsts.lcsts_gen_0_shot_chat import lcsts_datasets as datasets`| [lcsts_gen_0_shot_chat.py](lcsts_gen_0_shot_chat.py) | +| lcsts_gen_0_shot_str | Generative task for the LCSTS dataset | Accuracy | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.lcsts.lcsts_gen_0_shot_str import lcsts_datasets as datasets`| [lcsts_gen_0_shot_str.py](lcsts_gen_0_shot_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/livecodebench/README.md b/ais_bench/benchmark/configs/datasets/livecodebench/README.md index 669fdd1c..975bab1a 100644 --- a/ais_bench/benchmark/configs/datasets/livecodebench/README.md +++ b/ais_bench/benchmark/configs/datasets/livecodebench/README.md @@ -33,8 +33,8 @@ git clone https://huggingface.co/datasets/livecodebench/code_generation_lite > ⚠️ **重要**：`code_generation_lite.py` 是数据集仓库自带的加载脚本，包含了数据集的**版本信息**与**数据导入逻辑**（数据集按版本区分，版本信息即来自此文件）。**该文件缺失将导致数据集加载失败**。通过 `git lfs install && git clone https://huggingface.co/datasets/livecodebench/code_generation_lite` 完整克隆仓库即可自动获得该文件，请勿删除或遗漏。 ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|livecodebench_0_shot_chat_v4_v5|code_generation_lite数据集的生成式任务，与DeepSeek-R1测评使用数据集一致：LiveCodeBench(2024-08 – 2025-01)|pass@1|0-shot|对话格式|[livecodebench_0_shot_chat_v4_v5.py](livecodebench_0_shot_chat_v4_v5.py)| -|livecodebench_0_shot_chat_v4_v5_v6|code_generation_lite数据集的生成式任务, 与DeepSeek-V3.1和DeepSeek-V3.2测评使用数据集一致：LiveCodeBench(2024-08 – 2025-05)|pass@1|0-shot|对话格式|[livecodebench_0_shot_chat_v4_v5_v6.py](livecodebench_0_shot_chat_v4_v5_v6.py)| -|livecodebench_0_shot_chat_v6|code_generation_lite数据集的生成式任务, 与Qwen3测评使用数据集一致：LiveCodeBench(2025-05)|pass@1|0-shot|对话格式|[livecodebench_0_shot_chat_v6.py](livecodebench_0_shot_chat_v6.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|livecodebench_0_shot_chat_v4_v5|code_generation_lite数据集的生成式任务，与DeepSeek-R1测评使用数据集一致：LiveCodeBench(2024-08 – 2025-01)|pass@1|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.livecodebench.livecodebench_0_shot_chat_v4_v5 import LCB_datasets as datasets`|[livecodebench_0_shot_chat_v4_v5.py](livecodebench_0_shot_chat_v4_v5.py)| +|livecodebench_0_shot_chat_v4_v5_v6|code_generation_lite数据集的生成式任务, 与DeepSeek-V3.1和DeepSeek-V3.2测评使用数据集一致：LiveCodeBench(2024-08 – 2025-05)|pass@1|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.livecodebench.livecodebench_0_shot_chat_v4_v5_v6 import LCB_datasets as datasets`|[livecodebench_0_shot_chat_v4_v5_v6.py](livecodebench_0_shot_chat_v4_v5_v6.py)| +|livecodebench_0_shot_chat_v6|code_generation_lite数据集的生成式任务, 与Qwen3测评使用数据集一致：LiveCodeBench(2025-05)|pass@1|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.livecodebench.livecodebench_0_shot_chat_v6 import LCB_datasets as datasets`|[livecodebench_0_shot_chat_v6.py](livecodebench_0_shot_chat_v6.py)| diff --git a/ais_bench/benchmark/configs/datasets/livecodebench/README_en.md b/ais_bench/benchmark/configs/datasets/livecodebench/README_en.md index dbe29b51..9c772b58 100644 --- a/ais_bench/benchmark/configs/datasets/livecodebench/README_en.md +++ b/ais_bench/benchmark/configs/datasets/livecodebench/README_en.md @@ -33,8 +33,8 @@ git clone https://huggingface.co/datasets/livecodebench/code_generation_lite > ⚠️ **Important**: `code_generation_lite.py` is a loading script shipped with the dataset repository. It contains the dataset's **version information** and **data loading logic** (the dataset is versioned, and the version info comes from this file). **Missing this file will cause dataset loading to fail**. The file is automatically included by cloning the repository with `git lfs install && git clone https://huggingface.co/datasets/livecodebench/code_generation_lite`—do not delete or omit it. ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -|livecodebench_0_shot_chat_v4_v5|Generative task for the code_generation_lite dataset, same with DeepSeek-R1 Evaluation: LiveCodeBench(2024-08 – 2025-01)|pass@1|0-shot|Chat format|[livecodebench_0_shot_chat_v4_v5.py](livecodebench_0_shot_chat_v4_v5.py)| -|livecodebench_0_shot_chat_v4_v5_v6|Generative task for the code_generation_lite dataset, same with DeepSeek-V3.1 and DeepSeek-V3.2 Evaluation: LiveCodeBench(2024-08 – 2025-05)|pass@1|0-shot|Chat format|[livecodebench_0_shot_chat_v4_v5_v6.py](livecodebench_0_shot_chat_v4_v5_v6.py)| -|livecodebench_0_shot_chat_v6|Generative task for the code_generation_lite dataset, same with Qwen3 Evaluation: LiveCodeBench(2025-05)|pass@1|0-shot|Chat format|[livecodebench_0_shot_chat_v6.py](livecodebench_0_shot_chat_v6.py)| +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +|livecodebench_0_shot_chat_v4_v5|Generative task for the code_generation_lite dataset, same with DeepSeek-R1 Evaluation: LiveCodeBench(2024-08 – 2025-01)|pass@1|0-shot|Chat format|`from ais_bench.benchmark.configs.datasets.livecodebench.livecodebench_0_shot_chat_v4_v5 import LCB_datasets as datasets`|[livecodebench_0_shot_chat_v4_v5.py](livecodebench_0_shot_chat_v4_v5.py)| +|livecodebench_0_shot_chat_v4_v5_v6|Generative task for the code_generation_lite dataset, same with DeepSeek-V3.1 and DeepSeek-V3.2 Evaluation: LiveCodeBench(2024-08 – 2025-05)|pass@1|0-shot|Chat format|`from ais_bench.benchmark.configs.datasets.livecodebench.livecodebench_0_shot_chat_v4_v5_v6 import LCB_datasets as datasets`|[livecodebench_0_shot_chat_v4_v5_v6.py](livecodebench_0_shot_chat_v4_v5_v6.py)| +|livecodebench_0_shot_chat_v6|Generative task for the code_generation_lite dataset, same with Qwen3 Evaluation: LiveCodeBench(2025-05)|pass@1|0-shot|Chat format|`from ais_bench.benchmark.configs.datasets.livecodebench.livecodebench_0_shot_chat_v6 import LCB_datasets as datasets`|[livecodebench_0_shot_chat_v6.py](livecodebench_0_shot_chat_v6.py)| diff --git a/ais_bench/benchmark/configs/datasets/longbench/README.md b/ais_bench/benchmark/configs/datasets/longbench/README.md index 86a27eab..300745ff 100644 --- a/ais_bench/benchmark/configs/datasets/longbench/README.md +++ b/ais_bench/benchmark/configs/datasets/longbench/README.md @@ -48,9 +48,9 @@ LongBench包含14个英文任务、5个中文任务和2个代码任务，大部 └── LongBench.py ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|longbench|longbench|准确率(accuracy)|0-shot|对话格式|[longbench.py](longbench.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|longbench|longbench|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.longbench.longbench import longbench_datasets as datasets`|[longbench.py](longbench.py)| |longbench_2wikimqa_gen|longbench_2wikimqa_gen|准确率(accuracy)|0-shot|对话格式|[longbench_2wikimqa_gen.py](longbench2wikimqa/longbench_2wikimqa_gen.py)| |longbench_dureader_gen|longbench_dureader_gen|准确率(accuracy)|0-shot|对话格式|[longbench_dureader_gen.py](longbenchdureader/longbench_dureader_gen.py)| |longbench_gov_report_gen|longbench_gov_report_gen|准确率(accuracy)|0-shot|对话格式|[longbench_gov_report_gen.py](longbenchgov_report/longbench_gov_report_gen.py)| @@ -66,7 +66,7 @@ LongBench包含14个英文任务、5个中文任务和2个代码任务，大部 |longbench_passage_retrieval_en_gen|longbench_passage_retrieval_en_gen|准确率(accuracy)|0-shot|对话格式|[longbench_passage_retrieval_en_gen.py](longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen.py)| |longbench_passage_retrieval_zh_gen|longbench_passage_retrieval_zh_gen|准确率(accuracy)|0-shot|对话格式|[longbench_passage_retrieval_zh_gen.py](longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen.py)| |longbench_qasper_gen|longbench_qasper_gen|准确率(accuracy)|0-shot|对话格式|[longbench_qasper_gen.py](longbenchqasper/longbench_qasper_gen.py)| -|longbench_qmsum_gen|longbench_qmsum_gen|准确率(accuracy)|0-shot|对话格式|[longbench_qmsum_gen.py](longbenchqmsum/longbenchqmsum_gen.py)| +|longbench_qmsum_gen|longbench_qmsum_gen|准确率(accuracy)|0-shot|对话格式|[longbench_qmsum_gen.py](longbenchqmsum/longbench_qmsum_gen.py)| |longbench_repobench_gen|longbench_repobench_gen|准确率(accuracy)|0-shot|对话格式|[longbench_repobench_gen.py](longbenchrepobench/longbench_repobench_gen.py)| |longbench_samsum_gen|longbench_samsum_gen|准确率(accuracy)|0-shot|对话格式|[longbench_samsum_gen.py](longbenchsamsum/longbench_samsum_gen.py)| |longbench_trec_gen|longbench_trec_gen|准确率(accuracy)|0-shot|对话格式|[longbench_trec_gen.py](longbenchtrec/longbench_trec_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/longbench/README_en.md b/ais_bench/benchmark/configs/datasets/longbench/README_en.md index a43f2ec2..68bc4fe0 100644 --- a/ais_bench/benchmark/configs/datasets/longbench/README_en.md +++ b/ais_bench/benchmark/configs/datasets/longbench/README_en.md @@ -52,9 +52,9 @@ It is recommended to download the dataset from Hugging Face: [https://huggingfac ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| longbench | LongBench main task | Accuracy | 0-shot | Chat format | [longbench.py](longbench.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| longbench | LongBench main task | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.longbench.longbench import longbench_datasets as datasets`| [longbench.py](longbench.py) | | longbench_2wikimqa_gen | LongBench 2WikiMQA generative task | Accuracy | 0-shot | Chat format | [longbench_2wikimqa_gen.py](longbench2wikimqa/longbench_2wikimqa_gen.py) | | longbench_dureader_gen | LongBench DuReader generative task | Accuracy | 0-shot | Chat format | [longbench_dureader_gen.py](longbenchdureader/longbench_dureader_gen.py) | | longbench_gov_report_gen | LongBench GovReport generative task | Accuracy | 0-shot | Chat format | [longbench_gov_report_gen.py](longbenchgov_report/longbench_gov_report_gen.py) | @@ -70,7 +70,7 @@ It is recommended to download the dataset from Hugging Face: [https://huggingfac | longbench_passage_retrieval_en_gen | LongBench PassageRetrieval-EN generative task | Accuracy | 0-shot | Chat format | [longbench_passage_retrieval_en_gen.py](longbenchpassage_retrieval_en/longbench_passage_retrieval_en_gen.py) | | longbench_passage_retrieval_zh_gen | LongBench PassageRetrieval-ZH generative task | Accuracy | 0-shot | Chat format | [longbench_passage_retrieval_zh_gen.py](longbenchpassage_retrieval_zh/longbench_passage_retrieval_zh_gen.py) | | longbench_qasper_gen | LongBench QASPER generative task | Accuracy | 0-shot | Chat format | [longbench_qasper_gen.py](longbenchqasper/longbench_qasper_gen.py) | -| longbench_qmsum_gen | LongBench QMSum generative task | Accuracy | 0-shot | Chat format | [longbench_qmsum_gen.py](longbenchqmsum/longbenchqmsum_gen.py) | +| longbench_qmsum_gen | LongBench QMSum generative task | Accuracy | 0-shot | Chat format | [longbench_qmsum_gen.py](longbenchqmsum/longbench_qmsum_gen.py) | | longbench_repobench_gen | LongBench RepoBench generative task | Accuracy | 0-shot | Chat format | [longbench_repobench_gen.py](longbenchrepobench/longbench_repobench_gen.py) | | longbench_samsum_gen | LongBench SamSum generative task | Accuracy | 0-shot | Chat format | [longbench_samsum_gen.py](longbenchsamsum/longbench_samsum_gen.py) | | longbench_trec_gen | LongBench TREC generative task | Accuracy | 0-shot | Chat format | [longbench_trec_gen.py](longbenchtrec/longbench_trec_gen.py) | diff --git a/ais_bench/benchmark/configs/datasets/longbenchv2/README.md b/ais_bench/benchmark/configs/datasets/longbenchv2/README.md index c619ec8b..9f26a48b 100644 --- a/ais_bench/benchmark/configs/datasets/longbenchv2/README.md +++ b/ais_bench/benchmark/configs/datasets/longbenchv2/README.md @@ -17,6 +17,6 @@ LongBench v2包含503道富有挑战性的多项选择题，涵盖六大任务 └── data.json ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|longbenchv2_gen|longbenchv2|准确率(accuracy)|0-shot|对话格式|[longbenchv2_gen.py](longbenchv2_gen.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|longbenchv2_gen|longbenchv2|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.longbenchv2.longbenchv2_gen import LongBenchv2_datasets as datasets`|[longbenchv2_gen.py](longbenchv2_gen.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/longbenchv2/README_en.md b/ais_bench/benchmark/configs/datasets/longbenchv2/README_en.md index 7255db3d..9632f886 100644 --- a/ais_bench/benchmark/configs/datasets/longbenchv2/README_en.md +++ b/ais_bench/benchmark/configs/datasets/longbenchv2/README_en.md @@ -21,6 +21,6 @@ It is recommended to download the dataset from Hugging Face: [https://huggingfac ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| longbenchv2_gen | LongBench v2 task | Accuracy | 0-shot | Chat format | [longbenchv2_gen.py](longbenchv2_gen.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| longbenchv2_gen | LongBench v2 task | Accuracy | 0-shot | Chat format | `from ais_bench.benchmark.configs.datasets.longbenchv2.longbenchv2_gen import LongBenchv2_datasets as datasets` | [longbenchv2_gen.py](longbenchv2_gen.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/math/README.md b/ais_bench/benchmark/configs/datasets/math/README.md index df077d8a..6735901d 100644 --- a/ais_bench/benchmark/configs/datasets/math/README.md +++ b/ais_bench/benchmark/configs/datasets/math/README.md @@ -34,8 +34,8 @@ rm math.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|math_prm800k_500_0shot_cot_gen|MATH500数据集生成式任务, 默认max out tokens长度取32768，prompt带逻辑链|accuracy(pass@1)|0-shot|字符串格式|[math_prm800k_500_0shot_cot_gen.py](math_prm800k_500_0shot_cot_gen.py)| -|math_prm800k_500_5shot_cot_gen|MATH500数据集生成式任务, 默认max out tokens长度取32768，prompt带逻辑链|accuracy(pass@1)|5-shot|字符串格式|[math_prm800k_500_5shot_cot_gen.py](math_prm800k_500_5shot_cot_gen.py)| -|math500_gen_0_shot_cot_chat_prompt|MATH500数据集生成式任务，prompt带逻辑链（对齐DeepSeek R1精度测试）|accuracy(pass@1)|0-shot|对话格式|[math500_gen_0_shot_cot_chat_prompt.py](math500_gen_0_shot_cot_chat_prompt.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|math_prm800k_500_0shot_cot_gen|MATH500数据集生成式任务, 默认max out tokens长度取32768，prompt带逻辑链|accuracy(pass@1)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.math.math_prm800k_500_0shot_cot_gen import math_datasets as datasets`|[math_prm800k_500_0shot_cot_gen.py](math_prm800k_500_0shot_cot_gen.py)| +|math_prm800k_500_5shot_cot_gen|MATH500数据集生成式任务, 默认max out tokens长度取32768，prompt带逻辑链|accuracy(pass@1)|5-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.math.math_prm800k_500_5shot_cot_gen import math_datasets as datasets`|[math_prm800k_500_5shot_cot_gen.py](math_prm800k_500_5shot_cot_gen.py)| +|math500_gen_0_shot_cot_chat_prompt|MATH500数据集生成式任务，prompt带逻辑链（对齐DeepSeek R1精度测试）|accuracy(pass@1)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as datasets`|[math500_gen_0_shot_cot_chat_prompt.py](math500_gen_0_shot_cot_chat_prompt.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/math/README_en.md b/ais_bench/benchmark/configs/datasets/math/README_en.md index 0c6c025c..a7d87047 100644 --- a/ais_bench/benchmark/configs/datasets/math/README_en.md +++ b/ais_bench/benchmark/configs/datasets/math/README_en.md @@ -33,8 +33,8 @@ rm math.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| math_prm800k_500_0shot_cot_gen | Generative task for the MATH500 dataset. The default maximum output token length is 32768, with a logical chain in the prompt. | Accuracy (pass@1) | 0-shot | String format | [math_prm800k_500_0shot_cot_gen.py](math_prm800k_500_0shot_cot_gen.py) | -| math_prm800k_500_5shot_cot_gen | Generative task for the MATH500 dataset. The default maximum output token length is 32768, with a logical chain in the prompt. | Accuracy (pass@1) | 5-shot | String format | [math_prm800k_500_5shot_cot_gen.py](math_prm800k_500_5shot_cot_gen.py) | -| math500_gen_0_shot_cot_chat_prompt | Generative task for the MATH500 dataset, with a logical chain in the prompt (aligned with DeepSeek R1 accuracy test) | Accuracy (pass@1) | 0-shot | Chat format | [math500_gen_0_shot_cot_chat_prompt.py](math500_gen_0_shot_cot_chat_prompt.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| math_prm800k_500_0shot_cot_gen | Generative task for the MATH500 dataset. The default maximum output token length is 32768, with a logical chain in the prompt. | Accuracy (pass@1) | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.math.math_prm800k_500_0shot_cot_gen import math_datasets as datasets`| [math_prm800k_500_0shot_cot_gen.py](math_prm800k_500_0shot_cot_gen.py) | +| math_prm800k_500_5shot_cot_gen | Generative task for the MATH500 dataset. The default maximum output token length is 32768, with a logical chain in the prompt. | Accuracy (pass@1) | 5-shot | String format |`from ais_bench.benchmark.configs.datasets.math.math_prm800k_500_5shot_cot_gen import math_datasets as datasets`| [math_prm800k_500_5shot_cot_gen.py](math_prm800k_500_5shot_cot_gen.py) | +| math500_gen_0_shot_cot_chat_prompt | Generative task for the MATH500 dataset, with a logical chain in the prompt (aligned with DeepSeek R1 accuracy test) | Accuracy (pass@1) | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as datasets`| [math500_gen_0_shot_cot_chat_prompt.py](math500_gen_0_shot_cot_chat_prompt.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/mathvision/README.md b/ais_bench/benchmark/configs/datasets/mathvision/README.md index 446c775d..4b76f741 100644 --- a/ais_bench/benchmark/configs/datasets/mathvision/README.md +++ b/ais_bench/benchmark/configs/datasets/mathvision/README.md @@ -32,6 +32,6 @@ mathvision ``` ## 可用数据集任务 -| 任务名称 | 简介 | 评估指标 | few-shot | prompt 格式 | 对应源码配置文件路径 | -| --- | --- | --- | --- | --- | --- | -| mathvision_gen | MathVision 数据集生成式多模态数学推理任务，支持选择题和自由作答题；选择题要求最后一行输出 `ANSWER: [LETTER]`，自由作答题要求最终答案放在 `\boxed{}` 中 | Accuracy | 0-shot | 多模态对话格式（文本 + 图片） | [mathvision_gen.py](mathvision_gen.py) | +| 任务名称 | 简介 | 评估指标 | few-shot | prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| mathvision_gen | MathVision 数据集生成式多模态数学推理任务，支持选择题和自由作答题；选择题要求最后一行输出 `ANSWER: [LETTER]`，自由作答题要求最终答案放在 `\boxed{}` 中 | Accuracy | 0-shot | 多模态对话格式（文本 + 图片） | `from ais_bench.benchmark.configs.datasets.mathvision.mathvision_gen import mathvision_datasets as datasets` | [mathvision_gen.py](mathvision_gen.py) | diff --git a/ais_bench/benchmark/configs/datasets/mathvision/README_en.md b/ais_bench/benchmark/configs/datasets/mathvision/README_en.md index b0dc3be6..f09debd3 100644 --- a/ais_bench/benchmark/configs/datasets/mathvision/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mathvision/README_en.md @@ -32,6 +32,6 @@ mathvision ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| mathvision_gen | Generative multimodal mathematical reasoning task for MathVision. It supports both multiple-choice and open-answer questions. Multiple-choice questions require the final line to be `ANSWER: [LETTER]`, while open-answer questions require the final answer in `\boxed{}` | Accuracy | 0-shot | Multimodal chat format (text + image) | [mathvision_gen.py](mathvision_gen.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| mathvision_gen | Generative multimodal mathematical reasoning task for MathVision. It supports both multiple-choice and open-answer questions. Multiple-choice questions require the final line to be `ANSWER: [LETTER]`, while open-answer questions require the final answer in `\boxed{}` | Accuracy | 0-shot | Multimodal chat format (text + image) |`from ais_bench.benchmark.configs.datasets.mathvision.mathvision_gen import mathvision_datasets as datasets`| [mathvision_gen.py](mathvision_gen.py) | diff --git a/ais_bench/benchmark/configs/datasets/mbpp/README.md b/ais_bench/benchmark/configs/datasets/mbpp/README.md index d66ae589..ba4a2b73 100644 --- a/ais_bench/benchmark/configs/datasets/mbpp/README.md +++ b/ais_bench/benchmark/configs/datasets/mbpp/README.md @@ -25,7 +25,7 @@ rm mbpp.zip ## 可用数据集任务 ### mbpp_passk_gen_3_shot_chat_prompt #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mbpp_passk_gen_3_shot_chat_prompt|mbpp数据集生成式任务，支持测pass@k(默认pass@1)|pass@1|3-shot|对话格式|[mbpp_passk_gen_3_shot_chat_prompt.py](mbpp_passk_gen_3_shot_chat_prompt.py)| -|sanitized_mbpp_passk_gen_3_shot_chat_prompt|sanitized mbpp数据集生成式任务，支持测pass@k(默认pass@1)|pass@1|3-shot|对话格式|[sanitized_mbpp_passk_gen_3_shot_chat_prompt.py](sanitized_mbpp_passk_gen_3_shot_chat_prompt.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mbpp_passk_gen_3_shot_chat_prompt|mbpp数据集生成式任务，支持测pass@k(默认pass@1)|pass@1|3-shot|对话格式|`from ais_bench.benchmark.configs.datasets.mbpp.mbpp_passk_gen_3_shot_chat_prompt import mbpp_datasets as datasets`|[mbpp_passk_gen_3_shot_chat_prompt.py](mbpp_passk_gen_3_shot_chat_prompt.py)| +|sanitized_mbpp_passk_gen_3_shot_chat_prompt|sanitized mbpp数据集生成式任务，支持测pass@k(默认pass@1)|pass@1|3-shot|对话格式|`from ais_bench.benchmark.configs.datasets.mbpp.sanitized_mbpp_passk_gen_3_shot_chat_prompt import sanitized_mbpp_datasets as datasets`|[sanitized_mbpp_passk_gen_3_shot_chat_prompt.py](sanitized_mbpp_passk_gen_3_shot_chat_prompt.py)| diff --git a/ais_bench/benchmark/configs/datasets/mbpp/README_en.md b/ais_bench/benchmark/configs/datasets/mbpp/README_en.md index e87c3a92..29b3f2b7 100644 --- a/ais_bench/benchmark/configs/datasets/mbpp/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mbpp/README_en.md @@ -25,7 +25,7 @@ rm mbpp.zip ## Available Dataset Tasks ### mbpp_passk_gen_3_shot_chat_prompt #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| mbpp_passk_gen_3_shot_chat_prompt | Generative task for the mbpp dataset, supporting pass@k evaluation (default: pass@1) | pass@1 | 3-shot | Chat format | [mbpp_passk_gen_3_shot_chat_prompt.py](mbpp_passk_gen_3_shot_chat_prompt.py) | -| sanitized_mbpp_passk_gen_3_shot_chat_prompt | Generative task for the sanitized mbpp dataset, supporting pass@k evaluation (default: pass@1) | pass@1 | 3-shot | Chat format | [sanitized_mbpp_passk_gen_3_shot_chat_prompt.py](sanitized_mbpp_passk_gen_3_shot_chat_prompt.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| mbpp_passk_gen_3_shot_chat_prompt | Generative task for the mbpp dataset, supporting pass@k evaluation (default: pass@1) | pass@1 | 3-shot | Chat format |`from ais_bench.benchmark.configs.datasets.mbpp.mbpp_passk_gen_3_shot_chat_prompt import mbpp_datasets as datasets`| [mbpp_passk_gen_3_shot_chat_prompt.py](mbpp_passk_gen_3_shot_chat_prompt.py) | +| sanitized_mbpp_passk_gen_3_shot_chat_prompt | Generative task for the sanitized mbpp dataset, supporting pass@k evaluation (default: pass@1) | pass@1 | 3-shot | Chat format |`from ais_bench.benchmark.configs.datasets.mbpp.sanitized_mbpp_passk_gen_3_shot_chat_prompt import sanitized_mbpp_datasets as datasets`| [sanitized_mbpp_passk_gen_3_shot_chat_prompt.py](sanitized_mbpp_passk_gen_3_shot_chat_prompt.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/mgsm/README.md b/ais_bench/benchmark/configs/datasets/mgsm/README.md index 1b797477..b301e9f4 100644 --- a/ais_bench/benchmark/configs/datasets/mgsm/README.md +++ b/ais_bench/benchmark/configs/datasets/mgsm/README.md @@ -34,7 +34,7 @@ git clone https://huggingface.co/datasets/juletxara/mgsm ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mgsm_gen_0_shot_cot_chat_prompt|mgsm数据集生成式任务，prompt带逻辑链|accuracy|0-shot|对话格式|[mgsm_gen_0_shot_cot_chat_prompt.py](mgsm_gen_0_shot_cot_chat_prompt.py)| -|mgsm_gen_8_shot_cot_chat_prompt|mgsm数据集生成式任务，prompt带逻辑链|accuracy|8-shot|对话格式|[mgsm_gen_8_shot_cot_chat_prompt.py](mgsm_gen_8_shot_cot_chat_prompt.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mgsm_gen_0_shot_cot_chat_prompt|mgsm数据集生成式任务，prompt带逻辑链|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.mgsm.mgsm_gen_0_shot_cot_chat_prompt import mgsm_datasets as datasets`|[mgsm_gen_0_shot_cot_chat_prompt.py](mgsm_gen_0_shot_cot_chat_prompt.py)| +|mgsm_gen_8_shot_cot_chat_prompt|mgsm数据集生成式任务，prompt带逻辑链|accuracy|8-shot|对话格式|`from ais_bench.benchmark.configs.datasets.mgsm.mgsm_gen_8_shot_cot_chat_prompt import mgsm_datasets as datasets`|[mgsm_gen_8_shot_cot_chat_prompt.py](mgsm_gen_8_shot_cot_chat_prompt.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/mgsm/README_en.md b/ais_bench/benchmark/configs/datasets/mgsm/README_en.md index 0b05143e..9d453a4e 100644 --- a/ais_bench/benchmark/configs/datasets/mgsm/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mgsm/README_en.md @@ -34,7 +34,7 @@ git clone https://huggingface.co/datasets/juletxara/mgsm ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| mgsm_gen_0_shot_cot_chat_prompt | Generative task for the mgsm dataset, with a logical chain in the prompt | Accuracy | 0-shot | Chat format | [mgsm_gen_0_shot_cot_chat_prompt.py](mgsm_gen_0_shot_cot_chat_prompt.py) | -| mgsm_gen_8_shot_cot_chat_prompt | Generative task for the mgsm dataset, with a logical chain in the prompt | Accuracy | 8-shot | Chat format | [mgsm_gen_8_shot_cot_chat_prompt.py](mgsm_gen_8_shot_cot_chat_prompt.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| mgsm_gen_0_shot_cot_chat_prompt | Generative task for the mgsm dataset, with a logical chain in the prompt | Accuracy | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.mgsm.mgsm_gen_0_shot_cot_chat_prompt import mgsm_datasets as datasets`| [mgsm_gen_0_shot_cot_chat_prompt.py](mgsm_gen_0_shot_cot_chat_prompt.py) | +| mgsm_gen_8_shot_cot_chat_prompt | Generative task for the mgsm dataset, with a logical chain in the prompt | Accuracy | 8-shot | Chat format |`from ais_bench.benchmark.configs.datasets.mgsm.mgsm_gen_8_shot_cot_chat_prompt import mgsm_datasets as datasets`| [mgsm_gen_8_shot_cot_chat_prompt.py](mgsm_gen_8_shot_cot_chat_prompt.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/mmlu/README.md b/ais_bench/benchmark/configs/datasets/mmlu/README.md index 0ff33e51..430ea23a 100644 --- a/ais_bench/benchmark/configs/datasets/mmlu/README.md +++ b/ais_bench/benchmark/configs/datasets/mmlu/README.md @@ -198,10 +198,10 @@ rm mmlu.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mmlu_gen_5_shot_str|MMLU数据集生成式任务|accuracy(naive_average)|5-shot|字符串格式|[mmlu_gen_5_shot_str.py](mmlu_gen_5_shot_str.py)| -|mmlu_gen_5_shot_chat_prompt|MMLU数据集生成式任务|accuracy(naive_average)|5-shot|对话格式|[mmlu_gen_5_shot_chat_prompt.py](mmlu_gen_5_shot_chat_prompt.py)| -|mmlu_ppl_0_shot_str|MMLU数据集PPL任务|accuracy(naive_average)|0-shot|字符串格式|[mmlu_ppl_0_shot_str.py](mmlu_ppl_0_shot_str.py)| -|mmlu_gen_0_shot_cot_chat_prompt|MMLU数据集生成式任务，prompt带逻辑链（对齐DeepSeek R1精度测试）|accuracy(naive_average)|0-shot|对话格式|[mmlu_gen_0_shot_cot_chat_prompt.py](mmlu_gen_0_shot_cot_chat_prompt.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mmlu_gen_5_shot_str|MMLU数据集生成式任务|accuracy(naive_average)|5-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_5_shot_str import mmlu_datasets as datasets`|[mmlu_gen_5_shot_str.py](mmlu_gen_5_shot_str.py)| +|mmlu_gen_5_shot_chat_prompt|MMLU数据集生成式任务|accuracy(naive_average)|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_5_shot_chat_prompt import mmlu_datasets as datasets`|[mmlu_gen_5_shot_chat_prompt.py](mmlu_gen_5_shot_chat_prompt.py)| +|mmlu_ppl_0_shot_str|MMLU数据集PPL任务|accuracy(naive_average)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_ppl_0_shot_str import mmlu_datasets as datasets`|[mmlu_ppl_0_shot_str.py](mmlu_ppl_0_shot_str.py)| +|mmlu_gen_0_shot_cot_chat_prompt|MMLU数据集生成式任务，prompt带逻辑链（对齐DeepSeek R1精度测试）|accuracy(naive_average)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_0_shot_cot_chat_prompt import mmlu_datasets as datasets`|[mmlu_gen_0_shot_cot_chat_prompt.py](mmlu_gen_0_shot_cot_chat_prompt.py)| diff --git a/ais_bench/benchmark/configs/datasets/mmlu/README_en.md b/ais_bench/benchmark/configs/datasets/mmlu/README_en.md index ad962b03..45256d08 100644 --- a/ais_bench/benchmark/configs/datasets/mmlu/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mmlu/README_en.md @@ -198,10 +198,10 @@ rm mmlu.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| mmlu_gen_5_shot_str | Generative task for the MMLU dataset | Accuracy (naive_average) | 5-shot | String format | [mmlu_gen_5_shot_str.py](mmlu_gen_5_shot_str.py) | -| mmlu_gen_5_shot_chat_prompt | Generative task for the MMLU dataset, with a logical chain in the prompt| Accuracy (naive_average) | 5-shot | Chat format | [mmlu_gen_5_shot_chat_prompt.py](mmlu_gen_5_shot_chat_prompt.py) | -| mmlu_ppl_0_shot_str | MMLU dataset PPL task | Accuracy (naive_average) | 0-shot | String format | [mmlu_ppl_0_shot_str.py](mmlu_ppl_0_shot_str.py) | -| mmlu_gen_0_shot_cot_chat_prompt | Generative task for the MMLU dataset, with a logical chain in the prompt (aligned with DeepSeek R1 accuracy test) | Accuracy (naive_average) | 0-shot | Chat format | [mmlu_gen_0_shot_cot_chat_prompt.py](mmlu_gen_0_shot_cot_chat_prompt.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| mmlu_gen_5_shot_str | Generative task for the MMLU dataset | Accuracy (naive_average) | 5-shot | String format |`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_5_shot_str import mmlu_datasets as datasets`| [mmlu_gen_5_shot_str.py](mmlu_gen_5_shot_str.py) | +| mmlu_gen_5_shot_chat_prompt | Generative task for the MMLU dataset, with a logical chain in the prompt| Accuracy (naive_average) | 5-shot | Chat format |`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_5_shot_chat_prompt import mmlu_datasets as datasets`| [mmlu_gen_5_shot_chat_prompt.py](mmlu_gen_5_shot_chat_prompt.py) | +| mmlu_ppl_0_shot_str | MMLU dataset PPL task | Accuracy (naive_average) | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_ppl_0_shot_str import mmlu_datasets as datasets`| [mmlu_ppl_0_shot_str.py](mmlu_ppl_0_shot_str.py) | +| mmlu_gen_0_shot_cot_chat_prompt | Generative task for the MMLU dataset, with a logical chain in the prompt (aligned with DeepSeek R1 accuracy test) | Accuracy (naive_average) | 0-shot | Chat format |`from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_0_shot_cot_chat_prompt import mmlu_datasets as datasets`| [mmlu_gen_0_shot_cot_chat_prompt.py](mmlu_gen_0_shot_cot_chat_prompt.py) | diff --git a/ais_bench/benchmark/configs/datasets/mmlu_pro/README.md b/ais_bench/benchmark/configs/datasets/mmlu_pro/README.md index 1f18821d..f48961eb 100644 --- a/ais_bench/benchmark/configs/datasets/mmlu_pro/README.md +++ b/ais_bench/benchmark/configs/datasets/mmlu_pro/README.md @@ -25,7 +25,7 @@ rm mmlu_pro.zip ## 可用数据集任务 ### mmlu_pro_gen_0_shot_str #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mmlu_pro_gen_0_shot_str|mmlu-pro数据集生成式任务|pass@1|0-shot|字符串格式|[mmlu_pro_gen_0_shot_str.py](mmlu_pro_gen_0_shot_str.py)| -|mmlu_pro_gen_5_shot_str|mmlu-pro数据集生成式任务|pass@1|0-shot|字符串格式|[mmlu_pro_gen_5_shot_str.py](mmlu_pro_gen_5_shot_str.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mmlu_pro_gen_0_shot_str|mmlu-pro数据集生成式任务|pass@1|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmlu_pro.mmlu_pro_gen_0_shot_str import mmlu_pro_datasets as datasets`|[mmlu_pro_gen_0_shot_str.py](mmlu_pro_gen_0_shot_str.py)| +|mmlu_pro_gen_5_shot_str|mmlu-pro数据集生成式任务|pass@1|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmlu_pro.mmlu_pro_gen_5_shot_str import mmlu_pro_datasets as datasets`|[mmlu_pro_gen_5_shot_str.py](mmlu_pro_gen_5_shot_str.py)| diff --git a/ais_bench/benchmark/configs/datasets/mmlu_pro/README_en.md b/ais_bench/benchmark/configs/datasets/mmlu_pro/README_en.md index 857b90f0..11983390 100644 --- a/ais_bench/benchmark/configs/datasets/mmlu_pro/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mmlu_pro/README_en.md @@ -25,10 +25,10 @@ rm mmlu_pro.zip ## Available Dataset Tasks ### mmlu_pro_gen_0_shot_str #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| mmlu_pro_gen_0_shot_str | Generative task for the mmlu-pro dataset | pass@1 | 0-shot | String format | [mmlu_pro_gen_0_shot_str.py](mmlu_pro_gen_0_shot_str.py) | -| mmlu_pro_gen_5_shot_str | Generative task for the mmlu-pro dataset | pass@1 | 5-shot | String format | [mmlu_pro_gen_5_shot_str.py](mmlu_pro_gen_5_shot_str.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| mmlu_pro_gen_0_shot_str | Generative task for the mmlu-pro dataset | pass@1 | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.mmlu_pro.mmlu_pro_gen_0_shot_str import mmlu_pro_datasets as datasets`| [mmlu_pro_gen_0_shot_str.py](mmlu_pro_gen_0_shot_str.py) | +| mmlu_pro_gen_5_shot_str | Generative task for the mmlu-pro dataset | pass@1 | 5-shot | String format |`from ais_bench.benchmark.configs.datasets.mmlu_pro.mmlu_pro_gen_5_shot_str import mmlu_pro_datasets as datasets`| [mmlu_pro_gen_5_shot_str.py](mmlu_pro_gen_5_shot_str.py) | ### Note on Accuracy Correction diff --git a/ais_bench/benchmark/configs/datasets/mmmu/README.md b/ais_bench/benchmark/configs/datasets/mmmu/README.md index e9fb09dc..162a4f97 100644 --- a/ais_bench/benchmark/configs/datasets/mmmu/README.md +++ b/ais_bench/benchmark/configs/datasets/mmmu/README.md @@ -29,6 +29,6 @@ git clone https://www.modelscope.cn/datasets/AI-ModelScope/MMMU.git mmmu ## 可用数据集任务 ### mmmu_gen #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mmmu_gen|MMMU 数据集生成式任务：选择题使用CoT单选模板，开放题使用 `ANSWER: [ANSWER]` 模板|acc|0-shot|多模态对话格式|[mmmu_gen.py](mmmu_gen.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mmmu_gen|MMMU 数据集生成式任务：选择题使用CoT单选模板，开放题使用 `ANSWER: [ANSWER]` 模板|acc|0-shot|多模态对话格式|`from ais_bench.benchmark.configs.datasets.mmmu.mmmu_gen import mmmu_datasets as datasets`|[mmmu_gen.py](mmmu_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/mmmu/README_en.md b/ais_bench/benchmark/configs/datasets/mmmu/README_en.md index 70f9bcbc..0de6773c 100644 --- a/ais_bench/benchmark/configs/datasets/mmmu/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mmmu/README_en.md @@ -29,6 +29,6 @@ git clone https://www.modelscope.cn/datasets/AI-ModelScope/MMMU.git mmmu ## Available Dataset Tasks ### mmmu_gen #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -|mmmu_gen|Generative MMMU task: multiple-choice questions use the CoT single-answer template, while open questions use the `ANSWER: [ANSWER]` template|acc|0-shot|Multimodal chat format|[mmmu_gen.py](mmmu_gen.py)| +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +|mmmu_gen|Generative MMMU task: multiple-choice questions use the CoT single-answer template, while open questions use the `ANSWER: [ANSWER]` template|acc|0-shot|Multimodal chat format|`from ais_bench.benchmark.configs.datasets.mmmu.mmmu_gen import mmmu_datasets as datasets`|[mmmu_gen.py](mmmu_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/mmmu_pro/README.md b/ais_bench/benchmark/configs/datasets/mmmu_pro/README.md index 3b10af12..27365578 100644 --- a/ais_bench/benchmark/configs/datasets/mmmu_pro/README.md +++ b/ais_bench/benchmark/configs/datasets/mmmu_pro/README.md @@ -28,9 +28,9 @@ wget https://opencompass.openxlab.space/utils/VLMEval/MMMU_Pro_V.tsv ## 可用数据集任务 #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mmmu_pro_options10_cot_gen|mmmu_pro options10数据集思维链生成式任务|acc|0-shot|字符串格式|[mmmu_pro_options10_cot_gen.py](mmmu_pro_options10_cot_gen.py)| -|mmmu_pro_options10_gen|mmmu_pro options10数据集生成式任务|acc|0-shot|字符串格式|[mmmu_pro_options10_gen.py](mmmu_pro_options10_gen.py)| -|mmmu_pro_vision_cot_gen|mmmu_pro vision数据集思维链生成式任务|acc|0-shot|字符串格式|[mmmu_pro_vision_cot_gen.py](mmmu_pro_vision_cot_gen.py)| -|mmmu_pro_vision_gen|mmmu_pro vision数据集生成式任务|acc|0-shot|字符串格式|[mmmu_pro_vision_gen.py](mmmu_pro_vision_gen.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mmmu_pro_options10_cot_gen|mmmu_pro options10数据集思维链生成式任务|acc|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_options10_cot_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_options10_cot_gen.py](mmmu_pro_options10_cot_gen.py)| +|mmmu_pro_options10_gen|mmmu_pro options10数据集生成式任务|acc|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_options10_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_options10_gen.py](mmmu_pro_options10_gen.py)| +|mmmu_pro_vision_cot_gen|mmmu_pro vision数据集思维链生成式任务|acc|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_vision_cot_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_vision_cot_gen.py](mmmu_pro_vision_cot_gen.py)| +|mmmu_pro_vision_gen|mmmu_pro vision数据集生成式任务|acc|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_vision_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_vision_gen.py](mmmu_pro_vision_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/mmmu_pro/README_en.md b/ais_bench/benchmark/configs/datasets/mmmu_pro/README_en.md index 3262a911..d4539d7a 100644 --- a/ais_bench/benchmark/configs/datasets/mmmu_pro/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mmmu_pro/README_en.md @@ -28,9 +28,9 @@ wget https://opencompass.openxlab.space/utils/VLMEval/MMMU_Pro_V.tsv ## Available Dataset Tasks #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -|mmmu_pro_options10_cot_gen|mmmu_pro options10 dataset thinking chain generative task|acc|0-shot|String format|[mmmu_pro_options10_cot_gen.py](mmmu_pro_options10_cot_gen.py)| -|mmmu_pro_options10_gen|mmmu_pro options10 dataset generative task|acc|0-shot|String format|[mmmu_pro_options10_gen.py](mmmu_pro_options10_gen.py)| -|mmmu_pro_vision_cot_gen|mmmu_pro vision dataset thinking chain generative task|acc|0-shot|String format|[mmmu_pro_vision_cot_gen.py](mmmu_pro_vision_cot_gen.py)| -|mmmu_pro_vision_gen|mmmu_pro vision dataset generative task|acc|0-shot|String format|[mmmu_pro_vision_gen.py](mmmu_pro_vision_gen.py)| +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +|mmmu_pro_options10_cot_gen|mmmu_pro options10 dataset thinking chain generative task|acc|0-shot|String format|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_options10_cot_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_options10_cot_gen.py](mmmu_pro_options10_cot_gen.py)| +|mmmu_pro_options10_gen|mmmu_pro options10 dataset generative task|acc|0-shot|String format|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_options10_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_options10_gen.py](mmmu_pro_options10_gen.py)| +|mmmu_pro_vision_cot_gen|mmmu_pro vision dataset thinking chain generative task|acc|0-shot|String format|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_vision_cot_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_vision_cot_gen.py](mmmu_pro_vision_cot_gen.py)| +|mmmu_pro_vision_gen|mmmu_pro vision dataset generative task|acc|0-shot|String format|`from ais_bench.benchmark.configs.datasets.mmmu_pro.mmmu_pro_vision_gen import mmmu_pro_datasets as datasets`|[mmmu_pro_vision_gen.py](mmmu_pro_vision_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/mmstar/README.md b/ais_bench/benchmark/configs/datasets/mmstar/README.md index 5a00f29e..ce9d2d63 100644 --- a/ais_bench/benchmark/configs/datasets/mmstar/README.md +++ b/ais_bench/benchmark/configs/datasets/mmstar/README.md @@ -24,7 +24,7 @@ wget https://www.modelscope.cn/datasets/evalscope/MMStar/resolve/master/MMStar.t ## 可用数据集任务 ### mmstar_gen #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mmstar_gen|mmstar数据集生成式任务|acc|0-shot|字符串格式|[mmstar_gen.py](mmstar_gen.py)| -|mmstar_gen_cot|mmstar数据集思维链生成式任务|acc|0-shot|字符串格式|[mmstar_gen_cot.py](mmstar_gen_cot.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mmstar_gen|mmstar数据集生成式任务|acc|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmstar.mmstar_gen import mmstar_datasets as datasets`|[mmstar_gen.py](mmstar_gen.py)| +|mmstar_gen_cot|mmstar数据集思维链生成式任务|acc|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.mmstar.mmstar_gen_cot import mmstar_datasets as datasets`|[mmstar_gen_cot.py](mmstar_gen_cot.py)| diff --git a/ais_bench/benchmark/configs/datasets/mmstar/README_en.md b/ais_bench/benchmark/configs/datasets/mmstar/README_en.md index 482c0019..3b352239 100644 --- a/ais_bench/benchmark/configs/datasets/mmstar/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mmstar/README_en.md @@ -24,7 +24,7 @@ wget https://www.modelscope.cn/datasets/evalscope/MMStar/resolve/master/MMStar.t ## Available Dataset Tasks ### mmstar_gen #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -|mmstar_gen|Generative task for the mmstar dataset|acc|0-shot|String format|[mmstar_gen.py](mmstar_gen.py)| -|mmstar_gen_cot|COT Generative task for the mmstar dataset|acc|0-shot|String format|[mmstar_gen_cot.py](mmstar_gen_cot.py)| \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +|mmstar_gen|Generative task for the mmstar dataset|acc|0-shot|String format|`from ais_bench.benchmark.configs.datasets.mmstar.mmstar_gen import mmstar_datasets as datasets`|[mmstar_gen.py](mmstar_gen.py)| +|mmstar_gen_cot|COT Generative task for the mmstar dataset|acc|0-shot|String format|`from ais_bench.benchmark.configs.datasets.mmstar.mmstar_gen_cot import mmstar_datasets as datasets`|[mmstar_gen_cot.py](mmstar_gen_cot.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/mooncake_trace/README.md b/ais_bench/benchmark/configs/datasets/mooncake_trace/README.md index 109b5585..e855f399 100644 --- a/ais_bench/benchmark/configs/datasets/mooncake_trace/README.md +++ b/ais_bench/benchmark/configs/datasets/mooncake_trace/README.md @@ -136,9 +136,9 @@ Mooncake Trace 数据集是一个用于性能评测的 trace 数据集，支持 ## 可用数据集任务 -| 任务名称 | 简介 | 评估指标 | few-shot | prompt格式 | 对应源码配置文件路径 | -| --- | --- | --- | --- | --- | --- | -| mooncake_trace_gen | Mooncake trace 数据集生成式任务 | 性能测评 | 0-shot | 字符串格式 | [mooncake_trace_gen.py](mooncake_trace_gen.py) | +| 任务名称 | 简介 | 评估指标 | few-shot | prompt格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| mooncake-trace | Mooncake trace 数据集生成式任务 | 性能测评 | 0-shot | 字符串格式 | `from ais_bench.benchmark.configs.datasets.mooncake_trace.mooncake_trace_gen import mooncake_trace_datasets as datasets` | [mooncake_trace_gen.py](mooncake_trace_gen.py) | ## 使用示例 diff --git a/ais_bench/benchmark/configs/datasets/mooncake_trace/README_en.md b/ais_bench/benchmark/configs/datasets/mooncake_trace/README_en.md index ceea0ae5..a45ecce9 100644 --- a/ais_bench/benchmark/configs/datasets/mooncake_trace/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mooncake_trace/README_en.md @@ -134,9 +134,9 @@ When using `hash_ids`, `input_length` must satisfy: ## Available Dataset Tasks -| Task Name | Description | Metrics | few-shot | Prompt Format | Config Path | -| --- | --- | --- | --- | --- | --- | -| mooncake_trace_gen | Mooncake trace generative task | Performance | 0-shot | String | [mooncake_trace_gen.py](mooncake_trace_gen.py) | +| Task Name | Description | Metrics | few-shot | Prompt Format | Import Statement | Config Path | +| --- | --- | --- | --- | --- | --- | --- | +| mooncake-trace | Mooncake trace generative task | Performance | 0-shot | String | `from ais_bench.benchmark.configs.datasets.mooncake_trace.mooncake_trace_gen import mooncake_trace_datasets as datasets` | [mooncake_trace_gen.py](mooncake_trace_gen.py) | ## Usage Examples diff --git a/ais_bench/benchmark/configs/datasets/mtbench/README.md b/ais_bench/benchmark/configs/datasets/mtbench/README.md index 0c9cc7e2..b6d4f31b 100644 --- a/ais_bench/benchmark/configs/datasets/mtbench/README.md +++ b/ais_bench/benchmark/configs/datasets/mtbench/README.md @@ -29,9 +29,9 @@ wget https://huggingface.co/datasets/HuggingFaceH4/mt_bench_prompts/blob/main/ra ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|mtbench_gen|mtbench生成式任务|暂不支持精度评测|0-shot|列表格式|[mtbench_gen.py](mtbench_gen.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|mtbench_gen|mtbench生成式任务|暂不支持精度评测|0-shot|列表格式|`from ais_bench.benchmark.configs.datasets.mtbench.mtbench_gen import mtbench_datasets as datasets`|[mtbench_gen.py](mtbench_gen.py)| *注意：该多轮对话数据集的测评支持vLLM、SGLang、MindIE Service等服务化，使用时需指定--models为vllm_api_stream_chat_multiturn* \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/mtbench/README_en.md b/ais_bench/benchmark/configs/datasets/mtbench/README_en.md index f5f5b0ad..4a1e3ffe 100644 --- a/ais_bench/benchmark/configs/datasets/mtbench/README_en.md +++ b/ais_bench/benchmark/configs/datasets/mtbench/README_en.md @@ -29,9 +29,9 @@ wget https://huggingface.co/datasets/HuggingFaceH4/mt_bench_prompts/blob/main/ra ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| mtbench_gen | Generative task for MTBench | Accuracy evaluation not supported temporarily | 0-shot | List format | [mtbench_gen.py](mtbench_gen.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| mtbench_gen | Generative task for MTBench | Accuracy evaluation not supported temporarily | 0-shot | List format |`from ais_bench.benchmark.configs.datasets.mtbench.mtbench_gen import mtbench_datasets as datasets`| [mtbench_gen.py](mtbench_gen.py) | *Note: The evaluation of this multi-turn conversation dataset supports service deployment frameworks such as vLLM, SGLang, and MindIE Service. When using it, you need to specify `--models` as `vllm_api_stream_chat_multiturn`.* diff --git a/ais_bench/benchmark/configs/datasets/needlebench_v2/README.md b/ais_bench/benchmark/configs/datasets/needlebench_v2/README.md index 7d738cd6..340a646f 100644 --- a/ais_bench/benchmark/configs/datasets/needlebench_v2/README.md +++ b/ais_bench/benchmark/configs/datasets/needlebench_v2/README.md @@ -47,35 +47,35 @@ NeedleBench V2引入了更平衡的评分系统。总体评分现在是通过三 └── zh_tech.jsonl ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|atc_0shot_nocot_2_power_en|atc_0shot_nocot_2_power_en|准确率(accuracy)|0-shot|对话格式|[atc_0shot_nocot_2_power_en.py](atc/atc_0shot_nocot_2_power_en.py)| -|needlebench_v2_4k|needlebench_v2_4k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_4k.py](needlebench_v2_4k/needlebench_v2_4k.py)| -|needlebench_v2_multi_reasoning_4k|needlebench_v2_multi_reasoning_4k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_reasoning_4k.py](needlebench_v2_4k/needlebench_v2_multi_reasoning_4k.py)| -|needlebench_v2_multi_retrieval_4k|needlebench_v2_multi_retrieval_4k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_4k.py](needlebench_v2_4k/needlebench_v2_multi_retrieval_4k.py)| -|needlebench_v2_single_4k|needlebench_v2_single_4k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_single_4k.py](needlebench_v2_4k/needlebench_v2_single_4k.py)| -|needlebench_v2_8k|needlebench_v2_8k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_8k.py](needlebench_v2_8k/needlebench_v2_8k.py)| -|needlebench_v2_multi_reasoning_8k|needlebench_v2_multi_reasoning_8k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_reasoning_8k.py](needlebench_v2_8k/needlebench_v2_multi_reasoning_8k.py)| -|needlebench_v2_multi_retrieval_8k|needlebench_v2_multi_retrieval_8k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_8k.py)| -|needlebench_v2_single_8k|needlebench_v2_single_8k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_single_8k.py](needlebench_v2_8k/needlebench_v2_single_8k.py)| -|needlebench_v2_multi_retrieval_compare_batch_8k|needlebench_v2_multi_retrieval_compare_batch_8k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_compare_batch_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_compare_batch_8k.py)| -|needlebench_v2_32k|needlebench_v2_32k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_32k.py](needlebench_v2_32k/needlebench_v2_32k.py)| -|needlebench_v2_multi_reasoning_32k|needlebench_v2_multi_reasoning_32k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_reasoning_32k.py](needlebench_v2_32k/needlebench_v2_multi_reasoning_32k.py)| -|needlebench_v2_multi_retrieval_32k|needlebench_v2_multi_retrieval_32k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_32k.py](needlebench_v2_32k/needlebench_v2_multi_retrieval_32k.py)| -|needlebench_v2_single_32k|needlebench_v2_single_32k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_single_32k.py](needlebench_v2_32k/needlebench_v2_single_32k.py)| -|needlebench_v2_128k|needlebench_v2_128k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_128k.py](needlebench_v2_128k/needlebench_v2_128k.py)| -|needlebench_v2_multi_reasoning_128k|needlebench_v2_multi_reasoning_128k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_reasoning_128k.py](needlebench_v2_128k/needlebench_v2_multi_reasoning_128k.py)| -|needlebench_v2_multi_retrieval_128k|needlebench_v2_multi_retrieval_128k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_128k.py](needlebench_v2_128k/needlebench_v2_multi_retrieval_128k.py)| -|needlebench_v2_single_128k|needlebench_v2_single_128k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_single_128k.py](needlebench_v2_128k/needlebench_v2_single_128k.py)| -|needlebench_v2_200k|needlebench_v2_200k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_200k.py](needlebench_v2_200k/needlebench_v2_200k.py)| -|needlebench_v2_multi_reasoning_200k|needlebench_v2_multi_reasoning_200k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_reasoning_200k.py](needlebench_v2_200k/needlebench_v2_multi_reasoning_200k.py)| -|needlebench_v2_multi_retrieval_200k|needlebench_v2_multi_retrieval_200k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_200k.py](needlebench_v2_200k/needlebench_v2_multi_retrieval_200k.py)| -|needlebench_v2_single_200k|needlebench_v2_single_200k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_single_200k.py](needlebench_v2_200k/needlebench_v2_single_200k.py)| -|needlebench_v2_256k|needlebench_v2_256k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_256k.py](needlebench_v2_256k/needlebench_v2_256k.py)| -|needlebench_v2_multi_reasoning_256k|needlebench_v2_multi_reasoning_256k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_reasoning_256k.py](needlebench_v2_256k/needlebench_v2_multi_reasoning_256k.py)| -|needlebench_v2_multi_retrieval_256k|needlebench_v2_multi_retrieval_256k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_256k.py](needlebench_v2_256k/needlebench_v2_multi_retrieval_256k.py)| -|needlebench_v2_single_256k|needlebench_v2_single_256k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_single_256k.py](needlebench_v2_256k/needlebench_v2_single_256k.py)| -|needlebench_v2_1000k|needlebench_v2_1000k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_1000k.py](needlebench_v2_1000k/needlebench_v2_1000k.py)| -|needlebench_v2_multi_reasoning_1000k|needlebench_v2_multi_reasoning_1000k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_reasoning_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_reasoning_1000k.py)| -|needlebench_v2_multi_retrieval_1000k|needlebench_v2_multi_retrieval_1000k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_multi_retrieval_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_retrieval_1000k.py)| -|needlebench_v2_single_1000k|needlebench_v2_single_1000k|准确率(accuracy)|0-shot|对话格式|[needlebench_v2_single_1000k.py](needlebench_v2_1000k/needlebench_v2_single_1000k.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|atc_0shot_nocot_2_power_en|atc_0shot_nocot_2_power_en|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.atc.atc_0shot_nocot_2_power_en import needlebench_datasets as datasets`|[atc_0shot_nocot_2_power_en.py](atc/atc_0shot_nocot_2_power_en.py)| +|needlebench_v2_4k|needlebench_v2_4k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_4k import needlebench_datasets as datasets`|[needlebench_v2_4k.py](needlebench_v2_4k/needlebench_v2_4k.py)| +|needlebench_v2_multi_reasoning_4k|needlebench_v2_multi_reasoning_4k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_multi_reasoning_4k import needlebench_2needle_en_datasets as datasets`|[needlebench_v2_multi_reasoning_4k.py](needlebench_v2_4k/needlebench_v2_multi_reasoning_4k.py)| +|needlebench_v2_multi_retrieval_4k|needlebench_v2_multi_retrieval_4k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_multi_retrieval_4k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_4k.py](needlebench_v2_4k/needlebench_v2_multi_retrieval_4k.py)| +|needlebench_v2_single_4k|needlebench_v2_single_4k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_single_4k import needlebench_en_datasets as datasets`|[needlebench_v2_single_4k.py](needlebench_v2_4k/needlebench_v2_single_4k.py)| +|needlebench_v2_8k|needlebench_v2_8k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_8k import needlebench_datasets as datasets`|[needlebench_v2_8k.py](needlebench_v2_8k/needlebench_v2_8k.py)| +|needlebench_v2_multi_reasoning_8k|needlebench_v2_multi_reasoning_8k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_multi_reasoning_8k import needlebench_2needle_en_datasets as datasets`|[needlebench_v2_multi_reasoning_8k.py](needlebench_v2_8k/needlebench_v2_multi_reasoning_8k.py)| +|needlebench_v2_multi_retrieval_8k|needlebench_v2_multi_retrieval_8k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_multi_retrieval_8k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_8k.py)| +|needlebench_v2_single_8k|needlebench_v2_single_8k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_single_8k import needlebench_en_datasets as datasets`|[needlebench_v2_single_8k.py](needlebench_v2_8k/needlebench_v2_single_8k.py)| +|needlebench_v2_multi_retrieval_compare_batch_8k|needlebench_v2_multi_retrieval_compare_batch_8k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_multi_retrieval_compare_batch_8k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_compare_batch_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_compare_batch_8k.py)| +|needlebench_v2_32k|needlebench_v2_32k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_32k import needlebench_datasets as datasets`|[needlebench_v2_32k.py](needlebench_v2_32k/needlebench_v2_32k.py)| +|needlebench_v2_multi_reasoning_32k|needlebench_v2_multi_reasoning_32k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_multi_reasoning_32k import needlebench_2needle_en_datasets as datasets`|[needlebench_v2_multi_reasoning_32k.py](needlebench_v2_32k/needlebench_v2_multi_reasoning_32k.py)| +|needlebench_v2_multi_retrieval_32k|needlebench_v2_multi_retrieval_32k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_multi_retrieval_32k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_32k.py](needlebench_v2_32k/needlebench_v2_multi_retrieval_32k.py)| +|needlebench_v2_single_32k|needlebench_v2_single_32k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_single_32k import needlebench_en_datasets as datasets`|[needlebench_v2_single_32k.py](needlebench_v2_32k/needlebench_v2_single_32k.py)| +|needlebench_v2_128k|needlebench_v2_128k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_128k import needlebench_datasets as datasets`|[needlebench_v2_128k.py](needlebench_v2_128k/needlebench_v2_128k.py)| +|needlebench_v2_multi_reasoning_128k|needlebench_v2_multi_reasoning_128k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_multi_reasoning_128k import needlebench_2needle_en_datasets as datasets`|[needlebench_v2_multi_reasoning_128k.py](needlebench_v2_128k/needlebench_v2_multi_reasoning_128k.py)| +|needlebench_v2_multi_retrieval_128k|needlebench_v2_multi_retrieval_128k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_multi_retrieval_128k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_128k.py](needlebench_v2_128k/needlebench_v2_multi_retrieval_128k.py)| +|needlebench_v2_single_128k|needlebench_v2_single_128k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_single_128k import needlebench_en_datasets as datasets`|[needlebench_v2_single_128k.py](needlebench_v2_128k/needlebench_v2_single_128k.py)| +|needlebench_v2_200k|needlebench_v2_200k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_200k import needlebench_datasets as datasets`|[needlebench_v2_200k.py](needlebench_v2_200k/needlebench_v2_200k.py)| +|needlebench_v2_multi_reasoning_200k|needlebench_v2_multi_reasoning_200k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_multi_reasoning_200k import needlebench_2needle_en_datasets as datasets`|[needlebench_v2_multi_reasoning_200k.py](needlebench_v2_200k/needlebench_v2_multi_reasoning_200k.py)| +|needlebench_v2_multi_retrieval_200k|needlebench_v2_multi_retrieval_200k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_multi_retrieval_200k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_200k.py](needlebench_v2_200k/needlebench_v2_multi_retrieval_200k.py)| +|needlebench_v2_single_200k|needlebench_v2_single_200k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_single_200k import needlebench_en_datasets as datasets`|[needlebench_v2_single_200k.py](needlebench_v2_200k/needlebench_v2_single_200k.py)| +|needlebench_v2_256k|needlebench_v2_256k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_256k import needlebench_datasets as datasets`|[needlebench_v2_256k.py](needlebench_v2_256k/needlebench_v2_256k.py)| +|needlebench_v2_multi_reasoning_256k|needlebench_v2_multi_reasoning_256k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_multi_reasoning_256k import needlebench_2needle_en_datasets as datasets`|[needlebench_v2_multi_reasoning_256k.py](needlebench_v2_256k/needlebench_v2_multi_reasoning_256k.py)| +|needlebench_v2_multi_retrieval_256k|needlebench_v2_multi_retrieval_256k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_multi_retrieval_256k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_256k.py](needlebench_v2_256k/needlebench_v2_multi_retrieval_256k.py)| +|needlebench_v2_single_256k|needlebench_v2_single_256k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_single_256k import needlebench_en_datasets as datasets`|[needlebench_v2_single_256k.py](needlebench_v2_256k/needlebench_v2_single_256k.py)| +|needlebench_v2_1000k|needlebench_v2_1000k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_1000k import needlebench_datasets as datasets`|[needlebench_v2_1000k.py](needlebench_v2_1000k/needlebench_v2_1000k.py)| +|needlebench_v2_multi_reasoning_1000k|needlebench_v2_multi_reasoning_1000k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_multi_reasoning_1000k import needlebench_2needle_en_datasets as datasets`|[needlebench_v2_multi_reasoning_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_reasoning_1000k.py)| +|needlebench_v2_multi_retrieval_1000k|needlebench_v2_multi_retrieval_1000k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_multi_retrieval_1000k import needlebench_en_datasets as datasets`|[needlebench_v2_multi_retrieval_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_retrieval_1000k.py)| +|needlebench_v2_single_1000k|needlebench_v2_single_1000k|准确率(accuracy)|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_single_1000k import needlebench_en_datasets as datasets`|[needlebench_v2_single_1000k.py](needlebench_v2_1000k/needlebench_v2_single_1000k.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/needlebench_v2/README_en.md b/ais_bench/benchmark/configs/datasets/needlebench_v2/README_en.md index 6344d8d6..debab30f 100644 --- a/ais_bench/benchmark/configs/datasets/needlebench_v2/README_en.md +++ b/ais_bench/benchmark/configs/datasets/needlebench_v2/README_en.md @@ -51,35 +51,35 @@ It is recommended to download the dataset from Hugging Face: [https://huggingfac ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| atc_0shot_nocot_2_power_en | atc_0shot_nocot_2_power_en | Accuracy | 0-shot | Chat Format | [atc/atc_0shot_nocot_2_power_en.py]() | -| needlebench_v2_4k | needlebench_v2_4k | Accuracy | 0-shot | Chat Format | [needlebench_v2_4k.py](needlebench_v2_4k/needlebench_v2_4k.py) | -| needlebench_v2_multi_reasoning_4k | needlebench_v2_multi_reasoning_4k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_reasoning_4k.py](needlebench_v2_4k/needlebench_v2_multi_reasoning_4k.py) | -| needlebench_v2_multi_retrieval_4k | needlebench_v2_multi_retrieval_4k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_4k.py](needlebench_v2_4k/needlebench_v2_multi_retrieval_4k.py) | -| needlebench_v2_single_4k | needlebench_v2_single_4k | Accuracy | 0-shot | Chat Format | [needlebench_v2_single_4k.py](needlebench_v2_4k/needlebench_v2_single_4k.py) | -| needlebench_v2_8k | needlebench_v2_8k | Accuracy | 0-shot | Chat Format | [needlebench_v2_8k.py](needlebench_v2_8k/needlebench_v2_8k.py) | -| needlebench_v2_multi_reasoning_8k | needlebench_v2_multi_reasoning_8k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_reasoning_8k.py](needlebench_v2_8k/needlebench_v2_multi_reasoning_8k.py) | -| needlebench_v2_multi_retrieval_8k | needlebench_v2_multi_retrieval_8k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_8k.py) | -| needlebench_v2_single_8k | needlebench_v2_single_8k | Accuracy | 0-shot | Chat Format | [needlebench_v2_single_8k.py](needlebench_v2_8k/needlebench_v2_single_8k.py) | -| needlebench_v2_multi_retrieval_compare_batch_8k | needlebench_v2_multi_retrieval_compare_batch_8k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_compare_batch_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_compare_batch_8k.py) | -| needlebench_v2_32k | needlebench_v2_32k | Accuracy | 0-shot | Chat Format | [needlebench_v2_32k.py](needlebench_v2_32k/needlebench_v2_32k.py) | -| needlebench_v2_multi_reasoning_32k | needlebench_v2_multi_reasoning_32k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_reasoning_32k.py](needlebench_v2_32k/needlebench_v2_multi_reasoning_32k.py) | -| needlebench_v2_multi_retrieval_32k | needlebench_v2_multi_retrieval_32k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_32k.py](needlebench_v2_32k/needlebench_v2_multi_retrieval_32k.py) | -| needlebench_v2_single_32k | needlebench_v2_single_32k | Accuracy | 0-shot | Chat Format | [needlebench_v2_single_32k.py](needlebench_v2_32k/needlebench_v2_single_32k.py) | -| needlebench_v2_128k | needlebench_v2_128k | Accuracy | 0-shot | Chat Format | [needlebench_v2_128k.py](needlebench_v2_128k/needlebench_v2_128k.py) | -| needlebench_v2_multi_reasoning_128k | needlebench_v2_multi_reasoning_128k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_reasoning_128k.py](needlebench_v2_128k/needlebench_v2_multi_reasoning_128k.py) | -| needlebench_v2_multi_retrieval_128k | needlebench_v2_multi_retrieval_128k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_128k.py](needlebench_v2_128k/needlebench_v2_multi_retrieval_128k.py) | -| needlebench_v2_single_128k | needlebench_v2_single_128k | Accuracy | 0-shot | Chat Format | [needlebench_v2_single_128k.py](needlebench_v2_128k/needlebench_v2_single_128k.py) | -| needlebench_v2_200k | needlebench_v2_200k | Accuracy | 0-shot | Chat Format | [needlebench_v2_200k.py](needlebench_v2_200k/needlebench_v2_200k.py) | -| needlebench_v2_multi_reasoning_200k | needlebench_v2_multi_reasoning_200k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_reasoning_200k.py](needlebench_v2_200k/needlebench_v2_multi_reasoning_200k.py) | -| needlebench_v2_multi_retrieval_200k | needlebench_v2_multi_retrieval_200k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_200k.py](needlebench_v2_200k/needlebench_v2_multi_retrieval_200k.py) | -| needlebench_v2_single_200k | needlebench_v2_single_200k | Accuracy | 0-shot | Chat Format | [needlebench_v2_single_200k.py](needlebench_v2_200k/needlebench_v2_single_200k.py) | -| needlebench_v2_256k | needlebench_v2_256k | Accuracy | 0-shot | Chat Format | [needlebench_v2_256k.py](needlebench_v2_256k/needlebench_v2_256k.py) | -| needlebench_v2_multi_reasoning_256k | needlebench_v2_multi_reasoning_256k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_reasoning_256k.py](needlebench_v2_256k/needlebench_v2_multi_reasoning_256k.py) | -| needlebench_v2_multi_retrieval_256k | needlebench_v2_multi_retrieval_256k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_256k.py](needlebench_v2_256k/needlebench_v2_multi_retrieval_256k.py) | -| needlebench_v2_single_256k | needlebench_v2_single_256k | Accuracy | 0-shot | Chat Format | [needlebench_v2_single_256k.py](needlebench_v2_256k/needlebench_v2_single_256k.py) | -| needlebench_v2_1000k | needlebench_v2_1000k | Accuracy | 0-shot | Chat Format | [needlebench_v2_1000k.py](needlebench_v2_1000k/needlebench_v2_1000k.py) | -| needlebench_v2_multi_reasoning_1000k | needlebench_v2_multi_reasoning_1000k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_reasoning_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_reasoning_1000k.py) | -| needlebench_v2_multi_retrieval_1000k | needlebench_v2_multi_retrieval_1000k | Accuracy | 0-shot | Chat Format | [needlebench_v2_multi_retrieval_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_retrieval_1000k.py) | -| needlebench_v2_single_1000k | needlebench_v2_single_1000k | Accuracy | 0-shot | Chat Format | [needlebench_v2_single_1000k.py](needlebench_v2_1000k/needlebench_v2_single_1000k.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| atc_0shot_nocot_2_power_en | atc_0shot_nocot_2_power_en | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.atc.atc_0shot_nocot_2_power_en import needlebench_datasets as datasets`| [atc/atc_0shot_nocot_2_power_en.py]() | +| needlebench_v2_4k | needlebench_v2_4k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_4k import needlebench_datasets as datasets`| [needlebench_v2_4k.py](needlebench_v2_4k/needlebench_v2_4k.py) | +| needlebench_v2_multi_reasoning_4k | needlebench_v2_multi_reasoning_4k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_multi_reasoning_4k import needlebench_2needle_en_datasets as datasets`| [needlebench_v2_multi_reasoning_4k.py](needlebench_v2_4k/needlebench_v2_multi_reasoning_4k.py) | +| needlebench_v2_multi_retrieval_4k | needlebench_v2_multi_retrieval_4k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_multi_retrieval_4k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_4k.py](needlebench_v2_4k/needlebench_v2_multi_retrieval_4k.py) | +| needlebench_v2_single_4k | needlebench_v2_single_4k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_4k.needlebench_v2_single_4k import needlebench_en_datasets as datasets`| [needlebench_v2_single_4k.py](needlebench_v2_4k/needlebench_v2_single_4k.py) | +| needlebench_v2_8k | needlebench_v2_8k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_8k import needlebench_datasets as datasets`| [needlebench_v2_8k.py](needlebench_v2_8k/needlebench_v2_8k.py) | +| needlebench_v2_multi_reasoning_8k | needlebench_v2_multi_reasoning_8k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_multi_reasoning_8k import needlebench_2needle_en_datasets as datasets`| [needlebench_v2_multi_reasoning_8k.py](needlebench_v2_8k/needlebench_v2_multi_reasoning_8k.py) | +| needlebench_v2_multi_retrieval_8k | needlebench_v2_multi_retrieval_8k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_multi_retrieval_8k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_8k.py) | +| needlebench_v2_single_8k | needlebench_v2_single_8k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_single_8k import needlebench_en_datasets as datasets`| [needlebench_v2_single_8k.py](needlebench_v2_8k/needlebench_v2_single_8k.py) | +| needlebench_v2_multi_retrieval_compare_batch_8k | needlebench_v2_multi_retrieval_compare_batch_8k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_8k.needlebench_v2_multi_retrieval_compare_batch_8k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_compare_batch_8k.py](needlebench_v2_8k/needlebench_v2_multi_retrieval_compare_batch_8k.py) | +| needlebench_v2_32k | needlebench_v2_32k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_32k import needlebench_datasets as datasets`| [needlebench_v2_32k.py](needlebench_v2_32k/needlebench_v2_32k.py) | +| needlebench_v2_multi_reasoning_32k | needlebench_v2_multi_reasoning_32k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_multi_reasoning_32k import needlebench_2needle_en_datasets as datasets`| [needlebench_v2_multi_reasoning_32k.py](needlebench_v2_32k/needlebench_v2_multi_reasoning_32k.py) | +| needlebench_v2_multi_retrieval_32k | needlebench_v2_multi_retrieval_32k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_multi_retrieval_32k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_32k.py](needlebench_v2_32k/needlebench_v2_multi_retrieval_32k.py) | +| needlebench_v2_single_32k | needlebench_v2_single_32k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_32k.needlebench_v2_single_32k import needlebench_en_datasets as datasets`| [needlebench_v2_single_32k.py](needlebench_v2_32k/needlebench_v2_single_32k.py) | +| needlebench_v2_128k | needlebench_v2_128k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_128k import needlebench_datasets as datasets`| [needlebench_v2_128k.py](needlebench_v2_128k/needlebench_v2_128k.py) | +| needlebench_v2_multi_reasoning_128k | needlebench_v2_multi_reasoning_128k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_multi_reasoning_128k import needlebench_2needle_en_datasets as datasets`| [needlebench_v2_multi_reasoning_128k.py](needlebench_v2_128k/needlebench_v2_multi_reasoning_128k.py) | +| needlebench_v2_multi_retrieval_128k | needlebench_v2_multi_retrieval_128k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_multi_retrieval_128k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_128k.py](needlebench_v2_128k/needlebench_v2_multi_retrieval_128k.py) | +| needlebench_v2_single_128k | needlebench_v2_single_128k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_128k.needlebench_v2_single_128k import needlebench_en_datasets as datasets`| [needlebench_v2_single_128k.py](needlebench_v2_128k/needlebench_v2_single_128k.py) | +| needlebench_v2_200k | needlebench_v2_200k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_200k import needlebench_datasets as datasets`| [needlebench_v2_200k.py](needlebench_v2_200k/needlebench_v2_200k.py) | +| needlebench_v2_multi_reasoning_200k | needlebench_v2_multi_reasoning_200k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_multi_reasoning_200k import needlebench_2needle_en_datasets as datasets`| [needlebench_v2_multi_reasoning_200k.py](needlebench_v2_200k/needlebench_v2_multi_reasoning_200k.py) | +| needlebench_v2_multi_retrieval_200k | needlebench_v2_multi_retrieval_200k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_multi_retrieval_200k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_200k.py](needlebench_v2_200k/needlebench_v2_multi_retrieval_200k.py) | +| needlebench_v2_single_200k | needlebench_v2_single_200k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_200k.needlebench_v2_single_200k import needlebench_en_datasets as datasets`| [needlebench_v2_single_200k.py](needlebench_v2_200k/needlebench_v2_single_200k.py) | +| needlebench_v2_256k | needlebench_v2_256k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_256k import needlebench_datasets as datasets`| [needlebench_v2_256k.py](needlebench_v2_256k/needlebench_v2_256k.py) | +| needlebench_v2_multi_reasoning_256k | needlebench_v2_multi_reasoning_256k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_multi_reasoning_256k import needlebench_2needle_en_datasets as datasets`| [needlebench_v2_multi_reasoning_256k.py](needlebench_v2_256k/needlebench_v2_multi_reasoning_256k.py) | +| needlebench_v2_multi_retrieval_256k | needlebench_v2_multi_retrieval_256k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_multi_retrieval_256k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_256k.py](needlebench_v2_256k/needlebench_v2_multi_retrieval_256k.py) | +| needlebench_v2_single_256k | needlebench_v2_single_256k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_256k.needlebench_v2_single_256k import needlebench_en_datasets as datasets`| [needlebench_v2_single_256k.py](needlebench_v2_256k/needlebench_v2_single_256k.py) | +| needlebench_v2_1000k | needlebench_v2_1000k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_1000k import needlebench_datasets as datasets`| [needlebench_v2_1000k.py](needlebench_v2_1000k/needlebench_v2_1000k.py) | +| needlebench_v2_multi_reasoning_1000k | needlebench_v2_multi_reasoning_1000k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_multi_reasoning_1000k import needlebench_2needle_en_datasets as datasets`| [needlebench_v2_multi_reasoning_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_reasoning_1000k.py) | +| needlebench_v2_multi_retrieval_1000k | needlebench_v2_multi_retrieval_1000k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_multi_retrieval_1000k import needlebench_en_datasets as datasets`| [needlebench_v2_multi_retrieval_1000k.py](needlebench_v2_1000k/needlebench_v2_multi_retrieval_1000k.py) | +| needlebench_v2_single_1000k | needlebench_v2_single_1000k | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.needlebench_v2.needlebench_v2_1000k.needlebench_v2_single_1000k import needlebench_en_datasets as datasets`| [needlebench_v2_single_1000k.py](needlebench_v2_1000k/needlebench_v2_single_1000k.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/ocrbench_v2/README.md b/ais_bench/benchmark/configs/datasets/ocrbench_v2/README.md index 17fabcc9..bae99145 100644 --- a/ais_bench/benchmark/configs/datasets/ocrbench_v2/README.md +++ b/ais_bench/benchmark/configs/datasets/ocrbench_v2/README.md @@ -34,9 +34,9 @@ pip3 install -r requirements/datasets/ocrbench_v2.txt ``` ## 可用数据集任务 -| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 对应源码配置文件路径 | -| --- | --- | --- | --- | --- | --- | -| ocrbench_v2_gen_0_shot_chat | OCRBench_v2 数据集生成式任务，支持多模态输入（图像+文本） | 多种指标（根据任务类型） | 0-shot | 对话格式（多模态） | [ocrbench_v2_gen_0_shot_chat.py](ocrbench_v2_gen_0_shot_chat.py) | +| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| ocrbench_v2_gen_0_shot_chat | OCRBench_v2 数据集生成式任务，支持多模态输入（图像+文本） | 多种指标（根据任务类型） | 0-shot | 对话格式（多模态） | `from ais_bench.benchmark.configs.datasets.ocrbench_v2.ocrbench_v2_gen_0_shot_chat import ocrbench_v2_datasets as datasets` | [ocrbench_v2_gen_0_shot_chat.py](ocrbench_v2_gen_0_shot_chat.py) | ## 支持的任务类型 OCRBench_v2 数据集涵盖以下任务类型： diff --git a/ais_bench/benchmark/configs/datasets/ocrbench_v2/README_en.md b/ais_bench/benchmark/configs/datasets/ocrbench_v2/README_en.md index 736fbb0d..e87742e8 100644 --- a/ais_bench/benchmark/configs/datasets/ocrbench_v2/README_en.md +++ b/ais_bench/benchmark/configs/datasets/ocrbench_v2/README_en.md @@ -34,9 +34,9 @@ pip3 install -r requirements/datasets/ocrbench_v2.txt ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| ocrbench_v2_gen_0_shot_chat | Generative task for OCRBench_v2 dataset, supporting multimodal input (image + text) | Multiple metrics (depending on task type) | 0-shot | Chat format (multimodal) | [ocrbench_v2_gen_0_shot_chat.py](ocrbench_v2_gen_0_shot_chat.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| ocrbench_v2_gen_0_shot_chat | Generative task for OCRBench_v2 dataset, supporting multimodal input (image + text) | Multiple metrics (depending on task type) | 0-shot | Chat format (multimodal) |`from ais_bench.benchmark.configs.datasets.ocrbench_v2.ocrbench_v2_gen_0_shot_chat import ocrbench_v2_datasets as datasets`| [ocrbench_v2_gen_0_shot_chat.py](ocrbench_v2_gen_0_shot_chat.py) | ## Supported Task Types The OCRBench_v2 dataset covers the following task types: diff --git a/ais_bench/benchmark/configs/datasets/omnidocbench/README.md b/ais_bench/benchmark/configs/datasets/omnidocbench/README.md index b196e415..b234b92f 100644 --- a/ais_bench/benchmark/configs/datasets/omnidocbench/README.md +++ b/ais_bench/benchmark/configs/datasets/omnidocbench/README.md @@ -33,9 +33,9 @@ git lfs pull ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|omnidocbench_gen|OmniDocBench数据集生成式任务|accuracy (pass@1)|0-shot|字符串格式|[omnidocbench_gen.py](omnidocbench_gen.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|omnidocbench_gen|OmniDocBench数据集生成式任务|accuracy (pass@1)|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.omnidocbench.omnidocbench_gen import omnidocbench_datasets as datasets`|[omnidocbench_gen.py](omnidocbench_gen.py)| ## 使用约束 - 当前仅支持Edit_dist指标（用于测评DeepSeek-OCR模型），其他指标暂不支持，overall为各个维度的Edit_dist评分的均值 diff --git a/ais_bench/benchmark/configs/datasets/omnidocbench/README_en.md b/ais_bench/benchmark/configs/datasets/omnidocbench/README_en.md index 34955bd0..fc293070 100644 --- a/ais_bench/benchmark/configs/datasets/omnidocbench/README_en.md +++ b/ais_bench/benchmark/configs/datasets/omnidocbench/README_en.md @@ -33,9 +33,9 @@ git lfs pull ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| omnidocbench_gen | Generative task for the OmniDocBench dataset | accuracy (pass@1) | 0-shot | String format | [omnidocbench_gen.py](omnidocbench_gen.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| omnidocbench_gen | Generative task for the OmniDocBench dataset | accuracy (pass@1) | 0-shot | String format |`from ais_bench.benchmark.configs.datasets.omnidocbench.omnidocbench_gen import omnidocbench_datasets as datasets`| [omnidocbench_gen.py](omnidocbench_gen.py) | ## Usage Constraints: - Currently, only the Edit_dist metric is supported (used to evaluate the DeepSeek-OCR model); other metrics are not supported yet. The "overall" score is the average of the Edit_dist scores across all dimensions. diff --git a/ais_bench/benchmark/configs/datasets/piqa/README.md b/ais_bench/benchmark/configs/datasets/piqa/README.md index 22697229..0b9c2983 100644 --- a/ais_bench/benchmark/configs/datasets/piqa/README.md +++ b/ais_bench/benchmark/configs/datasets/piqa/README.md @@ -37,8 +37,8 @@ rm physicaliqa-train-dev.zip ## 可用数据集任务 ### piqa_gen_0_shot_chat_prompt #### 基本信息 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|piqa_gen_0_shot_chat_prompt|piqa数据集生成式任务|accuracy|0-shot|对话格式|[piqa_gen_0_shot_chat_prompt.py](piqa_gen_0_shot_chat_prompt.py)| -|piqa_gen_0_shot_str|piqa数据集生成式任务|accuracy|0-shot|字符串格式|[piqa_gen_0_shot_str.py](piqa_gen_0_shot_str.py)| -|piqa_ppl_0_shot_str|piqa数据集PPL任务|accuracy|0-shot|字符串格式|[piqa_ppl_0_shot_str.py](piqa_ppl_0_shot_str.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|piqa_gen_0_shot_chat_prompt|piqa数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.piqa.piqa_gen_0_shot_chat_prompt import piqa_datasets as datasets`|[piqa_gen_0_shot_chat_prompt.py](piqa_gen_0_shot_chat_prompt.py)| +|piqa_gen_0_shot_str|piqa数据集生成式任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.piqa.piqa_gen_0_shot_str import piqa_datasets as datasets`|[piqa_gen_0_shot_str.py](piqa_gen_0_shot_str.py)| +|piqa_ppl_0_shot_str|piqa数据集PPL任务|accuracy|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.piqa.piqa_ppl_0_shot_str import piqa_datasets as datasets`|[piqa_ppl_0_shot_str.py](piqa_ppl_0_shot_str.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/piqa/README_en.md b/ais_bench/benchmark/configs/datasets/piqa/README_en.md index 049bb73a..f59db0d2 100644 --- a/ais_bench/benchmark/configs/datasets/piqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/piqa/README_en.md @@ -29,8 +29,8 @@ rm physicaliqa-train-dev.zip ## Available Dataset Tasks ### piqa_gen_0_shot_chat_prompt #### Basic Information -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -| piqa_gen_0_shot_chat_prompt | Generative task for the piqa dataset | Accuracy | 0-shot | Chat Format | [piqa_gen_0_shot_chat_prompt.py](piqa_gen_0_shot_chat_prompt.py) | -| piqa_gen_0_shot_str | Generative task for the piqa dataset | Accuracy | 0-shot | String Format | [piqa_gen_0_shot_str.py](piqa_gen_0_shot_str.py) | -| piqa_ppl_0_shot_str | PPL task for the piqa dataset | Accuracy | 0-shot | String Format | [piqa_ppl_0_shot_str.py](piqa_ppl_0_shot_str.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| piqa_gen_0_shot_chat_prompt | Generative task for the piqa dataset | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.piqa.piqa_gen_0_shot_chat_prompt import piqa_datasets as datasets`| [piqa_gen_0_shot_chat_prompt.py](piqa_gen_0_shot_chat_prompt.py) | +| piqa_gen_0_shot_str | Generative task for the piqa dataset | Accuracy | 0-shot | String Format |`from ais_bench.benchmark.configs.datasets.piqa.piqa_gen_0_shot_str import piqa_datasets as datasets`| [piqa_gen_0_shot_str.py](piqa_gen_0_shot_str.py) | +| piqa_ppl_0_shot_str | PPL task for the piqa dataset | Accuracy | 0-shot | String Format |`from ais_bench.benchmark.configs.datasets.piqa.piqa_ppl_0_shot_str import piqa_datasets as datasets`| [piqa_ppl_0_shot_str.py](piqa_ppl_0_shot_str.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/race/README.md b/ais_bench/benchmark/configs/datasets/race/README.md index a6f015d2..506087ec 100644 --- a/ais_bench/benchmark/configs/datasets/race/README.md +++ b/ais_bench/benchmark/configs/datasets/race/README.md @@ -30,10 +30,10 @@ rm -r OpenCompassData-core-20240207.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|race_middle_gen_5_shot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|[race_middle_gen_5_shot_chat.py](race_middle_gen_5_shot_chat.py)| -|race_middle_gen_5_shot_cot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|[race_middle_gen_5_shot_cot_chat.py](race_middle_gen_5_shot_cot_chat.py)| -|race_high_gen_5_shot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|[race_high_gen_5_shot_chat.py](race_high_gen_5_shot_chat.py)| -|race_high_gen_5_shot_cot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|[race_high_gen_5_shot_cot_chat.py](race_high_gen_5_shot_cot_chat.py)| -|race_ppl_0_shot_chat|race数据集PPL任务|accuracy|0-shot|对话格式|[race_ppl_0_shot_chat.py](race_ppl_0_shot_chat.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|race_middle_gen_5_shot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.race.race_middle_gen_5_shot_chat import race_datasets as datasets`|[race_middle_gen_5_shot_chat.py](race_middle_gen_5_shot_chat.py)| +|race_middle_gen_5_shot_cot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.race.race_middle_gen_5_shot_cot_chat import race_datasets as datasets`|[race_middle_gen_5_shot_cot_chat.py](race_middle_gen_5_shot_cot_chat.py)| +|race_high_gen_5_shot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.race.race_high_gen_5_shot_chat import race_datasets as datasets`|[race_high_gen_5_shot_chat.py](race_high_gen_5_shot_chat.py)| +|race_high_gen_5_shot_cot_chat|race数据集生成式任务|accuracy|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.race.race_high_gen_5_shot_cot_chat import race_datasets as datasets`|[race_high_gen_5_shot_cot_chat.py](race_high_gen_5_shot_cot_chat.py)| +|race_ppl_0_shot_chat|race数据集PPL任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.race.race_ppl_0_shot_chat import race_datasets as datasets`|[race_ppl_0_shot_chat.py](race_ppl_0_shot_chat.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/race/README_en.md b/ais_bench/benchmark/configs/datasets/race/README_en.md index 492877fd..1a4943c3 100644 --- a/ais_bench/benchmark/configs/datasets/race/README_en.md +++ b/ais_bench/benchmark/configs/datasets/race/README_en.md @@ -30,10 +30,10 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| race_middle_gen_5_shot_chat | Generative task for the RACE dataset (middle school level) | Accuracy | 5-shot | Chat Format | [race_middle_gen_5_shot_chat.py](race_middle_gen_5_shot_chat.py) | -| race_middle_gen_5_shot_cot_chat | Generative task for the RACE dataset (middle school level) with chain-of-thought in prompt | Accuracy | 5-shot | Chat Format | [race_middle_gen_5_shot_cot_chat.py](race_middle_gen_5_shot_cot_chat.py) | -| race_high_gen_5_shot_chat | Generative task for the RACE dataset (senior high school level) | Accuracy | 5-shot | Chat Format | [race_high_gen_5_shot_chat.py](race_high_gen_5_shot_chat.py) | -| race_high_gen_5_shot_cot_chat | Generative task for the RACE dataset (senior high school level) with chain-of-thought in prompt | Accuracy | 5-shot | Chat Format | [race_high_gen_5_shot_cot_chat.py](race_high_gen_5_shot_cot_chat.py) | -| race_ppl_0_shot_chat | PPL task for the RACE dataset | Accuracy | 0-shot | Chat Format | [race_ppl_0_shot_chat.py](race_ppl_0_shot_chat.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| race_middle_gen_5_shot_chat | Generative task for the RACE dataset (middle school level) | Accuracy | 5-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.race.race_middle_gen_5_shot_chat import race_datasets as datasets`| [race_middle_gen_5_shot_chat.py](race_middle_gen_5_shot_chat.py) | +| race_middle_gen_5_shot_cot_chat | Generative task for the RACE dataset (middle school level) with chain-of-thought in prompt | Accuracy | 5-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.race.race_middle_gen_5_shot_cot_chat import race_datasets as datasets`| [race_middle_gen_5_shot_cot_chat.py](race_middle_gen_5_shot_cot_chat.py) | +| race_high_gen_5_shot_chat | Generative task for the RACE dataset (senior high school level) | Accuracy | 5-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.race.race_high_gen_5_shot_chat import race_datasets as datasets`| [race_high_gen_5_shot_chat.py](race_high_gen_5_shot_chat.py) | +| race_high_gen_5_shot_cot_chat | Generative task for the RACE dataset (senior high school level) with chain-of-thought in prompt | Accuracy | 5-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.race.race_high_gen_5_shot_cot_chat import race_datasets as datasets`| [race_high_gen_5_shot_cot_chat.py](race_high_gen_5_shot_cot_chat.py) | +| race_ppl_0_shot_chat | PPL task for the RACE dataset | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.race.race_ppl_0_shot_chat import race_datasets as datasets`| [race_ppl_0_shot_chat.py](race_ppl_0_shot_chat.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/realworldqa/README.md b/ais_bench/benchmark/configs/datasets/realworldqa/README.md index 2913cb44..4108452e 100644 --- a/ais_bench/benchmark/configs/datasets/realworldqa/README.md +++ b/ais_bench/benchmark/configs/datasets/realworldqa/README.md @@ -25,7 +25,7 @@ git clone https://huggingface.co/datasets/xai-community/realworldqa RealworldQA ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|realworldqa_gen|RealworldQA数据集生成式任务，⚠️该数据集任务下，会从Parquet文件中提取图片并保存到本地路径，然后将图片路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径图片。|accuracy|0-shot|列表格式（包含文本和图片两种数据）|[realworldqa_gen.py](realworldqa_gen.py)| -|realworldqa_gen_base64|RealworldQA数据集生成式任务，⚠️该数据集任务下，会将图片数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据。|accuracy|0-shot|列表格式（包含文本和图片两种数据）|[realworldqa_gen_base64.py](realworldqa_gen_base64.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|realworldqa_gen|RealworldQA数据集生成式任务，⚠️该数据集任务下，会从Parquet文件中提取图片并保存到本地路径，然后将图片路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径图片。|accuracy|0-shot|列表格式（包含文本和图片两种数据）|`from ais_bench.benchmark.configs.datasets.realworldqa.realworldqa_gen import realworldqa_datasets as datasets`|[realworldqa_gen.py](realworldqa_gen.py)| +|realworldqa_gen_base64|RealworldQA数据集生成式任务，⚠️该数据集任务下，会将图片数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据。|accuracy|0-shot|列表格式（包含文本和图片两种数据）|`from ais_bench.benchmark.configs.datasets.realworldqa.realworldqa_gen_base64 import realworldqa_datasets as datasets`|[realworldqa_gen_base64.py](realworldqa_gen_base64.py)| diff --git a/ais_bench/benchmark/configs/datasets/realworldqa/README_en.md b/ais_bench/benchmark/configs/datasets/realworldqa/README_en.md index 3a77c283..5cf8bdc3 100644 --- a/ais_bench/benchmark/configs/datasets/realworldqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/realworldqa/README_en.md @@ -25,7 +25,7 @@ git clone https://huggingface.co/datasets/xai-community/realworldqa RealworldQA ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| realworldqa_gen | Generative task for the RealworldQA dataset. ⚠️ For this dataset task, images will be extracted from Parquet files and saved to a local path, then the image paths will be passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the images at the specified path. | accuracy | 0-shot | List format (contains two types of data: text and image) | [realworldqa_gen.py](realworldqa_gen.py) | -| realworldqa_gen_base64 | Generative task for the RealworldQA dataset. ⚠️ For this dataset task, the image data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | accuracy | 0-shot | List format (contains two types of data: text and image) | [realworldqa_gen_base64.py](realworldqa_gen_base64.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| realworldqa_gen | Generative task for the RealworldQA dataset. ⚠️ For this dataset task, images will be extracted from Parquet files and saved to a local path, then the image paths will be passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the images at the specified path. | accuracy | 0-shot | List format (contains two types of data: text and image) |`from ais_bench.benchmark.configs.datasets.realworldqa.realworldqa_gen import realworldqa_datasets as datasets`| [realworldqa_gen.py](realworldqa_gen.py) | +| realworldqa_gen_base64 | Generative task for the RealworldQA dataset. ⚠️ For this dataset task, the image data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | accuracy | 0-shot | List format (contains two types of data: text and image) |`from ais_bench.benchmark.configs.datasets.realworldqa.realworldqa_gen_base64 import realworldqa_datasets as datasets`| [realworldqa_gen_base64.py](realworldqa_gen_base64.py) | diff --git a/ais_bench/benchmark/configs/datasets/refcoco/README.md b/ais_bench/benchmark/configs/datasets/refcoco/README.md index 4f2624f1..5059b4a3 100644 --- a/ais_bench/benchmark/configs/datasets/refcoco/README.md +++ b/ais_bench/benchmark/configs/datasets/refcoco/README.md @@ -49,10 +49,10 @@ RefCOCO/ ## 可用数据集任务 -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Config File | -| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -------- | ---------------------------------- | ---------------------------------------------- | -| refcoco_gen | RefCOCO 生成式定位任务配置，使用文件路径图像输入（`file://{image}`），导出 `RefCOCO_val`、`RefCOCO_test`、`RefCOCO_testA`、`RefCOCO_testB` 四个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | [refcoco_gen.py](refcoco_gen.py) | -| refcoco_gen_base64 | RefCOCO 生成式定位任务配置，使用 base64 data URL 图像输入（`data:image/jpeg;base64,{image}`），导出 `RefCOCO_base64_val`、`RefCOCO_base64_test`、`RefCOCO_base64_testA`、`RefCOCO_base64_testB` 四个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | [refcoco_gen_base64.py](refcoco_gen_base64.py) | +| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| refcoco_gen | RefCOCO 生成式定位任务配置，使用文件路径图像输入（`file://{image}`），导出 `RefCOCO_val`、`RefCOCO_test`、`RefCOCO_testA`、`RefCOCO_testB` 四个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | `from ais_bench.benchmark.configs.datasets.refcoco.refcoco_gen import refcoco_datasets as datasets` | [refcoco_gen.py](refcoco_gen.py) | +| refcoco_gen_base64 | RefCOCO 生成式定位任务配置，使用 base64 data URL 图像输入（`data:image/jpeg;base64,{image}`），导出 `RefCOCO_base64_val`、`RefCOCO_base64_test`、`RefCOCO_base64_testA`、`RefCOCO_base64_testB` 四个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | `from ais_bench.benchmark.configs.datasets.refcoco.refcoco_gen_base64 import refcoco_datasets as datasets` | [refcoco_gen_base64.py](refcoco_gen_base64.py) | ## 数据集分类 diff --git a/ais_bench/benchmark/configs/datasets/refcoco/README_en.md b/ais_bench/benchmark/configs/datasets/refcoco/README_en.md index c18f6e9c..3b42a11b 100644 --- a/ais_bench/benchmark/configs/datasets/refcoco/README_en.md +++ b/ais_bench/benchmark/configs/datasets/refcoco/README_en.md @@ -51,10 +51,10 @@ RefCOCO/ ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Config File | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------- | -------- | ----------------------------------------- | ---------------------------------------------- | -| refcoco_gen | RefCOCO generative grounding config that uses file-path image input (`file://{image}`) and exports `RefCOCO_val`, `RefCOCO_test`, `RefCOCO_testA`, and `RefCOCO_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | [refcoco_gen.py](refcoco_gen.py) | -| refcoco_gen_base64 | RefCOCO generative grounding config that uses base64 data-URL image input (`data:image/jpeg;base64,{image}`) and exports `RefCOCO_base64_val`, `RefCOCO_base64_test`, `RefCOCO_base64_testA`, and `RefCOCO_base64_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | [refcoco_gen_base64.py](refcoco_gen_base64.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Config File | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------- | -------- | ----------------------------------------- | --- | ---------------------------------------------- | +| refcoco_gen | RefCOCO generative grounding config that uses file-path image input (`file://{image}`) and exports `RefCOCO_val`, `RefCOCO_test`, `RefCOCO_testA`, and `RefCOCO_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | `from ais_bench.benchmark.configs.datasets.refcoco.refcoco_gen import refcoco_datasets as datasets` | [refcoco_gen.py](refcoco_gen.py) | +| refcoco_gen_base64 | RefCOCO generative grounding config that uses base64 data-URL image input (`data:image/jpeg;base64,{image}`) and exports `RefCOCO_base64_val`, `RefCOCO_base64_test`, `RefCOCO_base64_testA`, and `RefCOCO_base64_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | `from ais_bench.benchmark.configs.datasets.refcoco.refcoco_gen_base64 import refcoco_datasets as datasets` | [refcoco_gen_base64.py](refcoco_gen_base64.py) | ## Dataset Classification diff --git a/ais_bench/benchmark/configs/datasets/refcoco_plus/README.md b/ais_bench/benchmark/configs/datasets/refcoco_plus/README.md index 4e1cad17..8471d76a 100644 --- a/ais_bench/benchmark/configs/datasets/refcoco_plus/README.md +++ b/ais_bench/benchmark/configs/datasets/refcoco_plus/README.md @@ -44,10 +44,10 @@ RefCOCOplus/ ## 可用数据集任务 -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Config File | -| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -------- | ---------------------------------- | -------------------------------------------------------- | -| refcoco_plus_gen | RefCOCO+ 生成式定位任务配置，使用文件路径图像输入（`file://{image}`），导出 `RefCOCOPlus_val`、`RefCOCOPlus_testA`、`RefCOCOPlus_testB` 三个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | [refcoco_plus_gen.py](refcoco_plus_gen.py) | -| refcoco_plus_gen_base64 | RefCOCO+ 生成式定位任务配置，使用 base64 data URL 图像输入（`data:image/jpeg;base64,{image}`），导出 `RefCOCOPlus_base64_val`、`RefCOCOPlus_base64_testA`、`RefCOCOPlus_base64_testB` 三个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | [refcoco_plus_gen_base64.py](refcoco_plus_gen_base64.py) | +| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| refcoco_plus_gen | RefCOCO+ 生成式定位任务配置，使用文件路径图像输入（`file://{image}`），导出 `RefCOCOPlus_val`、`RefCOCOPlus_testA`、`RefCOCOPlus_testB` 三个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | `from ais_bench.benchmark.configs.datasets.refcoco_plus.refcoco_plus_gen import refcoco_plus_datasets as datasets` | [refcoco_plus_gen.py](refcoco_plus_gen.py) | +| refcoco_plus_gen_base64 | RefCOCO+ 生成式定位任务配置，使用 base64 data URL 图像输入（`data:image/jpeg;base64,{image}`），导出 `RefCOCOPlus_base64_val`、`RefCOCOPlus_base64_testA`、`RefCOCOPlus_base64_testB` 三个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | `from ais_bench.benchmark.configs.datasets.refcoco_plus.refcoco_plus_gen_base64 import refcoco_plus_datasets as datasets` | [refcoco_plus_gen_base64.py](refcoco_plus_gen_base64.py) | ## 数据集分类 diff --git a/ais_bench/benchmark/configs/datasets/refcoco_plus/README_en.md b/ais_bench/benchmark/configs/datasets/refcoco_plus/README_en.md index 0a90c023..3a833e2b 100644 --- a/ais_bench/benchmark/configs/datasets/refcoco_plus/README_en.md +++ b/ais_bench/benchmark/configs/datasets/refcoco_plus/README_en.md @@ -44,10 +44,10 @@ RefCOCOplus/ ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Config File | -| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -------- | ----------------------------------------- | -------------------------------------------------------- | -| refcoco_plus_gen | RefCOCO+ generative grounding config that uses file-path image input (`file://{image}`) and exports `RefCOCOPlus_val`, `RefCOCOPlus_testA`, and `RefCOCOPlus_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | [refcoco_plus_gen.py](refcoco_plus_gen.py) | -| refcoco_plus_gen_base64 | RefCOCO+ generative grounding config that uses base64 data-URL image input (`data:image/jpeg;base64,{image}`) and exports `RefCOCOPlus_base64_val`, `RefCOCOPlus_base64_testA`, and `RefCOCOPlus_base64_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | [refcoco_plus_gen_base64.py](refcoco_plus_gen_base64.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Config File | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -------- | ----------------------------------------- | --- | -------------------------------------------------------- | +| refcoco_plus_gen | RefCOCO+ generative grounding config that uses file-path image input (`file://{image}`) and exports `RefCOCOPlus_val`, `RefCOCOPlus_testA`, and `RefCOCOPlus_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | `from ais_bench.benchmark.configs.datasets.refcoco_plus.refcoco_plus_gen import refcoco_plus_datasets as datasets` | [refcoco_plus_gen.py](refcoco_plus_gen.py) | +| refcoco_plus_gen_base64 | RefCOCO+ generative grounding config that uses base64 data-URL image input (`data:image/jpeg;base64,{image}`) and exports `RefCOCOPlus_base64_val`, `RefCOCOPlus_base64_testA`, and `RefCOCOPlus_base64_testB` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | `from ais_bench.benchmark.configs.datasets.refcoco_plus.refcoco_plus_gen_base64 import refcoco_plus_datasets as datasets` | [refcoco_plus_gen_base64.py](refcoco_plus_gen_base64.py) | ## Dataset Classification diff --git a/ais_bench/benchmark/configs/datasets/refcocog/README.md b/ais_bench/benchmark/configs/datasets/refcocog/README.md index 865355b2..03294558 100644 --- a/ais_bench/benchmark/configs/datasets/refcocog/README.md +++ b/ais_bench/benchmark/configs/datasets/refcocog/README.md @@ -43,10 +43,10 @@ RefCOCOg/ ## 可用数据集任务 -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Config File | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -------- | ---------------------------------- | ------------------------------------------------ | -| refcocog_gen | RefCOCOg 生成式定位任务配置，使用文件路径图像输入（`file://{image}`），导出 `RefCOCOg_val` 和 `RefCOCOg_test` 两个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | [refcocog_gen.py](refcocog_gen.py) | -| refcocog_gen_base64 | RefCOCOg 生成式定位任务配置，使用 base64 data URL 图像输入（`data:image/jpeg;base64,{image}`），导出 `RefCOCOg_base64_val` 和 `RefCOCOg_base64_test` 两个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | [refcocog_gen_base64.py](refcocog_gen_base64.py) | +| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| refcocog_gen | RefCOCOg 生成式定位任务配置，使用文件路径图像输入（`file://{image}`），导出 `RefCOCOg_val` 和 `RefCOCOg_test` 两个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | `from ais_bench.benchmark.configs.datasets.refcocog.refcocog_gen import refcocog_datasets as datasets` | [refcocog_gen.py](refcocog_gen.py) | +| refcocog_gen_base64 | RefCOCOg 生成式定位任务配置，使用 base64 data URL 图像输入（`data:image/jpeg;base64,{image}`），导出 `RefCOCOg_base64_val` 和 `RefCOCOg_base64_test` 两个 split 任务 | Accuracy@0.5 | 0-shot | 多模态对话格式（MMPromptTemplate） | `from ais_bench.benchmark.configs.datasets.refcocog.refcocog_gen_base64 import refcocog_datasets as datasets` | [refcocog_gen_base64.py](refcocog_gen_base64.py) | ## 数据集分类 diff --git a/ais_bench/benchmark/configs/datasets/refcocog/README_en.md b/ais_bench/benchmark/configs/datasets/refcocog/README_en.md index 8e803ed3..e4633b63 100644 --- a/ais_bench/benchmark/configs/datasets/refcocog/README_en.md +++ b/ais_bench/benchmark/configs/datasets/refcocog/README_en.md @@ -43,10 +43,10 @@ RefCOCOg/ ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Config File | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -------- | ----------------------------------------- | ------------------------------------------------ | -| refcocog_gen | RefCOCOg generative grounding config that uses file-path image input (`file://{image}`) and exports `RefCOCOg_val` and `RefCOCOg_test` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | [refcocog_gen.py](refcocog_gen.py) | -| refcocog_gen_base64 | RefCOCOg generative grounding config that uses base64 data-URL image input (`data:image/jpeg;base64,{image}`) and exports `RefCOCOg_base64_val` and `RefCOCOg_base64_test` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | [refcocog_gen_base64.py](refcocog_gen_base64.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Config File | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -------- | ----------------------------------------- | --- | ------------------------------------------------ | +| refcocog_gen | RefCOCOg generative grounding config that uses file-path image input (`file://{image}`) and exports `RefCOCOg_val` and `RefCOCOg_test` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | `from ais_bench.benchmark.configs.datasets.refcocog.refcocog_gen import refcocog_datasets as datasets` | [refcocog_gen.py](refcocog_gen.py) | +| refcocog_gen_base64 | RefCOCOg generative grounding config that uses base64 data-URL image input (`data:image/jpeg;base64,{image}`) and exports `RefCOCOg_base64_val` and `RefCOCOg_base64_test` split tasks | Accuracy@0.5 | 0-shot | Multimodal chat format (MMPromptTemplate) | `from ais_bench.benchmark.configs.datasets.refcocog.refcocog_gen_base64 import refcocog_datasets as datasets` | [refcocog_gen_base64.py](refcocog_gen_base64.py) | ## Dataset Classification diff --git a/ais_bench/benchmark/configs/datasets/sharegpt/README.md b/ais_bench/benchmark/configs/datasets/sharegpt/README.md index 1e93d103..f91e97d4 100644 --- a/ais_bench/benchmark/configs/datasets/sharegpt/README.md +++ b/ais_bench/benchmark/configs/datasets/sharegpt/README.md @@ -54,9 +54,9 @@ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/b ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|sharegpt_gen|sharegpt生成式任务|暂不支持精度评测|0-shot|列表格式|[sharegpt_gen.py](sharegpt_gen.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|sharegpt_gen|sharegpt生成式任务|暂不支持精度评测|0-shot|列表格式|`from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets`|[sharegpt_gen.py](sharegpt_gen.py)| *注意：该多轮对话数据集的测评支持vLLM、SGLang、MindIE Service等服务化，使用时需指定--models为vllm_api_stream_chat_multiturn* \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/sharegpt/README_en.md b/ais_bench/benchmark/configs/datasets/sharegpt/README_en.md index 39fb7a16..9ff71e72 100644 --- a/ais_bench/benchmark/configs/datasets/sharegpt/README_en.md +++ b/ais_bench/benchmark/configs/datasets/sharegpt/README_en.md @@ -57,9 +57,9 @@ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/b ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| sharegpt_gen | Generative task for ShareGPT | Accuracy evaluation not supported temporarily | 0-shot | List Format | [sharegpt_gen.py](sharegpt_gen.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| sharegpt_gen | Generative task for ShareGPT | Accuracy evaluation not supported temporarily | 0-shot | List Format |`from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets`| [sharegpt_gen.py](sharegpt_gen.py) | *Note: The evaluation of this multi-turn conversation dataset supports service deployment frameworks such as vLLM, SGLang, and MindIE Service. When using it, you need to specify `--models` as `vllm_api_stream_chat_multiturn`.* \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/siqa/README.md b/ais_bench/benchmark/configs/datasets/siqa/README.md index cca0bc6f..66724396 100644 --- a/ais_bench/benchmark/configs/datasets/siqa/README.md +++ b/ais_bench/benchmark/configs/datasets/siqa/README.md @@ -27,7 +27,7 @@ rm -r OpenCompassData-core-20240207.zip ├── train-labels.lst ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|siqa_gen_0_shot_chat|siqa数据集生成式任务；`EDAccEvaluator`精度评估方式会通过`Levenshtein距离算法`选取最接近的答案，可能会造成误判，导致精度得分结果偏高。|accuracy|0-shot|对话格式|[siqa_gen_0_shot_chat.py](siqa_gen_0_shot_chat.py)| -|siqa_ppl_0_shot_chat|siqa数据集PPL任务|accuracy|0-shot|对话格式|[siqa_ppl_0_shot_chat.py](siqa_ppl_0_shot_chat.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|siqa_gen_0_shot_chat|siqa数据集生成式任务；`EDAccEvaluator`精度评估方式会通过`Levenshtein距离算法`选取最接近的答案，可能会造成误判，导致精度得分结果偏高。|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.siqa.siqa_gen_0_shot_chat import siqa_datasets as datasets`|[siqa_gen_0_shot_chat.py](siqa_gen_0_shot_chat.py)| +|siqa_ppl_0_shot_chat|siqa数据集PPL任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.siqa.siqa_ppl_0_shot_chat import siqa_datasets as datasets`|[siqa_ppl_0_shot_chat.py](siqa_ppl_0_shot_chat.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/siqa/README_en.md b/ais_bench/benchmark/configs/datasets/siqa/README_en.md index e05ee751..3487356f 100644 --- a/ais_bench/benchmark/configs/datasets/siqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/siqa/README_en.md @@ -28,7 +28,7 @@ rm -r OpenCompassData-core-20240207.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| siqa_gen_0_shot_chat | Generative task for the SIQA dataset; The `EDAccEvaluator` accuracy evaluation method selects the closest answer using the `Levenshtein distance algorithm`, which may cause misjudgment and result in an artificially high accuracy score. | Accuracy | 0-shot | Chat Format | [siqa_gen_0_shot_chat.py](siqa_gen_0_shot_chat.py) | -| siqa_ppl_0_shot_chat | PPL task for SIQA dataset | Accuracy | 0-shot | Chat Format | [siqa_ppl_0_shot_chat.py](siqa_ppl_0_shot_chat.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| siqa_gen_0_shot_chat | Generative task for the SIQA dataset; The `EDAccEvaluator` accuracy evaluation method selects the closest answer using the `Levenshtein distance algorithm`, which may cause misjudgment and result in an artificially high accuracy score. | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.siqa.siqa_gen_0_shot_chat import siqa_datasets as datasets`| [siqa_gen_0_shot_chat.py](siqa_gen_0_shot_chat.py) | +| siqa_ppl_0_shot_chat | PPL task for SIQA dataset | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.siqa.siqa_ppl_0_shot_chat import siqa_datasets as datasets`| [siqa_ppl_0_shot_chat.py](siqa_ppl_0_shot_chat.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/textvqa/README.md b/ais_bench/benchmark/configs/datasets/textvqa/README.md index a3d63f5e..ca0b0301 100644 --- a/ais_bench/benchmark/configs/datasets/textvqa/README.md +++ b/ais_bench/benchmark/configs/datasets/textvqa/README.md @@ -32,8 +32,8 @@ mv textvqa/*.jsonl textvqa/textvqa_json/ ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|textvqa_gen|TextVQA数据集生成式任务, ⚠️该数据集任务下，会直接将图片路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径图片。|VQA|0-shot|列表格式（包含文本和图片两种数据）|[textvqa_gen.py](textvqa_gen.py)| -|textvqa_gen_base64|TextVQA数据集生成式任务，⚠️该数据集任务下，会将图片数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据|VQA|0-shot|列表格式（包含文本和图片两种数据）|[textvqa_gen_base64.py](textvqa_gen_base64.py)| -|glm4v_textvqa_gen_base64|Glm4.1v-Thinking模型专用TextVQA数据集生成式任务，以适配该模型特殊的输出文本格式，⚠️该数据集任务下，会将图片数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据|VQA|0-shot|列表格式（包含文本和图片两种数据）|[glm4v_textvqa_gen_base64.py](glm4v_textvqa_gen_base64.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|textvqa_gen|TextVQA数据集生成式任务, ⚠️该数据集任务下，会直接将图片路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径图片。|VQA|0-shot|列表格式（包含文本和图片两种数据）|`from ais_bench.benchmark.configs.datasets.textvqa.textvqa_gen import textvqa_datasets as datasets`|[textvqa_gen.py](textvqa_gen.py)| +|textvqa_gen_base64|TextVQA数据集生成式任务，⚠️该数据集任务下，会将图片数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据|VQA|0-shot|列表格式（包含文本和图片两种数据）|`from ais_bench.benchmark.configs.datasets.textvqa.textvqa_gen_base64 import textvqa_datasets as datasets`|[textvqa_gen_base64.py](textvqa_gen_base64.py)| +|glm4v_textvqa_gen_base64|Glm4.1v-Thinking模型专用TextVQA数据集生成式任务，以适配该模型特殊的输出文本格式，⚠️该数据集任务下，会将图片数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据|VQA|0-shot|列表格式（包含文本和图片两种数据）|`from ais_bench.benchmark.configs.datasets.textvqa.glm4v_textvqa_gen_base64 import textvqa_datasets as datasets`|[glm4v_textvqa_gen_base64.py](glm4v_textvqa_gen_base64.py)| diff --git a/ais_bench/benchmark/configs/datasets/textvqa/README_en.md b/ais_bench/benchmark/configs/datasets/textvqa/README_en.md index fa505151..39705a6c 100644 --- a/ais_bench/benchmark/configs/datasets/textvqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/textvqa/README_en.md @@ -32,8 +32,8 @@ mv textvqa/*.jsonl textvqa/textvqa_json/ ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| textvqa_gen | Generative task for the TextVQA dataset. ⚠️ For this dataset task, the image path will be directly passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the images at the specified path. | VQA | 0-shot | List format (contains two types of data: text and image) | [textvqa_gen.py](textvqa_gen.py) | -| textvqa_gen_base64 | Generative task for the TextVQA dataset. ⚠️ For this dataset task, the image data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | VQA | 0-shot | List format (contains two types of data: text and image) | [textvqa_gen_base64.py](textvqa_gen_base64.py) | -| glm4v_textvqa_gen_base64 | Generative task for the TextVQA dataset limited to Glm4.1v-Thinking because of special output layout. ⚠️ For this dataset task, the image data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | VQA | 0-shot | List format (contains two types of data: text and image) | [glm4v_textvqa_gen_base64.py](glm4v_textvqa_gen_base64.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| textvqa_gen | Generative task for the TextVQA dataset. ⚠️ For this dataset task, the image path will be directly passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the images at the specified path. | VQA | 0-shot | List format (contains two types of data: text and image) |`from ais_bench.benchmark.configs.datasets.textvqa.textvqa_gen import textvqa_datasets as datasets`| [textvqa_gen.py](textvqa_gen.py) | +| textvqa_gen_base64 | Generative task for the TextVQA dataset. ⚠️ For this dataset task, the image data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | VQA | 0-shot | List format (contains two types of data: text and image) |`from ais_bench.benchmark.configs.datasets.textvqa.textvqa_gen_base64 import textvqa_datasets as datasets`| [textvqa_gen_base64.py](textvqa_gen_base64.py) | +| glm4v_textvqa_gen_base64 | Generative task for the TextVQA dataset limited to Glm4.1v-Thinking because of special output layout. ⚠️ For this dataset task, the image data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | VQA | 0-shot | List format (contains two types of data: text and image) |`from ais_bench.benchmark.configs.datasets.textvqa.glm4v_textvqa_gen_base64 import textvqa_datasets as datasets`| [glm4v_textvqa_gen_base64.py](glm4v_textvqa_gen_base64.py) | diff --git a/ais_bench/benchmark/configs/datasets/triviaqa/README.md b/ais_bench/benchmark/configs/datasets/triviaqa/README.md index b116a939..bb2103cc 100644 --- a/ais_bench/benchmark/configs/datasets/triviaqa/README.md +++ b/ais_bench/benchmark/configs/datasets/triviaqa/README.md @@ -25,7 +25,7 @@ rm triviaqa.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|triviaqa_gen_5_shot_chat_prompt|TriviaQA数据集生成式任务|accuracy|5-shot|对话格式|[triviaqa_gen_5_shot_chat_prompt.py](triviaqa_gen_5_shot_chat_prompt.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|triviaqa_gen_5_shot_chat_prompt|TriviaQA数据集生成式任务|accuracy|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.triviaqa.triviaqa_gen_5_shot_chat_prompt import triviaqa_datasets as datasets`|[triviaqa_gen_5_shot_chat_prompt.py](triviaqa_gen_5_shot_chat_prompt.py)| diff --git a/ais_bench/benchmark/configs/datasets/triviaqa/README_en.md b/ais_bench/benchmark/configs/datasets/triviaqa/README_en.md index afc2741e..0cc8db44 100644 --- a/ais_bench/benchmark/configs/datasets/triviaqa/README_en.md +++ b/ais_bench/benchmark/configs/datasets/triviaqa/README_en.md @@ -25,6 +25,6 @@ rm triviaqa.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| triviaqa_gen_5_shot_chat_prompt | Generative task for the TriviaQA dataset | Accuracy | 5-shot | Chat Format | [triviaqa_gen_5_shot_chat_prompt.py](triviaqa_gen_5_shot_chat_prompt.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| triviaqa_gen_5_shot_chat_prompt | Generative task for the TriviaQA dataset | Accuracy | 5-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.triviaqa.triviaqa_gen_5_shot_chat_prompt import triviaqa_datasets as datasets`| [triviaqa_gen_5_shot_chat_prompt.py](triviaqa_gen_5_shot_chat_prompt.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/videobench/README.md b/ais_bench/benchmark/configs/datasets/videobench/README.md index 374777c0..2c788c0c 100644 --- a/ais_bench/benchmark/configs/datasets/videobench/README.md +++ b/ais_bench/benchmark/configs/datasets/videobench/README.md @@ -33,7 +33,7 @@ mv videobench_subset/ videobench/ ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|videobench_gen|VideoBench数据集生成式任务，⚠️该数据集任务下，会直接将视频路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径视频。|accuracy|0-shot|列表格式（包含文本和视频两种数据）|[videobench_gen.py](videobench_gen.py)| -|videobench_gen_base64|VideoBench数据集生成式任务，⚠️该数据集任务下，会先将视频进行抽帧再转化为base64格式传入服务化，需确保服务化支持该输入格式数据。其中num_frames表示视频抽帧数，默认为5|accuracy|0-shot|列表格式（包含文本和视频两种数据）|[videobench_gen_base64.py](videobench_gen_base64.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|videobench_gen|VideoBench数据集生成式任务，⚠️该数据集任务下，会直接将视频路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径视频。|accuracy|0-shot|列表格式（包含文本和视频两种数据）|`from ais_bench.benchmark.configs.datasets.videobench.videobench_gen import videobench_datasets as datasets`|[videobench_gen.py](videobench_gen.py)| +|videobench_gen_base64|VideoBench数据集生成式任务，⚠️该数据集任务下，会先将视频进行抽帧再转化为base64格式传入服务化，需确保服务化支持该输入格式数据。其中num_frames表示视频抽帧数，默认为5|accuracy|0-shot|列表格式（包含文本和视频两种数据）|`from ais_bench.benchmark.configs.datasets.videobench.videobench_gen_base64 import videobench_datasets as datasets`|[videobench_gen_base64.py](videobench_gen_base64.py)| diff --git a/ais_bench/benchmark/configs/datasets/videobench/README_en.md b/ais_bench/benchmark/configs/datasets/videobench/README_en.md index 9acdae07..a073adb7 100644 --- a/ais_bench/benchmark/configs/datasets/videobench/README_en.md +++ b/ais_bench/benchmark/configs/datasets/videobench/README_en.md @@ -36,7 +36,7 @@ mv videobench_subset/ videobench/ ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| videobench_gen | Generative task for the VideoBench dataset. ⚠️ For this dataset task, the video path will be directly passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the videos at the specified path. | Accuracy | 0-shot | List format (contains two types of data: text and video) | [videobench_gen.py](videobench_gen.py) | -| videobench_gen_base64 | Generative task for the VideoBench dataset. ⚠️ For this dataset task, videos will first undergo frame extraction and then be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. Among the parameters, `num_frames` refers to the number of frames extracted from the video, with a default value of 5. | Accuracy | 0-shot | List format (contains two types of data: text and video) | [videobench_gen_base64.py](videobench_gen_base64.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| videobench_gen | Generative task for the VideoBench dataset. ⚠️ For this dataset task, the video path will be directly passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the videos at the specified path. | Accuracy | 0-shot | List format (contains two types of data: text and video) |`from ais_bench.benchmark.configs.datasets.videobench.videobench_gen import videobench_datasets as datasets`| [videobench_gen.py](videobench_gen.py) | +| videobench_gen_base64 | Generative task for the VideoBench dataset. ⚠️ For this dataset task, videos will first undergo frame extraction and then be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. Among the parameters, `num_frames` refers to the number of frames extracted from the video, with a default value of 5. | Accuracy | 0-shot | List format (contains two types of data: text and video) |`from ais_bench.benchmark.configs.datasets.videobench.videobench_gen_base64 import videobench_datasets as datasets`| [videobench_gen_base64.py](videobench_gen_base64.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/videomme/README.md b/ais_bench/benchmark/configs/datasets/videomme/README.md index 0b308d0d..adfd2e5b 100644 --- a/ais_bench/benchmark/configs/datasets/videomme/README.md +++ b/ais_bench/benchmark/configs/datasets/videomme/README.md @@ -13,7 +13,7 @@ Video-MME 是面向多模态大语言模型（MLLM）的视频理解评测基准 Video-MME ├── videomme │   └── test-00000-of-00001.parquet - │ + │ ├── subtitle │  ├── 068rdc75mHM.srt │   └── 08km9Y1bt-A.srt @@ -30,6 +30,6 @@ Video-MME 是面向多模态大语言模型（MLLM）的视频理解评测基准 #### 基本信息 - 当前对于Video-MME数据集的测评暂不支持字幕数据的传入 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|videomme_gen|videomme数据集生成式任务|acc|0-shot|字符串格式|[videomme_gen.py](videomme_gen.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|videomme_gen|videomme数据集生成式任务|acc|0-shot|字符串格式|`from ais_bench.benchmark.configs.datasets.videomme.videomme_gen import videomme_datasets as datasets`|[videomme_gen.py](videomme_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/videomme/README_en.md b/ais_bench/benchmark/configs/datasets/videomme/README_en.md index 2bb588a1..84ba0a9f 100644 --- a/ais_bench/benchmark/configs/datasets/videomme/README_en.md +++ b/ais_bench/benchmark/configs/datasets/videomme/README_en.md @@ -13,7 +13,7 @@ Video-MME is a Video understanding evaluation benchmark for multimodal large lan Video-MME ├── videomme │   └── test-00000-of-00001.parquet - │ + │ ├── subtitle │  ├── 068rdc75mHM.srt │   └── 08km9Y1bt-A.srt @@ -30,6 +30,6 @@ Video-MME is a Video understanding evaluation benchmark for multimodal large lan #### Basic Information - Currently, the evaluation of the Video-MME dataset does not support the input of subtitle data for the time being -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | --- | -|videomme_gen|Generative task for the videomme dataset|acc|0-shot|String format|[videomme_gen.py](videomme_gen.py)| +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +|videomme_gen|Generative task for the videomme dataset|acc|0-shot|String format|`from ais_bench.benchmark.configs.datasets.videomme.videomme_gen import videomme_datasets as datasets`|[videomme_gen.py](videomme_gen.py)| diff --git a/ais_bench/benchmark/configs/datasets/vocalsound/README.md b/ais_bench/benchmark/configs/datasets/vocalsound/README.md index c52b1fda..864b83e0 100644 --- a/ais_bench/benchmark/configs/datasets/vocalsound/README.md +++ b/ais_bench/benchmark/configs/datasets/vocalsound/README.md @@ -30,7 +30,7 @@ mv vocalsound/subset5/* vocalsound/ ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|vocalsound_gen|VocalSound数据集生成式任务，⚠️该数据集任务下会直接将音频路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径音频。|accuracy|0-shot|列表格式（包含文本和音频两种数据）|[vocalsound_gen.py](vocalsound_gen.py)| -|vocalsound_gen_base64|VocalSound数据集生成式任务，⚠️该数据集任务下，会将音频数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据。|accuracy|0-shot|列表格式（包含文本和音频两种数据）|[vocalsound_gen_base64.py](vocalsound_gen_base64.py)| \ No newline at end of file +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|vocalsound_gen|VocalSound数据集生成式任务，⚠️该数据集任务下会直接将音频路径传入服务化，需确保服务化支持该格式输入并且有权限访问该路径音频。|accuracy|0-shot|列表格式（包含文本和音频两种数据）|`from ais_bench.benchmark.configs.datasets.vocalsound.vocalsound_gen import vocalsound_datasets as datasets`|[vocalsound_gen.py](vocalsound_gen.py)| +|vocalsound_gen_base64|VocalSound数据集生成式任务，⚠️该数据集任务下，会将音频数据转化为base64格式再传入服务化，需确保服务化支持该输入格式数据。|accuracy|0-shot|列表格式（包含文本和音频两种数据）|`from ais_bench.benchmark.configs.datasets.vocalsound.vocalsound_gen_base64 import vocalsound_datasets as datasets`|[vocalsound_gen_base64.py](vocalsound_gen_base64.py)| \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/vocalsound/README_en.md b/ais_bench/benchmark/configs/datasets/vocalsound/README_en.md index e76eded0..6f6e0704 100644 --- a/ais_bench/benchmark/configs/datasets/vocalsound/README_en.md +++ b/ais_bench/benchmark/configs/datasets/vocalsound/README_en.md @@ -30,7 +30,7 @@ mv vocalsound/subset5/* vocalsound/ ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| vocalsound_gen | Generative task for the VocalSound dataset. ⚠️ For this dataset task, the audio path will be directly passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the audio at the specified path. | Accuracy | 0-shot | List format (contains two types of data: text and audio) | [vocalsound_gen.py](vocalsound_gen.py) | -| vocalsound_gen_base64 | Generative task for the VocalSound dataset. ⚠️ For this dataset task, the audio data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | Accuracy | 0-shot | List format (contains two types of data: text and audio) | [vocalsound_gen_base64.py](vocalsound_gen_base64.py) | \ No newline at end of file +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| vocalsound_gen | Generative task for the VocalSound dataset. ⚠️ For this dataset task, the audio path will be directly passed to the service deployment. Ensure that the service deployment supports this input format and has permission to access the audio at the specified path. | Accuracy | 0-shot | List format (contains two types of data: text and audio) |`from ais_bench.benchmark.configs.datasets.vocalsound.vocalsound_gen import vocalsound_datasets as datasets`| [vocalsound_gen.py](vocalsound_gen.py) | +| vocalsound_gen_base64 | Generative task for the VocalSound dataset. ⚠️ For this dataset task, the audio data will be converted to Base64 format before being passed to the service deployment. Ensure that the service deployment supports this input format. | Accuracy | 0-shot | List format (contains two types of data: text and audio) |`from ais_bench.benchmark.configs.datasets.vocalsound.vocalsound_gen_base64 import vocalsound_datasets as datasets`| [vocalsound_gen_base64.py](vocalsound_gen_base64.py) | \ No newline at end of file diff --git a/ais_bench/benchmark/configs/datasets/winogrande/README.md b/ais_bench/benchmark/configs/datasets/winogrande/README.md index c8330ef0..ff3c54bb 100644 --- a/ais_bench/benchmark/configs/datasets/winogrande/README.md +++ b/ais_bench/benchmark/configs/datasets/winogrande/README.md @@ -39,7 +39,7 @@ rm winogrande.zip ``` ## 可用数据集任务 -|任务名称|简介|评估指标|few-shot|prompt格式|对应源码配置文件路径| -| --- | --- | --- | --- | --- | --- | -|winogrande_gen_0_shot_chat_prompt|winogrande数据集生成式任务|accuracy|0-shot|对话格式|[winogrande_gen_0_shot_chat_prompt.py](winogrande_gen_0_shot_chat_prompt.py)| -|winogrande_gen_5_shot_chat_prompt|piqa数据集生成式任务|accuracy|5-shot|对话格式|[winogrande_gen_5_shot_chat_prompt.py](winogrande_gen_5_shot_chat_prompt.py)| +|任务名称|简介|评估指标|few-shot|prompt格式|配套文件导入方式|对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | --- | +|winogrande_gen_0_shot_chat_prompt|winogrande数据集生成式任务|accuracy|0-shot|对话格式|`from ais_bench.benchmark.configs.datasets.winogrande.winogrande_gen_0_shot_chat_prompt import winogrande_datasets as datasets`|[winogrande_gen_0_shot_chat_prompt.py](winogrande_gen_0_shot_chat_prompt.py)| +|winogrande_gen_5_shot_chat_prompt|piqa数据集生成式任务|accuracy|5-shot|对话格式|`from ais_bench.benchmark.configs.datasets.winogrande.winogrande_gen_5_shot_chat_prompt import winogrande_datasets as datasets`|[winogrande_gen_5_shot_chat_prompt.py](winogrande_gen_5_shot_chat_prompt.py)| diff --git a/ais_bench/benchmark/configs/datasets/winogrande/README_en.md b/ais_bench/benchmark/configs/datasets/winogrande/README_en.md index 773ca037..1f98fc1e 100644 --- a/ais_bench/benchmark/configs/datasets/winogrande/README_en.md +++ b/ais_bench/benchmark/configs/datasets/winogrande/README_en.md @@ -39,10 +39,10 @@ rm winogrande.zip ``` ## Available Dataset Tasks -| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Corresponding Source Code File Path | -| --- | --- | --- | --- | --- | --- | -| winogrande_gen_0_shot_chat_prompt | Generative task for the WinoGrande dataset | Accuracy | 0-shot | Chat Format | [winogrande_gen_0_shot_chat_prompt.py](winogrande_gen_0_shot_chat_prompt.py) | -| winogrande_gen_5_shot_chat_prompt | Generative task for the WinoGrande dataset (Note: The original "piqa dataset" in the introduction is a typo, corrected to "WinoGrande dataset" for consistency) | Accuracy | 5-shot | Chat Format | [winogrande_gen_5_shot_chat_prompt.py](winogrande_gen_5_shot_chat_prompt.py) | +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code File Path | +| --- | --- | --- | --- | --- | --- | --- | +| winogrande_gen_0_shot_chat_prompt | Generative task for the WinoGrande dataset | Accuracy | 0-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.winogrande.winogrande_gen_0_shot_chat_prompt import winogrande_datasets as datasets`| [winogrande_gen_0_shot_chat_prompt.py](winogrande_gen_0_shot_chat_prompt.py) | +| winogrande_gen_5_shot_chat_prompt | Generative task for the WinoGrande dataset (Note: The original "piqa dataset" in the introduction is a typo, corrected to "WinoGrande dataset" for consistency) | Accuracy | 5-shot | Chat Format |`from ais_bench.benchmark.configs.datasets.winogrande.winogrande_gen_5_shot_chat_prompt import winogrande_datasets as datasets`| [winogrande_gen_5_shot_chat_prompt.py](winogrande_gen_5_shot_chat_prompt.py) | ### Note diff --git a/ais_bench/configs/accuracy_benchmark/ceval_merge_en.py b/ais_bench/configs/accuracy_benchmark/ceval_merge_en.py new file mode 100644 index 00000000..2a178779 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/ceval_merge_en.py @@ -0,0 +1,23 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + +models = vllm_api_general +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py b/ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py new file mode 100644 index 00000000..2a178779 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py @@ -0,0 +1,23 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + +models = vllm_api_general +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py b/ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py new file mode 100644 index 00000000..6006cb04 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py @@ -0,0 +1,23 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py b/ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py new file mode 100644 index 00000000..6006cb04 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py @@ -0,0 +1,23 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py b/ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py new file mode 100644 index 00000000..741ac8c0 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py @@ -0,0 +1,28 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +models = vllm_api_general_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +# Key: replace or modify the implementation of the answer extraction function +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py b/ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py new file mode 100644 index 00000000..e5996975 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py @@ -0,0 +1,28 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +models = vllm_api_general_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +# 替换或修改答案的提取函数实现 +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_repeat_en.py b/ais_bench/configs/accuracy_benchmark/multi_repeat_en.py new file mode 100644 index 00000000..74361056 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_repeat_en.py @@ -0,0 +1,30 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict( + temperature=0.01, + ignore_eos=False, + num_return_sequences=5, # For the specific usage and constraints, refer to the accuracy_metric.md documentation +) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py b/ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py new file mode 100644 index 00000000..763e4ec0 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py @@ -0,0 +1,30 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict( + temperature=0.01, + ignore_eos=False, + num_return_sequences=5, # 具体作用和约束请参考文档 accuracy_metric.md +) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_task_en.py b/ais_bench/configs/accuracy_benchmark/multi_task_en.py new file mode 100644 index 00000000..fe933173 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_task_en.py @@ -0,0 +1,33 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[1]["host_ip"] = "localhost" +models[1]["host_port"] = 8081 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[1]["max_out_len"] = 512 +models[1]["batch_size"] = 1 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py b/ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py new file mode 100644 index 00000000..fe933173 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py @@ -0,0 +1,33 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[1]["host_ip"] = "localhost" +models[1]["host_port"] = 8081 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[1]["max_out_len"] = 512 +models[1]["batch_size"] = 1 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py b/ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py new file mode 100644 index 00000000..fe933173 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py @@ -0,0 +1,33 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[1]["host_ip"] = "localhost" +models[1]["host_port"] = 8081 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[1]["max_out_len"] = 512 +models[1]["batch_size"] = 1 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py b/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py new file mode 100644 index 00000000..ce94c64c --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py @@ -0,0 +1,24 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_datasets +models = vllm_api_general_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py b/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py new file mode 100644 index 00000000..ce94c64c --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py @@ -0,0 +1,24 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_datasets +models = vllm_api_general_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py b/ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py new file mode 100644 index 00000000..fe933173 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py @@ -0,0 +1,33 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[1]["host_ip"] = "localhost" +models[1]["host_port"] = 8081 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[1]["max_out_len"] = 512 +models[1]["batch_size"] = 1 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/single_task_en.py b/ais_bench/configs/accuracy_benchmark/single_task_en.py new file mode 100644 index 00000000..d9abf4d4 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/single_task_en.py @@ -0,0 +1,26 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +models = vllm_api_general_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=False) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py b/ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py new file mode 100644 index 00000000..d9abf4d4 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py @@ -0,0 +1,26 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +models = vllm_api_general_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=False) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py b/ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py new file mode 100644 index 00000000..7f87fb9c --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py @@ -0,0 +1,38 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py b/ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py new file mode 100644 index 00000000..034d0d2d --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py @@ -0,0 +1,38 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py b/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py new file mode 100644 index 00000000..4aeb54dd --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py @@ -0,0 +1,43 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +# Key: replace or modify the implementation of the answer extraction function +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py b/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py new file mode 100644 index 00000000..94aa1542 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py @@ -0,0 +1,43 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +# 关键：替换或修改答案的提取函数实现 +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/multi_task_en.py b/ais_bench/configs/accuracy_benchmark_local/multi_task_en.py new file mode 100644 index 00000000..57a29148 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/multi_task_en.py @@ -0,0 +1,41 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + +datasets = gsm8k_datasets + aime2024_datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py b/ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py new file mode 100644 index 00000000..236e466b --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py @@ -0,0 +1,41 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + +datasets = gsm8k_datasets + aime2024_datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/single_task_en.py b/ais_bench/configs/accuracy_benchmark_local/single_task_en.py new file mode 100644 index 00000000..12b9ec15 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/single_task_en.py @@ -0,0 +1,38 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py b/ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py new file mode 100644 index 00000000..695f7ac4 --- /dev/null +++ b/ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py @@ -0,0 +1,38 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.01, + do_sample=False, + ), + max_out_len=512, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +work_dir = 'outputs/default/' + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py b/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py index 1a3ca5b9..36379739 100644 --- a/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py +++ b/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py @@ -1,8 +1,8 @@ from mmengine.config import read_base from ais_bench.benchmark.models import MindieStreamApi from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask with read_base(): from ais_bench.benchmark.configs.summarizers.example import summarizer @@ -44,8 +44,8 @@ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/api-mindie-stream/' # 工作路径 \ No newline at end of file diff --git a/ais_bench/configs/api_examples/infer_vllm_api_general.py b/ais_bench/configs/api_examples/infer_vllm_api_general.py index 6c07ff49..fec91344 100644 --- a/ais_bench/configs/api_examples/infer_vllm_api_general.py +++ b/ais_bench/configs/api_examples/infer_vllm_api_general.py @@ -1,8 +1,8 @@ from mmengine.config import read_base from ais_bench.benchmark.models import VLLMCustomAPI from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask with read_base(): from ais_bench.benchmark.configs.summarizers.example import summarizer @@ -18,9 +18,7 @@ type=VLLMCustomAPI, abbr='vllm-api-general', model="", - max_seq_len = 4096, request_rate = 0, - rpm_verbose = False, retry = 2, host_ip = "localhost", host_port = 8080, @@ -40,8 +38,8 @@ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/api-vllm-general/' diff --git a/ais_bench/configs/api_examples/infer_vllm_api_general_chat.py b/ais_bench/configs/api_examples/infer_vllm_api_general_chat.py index 8a4e023e..9271ce1d 100644 --- a/ais_bench/configs/api_examples/infer_vllm_api_general_chat.py +++ b/ais_bench/configs/api_examples/infer_vllm_api_general_chat.py @@ -1,8 +1,8 @@ from mmengine.config import read_base from ais_bench.benchmark.models import VLLMCustomAPIChat from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask with read_base(): from ais_bench.benchmark.configs.summarizers.example import summarizer @@ -37,8 +37,8 @@ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/api-vllm-general-chat/' diff --git a/ais_bench/configs/api_examples/infer_vllm_api_multi_model_multi_dataset.py b/ais_bench/configs/api_examples/infer_vllm_api_multi_model_multi_dataset.py new file mode 100644 index 00000000..d56e9617 --- /dev/null +++ b/ais_bench/configs/api_examples/infer_vllm_api_multi_model_multi_dataset.py @@ -0,0 +1,15 @@ +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_5_shot_str import mmlu_datasets as mmlu_5_shot_str + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat + mmlu_5_shot_str +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat + +work_dir = 'outputs/multi_model_multi_dataset/' diff --git a/ais_bench/configs/api_examples/infer_vllm_api_old.py b/ais_bench/configs/api_examples/infer_vllm_api_old.py deleted file mode 100644 index 1055ac4a..00000000 --- a/ais_bench/configs/api_examples/infer_vllm_api_old.py +++ /dev/null @@ -1,46 +0,0 @@ -from mmengine.config import read_base -from ais_bench.benchmark.models import VLLMCustomAPIOld -from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask - -with read_base(): - from ais_bench.benchmark.configs.summarizers.example import summarizer - from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str - -datasets = [ - *gsm8k_0_shot_cot_str, -] - -models = [ - dict( - attr="service", - type=VLLMCustomAPIOld, - abbr='vllm-api-old', - max_seq_len = 4096, - request_rate = 0, - rpm_verbose = False, - retry = 2, - host_ip = "localhost", - host_port = 8080, - enable_ssl = False, - max_out_len = 512, - batch_size=1, - generation_kwargs = dict( - temperature = 0.5, - top_k = 10, - top_p = 0.95, - seed = None, - repetition_penalty = 1.03, - ) - ) -] - - -infer = dict(partitioner=dict(type=NaivePartitioner), - runner=dict( - type=LocalAPIRunner, - max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) - -work_dir = 'outputs/api-vllm-old/' \ No newline at end of file diff --git a/ais_bench/configs/api_examples/infer_vllm_api_stream_chat.py b/ais_bench/configs/api_examples/infer_vllm_api_stream_chat.py index c8daef24..bba3a371 100644 --- a/ais_bench/configs/api_examples/infer_vllm_api_stream_chat.py +++ b/ais_bench/configs/api_examples/infer_vllm_api_stream_chat.py @@ -1,8 +1,8 @@ from mmengine.config import read_base from ais_bench.benchmark.models import VLLMCustomAPIChatStream from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask with read_base(): from ais_bench.benchmark.configs.summarizers.example import summarizer @@ -18,9 +18,7 @@ type=VLLMCustomAPIChatStream, abbr='vllm-api-stream-chat', model="", - max_seq_len = 4096, request_rate = 0, - rpm_verbose = False, retry = 2, host_ip = "localhost", host_port = 8080, @@ -40,8 +38,8 @@ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/api-vllm-stream-chat/' diff --git a/ais_bench/configs/api_examples/infer_vllm_api_with_judge_model.py b/ais_bench/configs/api_examples/infer_vllm_api_with_judge_model.py new file mode 100644 index 00000000..93cd7c9f --- /dev/null +++ b/ais_bench/configs/api_examples/infer_vllm_api_with_judge_model.py @@ -0,0 +1,45 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.aime2025.aime2025_gen_0_shot_llmjudge import aime2025_datasets + +datasets = aime2025_datasets + +datasets[0]['judge_infer_cfg']['judge_model']['host_ip'] = 'localhost' +datasets[0]['judge_infer_cfg']['judge_model']['host_port'] = 8081 + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-judge-eval', + path="", + model="", + stream=True, + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + pred_postprocessor=dict(type=extract_non_reasoning_content), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/judge_eval/' diff --git a/ais_bench/configs/api_examples/infer_vllm_api_with_model_dataset_combinations.py b/ais_bench/configs/api_examples/infer_vllm_api_with_model_dataset_combinations.py new file mode 100644 index 00000000..1a50985a --- /dev/null +++ b/ais_bench/configs/api_examples/infer_vllm_api_with_model_dataset_combinations.py @@ -0,0 +1,20 @@ +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat + +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), + dict(models=[models[2]], datasets=[datasets[0], datasets[1]]), +] + +work_dir = 'outputs/custom_combinations/' diff --git a/ais_bench/configs/api_examples/perf_vllm_api_custom_dataset.py b/ais_bench/configs/api_examples/perf_vllm_api_custom_dataset.py new file mode 100644 index 00000000..b6c01b15 --- /dev/null +++ b/ais_bench/configs/api_examples/perf_vllm_api_custom_dataset.py @@ -0,0 +1,65 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever +from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer +from ais_bench.benchmark.datasets import CustomDataset +from ais_bench.benchmark.openicl.icl_evaluator import AccEvaluator + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + +datasets = [ + dict( + abbr='my_custom_dataset', + type=CustomDataset, + path='/path/to/your/dataset.jsonl', + reader_cfg=dict( + input_columns=['question'], + output_column='answer', + ), + infer_cfg=dict( + prompt_template=dict( + type=PromptTemplate, + template='{question}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + eval_cfg=dict( + evaluator=dict(type=AccEvaluator), + pred_role='BOT', + ), + meta_path='', + ) +] + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-custom-dataset', + model="", + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.5, top_k=10, top_p=0.95), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/custom_dataset/' diff --git a/ais_bench/configs/api_examples/perf_vllm_api_multiturn.py b/ais_bench/configs/api_examples/perf_vllm_api_multiturn.py new file mode 100644 index 00000000..e4fdd5e9 --- /dev/null +++ b/ais_bench/configs/api_examples/perf_vllm_api_multiturn.py @@ -0,0 +1,45 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets + +datasets = sharegpt_datasets + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr="vllm-multiturn-api-chat-stream", + path="", + model="", + stream=True, + request_rate=0, + retry=2, + api_key="", + host_ip="localhost", + host_port=8080, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + pred_postprocessor=dict(type=extract_non_reasoning_content), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/multi_turn_benchmark/' diff --git a/ais_bench/configs/api_examples/perf_vllm_api_rps_distribution.py b/ais_bench/configs/api_examples/perf_vllm_api_rps_distribution.py new file mode 100644 index 00000000..4c5e3db0 --- /dev/null +++ b/ais_bench/configs/api_examples/perf_vllm_api_rps_distribution.py @@ -0,0 +1,40 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPI + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import ( + synthetic_datasets, + ) + +datasets = synthetic_datasets + +models = [ + dict( + attr="service", + type=VLLMCustomAPI, + abbr='vllm-api-rps-distribution', + path="", + model="", + stream=True, + request_rate=100, + use_timestamp=False, + retry=2, + api_key="", + host_ip="localhost", + host_port=8080, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + traffic_cfg=dict( + burstiness=0.5, + ramp_up_strategy="linear", + ramp_up_start_rps=10, + ramp_up_end_rps=200, + ), + ) +] + +work_dir = 'outputs/rps_distribution_perf/' diff --git a/ais_bench/configs/api_examples/perf_vllm_api_stable_stage.py b/ais_bench/configs/api_examples/perf_vllm_api_stable_stage.py new file mode 100644 index 00000000..4701c701 --- /dev/null +++ b/ais_bench/configs/api_examples/perf_vllm_api_stable_stage.py @@ -0,0 +1,35 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPI + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import ( + synthetic_datasets, + ) + +datasets = synthetic_datasets + +models = [] +for rate in [0, 5, 10, 20]: + model_cfg = dict( + attr="service", + type=VLLMCustomAPI, + abbr=f'vllm-api-steady-rate-{rate}', + path="", + model="", + stream=True, + request_rate=rate, + use_timestamp=False, + retry=2, + api_key="", + host_ip="localhost", + host_port=8080, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + ) + models.append(model_cfg) + +work_dir = 'outputs/steady_state_perf/' diff --git a/ais_bench/configs/api_examples/perf_vllm_api_synthetic.py b/ais_bench/configs/api_examples/perf_vllm_api_synthetic.py new file mode 100644 index 00000000..1996719e --- /dev/null +++ b/ais_bench/configs/api_examples/perf_vllm_api_synthetic.py @@ -0,0 +1,48 @@ +from mmengine.config import read_base +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever +from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer +from ais_bench.benchmark.datasets import SyntheticDataset, MATHEvaluator, math_postprocess_v2 + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import ( + models as vllm_api_general_stream, + ) + +synthetic_config = { + "Type": "string", + "RequestCount": 100, + "TrustRemoteCode": False, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": 1, "MaxValue": 500} + }, + "Output": { + "Method": "gaussian", + "Params": {"Mean": 200, "Var": 100, "MinValue": 1, "MaxValue": 500} + } + }, +} + +datasets = [ + dict( + abbr='synthetic_custom', + type=SyntheticDataset, + config=synthetic_config, + reader_cfg=dict(input_columns=['question', 'max_out_len'], output_column='answer'), + infer_cfg=dict( + prompt_template=dict(type=PromptTemplate, template="{question}"), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + eval_cfg=dict( + evaluator=dict(type=MATHEvaluator, version='v2'), + pred_postprocessor=dict(type=math_postprocess_v2), + ), + ) +] + +models = vllm_api_general_stream +work_dir = 'outputs/synthetic_perf_custom/' diff --git a/ais_bench/configs/hf_example/infer_hf_base_model.py b/ais_bench/configs/hf_example/infer_hf_base_model.py index 5949ee1b..e77cb2b7 100644 --- a/ais_bench/configs/hf_example/infer_hf_base_model.py +++ b/ais_bench/configs/hf_example/infer_hf_base_model.py @@ -1,8 +1,8 @@ from mmengine.config import read_base from ais_bench.benchmark.models import HuggingFaceBaseModel from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask with read_base(): from ais_bench.benchmark.configs.summarizers.example import summarizer @@ -42,8 +42,8 @@ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/hf-base-model/' \ No newline at end of file diff --git a/ais_bench/configs/hf_example/infer_hf_chat_model.py b/ais_bench/configs/hf_example/infer_hf_chat_model.py index 4b676585..af42b5f5 100644 --- a/ais_bench/configs/hf_example/infer_hf_chat_model.py +++ b/ais_bench/configs/hf_example/infer_hf_chat_model.py @@ -1,8 +1,8 @@ from mmengine.config import read_base from ais_bench.benchmark.models import HuggingFacewithChatTemplate from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask with read_base(): from ais_bench.benchmark.configs.summarizers.example import summarizer @@ -42,8 +42,8 @@ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/hf-chat-model/' \ No newline at end of file diff --git a/ais_bench/configs/hf_example/infer_hf_multi_model_multi_dataset.py b/ais_bench/configs/hf_example/infer_hf_multi_model_multi_dataset.py new file mode 100644 index 00000000..dfed5320 --- /dev/null +++ b/ais_bench/configs/hf_example/infer_hf_multi_model_multi_dataset.py @@ -0,0 +1,46 @@ +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFaceBaseModel +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_chat_prompt import gsm8k_datasets as gsm8k_0_shot_cot_chat + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + +datasets = [*gsm8k_0_shot_cot_chat] + [*math500_gen_0_shot_cot_chat] + +models = [ + dict( + type=HuggingFaceBaseModel, + abbr='hf-base-model', + path='THUDM/chatglm-6b', + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.5, + top_k=10, + top_p=0.95, + do_sample=True, + seed=None, + repetition_penalty=1.03, + ), + max_out_len=100, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/hf-multi-model-multi-dataset/' diff --git a/ais_bench/configs/lmm_example/infer_lmm_multi_dataset.py b/ais_bench/configs/lmm_example/infer_lmm_multi_dataset.py new file mode 100644 index 00000000..511416d1 --- /dev/null +++ b/ais_bench/configs/lmm_example/infer_lmm_multi_dataset.py @@ -0,0 +1,12 @@ +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.textvqa.textvqa_gen_0_shot_str import textvqa_datasets + from ais_bench.benchmark.configs.datasets.docvqa.docvqa_gen_0_shot_str import docvqa_datasets + from ais_bench.benchmark.configs.models.lmm_models.lmm_vllm_api_chat import models as lmm_vllm_api_chat + +datasets = textvqa_datasets + docvqa_datasets +models = lmm_vllm_api_chat + +work_dir = 'outputs/lmm_multi_dataset/' diff --git a/ais_bench/configs/model_api_test_en.py b/ais_bench/configs/model_api_test_en.py new file mode 100644 index 00000000..c660e546 --- /dev/null +++ b/ais_bench/configs/model_api_test_en.py @@ -0,0 +1,36 @@ +from mmengine.config import read_base + +with read_base(): +# model tasks, choose one of them, other model tasks refer: https://ais-bench-benchmark-rf.readthedocs.io/en/latest/base_tutorials/all_params/models.html + # vllm_api_general is the base model, it only support text generation + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + # vllm_api_general_chat is the chat model, it support chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + # vllm_api_stream_chat is the stream chat model, it support stream chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + # vllm_api_general_stream is the stream model, it support stream generation + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + +# dataset task, get from https://ais-bench-benchmark-rf.readthedocs.io/en/latest/get_started/datasets.html + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = vllm_api_general_chat + +models[0]["path"] = "" # Specify the absolute path of the model serialized vocabulary file (generally not required for accuracy testing scenarios) +models[0]["model"] = "" # Specify the name of the model loaded on the server, configured according to the actual model name pulled by the VLLM inference service (configure as an empty string to get it automatically) +models[0]["request_rate"] = 0 # Request sending frequency: send 1 request to the server every 1/request_rate seconds; if less than 0.001, all requests are sent at once +models[0]["api_key"] = "" # Custom API key, default is an empty string +models[0]["host_ip"] = "localhost" # Specify the IP of the inference service +models[0]["host_port"] = 8080 # Specify the port of the inference service +models[0]["url"] = "" # Custom URL path for accessing the inference service (needs to be configured when the base URL is not a combination of http://host_ip:host_port; after configuration, host_ip and host_port will be ignored) +models[0]["max_out_len"] = 512 # Maximum number of tokens output by the inference service +models[0]["batch_size"] = 1 # Maximum concurrency for sending requests +models[0]["trust_remote_code"] = False # Whether the tokenizer trusts remote code, default is False; +models[0]["generation_kwargs"] = dict( # Model inference parameters, configured with reference to the VLLM documentation; the AISBench evaluation tool does not process them and attaches them to the sent request + temperature=0.01, + ignore_eos=False, +) + +# datasets[0]["path"] = ais_bench/datasets/gsm8k # Specify the absolute path of the dataset directory (required for accuracy testing scenarios) + +work_dir = 'outputs/default/' # Specify the working directory for saving task results and logs (default is outputs/default/) diff --git a/ais_bench/configs/model_api_test_zh_cn.py b/ais_bench/configs/model_api_test_zh_cn.py new file mode 100644 index 00000000..400c95c5 --- /dev/null +++ b/ais_bench/configs/model_api_test_zh_cn.py @@ -0,0 +1,36 @@ +from mmengine.config import read_base + +with read_base(): +# 模型任务，选择其中一个，其他模型任务参考：https://ais-bench-benchmark-rf.readthedocs.io/zh-cn/latest/base_tutorials/all_params/models.html 获取更多数据集任务 + # vllm_api_general 是基础模型，仅支持文本生成 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + # vllm_api_general_chat 是对话模型，支持对话 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + # vllm_api_stream_chat 是流式对话模型，支持流式对话 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + # vllm_api_general_stream 是流式模型，支持流式生成 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream +# 数据集任务，参考：https://ais-bench-benchmark-rf.readthedocs.io/zh-cn/latest/get_started/datasets.html 获取更多数据集任务 + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +# datasets = +models = vllm_api_general_chat + +models[0]["path"] = "" # 指定模型序列化词表文件的绝对路径（精度测试场景一般不需要配置） +models[0]["model"] = "" # 指定服务端加载的模型名称，根据 VLLM 推理服务实际拉取的模型名称配置（配置为空字符串则自动获取） +models[0]["request_rate"] = 0 # 请求发送频率：每 1/request_rate 秒向服务端发送 1 条请求；小于 0.001 时一次性发送所有请求 +models[0]["api_key"] = "" # 自定义 API key，默认为空字符串 +models[0]["host_ip"] = "localhost" # 指定推理服务的 IP +models[0]["host_port"] = 8080 # 指定推理服务的端口 +models[0]["url"] = "" # 自定义访问推理服务的 URL 路径（当基础 URL 不是 http://host_ip:host_port 的组合时需要配置；配置后 host_ip 和 host_port 将被忽略） +models[0]["max_out_len"] = 512 # 推理服务输出的最大 token 数 +models[0]["batch_size"] = 1 # 发送请求的最大并发数 +models[0]["trust_remote_code"] = False # tokenizer 是否信任远程代码，默认为 False +models[0]["generation_kwargs"] = dict( # 模型推理参数，参考 VLLM 文档配置；AISBench 评测工具不做处理，直接附加到发送的请求中 + temperature=0.01, + ignore_eos=False, +) + +# datasets[0]["path"] = ais_bench/datasets/gsm8k # 指定数据集目录的绝对路径（精度测试场景需要配置） + +work_dir = 'outputs/default/' # 指定任务结果和日志的保存工作目录（默认为 outputs/default/） diff --git a/ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py b/ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py new file mode 100644 index 00000000..e7635e0c --- /dev/null +++ b/ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py @@ -0,0 +1,26 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py b/ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py new file mode 100644 index 00000000..05c4c0e5 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py @@ -0,0 +1,79 @@ +import copy +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets as base_synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as base_vllm_api_stream_chat + +# 关键：统一收束 batch_size / request_rate / request_count / input_range / output_range 五个参数 +# 用户希望配几个任务，就在对应列表中追加几个元素（同一分组内的列表长度需保持一致） +# 注意：models 与 datasets 的列表长度需保持一致，二者会按下标一一配对，而非笛卡尔积 +tasks_params = { + "models": { + "batch_size": [1, 2, 4, 8, 16, 32], + "request_rate": [0, 0, 0, 0, 0, 0], + }, + "datasets": { + "request_count": [100, 100, 100, 100, 100, 100], + "input_range": [(1, 2), (2, 4), (4, 8), (8, 16), (16, 32), (32, 64)], + "output_range": [(1, 2), (2, 4), (4, 8), (8, 16), (16, 32), (32, 64)], + }, +} + +# 关键：通过 deepcopy 复制同一个基础模型配置，按 tasks_params["models"] 批量覆盖 batch_size / request_rate +models = [] +for idx, (batch_size, request_rate) in enumerate(zip(tasks_params["models"]["batch_size"], + tasks_params["models"]["request_rate"])): + model_cfg = copy.deepcopy(base_vllm_api_stream_chat[0]) + model_cfg["abbr"] = f"vllm-api-stream-chat-bs{batch_size}-rr{request_rate}" + model_cfg["host_ip"] = "localhost" + model_cfg["host_port"] = 8080 + model_cfg["max_out_len"] = 512 + model_cfg["batch_size"] = batch_size + model_cfg["request_rate"] = request_rate + # 关键：每个模型任务使用独立的 generation_kwargs + model_cfg["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + models.append(model_cfg) + +# 关键：按 tasks_params["datasets"] 批量构建合成数据集任务，名称按索引自动生成 +datasets = [] +for idx, (request_count, input_range, output_range) in enumerate( + zip(tasks_params["datasets"]["request_count"], + tasks_params["datasets"]["input_range"], + tasks_params["datasets"]["output_range"]) +): + ds = dict(base_synthetic_datasets[0]) + ds["abbr"] = f"synthetic-string-{idx}" + ds["config"] = { + "Type": "string", + "RequestCount": request_count, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": input_range[0], "MaxValue": input_range[1]}, + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": output_range[0], "MaxValue": output_range[1]}, + }, + }, + } + datasets.append(ds) + +# 关键：按索引一一配对 models[i] 与 datasets[i]，避免笛卡尔积 +# 例如 models[0](batch_size=1) 仅与 datasets[0](input_range=(1,2)) 配对，而非与所有数据集交叉组合 +model_dataset_combinations = [ + dict(models=[models[idx]], datasets=[datasets[idx]]) + for idx in range(min(len(models), len(datasets))) +] + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict(type=LocalRunner, task=dict(type=OpenICLApiInferTask)), +) \ No newline at end of file diff --git a/ais_bench/configs/performance_benchmark/multi_task_zh_cn.py b/ais_bench/configs/performance_benchmark/multi_task_zh_cn.py new file mode 100644 index 00000000..b3cbe739 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/multi_task_zh_cn.py @@ -0,0 +1,33 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_str import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_stream + vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[1]["host_ip"] = "localhost" +models[1]["host_port"] = 8081 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[1]["max_out_len"] = 512 +models[1]["batch_size"] = 1 + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py b/ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py new file mode 100644 index 00000000..23486797 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py @@ -0,0 +1,37 @@ +from mmengine.config import read_base +from ais_bench.benchmark.summarizers import DefaultPerfSummarizer +from ais_bench.benchmark.calculators import DefaultPerfMetricCalculator +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# 关键：自定义结果呈现任务中的 stats_list，调整要呈现的性能维度 +summarizer = dict( + attr="performance", + type=DefaultPerfSummarizer, + calculator=dict( + type=DefaultPerfMetricCalculator, + stats_list=["Average", "Min", "Max", "Median", "P75", "P90", "P95", "P99"], + ) +) + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/performance_fixed_request.py b/ais_bench/configs/performance_benchmark/performance_fixed_request.py new file mode 100644 index 00000000..20d58953 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_fixed_request.py @@ -0,0 +1,35 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Fixed Request Count Evaluation: +# Method 1 (basic): pass `--num-prompts N` on the command line to read only the first N samples. +# Method 2 (advanced): control the sampling range flexibly via `reader_cfg.test_range`, +# for example '[0:8]' reads the first 8 samples; '[10:20]' reads samples from index 10 to 20. +datasets[0]['reader_cfg']['test_range'] = '[0:8]' + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +# Fixed Request Count Performance Evaluation: +# Set request_rate to -1 to send requests concurrently without rate limiting (max throughput). +models[0]["request_rate"] = -1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/performance_multi_dataset.py b/ais_bench/configs/performance_benchmark/performance_multi_dataset.py new file mode 100644 index 00000000..3bd2e78e --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_multi_dataset.py @@ -0,0 +1,31 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Multi-Dataset Performance Evaluation: +# Specify multiple datasets and send them to the same service for performance evaluation. +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/performance_multi_model.py b/ais_bench/configs/performance_benchmark/performance_multi_model.py new file mode 100644 index 00000000..76e2a1e8 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_multi_model.py @@ -0,0 +1,31 @@ +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + +# Multi-Model Performance Evaluation: +# Evaluate multiple models on the same dataset simultaneously for performance comparison. +datasets = datasets + +# Rename the abbr of each model so that the results are distinguishable +vllm_api_stream_chat[0]["abbr"] = "vllm-qwen2.5-7b" +vllm_api_general_stream[0]["abbr"] = "vllm-qwen2.5-14b" + +vllm_api_stream_chat[0]["host_ip"] = "localhost" +vllm_api_stream_chat[0]["host_port"] = 8080 +vllm_api_stream_chat[0]["max_out_len"] = 1024 +vllm_api_stream_chat[0]["batch_size"] = 50 +vllm_api_stream_chat[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +vllm_api_general_stream[0]["host_ip"] = "localhost" +vllm_api_general_stream[0]["host_port"] = 8081 +vllm_api_general_stream[0]["max_out_len"] = 1024 +vllm_api_general_stream[0]["batch_size"] = 50 +vllm_api_general_stream[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +models = vllm_api_stream_chat + vllm_api_general_stream + +work_dir = "outputs/default/" diff --git a/ais_bench/configs/performance_benchmark/performance_multi_rate.py b/ais_bench/configs/performance_benchmark/performance_multi_rate.py new file mode 100644 index 00000000..edd736bf --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_multi_rate.py @@ -0,0 +1,27 @@ +import copy +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as base_vllm_api_stream_chat + +# Multi-Rate Performance Evaluation: +# Send the ShareGPT dataset to the service at request_rate=1, 2, 4, 8 (QPS) respectively. +# In AISBench, `request_rate` is a field of the model configuration, so build one model +# configuration per rate via `copy.deepcopy` and combine them with a single dataset. +datasets = datasets + +models = [] +for rate in [1, 2, 4, 8]: + model_cfg = copy.deepcopy(base_vllm_api_stream_chat[0]) + model_cfg["abbr"] = f"vllm-api-stream-chat-rate-{rate}" + model_cfg["host_ip"] = "localhost" + model_cfg["host_port"] = 8080 + model_cfg["max_out_len"] = 1024 + model_cfg["batch_size"] = 50 + model_cfg["request_rate"] = rate + model_cfg["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + models.append(model_cfg) + +work_dir = "outputs/default/" diff --git a/ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py b/ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py new file mode 100644 index 00000000..2ac288de --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py @@ -0,0 +1,20 @@ +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Single-Task Performance Evaluation: +# Send the ShareGPT dataset to the service at request_rate=1 (QPS) for performance evaluation. +datasets = datasets + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 1024 +models[0]["batch_size"] = 50 +models[0]["request_rate"] = 1 # Request sending frequency: send 1 request to the server every 1/request_rate seconds; if less than 0.001, all requests are sent at once +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) # When testing performance and needing to limit the output length, ignore_eos must be set to True + +work_dir = "outputs/default/" diff --git a/ais_bench/configs/performance_benchmark/performance_re_eval.py b/ais_bench/configs/performance_benchmark/performance_re_eval.py new file mode 100644 index 00000000..f36df4c7 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_re_eval.py @@ -0,0 +1,40 @@ +from mmengine.config import read_base +from ais_bench.benchmark.summarizers import DefaultPerfSummarizer +from ais_bench.benchmark.calculators import DefaultPerfMetricCalculator +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Performance Result Recalculation: +# Customize the `stats_list` of the result summarizer to adjust the statistical dimensions +# of the performance summary, then recalculate the summary with `--mode perf --reuse` without +# re-running the inference. +summarizer = dict( + attr="performance", + type=DefaultPerfSummarizer, + calculator=dict( + type=DefaultPerfMetricCalculator, + stats_list=["Average", "Min", "Max", "Median", "P75", "P90", "P95", "P99"], + ), +) + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/performance_seq_combinations.py b/ais_bench/configs/performance_benchmark/performance_seq_combinations.py new file mode 100644 index 00000000..9bd34b21 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_seq_combinations.py @@ -0,0 +1,57 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets as base_synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Custom Sequence Multi-Task Combinations: +# Build multiple synthetic sub-datasets with different input/output lengths, then use +# `model_dataset_combinations` to precisely pair models with partial datasets. +datasets = [] +for input_len in [256, 512]: + for output_len in [256, 512]: + ds = dict(base_synthetic_datasets[0]) + ds["abbr"] = f"syn_in{input_len}_out{output_len}" + ds["config"] = { + "Type": "string", + "RequestCount": 100, + "TrustRemoteCode": False, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": input_len, "MaxValue": input_len}, + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": output_len, "MaxValue": output_len}, + }, + }, + } + datasets.append(ds) + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +# Key: Only specify partial models for partial datasets +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0], datasets[1]]), + dict(models=[models[0]], datasets=[datasets[2]]), +] + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/performance_synthetic.py b/ais_bench/configs/performance_benchmark/performance_synthetic.py new file mode 100644 index 00000000..9f2b5d15 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/performance_synthetic.py @@ -0,0 +1,41 @@ +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets as base_synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Synthetic Dataset Multi-Task Combinations: +# Define multiple synthetic sub-datasets with different input/output lengths via the +# `config` field of `SyntheticDataset`, and combine them with the same model. +datasets = [] +for input_len in [256, 512, 1024]: + for output_len in [256, 512]: + ds = dict(base_synthetic_datasets[0]) + ds["abbr"] = f"syn_in{input_len}_out{output_len}" + ds["config"] = { + "Type": "string", + "RequestCount": 100, + "TrustRemoteCode": False, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": input_len, "MaxValue": input_len}, + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": output_len, "MaxValue": output_len}, + }, + }, + } + datasets.append(ds) + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["request_rate"] = 2 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" diff --git a/ais_bench/configs/performance_benchmark/single_task_zh_cn.py b/ais_bench/configs/performance_benchmark/single_task_zh_cn.py new file mode 100644 index 00000000..e7635e0c --- /dev/null +++ b/ais_bench/configs/performance_benchmark/single_task_zh_cn.py @@ -0,0 +1,26 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py b/ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py new file mode 100644 index 00000000..d130c787 --- /dev/null +++ b/ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py @@ -0,0 +1,49 @@ +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# 关键：自定义输入输出分布（可通过修改synthetic_config调整） +synthetic_config = { + "Type": "string", + "RequestCount": 1000, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": 50, "MaxValue": 500} + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": 20, "MaxValue": 200} + } + } +} + +datasets = [] +for ds in synthetic_datasets: + ds = dict(ds) + ds["config"] = synthetic_config + datasets.append(ds) + +models = vllm_api_stream_chat +# 关键：性能测试时需将 ignore_eos 设置为 True 以确保达到最大输出长度 +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + task=dict(type=OpenICLApiInferTask), + ), +) diff --git a/docs/requirements.txt b/docs/requirements.txt index 587d3c6c..b9ecd672 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,4 +3,5 @@ sphinx-rtd-theme sphinx-intl m2r2 linkify-it-py -myst_parser \ No newline at end of file +myst_parser +sphinx_design \ No newline at end of file diff --git a/docs/source_en/advanced_tutorials/custom_dataset.md b/docs/source_en/advanced_tutorials/custom_dataset.md index 3e420093..c4059fe0 100644 --- a/docs/source_en/advanced_tutorials/custom_dataset.md +++ b/docs/source_en/advanced_tutorials/custom_dataset.md @@ -114,10 +114,10 @@ This method currently only supports **accuracy evaluation scenarios**. Other par #### Example Command ```shell # Use vllm API -ais_bench ais_bench/configs/api_examples/infer_api_vllm_general.py +ais_bench ais_bench/configs/api_examples/infer_vllm_api_general.py # Use mindie API -ais_bench ais_bench/configs/api_examples/infer_api_mindie_stream_general.py +ais_bench ais_bench/configs/api_examples/infer_mindie_stream_api_general.py ``` @@ -133,6 +133,9 @@ datasets = [ ``` +> 💡 The above config file method is essentially a simplified application of the [Custom Config File Method](run_custom_config.md). For more complex scenarios (such as multi-model/multi-dataset combinations, custom model parameters, judge models, etc.), refer to the "Custom Dataset Evaluation" example in [Running AISBench with Custom Config Files](run_custom_config.md#custom-configuration-file-examples-for-each-scenario). + + ### Guide to Using Dataset Supplementary Info (`.meta.json`) This feature currently only supports **performance evaluation scenarios**. The `ais_bench` system will automatically attempt to parse the input dataset file, so in most cases, a `.meta.json` file is **not required**. However, if the original dataset does not specify `max_tokens`, or if you need to configure data sampling, you must define these settings in a `.meta.json` file. diff --git a/docs/source_en/advanced_tutorials/judge_model_evaluate.md b/docs/source_en/advanced_tutorials/judge_model_evaluate.md index 9d383fab..7f28aca0 100644 --- a/docs/source_en/advanced_tutorials/judge_model_evaluate.md +++ b/docs/source_en/advanced_tutorials/judge_model_evaluate.md @@ -222,6 +222,10 @@ The result display example is as follows: From the quick start section of the judge model, you can see that except for the additional need to modify the judge model configuration in the data configuration file, the other evaluation execution methods are exactly the same as the conventional evaluation execution methods. Therefore, the execution methods for other accuracy evaluation function scenarios are also exactly the same. +## Implement via Custom Config Files + +> 💡 The above judge model evaluation scenario can also be implemented through the [Custom Config File Method](run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax such as loops, conditional judgments, list comprehensions, etc. You can write the tested model, judge model, dataset, summarizer, and other configurations into a single file, write once and reuse multiple times. See the "Judge Model Evaluation" example in [Running AISBench with Custom Config Files](run_custom_config.md#custom-configuration-file-examples-for-each-scenario). + ### Multi-task Evaluation Refer to [Accuracy Evaluation Scenario Multi-task Evaluation](../base_tutorials/scenes_intro/accuracy_benchmark.md#multi-task-evaluation) @@ -232,13 +236,13 @@ Refer to [Accuracy Evaluation Scenario Multi-task Parallel Evaluation](../base_t ### Interrupted Evaluation & Failed Case Re-evaluation -Refer to [Accuracy Evaluation Scenario Interrupted Evaluation & Failed Case Re-evaluation](../base_tutorials/scenes_intro/accuracy_benchmark.md#interrupted-evaluation-failed-case-re-evaluation) +Refer to [Accuracy Evaluation Scenario Interrupted Evaluation & Failed Case Re-evaluation](../base_tutorials/scenes_intro/accuracy_benchmark.md#resumption-after-interruption--retesting-of-failed-cases) > ⚠️ Note: After `--reuse` re-completes the tested model inference results, the judge model will re-evaluate all complete inference results from scratch, and previously judged results will not be used. ### Merged Sub-dataset Inference -Refer to [Accuracy Evaluation Scenario Merged Sub-dataset Inference](../base_tutorials/scenes_intro/accuracy_benchmark.md#merged-sub-dataset-inference) +Refer to [Accuracy Evaluation Scenario Merged Sub-dataset Inference](../base_tutorials/scenes_intro/accuracy_benchmark.md#merging-sub-dataset-inference) ### Fixed Request Count Evaluation @@ -246,13 +250,13 @@ Refer to [Accuracy Evaluation Scenario Fixed Request Count Evaluation](../base_t ### Multiple Independent Repetitions Inference -Refer to [Accuracy Evaluation Scenario Multiple Independent Repetitions Inference](../base_tutorials/scenes_intro/accuracy_benchmark.md#multiple-independent-repetitions-inference) +Refer to [Accuracy Evaluation Scenario Multiple Independent Repetitions Inference](../base_tutorials/scenes_intro/accuracy_benchmark.md#multiple-independent-repeat-inference) > ⚠️ This scenario requires attention: only need to configure the parameters for multiple independent repetitions inference in the tested model, no need to configure in the judge model configuration. ### Inference Results Re-evaluation -Refer to [Accuracy Evaluation Scenario Inference Results Re-evaluation](../base_tutorials/scenes_intro/accuracy_benchmark.md#inference-results-re-evaluation) +Refer to [Accuracy Evaluation Scenario Inference Results Re-evaluation](../base_tutorials/scenes_intro/accuracy_benchmark.md#re-evaluation-of-inference-results) > ⚠️ This scenario requires attention that re-evaluation starts from the judge model inference diff --git a/docs/source_en/advanced_tutorials/multimodal_benchmark.md b/docs/source_en/advanced_tutorials/multimodal_benchmark.md index a4fca202..2a87980d 100644 --- a/docs/source_en/advanced_tutorials/multimodal_benchmark.md +++ b/docs/source_en/advanced_tutorials/multimodal_benchmark.md @@ -28,6 +28,9 @@ Supported Model backend ## Quick Start + +> 💡 The multimodal evaluation scenario can also be implemented through the [Custom Config File Method](run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax such as loops, conditional judgments, list comprehensions, etc. You can write multimodal models, multimodal datasets, summarizer, and other configurations into a single file, write once and reuse multiple times. See [Running AISBench with Custom Config Files](run_custom_config.md). + ### Multimodal input format There are various formats for service-oriented multimodal data input. Taking image + text input as an example, it is as follows: - Method 1: Local file format, default method diff --git a/docs/source_en/advanced_tutorials/multiturn_benchmark.md b/docs/source_en/advanced_tutorials/multiturn_benchmark.md index bbca822d..a8793129 100644 --- a/docs/source_en/advanced_tutorials/multiturn_benchmark.md +++ b/docs/source_en/advanced_tutorials/multiturn_benchmark.md @@ -172,6 +172,11 @@ After executing the AISBench command, detailed task execution data is saved to a This log indicates that detailed task data is stored in `outputs/default/20250628_151326` (relative to the directory where the command was executed). +## Implement via Custom Config Files + +> 💡 The above multi-turn dialogue performance evaluation scenario can also be implemented through the [Custom Config File Method](run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax such as loops, conditional judgments, list comprehensions, etc. You can write models, datasets, summarizer, and other configurations into a single file, write once and reuse multiple times. See the "Multi-Turn Dialogue Performance Evaluation" example in [Running AISBench with Custom Config Files](run_custom_config.md#custom-configuration-file-examples-for-each-scenario). + +### Viewing Detailed Performance Data ```shell 20250628_151326 # Unique directory generated for each experiment based on timestamp ├── configs # Auto-saved dump of all configuration files @@ -181,7 +186,7 @@ This log indicates that detailed task data is stored in `outputs/default/2025062 └── vllm-api-chat-stream/ # Name of the "service-based model configuration" (corresponds to the `abbr` parameter in the model task configuration file) ├── sharegptdataset.csv # Per-request performance output (CSV), matching the "Performance Parameters" table in the printed results ├── sharegptdataset.json # End-to-end performance output (JSON), matching the "Common Metric" table in the printed results - ├── sharegptdataset_details.h5 # Full打点 ITL data (Inter-Token Latency) + ├── sharegptdataset_details.h5 # Full-granularity ITL data (Inter-Token Latency) ├── sharegptdataset_details.json # Full detailed metrics └── sharegptdataset_plot.html # Request concurrency visualization report (HTML) ``` diff --git a/docs/source_en/advanced_tutorials/rps_distribution.md b/docs/source_en/advanced_tutorials/rps_distribution.md index 51d5d474..83ee8506 100644 --- a/docs/source_en/advanced_tutorials/rps_distribution.md +++ b/docs/source_en/advanced_tutorials/rps_distribution.md @@ -324,6 +324,9 @@ $\lambda_i = \lambda_{\text{start}} \times \left(\frac{\lambda_{end}}{\lambda_{s 4. **In stress testing scenarios, the frequency of connection creation is controlled, but not the request sending rate (after each connection is created, requests are sent and responses are processed continuously without interruption)**. 5. **In multi-turn dialogue scenarios, only the request distribution of the first turn is valid**. +## Implement via Custom Config Files + +> 💡 The above RPS distribution control parameters (`traffic_cfg`) are also applicable in the [Custom Config File Method](run_custom_config.md). You only need to add the `traffic_cfg` field in the model configuration dict. The configuration file is essentially a Python script that supports all Python syntax such as loops, conditional judgments, list comprehensions, etc. You can write models, datasets, summarizer, and other configurations into a single file, write once and reuse multiple times. See [Running AISBench with Custom Config Files](run_custom_config.md). --- diff --git a/docs/source_en/advanced_tutorials/run_custom_config.md b/docs/source_en/advanced_tutorials/run_custom_config.md index be417747..396f8585 100644 --- a/docs/source_en/advanced_tutorials/run_custom_config.md +++ b/docs/source_en/advanced_tutorials/run_custom_config.md @@ -1,6 +1,204 @@ # Running AISBench with a Custom Configuration File The standard command invocation method for AISBench specifies the model task via `--models`, the dataset task via `--datasets`, and the result presentation task via `--summarizer` to run an evaluation task. Additionally, AISBench supports specifying a **custom configuration file** that combines the configuration information of these three types of tasks, enabling the execution of custom task combinations. +## Why Use a Custom Configuration File +AISBench provides two ways to run tasks: **Command-Line Interface (CLI)** and **custom configuration file**. In actual use, it is recommended to prioritize the custom configuration file approach, for the following reasons: + +| Comparison Dimension | CLI Approach | Configuration File Approach | +| --- | --- | --- | +| **Reusability** | The complete command must be re-entered for each run | Configuration files can be saved, version-managed, and reused repeatedly | +| **Expressiveness** | Only model/dataset names can be specified via parameters | Allows precise control over all details including model parameters, dataset sampling range, and inference configuration | +| **Combination Flexibility** | Only Cartesian product combinations are supported | Supports `model_dataset_combinations` for arbitrary custom model-dataset pairings | +| **Parameter Override** | Internal parameters of preset models/datasets cannot be modified | Any field such as `abbr`, `test_range`, `host_ip`, `host_port` can be modified directly | +| **Batch Execution** | Requires running the command multiple times | A single configuration file can run multiple model and dataset combinations at once | +| **Team Collaboration** | Commands are hard to share and trace | Configuration files are code and can be committed to a repository for review and reuse | + +**Summary**: The CLI approach is suitable for quick validation, while the configuration file approach is suitable for formal, reproducible, and complex evaluation scenarios. + +## Configuration Files Are Python Scripts +The AISBench custom configuration file is essentially a Python script. This means you can use all Python syntax features in the configuration file to flexibly construct evaluation tasks. + +### Using `for` Loop to Batch Build Model Configurations + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + +datasets = gsm8k_0_shot_cot_str + +models = [] +for port in [8080, 8081, 8082]: + models.append( + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr=f'vllm-api-chat-port-{port}', + path="", + model="", + request_rate=0, + retry=2, + host_ip="localhost", + host_port=port, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.5, top_k=10, top_p=0.95), + ) + ) + +work_dir = 'outputs/multi_port_benchmark/' +``` + +### Using List Comprehension to Batch Add Datasets + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat +datasets = [ + dict(d, abbr=f'my_{d["abbr"]}', reader_cfg=dict(d.get('reader_cfg', {}), test_range='[0:100]')) + for d in datasets +] + +models = vllm_api_general_chat +work_dir = 'outputs/my_benchmark/' +``` + +### Conditional Configuration: Switch Based on Environment Variables + +```python +import os +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat, VLLMCustomAPI + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + +datasets = gsm8k_0_shot_cot_str + +use_stream = os.environ.get('USE_STREAM', 'false').lower() == 'true' +model_type = VLLMCustomAPIChat if use_stream else VLLMCustomAPI + +models = [ + dict( + attr="service", + type=model_type, + abbr='vllm-api-conditional', + path="", + model="", + stream=use_stream, + request_rate=0, + retry=2, + host_ip=os.environ.get('HOST_IP', 'localhost'), + host_port=int(os.environ.get('HOST_PORT', '8080')), + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.5, top_k=10, top_p=0.95), + ) +] + +work_dir = 'outputs/conditional_benchmark/' +``` + +### Using `.copy()` to Reuse and Modify Model Configurations + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_0_shot_cot_str + +model_high_temp = vllm_api_general_chat.copy() +model_high_temp[0]['abbr'] = vllm_api_general_chat[0]['abbr'] + '-high-temp' +model_high_temp[0]['generation_kwargs']['temperature'] = 0.9 + +model_low_temp = vllm_api_general_chat.copy() +model_low_temp[0]['abbr'] = vllm_api_general_chat[0]['abbr'] + '-low-temp' +model_low_temp[0]['generation_kwargs']['temperature'] = 0.1 + +models = model_high_temp + model_low_temp +work_dir = 'outputs/temperature_comparison/' +``` + +## Complete Configuration File Variable Reference +The following top-level variables can be defined in the custom configuration file. All variables are optional, but at least `models` and `datasets` must be defined to run an inference task. + +| Variable Name | Type | Required | Description | +| --- | --- | --- | --- | +| `models` | `list[dict]` | Yes (for inference) | List of model configurations. Each element is a dict that must at least include `type` (model class) and `abbr` (unique identifier) fields. Service-oriented models additionally require `attr="service"`, `host_ip`, `host_port`, etc.; local models additionally require `path`, `tokenizer_path`, etc. | +| `datasets` | `list[dict]` | Yes (for inference) | List of dataset configurations. Each element is a dict that must at least include `type` (dataset class), `abbr` (unique identifier), `reader_cfg`, `infer_cfg`, and `eval_cfg` fields | +| `summarizer` | `dict` | No | Result summarizer configuration. Usually imported from `ais_bench.benchmark.configs.summarizers.example`. Contains `attr` and `summary_groups` fields | +| `model_dataset_combinations` | `list[dict]` | No | List of custom model-dataset pairings. Each element is `dict(models=[...], datasets=[...])`. When not specified, the Cartesian product of `models` and `datasets` is used by default | +| `work_dir` | `str` | No | Working directory; inference results and logs will be output to this directory. Defaults to `outputs/default/` | +| `infer` | `dict` | No | Inference process configuration. Contains `partitioner` (partitioner) and `runner` (runner, with `max_num_workers` and `task` inside). Uses the default inference process when not specified | +| `eval` | `dict` | No | Evaluation process configuration. Same structure as `infer`. Only used when an independent evaluation phase is needed (e.g., SWE-Bench, VBench scenarios) | + +### Detailed `models` Field Description + +Common fields for each model configuration dict: + +| Field | Type | Description | +| --- | --- | --- | +| `type` | class | Model class, such as `VLLMCustomAPIChat`, `VLLMCustomAPI`, `HuggingFaceBaseModel`, `HuggingFacewithChatTemplate`, etc. | +| `abbr` | `str` | Unique identifier of the model, used as the column name in the result table. Model-dataset combinations with the same `abbr` in the same configuration file will be treated as duplicate tasks and skipped | +| `attr` | `str` | Model attribute; `"service"` for service-oriented models, `"local"` for local models | +| `path` | `str` | Model path (required for local models; can be an empty string for service-oriented models) | +| `model` | `str` | Model name specified for service-oriented inference | +| `host_ip` | `str` | IP address of the inference service (for service-oriented models) | +| `host_port` | `int` | Port of the inference service (for service-oriented models) | +| `stream` | `bool` | Whether to use streaming inference | +| `max_out_len` | `int` | Maximum output token count | +| `batch_size` | `int` | Inference batch size | +| `max_seq_len` | `int` | Maximum input sequence length | +| `request_rate` | `int` | Request rate limit; 0 means unlimited | +| `retry` | `int` | Number of retries for failed requests | +| `generation_kwargs` | `dict` | Generation parameters, such as `temperature`, `top_k`, `top_p`, `seed`, etc. | +| `tokenizer_path` | `str` | Tokenizer path (for local models) | +| `model_kwargs` | `dict` | Model loading parameters (for local models), such as `device_map` | +| `tokenizer_kwargs` | `dict` | Tokenizer parameters (for local models), such as `padding_side` | +| `run_cfg` | `dict` | Multi-GPU/multi-machine run configuration (for local models), such as `dict(num_gpus=1, num_procs=1)` | +| `pred_postprocessor` | `dict` | Model output post-processor, such as `dict(type=extract_non_reasoning_content)` | + +### Detailed `datasets` Field Description + +Common fields for each dataset configuration dict: + +| Field | Type | Description | +| --- | --- | --- | +| `type` | class | Dataset class, such as `GSM8KDataset`, `MATHDataset`, `SyntheticDataset`, etc. | +| `abbr` | `str` | Unique identifier of the dataset, used as the row name in the result table | +| `path` | `str` | Dataset file path | +| `reader_cfg` | `dict` | Reader configuration, containing `input_columns`, `output_column`, and optional `test_range` to control the sampling range (e.g., `'[0:100]'`) | +| `infer_cfg` | `dict` | Inference configuration, containing `prompt_template`, `retriever`, `inferencer` | +| `eval_cfg` | `dict` | Evaluation configuration, containing `evaluator` and optional `pred_postprocessor` | +| `judge_infer_cfg` | `dict` | Judge model inference configuration (for datasets requiring LLM Judge), containing `judge_model`, `judge_dataset_type`, `prompt_template`, `retriever`, `inferencer` | + +### Detailed `infer` Field Description + +```python +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) +``` ## Usage Instructions ```bash @@ -10,6 +208,459 @@ ais_bench ais_bench/configs/api_examples/infer_vllm_api_general.py ``` +## Custom Configuration File Examples for Each Scenario + +### 1. Service-Oriented Accuracy Evaluation +Access the inference service via API and perform accuracy evaluation using real datasets. Applicable to service-oriented deployment scenarios such as vLLM, MindIE, TGI, Triton, etc. + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_chat_prompt import gsm8k_datasets as gsm8k_0_shot_cot_chat + +datasets = [*gsm8k_0_shot_cot_chat] + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-general-chat', + model="", + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict( + temperature=0.5, + top_k=10, + top_p=0.95, + seed=None, + repetition_penalty=1.03, + ) + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/api-vllm-general-chat/' +``` + +### 2. Pure Model Accuracy Evaluation +Use a HuggingFace local model for direct inference and evaluation without deploying a service. + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFaceBaseModel +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_chat_prompt import gsm8k_datasets as gsm8k_0_shot_cot_chat + +datasets = [*gsm8k_0_shot_cot_chat] + +models = [ + dict( + type=HuggingFaceBaseModel, + abbr='hf-base-model', + path='THUDM/chatglm-6b', + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.5, + top_k=10, + top_p=0.95, + do_sample=True, + seed=None, + repetition_penalty=1.03, + ), + max_out_len=100, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/hf-base-model/' +``` + +### 3. Service-Oriented Performance Evaluation +Use a synthetic dataset to perform performance stress testing on the inference service, outputting metrics such as TTFT (Time To First Token), TPOT (Time Per Output Token), and E2EL (End-to-End Latency). + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import ( + synthetic_datasets, + ) + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import ( + models as vllm_api_general_stream, + ) + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import ( + models as vllm_api_stream_chat, + ) + +datasets = synthetic_datasets + +vllm_api_general_stream[0]["abbr"] = "demo-" + vllm_api_general_stream[0]["abbr"] +vllm_api_stream_chat[0]["abbr"] = "demo-" + vllm_api_stream_chat[0]["abbr"] + +models = vllm_api_general_stream + vllm_api_stream_chat + +work_dir = "outputs/demo_api-vllm-stream-perf/" +``` + +Run command: + +```bash +ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py -m perf +``` + +### 4. Synthetic Dataset Performance Evaluation +Customize the parameters of the synthetic dataset to control the number of requests and the input/output token length distribution. + +```python +from mmengine.config import read_base +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever +from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer +from ais_bench.benchmark.datasets import SyntheticDataset, MATHEvaluator, math_postprocess_v2 + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import ( + models as vllm_api_general_stream, + ) + +synthetic_config = { + "Type": "string", + "RequestCount": 100, + "TrustRemoteCode": False, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": 1, "MaxValue": 500} + }, + "Output": { + "Method": "gaussian", + "Params": {"Mean": 200, "Var": 100, "MinValue": 1, "MaxValue": 500} + } + }, +} + +datasets = [ + dict( + abbr='synthetic_custom', + type=SyntheticDataset, + config=synthetic_config, + reader_cfg=dict(input_columns=['question', 'max_out_len'], output_column='answer'), + infer_cfg=dict( + prompt_template=dict(type=PromptTemplate, template="{question}"), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + eval_cfg=dict( + evaluator=dict(type=MATHEvaluator, version='v2'), + pred_postprocessor=dict(type=math_postprocess_v2), + ), + ) +] + +models = vllm_api_general_stream +work_dir = 'outputs/synthetic_perf_custom/' +``` + +### 5. Multi-Model Multi-Dataset Combinations +Simultaneously evaluate the performance of multiple models on multiple datasets, automatically combined via the Cartesian product. + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_5_shot_str import mmlu_datasets as mmlu_5_shot_str + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat + mmlu_5_shot_str +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat + +work_dir = 'outputs/multi_model_multi_dataset/' +``` + +### 6. Custom Model-Dataset Pairings +Precisely control which models are paired with which datasets via `model_dataset_combinations` to avoid unnecessary Cartesian products. + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat + +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), + dict(models=[models[2]], datasets=[datasets[0], datasets[1]]), +] + +work_dir = 'outputs/custom_combinations/' +``` + +### 7. Judge Model Evaluation +For datasets that require LLM Judge evaluation (e.g., AIME 2025), configure the judge model in the dataset's `judge_infer_cfg`. + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.aime2025.aime2025_gen_0_shot_llmjudge import aime2025_datasets + +datasets = aime2025_datasets + +datasets[0]['judge_infer_cfg']['judge_model']['host_ip'] = 'localhost' +datasets[0]['judge_infer_cfg']['judge_model']['host_port'] = 8081 + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-judge-eval', + path="", + model="", + stream=True, + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + pred_postprocessor=dict(type=extract_non_reasoning_content), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/judge_eval/' +``` + +### 8. Steady-State Performance Evaluation +Simulate performance under steady-state load by controlling the `request_rate` parameter and `stream` parameter. + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPI + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import ( + synthetic_datasets, + ) + +datasets = synthetic_datasets + +models = [] +for rate in [0, 5, 10, 20]: + model_cfg = dict( + attr="service", + type=VLLMCustomAPI, + abbr=f'vllm-api-steady-rate-{rate}', + path="", + model="", + stream=True, + request_rate=rate, + use_timestamp=False, + retry=2, + api_key="", + host_ip="localhost", + host_port=8080, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + ) + models.append(model_cfg) + +work_dir = 'outputs/steady_state_perf/' +``` + +### 9. Multi-Turn Dialogue Performance Evaluation +Use the ShareGPT or MTBench multi-turn dialogue datasets for performance evaluation. + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets + +datasets = sharegpt_datasets + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr="vllm-multiturn-api-chat-stream", + path="", + model="", + stream=True, + request_rate=0, + retry=2, + api_key="", + host_ip="localhost", + host_port=8080, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + pred_postprocessor=dict(type=extract_non_reasoning_content), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/multi_turn_benchmark/' +``` + +### 10. Custom Dataset Evaluation +When you need to use your own dataset for evaluation, you can do so by customizing the dataset configuration. + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever +from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer +from ais_bench.benchmark.datasets import CustomDataset +from ais_bench.benchmark.openicl.icl_evaluator import AccEvaluator + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + +datasets = [ + dict( + abbr='my_custom_dataset', + type=CustomDataset, + path='/path/to/your/dataset.jsonl', + reader_cfg=dict( + input_columns=['question'], + output_column='answer', + ), + infer_cfg=dict( + prompt_template=dict( + type=PromptTemplate, + template='{question}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + eval_cfg=dict( + evaluator=dict(type=AccEvaluator), + pred_role='BOT', + ), + meta_path='', + ) +] + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-custom-dataset', + model="", + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.5, top_k=10, top_p=0.95), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/custom_dataset/' +``` + + ## Example of Using a Custom Configuration File for Accuracy Evaluation ### Editing the Example Content The following example demonstrates how to evaluate the performance of two service interfaces ([`v1/chat/completions`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py) and [`v1/completions`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general.py)) on the [GSM8K](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/README_en.md) and [MATH datasets](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/math/README_en.md). Refer to the sample file: [demo_infer_vllm_api.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/demo_infer_vllm_api.py): @@ -17,8 +668,8 @@ The following example demonstrates how to evaluate the performance of two servic ```python from mmengine.config import read_base from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask from ais_bench.benchmark.models import VLLMCustomAPIChat with read_base(): @@ -27,16 +678,13 @@ with read_base(): from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general -# Use only a subset of samples for demo testing gsm8k_0_shot_cot_str[0]['abbr'] = 'demo_' + gsm8k_0_shot_cot_str[0]['abbr'] gsm8k_0_shot_cot_str[0]['reader_cfg']['test_range'] = '[0:8]' math500_gen_0_shot_cot_chat[0]['abbr'] = 'demo_' + math500_gen_0_shot_cot_chat[0]['abbr'] math500_gen_0_shot_cot_chat[0]['reader_cfg']['test_range'] = '[0:8]' -# Specify the dataset list; add different dataset configurations by concatenation datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat -# Specify the model configuration list models = [ dict( attr="service", @@ -46,8 +694,8 @@ models = [ model="", request_rate = 0, retry = 2, - host_ip = "localhost", # Specify the IP address of the inference service - host_port = 8080, # Specify the port of the inference service + host_ip = "localhost", + host_port = 8080, max_out_len = 512, batch_size=1, generation_kwargs = dict( @@ -67,12 +715,12 @@ work_dir = 'outputs/demo_api-vllm-general-chat/' ### Executing the Custom Task Combination After modifying the configuration file, run the following command to start the accuracy evaluation: ```bash -ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_general_chat.py +ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api.py ``` If you need to execute multiple tasks in parallel, you can add the [`--max-num-workers`](../base_tutorials/all_params/cli_args.md#common-parameters) parameter to the command line to specify the maximum number of parallel tasks. Example: ```bash -ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_general_chat.py --max-num-workers 4 +ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api.py --max-num-workers 4 ``` @@ -103,12 +751,12 @@ with read_base(): models as vllm_api_stream_chat, ) -datasets = synthetic_datasets # Specify the dataset list +datasets = synthetic_datasets vllm_api_general_stream[0]["abbr"] = "demo-" + vllm_api_general_stream[0]["abbr"] vllm_api_stream_chat[0]["abbr"] = "demo-" + vllm_api_stream_chat[0]["abbr"] -models = vllm_api_general_stream + vllm_api_stream_chat # Specify the model list +models = vllm_api_general_stream + vllm_api_stream_chat work_dir = "outputs/demo_api-vllm-stream-perf/" ``` @@ -126,7 +774,7 @@ ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py -m perf --m ### Output Results ```bash -[2025-12-05 12:10:44,147] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-general-stream/syntheticdataset]: +[2025-12-05 12:10:44,147] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-general-stream/syntheticdataset]: ╒══════════════════════════╤═════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════╕ │ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ ╞══════════════════════════╪═════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════╡ @@ -135,7 +783,7 @@ ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py -m perf --m │ TTFT │ total │ 103.5 ms │ 102.4 ms │ 107.0 ms │ 103.1 ms │ 103.3 ms │ 104.2 ms │ 106.8 ms │ 10 │ ... [2025-12-05 12:10:44,149] [ais_bench] [INFO] Performance Result files located in outputs/demo_api-vllm-general-stream-chat-perf/20251205_121020/performances/demo-vllm-api-general-stream-chat. -[2025-12-05 12:10:44,149] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-stream-chat/syntheticdataset]: +[2025-12-05 12:10:44,149] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-stream-chat/syntheticdataset]: ╒══════════════════════════╤═════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤════════════════╤═════════════════╤═════════════════╤═════╕ │ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ ╞══════════════════════════╪═════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪════════════════╪═════════════════╪═════════════════╪═════╡ @@ -157,12 +805,12 @@ with read_base(): from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat -models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat model_dataset_combinations = [ - dict(models=[models[0]], datasets=[datasets[0]]), # Combination 1: Use model 0 (vllm_api_general) with dataset 0 (gsm8k_0_shot_cot_str) - dict(models=[models[1]], datasets=[datasets[1]]), # Combination 2: Use model 1 (vllm_api_general_chat) with dataset 1 (math500_gen_0_shot_cot_chat) - dict(models=[models[2]], datasets=[datasets[0], datasets[1]]), # Combination 3: Use model 2 (vllm_api_stream_chat) with dataset 0 (gsm8k_0_shot_cot_str) and dataset 1 (math500_gen_0_shot_cot_chat) + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), + dict(models=[models[2]], datasets=[datasets[0], datasets[1]]), ... ] ``` @@ -180,8 +828,8 @@ vllm_api_general_copy[0]['port'] = 8081 models = vllm_api_general_copy + vllm_api_general datasets = math500_gen_0_shot_cot_chat model_dataset_combinations = [ - dict(models=[models[1]], datasets=datasets), # Combination 1: Use model 1 (vllm_api_general) with dataset (math500_gen_0_shot_cot_chat) - dict(models=[models[0]], datasets=datasets), # Combination 2: Use model 0 (vllm_api_general_copy) with dataset 0 (math500_gen_0_shot_cot_chat). Since vllm_api_general_copy and vllm_api_general have the same abbr, this will be considered the same task as combination 1 and will be skipped, even if the internal parameters differ + dict(models=[models[1]], datasets=datasets), + dict(models=[models[0]], datasets=datasets), ] ``` @@ -189,21 +837,99 @@ Correct approach: When reusing model or dataset configurations, modify the `abbr ```python vllm_api_general_copy = vllm_api_general.copy() -vllm_api_general_copy[0]['abbr'] = vllm_api_general[0]['abbr'] + '-copy' # Modify abbr to identify the model +vllm_api_general_copy[0]['abbr'] = vllm_api_general[0]['abbr'] + '-copy' ``` In this way, `vllm_api_general_copy[0]` and `vllm_api_general[0]` have different `abbr` values, so combination 2 and combination 1 are different tasks and will be executed normally. ## List of Preset Custom Configuration File Samples +### Quick Start + +| Filename | Description | +| --- | --- | +| [model_api_test_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_en.py) | Quick-start example (English): configures the `vllm_api_general_chat` service model and the `demo_gsm8k_gen_4_shot_cot_chat_prompt` dataset for a single accuracy evaluation task. | +| [model_api_test_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_zh_cn.py) | Quick-start example (Chinese): same as `model_api_test_en.py`, with Chinese comments. | + +### Service-Oriented Accuracy Evaluation (`api_examples/`) + | Filename | Description | | --- | --- | | [infer_vllm_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_general.py) | Evaluates the `v1/completions` sub-service using vLLM API (version 0.6+) on the GSM8K dataset. The prompt format is a string, and the dataset path is customized. | -| [infer_mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py) | Evaluates the `infer` sub-service using MindIE Stream API on the GSM8K dataset. The prompt format is a string, and the dataset path is customized. | -| [infer_vllm_api_old.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_old.py) | Evaluates the `generate` sub-service using vLLM API (version 0.2.6) on the GSM8K dataset. The prompt format is a string, and the dataset path is customized. | | [infer_vllm_api_general_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_general_chat.py) | Evaluates the `v1/chat/completions` sub-service using vLLM API (version 0.6+) on the GSM8K dataset. The prompt format is a conversation format, and the dataset path is customized. | | [infer_vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_stream_chat.py) | Evaluates the `v1/chat/completions` sub-service with streaming inference using vLLM API (version 0.6+) on the GSM8K dataset. The prompt format is a conversation format, and the dataset path is customized. | +| [infer_vllm_api_old.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_old.py) | Evaluates the `v1/completions` sub-service using older vLLM API on the GSM8K dataset. The prompt format is a string. | +| [infer_mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py) | Evaluates the `infer` sub-service using MindIE Stream API on the GSM8K dataset. The prompt format is a string, and the dataset path is customized. | +| [demo_infer_vllm_api.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/demo_infer_vllm_api.py) | Demo example: Evaluates the accuracy of two interfaces `v1/chat/completions` and `v1/completions` simultaneously on the GSM8K and MATH datasets. | +| [infer_vllm_api_multi_model_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_multi_model_multi_dataset.py) | Multi-model multi-dataset accuracy evaluation: combines 3 vLLM service models (`general`, `general_chat`, `stream_chat`) with the GSM8K, MATH, and MMLU datasets via the Cartesian product. | +| [infer_vllm_api_with_model_dataset_combinations.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_with_model_dataset_combinations.py) | Custom model-dataset pairings: precisely controls which models are paired with which datasets via `model_dataset_combinations`. | +| [infer_vllm_api_with_judge_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_with_judge_model.py) | Judge model evaluation: evaluates the AIME 2025 dataset that requires an LLM Judge, configuring the judge model in `judge_infer_cfg`. | + +### Service-Oriented Performance Evaluation (`api_examples/`) + +| Filename | Description | +| --- | --- | +| [demo_infer_vllm_api_perf.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py) | Demo example: Evaluates the streaming performance of two interfaces `v1/chat/completions` and `v1/completions` simultaneously using synthetic datasets. | +| [perf_vllm_api_synthetic.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_synthetic.py) | Synthetic dataset performance evaluation: customizes the input/output token length distributions of the synthetic dataset. | +| [perf_vllm_api_stable_stage.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_stable_stage.py) | Steady-state performance evaluation: sends the synthetic dataset at multiple `request_rate`s (0/5/10/20) for steady-state performance testing. | +| [perf_vllm_api_multiturn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_multiturn.py) | Multi-turn dialogue performance evaluation: uses the ShareGPT multi-turn dialogue dataset. | +| [perf_vllm_api_custom_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_custom_dataset.py) | Custom dataset performance evaluation: evaluates performance on your own CSV/JSONL dataset. | +| [perf_vllm_api_rps_distribution.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_rps_distribution.py) | RPS distribution control performance evaluation: configures `traffic_cfg` (burstiness, ramp-up strategy) to control the request arrival distribution. | + +### Pure Model Accuracy Evaluation (`hf_example/`) + +| Filename | Description | +| --- | --- | | [infer_hf_base_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_base_model.py) | Evaluates using the inference interface of a Hugging Face base model on the GSM8K dataset. The prompt format is a string, and the dataset path is customized. | -| [infer_hf_chat_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_chat_model.py) | Evaluates using the inference interface of a Hugging Face chat model on the GSM8K dataset. The prompt format is a string, and the dataset path is customized. | +| [infer_hf_chat_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_chat_model.py) | Evaluates using the inference interface of a Hugging Face chat model on the GSM8K dataset. The prompt format is a conversation format, and the dataset path is customized. | +| [infer_hf_multi_model_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_multi_model_multi_dataset.py) | Multi-model multi-dataset pure model evaluation: evaluates multiple Hugging Face local models on multiple datasets. | + +### Multimodal Evaluation (`lmm_example/`) + +| Filename | Description | +| --- | --- | +| [multi_device_run_qwen_image_edit.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/lmm_example/multi_device_run_qwen_image_edit.py) | Multimodal image-edit model evaluation (Qwen image edit, multi-device). | +| [infer_lmm_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/lmm_example/infer_lmm_multi_dataset.py) | Multimodal multi-dataset accuracy evaluation: evaluates a multimodal model on multiple multimodal datasets. | + +### Accuracy Evaluation Scenario Samples (`accuracy_benchmark/`) + +| Filename | Description | +| --- | --- | +| [single_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/single_task_en.py) | Single-task accuracy evaluation | +| [multi_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_en.py) | Multi-task accuracy evaluation | +| [multi_task_parallel_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py) | Multi-task parallel accuracy evaluation | +| [multi_task_resume_partial_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py) | Resumption after interruption & retesting of failed cases (partial tasks) | +| [ceval_merge_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/ceval_merge_en.py) | Merging sub-dataset inference | +| [fixed_prompts_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py) | Fixed request count evaluation | +| [multi_repeat_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_repeat_en.py) | Multiple independent repeat inference | +| [inference_re_eval_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py) | Re-evaluation of inference results | + +### Pure Model Accuracy Evaluation Scenario Samples (`accuracy_benchmark_local/`) + +| Filename | Description | +| --- | --- | +| [single_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/single_task_en.py) | Single-task pure model evaluation | +| [multi_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/multi_task_en.py) | Pure model multi-task / multi-task parallel evaluation | +| [ceval_merge_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py) | Merged sub-dataset inference | +| [inference_re_eval_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py) | Re-evaluation of pure model inference results | + +### Performance Evaluation Scenario Samples (`performance_benchmark/`) + +| Filename | Description | +| --- | --- | +| [performance_qwen2_7b_sharegpt.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py) | Single-task performance evaluation (ShareGPT) | +| [performance_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_multi_dataset.py) | Multi-dataset performance evaluation | +| [performance_multi_rate.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_multi_rate.py) | Multi-rate performance evaluation | +| [performance_multi_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_multi_model.py) | Multi-model performance evaluation | +| [performance_synthetic.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_synthetic.py) | Synthetic dataset multi-task combinations | +| [performance_seq_combinations.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_seq_combinations.py) | Custom sequence multi-task combinations | +| [performance_fixed_request.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_fixed_request.py) | Fixed request count performance evaluation | +| [performance_re_eval.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_re_eval.py) | Performance result recalculation | + +### Common Utilities + +| Filename | Description | +| --- | --- | +| [all_dataset_configs.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/all_dataset_configs.py) | A consolidated import of all supported dataset configurations; can be used directly via `from ... import` in custom configuration files. | -**Note**: To evaluate other datasets using the above custom configuration files, import additional datasets from [ais_bench/configs/api_examples/all_dataset_configs.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/all_dataset_configs.py). +**Note**: To evaluate other datasets using the above custom configuration files, import additional datasets from [ais_bench/configs/api_examples/all_dataset_configs.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/all_dataset_configs.py). \ No newline at end of file diff --git a/docs/source_en/advanced_tutorials/stable_stage.md b/docs/source_en/advanced_tutorials/stable_stage.md index 8bad7831..aadb6c15 100644 --- a/docs/source_en/advanced_tutorials/stable_stage.md +++ b/docs/source_en/advanced_tutorials/stable_stage.md @@ -208,9 +208,14 @@ After the command execution is completed, the task execution details in `outputs For instructions on how to view the charts in this HTML file, please refer to 📚 [Instructions for Using Performance Test Visualization Concurrency Charts](../base_tutorials/results_intro/performance_visualization.md) +## Implement via Custom Config Files + +> 💡 The above steady-state performance evaluation scenario can also be implemented through the [Custom Config File Method](run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax such as loops, conditional judgments, list comprehensions, etc. You can write models, datasets, summarizer, and other configurations into a single file, write once and reuse multiple times. See the "Steady-State Performance Evaluation" example in [Running AISBench with Custom Config Files](run_custom_config.md#custom-configuration-file-examples-for-each-scenario). + + ## Other Functional Scenarios ### Recalculating Performance Results -Refer to 📚 [Recalculation of Performance Results](../base_tutorials/scenes_intro/performance_benchmark.md#recalculation-of-performance-results) +Refer to 📚 [Recalculation of Performance Results](../base_tutorials/scenes_intro/performance_benchmark.md#performance-result-recalculation) #### Configuration Differences Modify the configuration file `stable_stage.py` corresponding to the `stable_stage` result presentation task specified by `--summarizer`. diff --git a/docs/source_en/advanced_tutorials/synthetic_dataset.md b/docs/source_en/advanced_tutorials/synthetic_dataset.md index 231b052b..9942d843 100644 --- a/docs/source_en/advanced_tutorials/synthetic_dataset.md +++ b/docs/source_en/advanced_tutorials/synthetic_dataset.md @@ -215,6 +215,7 @@ synthetic_config = { } ``` +------ ### 4.2 TokenId Type Examples @@ -225,7 +226,8 @@ synthetic_config = { "Type": "tokenid", "RequestCount": 1000, "TokenIdConfig": { - "RequestSize": 2048 # 2048 tokens per request + "RequestSize": 2048, # 2048 tokens per request + "PrefixLen": 0 } } ``` @@ -237,12 +239,13 @@ synthetic_config = { "Type": "tokenid", "RequestCount": 5000, "TokenIdConfig": { - "RequestSize": 128 # Short text processing scenario + "RequestSize": 128, # Short text processing scenario + "PrefixLen": 0 } } ``` -#### prefix Cache Performance Testing +#### Prefix Cache Performance Testing ```python synthetic_config = { @@ -255,6 +258,8 @@ synthetic_config = { } ``` +------ + ## V. Frequently Asked Questions ### Q1: How to choose a distribution type? @@ -266,6 +271,7 @@ synthetic_config = { - **Stress testing**: Use zipf distribution for Input and uniform distribution for Output. - **Stability testing**: Use gaussian distribution for both Input and Output. +------ ### Q2: Why does the performance evaluation result matrix show unexpected values even after specifying the *input length*? @@ -273,13 +279,19 @@ synthetic_config = { - **`string` mode**: The input length here refers to the length of the input string, not the number of tokens. - **Preprocessing stage**: Additional string concatenation may be performed before/after using chat-related APIs. +------ ### Q3: Why does the performance evaluation result matrix show unexpected values even after specifying the *output length* in String mode? - **Significant discrepancy**: Check if the `ignore_eos` parameter in `generation_kwargs` of the model API configuration file is correctly set to `True` (this ensures the service ignores the end-of-sequence token until the preset output length is reached). +------ ## VI. Notes 1. **`tokenid` mode**: The value range of `tokenid` depends on the vocabulary range of the model specified in the model configuration file. -2. **`string` mode**: A fixed-length sequence is generated when MinValue=MaxValue. \ No newline at end of file +2. **`string` mode**: A fixed-length sequence is generated when MinValue=MaxValue. + +## VII. Implement via Custom Config Files + +> 💡 The above synthetic dataset evaluation scenario can also be implemented through the [Custom Config File Method](run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax such as loops, conditional judgments, list comprehensions, etc. You can write models, datasets, summarizer, and other configurations into a single file, write once and reuse multiple times. See the "Synthetic Dataset Performance Evaluation" example in [Running AISBench with Custom Config Files](run_custom_config.md#custom-configuration-file-examples-for-each-scenario). \ No newline at end of file diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index c3868853..bb92aa6f 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -17,14 +17,15 @@ Based on the execution scenario, command line parameters are divided into three `Accuracy Evaluation Parameters` take effect only when the `--mode` parameter is specified as `"all", "infer", "eval", "viz"`. `Performance Evaluation Parameters` take effect only when the `--mode` parameter is specified as `"perf", "perf_viz"`. `Common Parameters` are not restricted by the task execution mode and can be specified in all modes. -# ### Common Parameters +### Common Parameters Applicable to all modes and can be used in combination with accuracy or performance parameters. | Parameter | Description | Example | | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------- | -| `--models` | Specifies the name of the model inference backend task (corresponding to a pre-implemented default model configuration file under the path `ais_bench/benchmark/configs/models`). Multiple task names are supported. For details, refer to 📚 [Supported Models](./models.md) | `--models vllm_api_general` | -| `--datasets` | Specifies the name of the dataset task (corresponding to a pre-implemented default dataset configuration file under the path `ais_bench/benchmark/configs/datasets`). Multiple dataset names are supported. For details, refer to 📚 [Supported Dataset Types](./datasets.md) | `--datasets gsm8k_gen` | -| `--summarizer` | Specifies the name of the result summary task (corresponding to a pre-implemented default configuration file under the path `ais_bench/benchmark/configs/summarizers`). For details, refer to 📚 [Supported Result Summary Tasks](./summarizer.md) | `--summarizer medium`| +| `config` | Specifies the path to a custom configuration file. | `ais_bench /path/to/custom_config.py {other optional arguments}` | +| `--models` | Specifies the name of the model inference backend task (corresponding to a pre-implemented default model configuration file under the path `ais_bench/benchmark/configs/models`). Multiple task names are supported. For details, refer to 📚 [Supported Models](./models.md).
⚠️ **Note**: This parameter is invalid when a custom configuration file path is specified. | `--models vllm_api_general` | +| `--datasets` | Specifies the name of the dataset task (corresponding to a pre-implemented default dataset configuration file under the path `ais_bench/benchmark/configs/datasets`). Multiple dataset names are supported. For details, refer to 📚 [Supported Dataset Types](../../get_started/datasets.md).
⚠️ **Note**: This parameter is invalid when a custom configuration file path is specified. | `--datasets gsm8k_gen` | +| `--summarizer` | Specifies the name of the result summary task (corresponding to a pre-implemented default configuration file under the path `ais_bench/benchmark/configs/summarizers`). For details, refer to 📚 [Supported Result Summary Tasks](./summarizer.md).
⚠️ **Note**: This parameter is invalid when a custom configuration file path is specified. | `--summarizer medium`| | `--mode` or `-m` | Running mode, optional values: `all`, `infer`, `eval`, `viz`, `perf`, `perf_viz`; default value is `all`.
For details, refer to 📚 [Running Mode Description](./mode.md). | `--mode infer`
`-m all`| | `--reuse` or `-r` | Specifies the timestamp in an existing working directory to continue execution and overwrite original results. Used in conjunction with the `--mode` parameter, it can resume interrupted inference, or perform accuracy calculation/visualization result printing based on existing inference results. If no parameter is added, the latest timestamp in the `--work-dir` is automatically selected. | `--reuse 20250126_144254`
`-r 20250126_144254` | | `--work-dir` or `-w` | Specifies the evaluation working directory for saving output results. Default path: `outputs/default`. | `--work-dir /path/to/work`
`-w /path/to/work` | @@ -34,13 +35,13 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--max-workers-per-gpu` | Reserved parameter; not currently supported. | `--max-workers-per-gpu 1` | | `--merge-ds` | Enables merged inference for datasets of the same type (runs multiple datasets for the same task together). | `--merge-ds` | | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | -| `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. | `--max-num-workers 2` | +| `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. Note: In performance evaluation scenarios, an excessively high concurrency may cause resource contention among different processes, leading to inaccurate test results. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | | `--response-anomaly` / `--no-response-anomaly` | Enables or disables msProbe response anomaly detection. The command-line value overrides `response_anomaly.enabled` in the config file. Detection runs in a thread in parallel with Eval; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | | `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | -# ### Accuracy Evaluation Parameters +### Accuracy Evaluation Parameters Valid only when the mode is `all`, `infer`, `eval`, or `viz`. | Parameter | Description | Example | @@ -49,7 +50,7 @@ Valid only when the mode is `all`, `infer`, `eval`, or `viz`. | `--dump-extract-rate` | Toggle to dump evaluation speed data. Enabled if configured, disabled if not; disabled by default. | `--dump-extract-rate`| -# ### Performance Evaluation Parameters +### Performance Evaluation Parameters Valid only when the mode is `perf` or `perf_viz`. | Parameter | Description | Example | diff --git a/docs/source_en/base_tutorials/all_params/models.md b/docs/source_en/base_tutorials/all_params/models.md index 682aa379..29a00077 100644 --- a/docs/source_en/base_tutorials/all_params/models.md +++ b/docs/source_en/base_tutorials/all_params/models.md @@ -13,20 +13,20 @@ Taking the vLLM inference service deployed on GPU as an example, you can refer t The model configurations corresponding to different service-oriented backends are as follows: -| Model Configuration Name | Description | Prerequisites for Use | Supported Evaluation Modes | Interface Type | Supported Dataset Prompt Formats | Configuration File Path | -| ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | -| `vllm_api_general` | Access the inference service via vLLM's OpenAI-compatible API, with the interface `v1/completions` | The vLLM version used supports the `v1/completions` sub-service | Generative Evaluation, PPL Mode Evaluation | Text Interface | String Format | [vllm_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general.py) | -| `vllm_api_general_stream` | Access the vLLM inference service in streaming mode, with the interface `v1/completions` | The vLLM version used supports the `v1/completions` sub-service | Generative Evaluation | Streaming Interface | String Format | [vllm_api_general_stream.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py) | -| `vllm_api_general_chat` | Access the inference service via vLLM's OpenAI-compatible API, with the interface `v1/chat/completions` | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation, PPL Mode Evaluation | Text Interface | String Format, Dialogue Format, Multimodal Format | [vllm_api_general_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py) | -| `vllm_api_stream_chat` | Access the vLLM inference service in streaming mode, with the interface `v1/chat/completions` | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation | Streaming Interface | String Format, Dialogue Format, Multimodal Format | [vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) | -| `vllm_api_stream_chat_multiturn` | Access the vLLM inference service in streaming mode for multi-turn dialogue scenarios, with the interface `v1/chat/completions` | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation | Streaming Interface | Dialogue Format | [vllm_api_stream_chat_multiturn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat_multiturn.py) | -| `vllm_api_function_call_chat` | API for accessing the vLLM inference service in function call accuracy evaluation scenarios, with the interface `v1/chat/completions` (only applicable to the [BFCL](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/BFCL/README_en.md) evaluation scenario) | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation | Text Interface | Dialogue Format | [vllm_api_function_call_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_function_call_chat.py) | -| `vllm_api_old` | Access the inference service via vLLM-compatible API, with the interface `generate` | The vLLM version used supports the `generate` sub-service | Generative Evaluation | Text Interface | String Format, Multimodal Format | [vllm_api_old.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_old.py) | -| `mindie_stream_api_general` | Access the inference service via MindIE streaming API, with the interface `infer` | The MindIE version used supports the `infer` sub-service | Generative Evaluation | Streaming Interface | String Format, Multimodal Format | [mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/mindie_api/mindie_stream_api_general.py) | -| `triton_api_general` | Access the inference service via Triton API, with the interface `v2/models/{model name}/generate` | Start an inference service that supports Triton API | Generative Evaluation | Text Interface | String Format, Multimodal Format | [triton_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_api_general.py) | -| `triton_stream_api_general` | Access the inference service via Triton streaming API, with the interface `v2/models/{model name}/generate_stream` | Start an inference service that supports Triton API | Generative Evaluation | Streaming Interface | String Format, Multimodal Format | [triton_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_stream_api_general.py) | -| `tgi_api_general` | Access the inference service via TGI API, with the interface `generate` | Start an inference service that supports TGI API | Generative Evaluation | Text Interface | String Format, Multimodal Format | [tgi_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_api_general.py) | -| `tgi_stream_api_general` | Access the inference service via TGI streaming API, with the interface `generate_stream` | Start an inference service that supports TGI API | Generative Evaluation | Streaming Interface | String Format, Multimodal Format | [tgi_stream_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_stream_api_general.py) | +| Model Configuration Name | Description | Prerequisites for Use | Supported Evaluation Modes | Interface Type | Supported Dataset Prompt Formats | Configuration File Import Method | Configuration File Path | +| ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | +| `vllm_api_general` | Access the inference service via vLLM's OpenAI-compatible API, with the interface `v1/completions` | The vLLM version used supports the `v1/completions` sub-service | Generative Evaluation, PPL Mode Evaluation | Text Interface | String Format | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general` | [vllm_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general.py) | +| `vllm_api_general_stream` | Access the vLLM inference service in streaming mode, with the interface `v1/completions` | The vLLM version used supports the `v1/completions` sub-service | Generative Evaluation | Streaming Interface | String Format | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream` | [vllm_api_general_stream.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py) | +| `vllm_api_general_chat` | Access the inference service via vLLM's OpenAI-compatible API, with the interface `v1/chat/completions` | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation, PPL Mode Evaluation | Text Interface | String Format, Dialogue Format, Multimodal Format | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat` | [vllm_api_general_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py) | +| `vllm_api_stream_chat` | Access the vLLM inference service in streaming mode, with the interface `v1/chat/completions` | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation | Streaming Interface | String Format, Dialogue Format, Multimodal Format | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat` | [vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) | +| `vllm_api_stream_chat_multiturn` | Access the vLLM inference service in streaming mode for multi-turn dialogue scenarios, with the interface `v1/chat/completions` | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation | Streaming Interface | Dialogue Format | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat_multiturn import models as vllm_api_stream_chat_multiturn` | [vllm_api_stream_chat_multiturn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat_multiturn.py) | +| `vllm_api_function_call_chat` | API for accessing the vLLM inference service in function call accuracy evaluation scenarios, with the interface `v1/chat/completions` (only applicable to the [BFCL](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/BFCL/README_en.md) evaluation scenario) | The vLLM version used supports the `v1/chat/completions` sub-service | Generative Evaluation | Text Interface | Dialogue Format | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_function_call_chat import models as vllm_api_function_call_chat` | [vllm_api_function_call_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_function_call_chat.py) | +| `vllm_api_old` | Access the inference service via vLLM-compatible API, with the interface `generate` | The vLLM version used supports the `generate` sub-service | Generative Evaluation | Text Interface | String Format, Multimodal Format | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_old import models as vllm_api_old` | [vllm_api_old.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_old.py) | +| `mindie_stream_api_general` | Access the inference service via MindIE streaming API, with the interface `infer` | The MindIE version used supports the `infer` sub-service | Generative Evaluation | Streaming Interface | String Format, Multimodal Format | `from ais_bench.benchmark.configs.models.mindie_api.mindie_stream_api_general import models as mindie_stream_api_general` | [mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/mindie_api/mindie_stream_api_general.py) | +| `triton_api_general` | Access the inference service via Triton API, with the interface `v2/models/{model name}/generate` | Start an inference service that supports Triton API | Generative Evaluation | Text Interface | String Format, Multimodal Format | `from ais_bench.benchmark.configs.models.triton_api.triton_api_general import models as triton_api_general` | [triton_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_api_general.py) | +| `triton_stream_api_general` | Access the inference service via Triton streaming API, with the interface `v2/models/{model name}/generate_stream` | Start an inference service that supports Triton API | Generative Evaluation | Streaming Interface | String Format, Multimodal Format | `from ais_bench.benchmark.configs.models.triton_api.triton_stream_api_general import models as triton_stream_api_general` | [triton_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_stream_api_general.py) | +| `tgi_api_general` | Access the inference service via TGI API, with the interface `generate` | Start an inference service that supports TGI API | Generative Evaluation | Text Interface | String Format, Multimodal Format | `from ais_bench.benchmark.configs.models.tgi_api.tgi_api_general import models as tgi_api_general` | [tgi_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_api_general.py) | +| `tgi_stream_api_general` | Access the inference service via TGI streaming API, with the interface `generate_stream` | Start an inference service that supports TGI API | Generative Evaluation | Streaming Interface | String Format, Multimodal Format | `from ais_bench.benchmark.configs.models.tgi_api.tgi_stream_api_general import models as tgi_stream_api_general` | [tgi_stream_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_stream_api_general.py) | ### Parameter Description for Service-Oriented Inference Backend Configuration @@ -37,7 +37,7 @@ The common model templates do not preconfigure `response_anomaly`. Add the `resp ```python from ais_bench.benchmark.models import VLLMCustomAPI -models = [ +models = [ # Equivalent to the `models` imported via `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general` in a custom configuration file dict( attr="service", type=VLLMCustomAPI, @@ -80,8 +80,8 @@ The description of configurable parameters for the service-oriented inference ba | `path` | String | Tokenizer path, usually the same as the model path. The Tokenizer is loaded using `AutoTokenizer.from_pretrained(path)`. Specify an accessible local path, e.g., `/weight/DeepSeek-R1` | | `model` | String | Name of the model accessible on the server, which must be consistent with the name specified during service-oriented deployment | | `model_name` | String | Applicable only to Triton services. It is concatenated into the endpoint URI `/v2/models/{modelname}/{infer, generate, generate_stream}` and must be consistent with the name used during deployment | -| `stream` | Boolean | Whether the inference service is a streaming interface. Required Parameter. | -| `request_rate` | Float | Request sending rate (unit: requests per second). A request is sent every `1/request_rate` seconds; if the value is less than 0.1, requests are automatically merged and sent in batches. Valid range: [0, 64000]. When the `traffic_cfg` item is enabled, this function may be overwritten (for specific reasons, refer to 🔗 [Parameter Interpretation Section in the Description of Request Rate (RPS) Distribution Control and Visualization](../../advanced_tutorials/rps_distribution.md#parameter-interpretation)) | +| `stream` | Boolean | API model inference interface type. The default is False, meaning a non-streaming interface. When True, it indicates a streaming interface (for details, refer to 🔗 [Service-Oriented Inference Backend](#service-oriented-inference-backend)) | +| `request_rate` | Float | Request sending rate (unit: seconds), a request is sent every `1/request_rate` seconds; in pressure testing scenarios, it represents the number of new server-side connections added per second; if the value is less than 0.1, the request sending rate is unlimited. Valid range: [0, 64000]. When the `traffic_cfg` item is enabled, this function may be overwritten (for specific reasons, refer to 🔗 [Parameter Interpretation Section in the Description of Request Rate (RPS) Distribution Control and Visualization](../../advanced_tutorials/rps_distribution.md#parameter-interpretation)) | | `use_timestamp` | Boolean | Whether to schedule requests according to the dataset's timestamp field. When True and the dataset contains timestamps, requests are sent by timestamp and **request_rate** / **traffic_cfg** are ignored; when False, request_rate and traffic_cfg apply. Default False. Used with timestamped datasets (e.g. Mooncake Trace). | | `traffic_cfg` | Dict | Parameters for controlling fluctuations in the request sending rate (for detailed usage instructions, refer to 🔗 [Description of Request Rate (RPS) Distribution Control and Visualization](../../advanced_tutorials/rps_distribution.md)). If this item is not filled in, the function is disabled by default | | `retry` | Int | Maximum number of retries after failing to connect to the server. Valid range: [0, 1000] | @@ -150,12 +150,12 @@ models = [ ## Local Model Backend -| Model Configuration Name | Description | Prerequisites for Use | Supported Prompt Formats (String Format or Dialogue Format) | Corresponding Source Code Configuration File Path | -| --- | --- | --- | --- | --- | -| `hf_base_model` | HuggingFace Base Model Backend | The basic dependencies of the evaluation tool have been installed; the HuggingFace model weight path must be specified in the configuration file (automatic download is not supported currently) | String Format | [hf_base_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_base_model.py) | -| `hf_chat_model` | HuggingFace Chat Model Backend | The basic dependencies of the evaluation tool have been installed; the HuggingFace model weight path must be specified in the configuration file (automatic download is not supported currently) | Dialogue Format | [hf_chat_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_chat_model.py) | -|`hf_qwenvl_model`| HuggingFace Chat QwenVL Model Backend|The basic dependencies of the evaluation tool have been installed; the HuggingFace model weight path must be specified in the configuration file (automatic download is not supported currently)|Dialogue Format|[hf_qwenvl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_qwenvl_model.py)| -|`vllm_offline_vl_model`| vLLM Chat QwenVL Offline Inference Model Backend|The basic dependencies of the evaluation tool have been installed; the model weight path must be specified in the configuration file (automatic download is not supported currently)|Dialogue Format|[vllm_offline_vl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_offline_models/vllm_offline_vl_model.py)| +| Model Configuration Name | Description | Prerequisites for Use | Supported Prompt Formats (String Format or Dialogue Format) | Configuration File Import Method | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | +| `hf_base_model` | HuggingFace Base Model Backend | The basic dependencies of the evaluation tool have been installed; the HuggingFace model weight path must be specified in the configuration file (automatic download is not supported currently) | String Format | `from ais_bench.benchmark.configs.models.hf_models.hf_base_model import models as hf_base_model` | [hf_base_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_base_model.py) | +| `hf_chat_model` | HuggingFace Chat Model Backend | The basic dependencies of the evaluation tool have been installed; the HuggingFace model weight path must be specified in the configuration file (automatic download is not supported currently) | Dialogue Format | `from ais_bench.benchmark.configs.models.hf_models.hf_chat_model import models as hf_chat_model` | [hf_chat_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_chat_model.py) | +|`hf_qwenvl_model`| HuggingFace Chat QwenVL Model Backend|The basic dependencies of the evaluation tool have been installed; the HuggingFace model weight path must be specified in the configuration file (automatic download is not supported currently)|Dialogue Format|`from ais_bench.benchmark.configs.models.hf_models.hf_qwenvl_model import models as hf_qwenvl_model`|[hf_qwenvl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_qwenvl_model.py)| +|`vllm_offline_vl_model`| vLLM Chat QwenVL Offline Inference Model Backend|The basic dependencies of the evaluation tool have been installed; the model weight path must be specified in the configuration file (automatic download is not supported currently)|Dialogue Format|`from ais_bench.benchmark.configs.models.vllm_offline_models.vllm_offline_vl_model import models as vllm_offline_vl_model`|[vllm_offline_vl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_offline_models/vllm_offline_vl_model.py)| ### Parameter Description for Huggingface Local Model Backend Configuration @@ -163,7 +163,7 @@ The configuration file for the huggingface local model backend is configured usi ```python from ais_bench.benchmark.models import HuggingFacewithChatTemplate -models = [ +models = [ # Equivalent to the `models` imported via `from ais_bench.benchmark.configs.models.hf_models.hf_chat_model import models as hf_chat_model` in a custom configuration file dict( attr="local", # Backend type identifier type=HuggingFacewithChatTemplate, # Model type diff --git a/docs/source_en/base_tutorials/all_params/summarizer.md b/docs/source_en/base_tutorials/all_params/summarizer.md index 5115f8ea..69a1c17f 100644 --- a/docs/source_en/base_tutorials/all_params/summarizer.md +++ b/docs/source_en/base_tutorials/all_params/summarizer.md @@ -1,7 +1,7 @@ # Supported Result Summary Tasks -| Task Name | Description | Configuration File Path | -| -------------- | -------------- | -------------- | -| `example` | A simplified accuracy evaluation result summary template that covers all currently supported datasets and is the default template used. | [example.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/example.py) | -| `medium` | A general accuracy evaluation result summary template, suitable for multiple basic datasets. | [medium.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/medium.py) | -| `default_perf` | A full-scale performance evaluation result summary template that aggregates performance data of all requests. It supports manual configuration of performance statistics indicators via `default_perf.py`. | [default_perf.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/default_perf.py) | -| `stable_stage` | A performance evaluation result summary template for the stable stage, which only aggregates request data when the system reaches the configured maximum concurrency. It supports manual configuration of performance statistics indicators via `stable_stage.py`. | [stable_stage.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/stable_stage.py) | \ No newline at end of file +| Task Name | Description | Configuration File Import Method | Configuration File Path | +| -------------- | -------------- | -------------- | -------------- | +| `example` | A simplified accuracy evaluation result summary template that covers all currently supported datasets and is the default template used. | `from ais_bench.benchmark.configs.summarizers.example import summarizer` | [example.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/example.py) | +| `medium` | A general accuracy evaluation result summary template, suitable for multiple basic datasets.| `from ais_bench.benchmark.configs.summarizers.medium import summarizer` | [medium.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/medium.py) | +| `default_perf` | A full-scale performance evaluation result summary template that aggregates performance data of all requests. It supports manual configuration of performance statistics indicators via `default_perf.py`. | `from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer` | [default_perf.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/default_perf.py) | +| `stable_stage` | A performance evaluation result summary template for the stable stage, which only aggregates request data when the system reaches the configured maximum concurrency. It supports manual configuration of performance statistics indicators via `stable_stage.py`. | `from ais_bench.benchmark.configs.summarizers.perf.stable_stage import summarizer` | [stable_stage.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/stable_stage.py) | \ No newline at end of file diff --git a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md index 120bd2b3..b465318c 100644 --- a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -7,30 +7,99 @@ Before performing service-oriented inference, the following conditions must be m - Accessible service-oriented model service: Ensure the service process can be directly accessed in the current environment. - Dataset task preparation: - - Open-source datasets: Select a dataset from 📚 [Open-Source Datasets](../all_params/datasets.md#open-source-datasets), and choose the dataset task to execute from the "detailed introduction" document corresponding to the dataset. Prepare the dataset files by referring to the "detailed introduction" document of the selected dataset task. It is recommended to manually place the open-source dataset in the default directory `ais_bench/datasets/`; the program will automatically load the dataset files during task execution. + - Open-source datasets: Select a dataset from 📚 [Open-Source Datasets](../../get_started/datasets.md#open-source-datasets), and choose the dataset task to execute from the "detailed introduction" document corresponding to the dataset. Prepare the dataset files by referring to the "detailed introduction" document of the selected dataset task. It is recommended to manually place the open-source dataset in the default directory `ais_bench/datasets/`; the program will automatically load the dataset files during task execution. - Custom datasets: No need to specify a dataset task; refer to 📚 [Custom Dataset](../../advanced_tutorials/custom_dataset.md) for other configurations. - Model task preparation: Select the model task to execute from 📚 [Service-Oriented Inference Backend](../all_params/models.md#service-oriented-inference-backend). ## Main Functional Scenarios ### Single-Task Evaluation -Please refer to 📚 [Quick Start](../../get_started/quick_start.md) on the homepage for details; no further elaboration here. +Please refer to 📚 [Quick Start](../../get_started/quick_start.md) on the homepage for details. ### Multi-Task Evaluation It supports configuring multiple models or multiple dataset tasks simultaneously and conducting batch evaluations with a single command, which is suitable for large-scale model horizontal comparison or multi-dataset accuracy comparison analysis. -#### Command Description -Users can specify multiple configuration tasks via the `--models` and `--datasets` parameters. The number of subtasks is the product of the number of tasks configured by `--models` and `--datasets`—that is, one model configuration and one dataset configuration form a subtask. Example command: -```bash -ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt -``` -The above command specifies 2 model tasks (`vllm_api_general_chat`, `vllm_api_stream_chat`) and 2 dataset tasks (`gsm8k_gen_4_shot_cot_str`, `aime2024_gen_0_shot_chat_prompt`), and will execute the following 4 combined accuracy test tasks: +#### Description of Sub-task Combinations + +In multi-task evaluation scenarios, the number of subtasks is the product of the number of tasks configured by `models` and the number of tasks configured by `datasets`—that is, one model configuration and one dataset configuration form a subtask. The following example simultaneously evaluates 2 model tasks (`vllm_api_general_chat`, `vllm_api_stream_chat`) and 2 dataset tasks (`gsm8k_gen_4_shot_cot_str`, `aime2024_gen_0_shot_chat_prompt`), and will execute the following 4 combined accuracy test tasks: + [vllm_api_general_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py) model task + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) dataset task + [vllm_api_general_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py) model task + [aime2024_gen_0_shot_chat_prompt](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_chat_prompt.py) dataset task + [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) model task + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) dataset task + [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) model task + [aime2024_gen_0_shot_chat_prompt](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_chat_prompt.py) dataset task +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +Refer to the [model_api_test_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_en.py) file from the quick start. Import multiple model tasks and dataset tasks within `with read_base():`, then combine them into the `models` and `datasets` lists. For a complete example, refer to [multi_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +# ...For other parameter configurations, please refer to the configuration file +``` + +After modifying the configuration file, execute the command: + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_en.py +``` + +#### Custom Model-Dataset Pairings (Optional) + +By default, the `models` list and `datasets` list in the above configuration are automatically combined as a Cartesian product, with the number of subtasks equal to the number of models × the number of datasets (in this example, 2 × 2 = 4). If you want to precisely control which models are paired with which datasets (e.g., letting some models only run on some datasets to avoid meaningless combinations), you can explicitly declare the pairing relationship in the configuration file via the `model_dataset_combinations` field: + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets +models = vllm_api_general_chat + vllm_api_stream_chat + +# Key: Precisely control pairings via model_dataset_combinations +# The following example generates only 2 subtasks (the Cartesian product would generate 4): +# - vllm_api_general_chat + gsm8k_gen_4_shot_cot_str +# - vllm_api_stream_chat + aime2024_gen_0_shot_chat_prompt +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), +] +``` + +> ⚠️ **Note**: The unique identifier for models and datasets is determined by the `abbr` field. In the same configuration file, repeated combinations of models or datasets with the same `abbr` will be treated as duplicate tasks and skipped. When reusing model/dataset configurations via methods such as `.copy()`, the `abbr` must be explicitly modified to ensure uniqueness. See 📚 [Custom Model and Dataset Combinations](../../advanced_tutorials/run_custom_config.md#custom-model-and-dataset-combinations) for details. + +::: + +:::{tab-item} Alternative: Using Command-Line Parameters + +Users can specify multiple configuration tasks via the `--models` and `--datasets` parameters. Example command: + +```bash +ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt +``` + #### Modify Configuration Files Corresponding to Tasks The actual paths of the configuration files for model tasks and dataset tasks can be queried by executing the command with the `--search` parameter: ```bash @@ -50,14 +119,20 @@ The following configuration files to be modified will be queried: │ --datasets │ aime2024_gen_0_shot_chat_prompt │ /your_workspace/benchmark_test/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_chat_prompt.py │ ╘═════════════╧═════════════════════════════════╧═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ ``` -- Refer to 📚 [Service-Oriented Inference Backend Configuration Parameter Description](../all_params/models.md#parameter-description-for-local-model-backend-configuration) to configure the configuration files corresponding to the model tasks `vllm_api_general_chat` and `vllm_api_stream_chat` according to the actual situation. -- Refer to 📚 [Configure Open-Source Datasets](../all_params/datasets.md#configuring-open-source-datasets) to configure the configuration files corresponding to the dataset tasks `gsm8k_gen_4_shot_cot_str` and `aime2024_gen_0_shot_chat_prompt` according to the actual situation. **Note**: If the dataset is placed in the default directory `ais_bench/datasets/`, no configuration is generally required. +- Refer to 📚 [Service-Oriented Inference Backend Configuration Parameter Description](../all_params/models.md#parameter-description-for-service-oriented-inference-backend-configuration) to configure the configuration files corresponding to the model tasks `vllm_api_general_chat` and `vllm_api_stream_chat` according to the actual situation. +- Refer to 📚 [Configure Open-Source Datasets](../../get_started/datasets.md#configuring-open-source-datasets) to configure the configuration files corresponding to the dataset tasks `gsm8k_gen_4_shot_cot_str` and `aime2024_gen_0_shot_chat_prompt` according to the actual situation. **Note**: If the dataset is placed in the default directory `ais_bench/datasets/`, no configuration is generally required. #### Execute the Evaluation Command + Execute the command: + ```bash ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt ``` + +::: +:::: + During execution, a timestamp directory will be created under the path specified by 📚 [`--work-dir`](../all_params/cli_args.md#common-parameters) (default: `outputs/default/`) to store execution details. After the task is completed, an example of the on-screen log showing the results is as follows: @@ -109,18 +184,55 @@ At the same time, the final generated directory structure is as follows: ``` ### Multi-Task Parallel Evaluation -By default, multiple subtasks are executed serially. Continuous Batch is enabled by default within a single task, and multiple processes will be launched to send and process requests according to the maximum concurrency configured by the user, allowing for large concurrency settings. When the concurrency of a single task is low, multi-task parallelism can be achieved by setting the 📚 [`--max-num-workers`](../all_params/cli_args.md#accuracy-evaluation-parameters) parameter. Example as follows: +By default, multiple subtasks are executed serially. Continuous Batch is enabled by default within a single task, and multiple processes will be launched to send and process requests according to the maximum concurrency configured by the user, allowing for large concurrency settings. When the concurrency of a single task is low, multi-task parallelism can be achieved by setting the 📚 [`--max-num-workers`](../all_params/cli_args.md#common-parameters) parameter. Example as follows: + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +In the custom configuration file, `max_num_workers` no longer needs to be set; instead, it is passed via the command-line parameter [`--max-num-workers`](../all_params/cli_args.md#common-parameters). The configuration file example is identical to that in [Multi-Task Evaluation](#multi-task-evaluation). For a complete example, refer to [multi_task_parallel_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py): + +```python +# The complete example is identical to the configuration in Multi-Task Evaluation; the only difference lies in the execution command +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +# ...For other parameter configurations, please refer to the configuration file +``` + +Execute the command (specify the parallelism count via `--max-num-workers 4`): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py --max-num-workers 4 +``` + +::: +:::{tab-item} Alternative: Using Command-Line Parameters + ```bash ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt --max-num-workers 4 ``` -In the example above, the maximum number of concurrent tasks is set to 4, so four subtasks will be executed simultaneously. This can be viewed on the command line dashboard: +::: +:::: +In the example above, the maximum number of concurrent tasks is set to 4, so four subtasks will be executed simultaneously. This can be viewed on the command-line dashboard: ``` Base path of result&log : outputs/default/20251106_113926 Task Progress Table (Updated at: 2025-11-06 11:39:58) Page: 1/1 Total 5 rows of data -Press Up/Down arrow to page, 'P' to PAUSE/RESUME screen refresh, 'Ctrl + C' to exit +Press Up/Down arrow to page, 'P' to PAUZE/RESUME screen refresh, 'Ctrl + C' to exit +--------------------------------+-----------+----------------------------------------------------+-------------+-------------+-----------------------------------------------+---------------------------------------------------+ | Task Name | Process | Progress | Time Cost | Status | Log Path | Extend Parameters | @@ -133,18 +245,28 @@ Press Up/Down arrow to page, 'P' to PAUSE/RESUME screen refresh, 'Ctrl + C' to +--------------------------------+-----------+----------------------------------------------------+-------------+-------------+-----------------------------------------------+---------------------------------------------------+ | vllm-api-stream-chat/aime2024 | 1250138 | [############### ] 15/30 [5.0 it/s] | 0:00:07 | inferencing | logs/infer/vllm-api-stream-chat/aime2024.out | {'POST': 20, 'RECV': 15, 'FINISH': 15, 'FAIL': 0} | +--------------------------------+-----------+----------------------------------------------------+-------------+-------------+-----------------------------------------------+---------------------------------------------------+ + ``` The generated result is consistent with the example in [Multi-Task Evaluation](#multi-task-evaluation). + ### Resumption After Interruption & Retesting of Failed Cases If the inference task fails due to an unexpected interruption or server exception during the evaluation, the breakpoint management function can be enabled via `--reuse` to resume the task. It also supports automatic retesting of only failed cases without re-running all tasks. Example as follows: 1. Assume the user first executes the inference evaluation with the following command. If the task is interrupted due to an abnormal exit or some requests fail due to server exceptions: + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +First execution command (based on [single_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/single_task_en.py)): + ```bash -ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt +ais_bench ais_bench/configs/accuracy_benchmark/single_task_en.py ``` + At this point, some inference results will be saved, and the following file content will be generated under the 📚 [`--work-dir`](../all_params/cli_args.md#common-parameters) directory: + ```bash # Under output/default 20250628_151326/ # Timestamp directory created by the test task @@ -158,11 +280,41 @@ At this point, some inference results will be saved, and the following file cont └── tmp_0_2766386_1749107195.json # Cache file, named in the format: tmp_{task_process_ID}_{process_number}_{timestamp}.json ``` +2. Resume the inference by specifying the task timestamp directory via the `--reuse` parameter (`--reuse` is a common parameter; when using a custom configuration file, it can still be appended via the command line): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/single_task_en.py --reuse 20250628_151326 +``` + +::: +:::{tab-item} Alternative: Using Command-Line Parameters + +```bash +ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt +``` +At this point, some inference results will be saved, and the following file content will be generated under the 📚 [`--work-dir`](../all_params/cli_args.md#common-parameters) directory: +```bash +# Under output/default +20250628_151326/ # Timestamp directory created by the test task +├── configs # A combined configuration file of the configuration files for model tasks, dataset tasks, and structure presentation tasks +│ └── 20250628_151326_29317.py +├── logs # Logs during execution; if --debug is added to the command, no process logs will be saved to disk (all will be printed directly) +│ └── infer # Logs of the inference phase +└── predictions # Directory for inference results, recording the input of each request, model output, and answers (for accuracy evaluation) + └── vllm-api-general-chat + └── tmp_demo_gsm8k # Inference output of completed requests + └── tmp_0_2766386_1749107195.json # Cache file, named in the format: tmp_{task_process_ID}_{process_number}_{timestamp}.json +``` 2. Resume the inference by specifying the task timestamp directory via the `--reuse` parameter: ```bash ais_bench --models vllm_api_general --datasets gsm8k_gen --reuse 20250628_151326 ``` + +::: +:::: + The following content will be printed in the log, indicating that the resumption task has started: + ```bash 02/20 13:14:15 - AISBench - INFO - Found 10 tmp items, run infer task from the last interrupted position ``` @@ -170,8 +322,56 @@ After the resumption is completed, the accuracy results of all requests will be > ⚠️ Note: Resumption after interruption and retesting of failed cases may change the order of requests, which may cause slight fluctuations in results. -💡 [Multi-Task Evaluation](#multi-task-evaluation) also supports resumption after interruption and retesting of failed cases for all or part of the tasks. -For example, if an interruption occurs when executing the following multi-task evaluation command: +💡[Multi-Task Evaluation](#multi-task-evaluation) also supports resumption after interruption and retesting of failed cases for all or part of the tasks. + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +For example, an interruption occurs when executing the following multi-task evaluation command: + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_en.py +``` + +Resume all tasks after interruption in the following way: + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_en.py --reuse 20250628_151326 +``` + +You can also resume only part of the tasks after editing the custom configuration file. For a complete example, refer to [multi_task_resume_partial_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_datasets +models = vllm_api_general_chat +# ...For other parameter configurations, please refer to the configuration file +``` + +Then execute: + +```bash +# Resume only the vllm_api_general_chat + gsm8k_gen_4_shot_cot_str task after interruption +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py --reuse 20250628_151326 + +# Resume the two tasks of vllm_api_general_chat + gsm8k_gen_4_shot_cot_str and vllm_api_general_chat + aime2024_gen_0_shot_chat_prompts +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py --reuse 20250628_151326 +``` + +> 💡 If you need to resume only part of the combinations (e.g., `vllm_api_general_chat + aime2024`, `vllm_api_stream_chat + aime2024`), simply specify the corresponding model tasks and dataset tasks in the custom configuration file and then specify the timestamp via `--reuse`. See 📚 [Custom Model-Dataset Pairings](../../advanced_tutorials/run_custom_config.md#6-custom-model-dataset-pairings) for details. + +::: +:::{tab-item} Alternative: Using Command-Line Parameters + ```bash ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt ``` @@ -181,27 +381,179 @@ ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_g ``` You can also resume only part of the tasks in the following ways: ```bash -# Resume only the task of vllm_api_general_chat + gsm8k_gen_4_shot_cot_str +# Resume only the vllm_api_general_chat + gsm8k_gen_4_shot_cot_str task after interruption ais_bench --models vllm_api_general_chat --datasets gsm8k_gen_4_shot_cot_str --reuse 20250628_151326 # Resume the two tasks of vllm_api_general_chat + gsm8k_gen_4_shot_cot_str and vllm_api_general_chat + aime2024_gen_0_shot_chat_prompts ais_bench --models vllm_api_general_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt --reuse 20250628_151326 # Resume the two tasks of vllm_api_general_chat + aime2024_gen_0_shot_chat_prompts and vllm_api_stream_chat + aime2024_gen_0_shot_chat_prompts -ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets aime2024_gen_0_shot_chat_prompt --reuse 20250628 +ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets aime2024_gen_0_shot_chat_prompt --reuse 20250628_151326 ``` +::: +:::: + ### Merging Sub-dataset Inference -Some datasets are categorized into different sub-datasets, which will be split into multiple subtasks for inference during the inference process. Examples include 📚 [MMLU](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/mmlu/README_en.md) and 📚 [CEVAL](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/ceval/README_en.md). AISBench Benchmark supports merging datasets that consist of multiple small-scale datasets into a single task for unified evaluation. An example command is as follows: +Some datasets are categorized into different sub-datasets, which will be split into multiple subtasks for inference during the inference process. Examples include 📚 [MMLU](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/mmlu/README_en.md) and 📚 [CEVAL](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/ceval/README_en.md). AISBench Benchmark supports merging datasets that consist of multiple small-scale datasets into a single task for unified evaluation. An example is as follows: + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +Modify the custom configuration file to import a dataset task that supports merged inference. For a complete example, refer to [ceval_merge_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/ceval_merge_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + +models = vllm_api_general +# ...For other parameter configurations, please refer to the configuration file +``` + +Execute the command (`--merge-ds` is a common parameter; when using a custom configuration file, it can still be appended via the command line): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/ceval_merge_en.py --merge-ds +``` + +::: +:::{tab-item} Alternative: Using Command-Line Parameters + ```bash ais_bench --models vllm_api_general --datasets ceval_gen --merge-ds ``` + +::: +:::: + > ⚠️ Note: In merge mode, only the overall result will be generated, and the accuracy of individual sub-datasets will no longer be listed separately. Additionally, if you need to resume interrupted inference or re-run failed cases for inference results that were interrupted or failed in merge mode, you must also add `--merge-ds` to the command. +### Fixed Request Count Evaluation + +When the dataset scale is too large and you only want to perform accuracy testing on a subset of samples, you can use either of the following two approaches to control the data reading range. They achieve the same goal, so just pick the one that fits your habit: + +- **Basic approach**: Specify the number of data entries to read directly via the command-line parameter 📚 [`--num-prompts`](../all_params/cli_args.md#common-parameters). No configuration file modification is required, and it is the simplest to use. +- **Advanced approach (more powerful)**: Set the `reader_cfg.test_range` field of the dataset in the custom configuration file, which supports a more flexible sampling range (e.g., specifying a start index and custom step). For detailed usage, refer to 📚 [Custom Configuration Files](../../advanced_tutorials/run_custom_config.md). + +Example as follows: + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +**Method 1: Basic approach — Use `--num-prompts` to specify the number of entries to read** + +For a complete example, refer to [fixed_prompts_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +# ...For other parameter configurations, please refer to the configuration file +``` + +Execute the command (specify reading only 1 sample via `--num-prompts 1`): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py --num-prompts 1 +``` + +**Method 2: Advanced approach — Use `test_range` to flexibly specify the reading range** + +If you need more flexible range control (e.g., specifying a start index and custom step), you can set the `reader_cfg.test_range` field of the dataset directly in the custom configuration file, without passing any command-line parameter. For a complete example, refer to [fixed_prompts_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Key: control the sampling range flexibly via reader_cfg.test_range +# For example, '[0:8]' reads the first 8 samples; '[10:20]' reads samples from index 10 to 20 +datasets[0]['reader_cfg']['test_range'] = '[0:8]' + +models = vllm_api_stream_chat +# ...For other parameter configurations, please refer to the configuration file +``` + +Execute the command (test_range has been specified in the configuration file, no need to pass `--num-prompts`): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py +``` + +::: +:::{tab-item} Alternative: Using Command-Line Parameters + +```bash +ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --num-prompts 1 +``` +The above command only performs inference on the first entry in the sample dataset and only evaluates the accuracy of this one entry. + +::: +:::: + +> ⚠️ Note: Currently, the dataset is read sequentially in the default queue order; random sampling or shuffling is not supported. When `reader_cfg.test_range` in the configuration file and the command-line `--num-prompts` are both specified, the command-line parameter `--num-prompts` takes precedence. ### Multiple Independent Repeat Inference > After enabling this feature, the `dataset`/`number of requests` will be expanded exponentially at the `data point level`, which will significantly increase inference time and memory usage. Please read 📚 [Accuracy Evaluation Scenario: Interpretation of Evaluation Metrics](../results_intro/accuracy_metric.md) first, and **confirm whether this feature is necessary for your current scenario** before enabling it. -This scenario aims to explore model capabilities from multiple dimensions such as reliability, stability, and overall accuracy. To enable it, configure the value of the 🔗[`num_return_sequences` parameter](../all_params/models.md#parameter-description-for-service-oriented-inference-backend-configuration) in the hyperparameter `generation_kwargs` within the `service-side inference backend configuration parameters`. Refer to the following example for the format (the value provided is for reference only): +This scenario aims to explore model capabilities from multiple dimensions such as reliability, stability, and overall accuracy. To enable it, configure the value of the 🔗[`num_return_sequences` parameter](../all_params/models.md#parameter-description-for-service-oriented-inference-backend-configuration) in the hyperparameter `generation_kwargs` within the `service-side inference backend configuration parameters`. + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +For a complete example, refer to [multi_repeat_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_repeat_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +# Key: Enable multiple independent repeat inference via generation_kwargs.num_return_sequences +models[0]["generation_kwargs"] = dict( + temperature=0.01, + ignore_eos=False, + num_return_sequences=5, # For specific functions and constraints, refer to the document accuracy_metric.md +) +# ...For other parameter configurations, please refer to the configuration file +``` + +Execute the command: + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_repeat_en.py +``` + +::: +:::{tab-item} Alternative: Using Command-Line Parameters + +Modify `generation_kwargs` in the model task configuration file: ```python models = [ @@ -211,11 +563,14 @@ models = [ num_return_sequences = 5, # For specific functions and constraints, refer to the document accuracy_metric.md ... # Other parameters ), - ... + ... # Other parameters ) ] ``` +::: +:::: + After the accuracy evaluation phase is completed, the results will be recorded in the log and printed in the running window. The format is as shown in the following example (data is for reference only): ```bash @@ -229,6 +584,24 @@ After the accuracy evaluation phase is completed, the results will be recorded i For **specific interpretation of indicators** and **parameter constraints** in the table above, please refer to 📚 [Accuracy Evaluation Scenario: Interpretation of Evaluation Metrics](../results_intro/accuracy_metric.md). +## Implementation via Custom Configuration Files + +> 💡 All the above functional scenarios (multi-task evaluation, multi-task parallelism, resumption after interruption, merged sub-datasets, fixed request count evaluation, multiple independent repeat inference, re-evaluation of inference results, etc.) provide two startup methods (**⭐ Recommended: Using a Custom Configuration File**, **Alternative: Using Command-Line Parameters**). The custom configuration file is essentially a Python script, which supports all Python syntax such as loops, conditional statements, and list comprehensions. You can write model, dataset, summarizer, and other configurations into a single file—write once, reuse multiple times. + +All custom configuration file examples involved in this section have been uniformly stored in the `ais_bench/configs/accuracy_benchmark/` directory for easy reference and reuse: + +| Filename | Corresponding Scenario | +| --- | --- | +| [single_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/single_task_en.py) | Single-task evaluation | +| [multi_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_en.py) | Multi-task evaluation | +| [multi_task_parallel_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_parallel_en.py) | Multi-task parallel evaluation | +| [multi_task_resume_partial_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py) | Resumption after interruption & retesting of failed cases (partial tasks) | +| [ceval_merge_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/ceval_merge_en.py) | Merging sub-dataset inference | +| [fixed_prompts_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_en.py) | Fixed request count evaluation | +| [multi_repeat_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_repeat_en.py) | Multiple independent repeat inference | +| [inference_re_eval_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py) | Re-evaluation of inference results | + +> For a complete description of the custom configuration file syntax (including the top-level variables that can be defined, detailed field descriptions, advanced Python usage, etc.), please refer to 📚 [Running AISBench with a Custom Configuration File](../../advanced_tutorials/run_custom_config.md). The "Custom Configuration File Examples for Each Scenario" section also provides complete examples of 10 typical scenarios (such as service-oriented performance evaluation, synthetic dataset performance evaluation, steady-state performance evaluation, multi-turn dialogue performance evaluation, judge model evaluation, custom dataset evaluation, etc.). ## Other Functional Scenarios ### Re-evaluation of Inference Results @@ -241,25 +614,70 @@ graph LR; D --> E[Generate a summary report based on accuracy data] E --> F((Present results)) ``` - -Each link in the entire execution process is independently decoupled, and inference results can be re-evaluated repeatedly. If there is an issue with the accuracy data obtained from the first accuracy evaluation (e.g., failure to accurately extract valuable content from the response), you can modify the answer extraction method and perform re-evaluation of the inference results. The specific operations are as follows: +Each link in the entire execution process is independently decoupled, and inference results can be re-evaluated repeatedly. If there is an issue with the accuracy data obtained from the first accuracy evaluation (e.g., failure to accurately extract valuable content from the response), you can modify the answer extraction method and perform re-evaluation of the inference results. The specific operations are as follows. Assume the command used for the previous performance evaluation was: + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + ```bash -ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt +ais_bench ais_bench/configs/accuracy_benchmark/single_task_en.py ``` -And the timestamp of the saved results is `20250628_151326`. However, the accuracy data for 8 cases is incorrect, showing a score of 0: +At the same time, the timestamp of the saved results is `20250628_151326`. However, the accuracy data for 8 cases is incorrect, showing a score of 0: ```bash dataset version metric mode vllm_api_general_chat ----------------------- -------- -------- ----- ---------------------- demo_gsm8k 401e4c accuracy gen 00.00 ``` +Check `20250628_151326/predictions/vllm-api-general-chat/gsm8k.json` and find that the inference results actually contain the correct answers. + +**Re-evaluation steps:** + +1. Edit the custom configuration file (e.g., [inference_re_eval_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py)) to override the answer extraction function in the `eval_cfg` of the corresponding dataset according to actual needs (refer to the following example). The `pred_postprocessor` is responsible for extracting the answer from the model output and can be replaced or customized according to the actual situation. The complete example is as follows: + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +models = vllm_api_general_chat +# ...For other parameter configurations, please refer to the configuration file + +# Key: Replace or modify the implementation of the answer extraction function +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) +``` + +2. On the basis of the first accuracy evaluation command, add `--mode eval` and `--reuse {timestamp of the inference results to be reused}` to perform repeated re-evaluation (`--mode` and `--reuse` are common parameters; when using a custom configuration file, they can still be appended via the command line): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/inference_re_eval_en.py --mode eval --reuse 20250628_151326 +``` + +::: +:::{tab-item} Alternative: Using Command-Line Parameters +```bash +ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt +``` +At the same time, the timestamp of the saved results is `20250628_151326`. However, the accuracy data for 8 cases is incorrect, showing a score of 0: +```bash +dataset version metric mode vllm_api_general_chat +----------------------- -------- -------- ----- ---------------------- +demo_gsm8k 401e4c accuracy gen 00.00 +``` Check `20250628_151326/predictions/vllm-api-general-chat/gsm8k.json` and find that the inference results actually contain the correct answers. At this point, you can modify the configuration file corresponding to the `gsm8k_gen_4_shot_cot_chat_prompt` dataset task. Use the `--search` command to query the path of the corresponding configuration file: ```bash ais_bench --datasets gsm8k_gen_4_shot_cot_chat_prompt --search ``` - The configuration file path will be displayed as follows: ```bash ╒═════════════╤═══════════════════════════════════════╤═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ @@ -267,6 +685,7 @@ The configuration file path will be displayed as follows: ╞═════════════╪═══════════════════════════════════════╪═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ │ --datasets │ gsm8k_gen_4_shot_cot_chat_prompt │ /your_workspace/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_chat_prompt.py │ ╘═════════════╧═══════════════════════════════════════╧═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ + ``` Open `gsm8k_gen_4_shot_cot_chat_prompt.py` and replace or modify the answer extraction function: @@ -281,9 +700,14 @@ gsm8k_eval_cfg = dict(evaluator=dict(type=Gsm8kEvaluator), pred_postprocessor=dict(type=gsm8k_postprocess), # Replace or modify the implementation of the answer extraction function dataset_postprocessor=dict(type=gsm8k_dataset_postprocess)) # ...... + ``` You can add `--mode eval` and `--reuse {timestamp of the inference results to be reused}` to the command of the first accuracy evaluation to perform repeated re-evaluation: ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode eval --reuse 20250628_151326 -``` \ No newline at end of file + +``` + +::: +:::: \ No newline at end of file diff --git a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark_local.md b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark_local.md index fc2d2d46..9ece0638 100644 --- a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark_local.md +++ b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark_local.md @@ -2,29 +2,210 @@ Load models and datasets in a local environment, compare outputs with reference answers through a unified inference process, and evaluate the inherent accuracy of the model. Customize parameters such as batch size and sequence length, applicable to the **Huggingface Transformers** inference framework. ## Test Preparation -Before performing local model inference, the following conditions must be met: +Before performing service-oriented inference, the following conditions must be met: - Available model weights: Ensure that the model weight files to be tested are already available locally. Open-source weights can be obtained from 🔗 [Hugging Face Community](https://huggingface.co/models). -- Dataset task preparation: Select a dataset from 📚 [Open-Source Datasets](../all_params/datasets.md#open-source-datasets), and choose the dataset task to execute in the "detailed introduction" document corresponding to the dataset. Prepare the dataset files according to the "detailed introduction" document of the selected dataset task. It is recommended to manually place the open-source dataset in the default directory `ais_bench/datasets/`, and the program will automatically load the dataset files during task execution. +- Dataset task preparation: Select a dataset from 📚 [Open-Source Datasets](../../get_started/datasets.md#open-source-datasets), and choose the dataset task to execute in the "detailed introduction" document corresponding to the dataset. Prepare the dataset files according to the "detailed introduction" document of the selected dataset task. It is recommended to manually place the open-source dataset in the default directory `ais_bench/datasets/`, and the program will automatically load the dataset files during task execution. - Model task preparation: Select the model task to execute from 📚 [Local Model Backend](../all_params/models.md#local-model-backend). ## Main Functions -The main functions in the pure model accuracy evaluation scenario are similar to those in the service-oriented accuracy evaluation scenario. + +The main functions in the pure model accuracy evaluation scenario are similar to those in the service-oriented accuracy evaluation scenario, but the model task needs to be replaced with a local HuggingFace model task (such as [`HuggingFacewithChatTemplate`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/models/huggingface_chat_model.py) or [`HuggingFaceBaseModel`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/models/huggingface_base_model.py)). ### Pure Model Multi-Task Evaluation -Refer to [Usage of Service-Oriented Accuracy Multi-Task Evaluation](accuracy_benchmark.md#multi-task-evaluation). + +Supports simultaneous configuration of multiple dataset tasks through a single command for batch evaluation. For a complete example, refer to [multi_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/multi_task_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + +datasets = gsm8k_datasets + aime2024_datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + # ...For other parameter configurations, see the configuration file + ) +] +``` + +Execution command: + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/multi_task_en.py +``` + +#### Custom Model-Dataset Pairings (Optional) + +By default, the `models` list and `datasets` list in the above configuration will automatically be combined in a Cartesian product, and the number of sub-tasks is the number of models × the number of datasets (1 × 2 = 2 in this example). If you want to precisely control which models are paired with which datasets (for example, only let the model run a subset of datasets), you can explicitly declare the pairing relationship through the `model_dataset_combinations` field in the configuration file: + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + +datasets = gsm8k_datasets + aime2024_datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + ) +] + +# Key: Precisely control pairings through model_dataset_combinations +# The following example generates only 1 sub-task (the Cartesian product would generate 2): +# - hf-chat-model + gsm8k +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), +] +``` + +> ⚠️ **Note**: The unique identifier of a model or dataset is determined by the `abbr` field. In the same configuration file, combinations where models or datasets with the same `abbr` appear repeatedly will be considered duplicate tasks and will be skipped. When reusing model/dataset configurations through methods like `.copy()`, you must explicitly modify `abbr` to ensure uniqueness. For details, refer to 📚 [Custom Model-Dataset Combinations](../../advanced_tutorials/run_custom_config.md#custom-model-and-dataset-combinations). + +> 💡 For detailed usage, you can also refer to [Usage of Service-Oriented Accuracy Multi-Task Evaluation](accuracy_benchmark.md#multi-task-evaluation). ### Pure Model Multi-Task Parallel Evaluation -Refer to [Usage of Service-Oriented Accuracy Multi-Task Parallel Evaluation](accuracy_benchmark.md#multi-task-parallel-evaluation). + +Supports multi-task parallelism through the [`--max-num-workers`](../all_params/cli_args.md#common-parameters) command-line parameter. The configuration file example is exactly the same as [Pure Model Multi-Task Evaluation](#pure-model-multi-task-evaluation), the only difference is the execution command. + +Execution command (taking `max-num-workers 4` as an example): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/multi_task_en.py --max-num-workers 4 +``` + > ⚠️ Note: Multi-task parallel evaluation in pure model accuracy evaluation will occupy different GPU units. The number of GPU units required for parallel tasks should be less than or equal to the total number of available GPUs. +> 💡 For detailed usage, you can also refer to [Usage of Service-Oriented Accuracy Multi-Task Parallel Evaluation](accuracy_benchmark.md#multi-task-parallel-evaluation). + ### Pure Model Resumption After Interruption -During the pure model accuracy evaluation, if the task is interrupted, you can use the `--reuse` parameter to specify the task timestamp directory to continue the unfinished inference task, realizing breakpoint resumption. This function does not require re-running all tasks, but only performs supplementary inference on the unfinished parts. For details on usage, refer to [Usage of Service-Oriented Accuracy Resumption After Interruption](accuracy_benchmark.md#resumption-after-interruption-&-retesting-of-failed-cases). + +During the pure model accuracy evaluation, if the task is interrupted, you can use the `--reuse` parameter to specify the task timestamp directory to continue the unfinished inference task, realizing breakpoint resumption. This function does not require re-running all tasks, but only performs supplementary inference on the unfinished parts. + +First execution command: + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/single_task_en.py +``` + +Specify the task timestamp directory through the `--reuse` parameter to continue (`--reuse` is a common parameter, and can still be appended through the command line when using a custom configuration file): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/single_task_en.py --reuse 20250628_151326 +``` + > ⚠️ Note: Currently, pure model accuracy evaluation does not support automatic retesting of failed cases. +> 💡 For detailed usage, you can also refer to [Usage of Service-Oriented Accuracy Resumption After Interruption](accuracy_benchmark.md#resumption-after-interruption--retesting-of-failed-cases). + ### Pure Model Merged Sub-Dataset Inference -Refer to [Usage of Service-Oriented Accuracy Merged Sub-Dataset Inference](accuracy_benchmark.md#merging-sub-dataset-inference). + +Supports merging datasets containing multiple small-scale sub-datasets into a single task for unified evaluation. For a complete example, refer to [ceval_merge_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + # ...For other parameter configurations, see the configuration file + ) +] +``` + +Execution command (`--merge-ds` is a common parameter, and can still be appended through the command line when using a custom configuration file): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py --merge-ds +``` + +> 💡 For detailed usage, you can also refer to [Usage of Service-Oriented Accuracy Merged Sub-Dataset Inference](accuracy_benchmark.md#merging-sub-dataset-inference). + +## Implementation via Custom Configuration Files + +> 💡 All the above functional scenarios (multi-task evaluation, multi-task parallel, resumption after interruption, merged sub-dataset, etc.) can be implemented through the [Custom Configuration File](../../advanced_tutorials/run_custom_config.md) approach. The configuration file is essentially a Python script, which supports all Python syntaxes such as loops, conditional judgments, and list comprehensions. Model, dataset, summarizer, and other configurations can be written into one file for one-time writing and multiple reuse. + +All custom configuration file examples involved in this section are uniformly stored in the `ais_bench/configs/accuracy_benchmark_local/` directory for easy reference and reuse: + +| File Name | Corresponding Scenario | +| --- | --- | +| [single_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/single_task_en.py) | Single-Task Evaluation | +| [multi_task_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/multi_task_en.py) | Pure Model Multi-Task Evaluation / Multi-Task Parallel Evaluation | +| [ceval_merge_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/ceval_merge_en.py) | Merged Sub-Dataset Inference | +| [inference_re_eval_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py) | Re-Evaluation of Pure Model Inference Results | + +For details, refer to the "Pure Model Accuracy Evaluation" example in [Running AISBench via Custom Configuration Files](../../advanced_tutorials/run_custom_config.md#custom-configuration-file-examples-for-each-scenario). ## Other Functions + ### Re-Evaluation of Pure Model Inference Results -Refer to [Usage of Service-Oriented Accuracy Re-Evaluation of Inference Results](accuracy_benchmark.md#re-evaluation-of-inference-results). \ No newline at end of file + +For a complete example, refer to [inference_re_eval_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # Replace with the actual local model weight path + tokenizer_path='THUDM/chatglm-6b', + # ...For other parameter configurations, see the configuration file + ) +] + +# Key: Replace or modify the answer extraction function implementation +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) +``` + +Execution command (`--mode eval` and `--reuse` are common parameters, and can still be appended through the command line when using a custom configuration file): + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/inference_re_eval_en.py --mode eval --reuse 20250628_151326 +``` + +> 💡 For detailed usage, you can also refer to [Usage of Service-Oriented Accuracy Re-Evaluation of Inference Results](accuracy_benchmark.md#re-evaluation-of-inference-results). \ No newline at end of file diff --git a/docs/source_en/base_tutorials/scenes_intro/home.md b/docs/source_en/base_tutorials/scenes_intro/home.md index 3d7a3784..93147190 100644 --- a/docs/source_en/base_tutorials/scenes_intro/home.md +++ b/docs/source_en/base_tutorials/scenes_intro/home.md @@ -7,7 +7,7 @@ - **Model Tasks and Dataset Tasks Supported by This Scenario**: - **Model Tasks**: 📚 [Service-Oriented Inference Backend](../all_params/models.md#service-oriented-inference-backend) - - **Dataset Tasks**: 📚 [Open-Source Datasets](../all_params/datasets.md#open-source-datasets) and 📚 [Custom Datasets](../all_params/datasets.md#custom-datasets) + - **Dataset Tasks**: 📚 [Open-Source Datasets](../../get_started/datasets.md#open-source-datasets) and 📚 [Custom Datasets](../../get_started/datasets.md#custom-datasets) - **Constraint**: Currently, PPL mode accuracy evaluation tasks only support `vllm_api_general` and `vllm_api_general_chat` model configurations; other configurations are not supported. @@ -20,7 +20,7 @@ After selecting the **model task** and **dataset task** according to your usage - **Supported Items**: - **Model Tasks**: 📚 [Local Model Backend](../all_params/models.md#local-model-backend) - - **Dataset Tasks**: 📚 [Open-Source Datasets](../all_params/datasets.md#open-source-datasets) and 📚 [Custom Datasets](../all_params/datasets.md#custom-datasets) + - **Dataset Tasks**: 📚 [Open-Source Datasets](../../get_started/datasets.md#open-source-datasets) and 📚 [Custom Datasets](../../get_started/datasets.md#custom-datasets) - **Constraint**: PPL mode evaluation tasks are not supported. @@ -34,7 +34,7 @@ After selecting the **model task** and **dataset task** according to your usage - **Supported Items**: - **Model Tasks**: Streaming interface types in 📚 [Service-Oriented Inference Backend](../all_params/models.md#service-oriented-inference-backend) - - **Dataset Tasks**: All data types in 📚 [Supported Dataset Types](../all_params/datasets.md#supported-dataset-types) + - **Dataset Tasks**: All data types in 📚 [Supported Dataset Types](../../get_started/datasets.md#supported-dataset-types) - **Note**: The cache size occupied by performance evaluation is proportional to the context length of requests and the number of requests, so it usually increases positively with the evaluation duration. diff --git a/docs/source_en/base_tutorials/scenes_intro/performance_benchmark.md b/docs/source_en/base_tutorials/scenes_intro/performance_benchmark.md index 0ff427b7..726e524c 100644 --- a/docs/source_en/base_tutorials/scenes_intro/performance_benchmark.md +++ b/docs/source_en/base_tutorials/scenes_intro/performance_benchmark.md @@ -1,40 +1,97 @@ -# Guide to Service-Oriented Performance Evaluation -## Introduction -AISBench Benchmark provides service-oriented performance evaluation capabilities. For streaming inference scenarios, it systematically evaluates key performance indicators of model services in real-world deployment environments—such as response latency (e.g., TTFT, Inter-Token Latency), throughput capacity (e.g., QPS, TPUT), and concurrent processing capability—by accurately recording the send time of each request, the return time of each stage, and the response content. +# Service-Oriented Performance Evaluation +Send batch requests to the service through a unified request interface to evaluate the service performance of the model in actual deployment scenarios. The request sending mode and request data can be customized to obtain performance indicators such as throughput and latency. It supports two deployment frameworks: **vLLM** and **vLLM-Ascend**, and provides complete performance analysis reports. -Users can flexibly control request content, request intervals, concurrent quantities, and other parameters by configuring service-oriented backend parameters to adapt to different evaluation scenarios (e.g., low-concurrency latency-sensitive scenarios, high-concurrency throughput-priority scenarios). The evaluation supports automated execution and outputs structured results, facilitating horizontal comparison of service performance differences across different models, deployment solutions, and hardware configurations. +## Quick Start +### Prerequisite + +The performance evaluation requires **first preparing a service environment** (i.e., a service program that provides OpenAI-compatible interfaces). + +Here is the reference service startup method (vLLM OpenAI-compatible service): + +```bash +vllm serve Qwen/Qwen2.5-7B-Instruct --port 8080 --max-model-len 4096 +``` + +Wait for the service to start successfully (the port shows that the service process is listening), then use the following configuration file for evaluation. + +:::{admonition} Recommended Practice +:class: tip + +For details on how to write the following custom configuration file, please refer to [Custom Configuration Files](../../advanced_tutorials/run_custom_config.md#custom-configuration-file-examples-for-each-scenario). Using a custom configuration file can support richer custom parameter configurations, such as supporting `num_prompts`, `request_rate` (QPS sending mode), etc. +::: + +### One-Click Evaluation + +After the service is started, the following **custom configuration file** can be used to send the `ShareGPT` dataset to the service at `request_rate=1` (QPS) for performance evaluation. + +- Configuration file content: + ```python + from mmengine.config import read_base + + with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + + models = vllm_api_stream_chat + models[0]["host_ip"] = "localhost" + models[0]["host_port"] = 8080 + models[0]["max_out_len"] = 1024 + models[0]["batch_size"] = 50 + models[0]["request_rate"] = 1 # Request sending frequency: send 1 request to the server every 1/request_rate seconds; if less than 0.001, all requests are sent at once + models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) # When testing performance and needing to limit the output length, ignore_eos must be set to True + + work_dir = "outputs/default/" + ``` + +- Execution command (you can also append `--num-prompts N` to limit the number of requests sent): + ```bash + ais_bench ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py + ``` + +After the task is completed, you can view the performance result report in the `summary/` directory under the task output directory. -## Quick Start for Service-Oriented Performance Evaluation ### Command Meaning -The meaning of the AISBench service-oriented performance evaluation command is the same as explained in 📚 [Tool Quick Start/Command Meaning](../../get_started/quick_start.md#command-meaning). On this basis, you need to add `--mode perf` or `-m perf` to enter the performance evaluation scenario. Take the following AISBench command as an example: + +The meaning of the AISBench service-oriented performance evaluation command is the same as explained in 📚 [Tool Quick Start/Command Meaning](../../get_started/quick_start.md#start-evaluation-choose-one-of-two-methods). On this basis, you need to add `--mode perf` or `-m perf` to enter the performance evaluation scenario. Take the following AISBench command as an example: + ```shell ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer default_perf --mode perf ``` + Among them: + - `--models` specifies the model task, i.e., the `vllm_api_stream_chat` model task. - `--datasets` specifies the dataset task, i.e., the `demo_gsm8k_gen_4_shot_cot_chat_prompt` dataset task. -- `--summarizer` specifies the result presentation task, i.e., the `default_perf` result presentation task (if `--summarizer` is not specified, the `default_perf` task is used by default in accuracy evaluation scenarios). It is generally used by default and does not need to be specified in the command line; subsequent commands will omit this parameter. +- `--summarizer` specifies the result presentation task, i.e., the `default_perf` result presentation task (if `--summarizer` is not specified, the `default_perf` task is used by default in performance evaluation scenarios). It is generally used by default and does not need to be specified in the command line; subsequent commands will omit this parameter. ### Task Meaning Query (Optional) + Specific information (introduction, usage constraints, etc.) about the selected model task `vllm_api_stream_chat`, dataset task `demo_gsm8k_gen_4_shot_cot_chat_prompt`, and result presentation task `default_perf` can be queried from the following links: + - `--models`: 📚 [Service-Oriented Inference Backend](../all_params/models.md#service-oriented-inference-backend) -- `--datasets`: 📚 [Open-Source Datasets](../all_params/datasets.md#open-source-datasets) → 📚 [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README_en.md) +- `--datasets`: 📚 [Open-Source Datasets](../../get_started/datasets.md#open-source-datasets) → 📚 [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README_en.md) - `--summarizer`: 📚 [Result Summary Tasks](../all_params/summarizer.md#supported-result-summary-tasks) ### Preparations Before Running the Command + - `--models`: To use the `vllm_api_stream_chat` model task, you need to prepare an inference service that supports the `v1/chat/completions` sub-service. You can refer to 🔗 [VLLM Launch OpenAI-Compatible Server](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server) to start the inference service. - `--datasets`: To use the `demo_gsm8k_gen_4_shot_cot_chat_prompt` dataset task, you need to prepare the GSM8K dataset, which can be downloaded from 🔗 [GSM8K Dataset Compressed Package Provided by OpenCompass](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip). Deploy the unzipped `gsm8k/` folder to the `ais_bench/datasets` folder in the root path of the AISBench evaluation tool. -# Modification of Configuration Files Corresponding to Tasks +### Modification of Configuration Files Corresponding to Tasks + Each model task, dataset task, and result presentation task corresponds to a configuration file. The content of these configuration files must be modified before executing commands. The paths of these configuration files can be queried by adding `--search` to the original AISBench command. For example: + ```shell # Note: Whether to add "--mode perf" to the search command does not affect the search results ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode perf --search ``` + > ⚠️ **Note**: Executing a command with the `search` option will print the absolute path of the configuration file corresponding to the task. Executing the query command will yield the following results: + ```shell ╒══════════════╤═══════════════════════════════════════╤════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ │ Task Type │ Task Name │ Config File Path │ @@ -43,12 +100,12 @@ Executing the query command will yield the following results: ├──────────────┼───────────────────────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ │ --datasets │ demo_gsm8k_gen_4_shot_cot_chat_prompt │ /your_workspace/benchmark/ais_bench/benchmark/configs/datasets/demo/demo_gsm8k_gen_4_shot_cot_chat_prompt.py │ ╘══════════════╧═══════════════════════════════════════╧════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ - ``` -- The dataset task configuration file `demo_gsm8k_gen_4_shot_cot_chat_prompt.py` in the quick start does not require additional modifications. For an introduction to the content of the dataset task configuration file, please refer to 📚 [Configure Open-Source Datasets](../all_params/datasets.md#configure-open-source-datasets) +- The dataset task configuration file `demo_gsm8k_gen_4_shot_cot_chat_prompt.py` in the quick start does not require additional modifications. For an introduction to the content of the dataset task configuration file, please refer to 📚 [Configure Open-Source Datasets](../../get_started/datasets.md#configuring-open-source-datasets) The model configuration file `vllm_api_stream_chat.py` contains configuration content related to model operation and needs to be modified according to actual conditions. The content that needs to be modified in the quick start is marked with comments. + ```python from ais_bench.benchmark.models import VLLMCustomAPIChatStream @@ -79,14 +136,14 @@ models = [ ``` -# Execute Commands +### Execute Commands After modifying the configuration files, execute the command to start the service performance evaluation: ```bash ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt -m perf ``` -## View Task Execution Details After executing the AISBench command, the status of the ongoing task will be displayed on a real-time refreshing dashboard in the command line (press the "P" key on the keyboard to stop refreshing for copying dashboard information, and press "P" again to resume refreshing). For example: + ``` Base path of result&log : outputs/default/20251106_103326 Task Progress Table (Updated at: 2025-11-06 10:34:41) @@ -98,21 +155,24 @@ Press Up/Down arrow to page, 'P' to PAUZE/RESUME screen refresh, 'Ctrl + C' to +=================================+===========+=================================================+=============+=============+================================================+================================================+ | vllm-api-stream-chat/demo_gsm8k | 744887 | [########### ] 3/8 [0.1 it/s] | 0:00:54 | inferencing | logs/infer/vllm-api-stream-chat/demo_gsm8k.out | {'POST': 4, 'RECV': 3, 'FINISH': 3, 'FAIL': 0} | +---------------------------------+-----------+-------------------------------------------------+-------------+-------------+------------------------------------------------+------------------------------------------------+ -` - ``` Detailed logs of task execution will be continuously saved to the default output path, which is displayed on the real-time refreshing dashboard as `Log Path`. The `Log Path` (`logs/infer/vllm-api-stream-chat/demo_gsm8k.out`) is a subpath under the `Base path` (`outputs/default/20251106_103326`). Taking the above dashboard information as an example, the path to the detailed logs of task execution is: + ```shell # {Base path}/{Log Path} outputs/default/20251106_103326/logs/infer/vllm-api-stream-chat/demo_gsm8k.out ``` > 💡 If you want detailed logs to be printed directly during execution, you can add `--debug` to the command: -`ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt -m perf --debug` +> +> ```bash +> ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt -m perf --debug +> ``` -# View Performance Results -An example of performance results printed on the screen is as follows: +### View Performance Results + +The on-screen performance results are displayed as follows: ```bash [2025-11-06 10:35:43,667] [ais_bench] [INFO] Performance Results of task: vllm-api-stream-chat/demo_gsm8k: @@ -164,344 +224,676 @@ An example of performance results printed on the screen is as follows: ╘══════════════════════════╧═════════╧══════════════════╛ [2025-11-06 10:35:43,672] [ais_bench] [INFO] Performance Result files located in outputs/default/20251106_103326/performances/vllm-api-stream-chat. ``` -💡 For the meaning of specific performance parameters, please refer to 📚 [Explanation of Performance Evaluation Results](../results_intro/performance_metric.md) -# View Performance Details -After executing the AISBench command, more details of task execution will eventually be saved to the `Base path` (`outputs/default/20251106_103326`) +💡 For the meaning of specific performance parameters, refer to 📚 [Performance Evaluation Results Description](../results_intro/performance_metric.md) + +### Performance Details View + +After executing the AISBench command, more details of task execution will eventually be saved to the `Base path` (`outputs/default/20251106_103326`). + +After the command execution ends, the task execution details in `outputs/default/20250628_151326` are as follows: -After the command execution is completed, the details of task execution in `outputs/default/20250628_151326` are as follows: ```shell 20251106_103326 # Unique directory generated based on timestamp for each experiment -├── configs # Automatically stored all dumped configuration files -├── logs # Logs during execution; if --debug is added to the command, no process logs will be saved to disk (all will be printed directly) -│ └── performance/ # Log files of the inference phase +├── configs # Automatically stored configuration files of all dumped configurations +├── logs # Logs during execution; if --debug is added to the command, there will be no on-disk logs (all printed directly) +│ └── performance/ # Log files from the inference phase └── performance # Performance evaluation results -│ └── vllm-api-stream-chat/ # Name of "service model configuration", corresponding to the abbr parameter of models in the model task configuration file -│ ├── demo_gsm8k.csv # Single-request performance output (CSV), consistent with the Performance Parameters table in the on-screen performance results -│ ├── demo_gsm8k.json # End-to-end performance output (JSON), consistent with the Common Metric table in the on-screen performance results -│ ├── demo_gsm8k_plot.html # Request concurrency visualization report (HTML) -│ └── ...... -``` -💡 It is recommended to open the request concurrency visualization report `demo_gsm8k_plot.html` using browsers such as Chrome or Edge. You can view the latency of each request and the number of concurrent service times perceived by the client at each moment: - ![full_plot_example.img](../../img/request_concurrency/full_plot_example.png) + └── vllm-api-stream-chat/ # "Service-oriented model configuration" name, corresponding to the abbr parameter of models in the model task configuration file + ├── demo_gsm8k.csv # Single-request performance output (CSV), consistent with the Performance Parameters table in the on-screen performance results + ├── demo_gsm8k.json # End-to-end performance output (JSON), consistent with the Common Metric table in the on-screen performance results + ├── demo_gsm8k_plot.html # Request concurrency visualization report (HTML) + └── ...... +``` + +💡 The `demo_gsm8k_plot.html` request concurrency visualization report is recommended to be opened with browsers such as Chrome or Edge, where you can see the latency of each request and the number of concurrent service requests perceived by the client at each moment: +![full_plot_example](../../img/request_concurrency/full_plot_example.png) + For instructions on using this HTML visualization file, please refer to 📚 [Instructions for Using Performance Test Visualization Concurrency Graphs](../results_intro/performance_visualization.md) -# Preconditions for Service-Oriented Performance Evaluation -Before conducting service-oriented inference, the following conditions must be met: +## Test Preparation + +Before performing service-oriented inference, the following conditions must be met: + +- Available model weights: Ensure that the model weight files to be tested are already available locally. Open-source weights can be obtained from 🔗 [Hugging Face Community](https://huggingface.co/models). +- Service environment preparation: Ensure that the model inference service is started through inference engines such as vLLM/vLLM-Ascend. The startup parameters need to ensure that the server's `max-model-len` and other configurations can accommodate the length of the prompt and output to be sent. +- Dataset preparation: Select a dataset suitable for performance evaluation scenarios, such as `ShareGPT`. For details, refer to 📚 [Datasets](../../get_started/datasets.md#open-source-datasets). The user can also prepare a custom dataset, see [Custom Dataset Evaluation](../../advanced_tutorials/custom_dataset.md). +- Model task preparation: Select the model task to execute from 📚 [vLLM Model Backend](../all_params/models.md#service-oriented-inference-backend). + +:::{admonition} Service Startup Precautions +:class: warning + +- It is recommended to ensure that the service is fully started before starting the evaluation task, otherwise the task may fail due to connection failure. +- When the service fails, the tool will record the failure cause in the logs, and the user can troubleshoot based on the error information. +::: + +## Main Functional Scenarios -- **Accessible Service-Oriented Model Service**: Ensure the service process can be directly accessed in the current environment. -- **Dataset Preparation**: - - **Open-Source Dataset**: Select a dataset from 📚 [Open-Source Datasets](../all_params/datasets.md#开源数据集), and choose the dataset task to execute from the "Detailed Introduction" document corresponding to the dataset. Prepare the dataset files by referring to the "Detailed Introduction" document of the selected dataset task. It is recommended to manually place the open-source dataset in the default directory `ais_bench/datasets/`; the program will automatically load the dataset files during task execution. - - **Randomly Synthesized Dataset**: Select `synthetic_gen` as the dataset task, and refer to 📚 [Randomly Synthesized Dataset](../../advanced_tutorials/synthetic_dataset.md) for other configurations. - - **Custom Dataset**: No need to specify a dataset task; refer to 📚 [Custom Dataset](../../advanced_tutorials/custom_dataset.md) for other configurations. -- **Service-Oriented Model Backend Configuration**: From [Service-Oriented Inference Backend](../all_params/models.md#服务化推理后端), select a sub-service with the interface type of `Streaming Interface` (⚠️ Other types are not supported). +### Single-Task Performance Evaluation +#### Using a Custom Configuration File (Recommended) -# Main Functional Scenarios -## Single-Task Evaluation -Refer to [Quick Start for Service-Oriented Performance Evaluation](#服务化性能测评快速入门) +:::{tab-set} +:::{tab-item} ⭐ Custom Configuration File -## Multi-Task Evaluation -Supports simultaneous configuration of multiple models or multiple dataset tasks, enabling batch evaluation through a single command. This is suitable for serial execution of multiple test commands. +The configuration file content is consistent with the [Quick Start One-Click Evaluation](#one-click-evaluation). + +Execution command: + +```bash +ais_bench ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py +``` + +::: +:::{tab-item} Alternative: Command-Line Parameters + +You can also use the preset configuration file for one-click evaluation (the service address needs to be configured in the model configuration file [vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py)): -### Command Description -Users can specify multiple configuration tasks via the `--models` and `--datasets` parameters. The number of subtasks is the product of the number of tasks configured in `--models` and `--datasets`—that is, one model configuration and one dataset configuration form a subtask. Example: ```bash -ais_bench --models vllm_api_general_stream vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_str --mode perf +ais_bench --models vllm_api_stream_chat --datasets sharegpt_gen -m perf +``` + +::: +::: + +#### Specifying Custom Performance Dimensions + +AISBench supports users in customizing the statistical items of performance reports. In the custom configuration file, you can customize the `summarizer` and modify the `stats_list` field of its calculator to control which statistical dimensions (e.g., `Average`, `Min`, `Max`, `Median`, `P75`, `P90`, `P99`) are computed for each performance parameter (E2EL, TTFT, TPOT, etc.) in the summary report. + +Commonly used statistical items include: + +| Statistic | Description | +| --- | --- | +| `Average` | Average value | +| `Min` | Minimum value | +| `Max` | Maximum value | +| `Median` | Median value | +| `P75` | 75th percentile | +| `P90` | 90th percentile | +| `P95` | 95th percentile | +| `P99` | 99th percentile | + +The following is an example configuration that contains the most commonly used statistical items: + +```python +from mmengine.config import read_base +from ais_bench.benchmark.summarizers import DefaultPerfSummarizer +from ais_bench.benchmark.calculators import DefaultPerfMetricCalculator + +with read_base(): + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Key: customize the stats_list of the performance summarizer calculator +summarizer = dict( + attr="performance", + type=DefaultPerfSummarizer, + calculator=dict( + type=DefaultPerfMetricCalculator, + stats_list=["Average", "Min", "Max", "Median", "P75", "P90", "P95", "P99"], + ), +) + +models = vllm_api_stream_chat ``` -The above command specifies 2 model tasks (`vllm_api_general_stream` `vllm_api_stream_chat`) and 2 dataset tasks (`gsm8k_gen_4_shot_cot_str` `aime2024_gen_0_shot_str`), and will execute the following 4 combined performance test tasks: + +For a complete runnable example, refer to [performance_re_eval.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_re_eval.py). + +### Multi-Task Performance Evaluation + +Supports simultaneous configuration of multiple datasets or multiple sending parameter combinations (such as different `request_rate`s) for performance evaluation through a single command, facilitating the comparison of performance indicators of different sending strategies. + +#### Description of Sub-task Combinations + +In multi-task evaluation scenarios, the number of subtasks is the product of the number of tasks configured by `models` and the number of tasks configured by `datasets`—that is, one model configuration and one dataset configuration form a subtask. + +The following example simultaneously evaluates 2 model tasks (`vllm_api_general_stream`, `vllm_api_stream_chat`) and 2 dataset tasks (`gsm8k_gen_4_shot_cot_str`, `aime2024_gen_0_shot_str`), and will execute the following 4 combined performance test tasks: + + [vllm_api_general_stream](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py) Model Task + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) Dataset Task + [vllm_api_general_stream](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py) Model Task + [aime2024_gen_0_shot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_str) Dataset Task + [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) Model Task + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) Dataset Task + [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) Model Task + [aime2024_gen_0_shot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_str.py) Dataset Task -### Modify Configuration Files Corresponding to Tasks -The actual paths of the configuration files corresponding to model tasks and dataset tasks can be queried by executing the command with the `--search` parameter: -```bash -ais_bench --models vllm_api_general_stream vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_str --mode perf --search +#### Custom Model-Dataset Pairings (Optional) + +By default, the `models` list and `datasets` list in the configuration file are automatically combined as a Cartesian product, with the number of subtasks equal to the number of models × the number of datasets (in this example, 2 × 2 = 4). If you want to precisely control which models are paired with which datasets (e.g., letting some models only run on some datasets to avoid meaningless combinations), you can explicitly declare the pairing relationship in the configuration file via the `model_dataset_combinations` field: + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_str import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets +models = vllm_api_general_stream + vllm_api_stream_chat + +# Key: Precisely control pairings via model_dataset_combinations +# The following example generates only 2 subtasks (the Cartesian product would generate 4): +# - vllm_api_general_stream + gsm8k_gen_4_shot_cot_str +# - vllm_api_stream_chat + aime2024_gen_0_shot_str +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), +] ``` -The following configuration files to be modified will be displayed: -```bash -╒═════════════╤══════════════════════════╤═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ -│ Task Type │ Task Name │ Config File Path │ -╞═════════════╪══════════════════════════╪═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ -│ --models │ vllm_api_general_stream │ /your_workspace/benchmark_test/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py │ -├─────────────┼──────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ -│ --models │ vllm_api_stream_chat │ /your_workspace/benchmark_test/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py │ -├─────────────┼──────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ -│ --datasets │ gsm8k_gen_4_shot_cot_str │ /your_workspace/benchmark_test/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py │ -├─────────────┼──────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ -│ --datasets │ aime2024_gen_0_shot_str │ /your_workspace/benchmark_test/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_str.py │ -╘═════════════╧══════════════════════════╧═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ -``` -- Refer to 📚 [Description of Service-Oriented Inference Backend Configuration Parameters](../all_params/models.md#服务化推理后端配置参数说明) to configure the configuration files corresponding to the model tasks `vllm_api_general_stream` and `vllm_api_stream_chat` according to the actual situation. -- Refer to 📚 [Configure Open-Source Dataset](../all_params/datasets.md#配置开源数据集) to configure the configuration files corresponding to the dataset tasks `gsm8k_gen_4_shot_cot_str` and `aime2024_gen_0_shot_str` according to the actual situation. **Note**: If the dataset is placed in the default directory `ais_bench/datasets/`, no configuration is generally required. - -### Execute the Evaluation Command -Execute the command: -```bash -ais_bench --models vllm_api_general_stream vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_str --mode perf + +> ⚠️ **Note**: The unique identifier for models and datasets is determined by the `abbr` field. In the same configuration file, repeated combinations of models or datasets with the same `abbr` will be treated as duplicate tasks and skipped. When reusing model/dataset configurations via methods such as `.copy()`, the `abbr` must be explicitly modified to ensure uniqueness. See 📚 [Custom Model and Dataset Combinations](../../advanced_tutorials/run_custom_config.md#custom-model-and-dataset-combinations) for details. + +#### Multi-Task Parallel + +Supports multi-task parallelism through the [`--max-num-workers`](../all_params/cli_args.md#common-parameters) command-line parameter. Different sub-tasks will be distributed to different processes for parallel execution. + +#### Specifying Multiple Datasets for Performance Evaluation + +:::{tab-set} +:::{tab-item} ⭐ Custom Configuration File + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" ``` -During execution, a timestamp directory will be created under the path specified by 📚 [`--work-dir`](../all_params/cli_args.md#公共参数) (the default path is `outputs/default/`) to save execution details. -After the 4 performance evaluation tasks are completed, the performance results of all 4 tasks will be printed at once: +Execution command: + ```bash -[2025-11-06 10:35:43,667] [ais_bench] [INFO] Performance Results of task: vllm-api-stream-chat/demo_gsm8k: -╒══════════════════════════╤═════════╤═════════════════╤═══════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤══════╕ -│ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ -╞══════════════════════════╪═════════╪═════════════════╪═══════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪══════╡ -│ E2EL │ total │ 2754.0929 ms │ 2189.0804 ms │ 3366.1463 ms │ 2753.1668 ms │ 3048.2929 ms │ 3222.573 ms │ 3303.3894 ms │ 1319 │ -...... -╒══════════════════════════╤═════════╤═══════════════════╕ -│ Common Metric │ Stage │ Value │ -╞══════════════════════════╪═════════╪═══════════════════╡ -│ Benchmark Duration │ total │ 38039.9928 ms │ -...... -[2025-11-06 11:11:33,468] [ais_bench] [INFO] Performance Result files located in outputs/default/20251106_110904/performances/vllm-api-general-stream. -[2025-11-06 11:11:33,468] [ais_bench] [INFO] Performance Results of task: vllm-api-general-stream/aime2024: -╒══════════════════════════╤═════════╤═════════════════╤════════════════╤════════════════╤═══════════════╤═════════════════╤═════════════════╤═════════════════╤═════╕ -│ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ -╞══════════════════════════╪═════════╪═════════════════╪════════════════╪════════════════╪═══════════════╪═════════════════╪═════════════════╪═════════════════╪═════╡ -│ E2EL │ total │ 2868.1822 ms │ 2277.1049 ms │ 3307.2084 ms │ 2941.6767 ms │ 3158.5361 ms │ 3220.2141 ms │ 3307.0174 ms │ 30 │ -...... -╒══════════════════════════╤═════════╤═══════════════════╕ -│ Common Metric │ Stage │ Value │ -╞══════════════════════════╪═════════╪═══════════════════╡ -│ Benchmark Duration │ total │ 3346.9782 ms │ -...... -[2025-11-06 11:11:33,471] [ais_bench] [INFO] Performance Result files located in outputs/default/20251106_110904/performances/vllm-api-general-stream. -[2025-11-06 11:11:33,471] [ais_bench] [INFO] Performance Results of task: vllm-api-stream-chat/gsm8k: -╒══════════════════════════╤═════════╤═════════════════╤════════════════╤═════════════════╤═════════════════╤═════════════════╤════════════════╤═════════════════╤══════╕ -│ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ -╞══════════════════════════╪═════════╪═════════════════╪════════════════╪═════════════════╪═════════════════╪═════════════════╪════════════════╪═════════════════╪══════╡ -│ E2EL │ total │ 2753.3518 ms │ 2189.5185 ms │ 3339.4463 ms │ 2755.8153 ms │ 3039.7431 ms │ 3219.6642 ms │ 3313.0408 ms │ 1319 │ -...... -╒══════════════════════════╤═════════╤═══════════════════╕ -│ Common Metric │ Stage │ Value │ -╞══════════════════════════╪═════════╪═══════════════════╡ -│ Benchmark Duration │ total │ 38101.2396 ms │ -...... -[2025-11-06 11:11:33,474] [ais_bench] [INFO] Performance Result files located in outputs/default/20251106_110904/performances/vllm-api-stream-chat. -[2025-11-06 11:11:33,474] [ais_bench] [INFO] Performance Results of task: vllm-api-stream-chat/aime2024: -╒══════════════════════════╤═════════╤═════════════════╤═══════════════╤════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════╕ -│ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ -╞══════════════════════════╪═════════╪═════════════════╪═══════════════╪════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════╡ -│ E2EL │ total │ 2745.4115 ms │ 2187.5882 ms │ 3288.4635 ms │ 2820.7541 ms │ 2988.8338 ms │ 3188.436 ms │ 3273.7475 ms │ 30 │ -...... -╒══════════════════════════╤═════════╤═══════════════════╕ -│ Common Metric │ Stage │ Value │ -╞══════════════════════════╪═════════╪═══════════════════╡ -│ Benchmark Duration │ total │ 3335.7672 ms │ -...... -[2025-11-06 11:11:33,477] [ais_bench] [INFO] Performance Result files located in outputs/default/20251106_110904/performances/vllm-api-stream-chat. -``` - -At the same time, the final generated directory structure is as follows: +ais_bench ais_bench/configs/performance_benchmark/performance_multi_dataset.py +``` + +::: +:::{tab-item} Alternative: Command-Line Parameters + +Use the `--datasets` parameter to specify multiple datasets: + ```bash -# Under output/default -20251106_110904/ # Output directory corresponding to the task creation time -├── configs # A combined configuration file integrating configs for model tasks, dataset tasks, and structure presentation tasks -├── logs # Contains logs from the inference and accuracy evaluation phases; when the --debug command is added, logs will be printed directly to the screen without generating disk-stored files -│ └── performance # Log files from the inference phase -└── performances # Performance evaluation results - ├── vllm-api-general-stream # Name of the "service-oriented model configuration", corresponding to the abbr parameter in the models section of the model task configuration file - │ ├── aime2024.csv # Single-request performance output (CSV), consistent with the Performance Parameters table in the on-screen performance results display - │ ├── aime2024.json # End-to-end performance output (JSON), consistent with the Common Metric table in the on-screen performance results display - │ ├── aime2024_plot.html # Request concurrency visualization report (HTML) - │ ├── gsm8k.csv - │ ├── gsm8k.json - │ ├── gsm8k_plot.html - │ └── ...... - └── vllm-api-stream-chat - ├── aime2024.csv - ├── aime2024.json - ├── aime2024_plot.html - ├── gsm8k.csv - ├── gsm8k.json - ├── gsm8k_plot.html - └── ...... +ais_bench --models vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str,aime2024_gen_0_shot_chat_prompt -m perf +``` + +::: +::: + +#### Specifying Multiple Sending Rates for Performance Evaluation + +The following configuration file example sends the `ShareGPT` dataset to the service at `request_rate=1, 2, 4, 8` respectively for performance evaluation. +:::{tab-set} +:::{tab-item} ⭐ Custom Configuration File + +```python +import copy +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as base_vllm_api_stream_chat + +# In AISBench, `request_rate` is a field of the model configuration, so build one +# model configuration per rate via `copy.deepcopy` and combine them with one dataset. +models = [] +for rate in [1, 2, 4, 8]: + model_cfg = copy.deepcopy(base_vllm_api_stream_chat[0]) + model_cfg["abbr"] = f"vllm-api-stream-chat-rate-{rate}" + model_cfg["host_ip"] = "localhost" + model_cfg["host_port"] = 8080 + model_cfg["max_out_len"] = 1024 + model_cfg["batch_size"] = 50 + model_cfg["request_rate"] = rate + model_cfg["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + models.append(model_cfg) + +work_dir = "outputs/default/" ``` -> ⚠️ Note: -> - In multi-task performance evaluation scenarios, the dataset tasks specified by `--datasets` must belong to different dataset types. Otherwise, performance data may be missing due to overwriting. For example, you cannot use `--datasets` to specify both the `aime2024_gen_0_shot_str` and `aime2024_gen_0_shot_chat_prompt` dataset tasks simultaneously. +Execution command: -### Custom Sequence Length Evaluation -#### 1 Configure Input and Output Distribution for Custom Sequence Datasets -To perform custom sequence length evaluation, you need to specify the special dataset task `synthetic_gen_string`. Execute the following command to retrieve the path of the configuration file corresponding to `synthetic_gen_string`: ```bash -ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_string --search +ais_bench ais_bench/configs/performance_benchmark/performance_multi_rate.py +``` + +::: +:::{tab-item} Alternative: Command-Line Parameters + +It is not supported to specify multiple sending rates for one dataset in a single command. It is recommended to use a custom configuration file. + +::: +::: + +#### Specifying Multiple Models for Performance Evaluation + +Supports simultaneous evaluation of multiple models on the same dataset, suitable for comparing the performance of different models. + +:::{tab-set} +:::{tab-item} ⭐ Custom Configuration File + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + +# Rename the abbr of each model so that the results are distinguishable +vllm_api_stream_chat[0]["abbr"] = "vllm-qwen2.5-7b" +vllm_api_general_stream[0]["abbr"] = "vllm-qwen2.5-14b" + +vllm_api_stream_chat[0]["host_ip"] = "localhost" +vllm_api_stream_chat[0]["host_port"] = 8080 +vllm_api_stream_chat[0]["max_out_len"] = 1024 +vllm_api_stream_chat[0]["batch_size"] = 50 +vllm_api_stream_chat[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +vllm_api_general_stream[0]["host_ip"] = "localhost" +vllm_api_general_stream[0]["host_port"] = 8081 +vllm_api_general_stream[0]["max_out_len"] = 1024 +vllm_api_general_stream[0]["batch_size"] = 50 +vllm_api_general_stream[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +models = vllm_api_stream_chat + vllm_api_general_stream + +work_dir = "outputs/default/" ``` -The result will be: + +Execution command: + +```bash +ais_bench ais_bench/configs/performance_benchmark/performance_multi_model.py ``` -╒══════════════╤═══════════════════════════════════════╤════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ -│ Task Type │ Task Name │ Config File Path │ -╞══════════════╪═══════════════════════════════════════╪════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ -│ --models │ vllm_api_stream_chat │ /your_workspace/benchmark/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py │ -├──────────────┼───────────────────────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ -│ --datasets │ synthetic_gen_string │ /your_workspace/benchmark/ais_bench/benchmark/configs/datasets/synthetic/synthetic_gen_string.py │ -╘══════════════╧═══════════════════════════════════════╧════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ + +::: +:::{tab-item} Alternative: Command-Line Parameters + +```bash +ais_bench --models vllm_api_stream_chat,vllm_api_general_stream --datasets sharegpt_gen -m perf ``` -Modify the `synthetic_config` in `/your_workspace/benchmark/ais_bench/benchmark/configs/datasets/synthetic/synthetic_gen_string.py`. The configuration content is as follows: +::: +::: + +### Synthetic Dataset Multi-Task Combinations + +In actual performance evaluation, it is sometimes necessary to simulate the input load in production environments, such as fixed-length inputs, Poisson-distributed request arrival, etc. AISBench supports users in defining custom performance evaluation datasets through the `SyntheticDataset`, and supports configuring the distribution of input sequence lengths, the distribution of output sequence lengths, the request arrival rate (QPS), etc. through parameters. The model-dataset sub-tasks generated by the synthetic dataset support combinations with each other. + +The following configuration file example sends synthetic datasets of different input lengths to the service for performance evaluation at `request_rate=2`: + +:::{tab-set} +:::{tab-item} ⭐ Custom Configuration File + ```python -synthetic_config = { - "Type": "string", - "RequestCount": 1000, # Number of requests (number of dataset entries) - "StringConfig": { - "Input": { - "Method": "uniform", - "Params": {"MinValue": 50, "MaxValue": 500} # Input length: 50-500 - }, - "Output": { - "Method": "uniform", - "Params": {"MinValue": 20, "MaxValue": 200} # Output length: 20-200 +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets as base_synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Define multiple sub-datasets with different input/output lengths via the `config` field +datasets = [] +for input_len in [256, 512, 1024]: + for output_len in [256, 512]: + ds = dict(base_synthetic_datasets[0]) + ds["abbr"] = f"syn_in{input_len}_out{output_len}" + ds["config"] = { + "Type": "string", + "RequestCount": 100, + "TrustRemoteCode": False, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": input_len, "MaxValue": input_len}, + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": output_len, "MaxValue": output_len}, + }, + }, } - } -} + datasets.append(ds) + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["request_rate"] = 2 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" +``` + +Execution command: + +```bash +ais_bench ais_bench/configs/performance_benchmark/performance_synthetic.py ``` -💡 For more custom input and output distributions, refer to 📚 [Random Synthetic Dataset](../../advanced_tutorials/synthetic_dataset.md) -#### 2 Ensure the Inference Service Reaches the Set Maximum Output -To ensure the inference service achieves the set maximum output, you need to configure the special post-processing parameter `ignore_eos = True` in `generation_kwargs` of the 📚 [Service-Oriented Model Configuration](../all_params/models.md#Service-Oriented Inference Backend Configuration Parameter Description) to control the maximum output length of requests (preventing early termination). +::: +:::{tab-item} Alternative: Command-Line Parameters + +It is not supported to specify multiple synthetic datasets with different lengths in a single command. It is recommended to use a custom configuration file. + +::: +::: + +> 💡 For more configuration details of `SyntheticDataset`, please refer to 📚 [Datasets](../../get_started/datasets.md#randomly-synthesized-datasets). + +### Custom Sequence Length Usage through Custom Config File Approach + +:::{admonition} Why use a custom config file? +:class: tip + +For the synthetic dataset scenario, in order to fully support the user's combination of multiple different input/output lengths, multiple different QPS sending rates, etc., it is **strongly recommended to use a custom configuration file**, because the command-line parameters can only support a single fixed length and a single QPS, and cannot satisfy the combinatorial requirements. +::: + +For detailed instructions on writing custom configuration files, please refer to [Custom Configuration Files](../../advanced_tutorials/run_custom_config.md#4-synthetic-dataset-performance-evaluation). + +### Custom Sequence Multi-Task Combinations + +For multi-task combinations based on custom sequence lengths, the user can combine different models and datasets for evaluation through the `model_dataset_combinations` field. + +:::{tab-set} +:::{tab-item} ⭐ Custom Configuration File -For example, modify the content of the configuration file [vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/models/vllm_api/vllm_api_stream_chat.py) corresponding to the `vllm_api_stream_chat` model task: ```python -from ais_bench.benchmark.models import VLLMCustomAPIChatStream -models = [ - dict( - attr="service", - type=VLLMCustomAPIChatStream, - abbr='vllm-api-stream-chat', - # Configure other model task parameters such as port and IP by yourself - generation_kwargs = dict( - # ..... - ignore_eos = True, # The inference service output ignores EOS (output length will definitely reach max_out_len) - ) - ) +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets as base_synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = [] +for input_len in [256, 512]: + for output_len in [256, 512]: + ds = dict(base_synthetic_datasets[0]) + ds["abbr"] = f"syn_in{input_len}_out{output_len}" + ds["config"] = { + "Type": "string", + "RequestCount": 100, + "TrustRemoteCode": False, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": input_len, "MaxValue": input_len}, + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": output_len, "MaxValue": output_len}, + }, + }, + } + datasets.append(ds) + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +# Key: Only specify partial models for partial datasets +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0], datasets[1]]), + dict(models=[models[0]], datasets=[datasets[2]]), ] +work_dir = "outputs/default/" ``` -#### 3 Start Performance Evaluation -Execute the following command: +Execution command: + ```bash -ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_string -m perf +ais_bench ais_bench/configs/performance_benchmark/performance_seq_combinations.py ``` -After completion, the output directory structure is the same as that described in the [Multi-Task Evaluation](#Multi-Task Evaluation) section. Corresponding CSV/JSON/HTML files will be generated under performance/vllm-api-stream-chat/synthetic*. -> ⚠️ Note: -> - Some service-oriented backends do not support the `ignore_eos` post-processing parameter. In such cases, the actual number of output `Tokens` may not reach the configured maximum output length. You need to configure other post-processing parameters (e.g., parameters for limiting minimum output) to achieve the maximum output length. +::: +:::{tab-item} Alternative: Command-Line Parameters + +Not supported. + +::: +:::: ### Fixed Request Count Evaluation -When the dataset scale is too large and you only want to perform performance testing on a subset of samples, you can use the 📚 [`--num-prompts`](../all_params/cli_args.md#Performance Evaluation Parameters) parameter to specify the number of data entries to read. An example is as follows: + +When the dataset scale is too large and you only want to perform performance testing on a subset of samples, you can use either of the following two approaches to control the data reading range. They achieve the same goal, so just pick the one that fits your habit: + +- **Basic approach**: Specify the number of data entries to read directly via the command-line parameter 📚 [`--num-prompts`](../all_params/cli_args.md#common-parameters). No configuration file modification is required, and it is the simplest to use. +- **Advanced approach (more powerful)**: Set the `reader_cfg.test_range` field of the dataset in the custom configuration file, which supports a more flexible sampling range (e.g., specifying a start index and custom step). For detailed usage, refer to 📚 [Custom Configuration Files](../../advanced_tutorials/run_custom_config.md). + +Example as follows: + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +**Method 1: Basic approach — Use `--num-prompts` to specify the number of entries to read** + +For a complete example, refer to [performance_fixed_request.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_fixed_request.py): + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +# ...For other parameter configurations, please refer to the configuration file +``` + +Execute the command (specify reading only 1 sample via `--num-prompts 1`): + ```bash -ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt -m perf --num-prompts 1 +ais_bench ais_bench/configs/performance_benchmark/performance_fixed_request.py --mode perf --num-prompts 1 ``` -The above command only performs inference on the first entry in the sample dataset and measures its performance. -> ⚠️ Note: Currently, the dataset is read sequentially in the default queue order; random sampling or shuffling is not supported. +**Method 2: Advanced approach — Use `test_range` to flexibly specify the reading range** -## Other Functional Scenarios -### Speculative Decoding Metrics Collection +If you need more flexible range control (e.g., specifying a start index and custom step), you can set the `reader_cfg.test_range` field of the dataset directly in the custom configuration file, without passing any command-line parameter. For a complete example, refer to [performance_fixed_request.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_fixed_request.py): -When running performance evaluation against a vLLM inference server with speculative decoding enabled, you can append `--spec-decode` to collect spec decode performance metrics (acceptance rate, acceptance length, etc.) from the server's Prometheus `/metrics` endpoint. The metrics are displayed alongside the standard performance results and saved to `spec_decode_*.json` under the `performances/` directory. +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask -```shell -ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode perf --spec-decode +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# Key: control the sampling range flexibly via reader_cfg.test_range +# For example, '[0:8]' reads the first 8 samples; '[10:20]' reads samples from index 10 to 20 +datasets[0]['reader_cfg']['test_range'] = '[0:8]' + +models = vllm_api_stream_chat +# ...For other parameter configurations, please refer to the configuration file ``` -For prerequisites, configuration details, and metric explanations, see 📚 [Speculative Decoding Metrics Collection](../../advanced_tutorials/spec_decode.md). +Execute the command (test_range has been specified in the configuration file, no need to pass `--num-prompts`): -### Performance Result Recalculation -The main functional scenario evaluation tool for performance testing executes a complete workflow of performance sampling → calculation → aggregation: -```mermaid -graph LR; - A[Execute inference based on the given dataset] --> B((Performance打点数据)) - B --> C[Calculate metrics based on the打点数据] - C --> D((Performance data)) - D --> E[Generate an aggregated report based on the performance data] - E --> F((Present results)) +```bash +ais_bench ais_bench/configs/performance_benchmark/performance_fixed_request.py --mode perf ``` -*Note: "打点数据" (dǎdiǎn shùjù) refers to "instrumented data" or "sampled performance metrics" in this technical context.* -Each link in the execution workflow is independently decoupled. Calculation and aggregation can be repeatedly performed based on the results of performance sampling. If the directly printed performance data does not include data for relevant dimensions (e.g., missing 95th percentile data), you need to modify some configurations for recalculation. The specific operations are as follows: +::: +:::{tab-item} Alternative: Using Command-Line Parameters -Assume the command used for the previous performance evaluation was: ```bash -ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode perf +ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode perf --num-prompts 1 ``` -The printed `Performance Parameters` table is as follows: +The above command only performs inference on the first entry in the sample dataset and only measures the performance of this one entry. + +::: +:::: + +> ⚠️ Note: Currently, the dataset is read sequentially in the default queue order; random sampling or shuffling is not supported. When `reader_cfg.test_range` in the configuration file and the command-line `--num-prompts` are both specified, the command-line parameter `--num-prompts` takes precedence. + +### Fixed Request Count Performance Evaluation + +In some scenarios, the user wants to fix the total number of requests sent without limiting the sending rate, that is, to send requests at the maximum throughput. In this case, `request_rate` needs to be set to `-1`, indicating that requests are sent concurrently without rate limiting. + +:::{tab-set} +:::{tab-item} ⭐ Custom Configuration File + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["request_rate"] = -1 # -1 indicates concurrent sending without rate limiting +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" +``` + +Execution command (you can append `--num-prompts 100` to fix the total number of requests): + ```bash -[2025-11-06 11:11:33,463] [ais_bench] [INFO] Performance Results of task: vllm-api-general-stream/gsm8k: -╒══════════════════════════╤═════════╤═════════════════╤════════════════╤═════════════════╤═════════════════╤═════════════════╤════════════════╤═════════════════╤══════╕ -│ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ -╞══════════════════════════╪═════════╪═════════════════╪════════════════╪═════════════════╪═════════════════╪═════════════════╪════════════════╪═════════════════╪══════╡ -│ E2EL │ total │ 2753.3518 ms │ 2189.5185 ms │ 3339.4463 ms │ 2755.8153 ms │ 3039.7431 ms │ 3219.6642 ms │ 3313.0408 ms │ 1319 │ -...... +ais_bench ais_bench/configs/performance_benchmark/performance_fixed_request.py --mode perf ``` -*Note: "E2EL" stands for "End-to-End Latency" in this performance context.* -If you want to view performance data for the "P95" (95th percentile) dimension, you need to modify the content of the configuration file corresponding to the default result presentation task `default_perf` for `--summarizer`. The path of `default_perf` can be queried using the `--search` command: +::: +:::{tab-item} Alternative: Command-Line Parameters + ```bash -╒══════════════╤══════════════╤═══════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ -│ Task Type │ Task Name │ Config File Path │ -╞══════════════╪══════════════╪═══════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ -│ --summarizer │ default_perf │ /your_workspace/benchmark/ais_bench/benchmark/configs/summarizers/perf/default_perf.py │ -╘══════════════╧══════════════╧═══════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ +ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode perf --num-prompts 100 +``` +::: +::: + +## Implementation via Custom Configuration Files + +> 💡 All the above functional scenarios (multi-task evaluation, multi-task parallel, fixed request count, etc.) can be implemented through the [Custom Configuration File](../../advanced_tutorials/run_custom_config.md) approach. The configuration file is essentially a Python script, which supports all Python syntaxes such as loops, conditional judgments, and list comprehensions. Model, dataset, summarizer, and other configurations can be written into one file for one-time writing and multiple reuse. + +All custom configuration file examples involved in this section are uniformly stored in the `ais_bench/configs/performance_benchmark/` directory for easy reference and reuse: + +| File Name | Corresponding Scenario | +| --- | --- | +| [performance_qwen2_7b_sharegpt.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_qwen2_7b_sharegpt.py) | Single-Task Performance Evaluation | +| [performance_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_multi_dataset.py) | Multi-Dataset Performance Evaluation | +| [performance_multi_rate.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_multi_rate.py) | Multi-Rate Performance Evaluation | +| [performance_multi_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_multi_model.py) | Multi-Model Performance Evaluation | +| [performance_synthetic.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_synthetic.py) | Synthetic Dataset Multi-Task Combinations | +| [performance_seq_combinations.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_seq_combinations.py) | Custom Sequence Multi-Task Combinations | +| [performance_fixed_request.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_fixed_request.py) | Fixed Request Count Performance Evaluation | +| [performance_re_eval.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_re_eval.py) | Performance Result Recalculation | + +For details, refer to the "Service-Oriented Performance Evaluation" example in [Running AISBench via Custom Configuration Files](../../advanced_tutorials/run_custom_config.md#custom-configuration-file-examples-for-each-scenario). + +## Other Functional Scenarios + +### Speculative Decoding Metrics Collection + +When running performance evaluation against a vLLM inference server with speculative decoding enabled, you can append `--spec-decode` to collect spec decode performance metrics (acceptance rate, acceptance length, etc.) from the server's Prometheus `/metrics` endpoint. The metrics are displayed alongside the standard performance results and saved to `spec_decode_*.json` under the `performances/` directory. + +```shell +ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode perf --spec-decode ``` -Modify the content of `default_perf.py`: -```py +For prerequisites, configuration details, and metric explanations, see 📚 [Speculative Decoding Metrics Collection](../../advanced_tutorials/spec_decode.md). + +### Performance Result Recalculation + +In the actual evaluation process, the user may want to update the performance summary based on the existing inference results, for example, after modifying the `stats_list` configuration, recalculate the summary report without re-running the inference. + +AISBench supports recalculating performance summaries based on existing inference results through the `--mode perf` and `--reuse` parameters. + +For a complete example, refer to [performance_re_eval.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/performance_re_eval.py): + +```python from mmengine.config import read_base from ais_bench.benchmark.summarizers import DefaultPerfSummarizer from ais_bench.benchmark.calculators import DefaultPerfMetricCalculator +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat +# Key: customize the stats_list of the performance summarizer to control the +# statistical dimensions of the performance summary summarizer = dict( + attr="performance", type=DefaultPerfSummarizer, calculator=dict( type=DefaultPerfMetricCalculator, - stats_list=["Average", "Min", "Max", "Median", "P95"], - ) + stats_list=["Average", "Min", "Max", "Median", "P75", "P90", "P95", "P99"], + ), ) + +models = vllm_api_stream_chat +models[0]["host_ip"] = "localhost" +models[0]["host_port"] = 8080 +models[0]["max_out_len"] = 512 +models[0]["batch_size"] = 1 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + +work_dir = "outputs/default/" ``` -Among them, the `stats_list` can hold data for up to 8 performance dimensions at the same time. -After the modification is completed, you can execute the following command to recalculate the performance metrics: +Execution command (`--mode perf` and `--reuse` are common parameters, and can still be appended through the command line when using a custom configuration file): ```bash -## Note: --summarizer default_perf must be specified -ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer default_perf --mode perf_viz --pressure --debug --reuse 20250628_151326 +ais_bench ais_bench/configs/performance_benchmark/performance_re_eval.py --mode perf --reuse 20250628_151326 ``` -The on-screen performance results will be as follows: -```bash -[2025-11-06 11:11:33,463] [ais_bench] [INFO] Performance Results of task: vllm-api-general-stream/gsm8k: -╒══════════════════════════╤═════════╤════════════════╤═════════════════╤═════════════════╤════════════════╤═════════════════╤═════╕ -│ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P95 │ N │ -╞══════════════════════════╪═════════╪════════════════╪═════════════════╪═════════════════╪════════════════╪═════════════════╪═════╡ -│ E2EL │ total │ 2761.6153 ms │ 2493.8016 ms │ 3086.0523 ms │ 2848.9603 ms │ 3021.0043 ms │ 8 │ -...... -╒══════════════════════════╤═════════╤═══════════════════╕ -│ Common Metric │ Stage │ Value │ -╞══════════════════════════╪═════════╪═══════════════════╡ -│ Benchmark Duration │ total │ 3090.7835 ms │ -...... -[2025-11-06 11:11:33,468] [ais_bench] [INFO] Performance Result files located in outputs/default/20251106_110904/performances/vllm-api-general-stream. - -``` -> ⚠️ The files `gsm8kdataset.csv`, `gsm8kdataset_details.json`, and `gsm8kdataset_plot.html` under `20251106_110904/performance/` will be regenerated (overwriting the original ones). - - -## Specifications for Service-Oriented Performance Testing -The scale of service-oriented performance testing determines the resource usage of the AISBench evaluation tool. Taking [Custom Sequence Length Evaluation](#Custom Sequence Length Evaluation) as an example, the test scale is mainly determined by the total number of requests (`RequestCount`), dataset input token length (`Input`), and output token length (`Output`). When tested on a CPU of model `Intel(R) Xeon(R) Platinum 8480P`, the resource usage under typical test scales is approximately as follows: - -| Total Number of Requests (`RequestCount`) | Dataset Input Token Length (`Input`) | Output Token Length (`Output`) | Maximum Memory Usage (GB) | Maximum Disk Usage (GB) | Performance Data Calculation Time (s) | Remarks | -|-------------------------------------------|--------------------------------------|---------------------------------|---------------------------|--------------------------|----------------------------------------|---------| -| 10,000 | 1024 | 1024 | < 16 | 0.12 | 3 | | -| 10,000 | 1024 | 4096 | < 16 | 0.16 | 4 | | -| 10,000 | 4096 | 4096 | < 16 | 0.17 | 6 | | -| 50,000 | 4096 | 4096 | < 32 | 0.80 | 30 | | -| 250,000 | 4096 | 4096 | < 64 | 4.00 | 150 | Maximum specification | - -> ⚠️ The maximum memory usage, maximum disk usage, and calculation time of performance data are roughly proportional to the value of (`RequestCount × (Input + Output)`). The maximum specification supported by a single machine in AISBench is `RequestCount × (Input + Output) = 250,000 × (4096 + 4096) = 2,024,000,000`. \ No newline at end of file + +## Specifications + +The following specifications are required when using AISBench for performance evaluation: + +| Item | Specification | +| --- | --- | +| Service Status | The service must be running normally, and the listening port is consistent with the `url` field in the configuration | +| `max-model-len` | Must be greater than or equal to `prompt length + output length`, otherwise the service will reject the request | +| Network | The evaluation machine needs to be able to access the service address normally | +| Concurrency | The number of concurrent evaluations should not exceed the service's processing capacity to avoid request timeout/failure | +| Output Directory | Each task generates a timestamp directory containing `configs/`, `logs/`, `predictions/`, `results/`, `summary/`, `performances/` | \ No newline at end of file diff --git a/docs/source_en/best_practices/practice_ascend.md b/docs/source_en/best_practices/practice_ascend.md index a597841b..5ffc7c50 100644 --- a/docs/source_en/best_practices/practice_ascend.md +++ b/docs/source_en/best_practices/practice_ascend.md @@ -2,6 +2,8 @@ ### Version of AISBench Evaluation Tool Used for Reproduction The version of the AISBench evaluation tool used for reproduction in this paper is [v3.0-20250331](https://github.com/AISBench/benchmark/releases/tag/v3.0-20250331). +> 💡 All evaluation commands in this document can be implemented through the [custom configuration file approach](../advanced_tutorials/run_custom_config.md). Write configurations for models, datasets, summarizers, etc. into a single Python file for one-time writing and multiple reuse. The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. See [Running AISBench with Custom Configuration Files](../advanced_tutorials/run_custom_config.md) for details. + ### I. Background and Objectives #### 1.1 Significance of Reproduction ##### 1.1.1 Mathematical Reasoning Advantages of DeepSeek-R1 diff --git a/docs/source_en/best_practices/practice_nvidia.md b/docs/source_en/best_practices/practice_nvidia.md index 9b7ca0ae..3eea0369 100644 --- a/docs/source_en/best_practices/practice_nvidia.md +++ b/docs/source_en/best_practices/practice_nvidia.md @@ -2,6 +2,7 @@ ### Version of AISBench Evaluation Tool Used for Reproduction The version of the AISBench evaluation tool used for reproduction in this paper is [v3.0-20250412](https://github.com/AISBench/benchmark/releases/tag/v3.0-20250412). +> 💡 All evaluation commands in this document can be implemented through the [custom configuration file approach](../advanced_tutorials/run_custom_config.md). Write configurations for models, datasets, summarizers, etc. into a single Python file for one-time writing and multiple reuse. The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. See [Running AISBench with Custom Configuration Files](../advanced_tutorials/run_custom_config.md) for details. ### I. Background and Objectives #### 1.1 Significance of Reproduction @@ -272,7 +273,7 @@ The inference backend configuration file contains settings related to the local ```bash # Ensure you are in the outermost directory of the source code: your/work/dir/benchmark -vim ais_bench/benchmark/configs/models/hf_model/hf_chat_model.py +vim ais_bench/benchmark/configs/models/hf_models/hf_chat_model.py ``` Modify the content of the inference backend configuration file as follows: diff --git a/docs/source_en/best_practices/replicate_llm_datasets_accuracy.md b/docs/source_en/best_practices/replicate_llm_datasets_accuracy.md index e43afe64..e2622ac7 100644 --- a/docs/source_en/best_practices/replicate_llm_datasets_accuracy.md +++ b/docs/source_en/best_practices/replicate_llm_datasets_accuracy.md @@ -1,47 +1,45 @@ # Reproducing Dataset Evaluation Results from Large Language Model (LLM) Papers (Technical Reports) — Taking the GPQA Dataset Used by DeepSeek R1 as an Example +> 💡 All evaluation commands in this document can be implemented through the [custom configuration file approach](../advanced_tutorials/run_custom_config.md). Write configurations for models, datasets, summarizers, etc. into a single Python file for one-time writing and multiple reuse. The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. See [Running AISBench with Custom Configuration Files](../advanced_tutorials/run_custom_config.md) for details. + ## Preface - Methodology -To reproduce the accuracy results reported in papers using the AISBench evaluation tool, it is essential to align with the testing methodology for the dataset as described in the model’s technical report or paper. The following configurations in the evaluation tool need to be aligned accordingly: +To reproduce the accuracy results reported in papers using the AISBench evaluation tool, it is essential to align with the testing methodology for the dataset as described in the model's technical report or paper. The following configurations in the evaluation tool need to be aligned accordingly: -### Model - Related Configurations +**Model - Related Configurations**: - Select the appropriate model task corresponding to the endpoint - Fully align the maximum output length -- Fully align the post - processing parameters +- Fully align the post-processing parameters -### Dataset - Related Configurations +**Dataset - Related Configurations**: - Fully align the prompt engineering - Fully align the answer extraction method - Align the accuracy evaluation metrics ---- - ## Example: Reproducing the Evaluation Results of the DeepSeek R1 Model on the GPQA Dataset ### Select the Appropriate Model Configuration File Corresponding to the Endpoint -For execution efficiency, inference services are generally used as the subjects under test when reproducing model accuracy. Inference services can be accessed via various endpoints, and the industry standard mainly adopts OpenAI - style endpoints. There are two primary OpenAI endpoints: `v1/completions` and `v1/chat/completions`. +For execution efficiency, inference services are generally used as the subjects under test when reproducing model accuracy. Inference services can be accessed via various endpoints, and the industry standard mainly adopts OpenAI-style endpoints. There are two primary OpenAI endpoints: `v1/completions` and `v1/chat/completions`. - **v1/completions**: The model generates text based on a "prefix continuation" logic and does not inherently distinguish between "instructions" and "content". Strong guidance through prompt engineering (e.g., adding "Please answer:") is required; otherwise, it may produce imitative outputs rather than executing instructions. For instance, inputting "Translate the following English to Chinese: Hello" might result in the continuation "Translate the following Chinese to English: Nihao" instead of a direct translation. - Therefore, it is suitable for single - turn text generation tasks (such as code completion, short - text writing, text continuation, and simple text classification) or scenarios that need to be compatible with legacy base models. + Therefore, it is suitable for single-turn text generation tasks (such as code completion, short-text writing, text continuation, and simple text classification) or scenarios that need to be compatible with legacy base models. - **v1/chat/completions**: The model natively understands the semantic roles of system/user/assistant, prioritizes executing user instructions, and ensures more stable dialogue consistency and intent alignment. It can complete tasks like translation and summarization without complex prompt wrapping. - Hence, it is ideal for modern LLM application scenarios such as multi - turn dialogues (customer service, chatbots), instruction - driven tasks (translation, summarization, data analysis), tool integration (function calling, retrieval - augmented generation), and multimodal interactions. + Hence, it is ideal for modern LLM application scenarios such as multi-turn dialogues (customer service, chatbots), instruction-driven tasks (translation, summarization, data analysis), tool integration (function calling, retrieval-augmented generation), and multimodal interactions. -💡 As of January 2025, nearly all newly released LLM models support the `v1/chat/completions` endpoint, and the `v1/completions` endpoint has been largely deprecated. Consequently, model configuration files typically only use the model tasks for accessing the `v1/chat/completions` endpoint: **vllm_api_general_chat** (accessing the service via a non - streaming interface) and **vllm_api_stream_chat** (accessing the service via a streaming interface). +💡 As of January 2025, nearly all newly released LLM models support the `v1/chat/completions` endpoint, and the `v1/completions` endpoint has been largely deprecated. Consequently, model configuration files typically only use the model tasks for accessing the `v1/chat/completions` endpoint: **vllm_api_general_chat** (accessing the service via a non-streaming interface) and **vllm_api_stream_chat** (accessing the service via a streaming interface). Taking the model task `vllm_api_general_chat` as an example, the absolute path to its corresponding model configuration file can be obtained by running the following command: ```bash ais_bench --models vllm_api_general_chat --search ``` -⚠️ All subsequent model - related configurations will be modified in this configuration file. - ---- +⚠️ All subsequent model-related configurations will be modified in this configuration file. ### Fully Align the Maximum Output Length -The following description can be found on the [DeepSeek R1 Hugging Face Model Card](https://huggingface.co/deepseek - ai/DeepSeek - R1): +The following description can be found on the [DeepSeek R1 Hugging Face Model Card](https://huggingface.co/deepseek-ai/DeepSeek-R1): > ## 4. Evaluation Results -> ### DeepSeek - R1 - Evaluation +> ### DeepSeek-R1-Evaluation > For all our models, the maximum generation length is set to 32,768 tokens.... This indicates that the maximum output length of the DeepSeek R1 model is set to 32,768 tokens. @@ -54,7 +52,7 @@ models = [ dict( attr="service", type=VLLMCustomAPIChat, - abbr='vllm - api - general - chat', + abbr='vllm-api-general-chat', # ...... max_out_len=32768, # Maximum number of tokens output by the inference service # ...... @@ -62,17 +60,15 @@ models = [ ] ``` ---- - -### Fully Align the Post - processing Parameters -The following description is available on the [DeepSeek R1 Hugging Face Model Card](https://huggingface.co/deepseek - ai/DeepSeek - R1): +### Fully Align the Post-processing Parameters +The following description is available on the [DeepSeek R1 Hugging Face Model Card](https://huggingface.co/deepseek-ai/DeepSeek-R1): > ## 4. Evaluation Results -> ### DeepSeek - R1 - Evaluation -> ..., For benchmarks requiring sampling, we use a temperature of $0.6$, a top - p value of $0.95$, ... +> ### DeepSeek-R1-Evaluation +> ..., For benchmarks requiring sampling, we use a temperature of $0.6$, a top-p value of $0.95$, ... -It can be seen from this that the post - processing parameters of the DeepSeek R1 model include a temperature of 0.6 and a top - p value of 0.95. +It can be seen from this that the post-processing parameters of the DeepSeek R1 model include a temperature of 0.6 and a top-p value of 0.95. -Taking the model task `vllm_api_general_chat` as an example, the configuration for the post - processing parameters is as follows: +Taking the model task `vllm_api_general_chat` as an example, the configuration for the post-processing parameters is as follows: ```python from ais_bench.benchmark.models import VLLMCustomAPIChat @@ -80,77 +76,125 @@ models = [ dict( attr="service", type=VLLMCustomAPIChat, - abbr='vllm - api - general - chat', + abbr='vllm-api-general-chat', # ...... - temperature=0.6, # Sampling temperature for text generation - top_p=0.95, # Top - p sampling parameter + generation_kwargs=dict( # Post-processing parameters are filled in here + temperature=0.6, + top_p=0.95, + ), # ...... ) ] ``` ---- - ### Fully Align Prompt Engineering -In the [DeepSeek R1 technical report](https://github.com/deepseek - ai/DeepSeek - R1/blob/main/DeepSeek_R1.pdf), the prompt format for the GPQA dataset is specified as follows: -> For GPQA, we use the 0 - shot chain - of - thought (CoT) prompt from the original GPQA paper. The prompt template is as follows: -> Q: [question] -> A: Let's think step by step. +Prompt engineering has a significant impact on model accuracy. Generally, model papers or technical reports disclose the prompts used for testing. If third-party tools are used for testing, the specific open-source tool is also explicitly stated. + +In the DeepSeek R1 paper, the prompt engineering used for the GPQA dataset test is described as follows: -In the AISBench dataset configuration file, the prompt engineering can be aligned by modifying the reader configuration, as shown below: +> Evaluation Prompts Following the setup in DeepSeek-V3, standard benchmarks such as MMLU, DROP, GPQADiamond, and SimpleQA are evaluated using prompts from the simple-evals framework. + +This indicates that the prompt engineering for DeepSeek R1 uses the prompt template from the simple-evals tool. You can refer to the [simple-evals](https://github.com/openai/simple-evals) project. The relevant portion of the prompt used by GPQA is as follows (you need to look at the code): ```python -# https://github.com/AISBench/benchmark/blob/master/ais_bench/benchmark/configs/datasets/gpqa/gpqa_gen_0_shot_cot_chat_prompt.py +QUERY_TEMPLATE_MULTICHOICE = """ +Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. -gpqa_reader_cfg = dict( - # ...... - prompt_template='Q: {question}\nA: Let\'s think step by step.', - # ...... -) +{Question} + +A) {A} +B) {B} +C) {C} +D) {D} +""".strip() ``` ---- +Therefore, the prompt engineering in AISBench should be modified to: +```python +# https://github.com/AISBench/benchmark/blob/master/ais_bench/benchmark/configs/datasets/gpqa/gpqa_gen_0_shot_cot_chat_prompt.py + +## Prompt template identical to simple-evals +align_prompt = """ +Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. + +{question} + +A) {A} +B) {B} +C) {C} +D) {D} +""".strip() + +## ...... + +gpqa_infer_cfg = dict( + prompt_template=dict( # Prompt engineering + type=PromptTemplate, + template=dict( + round=[ + dict(role='HUMAN', prompt=align_prompt), # Pass in the prompt template + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer)) +``` +For a more detailed introduction to prompt engineering, please refer to [Prompt Template Introduction](../prompt/prompt_template.md). ### Fully Align the Answer Extraction Method -The answer format in the GPQA dataset is option - based (options A, B, C, D). In the DeepSeek R1 paper, the answer extraction method is to extract the final answer option (A/B/C/D) from the model - generated reasoning process. +How to extract answers from the model's inference results and evaluate them directly affects the evaluation scores. The answer extraction methods for LLM evaluation datasets generally fall into 3 categories: +1. For evaluation datasets of multiple-choice or Q&A types (such as ceval, gsm8k, etc.), the answer extraction method is generally based on fixed regular expressions. +2. For more complex datasets such as code-related ones or mathematical ones that need to include the problem-solving process (e.g., livecodebench, humaneval, math500, etc.), there is generally a unified evaluate library associated with the dataset that can be called. +3. For some divergent or subjective datasets, it may be necessary to introduce a judge model for evaluation. -Therefore, in AISBench, a custom post - processing function for answer extraction needs to be implemented in the dataset configuration file, as shown below: -```python -# https://github.com/AISBench/benchmark/blob/master/ais_bench/benchmark/configs/datasets/gpqa/gpqa_gen_0_shot_cot_chat_prompt.py +In general, only when the evaluation involves the third category of datasets will the model's paper or technical report explicitly state it (for example, using the GPT-4-1106 API as a judge model). For the second category of datasets, the evaluate library associated with the dataset is generally used by default for answer extraction. -import re +For the first category of datasets, if the model paper or technical report indicates which tool the prompt engineering comes from, the answer extraction method mentioned in that tool can be directly used. Taking GPQA as an example, in simple-evals, the answer extraction method for GPQA is based on a fixed regular expression: +```python +# https://github.com/openai/simple-evals/blob/main/common.py +ANSWER_PATTERN_MULTICHOICE = r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?" -def gpqa_extract_answer(text): +# https://github.com/openai/simple-evals/blob/main/gpqa_eval.py +match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) +``` +Therefore, the answer extraction method in AISBench should be modified to be exactly the same regular expression as in simple-evals: +```python +# https://github.com/AISBench/benchmark/blob/master/ais_bench/benchmark/datasets/gpqa.py +@TEXT_POSTPROCESSORS.register_module() # Answer extraction function: extracts one option from A, B, C, D from the original model response string +def GPQA_Simple_Eval_postprocess(text: str) -> str: """ - Extract the final answer option (A/B/C/D) from the model - generated reasoning text + Extract one option from A, B, C, D from the original model response string as the answer. + + :param text: The original model response string. + :return: The extracted answer option (A, B, C, D). Returns None if no match is found. """ - ANSWER_PATTERN = r"Answer[ \t]*:[ \t]*\$?([A - D])\$?" + ANSWER_PATTERN = r"(?i)Answer[ \t]*:[ \t]*\$?([A-D])\$?" match = re.search(ANSWER_PATTERN, text) if match: return match.group(1) return None + +# https://github.com/AISBench/benchmark/blob/master/ais_bench/benchmark/configs/datasets/gpqa/gpqa_gen_0_shot_cot_chat_prompt.py + from ais_bench.benchmark.datasets import GPQADataset, GPQA_Simple_Eval_postprocess, GPQAEvaluator gpqa_eval_cfg = dict(evaluator=dict(type=GPQAEvaluator), - pred_postprocessor=dict(type=GPQA_Simple_Eval_postprocess, func=gpqa_extract_answer)) # Pass in the custom answer extraction function, which can also be directly defined in the dataset configuration file + pred_postprocessor=dict(type=GPQA_Simple_Eval_postprocess)) # Pass in the custom answer extraction function. The function itself can also be defined directly in the dataset configuration file ``` ---- - ### Align the Accuracy Evaluation Metrics Typically, model evaluation results are presented in a table. Take the results from DeepSeek as an example: -| Model | AIME 2024 pass@1 | AIME 2024 cons@64 | MATH - 500 pass@1 | GPQA Diamond pass@1 | LiveCodeBench pass@1 | CodeForces rating | +| Model | AIME 2024 pass@1 | AIME 2024 cons@64 | MATH-500 pass@1 | GPQA Diamond pass@1 | LiveCodeBench pass@1 | CodeForces rating | | ----- | ---------------- | ----------------- | ----------------- | ------------------- | -------------------- | ----------------- | -| GPT - 4o - 0513 | 9.3 | 13.4 | 74.6 | 49.9 | 32.9 | 759 | -| Claude - 3.5 - Sonnet - 1022 | 16.0 | 26.7 | 78.3 | 65.0 | 38.9 | 717 | -| o1 - mini | 63.6 | 80.0 | 90.0 | 60.0 | 53.8 | 1820 | +| GPT-4o-0513 | 9.3 | 13.4 | 74.6 | 49.9 | 32.9 | 759 | +| Claude-3.5-Sonnet-1022 | 16.0 | 26.7 | 78.3 | 65.0 | 38.9 | 717 | +| o1-mini | 63.6 | 80.0 | 90.0 | 60.0 | 53.8 | 1820 | -Here, `cons@64` and `pass@1` represent accuracy evaluation metrics. For detailed explanations of these metrics, refer to [Accuracy Metric Description](../base_tutorials/results_intro/accuracy_metric.md#ii - definition - and - relationship - between - passk - consk - and - avgn). +Here, `cons@64` and `pass@1` represent accuracy evaluation metrics. For detailed explanations of these metrics, refer to [Accuracy Metric Description](../base_tutorials/results_intro/accuracy_metric.md#ii-definitions-and-relationships-of-passk-consk-avgn). Taking GPQA as an example, the table shows that `pass@1` is used as the accuracy evaluation metric. The description of pass@1 in the DeepSeek R1 paper is as follows: -> ..., and report pass@1 using a non - zero temperature. Specifically, we use a sampling temperature of 0.6 and a top - 𝑝 value of 0.95 to generate 𝑘 responses (typically between 4 and 64, depending on the test set size) for each question. Pass@1 is then calculated as -> ${\text{pass@1}} = \frac{1}{n} \sum_{i = 1}^{n} p_i$ + +> ..., and report pass@1 using a non-zero temperature. Specifically, we use a sampling temperature of 0.6 and a top-𝑝 value of 0.95 to generate 𝑘 responses (typically between 4 and 64, depending on the test set size) for each question. Pass@1 is then calculated as +> ${\text{pass@1}} = \frac{1}{n} \sum_{i=1}^{n} p_i$ Then in AISBench, configure the model configuration file as follows: ```python @@ -182,13 +226,10 @@ After the precision evaluation phase, the results will be recorded in the logs a ``` Among them, `avg@4` has the same meaning as `pass@1` (average over 4 runs) in DeepSeek. - > ⚠️ While `n` only affects the fluctuation range of the evaluation results and not the mathematical expectation, a larger `n` means more repeated runs for each test case, leading to higher resource consumption. When reproducing accuracy, adjustments should be made based on the actual resource availability. > 💡 If a paper does not specify the accuracy evaluation metric for a dataset, `pass@1` is generally used by default. Thus, omitting the configuration of `n` and `k` in the AISBench dataset configuration file defaults to `pass@1`. ---- - ## References -- DeepSeek R1 Hugging Face Model Card: https://huggingface.co/deepseek - ai/DeepSeek - R1 -- DeepSeek R1 Paper: https://github.com/deepseek - ai/DeepSeek - R1/blob/main/DeepSeek_R1.pdf \ No newline at end of file +- DeepSeek R1 Hugging Face Model Card: https://huggingface.co/deepseek-ai/DeepSeek-R1 +- DeepSeek R1 Paper: https://github.com/deepseek-ai/DeepSeek-R1/blob/main/DeepSeek_R1.pdf \ No newline at end of file diff --git a/docs/source_en/conf.py b/docs/source_en/conf.py index d90e03bb..6b9d5f58 100644 --- a/docs/source_en/conf.py +++ b/docs/source_en/conf.py @@ -35,6 +35,7 @@ 'sphinx.ext.imgconverter', # 支持图片格式转换 'sphinx.ext.mathjax', # 支持数学公式 'sphinx.ext.viewcode', # 查看代码源文件 + 'sphinx_design', # 支持 tab-set、card 等 UI 组件 ] # 4. 若使用 Markdown，需指定源文件后缀 @@ -58,6 +59,7 @@ 'dollarmath', # 支持 $ 分隔的数学公式 'html_admonition', # 支持 HTML 警告框 'replacements', # 支持文本替换 + 'colon_fence', # 支持 ::: 栅栏指令（用于 tab-set 等 sphinx_design 组件） ] # （可选）配置 Mermaid 输出格式 diff --git a/docs/source_en/extended_benchmark/agent/harbor_bench.md b/docs/source_en/extended_benchmark/agent/harbor_bench.md index bf0e6712..275a787a 100644 --- a/docs/source_en/extended_benchmark/agent/harbor_bench.md +++ b/docs/source_en/extended_benchmark/agent/harbor_bench.md @@ -163,6 +163,8 @@ AISBench modified dataset repository: [https://github.com/AISBench/terminal-benc Modify `ais_bench/configs/agent_example/harbor_terminal_bench_2_task.py` under AISBench tool root directory: +> 💡 The above `harbor_terminal_bench_2_task.py` is a concrete application of the [custom configuration file approach](../../advanced_tutorials/run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. You can refer to this example file to write a configuration file that meets specific needs. See [Running AISBench with Custom Configuration Files](../../advanced_tutorials/run_custom_config.md) for details. + ```python models = [ dict( @@ -204,7 +206,7 @@ datasets.append( # ...... ), ) -) + # ...... ``` diff --git a/docs/source_en/extended_benchmark/agent/swe_bench.md b/docs/source_en/extended_benchmark/agent/swe_bench.md index 59e4865e..ec1c7d49 100644 --- a/docs/source_en/extended_benchmark/agent/swe_bench.md +++ b/docs/source_en/extended_benchmark/agent/swe_bench.md @@ -21,6 +21,8 @@ Directory `ais_bench/configs/swe_bench_examples/` provides the following example - `mini_swe_agent_swe_bench_multilingual.py`: SWE-bench Multilingual (`SWE-bench/SWE-bench_Multilingual`) — multilingual issue statements. - `mini_swe_agent_swe_bench_multilingual_mini.py`: SWE-bench Multilingual Mini (**15**/**30**/**60** instances) — an AISBench-constructed Multilingual subset designed to significantly reduce evaluation cost; see the dataset card and construction repository: `https://modelers.cn/datasets/AISBench/SWE-Bench_Multilingual_mini` and `https://github.com/AISBench/datasets/tree/main/mini_datasets/swe_bench_multiligual_mini`. +> 💡 The example configuration files mentioned above are concrete applications of the [custom configuration file approach](../../advanced_tutorials/run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. You can refer to these example files to write a configuration file that meets specific needs. See [Running AISBench with Custom Configuration Files](../../advanced_tutorials/run_custom_config.md) for details. + ## 2. Prerequisites Before running, make sure the following dependencies are available: diff --git a/docs/source_en/extended_benchmark/agent/swe_bench_pro.md b/docs/source_en/extended_benchmark/agent/swe_bench_pro.md index 00501e8d..20303ab4 100644 --- a/docs/source_en/extended_benchmark/agent/swe_bench_pro.md +++ b/docs/source_en/extended_benchmark/agent/swe_bench_pro.md @@ -19,6 +19,8 @@ Directory `ais_bench/configs/swe_bench_pro_examples/` provides the following exa - `mini_swe_agent_swe_bench_pro_mini.py`: SWE-bench Pro Mini — commonly used for quick iterations. - `mini_swe_agent_swe_bench_pro_full.py`: SWE-bench Pro Full — the full test set. +> 💡 The example configuration files mentioned above are concrete applications of the [custom configuration file approach](../../advanced_tutorials/run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. You can refer to these example files to write a configuration file that meets specific needs. See [Running AISBench with Custom Configuration Files](../../advanced_tutorials/run_custom_config.md) for details. + ## 2. Prerequisites Before running, make sure the following dependencies are available: diff --git a/docs/source_en/extended_benchmark/agent/tau2_bench.md b/docs/source_en/extended_benchmark/agent/tau2_bench.md index d98c00c7..74a36126 100644 --- a/docs/source_en/extended_benchmark/agent/tau2_bench.md +++ b/docs/source_en/extended_benchmark/agent/tau2_bench.md @@ -64,6 +64,8 @@ Ensure local or cloud deployment of tested inference services following OpenAI c ### 3. Configure Custom Configuration File for τ²-Bench Tasks 1. Modify necessary configurations in `ais_bench/configs/agent_example/tau2_bench_task.py` under AISBench tool root directory (mainly configuring information about tested inference services and user-simulating inference services) + +> 💡 The above `tau2_bench_task.py` is a concrete application of the [custom configuration file approach](../../advanced_tutorials/run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. You can refer to this example file to write a configuration file that meets specific needs. See [Running AISBench with Custom Configuration Files](../../advanced_tutorials/run_custom_config.md) for details. ```python # ...... models = [ @@ -226,11 +228,11 @@ for task in sub_tasks: +-----------------------------------+-----------+------------------------------------------------------------+-------------+----------+-------------------------------------------------+---------------------+ | Task Name | Process | Progress | Time Cost | Status | Log Path | Extend Parameters | +===================================+===========+============================================================+=============+==========+=================================================+=====================+ -| openai-v1-chat/tau2_bench_airline | 1856223 | [###### ] 30/150 Running TAU2 Bench | 0:07:13 | running | logs/eval/openai-v1-chat/tau2_bench_airline.out | None | +| openai-v1-chat/tau2_bench_airline | 1856223 | [###### ] 30/250 Running TAU2 Bench | 0:07:13 | running | logs/eval/openai-v1-chat/tau2_bench_airline.out | None | +-----------------------------------+-----------+------------------------------------------------------------+-------------+----------+-------------------------------------------------+---------------------+ -| openai-v1-chat/tau2_bench_retail | 1856224 | [###### ] 75/342 Running TAU2 Bench | 0:11:56 | running | logs/eval/openai-v1-chat/tau2_bench_retail.out | None | +| openai-v1-chat/tau2_bench_retail | 1856224 | [###### ] 75/568 Running TAU2 Bench | 0:11:56 | running | logs/eval/openai-v1-chat/tau2_bench_retail.out | None | +-----------------------------------+-----------+------------------------------------------------------------+-------------+----------+-------------------------------------------------+---------------------+ -| openai-v1-chat/tau2_bench_telecom | 1856222 | [###### ] 76/342 Running TAU2 Bench | 1:09:51 | running | logs/eval/openai-v1-chat/tau2_bench_telecom.out | None | +| openai-v1-chat/tau2_bench_telecom | 1856222 | [###### ] 76/568 Running TAU2 Bench | 1:09:51 | running | logs/eval/openai-v1-chat/tau2_bench_telecom.out | None | +-----------------------------------+-----------+------------------------------------------------------------+-------------+----------+-------------------------------------------------+---------------------+ ``` diff --git a/docs/source_en/extended_benchmark/lmm_generate/gedit_bench.md b/docs/source_en/extended_benchmark/lmm_generate/gedit_bench.md index 63ab0b58..d8534c0f 100644 --- a/docs/source_en/extended_benchmark/lmm_generate/gedit_bench.md +++ b/docs/source_en/extended_benchmark/lmm_generate/gedit_bench.md @@ -116,6 +116,8 @@ Place the dataset in the `${PATH_TO_WORKSPACE}/benchmark/ais_bench/datasets` dir In the container, navigate to the `${PATH_TO_WORKSPACE}/benchmark/ais_bench/configs/lmm_example` directory, open the `multi_device_run_qwen_image_edit.py` file, and edit the following content to set the model configuration: +> 💡 The above `multi_device_run_qwen_image_edit.py` is a concrete application of the [custom configuration file approach](../../advanced_tutorials/run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. You can refer to this example file to write a configuration file that meets specific needs. See [Running AISBench with Custom Configuration Files](../../advanced_tutorials/run_custom_config.md) for details. + ```python # ...... # ====== User configuration parameters ========= @@ -134,7 +136,7 @@ Execute the following command to find the path where the `gedit_gen_0_shot_llmju ais_bench --datasets gedit_gen_0_shot_llmjudge --search ``` -Edit the judge model related configuration in the `gedit_gen_0_shot_llmjudge.py` file. The judge model configuration is the same as the regular API model configuration (you can refer to the relevant configuration tutorial in Quick Start [Model Configuration Introduction](../../get_started/quick_start.md#task-corresponding-configuration-file-modification)), but in the `judge_model` field: +Edit the judge model related configuration in the `gedit_gen_0_shot_llmjudge.py` file. The judge model configuration is the same as the regular API model configuration (you can refer to the relevant configuration tutorial in Quick Start [Model Configuration Introduction](../../get_started/quick_start.md#start-evaluation-choose-one-of-two-methods)), but in the `judge_model` field: ```python # ...... diff --git a/docs/source_en/extended_benchmark/lmm_generate/vbench.md b/docs/source_en/extended_benchmark/lmm_generate/vbench.md index 4bba0ddc..1344f337 100644 --- a/docs/source_en/extended_benchmark/lmm_generate/vbench.md +++ b/docs/source_en/extended_benchmark/lmm_generate/vbench.md @@ -2,7 +2,9 @@ **VBench** (VBench: Comprehensive Benchmark Suite for Video Generative Models) is a benchmark suite for video generative models. It organizes evaluation around perception-related metrics such as subject consistency, motion smoothness, temporal flickering, and spatial relationship (the official Standard suite has **16 dimensions**), and provides matching prompts, pipelines, and validation methods for each dimension. -AISBench has **adapted to VBench 1.0**. The repository directory `ais_bench/configs/vbench_examples/` contains **standalone configuration file** examples for running quality/semantic dimension evaluation on generated videos on **GPU** or **NPU**. **AISBench currently does not include multimodal video generation**, so please generate videos first and then run the evaluation. (For Standard mode, see the [Dataset Generation](#dataset-generation) section.) +AISBench has **adapted to VBench 1.0**. The repository directory `ais_bench/configs/vbench_examples/` contains **standalone configuration file** examples for running quality/semantic dimension evaluation on generated videos on **GPU** or **NPU**. **AISBench currently does not include multimodal video generation**, so please generate videos first and then run the evaluation. (For Standard mode, see the [Dataset Generation](#inference-result-video-generation) section.) + +> 💡 The example configuration files under `vbench_examples/` mentioned above are concrete applications of the [custom configuration file approach](../../advanced_tutorials/run_custom_config.md). The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. You can refer to these example files to write a configuration file that meets specific needs. See [Running AISBench with Custom Configuration Files](../../advanced_tutorials/run_custom_config.md) for details. ## Table of Contents @@ -11,7 +13,7 @@ AISBench has **adapted to VBench 1.0**. The repository directory `ais_bench/conf - [Configuration and Output](#configuration-and-output) - [Score Aggregation (Quality / Semantic / Total)](#score-aggregation-quality--semantic--total) - [Prompt Suite (Official Prompt Structure)](#prompt-suite-official-prompt-structure) -- [Dataset Generation](#dataset-generation) +- [Dataset Generation](#inference-result-video-generation) - [Sampling Pseudocode (Reference Official)](#sampling-pseudocode-reference-official) - [Format Requirements](#format-requirements) - [VBench-1.0-mini (AISBench Official Sampled Subset)](#vbench-10-mini-aisbench-official-sampled-subset) @@ -45,7 +47,7 @@ Some torchvision operators (such as `nms` and `roi_align`) may run only on CPU o ## Quick Start 1. **Prepare the video directory** - For both Standard and Custom modes, set `DATA_PATH` in the corresponding configuration to the root directory of the generated videos (absolute or relative path). You can also copy the configuration file, change `DATA_PATH`, and then run `ais_bench --mode eval`. (See [Dataset Generation](#dataset-generation) for video sampling notes.) + For both Standard and Custom modes, set `DATA_PATH` in the corresponding configuration to the root directory of the generated videos (absolute or relative path). You can also copy the configuration file, change `DATA_PATH`, and then run `ais_bench --mode eval`. (See [Dataset Generation](#inference-result-video-generation) for video sampling notes.) 2. **Download third-party dependencies to local cache** VBench loads multiple small model weights for video generation quality evaluation. It is recommended to download them in advance. By default, the evaluation will also try to download dependencies automatically, but downloads may fail and break the evaluation. For details, see [`vbench_cache_dependencies.md`](./vbench_cache_dependencies.md). diff --git a/docs/source_en/faqs/error_codes.md b/docs/source_en/faqs/error_codes.md index 4e604531..4ba9c72f 100644 --- a/docs/source_en/faqs/error_codes.md +++ b/docs/source_en/faqs/error_codes.md @@ -113,7 +113,7 @@ If you specified the configuration file folder path via `--config-dir` when exec When using the [randomly synthesized dataset](../advanced_tutorials/synthetic_dataset.md) in the `tokenid` scenario, the model configuration file must specify the tokenizer path. ### Solution -Assume the ais_bench evaluation tool command is `ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_tokenid --mode perf`. Then, all `path` parameters in the `models` section of the `vllm_api_stream_chat.py` configuration file (refer to [Modifying Configuration Files for Corresponding Tasks](../get_started/quick_start.md#Modifying Configuration Files for Corresponding Tasks) for the configuration file path retrieval method) must be set to the tokenizer path (usually the model weight folder path). +Assume the ais_bench evaluation tool command is `ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_tokenid --mode perf`. Then, all `path` parameters in the `models` section of the `vllm_api_stream_chat.py` configuration file (refer to [Modifying Configuration Files for Corresponding Tasks](../get_started/quick_start.md#start-evaluation-choose-one-of-two-methods) for the configuration file path retrieval method) must be set to the tokenizer path (usually the model weight folder path). ```python # ...... diff --git a/docs/source_en/get_started/quick_start.md b/docs/source_en/get_started/quick_start.md index 22af4efc..e5b8f0a7 100644 --- a/docs/source_en/get_started/quick_start.md +++ b/docs/source_en/get_started/quick_start.md @@ -1,38 +1,116 @@ # Quick Start -## Command Meaning -A single or multiple evaluation tasks executed by the AISBench command are defined by a combination of model tasks (single or multiple), dataset tasks (single or multiple), and result presentation tasks (single). Other command-line options of AISBench specify the scenario of the evaluation task (e.g., accuracy evaluation scenario, performance evaluation scenario). Take the following AISBench command as an example: + +## Preparations Before Running the Command + +- An inference service that supports the `v1/chat/completions` sub-service is required. You can refer to 🔗 [Launching an OpenAI-Compatible Server with VLLM](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server) to start the inference service. +- The gsm8k dataset is required, which can be downloaded from 🔗 [the gsm8k dataset zip package provided by opencompass](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip). Deploy the unzipped `gsm8k/` folder to the `ais_bench/datasets` folder in the root directory of the AISBench evaluation tool. + +## Start Evaluation (Choose One of Two Methods) + +| ⭐ Recommended: Using a Custom Configuration File | Alternative: Using Command-Line Arguments (Original Quick Start Method) | +| :--- | :--- | +| Modify a single file to centrally manage all configurations, with configuration written at any path | Specify via `--models` `--datasets` parameters | +| Write once, reuse multiple times | Each run requires inputting the full command | +| Supports all Python syntax for flexible extension | Only supports Cartesian product combinations | + +::::{tab-set} +:::{tab-item} ⭐ Recommended: Using a Custom Configuration File + +AISBench provides a pre-built custom configuration file [model_api_test_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_en.py), which centralizes common service-oriented inference test configurations (model selection, service address, port, generation parameters, etc.) in a single file, eliminating the need to find and modify multiple configuration files separately. This file is essentially a Python script that supports all Python syntax, allowing you to freely extend it. + +Open `ais_bench/configs/model_api_test_en.py` and modify the following configurations according to the actual situation (If you installed the tool via `pip3 install ais_bench_benchmark`, you can create `model_api_test_en.py` at any path and write the following configuration content into that file): + +```python +from mmengine.config import read_base + +with read_base(): +# Model tasks, choose one of them. For other model tasks, refer to: https://ais-bench-benchmark-rf.readthedocs.io/en/latest/base_tutorials/all_params/models.html to obtain more model tasks + # vllm_api_general is a base model that only supports text generation + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + # vllm_api_general_chat is a chat model that supports dialogue + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + # vllm_api_stream_chat is a streaming chat model that supports streaming dialogue + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + # vllm_api_general_stream is a streaming model that supports streaming generation + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + +# Dataset tasks, refer to: https://ais-bench-benchmark-rf.readthedocs.io/en/latest/get_started/datasets.html to obtain more dataset tasks + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = vllm_api_general_chat + +models[0]["path"] = "" # Specify the absolute path to the model serialized vocabulary file (generally not required for accuracy testing scenarios) +models[0]["model"] = "" # Specify the name of the model loaded on the server, configured according to the actual model name pulled by the VLLM inference service (configure as an empty string to automatically retrieve it) +models[0]["request_rate"] = 0 # Request sending frequency: send 1 request to the server every 1/request_rate seconds; if less than 0.001, all requests are sent at once +models[0]["api_key"] = "" # Custom API key, default is an empty string +models[0]["host_ip"] = "localhost" # Specify the IP of the inference service +models[0]["host_port"] = 8080 # Specify the port of the inference service +models[0]["url"] = "" # Custom URL path for accessing the inference service (needs to be configured when the base URL is not a combination of http://host_ip:host_port; after configuration, host_ip and host_port will be ignored) +models[0]["max_out_len"] = 512 # Maximum number of tokens output by the inference service +models[0]["batch_size"] = 1 # Maximum concurrency for sending requests +models[0]["trust_remote_code"] = False # Whether the tokenizer trusts remote code, default is False +models[0]["generation_kwargs"] = dict( # Model inference parameters, configured with reference to the VLLM documentation; the AISBench evaluation tool does not process these parameters and attaches them directly to the sent requests + temperature=0.01, + ignore_eos=False, +) + +# datasets[0]["path"] = ais_bench/datasets/gsm8k # Specify the absolute path of the dataset directory (required for accuracy testing scenarios) + +work_dir = 'outputs/default/' # Specify the working directory for saving task results and logs (default is outputs/default/) + +``` + +> 💡 The configuration file already pre-imports commonly used model types (`vllm_api_general`, `vllm_api_general_chat`, `vllm_api_stream_chat`, `vllm_api_general_stream`), just uncomment/modify the relevant lines to switch. For more usages of custom configuration files, please refer to 📚 [Running AISBench with a Custom Configuration File](../advanced_tutorials/run_custom_config.md). + +The selection, preparation, and usage of dataset tasks are described in the following steps: + +1. Select a dataset task from 📚 [Open Source Datasets](https://ais-bench-benchmark.readthedocs.io/en/latest/get_started/datasets.html#open-source-datasets). +2. Go to the 📚 [Detailed Introduction / Dataset Deployment](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README_en.md#dataset-deployment) for the dataset to prepare the dataset. +3. Refer to 📚 [Detailed Introduction / Available Dataset Tasks](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README_en.md#available-dataset-tasks) to select an available dataset task, and copy the corresponding task import method (e.g., `from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets`) into the custom configuration file. + +After modifying the configuration file, run the following command to start the service-oriented accuracy evaluation: + +```bash +ais_bench ais_bench/configs/model_api_test_en.py +``` + +::: +:::{tab-item} Alternative: Using Command-Line Arguments + +If you prefer the command-line argument approach, AISBench also supports specifying tasks directly via the `--models`, `--datasets`, `--summarizer` parameters. The following is the command-line approach that has **exactly the same execution effect** as the above custom configuration file approach. + +A single or multiple evaluation tasks executed by the AISBench command are defined by a combination of model tasks (single or multiple), dataset tasks (single or multiple), and result presentation tasks (single). Take the following AISBench command as an example: + ```shell ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer example ``` + This command does not specify other command-line options, so it defaults to an accuracy evaluation scenario task, where: - `--models` specifies the model task, i.e., the `vllm_api_general_chat` model task. - - `--datasets` specifies the dataset task, i.e., the `demo_gsm8k_gen_4_shot_cot_chat_prompt` dataset task. +- `--summarizer` specifies the result presentation task, i.e., the `example` result presentation task (if `--summarizer` is not specified, the `example` task is used by default in the accuracy evaluation scenario). It is generally recommended to use the default, so there is no need to specify it in the command line. -- `--summarizer` specifies the result presentation task, i.e., the `example` result presentation task (if `--summarizer` is not specified, the `example` task is used by default in the accuracy evaluation scenario). It is generally recommended to use the default, so there is no need to specify it in the command line, and subsequent commands will omit it. - -## Task Meaning Query (Optional) -The specific information (introduction, usage constraints, etc.) of the selected model task `vllm_api_general_chat`, dataset task `demo_gsm8k_gen_4_shot_cot_chat_prompt`, and result presentation task `example` can be queried from the following links respectively: -- `--models`: 📚 [Service-Oriented Inference Backend](../base_tutorials/all_params/models.md#service-oriented-inference-backend) +For multi-task evaluation, please refer to: 📚 [Multi-Task Evaluation](../base_tutorials/scenes_intro/accuracy_benchmark.md#multi-task-evaluation) for accuracy scenarios and 📚 [Multi-Task Evaluation](../base_tutorials/scenes_intro/performance_benchmark.md#multi-task-performance-evaluation) for performance scenarios. -- `--datasets`: 📚 [Open Source Datasets](../get_started/datasets.md#open-source-datasets) → 📚 [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README_en.md) +For more flexible evaluation methods with self-combined tasks, you can refer to: 📚 [Running AISBench with a Custom Configuration File](../advanced_tutorials/run_custom_config.md#running-aisbench-with-a-custom-configuration-file). -- `--summarizer`: 📚 [Result Summary Tasks](../base_tutorials/all_params/summarizer.md) +The specific information (introduction, usage constraints, etc.) of the selected model task `vllm_api_general_chat`, dataset task `demo_gsm8k_gen_4_shot_cot_chat_prompt`, and result presentation task `example` can be queried from the following links respectively: -## Preparations Before Running the Command -- `--models`: To use the `vllm_api_general_chat` model task, you need to prepare an inference service that supports the `v1/chat/completions` sub-service. You can refer to 🔗 [Launching an OpenAI-Compatible Server with VLLM](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server) to start the inference service. -- `--datasets`: To use the `demo_gsm8k_gen_4_shot_cot_chat_prompt` dataset task, you need to prepare the gsm8k dataset, which can be downloaded from 🔗 [the gsm8k dataset zip package provided by opencompass](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip). Deploy the unzipped `gsm8k/` folder to the `ais_bench/datasets` folder in the root directory of the AISBench evaluation tool. +- `--models`: 📚 [Service-Oriented Inference Backend](https://ais-bench-benchmark.readthedocs.io/en/latest/base_tutorials/all_params/models.html#service-oriented-inference-backend) +- `--datasets`: 📚 [Open Source Datasets](https://ais-bench-benchmark.readthedocs.io/en/latest/get_started/datasets.html#open-source-datasets) → 📚 [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README_en.md) +- `--summarizer`: 📚 [Result Summary Tasks](https://ais-bench-benchmark.readthedocs.io/en/latest/base_tutorials/all_params/summarizer.html) -## Modification of Configuration Files Corresponding to Tasks Each model task, dataset task, and result presentation task corresponds to a configuration file. You need to modify the content of these configuration files before running the command. The paths of these configuration files can be queried by adding `--search` to the original AISBench command. For example: + ```shell ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --search ``` + > ⚠️ **Note**: Executing the command with the `search` option will print the absolute paths of the configuration files corresponding to the tasks. Executing the query command will yield the following results: + ```shell -06/28 11:52:25 - AISBench - INFO - Searching configs... ╒══════════════╤═══════════════════════════════════════╤════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ │ Task Type │ Task Name │ Config File Path │ ╞══════════════╪═══════════════════════════════════════╪════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ @@ -43,9 +121,10 @@ Executing the query command will yield the following results: ``` -- The dataset task configuration file `demo_gsm8k_gen_4_shot_cot_chat_prompt.py` in the quick start does not require additional modifications. For an introduction to the content of the dataset task configuration file, please refer to 📚 [Configuring Open Source Datasets](../get_started/datasets.md#configuring-open-source-datasets). +- The dataset task configuration file `demo_gsm8k_gen_4_shot_cot_chat_prompt.py` in the quick start does not require additional modifications. For an introduction to the content of the dataset task configuration file, please refer to 📚 [Configuring Open Source Datasets](https://ais-bench-benchmark.readthedocs.io/en/latest/base_tutorials/all_params/datasets.html#configuring-open-source-datasets). The model configuration file `vllm_api_general_chat.py` contains configuration content related to model operation and needs to be modified according to the actual situation. The content that needs to be modified in the quick start is marked with comments. + ```python from ais_bench.benchmark.models import VLLMCustomAPIChat @@ -55,7 +134,7 @@ models = [ type=VLLMCustomAPIChat, abbr='vllm-api-general-chat', path="", # Specify the absolute path of the model serialized vocabulary file (configuration is generally not required for accuracy testing scenarios). - model="DeepSeek-R1", # Specify the name of the model loaded on the server, configured according to the actual model name pulled by the VLLM inference service (configure as an empty string to get it automatically) + model="", # Specify the name of the model loaded on the server, configured according to the actual model name pulled by the VLLM inference service (configure as an empty string to get it automatically) stream=False, request_rate=0, # Request sending frequency: send 1 request to the server every 1/request_rate seconds; if less than 0.1, all requests are sent at once use_timestamp=False, # Whether to schedule requests by dataset timestamp; used with timestamped datasets (e.g. Mooncake Trace) @@ -63,7 +142,7 @@ models = [ api_key="", # Custom API key, default is an empty string host_ip="localhost", # Specify the IP of the inference service host_port=8080, # Specify the port of the inference service - url="", # Custom access path for the inference service (required when the base URL is not http://host_ip:host_port, and will ignore host_ip and host_port) + url="", # Custom access path for the inference service (required when the base URL is not http://host_ip:host_port; after configuration, host_ip and host_port will be ignored) max_out_len=512, # Maximum number of tokens output by the inference service batch_size=1, # Maximum concurrency for sending requests trust_remote_code=False, # Whether to trust remote code in the tokenizer, default False; @@ -74,37 +153,46 @@ models = [ ) ] ``` -## Execute Command + After modifying the configuration file, run the following command to start the service-oriented accuracy evaluation: + ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt ``` +::: +:::: + ## View Task Execution Details -After executing the AISBench command, the status of the running task will be displayed on a real-time refreshing dashboard in the command line (press the "P" key to pause refreshing for copying dashboard information, and press "P" again to resume refreshing). Example: + +After executing the AISBench command, the task management dashboard will refresh in real time in the command line to show the task execution status (press the "P" key to pause/resume refreshing for copying dashboard information, and press "P" again to continue refreshing). The task management dashboard supports monitoring the detailed execution status of multiple tasks simultaneously, including task name, progress, time cost, status, log path, extended parameters, and other information. For example: + ``` Base path of result&log : outputs/default/20250628_151326 Task Progress Table (Updated at: 2025-11-06 10:08:21) Page: 1/1 Total 2 rows of data -Press Up/Down arrow to page, 'P' to PAUSE/RESUME screen refresh, 'Ctrl + C' to exit +Press Up/Down arrow to page, 'P' to PAUZE/RESUME screen refresh, 'Ctrl + C' to exit +----------------------------------+-----------+-------------------------------------------------+-------------+-------------+-------------------------------------------------+------------------------------------------------+ | Task Name | Process | Progress | Time Cost | Status | Log Path | Extend Parameters | -+==================================+===========+=================================================+=============+=============+=================================================+================================================+ ++==================================+===========+=================================================+=============+=============+================================================+================================================+ | vllm-api-general-chat/demo_gsm8k | 547141 | [############### ] 4/8 [0.5 it/s] | 0:00:11 | inferencing | logs/infer/vllm-api-general-chat/demo_gsm8k.out | {'POST': 5, 'RECV': 4, 'FINISH': 4, 'FAIL': 0} | +----------------------------------+-----------+-------------------------------------------------+-------------+-------------+-------------------------------------------------+------------------------------------------------+ + ``` Detailed logs of task execution are continuously written to the default output path, which is displayed on the real-time refreshing dashboard as `Log Path`. The `Log Path` (`logs/infer/vllm-api-general-chat/demo_gsm8k.out`) is located under the `Base path` (`outputs/default/20250628_151326`). Using the dashboard information above as an example, the path to the detailed task execution log is: + ```shell # {Base path}/{Log Path} outputs/default/20250628_151326/logs/infer/vllm-api-general-chat/demo_gsm8k.out ``` > 💡 To print detailed logs directly during execution, add the `--debug` parameter to the command: -`ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --debug` +> `ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --debug` The `Base path` (`outputs/default/20250628_151326`) contains all task execution details. After the command completes, the full execution details are structured as follows: + ```shell 20250628_151326/ ├── configs # Combined configuration file for model tasks, dataset tasks, and structure presentation tasks @@ -131,7 +219,9 @@ The `Base path` (`outputs/default/20250628_151326`) contains all task execution > ⚠️ **Note**: The content of task execution details written to disk varies across different evaluation scenarios. Please refer to the guide for the specific evaluation scenario. ### Output Results + Since there are only 8 data samples, the results will be generated quickly. Example output: + ```bash dataset version metric mode vllm_api_general_chat ----------------------- -------- -------- ----- ---------------------- diff --git a/docs/source_en/index.rst b/docs/source_en/index.rst index 5a44a5d1..26377c84 100644 --- a/docs/source_en/index.rst +++ b/docs/source_en/index.rst @@ -21,7 +21,7 @@ To help you quickly get started with AISBench Benchmark Tool, we recommend learn * The :doc:`Quick Start ` provided in this tutorial will guide you through basic accuracy evaluation configuration and execution. * The :doc:`Dataset Preparation Guide ` will help you understand the supported datasets and how to prepare them for evaluation. * The Basic Tutorial section will introduce :doc:`Evaluation Scenario Introduction `, :doc:`Evaluation Result Explanation `, and :doc:`Detailed Parameter Description ` to help you better understand the use of major evaluation scenarios. -* For a deeper understanding of advanced usage of AISBench Benchmark Tool, you can refer to the :doc:`Advanced Tutorial `. +* For a deeper understanding of advanced usage of AISBench Benchmark Tool, you can refer to the :doc:`Advanced Tutorial `. **Strongly recommended** to read :doc:`Running AISBench with a Custom Configuration File `. The configuration file is essentially a Python script that supports all Python syntax including loops, conditional statements, list comprehensions, etc. You can write configurations for models, datasets, summarizers, etc. into a single file, write once and reuse multiple times, covering nearly all evaluation scenarios. * You can refer to the :doc:`Best Practices ` section to learn best practices for using AISBench Benchmark Tool in different scenarios. * Finally, you can refer to the :doc:`Frequently Asked Questions ` section to solve problems encountered during the use of AISBench Benchmark Tool. @@ -63,6 +63,7 @@ To help you quickly get started with AISBench Benchmark Tool, we recommend learn :hidden: extended_benchmark/lmm_generate/index + extended_benchmark/agent/index .. toctree:: :maxdepth: 2 diff --git a/docs/source_zh_cn/advanced_tutorials/custom_dataset.md b/docs/source_zh_cn/advanced_tutorials/custom_dataset.md index 296737b6..e70e8fcf 100644 --- a/docs/source_zh_cn/advanced_tutorials/custom_dataset.md +++ b/docs/source_zh_cn/advanced_tutorials/custom_dataset.md @@ -105,11 +105,11 @@ ais_bench \ 命令行参考： ```shell -ais_bench ais_bench/configs/api_examples/infer_api_vllm_general.py +ais_bench ais_bench/configs/api_examples/infer_vllm_api_general.py ``` ```shell -ais_bench ais_bench/configs/api_examples/infer_api_mindie_stream_general.py +ais_bench ais_bench/configs/api_examples/infer_mindie_stream_api_general.py ``` 在原配置文件中，直接向 `datasets` 变量中添加新的项即可。同普通数据集一致，该方式下支持自定义数据集与普通数据集混用。 @@ -122,6 +122,8 @@ datasets = [ ] ``` +> 💡 上述配置文件方式本质上就是 [自定义配置文件方式](run_custom_config.md) 的简化应用。更复杂的场景（如多模型多数据集组合、自定义模型参数、裁判模型等）请参考 [自定义配置文件运行AISBench](run_custom_config.md#各场景自定义配置文件示例) 中"自定义数据集测评"示例。 + ### 数据集补充信息`.meta.json`使用指南 目前仅支持性能测评场景。ais_bench 会默认尝试对输入的数据集文件进行解析，因此在绝大多数情况下，`.meta.json` 文件都是 **不需要** 的。但是，如果原生数据集中没有指定max_tokens，或者需要通过配置进行数据采样等，则需要在 `.meta.json` 文件中进行指定。 diff --git a/docs/source_zh_cn/advanced_tutorials/judge_model_evaluate.md b/docs/source_zh_cn/advanced_tutorials/judge_model_evaluate.md index 0dd6c731..db89cba6 100644 --- a/docs/source_zh_cn/advanced_tutorials/judge_model_evaluate.md +++ b/docs/source_zh_cn/advanced_tutorials/judge_model_evaluate.md @@ -191,12 +191,17 @@ outputs/default/20260305_153318/logs/eval/vllm-api-general-chat/aime2025-judge.o ## 其他精度评测功能场景 从裁判模型的快速上手章节可以看到，除了需要额外修改数据配置文件中裁判模型的配置，其他测评执行方式是与常规测评执行方式是完全一致的，因此其他精度评测功能场景的执行方式也是完全一致的。 + +## 通过自定义配置文件实现 + +> 💡 上述裁判模型测评场景也可以通过 [自定义配置文件方式](run_custom_config.md) 实现。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将被测模型、裁判模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。详见 [自定义配置文件运行AISBench](run_custom_config.md#各场景自定义配置文件示例) 中"裁判模型测评"示例。 + ### 多任务测评 参考[精度评测场景多任务测评](../base_tutorials/scenes_intro/accuracy_benchmark.md#多任务测评) ### 多任务并行测评 参考[精度评测场景多任务并行测评](../base_tutorials/scenes_intro/accuracy_benchmark.md#多任务并行测评) ### 中断续测 & 失败用例重测 -参考[精度评测场景中断续测 & 失败用例重测](../base_tutorials/scenes_intro/accuracy_benchmark.md#中断续测-失败用例重测) +参考[精度评测场景中断续测 & 失败用例重测](../base_tutorials/scenes_intro/accuracy_benchmark.md#中断续测--失败用例重测) > ⚠️ 注意，--reuse 重新补全被测模型推理结果后，裁判模型会从0开始重新对全部完整的推理结果进行判断，历史判断过的结果将不会使用。 ### 合并子数据集推理 参考[精度评测场景合并子数据集推理](../base_tutorials/scenes_intro/accuracy_benchmark.md#合并子数据集推理) diff --git a/docs/source_zh_cn/advanced_tutorials/multimodal_benchmark.md b/docs/source_zh_cn/advanced_tutorials/multimodal_benchmark.md index 9d2f191b..891b458c 100644 --- a/docs/source_zh_cn/advanced_tutorials/multimodal_benchmark.md +++ b/docs/source_zh_cn/advanced_tutorials/multimodal_benchmark.md @@ -28,6 +28,8 @@ ## 快速入门 + +> 💡 多模态测评场景也可以通过 [自定义配置文件方式](run_custom_config.md) 实现。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将多模态模型、多模态数据集、summarizer 等配置写入一个文件，一次编写、多次复用。详见 [自定义配置文件运行AISBench](run_custom_config.md)。 ### 多模态输入格式 服务化的多模态数据输入有多种格式，以图片+文本输入举例如下： - 方式1：本地文件格式，默认方法 diff --git a/docs/source_zh_cn/advanced_tutorials/multiturn_benchmark.md b/docs/source_zh_cn/advanced_tutorials/multiturn_benchmark.md index a98b8efc..fb917eda 100644 --- a/docs/source_zh_cn/advanced_tutorials/multiturn_benchmark.md +++ b/docs/source_zh_cn/advanced_tutorials/multiturn_benchmark.md @@ -151,6 +151,12 @@ ais_bench --models vllm_api_stream_chat --datasets sharegpt_gen -m perf --debug ### 性能细节查看 执行AISBench命令后，任务执行更多细节最终会落盘在默认的输出路径，这个输出路径在运行中的打屏日志中有提示，例如： + +## 通过自定义配置文件实现 + +> 💡 上述多轮对话性能测评场景也可以通过 [自定义配置文件方式](run_custom_config.md) 实现。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。详见 [自定义配置文件运行AISBench](run_custom_config.md#各场景自定义配置文件示例) 中"多轮对话性能测评"示例。 + +### 性能细节查看 ```shell 06/28 15:13:26 - AISBench - INFO - Current exp folder: outputs/default/20250628_151326 ``` diff --git a/docs/source_zh_cn/advanced_tutorials/rps_distribution.md b/docs/source_zh_cn/advanced_tutorials/rps_distribution.md index c286b92e..6fa2913c 100644 --- a/docs/source_zh_cn/advanced_tutorials/rps_distribution.md +++ b/docs/source_zh_cn/advanced_tutorials/rps_distribution.md @@ -329,6 +329,10 @@ $\lambda_i = \lambda_{\text{start}} \times \left(\frac{\lambda_{end}}{\lambda_{s 4. **压测场景下，控制连接数创建的频率，不控制请求发送速率（每个连接创建后，会不间断的执行请求发送和处理返回）** 5. **多轮对话场景下，仅第一轮的请求分布有效** +## 通过自定义配置文件实现 + +> 💡 上述 RPS 分布控制参数（`traffic_cfg`）在 [自定义配置文件方式](run_custom_config.md) 中同样适用。只需在模型配置的 dict 中添加 `traffic_cfg` 字段即可。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。详见 [自定义配置文件运行AISBench](run_custom_config.md)。 + --- ## 配置与可视化示例 diff --git a/docs/source_zh_cn/advanced_tutorials/run_custom_config.md b/docs/source_zh_cn/advanced_tutorials/run_custom_config.md index 025832ef..f70c5698 100644 --- a/docs/source_zh_cn/advanced_tutorials/run_custom_config.md +++ b/docs/source_zh_cn/advanced_tutorials/run_custom_config.md @@ -2,13 +2,677 @@ AISBench常规命令调用方式是通过`--models`指定模型任务，通过`--datasets`指定数据集任务，通过`--summarizer`指定结果呈现任务来绝对运行的测评任务，AISBench同样也支持指定自定义的配置文件将这三类任务对应的配置文件信息组合在一起，从而实现自定义的任务组合运行。 +## 为什么使用自定义配置文件 + +AISBench 提供了两种运行方式：**命令行参数方式（CLI）** 与 **自定义配置文件方式**。在实际使用中，推荐优先使用自定义配置文件方式，原因如下： + +| 对比维度 | CLI 方式 | 配置文件方式 | +| --- | --- | --- | +| **可复用性** | 每次运行需要重新输入完整命令 | 配置文件可保存、版本管理、反复使用 | +| **表达能力** | 只能通过参数指定模型/数据集名称 | 可以精确控制模型参数、数据集采样范围、推理配置等所有细节 | +| **组合灵活性** | 仅支持笛卡尔积组合 | 支持 `model_dataset_combinations` 自定义任意模型-数据集配对 | +| **参数覆盖** | 无法修改预设模型/数据集内部参数 | 可直接修改 `abbr`、`test_range`、`host_ip`、`host_port` 等任意字段 | +| **批量运行** | 需要多次执行命令 | 一个配置文件即可同时运行多模型、多数据集组合 | +| **团队协作** | 命令难以共享和追溯 | 配置文件即代码，可提交到代码仓库进行 review 和复用 | + +**总结**：CLI 方式适合快速验证，配置文件方式适合正式的、可复现的、复杂的测评场景。 + +## 配置文件即 Python 脚本 + +AISBench 的自定义配置文件本质上就是一个 Python 脚本。这意味着你可以在配置文件中使用所有 Python 语法特性来灵活构建测评任务。 + +### 使用 for 循环批量构建模型配置 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + +datasets = gsm8k_0_shot_cot_str + +models = [] +for port in [8080, 8081, 8082]: + models.append( + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr=f'vllm-api-chat-port-{port}', + path="", + model="", + request_rate=0, + retry=2, + host_ip="localhost", + host_port=port, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.5, top_k=10, top_p=0.95), + ) + ) + +work_dir = 'outputs/multi_port_benchmark/' +``` + +### 使用列表推导式批量添加数据集 + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat +datasets = [ + dict(d, abbr=f'my_{d["abbr"]}', reader_cfg=dict(d.get('reader_cfg', {}), test_range='[0:100]')) + for d in datasets +] + +models = vllm_api_general_chat +work_dir = 'outputs/my_benchmark/' +``` + +### 条件配置：根据环境变量切换 + +```python +import os +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat, VLLMCustomAPI + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + +datasets = gsm8k_0_shot_cot_str + +use_stream = os.environ.get('USE_STREAM', 'false').lower() == 'true' +model_type = VLLMCustomAPIChat if use_stream else VLLMCustomAPI + +models = [ + dict( + attr="service", + type=model_type, + abbr='vllm-api-conditional', + path="", + model="", + stream=use_stream, + request_rate=0, + retry=2, + host_ip=os.environ.get('HOST_IP', 'localhost'), + host_port=int(os.environ.get('HOST_PORT', '8080')), + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.5, top_k=10, top_p=0.95), + ) +] + +work_dir = 'outputs/conditional_benchmark/' +``` + +### 使用 `.copy()` 复用并修改模型配置 + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_0_shot_cot_str + +model_high_temp = vllm_api_general_chat.copy() +model_high_temp[0]['abbr'] = vllm_api_general_chat[0]['abbr'] + '-high-temp' +model_high_temp[0]['generation_kwargs']['temperature'] = 0.9 + +model_low_temp = vllm_api_general_chat.copy() +model_low_temp[0]['abbr'] = vllm_api_general_chat[0]['abbr'] + '-low-temp' +model_low_temp[0]['generation_kwargs']['temperature'] = 0.1 + +models = model_high_temp + model_low_temp +work_dir = 'outputs/temperature_comparison/' +``` + +## 配置文件完整变量参考 + +自定义配置文件中可以定义以下顶层变量。所有变量均为可选，但至少需要定义 `models` 和 `datasets` 才能运行推理任务。 + +| 变量名 | 类型 | 是否必需 | 说明 | +| --- | --- | --- | --- | +| `models` | `list[dict]` | 是（推理时） | 模型配置列表。每个元素是一个字典，至少包含 `type`（模型类）、`abbr`（唯一标识）字段。服务化模型还需 `attr="service"`、`host_ip`、`host_port` 等；本地模型还需 `path`、`tokenizer_path` 等 | +| `datasets` | `list[dict]` | 是（推理时） | 数据集配置列表。每个元素是一个字典，至少包含 `type`（数据集类）、`abbr`（唯一标识）、`reader_cfg`、`infer_cfg`、`eval_cfg` 字段 | +| `summarizer` | `dict` | 否 | 结果汇总器配置。通常从 `ais_bench.benchmark.configs.summarizers.example` 导入。包含 `attr` 和 `summary_groups` 字段 | +| `model_dataset_combinations` | `list[dict]` | 否 | 自定义模型-数据集配对列表。每个元素为 `dict(models=[...], datasets=[...])`。不指定时，默认对 `models` 和 `datasets` 做笛卡尔积组合 | +| `work_dir` | `str` | 否 | 工作目录，推理结果和日志将输出到此目录下。默认为 `outputs/default/` | +| `infer` | `dict` | 否 | 推理流程配置。包含 `partitioner`（分区器）、`runner`（运行器，内含 `max_num_workers` 和 `task`）。不指定时使用默认推理流程 | +| `eval` | `dict` | 否 | 评测流程配置。结构同 `infer`。仅在需要独立评测阶段时使用（如 SWE-Bench、VBench 等场景） | + +### models 字段详解 + +每个模型配置字典的常用字段： + +| 字段 | 类型 | 说明 | +| --- | --- | --- | +| `type` | class | 模型类，如 `VLLMCustomAPIChat`、`VLLMCustomAPI`、`HuggingFaceBaseModel`、`HuggingFacewithChatTemplate` 等 | +| `abbr` | `str` | 模型唯一标识，用于结果表格中的列名。同一配置文件中相同 `abbr` 的模型与数据集组合会被视为重复任务而跳过 | +| `attr` | `str` | 模型属性，服务化模型为 `"service"`，本地模型为 `"local"` | +| `path` | `str` | 模型路径（本地模型必填，服务化模型可为空字符串） | +| `model` | `str` | 服务化推理时指定的模型名称 | +| `host_ip` | `str` | 推理服务 IP 地址（服务化模型） | +| `host_port` | `int` | 推理服务端口（服务化模型） | +| `stream` | `bool` | 是否使用流式推理 | +| `max_out_len` | `int` | 最大输出 token 数 | +| `batch_size` | `int` | 推理 batch size | +| `max_seq_len` | `int` | 最大输入序列长度 | +| `request_rate` | `int` | 请求速率限制，0 表示不限制 | +| `retry` | `int` | 请求失败重试次数 | +| `generation_kwargs` | `dict` | 生成参数，如 `temperature`、`top_k`、`top_p`、`seed` 等 | +| `tokenizer_path` | `str` | Tokenizer 路径（本地模型） | +| `model_kwargs` | `dict` | 模型加载参数（本地模型），如 `device_map` | +| `tokenizer_kwargs` | `dict` | Tokenizer 参数（本地模型），如 `padding_side` | +| `run_cfg` | `dict` | 多卡/多机运行配置（本地模型），如 `dict(num_gpus=1, num_procs=1)` | +| `pred_postprocessor` | `dict` | 模型输出后处理器，如 `dict(type=extract_non_reasoning_content)` | + +### datasets 字段详解 + +每个数据集配置字典的常用字段： + +| 字段 | 类型 | 说明 | +| --- | --- | --- | +| `type` | class | 数据集类，如 `GSM8KDataset`、`MATHDataset`、`SyntheticDataset` 等 | +| `abbr` | `str` | 数据集唯一标识，用于结果表格中的行名 | +| `path` | `str` | 数据集文件路径 | +| `reader_cfg` | `dict` | 读取器配置，包含 `input_columns`、`output_column`，可选 `test_range` 控制采样范围（如 `'[0:100]'`） | +| `infer_cfg` | `dict` | 推理配置，包含 `prompt_template`、`retriever`、`inferencer` | +| `eval_cfg` | `dict` | 评测配置，包含 `evaluator` 和可选的 `pred_postprocessor` | +| `judge_infer_cfg` | `dict` | 裁判模型推理配置（需要 LLM Judge 的数据集），包含 `judge_model`、`judge_dataset_type`、`prompt_template`、`retriever`、`inferencer` | + +### infer 字段详解 + +```python +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) +``` + ## 使用说明 ```bash ais_bench ais_bench/configs/{模型类型}_examples/{任务配置文件名} # 示例： ais_bench ais_bench/configs/api_examples/infer_vllm_api_general.py - ``` +``` + +## 各场景自定义配置文件示例 + +### 1. 服务化精度测评 + +通过 API 访问推理服务，使用真实数据集进行精度测评。适用于 vLLM、MindIE、TGI、Triton 等服务化部署场景。 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_chat_prompt import gsm8k_datasets as gsm8k_0_shot_cot_chat + +datasets = [*gsm8k_0_shot_cot_chat] + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-general-chat', + model="", + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict( + temperature=0.5, + top_k=10, + top_p=0.95, + seed=None, + repetition_penalty=1.03, + ) + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/api-vllm-general-chat/' +``` + +### 2. 纯模型精度测评 + +使用 HuggingFace 本地模型直接进行推理测评，无需部署服务。 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFaceBaseModel +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_chat_prompt import gsm8k_datasets as gsm8k_0_shot_cot_chat + +datasets = [*gsm8k_0_shot_cot_chat] + +models = [ + dict( + type=HuggingFaceBaseModel, + abbr='hf-base-model', + path='THUDM/chatglm-6b', + tokenizer_path='THUDM/chatglm-6b', + model_kwargs=dict(device_map='auto'), + tokenizer_kwargs=dict(padding_side='left'), + generation_kwargs=dict( + temperature=0.5, + top_k=10, + top_p=0.95, + do_sample=True, + seed=None, + repetition_penalty=1.03, + ), + max_out_len=100, + batch_size=1, + max_seq_len=2048, + batch_padding=True, + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/hf-base-model/' +``` + +### 3. 服务化性能测评 + +使用合成数据集对推理服务进行性能压测，输出 TTFT（首 Token 延迟）、TPOT（每 Token 延迟）、E2EL（端到端延迟）等指标。 + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import ( + synthetic_datasets, + ) + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import ( + models as vllm_api_general_stream, + ) + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import ( + models as vllm_api_stream_chat, + ) + +datasets = synthetic_datasets + +vllm_api_general_stream[0]["abbr"] = "demo-" + vllm_api_general_stream[0]["abbr"] +vllm_api_stream_chat[0]["abbr"] = "demo-" + vllm_api_stream_chat[0]["abbr"] + +models = vllm_api_general_stream + vllm_api_stream_chat + +work_dir = "outputs/demo_api-vllm-stream-perf/" +``` + +运行命令： + +```bash +ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py -m perf +``` + +### 4. 合成数据集性能测评 + +自定义合成数据集的参数，控制请求数量、输入/输出 token 长度分布等。 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever +from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer +from ais_bench.benchmark.datasets import SyntheticDataset, MATHEvaluator, math_postprocess_v2 + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import ( + models as vllm_api_general_stream, + ) + +synthetic_config = { + "Type": "string", + "RequestCount": 100, + "TrustRemoteCode": False, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": 1, "MaxValue": 500} + }, + "Output": { + "Method": "gaussian", + "Params": {"Mean": 200, "Var": 100, "MinValue": 1, "MaxValue": 500} + } + }, +} + +datasets = [ + dict( + abbr='synthetic_custom', + type=SyntheticDataset, + config=synthetic_config, + reader_cfg=dict(input_columns=['question', 'max_out_len'], output_column='answer'), + infer_cfg=dict( + prompt_template=dict(type=PromptTemplate, template="{question}"), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + eval_cfg=dict( + evaluator=dict(type=MATHEvaluator, version='v2'), + pred_postprocessor=dict(type=math_postprocess_v2), + ), + ) +] + +models = vllm_api_general_stream +work_dir = 'outputs/synthetic_perf_custom/' +``` + +### 5. 多模型多数据集组合 + +同时测评多个模型在多个数据集上的表现，利用笛卡尔积自动组合。 + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.datasets.mmlu.mmlu_gen_5_shot_str import mmlu_datasets as mmlu_5_shot_str + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat + mmlu_5_shot_str +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat + +work_dir = 'outputs/multi_model_multi_dataset/' +``` + +### 6. 自定义模型-数据集配对 + +通过 `model_dataset_combinations` 精确控制哪些模型与哪些数据集组合，避免不必要的笛卡尔积。 + +```python +from mmengine.config import read_base + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_0_shot_cot_str import gsm8k_datasets as gsm8k_0_shot_cot_str + from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat + +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), + dict(models=[models[2]], datasets=[datasets[0], datasets[1]]), +] + +work_dir = 'outputs/custom_combinations/' +``` + +### 7. 裁判模型测评 + +对于需要 LLM Judge 评判的数据集（如 AIME 2025），在数据集的 `judge_infer_cfg` 中配置裁判模型。 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.aime2025.aime2025_gen_0_shot_llmjudge import aime2025_datasets + +datasets = aime2025_datasets + +datasets[0]['judge_infer_cfg']['judge_model']['host_ip'] = 'localhost' +datasets[0]['judge_infer_cfg']['judge_model']['host_port'] = 8081 + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-judge-eval', + path="", + model="", + stream=True, + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + pred_postprocessor=dict(type=extract_non_reasoning_content), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/judge_eval/' +``` + +### 8. 稳态性能测评 + +通过控制 `request_rate` 参数和 `stream` 参数，模拟稳态负载下的性能表现。 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPI + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import ( + synthetic_datasets, + ) + +datasets = synthetic_datasets + +models = [] +for rate in [0, 5, 10, 20]: + model_cfg = dict( + attr="service", + type=VLLMCustomAPI, + abbr=f'vllm-api-steady-rate-{rate}', + path="", + model="", + stream=True, + request_rate=rate, + use_timestamp=False, + retry=2, + api_key="", + host_ip="localhost", + host_port=8080, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + ) + models.append(model_cfg) + +work_dir = 'outputs/steady_state_perf/' +``` + +### 9. 多轮对话性能测评 + +使用 ShareGPT 或 MTBench 多轮对话数据集进行性能测评。 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.utils.postprocess.model_postprocessors import extract_non_reasoning_content + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.sharegpt.sharegpt_gen import sharegpt_datasets + +datasets = sharegpt_datasets + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr="vllm-multiturn-api-chat-stream", + path="", + model="", + stream=True, + request_rate=0, + retry=2, + api_key="", + host_ip="localhost", + host_port=8080, + url="", + max_out_len=512, + batch_size=1, + trust_remote_code=False, + generation_kwargs=dict(temperature=0.01, ignore_eos=False), + pred_postprocessor=dict(type=extract_non_reasoning_content), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/multi_turn_benchmark/' +``` + +### 10. 自定义数据集测评 + +当需要使用自己的数据集进行测评时，可以通过自定义数据集配置实现。 + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever +from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer +from ais_bench.benchmark.datasets import CustomDataset +from ais_bench.benchmark.openicl.icl_evaluator import AccEvaluator + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + +datasets = [ + dict( + abbr='my_custom_dataset', + type=CustomDataset, + path='/path/to/your/dataset.jsonl', + reader_cfg=dict( + input_columns=['question'], + output_column='answer', + ), + infer_cfg=dict( + prompt_template=dict( + type=PromptTemplate, + template='{question}', + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), + ), + eval_cfg=dict( + evaluator=dict(type=AccEvaluator), + pred_role='BOT', + ), + meta_path='', + ) +] + +models = [ + dict( + attr="service", + type=VLLMCustomAPIChat, + abbr='vllm-api-custom-dataset', + model="", + request_rate=0, + retry=2, + host_ip="localhost", + host_port=8080, + max_out_len=512, + batch_size=1, + generation_kwargs=dict(temperature=0.5, top_k=10, top_p=0.95), + ) +] + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict( + type=LocalRunner, + max_num_workers=2, + task=dict(type=OpenICLApiInferTask), + ), +) + +work_dir = 'outputs/custom_dataset/' +``` ## 自定义配置文件精度测评使用样例 @@ -19,8 +683,8 @@ ais_bench ais_bench/configs/api_examples/infer_vllm_api_general.py ```python from mmengine.config import read_base from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask from ais_bench.benchmark.models import VLLMCustomAPIChat with read_base(): @@ -29,15 +693,14 @@ with read_base(): from ais_bench.benchmark.configs.datasets.math.math500_gen_0_shot_cot_chat_prompt import math_datasets as math500_gen_0_shot_cot_chat from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general -# 只取部分样本进行 demo 测试 gsm8k_0_shot_cot_str[0]['abbr'] = 'demo_' + gsm8k_0_shot_cot_str[0]['abbr'] gsm8k_0_shot_cot_str[0]['reader_cfg']['test_range'] = '[0:8]' math500_gen_0_shot_cot_chat[0]['abbr'] = 'demo_' + math500_gen_0_shot_cot_chat[0]['abbr'] math500_gen_0_shot_cot_chat[0]['reader_cfg']['test_range'] = '[0:8]' -datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat # 指定数据集列表，可通过累加添加不同的数据集配置 -models = [ # 指定模型配置列表 +datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat +models = [ dict( attr="service", type=VLLMCustomAPIChat, @@ -46,8 +709,8 @@ models = [ # 指定模型配置列表 model="", request_rate = 0, retry = 2, - host_ip = "localhost", # 指定推理服务的IP - host_port = 8080, # 指定推理服务的端口 + host_ip = "localhost", + host_port = 8080, max_out_len = 512, batch_size=1, generation_kwargs = dict( @@ -68,13 +731,13 @@ work_dir = 'outputs/demo_api-vllm-general-chat/' 修改好配置文件后，执行如下命令启动精度评测： ```bash -ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_general_chat.py +ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api.py ``` 如果需要执行多任务并行，可以在命令行中添加 [`--max-num-workers`](../base_tutorials/all_params/cli_args.md#公共参数)参数指定最大任务并行数，示例如下： ```bash -ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_general_chat.py --max-num-workers 4 +ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api.py --max-num-workers 4 ``` ### 输出结果 @@ -107,12 +770,12 @@ with read_base(): models as vllm_api_stream_chat, ) -datasets = synthetic_datasets # 指定数据集列表 +datasets = synthetic_datasets vllm_api_general_stream[0]["abbr"] = "demo-" + vllm_api_general_stream[0]["abbr"] vllm_api_stream_chat[0]["abbr"] = "demo-" + vllm_api_stream_chat[0]["abbr"] -models = vllm_api_general_stream + vllm_api_stream_chat # 指定模型列表 +models = vllm_api_general_stream + vllm_api_stream_chat work_dir = "outputs/demo_api-vllm-stream-perf/" ``` @@ -134,7 +797,7 @@ ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py -m perf --m ### 输出结果 ```bash -[2025-12-05 12:10:44,147] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-general-stream/syntheticdataset]: +[2025-12-05 12:10:44,147] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-general-stream/syntheticdataset]: ╒══════════════════════════╤═════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤═════╕ │ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ ╞══════════════════════════╪═════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪═════╡ @@ -143,7 +806,7 @@ ais_bench ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py -m perf --m │ TTFT │ total │ 103.5 ms │ 102.4 ms │ 107.0 ms │ 103.1 ms │ 103.3 ms │ 104.2 ms │ 106.8 ms │ 10 │ ... [2025-12-05 12:10:44,149] [ais_bench] [INFO] Performance Result files located in outputs/demo_api-vllm-general-stream-chat-perf/20251205_121020/performances/demo-vllm-api-general-stream-chat. -[2025-12-05 12:10:44,149] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-stream-chat/syntheticdataset]: +[2025-12-05 12:10:44,149] [ais_bench] [INFO] Performance Results of task [demo-vllm-api-stream-chat/syntheticdataset]: ╒══════════════════════════╤═════════╤═════════════════╤═════════════════╤═════════════════╤═════════════════╤════════════════╤═════════════════╤═════════════════╤═════╕ │ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ ╞══════════════════════════╪═════════╪═════════════════╪═════════════════╪═════════════════╪═════════════════╪════════════════╪═════════════════╪═════════════════╪═════╡ @@ -166,12 +829,12 @@ with read_base(): from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat -models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat +models = vllm_api_general + vllm_api_general_chat + vllm_api_stream_chat datasets = gsm8k_0_shot_cot_str + math500_gen_0_shot_cot_chat model_dataset_combinations = [ - dict(models=[models[0]], datasets=[datasets[0]]), # 组合1，使用模型0(vllm_api_general)与数据集0(gsm8k_0_shot_cot_str)进行组合 - dict(models=[models[1]], datasets=[datasets[1]]), # 组合2，使用模型1(vllm_api_general_chat)与数据集1(math500_gen_0_shot_cot_chat)进行组合 - dict(models=[models[2]], datasets=[datasets[0], datasets[1]]), # 组合3，使用模型2(vllm_api_stream_chat)与数据集0(gsm8k_0_shot_cot_str)和数据集1(math500_gen_0_shot_cot_chat)进行组合 + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), + dict(models=[models[2]], datasets=[datasets[0], datasets[1]]), ... ] ``` @@ -189,8 +852,8 @@ vllm_api_general_copy[0]['port'] = 8081 models = vllm_api_general_copy + vllm_api_general datasets = math500_gen_0_shot_cot_chat model_dataset_combinations = [ - dict(models=[models[1]], datasets=datasets), # 组合1，使用模型1(vllm_api_general)与数据集(math500_gen_0_shot_cot_chat)进行组合 - dict(models=[models[0]], datasets=datasets), # 组合2，使用模型0(vllm_api_general_copy)与数据集0(math500_gen_0_shot_cot_chat)进行组合，由于vllm_api_general_copy与vllm_api_general的abbr相同，所以会被认为与组合1是相同任务，会被跳过，即便内部参数存在区别 + dict(models=[models[1]], datasets=datasets), + dict(models=[models[0]], datasets=datasets), ] ``` @@ -198,20 +861,97 @@ model_dataset_combinations = [ ```python vllm_api_general_copy = vllm_api_general.copy() -vllm_api_general_copy[0]['abbr'] = vllm_api_general[0]['abbr'] + '-copy' # 修改abbr,标识模型 +vllm_api_general_copy[0]['abbr'] = vllm_api_general[0]['abbr'] + '-copy' ``` 这样vllm_api_general_copy[0]与vllm_api_general[0]的abbr不同，组合2与组合1是不同任务，会被正常执行。 -## 预设自定义配置文件文件样例列表 +## 预设自定义配置文件样例列表 + +### 快速上手 + +| 文件名 | 简介 | +| --- | --- | +| [model_api_test_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_zh_cn.py) | 快速上手样例（中文注释）：配置 `vllm_api_general_chat` 服务化模型与 `demo_gsm8k_gen_4_shot_cot_chat_prompt` 数据集，执行单任务精度测评 | +| [model_api_test_en.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_en.py) | 快速上手样例（英文注释）：与 `model_api_test_zh_cn.py` 内容一致，注释为英文 | + +### 服务化精度测评（`api_examples/`） + +| 文件名 | 简介 | +| --- | --- | +| [infer_vllm_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_general.py) | 基于gsm8k数据集使用vllm api(0.6+版本)访问v1/completions子服务进行评测，prompt格式为字符串格式，自定义了数据集路径 | +| [infer_vllm_api_general_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_general_chat.py) | 基于gsm8k数据集使用vllm api(0.6+版本)访问v1/chat/completions子服务进行评测，prompt格式为对话格式，自定义了数据集路径 | +| [infer_vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_stream_chat.py) | 基于gsm8k数据集使用vllm api(0.6+版本)访问v1/chat/completions子服务使用流式推理进行评测，prompt格式为对话格式，自定义了数据集路径 | +| [infer_vllm_api_old.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_old.py) | 基于gsm8k数据集使用旧版vllm api访问v1/completions子服务进行评测，prompt格式为字符串格式 | +| [infer_mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py) | 基于gsm8k数据集使用mindie stream api访问infer子服务进行评测，prompt格式为字符串格式，自定义了数据集路径 | +| [demo_infer_vllm_api.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/demo_infer_vllm_api.py) | Demo示例：同时评测v1/chat/completions与v1/completions两个接口在GSM8K与MATH数据集上的精度表现 | +| [infer_vllm_api_multi_model_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_multi_model_multi_dataset.py) | 多模型多数据集精度测评：将3个vllm服务化模型（general/general_chat/stream_chat）与GSM8K、MATH、MMLU数据集进行笛卡尔积组合 | +| [infer_vllm_api_with_model_dataset_combinations.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_with_model_dataset_combinations.py) | 自定义模型-数据集配对：通过 `model_dataset_combinations` 精确控制模型与数据集的配对关系 | +| [infer_vllm_api_with_judge_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_with_judge_model.py) | 裁判模型测评：评测需要LLM Judge的AIME 2025数据集，在 `judge_infer_cfg` 中配置裁判模型 | + +### 服务化性能测评（`api_examples/`） + +| 文件名 | 简介 | +| --- | --- | +| [demo_infer_vllm_api_perf.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/demo_infer_vllm_api_perf.py) | Demo示例：同时评测v1/chat/completions与v1/completions两个接口使用合成数据集进行流式性能测评 | +| [perf_vllm_api_synthetic.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_synthetic.py) | 合成数据集性能测评：自定义合成数据集的输入输出token长度分布 | +| [perf_vllm_api_stable_stage.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_stable_stage.py) | 稳态性能测评：以多个 `request_rate`（0/5/10/20）发送合成数据集进行稳态性能测试 | +| [perf_vllm_api_multiturn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_multiturn.py) | 多轮对话性能测评：使用ShareGPT多轮对话数据集 | +| [perf_vllm_api_custom_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_custom_dataset.py) | 自定义数据集性能测评：在自定义CSV/JSONL数据集上进行性能测评 | +| [perf_vllm_api_rps_distribution.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/perf_vllm_api_rps_distribution.py) | RPS分布控制性能测评：通过 `traffic_cfg`（burstiness、ramp-up策略）控制请求到达分布 | + +### 纯模型精度测评（`hf_example/`） + +| 文件名 | 简介 | +| --- | --- | +| [infer_hf_base_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_base_model.py) | 基于gsm8k数据集使用huggingface base模型的推理接口进行评测，prompt格式为字符串格式，自定义了数据集路径 | +| [infer_hf_chat_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_chat_model.py) | 基于gsm8k数据集使用huggingface chat模型的推理接口进行评测，prompt格式为对话格式，自定义了数据集路径 | +| [infer_hf_multi_model_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_multi_model_multi_dataset.py) | 多模型多数据集纯模型测评：在多个数据集上评测多个HuggingFace本地模型 | + +### 多模态测评（`lmm_example/`） + +| 文件名 | 简介 | +| --- | --- | +| [multi_device_run_qwen_image_edit.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/lmm_example/multi_device_run_qwen_image_edit.py) | 多模态图像编辑模型测评（Qwen图像编辑，多设备） | +| [infer_lmm_multi_dataset.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/lmm_example/infer_lmm_multi_dataset.py) | 多模态多数据集精度测评：在多个多模态数据集上评测多模态模型 | + +### 精度测评场景样例（`accuracy_benchmark/`） + +| 文件名 | 简介 | +| --- | --- | +| [single_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py) | 单任务精度测评 | +| [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py) | 多任务精度测评 | +| [multi_task_parallel_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py) | 多任务并行精度测评 | +| [multi_task_resume_partial_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py) | 中断续跑与失败用例重测（部分任务） | +| [ceval_merge_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py) | 子数据集合并推理 | +| [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py) | 固定请求数测评 | +| [multi_repeat_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py) | 多次独立重复推理 | +| [inference_re_eval_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py) | 推理结果重评估 | + +### 纯模型精度测评场景样例（`accuracy_benchmark_local/`） + +| 文件名 | 简介 | +| --- | --- | +| [single_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py) | 纯模型单任务测评 | +| [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py) | 纯模型多任务/多任务并行测评 | +| [ceval_merge_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py) | 子数据集合并推理 | +| [inference_re_eval_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py) | 纯模型推理结果重评估 | + +### 性能测评场景样例（`performance_benchmark/`） + +| 文件名 | 简介 | +| --- | --- | +| [single_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/single_task_zh_cn.py) | 单任务性能测评 | +| [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/multi_task_zh_cn.py) | 多任务性能测评 | +| [synthetic_gen_string_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py) | 自定义序列长度性能测评 | +| [multi_task_synthetic_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py) | 自定义序列的多任务组合性能测评 | +| [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py) | 固定请求数性能测评 | +| [perf_recalculate_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py) | 性能结果重计算 | + +### 通用工具 -|文件名|简介| +| 文件名 | 简介 | | --- | --- | -|[infer_vllm_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_general.py)|基于gsm8k数据集使用vllm api(0.6+版本)访问v1/completions子服务进行评测，prompt格式为字符串格式，自定义了数据集路径| -|[infer_mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_mindie_stream_api_general.py)|基于gsm8k数据集使用mindie stream api访问infer子服务进行评测，prompt格式为字符串格式，自定义了数据集路径| -|[infer_vllm_api_general_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_general_chat.py)|基于gsm8k数据集使用vllm api(0.6+版本)访问v1/chat/completions子服务进行评测，prompt格式为对话格式，自定义了数据集路径| -|[infer_vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/infer_vllm_api_stream_chat.py)|基于gsm8k数据集使用vllm api(0.6+版本)访问v1/chat/completions子服务使用流式推理进行评测，prompt格式为对话格式，自定义了数据集路径| -|[infer_hf_base_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_base_model.py)|基于gsm8k数据集使用huggingface base模型的推理接口进行评测，prompt格式为字符串格式，自定义了数据集路径| -|[infer_hf_chat_model.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/hf_example/infer_hf_chat_model.py)|基于gsm8k数据集使用huggingface chat模型的推理接口进行评测，prompt格式为字符串格式，自定义了数据集路径| +| [all_dataset_configs.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/all_dataset_configs.py) | 所有支持的数据集配置导入汇总，可在自定义配置文件中直接 `from ... import` 使用 | **注**: 上述自定义配置文件如果要评测其他数据集，请从[ais_bench/configs/api_examples/all_dataset_configs.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/api_examples/all_dataset_configs.py)导入其他数据集。 diff --git a/docs/source_zh_cn/advanced_tutorials/stable_stage.md b/docs/source_zh_cn/advanced_tutorials/stable_stage.md index a5d80643..5c9bea9f 100644 --- a/docs/source_zh_cn/advanced_tutorials/stable_stage.md +++ b/docs/source_zh_cn/advanced_tutorials/stable_stage.md @@ -188,6 +188,10 @@ ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_cha ![full_plot_example.img](../img/request_concurrency/full_plot_example.png) 具体这个html中的图标如何查看请参考📚 [性能测试可视化并发图使用说明](../base_tutorials/results_intro/performance_visualization.md) +## 通过自定义配置文件实现 + +> 💡 上述稳态性能测评场景也可以通过 [自定义配置文件方式](run_custom_config.md) 实现。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。详见 [自定义配置文件运行AISBench](run_custom_config.md#各场景自定义配置文件示例) 中"稳态性能测评"示例。 + ## 其他功能场景 ### 性能结果重计算 参考📚 [常规性能测试性能结果重计算](../base_tutorials/scenes_intro/performance_benchmark.md#性能结果重计算) @@ -205,7 +209,7 @@ ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_cha 压力测试的流程与[稳态测试快速入门](#稳态测试快速入门)基本一致，差异主要有如下两点： #### 压力测试参数说明 通过命令参数`--pressure-time`和指定压力测试的持续时间，压测持续时间不能超过86400秒（24小时）。 -通过配置[模型配置文件](../base_tutorials/all_params/models.md#配置模型)中的`request_rate`参数来指定每个进程新增线程（客户端）的频率。此参数取值越大，实际新增线程（客户端）的频率偏差越大（偏差和cpu单核处理能力有关）。 +通过配置[模型配置文件](../base_tutorials/all_params/models.md#服务化推理后端)中的`request_rate`参数来指定每个进程新增线程（客户端）的频率。此参数取值越大，实际新增线程（客户端）的频率偏差越大（偏差和cpu单核处理能力有关）。 通过修改[配置常量文件参数](../base_tutorials/all_params/cli_args.md#配置常量文件参数)中的`WORKERS_NUM`参数来指定压力测试中使用的进程数,提高压力测试的并发能力。 diff --git a/docs/source_zh_cn/advanced_tutorials/synthetic_dataset.md b/docs/source_zh_cn/advanced_tutorials/synthetic_dataset.md index 81260dc4..50fcbb2c 100644 --- a/docs/source_zh_cn/advanced_tutorials/synthetic_dataset.md +++ b/docs/source_zh_cn/advanced_tutorials/synthetic_dataset.md @@ -294,3 +294,7 @@ synthetic_config = { 1. **`tokenid`模式**：该模式下的`tokenid`取值范围取决于在模型配置文件中指定的模型的词表范围 2. **`string`模式**：当MinValue=MaxValue时生成固定长度序列 + +## 七. 通过自定义配置文件实现 + +> 💡 上述合成数据集测评场景也可以通过 [自定义配置文件方式](run_custom_config.md) 实现。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。详见 [自定义配置文件运行AISBench](run_custom_config.md#各场景自定义配置文件示例) 中"合成数据集性能测评"示例。 diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index a251e301..94458464 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -21,9 +21,10 @@ ais_bench [OPTIONS] 适用于所有模式，可同时与精度或性能参数联合使用。 | 参数| 说明| 示例| | ---- | ---- | ----| -| `--models`| 指定模型推理后端任务名称（对应 `ais_bench/benchmark/configs/models` 路径下一个已经实现的默认模型配置文件），支持传入多个任务名称。详情参考📚 [支持的模型](./models.md)| `--models vllm_api_general` | -| `--datasets` | 指定数据集任务名称（对应 `ais_bench/benchmark/configs/datasets` 路径下一个已经实现的默认数据集配置文件），可传入多个。详情参考📚 [支持的数据集类型](./datasets.md)| `--datasets gsm8k_gen` | -| `--summarizer` | 指定结果总结任务名称（对应 `ais_bench/benchmark/configs/summarizers` 路径下一个已经实现的默认模型配置文件）。详情参考📚 [支持的结果汇总任务](./summarizer.md) | `--summarizer medium`| +|`config`|指定自定义配置文件路径|`ais_bench /path/to/custom_config.py {other optional arguments}`| +| `--models`| 指定模型推理后端任务名称（对应 `ais_bench/benchmark/configs/models` 路径下一个已经实现的默认模型配置文件），支持传入多个任务名称。详情参考📚 [支持的模型](./models.md)。
⚠️注意：指定了自定义配置文件路径后此参数无效| `--models vllm_api_general` | +| `--datasets` | 指定数据集任务名称（对应 `ais_bench/benchmark/configs/datasets` 路径下一个已经实现的默认数据集配置文件），可传入多个。详情参考📚 [支持的数据集类型](../../get_started/datasets.md)。
⚠️注意：指定了自定义配置文件路径后此参数无效| `--datasets gsm8k_gen` | +| `--summarizer` | 指定结果总结任务名称（对应 `ais_bench/benchmark/configs/summarizers` 路径下一个已经实现的默认模型配置文件）。详情参考📚 [支持的结果汇总任务](./summarizer.md) 。
⚠️注意：指定了自定义配置文件路径后此参数无效| `--summarizer medium`| | `--mode` 或 `-m`| 运行模式，可选：`all`、`infer`、`eval`、`viz`、`perf`、`perf_viz`；默认 `all`。
详细请见 📚 [运行模式说明](./mode.md)。 | `--mode infer`
`-m all`| | `--reuse` 或 `-r`| 指定已有工作目录下的时间戳，继续执行并覆盖原有结果。结合`--mode`参数值，可用于推理中断续推，或基于已有推理结果执行精度计算、可视化结果打印。若不加参，则自动选取 `--work-dir` 下最新时间戳。| `--reuse 20250126_144254`
`-r 20250126_144254` | | `--work-dir` 或 `-w` | 指定评测工作目录，用于保存输出结果。默认 `outputs/default`。| `--work-dir /path/to/work`
`-w /path/to/work` | diff --git a/docs/source_zh_cn/base_tutorials/all_params/models.md b/docs/source_zh_cn/base_tutorials/all_params/models.md index 00e749c4..80ca3cd5 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/models.md +++ b/docs/source_zh_cn/base_tutorials/all_params/models.md @@ -10,20 +10,20 @@ AISBench Benchmark 支持多种服务化推理后端，包括 vLLM、SGLang、Tr 以在 GPU 上部署的 vLLM 推理服务为例，您可以参考 [vLLM 官方文档](https://docs.vllm.ai/en/stable/getting_started/quickstart.html) 启动服务。 不同服务化后端对应的模型配置如下： -| 模型配置名称| 简介| 使用前提| 支持的测评模式 | 接口类型 | 支持的数据集 Prompt 格式 | 配置文件路径| -| ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | -| `vllm_api_general` | 通过 vLLM 兼容 OpenAI 的 API 访问推理服务，接口为 `v1/completions`| 基于 vLLM 版本支持 `v1/completions` 子服务| 生成式测评、PPL模式测评 | 文本接口 | 字符串格式| [vllm_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general.py)| -| `vllm_api_general_stream`| 流式访问 vLLM 推理服务，接口为 `v1/completions`| 基于 vLLM 版本支持 `v1/completions` 子服务 | 生成式测评| 流式接口 | 字符串格式| [vllm_api_general_stream.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py) | -| `vllm_api_general_chat` | 通过 vLLM 兼容 OpenAI 的 API 访问推理服务，接口为 `v1/chat/completions` | 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评、PPL模式测评 | 文本接口 | 字符串格式、对话格式、多模态格式 | [vllm_api_general_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py) | -| `vllm_api_stream_chat`| 流式访问 vLLM 推理服务，接口为 `v1/chat/completions`| 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评 | 流式接口 | 字符串格式、对话格式、多模态格式 | [vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) | -| `vllm_api_stream_chat_multiturn`| 多轮对话场景的流式访问 vLLM 推理服务，接口为 `v1/chat/completions`| 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评 | 流式接口 | 对话格式 | [vllm_api_stream_chat_multiturn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat_multiturn.py) | -| `vllm_api_function_call_chat`| function call精度测评场景访问 vLLM 推理服务的API ，接口为 `v1/chat/completions`（只适用于[BFCL](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/BFCL/README.md)测评场景| 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评 | 文本接口 | 对话格式 | [vllm_api_function_call_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_function_call_chat.py) | -| `vllm_api_old` | 通过 vLLM 兼容 API 访问推理服务，接口为 `generate`| 基于 vLLM 版本支持 `generate` 子服务 | 生成式测评 | 文本接口 | 字符串格式、多模态格式| [vllm_api_old.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_old.py)| -| `mindie_stream_api_general` | 通过 MindIE 流式 API 访问推理服务，接口为 `infer`| 基于 MindIE 版本支持 `infer` 子服务 | 生成式测评 | 流式接口 | 字符串格式、多模态格式| [mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/mindie_api/mindie_stream_api_general.py) | -| `triton_api_general` | 通过 Triton API 访问推理服务，接口为 `v2/models/{model name}/generate` | 启动支持 Triton API 的推理服务 | 生成式测评 | 文本接口 | 字符串格式、多模态格式| [triton_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_api_general.py) | -| `triton_stream_api_general` | 通过 Triton 流式 API 访问推理服务，接口为 `v2/models/{model name}/generate_stream` | 启动支持 Triton API 的推理服务 | 生成式测评 | 流式接口 | 字符串格式、多模态格式 | [triton_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_stream_api_general.py) | -| `tgi_api_general` | 通过 TGI API 访问推理服务，接口为 `generate`| 启动支持 TGI API 的推理服务 | 生成式测评 | 文本接口 | 字符串格式、多模态格式| [tgi_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_api_general.py)| -| `tgi_stream_api_general` | 通过 TGI 流式 API 访问推理服务，接口为 `generate_stream`| 启动支持 TGI API 的推理服务 | 生成式测评 | 流式接口 | 字符串格式、多模态格式| [tgi_stream_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_stream_api_general.py) | +| 模型配置名称| 简介| 使用前提| 支持的测评模式 | 接口类型 | 支持的数据集 Prompt 格式 | 配套文件导入方式 | 配置文件路径| +| ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | ---------- | +| `vllm_api_general` | 通过 vLLM 兼容 OpenAI 的 API 访问推理服务，接口为 `v1/completions`| 基于 vLLM 版本支持 `v1/completions` 子服务| 生成式测评、PPL模式测评 | 文本接口 | 字符串格式|`from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general`| [vllm_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general.py)| +| `vllm_api_general_stream`| 流式访问 vLLM 推理服务，接口为 `v1/completions`| 基于 vLLM 版本支持 `v1/completions` 子服务 | 生成式测评| 流式接口 | 字符串格式| `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream` | [vllm_api_general_stream.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py) | +| `vllm_api_general_chat` | 通过 vLLM 兼容 OpenAI 的 API 访问推理服务，接口为 `v1/chat/completions` | 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评、PPL模式测评 | 文本接口 | 字符串格式、对话格式、多模态格式 | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat` | [vllm_api_general_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py) | +| `vllm_api_stream_chat`| 流式访问 vLLM 推理服务，接口为 `v1/chat/completions`| 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评 | 流式接口 | 字符串格式、对话格式、多模态格式 | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat` | [vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py) | +| `vllm_api_stream_chat_multiturn`| 多轮对话场景的流式访问 vLLM 推理服务，接口为 `v1/chat/completions`| 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评 | 流式接口 | 对话格式 | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat_multiturn import models as vllm_api_stream_chat_multiturn` | [vllm_api_stream_chat_multiturn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat_multiturn.py) | +| `vllm_api_function_call_chat`| function call精度测评场景访问 vLLM 推理服务的API ，接口为 `v1/chat/completions`（只适用于[BFCL](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/BFCL/README.md)测评场景| 基于 vLLM 版本支持 `v1/chat/completions` 子服务 | 生成式测评 | 文本接口 | 对话格式 | `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_function_call_chat import models as vllm_api_function_call_chat` | [vllm_api_function_call_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_function_call_chat.py) | +| `vllm_api_old` | 通过 vLLM 兼容 API 访问推理服务，接口为 `generate`| 基于 vLLM 版本支持 `generate` 子服务 | 生成式测评 | 文本接口 | 字符串格式、多模态格式| `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_old import models as vllm_api_old` | [vllm_api_old.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_old.py)| +| `mindie_stream_api_general` | 通过 MindIE 流式 API 访问推理服务，接口为 `infer`| 基于 MindIE 版本支持 `infer` 子服务 | 生成式测评 | 流式接口 | 字符串格式、多模态格式| `from ais_bench.benchmark.configs.models.mindie_api.mindie_stream_api_general import models as mindie_stream_api_general` | [mindie_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/mindie_api/mindie_stream_api_general.py) | +| `triton_api_general` | 通过 Triton API 访问推理服务，接口为 `v2/models/{model name}/generate` | 启动支持 Triton API 的推理服务 | 生成式测评 | 文本接口 | 字符串格式、多模态格式| `from ais_bench.benchmark.configs.models.triton_api.triton_api_general import models as triton_api_general` | [triton_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_api_general.py) | +| `triton_stream_api_general` | 通过 Triton 流式 API 访问推理服务，接口为 `v2/models/{model name}/generate_stream` | 启动支持 Triton API 的推理服务 | 生成式测评 | 流式接口 | 字符串格式、多模态格式 | `from ais_bench.benchmark.configs.models.triton_api.triton_stream_api_general import models as triton_stream_api_general` | [triton_stream_api_general.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/triton_api/triton_stream_api_general.py) | +| `tgi_api_general` | 通过 TGI API 访问推理服务，接口为 `generate`| 启动支持 TGI API 的推理服务 | 生成式测评 | 文本接口 | 字符串格式、多模态格式| `from ais_bench.benchmark.configs.models.tgi_api.tgi_api_general import models as tgi_api_general` | [tgi_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_api_general.py)| +| `tgi_stream_api_general` | 通过 TGI 流式 API 访问推理服务，接口为 `generate_stream`| 启动支持 TGI API 的推理服务 | 生成式测评 | 流式接口 | 字符串格式、多模态格式| `from ais_bench.benchmark.configs.models.tgi_api.tgi_stream_api_general import models as tgi_stream_api_general` | [tgi_stream_api_general](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/tgi_api/tgi_stream_api_general.py) | ### 服务化推理后端配置参数说明 服务化推理后端配置文件采用Python语法格式配置，示例如下： @@ -33,7 +33,7 @@ AISBench Benchmark 支持多种服务化推理后端，包括 vLLM、SGLang、Tr ```python from ais_bench.benchmark.models import VLLMCustomAPI -models = [ +models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general`中导入的models dict( attr="service", type=VLLMCustomAPI, @@ -141,19 +141,19 @@ models = [ - 启动时加 `--debug`（或在模型配置设 `verbose=True`）即可按 case 打印 `[Multi-LoRA] data_id=... lora_model_name=...` 日志，便于观测路由命中情况。 ## 本地模型后端 -|模型配置名称|简介|使用前提|支持的prompt格式(字符串格式或对话格式)|对应源码配置文件路径| -| --- | --- | --- | --- | --- | -|`hf_base_model`|HuggingFace Base 模型后端|已安装评测工具基础依赖，需在配置文件中指定 HuggingFace 模型权重路径（当前不支持自动下载）|字符串格式|[hf_base_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_base_model.py)| -|`hf_chat_model`| HuggingFace Chat 模型后端|已安装评测工具基础依赖，需在配置文件中指定 HuggingFace 模型权重路径（当前不支持自动下载）|对话格式|[hf_chat_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_chat_model.py)| -|`hf_qwenvl_model`| HuggingFace Chat QwenVL模型后端|已安装评测工具基础依赖，需在配置文件中指定 HuggingFace 模型权重路径（当前不支持自动下载）|对话格式|[hf_qwenvl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_qwenvl_model.py)| -|`vllm_offline_vl_model`| vllm Chat QwenVL离线推理模型后端|已安装评测工具基础依赖，需在配置文件中指定模型模型权重路径（当前不支持自动下载）|对话格式|[vllm_offline_vl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_offline_models/vllm_offline_vl_model.py)| +|模型配置名称|简介|使用前提|支持的prompt格式(字符串格式或对话格式)| 配套文件导入方式 |对应源码配置文件路径| +| --- | --- | --- | --- | --- | --- | +|`hf_base_model`|HuggingFace Base 模型后端|已安装评测工具基础依赖，需在配置文件中指定 HuggingFace 模型权重路径（当前不支持自动下载）|字符串格式|`from ais_bench.benchmark.configs.models.hf_models.hf_base_model import models as hf_base_model`|[hf_base_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_base_model.py)| +|`hf_chat_model`| HuggingFace Chat 模型后端|已安装评测工具基础依赖，需在配置文件中指定 HuggingFace 模型权重路径（当前不支持自动下载）|对话格式|`from ais_bench.benchmark.configs.models.hf_models.hf_chat_model import models as hf_chat_model`|[hf_chat_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_chat_model.py)| +|`hf_qwenvl_model`| HuggingFace Chat QwenVL模型后端|已安装评测工具基础依赖，需在配置文件中指定 HuggingFace 模型权重路径（当前不支持自动下载）|对话格式|`from ais_bench.benchmark.configs.models.hf_models.hf_qwenvl_model import models as hf_qwenvl_model`|[hf_qwenvl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/hf_models/hf_qwenvl_model.py)| +|`vllm_offline_vl_model`| vllm Chat QwenVL离线推理模型后端|已安装评测工具基础依赖，需在配置文件中指定模型模型权重路径（当前不支持自动下载）|对话格式|`from ais_bench.benchmark.configs.models.vllm_offline_models.vllm_offline_vl_model import models as vllm_offline_vl_model`|[vllm_offline_vl_model](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_offline_models/vllm_offline_vl_model.py)| ### 本地huggingface模型后端配置参数说明 本地huggingface模型后端配置文件采用Python语法格式配置，示例如下： ```python from ais_bench.benchmark.models import HuggingFacewithChatTemplate -models = [ +models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.configs.models.hf_models.hf_chat_model import models as hf_chat_model`中导入的models dict( attr="local", # 后端类型标识 type=HuggingFacewithChatTemplate, # 模型类型 diff --git a/docs/source_zh_cn/base_tutorials/all_params/summarizer.md b/docs/source_zh_cn/base_tutorials/all_params/summarizer.md index cd579a41..73b9e44e 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/summarizer.md +++ b/docs/source_zh_cn/base_tutorials/all_params/summarizer.md @@ -1,7 +1,7 @@ # 支持的结果汇总任务 -| 任务名称 | 简介 | 配置文件路径 | -| -------------- | -------------- | -------------- | -| `example` | 简化版精度评测结果汇总模板，覆盖当前支持的所有数据集，是默认使用的模板。 | [example.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/example.py) | -| `medium` | 通用精度评测结果汇总模板，适用于多个基础数据集。| [medium.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/medium.py) | -| `default_perf` | 全量性能评测结果汇总模板，汇总所有请求的性能数据。支持通过 `default_perf.py` 手动配置性能统计指标。 | [default\_perf.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/default_perf.py) | -| `stable_stage` | 稳定阶段性能评测结果汇总模板，仅汇总系统达到配置最大并发时的请求数据。支持通过 `stable_stage.py` 手动配置性能统计指标。 | [stable\_stage.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/stable_stage.py) | +| 任务名称 | 简介 | 配置文件导入方式 | 配置文件路径 | +| -------------- | -------------- | -------------- | -------------- | +| `example` | 简化版精度评测结果汇总模板，覆盖当前支持的所有数据集，是默认使用的模板。 | `from ais_bench.benchmark.configs.summarizers.example import summarizer` | [example.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/example.py) | +| `medium` | 通用精度评测结果汇总模板，适用于多个基础数据集。| `from ais_bench.benchmark.configs.summarizers.medium import summarizer` | [medium.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/medium.py) | +| `default_perf` | 全量性能评测结果汇总模板，汇总所有请求的性能数据。支持通过 `default_perf.py` 手动配置性能统计指标。 | `from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer` | [default\_perf.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/default_perf.py) | +| `stable_stage` | 稳定阶段性能评测结果汇总模板，仅汇总系统达到配置最大并发时的请求数据。支持通过 `stable_stage.py` 手动配置性能统计指标。 | `from ais_bench.benchmark.configs.summarizers.perf.stable_stage import summarizer` | [stable\_stage.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/summarizers/perf/stable_stage.py) | diff --git a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md index ab80dac1..5a8f9c8c 100644 --- a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -6,28 +6,98 @@ - 可访问的服务化模型服务：确保服务进程可在当前环境下直接访问。 - 数据集任务准备： - - 开源数据集：从📚 [开源数据集](../all_params/datasets.md#开源数据集)中选择数据集，并且在数据集对应的"详细介绍"文档中选择要执行的数据集任务。参考选取的数据集任务对应的"详细介绍"文档准备好数据集文件，建议将开源数据集手动放置在默认目录 `ais_bench/datasets/`下，程序将在任务执行时自动加载数据集文件。 + - 开源数据集：从📚 [开源数据集](../../get_started/datasets.md#开源数据集)中选择数据集，并且在数据集对应的"详细介绍"文档中选择要执行的数据集任务。参考选取的数据集任务对应的"详细介绍"文档准备好数据集文件，建议将开源数据集手动放置在默认目录 `ais_bench/datasets/`下，程序将在任务执行时自动加载数据集文件。 - 自定义数据集：无需指定数据集任务，其他配置参考📚 [自定义数据集](../../advanced_tutorials/custom_dataset.md)。 - 模型任务准备：从📚 [服务化推理后端](../all_params/models.md#服务化推理后端)中选择要执行的模型任务。 ## 主要功能场景 ### 单任务测评 -请参考主页📚 [快速入门](../../get_started/quick_start.md)，不做赘述。 +请参考主页📚 [快速入门](../../get_started/quick_start.md)。快速入门中已经提供了两种启动方式： ### 多任务测评 支持同时配置多个模型或多个数据集任务，通过单次命令进行批量测评，适用于大规模模型横向对比或多数据集精度对比分析。 -#### 命令说明 -用户可通过`--models`和`--datasets`参数指定多个配置任务，子任务数为`--models`配置任务数和`--datasets`配置任务数的乘积，即一个模型配置和一个数据集配置组成一个子任务，命令示例： -```bash -ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt -``` -上述命令指定了2个模型任务（`vllm_api_general_chat` `vllm_api_stream_chat`）和2个数据集任务（`gsm8k_gen_4_shot_cot_str` `aime2024_gen_0_shot_chat_prompt`），将执行以下4个组合精度测试任务： + +#### 子任务组合说明 + +多任务测评场景下，子任务数为`models`配置任务数和`datasets`配置任务数的乘积，即一个模型配置和一个数据集配置组成一个子任务。下面以同时测评2个模型任务（`vllm_api_general_chat`、`vllm_api_stream_chat`）和2个数据集任务（`gsm8k_gen_4_shot_cot_str`、`aime2024_gen_0_shot_chat_prompt`）为例，将执行以下4个组合精度测试任务： + [vllm_api_general_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py)模型任务 + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) 数据集任务 + [vllm_api_general_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_chat.py)模型任务 + [aime2024_gen_0_shot_chat_prompt](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_chat_prompt.py) 数据集任务 + [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py)模型任务 + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) 数据集任务 + [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py)模型任务 + [aime2024_gen_0_shot_chat_prompt](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_chat_prompt.py) 数据集任务 +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +参考快速入门中的 [model_api_test_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_zh_cn.py) 文件，在`with read_base():`中导入多个模型任务和数据集任务，然后将其合并到 `models`、`datasets` 列表即可。完整样例请参考 [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +修改好配置文件后，执行命令： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py +``` + +#### 自定义模型-数据集配对（可选） + +默认情况下，上述配置中 `models` 列表与 `datasets` 列表会自动按笛卡尔积组合，子任务数为模型数 × 数据集数（本例为 2 × 2 = 4 个）。若希望精确控制哪些模型与哪些数据集配对（例如让部分模型只跑部分数据集、避免无意义的组合），可在配置文件中通过 `model_dataset_combinations` 字段显式声明配对关系： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets +models = vllm_api_general_chat + vllm_api_stream_chat + +# 关键：通过 model_dataset_combinations 精确控制配对 +# 下例仅生成 2 个子任务（笛卡尔积会生成 4 个）： +# - vllm_api_general_chat + gsm8k_gen_4_shot_cot_str +# - vllm_api_stream_chat + aime2024_gen_0_shot_chat_prompt +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), +] +``` + +> ⚠️ **注意**：模型与数据集的唯一标识由 `abbr` 字段决定。同一配置文件中，相同 `abbr` 的模型或数据集重复出现的组合会被视为重复任务而被跳过。当通过 `.copy()` 等方式复用模型/数据集配置时，必须显式修改 `abbr` 以保证唯一性。详见 📚 [自定义模型与数据集组合](../../advanced_tutorials/run_custom_config.md#自定义模型与数据集组合)。 + +::: + +:::{tab-item} 备选：使用命令行参数 + +用户可通过`--models`和`--datasets`参数指定多个配置任务，命令示例： + +```bash +ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt +``` + #### 修改任务对应的配置文件 模型任务和数据集任务对应的配置文件实际路径通过执行加`--search`命令查询： ```bash @@ -48,7 +118,7 @@ ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_g ╘═════════════╧═════════════════════════════════╧═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ ``` - 参考📚 [服务化推理后端配置参数说明](../all_params/models.md#服务化推理后端配置参数说明)按照实际情况配置模型任务`vllm_api_general_chat`和`vllm_api_stream_chat`对应的配置文件。 -- 参考📚 [配置开源数据集](../all_params/datasets.md#配置开源数据集)按照实际情况配置数据集任务`gsm8k_gen_4_shot_cot_str`和`aime2024_gen_0_shot_chat_prompt`对应的配置文件。**注**：如果数据集放在默认目录 `ais_bench/datasets/`下，则一般不需要配置 +- 参考📚 [配置开源数据集](../../get_started/datasets.md#配置开源数据集)按照实际情况配置数据集任务`gsm8k_gen_4_shot_cot_str`和`aime2024_gen_0_shot_chat_prompt`对应的配置文件。**注**：如果数据集放在默认目录 `ais_bench/datasets/`下，则一般不需要配置 #### 执行评测命令 @@ -58,6 +128,9 @@ ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_g ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt ``` +::: +:::: + 执行过程中会在📚 [`--work-dir`](../all_params/cli_args.md#公共参数)路径（默认是`outputs/default/`）下创建时间戳目录用于保存执行细节。 任务结束后结果呈现的打屏日志示例如下： @@ -110,12 +183,50 @@ aime2024 604a78 accuracy gen 50.00 ├── summary_20250628_172032.md └── summary_20250628_172032.txt ``` + ### 多任务并行测评 -默认情况下，多个子任务采用串行执行，单个任务内默认开启Continuous Batch，会根据用户配置的最大并发拉起多个进程发送和处理请求，允许配置较大的并发。在单个任务并发较小时，可以通过设置📚 [`--max-num-workers`](../all_params/cli_args.md#精度测评参数)参数实现多任务并行，示例如下： +默认情况下，多个子任务采用串行执行，单个任务内默认开启Continuous Batch，会根据用户配置的最大并发拉起多个进程发送和处理请求，允许配置较大的并发。在单个任务并发较小时，可以通过设置📚 [`--max-num-workers`](../all_params/cli_args.md#公共参数)参数实现多任务并行，示例如下： + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +在自定义配置文件中不再需要设置 `max_num_workers`，而是通过命令行参数 [`--max-num-workers`](../all_params/cli_args.md#公共参数) 传递。配置文件样例与[多任务测评](#多任务测评)完全一致，完整样例请参考 [multi_task_parallel_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py)： + +```python +# 完整样例与多任务测评中的配置一致，区别仅在执行命令 +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_chat + vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +执行命令（通过 `--max-num-workers 4` 指定并行数）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py --max-num-workers 4 +``` + +::: +:::{tab-item} 备选：使用命令行参数 ```bash ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt --max-num-workers 4 ``` + +::: +:::: 示例中指定任务最大并发数为4，四个子任务将会同时执行，可以在命令行看板上看到： ``` Base path of result&log : outputs/default/20251106_113926 @@ -144,6 +255,40 @@ Press Up/Down arrow to page, 'P' to PAUZE/RESUME screen refresh, 'Ctrl + C' to 在测评过程中发生意外中断或服务器异常导致的推理任务失败时，可通过`--reuse`开启断点管理功能实现任务续测，亦支持仅对失败用例进行自动重测，无需重复运行全部任务。示例如下： 1、假设用户使用如下命令首次执行推理测评，若由于任务异常退出导致的任务中断或由于服务端异常导致部分请求失败 + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +首次执行命令（基于 [single_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py)）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py +``` + +此时部分推理结果会被保存下来，在📚 [`--work-dir`](../all_params/cli_args.md#公共参数)生成如下文件内容： + +```bash +# output/default下 +20250628_151326/ # 测试任务创建的时间戳目录 +├── configs # 模型任务、数据集任务和结构呈现任务对应的配置文件合成的一个配置 +│ └── 20250628_151326_29317.py +├── logs # 执行过程中日志，命令中如果加--debug，不会有过程日志落盘（都直接打印出来了） +│ └── infer # 推理阶段日志 +└── predictions # 推理结果目录，记录每条请求的输入、模型输出及答案（用于精度评估） + └── vllm-api-general-chat + └── tmp_demo_gsm8k # 已完成请求的推理输出 + └── tmp_0_2766386_1749107195.json # 缓存文件，命名格式为：tmp_{任务进程ID}_{进程编号}_{时间戳}.json +``` + +2、通过`--reuse`参数指定任务时间戳目录续推（`--reuse` 是公共参数，使用自定义配置文件时仍可通过命令行追加）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py --reuse 20250628_151326 +``` + +::: +:::{tab-item} 备选：使用命令行参数 + ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt ``` @@ -153,11 +298,11 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch # output/default下 20250628_151326/ # 测试任务创建的时间戳目录 ├── configs # 模型任务、数据集任务和结构呈现任务对应的配置文件合成的一个配置 -│   └── 20250628_151326_29317.py +│ └── 20250628_151326_29317.py ├── logs # 执行过程中日志，命令中如果加--debug，不会有过程日志落盘（都直接打印出来了） -│   └── infer # 推理阶段日志 +│ └── infer # 推理阶段日志 └── predictions # 推理结果目录，记录每条请求的输入、模型输出及答案（用于精度评估） -    └── vllm-api-general-chat + └── vllm-api-general-chat └── tmp_demo_gsm8k # 已完成请求的推理输出 └── tmp_0_2766386_1749107195.json # 缓存文件，命名格式为：tmp_{任务进程ID}_{进程编号}_{时间戳}.json ``` @@ -165,16 +310,70 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch ```bash ais_bench --models vllm_api_general --datasets gsm8k_gen --reuse 20250628_151326 ``` + +::: +:::: + 日志中会打印如下内容，提示续推任务开启： + ```bash 02/20 13:14:15 - AISBench - INFO - Found 10 tmp items, run infer task from the last interrupted position ``` + 续推结束后，会重新所有请求的精度结果并打印，生成结果与📚 [快速入门](../../get_started/quick_start.md)示例一致。 > ⚠️ 注意：中断续测与失败重测可能改变请求顺序，可能引发结果微小波动。 💡[多任务测评](#多任务测评) 也支持全量和部分任务的中断续测 & 失败用例重测。 + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + 例如，执行如下多任务评测命令出现中断： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py +``` + +通过如下方式对全量任务中断续测： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py --reuse 20250628_151326 +``` + +也可以通过编辑自定义配置文件后仅对部分任务中断续测。完整样例请参考 [multi_task_resume_partial_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +datasets = gsm8k_datasets +models = vllm_api_general_chat +# ...其余参数配置详见配置文件 +``` + +然后执行： + +```bash +# 仅对 vllm_api_general_chat + gsm8k_gen_4_shot_cot_str 任务中断续测 +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py --reuse 20250628_151326 + +# 对vllm_api_general_chat + gsm8k_gen_4_shot_cot_str, vllm_api_general_chat + aime2024_gen_0_shot_chat_prompts两个任务续测 +ais_bench ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py --reuse 20250628_151326 +``` + +> 💡 如果需要对部分组合（例如 `vllm_api_general_chat + aime2024`、`vllm_api_stream_chat + aime2024`）续测，只需在自定义配置文件中指定对应模型任务和数据集任务后通过 `--reuse` 指定时间戳即可，详见 📚 [自定义模型-数据集配对](../../advanced_tutorials/run_custom_config.md#6-自定义模型-数据集配对)。 + +::: +:::{tab-item} 备选：使用命令行参数 + ```bash ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt ``` @@ -192,27 +391,171 @@ ais_bench --models vllm_api_general_chat --datasets gsm8k_gen_4_shot_cot_str aim ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets aime2024_gen_0_shot_chat_prompt --reuse 20250628_151326 ``` +::: +:::: + ### 合并子数据集推理 部分数据集会分类成不同的子数据集，在推理时会被划分为多个子任务行推理，例如：📚 [MMLU](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/mmlu/README.md)、📚 [CEVAL](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/ceval/README.md)。AISBench Benchmark支持将存在多个小规模数据集的数据集合并为一个任务进行统一测评。示例如下： + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +修改自定义配置文件，引入支持合并推理的数据集任务即可。完整样例请参考 [ceval_merge_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + +models = vllm_api_general +# ...其余参数配置详见配置文件 +``` + +执行命令（`--merge-ds` 是公共参数，使用自定义配置文件时仍可通过命令行追加）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py --merge-ds +``` + +::: +:::{tab-item} 备选：使用命令行参数 + ```bash ais_bench --models vllm_api_general --datasets ceval_gen --merge-ds ``` -> ⚠️ 注意：合并模式下将只生成整体结果，子数据集精度不再单独列出。同时对合并模式下中断或失败的推理结果进行数据集中断续测 & 失败用例重测也必须在命令中加`--merge-ds` + +::: +:::: + +> ⚠️ 注意：合并模式下将只生成整体结果，子数据集精度不再单独列出。同时对合并模式下中断或失败的推理结果进行数据集中断续测 & 失败用例重测也必须在命令中加`--merge-ds`。 ### 固定请求数测评 -当集规模过大，只想针数据对部分样本执行性能测试时，可使用 📚 [`--num-prompts`](../all_params/cli_args.md#性能测评参数) 参数指定读取的数据条数。示例如下： +当数据集规模过大，只想针对部分样本执行精度测试时，可使用以下两种方式控制读取的数据范围，二者作用一致，按使用习惯选择即可： + +- **基础方式**：通过命令行参数 📚 [`--num-prompts`](../all_params/cli_args.md#公共参数) 直接指定读取的数据条数，无需修改配置文件，使用最简单。 +- **进阶方式（功能更强大）**：在自定义配置文件中设置数据集的 `reader_cfg.test_range` 字段，支持更灵活的采样范围（如指定起始位置、自定义步长等），详细用法可参考 📚 [自定义配置文件](../../advanced_tutorials/run_custom_config.md)。 + +示例如下： + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +**方式一：基础方式 — 通过 `--num-prompts` 指定读取条数** + +完整样例请参考 [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +执行命令（通过 `--num-prompts 1` 指定仅读取 1 条样本）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py --num-prompts 1 +``` + +**方式二：进阶方式 — 通过 `test_range` 灵活指定读取范围** + +如果需要更灵活的范围控制（如指定起始索引、自定义步长等），可在自定义配置文件中直接设置数据集的 `reader_cfg.test_range` 字段，无需通过命令行参数。完整样例请参考 [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# 关键：通过 reader_cfg.test_range 灵活控制采样范围 +# 例如：'[0:8]' 表示读取前 8 条样本；'[10:20]' 表示读取索引 10 到 20 的样本 +datasets[0]['reader_cfg']['test_range'] = '[0:8]' + +models = vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +执行命令（已在配置文件中指定 test_range，无需再传 `--num-prompts`）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py +``` + +::: +:::{tab-item} 备选：使用命令行参数 + ```bash ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --num-prompts 1 ``` 上述命令仅对示例数据集中的第一条记录进行推理并只对这一条记录进行精度评估。 -> ⚠️ 注意：当前数据集会按照默认队列顺序依次读取，不支持随机抽样或打乱顺序。 + +::: +:::: + +> ⚠️ 注意：当前数据集会按照默认队列顺序依次读取，不支持随机抽样或打乱顺序。同时配置文件中设置 `reader_cfg.test_range` 与命令行 `--num-prompts` 时，命令行参数 `--num-prompts` 优先级更高。 ### 多次独立重复推理 > 该功能开启后，由于`数据集`/`请求数量`将按照`数据点级别`成倍扩充，从而导致推理时间显著变长，且使用内存显著提高。请在阅读 📚 [精度评测场景：评估指标解析](../results_intro/accuracy_metric.md) 后，**确认当前场景是否需要开启该功能**。 -该场景旨在从可靠性、稳定性、整体准确性等多维度探究模型能力，开启方式为：在 `服务化推理后端配置参数` 中的超参 `generation_kwargs` 中配置 🔗[`num_return_sequences`参数数值](../all_params/models.md#服务化推理后端配置参数说明)，格式按照以下示例内容（取值仅供参考）： +该场景旨在从可靠性、稳定性、整体准确性等多维度探究模型能力，开启方式为：在 `服务化推理后端配置参数` 中的超参 `generation_kwargs` 中配置 🔗[`num_return_sequences`参数数值](../all_params/models.md#服务化推理后端配置参数说明)。 + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +完整样例请参考 [multi_repeat_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +# 关键：通过 generation_kwargs.num_return_sequences 启用多次独立重复推理 +models[0]["generation_kwargs"] = dict( + temperature=0.01, + ignore_eos=False, + num_return_sequences=5, # 具体作用和约束请参考文档 accuracy_metric.md +) +# ...其余参数配置详见配置文件 +``` + +执行命令： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py +``` + +::: +:::{tab-item} 备选：使用命令行参数 + +修改模型任务配置文件中的 `generation_kwargs`： ```python models = [ @@ -222,11 +565,14 @@ models = [ num_return_sequences = 5, # 具体作用和约束请参考文档 accuracy_metric.md ... # 其它参数 ), - ... + ... # 其它参数 ) ] ``` +::: +:::: + 精度评估阶段结束后，结果会记录在日志和打屏在运行窗口，格式按照以下示例内容（数据仅供参考）： ```bash @@ -240,6 +586,25 @@ models = [ 上表中，**具体指标解读**和**参数约束** 请参考📚 [精度评测场景：评估指标解析](../results_intro/accuracy_metric.md) +## 通过自定义配置文件实现 + +> 💡 上述所有功能场景（多任务测评、多任务并行、中断续测、合并子数据集、固定请求数测评、多次独立重复推理、推理结果重评估等）均提供了两种启动方式（**⭐ 推荐：使用自定义配置文件**、**备选：使用命令行参数**）。自定义配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。 + +本章节涉及的所有自定义配置文件样例已统一存放在 `ais_bench/configs/accuracy_benchmark/` 目录下，便于查阅与复用： + +| 文件名 | 对应场景 | +| --- | --- | +| [single_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py) | 单任务测评 | +| [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_zh_cn.py) | 多任务测评 | +| [multi_task_parallel_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_parallel_zh_cn.py) | 多任务并行测评 | +| [multi_task_resume_partial_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_zh_cn.py) | 中断续测 & 失败用例重测（部分任务） | +| [ceval_merge_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/ceval_merge_zh_cn.py) | 合并子数据集推理 | +| [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/fixed_prompts_zh_cn.py) | 固定请求数测评 | +| [multi_repeat_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/multi_repeat_zh_cn.py) | 多次独立重复推理 | +| [inference_re_eval_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py) | 推理结果重评估 | + +> 关于自定义配置文件语法的完整说明（包括可定义的顶层变量、字段详解、Python 高级用法等），请参考 📚 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)；其中"各场景自定义配置文件示例"章节还提供了 10 种典型场景的完整样例（如服务化性能测评、合成数据集性能测评、稳态性能测评、多轮对话性能测评、裁判模型测评、自定义数据集测评等）。 + ## 其他功能场景 ### 推理结果重评估 主要功能场景下评测任务的执行流程包括完整的推理 → 评估 → 汇总流程： @@ -254,6 +619,54 @@ graph LR; 整个执行流程中的每个环节都是独立解耦的，推理结果是可以反复重评估的，如果第一次执行精度评测的到的精度数据有问题（比如没有准确得提取出response中有价值的内容），就可以修改答案提取的方式，执行推理结果重评估。具体操作如下。 假设上次执行性能测评的命令是： + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/single_task_zh_cn.py +``` +同时提示落盘的时间戳为`20250628_151326`，但是8条case的精度数据有问题，只得了0分： +```bash +dataset version metric mode vllm_api_general_chat +----------------------- -------- -------- ----- ---------------------- +demo_gsm8k 401e4c accuracy gen 00.00 +``` +查看`20250628_151326/predictions/vllm-api-general-chat/gsm8k.json`，发现推理结果中实际给了正确的答案。 + +**重评估步骤：** + +1. 编辑自定义配置文件（如 [inference_re_eval_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py)），按照实际需求覆盖对应数据集的 `eval_cfg` 中答案提取函数（参考下面示例）；其中 `pred_postprocessor` 负责从模型输出中提取答案，可根据实际情况替换或自定义。完整样例如下： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + +models = vllm_api_general_chat +# ...其余参数配置详见配置文件 + +# 关键：替换或修改答案的提取函数实现 +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) +``` + +2. 在第一次精度评测命令的基础上叠加 `--mode eval` 和 `--reuse {复用的推理结果所在的时间戳}` 反复重评估（`--mode` 与 `--reuse` 是公共参数，使用自定义配置文件时仍可通过命令行追加）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark/inference_re_eval_zh_cn.py --mode eval --reuse 20250628_151326 +``` + +::: +:::{tab-item} 备选：使用命令行参数 + ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt ``` @@ -276,6 +689,7 @@ ais_bench --datasets gsm8k_gen_4_shot_cot_chat_prompt --search ╘═════════════╧═══════════════════════════════════════╧═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╛ ``` + 打开`gsm8k_gen_4_shot_cot_chat_prompt.py`替换或修改答案的提取函数 ```python # ...... @@ -295,4 +709,7 @@ gsm8k_eval_cfg = dict(evaluator=dict(type=Gsm8kEvaluator), ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode eval --reuse 20250628_151326 -``` \ No newline at end of file +``` + +::: +:::: \ No newline at end of file diff --git a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark_local.md b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark_local.md index 2187ddd6..c8bbad04 100644 --- a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark_local.md +++ b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark_local.md @@ -4,22 +4,207 @@ 在执行服务化推理前，需要满足以下条件： - 可用的模型权重：确保本地已有需测试的模型权重文件，开源权重可从🔗 [huggingface社区](https://huggingface.co/models)获取。 -- 数据集任务准备：从📚 [开源数据集](../all_params/datasets.md#开源数据集)中选择数据集，并且在数据集对应的"详细介绍"文档中选择要执行的数据集任务。参考选取的数据集任务对应的"详细介绍"文档准备好数据集文件，建议将开源数据集手动放置在默认目录 `ais_bench/datasets/`下，程序将在任务执行时自动加载数据集文件。 +- 数据集任务准备：从📚 [开源数据集](../../get_started/datasets.md#开源数据集)中选择数据集，并且在数据集对应的"详细介绍"文档中选择要执行的数据集任务。参考选取的数据集任务对应的"详细介绍"文档准备好数据集文件，建议将开源数据集手动放置在默认目录 `ais_bench/datasets/`下，程序将在任务执行时自动加载数据集文件。 - 模型任务准备：从📚 [本地模型后端](../all_params/models.md#本地模型后端)中选择要执行的模型任务。 ## 主要功能 -纯模型精度测评场景下主要功能与服务化精度测评场景相似。 + +纯模型精度测评场景下主要功能与服务化精度测评场景相似，但需要将模型任务替换为本地 HuggingFace 模型任务（如 [`HuggingFacewithChatTemplate`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/models/huggingface_chat_model.py) 或 [`HuggingFaceBaseModel`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/models/huggingface_base_model.py)）。 + ### 纯模型多任务测评 -参考[服务化精度多任务测评使用方法](accuracy_benchmark.md#多任务测评) + +支持同时配置多个数据集任务，通过单次命令进行批量测评。完整样例请参考 [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + +datasets = gsm8k_datasets + aime2024_datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + # ...其余参数配置详见配置文件 + ) +] +``` + +执行命令： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py +``` + +#### 自定义模型-数据集配对（可选） + +默认情况下，上述配置中 `models` 列表与 `datasets` 列表会自动按笛卡尔积组合，子任务数为模型数 × 数据集数（本例为 1 × 2 = 2 个）。若希望精确控制哪些模型与哪些数据集配对（例如只让该模型跑部分数据集），可在配置文件中通过 `model_dataset_combinations` 字段显式声明配对关系： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_chat_prompt import aime2024_datasets + +datasets = gsm8k_datasets + aime2024_datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + ) +] + +# 关键：通过 model_dataset_combinations 精确控制配对 +# 下例仅生成 1 个子任务（笛卡尔积会生成 2 个）： +# - hf-chat-model + gsm8k +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), +] +``` + +> ⚠️ **注意**：模型与数据集的唯一标识由 `abbr` 字段决定。同一配置文件中，相同 `abbr` 的模型或数据集重复出现的组合会被视为重复任务而被跳过。当通过 `.copy()` 等方式复用模型/数据集配置时，必须显式修改 `abbr` 以保证唯一性。详见 📚 [自定义模型与数据集组合](../../advanced_tutorials/run_custom_config.md#自定义模型与数据集组合)。 + +> 💡 详细使用方法也可参考[服务化精度多任务测评使用方法](accuracy_benchmark.md#多任务测评)。 + ### 纯模型多任务并行测评 -参考[服务化精度多任务并行测评使用方法](accuracy_benchmark.md#多任务并行测评)。 + +支持通过 [`--max-num-workers`](../all_params/cli_args.md#公共参数) 命令行参数实现多任务并行。配置文件样例与[纯模型多任务测评](#纯模型多任务测评)完全一致，区别仅在执行命令。 + +执行命令（以 `max-num-workers 4` 为例）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py --max-num-workers 4 +``` + > ⚠️ 注意：纯模型精度测评多任务并行会占用不同GPU单元，并行任务所需的GPU单元应小于等于可使用的GPU总数。 + +> 💡 详细使用方法也可参考[服务化精度多任务并行测评使用方法](accuracy_benchmark.md#多任务并行测评)。 + ### 纯模型中断续测 -在纯模型精度测评过程中，如遇任务中断，可通过 `--reuse` 参数指定任务时间戳目录，继续未完成的推理任务，实现断点续测。该功能无需重复运行全部任务，仅对未完成部分进行补充推理。使用详情可参考[服务化精度中断续测使用方法](accuracy_benchmark.md#中断续测--失败用例重测)。 + +在纯模型精度测评过程中，如遇任务中断，可通过 `--reuse` 参数指定任务时间戳目录，继续未完成的推理任务，实现断点续测。该功能无需重复运行全部任务，仅对未完成部分进行补充推理。 + +首次执行命令： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py +``` + +通过 `--reuse` 参数指定任务时间戳目录续推（`--reuse` 是公共参数，使用自定义配置文件时仍可通过命令行追加）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py --reuse 20250628_151326 +``` + > ⚠️ 注意，纯模型精度测评当前不支持失败用例自动重测。 + +> 💡 详细使用方法也可参考[服务化精度中断续测使用方法](accuracy_benchmark.md#中断续测--失败用例重测)。 + ### 纯模型合并子数据集推理 -参考[服务化精度合并子数据集推理使用方法](accuracy_benchmark.md#合并子数据集推理)。 + +支持将存在多个小规模数据集的数据集合并为一个任务进行统一测评。完整样例请参考 [ceval_merge_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.ceval.ceval_gen_5_shot_str import ceval_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + # ...其余参数配置详见配置文件 + ) +] +``` + +执行命令（`--merge-ds` 是公共参数，使用自定义配置文件时仍可通过命令行追加）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py --merge-ds +``` + +> 💡 详细使用方法也可参考[服务化精度合并子数据集推理使用方法](accuracy_benchmark.md#合并子数据集推理)。 + +## 通过自定义配置文件实现 + +> 💡 上述所有功能场景（多任务测评、多任务并行、中断续测、合并子数据集等）均可以通过 [自定义配置文件方式](../../advanced_tutorials/run_custom_config.md) 实现。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。 + +本章节涉及的所有自定义配置文件样例已统一存放在 `ais_bench/configs/accuracy_benchmark_local/` 目录下，便于查阅与复用： + +| 文件名 | 对应场景 | +| --- | --- | +| [single_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/single_task_zh_cn.py) | 单任务测评 | +| [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/multi_task_zh_cn.py) | 纯模型多任务测评 / 多任务并行测评 | +| [ceval_merge_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/ceval_merge_zh_cn.py) | 合并子数据集推理 | +| [inference_re_eval_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py) | 纯模型推理结果重评估 | + +详见 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md#各场景自定义配置文件示例) 中"纯模型精度测评"示例。 ## 其他功能 + ### 纯模型推理结果重评估 -参考[服务化精度推理结果重评估使用方法](accuracy_benchmark.md#推理结果重评估)。 \ No newline at end of file + +完整样例请参考 [inference_re_eval_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.models import HuggingFacewithChatTemplate +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask +from ais_bench.benchmark.datasets import gsm8k_postprocess, gsm8k_dataset_postprocess + +with read_base(): + from ais_bench.benchmark.configs.summarizers.example import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = [ + dict( + type=HuggingFacewithChatTemplate, + abbr='hf-chat-model', + path='THUDM/chatglm-6b', # 替换为实际的本地模型权重路径 + tokenizer_path='THUDM/chatglm-6b', + # ...其余参数配置详见配置文件 + ) +] + +# 关键：替换或修改答案的提取函数实现 +datasets[0]['eval_cfg']['pred_postprocessor'] = dict(type=gsm8k_postprocess) +datasets[0]['eval_cfg']['dataset_postprocessor'] = dict(type=gsm8k_dataset_postprocess) +``` + +执行命令（`--mode eval` 与 `--reuse` 是公共参数，使用自定义配置文件时仍可通过命令行追加）： + +```bash +ais_bench ais_bench/configs/accuracy_benchmark_local/inference_re_eval_zh_cn.py --mode eval --reuse 20250628_151326 +``` + +> 💡 详细使用方法也可参考[服务化精度推理结果重评估使用方法](accuracy_benchmark.md#推理结果重评估)。 \ No newline at end of file diff --git a/docs/source_zh_cn/base_tutorials/scenes_intro/home.md b/docs/source_zh_cn/base_tutorials/scenes_intro/home.md index b764dbde..bfb1d808 100644 --- a/docs/source_zh_cn/base_tutorials/scenes_intro/home.md +++ b/docs/source_zh_cn/base_tutorials/scenes_intro/home.md @@ -9,7 +9,7 @@ - **模型任务**：📚 [服务化推理后端](../all_params/models.md#服务化推理后端) - - **数据集任务**：📚 [开源数据集](../all_params/datasets.md#开源数据集) 与 📚 [自定义数据集](../all_params/datasets.md#自定义数据集) + - **数据集任务**：📚 [开源数据集](../../get_started/datasets.md#开源数据集) 与 📚 [自定义数据集](../../get_started/datasets.md#自定义数据集) - 约束：当前PPL模式精度测评任务只支持`vllm_api_general`和`vllm_api_general_chat`两种模型配置，其他均不支持。 @@ -24,7 +24,7 @@ - **模型任务**：📚 [本地模型后端](../all_params/models.md#本地模型后端) - - **数据集任务**：📚 [开源数据集](../all_params/datasets.md#开源数据集) 与 📚 [自定义数据集](../all_params/datasets.md#自定义数据集) + - **数据集任务**：📚 [开源数据集](../../get_started/datasets.md#开源数据集) 与 📚 [自定义数据集](../../get_started/datasets.md#自定义数据集) - 约束：不支持PPL模式测评任务 @@ -40,7 +40,7 @@ - **模型任务**：📚 [服务化推理后端](../all_params/models.md#服务化推理后端)中的流式接口类型 - - **数据集任务**：📚 [支持数据集类型](../all_params/datasets.md#支持数据集类型)中的所有数据类型 + - **数据集任务**：📚 [支持数据集类型](../../get_started/datasets.md#支持数据集类型)中的所有数据类型 - 注意：性能测评所占用的缓存大小与请求的上下文长度以及请求的数量成正比，因此通常与测评时长呈正相关增长 diff --git a/docs/source_zh_cn/base_tutorials/scenes_intro/performance_benchmark.md b/docs/source_zh_cn/base_tutorials/scenes_intro/performance_benchmark.md index 96c07a54..97cdc330 100644 --- a/docs/source_zh_cn/base_tutorials/scenes_intro/performance_benchmark.md +++ b/docs/source_zh_cn/base_tutorials/scenes_intro/performance_benchmark.md @@ -5,7 +5,7 @@ AISBench Benchmark 提供服务化性能测评能力。针对流式推理场景 用户可通过配置服务化后端参数，灵活控制请求内容、请求间隔、并发数量等，适配不同评测场景（如低并发延迟敏感型、高并发吞吐优先型等）。测评支持自动化执行并输出结构化结果，便于横向对比不同模型、部署方案、硬件配置下的服务性能差异。 ## 服务化性能测评快速入门 ### 命令含义 -AISBench服务化性能测评命令含义与📚 [工具快速入门/命令含义](../../get_started/quick_start.md#命令含义)中的解释相同。在此基础上需要额外加上`--mode perf`或`-m perf`来进入性能评测场景，以如下AISBench命令为例： +AISBench服务化性能测评命令含义与📚 [工具快速入门/命令含义](../../get_started/quick_start.md#启动测评两种方式任选其一)中的解释相同。在此基础上需要额外加上`--mode perf`或`-m perf`来进入性能评测场景，以如下AISBench命令为例： ```shell ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer default_perf --mode perf ``` @@ -20,7 +20,7 @@ ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_cha 所选模型任务`vllm_api_stream_chat`、数据集任务`demo_gsm8k_gen_4_shot_cot_chat_prompt`和结果呈现任务`default_perf`的具体信息(简介，使用约束等)可以分别从如下链接中查询含义： - `--models`: 📚 [服务化推理后端](../all_params/models.md#服务化推理后端) -- `--datasets`:📚 [开源数据集](../all_params/datasets.md#开源数据集) → 📚 [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README.md) +- `--datasets`:📚 [开源数据集](../../get_started/datasets.md#开源数据集) → 📚 [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README.md) - `--summarizer`:📚 [结果汇总任务](../all_params/summarizer.md#支持的结果汇总任务) @@ -49,7 +49,7 @@ ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_cha ``` -- 快速入门中数据集任务配置文件`demo_gsm8k_gen_4_shot_cot_chat_prompt.py`不需要做额外修改，数据集任务配置文件内容介绍可参考📚 [配置开源数据集](../all_params/datasets.md#配置开源数据集) +- 快速入门中数据集任务配置文件`demo_gsm8k_gen_4_shot_cot_chat_prompt.py`不需要做额外修改，数据集任务配置文件内容介绍可参考📚 [配置开源数据集](../../get_started/datasets.md#配置开源数据集) 模型配置文件`vllm_api_stream_chat.py`中包含了模型运行相关的配置内容，是需要依据实际情况修改的。快速入门中需要修改的内容用注释标明。 ```python @@ -82,10 +82,27 @@ models = [ ``` ### 执行命令 + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +完成配置后，执行命令启动服务化性能评测： + +```bash +ais_bench ais_bench/configs/performance_benchmark/single_task_zh_cn.py --mode perf +``` + +::: +:::{tab-item} 备选：使用命令行参数 + 修改好配置文件后，执行命令启动服务化性能评测： + ```bash ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt -m perf ``` + +::: +:::: #### 查看任务执行细节 执行AISBench命令后，正在执行的任务状态会在命令行实时刷新的看板上显示（键盘按"P"键可以停止刷新，用于复制看板信息，再按"P"可以继续刷新），例如： ``` @@ -192,18 +209,99 @@ outputs/default/20251106_103326/logs/infer/vllm-api-stream-chat/demo_gsm8k.out - 可访问的服务化模型服务：确保服务进程可在当前环境下直接访问。 - 数据集准备： - - 开源数据集：从📚 [开源数据集](../all_params/datasets.md#开源数据集)中选择数据集，并且在数据集对应的"详细介绍"文档中选择要执行的数据集任务。参考选取的数据集任务对应的"详细介绍"文档准备好数据集文件，建议将开源数据集手动放置在默认目录 `ais_bench/datasets/`下，程序将在任务执行时自动加载数据集文件。 + - 开源数据集：从📚 [开源数据集](../../get_started/datasets.md#开源数据集)中选择数据集，并且在数据集对应的"详细介绍"文档中选择要执行的数据集任务。参考选取的数据集任务对应的"详细介绍"文档准备好数据集文件，建议将开源数据集手动放置在默认目录 `ais_bench/datasets/`下，程序将在任务执行时自动加载数据集文件。 - 随机合成数据集：数据集任务选`synthetic_gen`，其他配置参考📚 [随机合成数据集](../../advanced_tutorials/synthetic_dataset.md)。 - 自定义数据集：无需指定数据集任务，其他配置参考📚 [自定义数据集](../../advanced_tutorials/custom_dataset.md)。 - 服务化模型后端配置：从[服务化推理后端](../all_params/models.md#服务化推理后端)中选择接口类型为`流式接口`的子服务（⚠️ 其他不支持）。 ## 主要功能场景 -### 单任务评测 -参考[服务化性能测评快速入门](#服务化性能测评快速入门) +### 单任务测评 +参考[服务化性能测评快速入门](#服务化性能测评快速入门)。快速入门已经提供了两种启动方式： + +- ⭐ 推荐：使用自定义配置文件（见上文"快速入门"中的"⭐ 推荐：使用自定义配置文件"标签页） +- 备选：使用命令行参数（见上文"快速入门"中的"备选：使用命令行参数"标签页） + ### 多任务测评 支持同时配置多个模型或多个数据集任务，通过单次命令进行批量测评，适用于多个测试命令串行执行。 -#### 命令说明 -用户可通过`--models`和`--datasets`参数指定多个配置任务，子任务数为`--models`配置任务数和`--datasets`配置任务数的乘积，即一个模型配置和一个数据集配置组成一个子任务，示例： + +#### 子任务组合说明 + +多任务测评场景下，子任务数为`models`配置任务数和`datasets`配置任务数的乘积，即一个模型配置和一个数据集配置组成一个子任务。 + +下面以同时测评2个模型任务（`vllm_api_general_stream`、`vllm_api_stream_chat`）和2个数据集任务（`gsm8k_gen_4_shot_cot_str`、`aime2024_gen_0_shot_str`）为例，将执行以下4个组合性能测试任务： + ++ [vllm_api_general_stream](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py)模型任务 + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) 数据集任务 ++ [vllm_api_general_stream](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_general_stream.py)模型任务 + [aime2024_gen_0_shot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_str) 数据集任务 ++ [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py)模型任务 + [gsm8k_gen_4_shot_cot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/gsm8k/gsm8k_gen_4_shot_cot_str.py) 数据集任务 ++ [vllm_api_stream_chat](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/models/vllm_api/vllm_api_stream_chat.py)模型任务 + [aime2024_gen_0_shot_str](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aime2024/aime2024_gen_0_shot_str.py) 数据集任务 + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +在`with read_base():`中导入多个模型任务和数据集任务，然后将其合并到 `models`、`datasets` 列表即可。完整样例请参考 [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/multi_task_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_str import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets + +models = vllm_api_general_stream + vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +修改好配置文件后，执行命令： + +```bash +ais_bench ais_bench/configs/performance_benchmark/multi_task_zh_cn.py --mode perf +``` + +#### 自定义模型-数据集配对（可选） + +默认情况下，上述配置中 `models` 列表与 `datasets` 列表会自动按笛卡尔积组合，子任务数为模型数 × 数据集数（本例为 2 × 2 = 4 个）。若希望精确控制哪些模型与哪些数据集配对（例如让部分模型只跑部分数据集、避免无意义的组合），可在配置文件中通过 `model_dataset_combinations` 字段显式声明配对关系： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.gsm8k.gsm8k_gen_4_shot_cot_str import gsm8k_datasets + from ais_bench.benchmark.configs.datasets.aime2024.aime2024_gen_0_shot_str import aime2024_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +datasets = gsm8k_datasets + aime2024_datasets +models = vllm_api_general_stream + vllm_api_stream_chat + +# 关键：通过 model_dataset_combinations 精确控制配对 +# 下例仅生成 2 个子任务（笛卡尔积会生成 4 个）： +# - vllm_api_general_stream + gsm8k_gen_4_shot_cot_str +# - vllm_api_stream_chat + aime2024_gen_0_shot_str +model_dataset_combinations = [ + dict(models=[models[0]], datasets=[datasets[0]]), + dict(models=[models[1]], datasets=[datasets[1]]), +] +``` + +> ⚠️ **注意**：模型与数据集的唯一标识由 `abbr` 字段决定。同一配置文件中，相同 `abbr` 的模型或数据集重复出现的组合会被视为重复任务而被跳过。当通过 `.copy()` 等方式复用模型/数据集配置时，必须显式修改 `abbr` 以保证唯一性。详见 📚 [自定义模型与数据集组合](../../advanced_tutorials/run_custom_config.md#自定义模型与数据集组合)。 + +::: + +:::{tab-item} 备选：使用命令行参数 + +用户可通过`--models`和`--datasets`参数指定多个配置任务，示例： ```bash ais_bench --models vllm_api_general_stream vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_str --mode perf ``` @@ -234,13 +332,15 @@ ais_bench --models vllm_api_general_stream vllm_api_stream_chat --datasets gsm8k ``` - 参考📚 [服务化推理后端配置参数说明](../all_params/models.md#服务化推理后端配置参数说明)按照实际情况配置模型任务`vllm_api_general_stream`和`vllm_api_stream_chat`对应的配置文件。 -- 参考📚 [配置开源数据集](../all_params/datasets.md#配置开源数据集)按照实际情况配置数据集任务`gsm8k_gen_4_shot_cot_str`和`aime2024_gen_0_shot_str`对应的配置文件。**注**：如果数据集放在默认目录 `ais_bench/datasets/`下，则一般不需要配置 +- 参考📚 [配置开源数据集](../../get_started/datasets.md#配置开源数据集)按照实际情况配置数据集任务`gsm8k_gen_4_shot_cot_str`和`aime2024_gen_0_shot_str`对应的配置文件。**注**：如果数据集放在默认目录 `ais_bench/datasets/`下，则一般不需要配置 #### 执行评测命令 执行命令： ```bash ais_bench --models vllm_api_general_stream vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_str --mode perf ``` +::: +:::: 执行过程中会在📚 [`--work-dir`](../all_params/cli_args.md#公共参数)路径（默认是`outputs/default/`）下创建时间戳目录用于保存执行细节。 4个性能评测任务结束后会一次性打印4个任务的性能结果： @@ -329,7 +429,64 @@ ais_bench --models vllm_api_general_stream vllm_api_stream_chat --datasets gsm8k ### 自定义序列长度测评 + +自定义序列长度测评需要指定特殊的数据集任务 `synthetic_gen_string`，并在模型任务的 `generation_kwargs` 中配置 `ignore_eos = True` 以确保达到最大输出长度。 + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +完整样例请参考 [synthetic_gen_string_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# 关键：自定义输入输出分布（可通过修改synthetic_config调整） +synthetic_config = { + "Type": "string", + "RequestCount": 1000, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": 50, "MaxValue": 500} + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": 20, "MaxValue": 200} + } + } +} + +datasets = [] +for ds in synthetic_datasets: + ds = dict(ds) + ds["config"] = synthetic_config + datasets.append(ds) + +models = vllm_api_stream_chat +# 关键：性能测试时需将 ignore_eos 设置为 True 以确保达到最大输出长度 +models[0]["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) +# ...其余参数配置详见配置文件 +``` + +执行命令： + +```bash +ais_bench ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py --mode perf +``` + +::: +:::{tab-item} 备选：使用命令行参数 + #### 1 配置自定义序列数据集输入输出分布 + 自定义序列长度测评需要指定特殊的数据集任务`synthetic_gen_string`，执行如下命令来检索`synthetic_gen_string`对应的配置文件所在路径”： ```bash ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_string --search @@ -365,6 +522,7 @@ synthetic_config = { 💡更多的自定义输入输出分布可参考📚 [随机合成数据集](../../advanced_tutorials/synthetic_dataset.md) #### 2 确保推理服务达到设置的最大输出 + 为了确保推理服务达到设置的最大输出，需要在📚 [服务化模型配置](../all_params/models.md#服务化推理后端配置参数说明)的 `generation_kwargs` 中配置特殊的后处理参数`ignore_eos = True`，以控制请求的最大输出长度（不提前结束）。 例如修改`vllm_api_stream_chat`模型任务对应的配置文件[vllm_api_stream_chat.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/models/vllm_api/vllm_api_stream_chat.py)内容： @@ -386,23 +544,236 @@ models = [ ``` #### 3 启动性能测评 + 执行以下命令： ```bash ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_string -m perf ``` + +::: +:::: 完成后，输出目录结构同[多任务测评](#多任务测评)章节所示，会在 performance/vllm-api-stream-chat/synthetic* 下生成相应的 CSV/JSON/HTML 文件。 + > ⚠️ 注意： > - 部分服务化后端不支持 `ignore_eos` 后处理参数，此时实际输出的 `Token` 数可能无法达到所配置的最大输出长度，需要通过其他后处理参数的配置达到最大输出长度（例如限定最小输出的后处理参数等）。 +### 自定义序列的多任务组合测评 + +实际性能测评中，经常需要在同一推理服务上批量验证不同并发（`batch_size`）、不同请求频率（`request_rate`）、不同生成参数（`generation_kwargs`）下的表现，同时还要对比不同请求个数、不同输入/输出长度的组合。**自定义配置文件仅靠 Python 脚本即可批量生成上述所有组合任务**，无需手工复制粘贴多个配置文件。 + +下面的示例演示：将 `batch_size`、`request_rate`、`request_count`、`input_range`、`output_range` 这 5 个参数统一收束到 `tasks_params` 字典中，**用户希望配几个任务就在对应列表中追加几个元素**，无需关心其余模板代码；后端代码会自动按 `tasks_params` 的列表长度生成对应数量的模型任务与数据集任务，**并通过 `model_dataset_combinations` 按索引一一配对**（`models[i]` 配 `datasets[i]`，而非笛卡尔积），数据集名称按索引自动生成（如 `synthetic-string-0`、`synthetic-string-1`...）。 + +完整样例请参考 [multi_task_synthetic_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py)： + +```python +import copy +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.synthetic.synthetic_gen_string import synthetic_datasets as base_synthetic_datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as base_vllm_api_stream_chat + +# 关键：统一收束 batch_size / request_rate / request_count / input_range / output_range 五个参数 +# 用户希望配几个任务，就在对应列表中追加几个元素（同一分组内的列表长度需保持一致） +# 注意：models 与 datasets 的列表长度需保持一致，二者会按下标一一配对，而非笛卡尔积 +tasks_params = { + "models": { + "batch_size": [1, 2, 4, 8, 16, 32], + "request_rate": [0, 0, 0, 0, 0, 0], + }, + "datasets": { + "request_count": [100, 100, 100, 100, 100, 100], + "input_range": [(1, 2), (2, 4), (4, 8), (8, 16), (16, 32), (32, 64)], + "output_range": [(1, 2), (2, 4), (4, 8), (8, 16), (16, 32), (32, 64)], + }, +} + +# 关键：通过 deepcopy 复制同一个基础模型配置，按 tasks_params["models"] 批量覆盖 batch_size / request_rate +models = [] +for idx, (batch_size, request_rate) in enumerate(zip(tasks_params["models"]["batch_size"], + tasks_params["models"]["request_rate"])): + model_cfg = copy.deepcopy(base_vllm_api_stream_chat[0]) + model_cfg["abbr"] = f"vllm-api-stream-chat-bs{batch_size}-rr{request_rate}" + model_cfg["host_ip"] = "localhost" + model_cfg["host_port"] = 8080 + model_cfg["max_out_len"] = 512 + model_cfg["batch_size"] = batch_size + model_cfg["request_rate"] = request_rate + # 关键：每个模型任务使用独立的 generation_kwargs + model_cfg["generation_kwargs"] = dict(temperature=0.01, ignore_eos=True) + models.append(model_cfg) + +# 关键：按 tasks_params["datasets"] 批量构建合成数据集任务，名称按索引自动生成 +datasets = [] +for idx, (request_count, input_range, output_range) in enumerate( + zip(tasks_params["datasets"]["request_count"], + tasks_params["datasets"]["input_range"], + tasks_params["datasets"]["output_range"]) +): + ds = dict(base_synthetic_datasets[0]) + ds["abbr"] = f"synthetic-string-{idx}" + ds["config"] = { + "Type": "string", + "RequestCount": request_count, + "StringConfig": { + "Input": { + "Method": "uniform", + "Params": {"MinValue": input_range[0], "MaxValue": input_range[1]}, + }, + "Output": { + "Method": "uniform", + "Params": {"MinValue": output_range[0], "MaxValue": output_range[1]}, + }, + }, + } + datasets.append(ds) + +# 关键：按索引一一配对 models[i] 与 datasets[i]，避免笛卡尔积 +# 例如 models[0](batch_size=1) 仅与 datasets[0](input_range=(1,2)) 配对，而非与所有数据集交叉组合 +model_dataset_combinations = [ + dict(models=[models[idx]], datasets=[datasets[idx]]) + for idx in range(min(len(models), len(datasets))) +] + +work_dir = "outputs/default/" + +infer = dict( + partitioner=dict(type=NaivePartitioner), + runner=dict(type=LocalRunner, task=dict(type=OpenICLApiInferTask)), +) +``` + +执行命令： + +```bash +ais_bench ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py --mode perf +``` + +上述示例中： + +- `tasks_params["models"]` 内 `batch_size` 与 `request_rate` 两个列表**逐位配对**，共同决定模型任务数量。本例中两个列表各有 6 个元素，因此生成 6 个模型任务，分别对应 `batch_size` = 1 / 2 / 4 / 8 / 16 / 32 与 `request_rate` = 0。每个任务使用唯一的 `abbr`（如 `vllm-api-stream-chat-bs1-rr0`）以保证结果可区分。 +- `tasks_params["datasets"]` 内 `request_count`、`input_range`、`output_range` 三个列表**逐位配对**，共同决定数据集任务数量。本例中三个列表各有 6 个元素，因此生成 6 个数据集任务；数据集名称按列表下标自动生成（`synthetic-string-0` ~ `synthetic-string-5`）。 +- 用户**仅需修改 `tasks_params` 的列表长度与元素值**，即可灵活调整任务数量与各项参数，无需触碰下方模板代码。 +- 通过 `model_dataset_combinations` 字段按索引一一配对 `models[i]` 与 `datasets[i]`，本例共生成 **6 个子任务**（而非笛卡尔积的 36 个），对应关系如下： + + | 子任务 | 模型 (`batch_size` / `request_rate`) | 数据集 (`input_range` / `output_range`) | + | --- | --- | --- | + | 1 | 1 / 0 | (1, 2) / (1, 2) | + | 2 | 2 / 0 | (2, 4) / (2, 4) | + | 3 | 4 / 0 | (4, 8) / (4, 8) | + | 4 | 8 / 0 | (8, 16) / (8, 16) | + | 5 | 16 / 0 | (16, 32) / (16, 32) | + | 6 | 32 / 0 | (32, 64) / (32, 64) | + +> ⚠️ **注意**：`tasks_params["models"]` 内 `batch_size` 与 `request_rate` 列表长度必须一致；`tasks_params["datasets"]` 内 `request_count`、`input_range`、`output_range` 列表长度必须一致；否则 `zip()` 截断后会丢失末尾元素。同时，由于本场景要求一一配对，`models` 列表与 `datasets` 列表的长度也建议保持一致；当长度不一致时，会以较短列表的长度为准。 + +> ⚠️ **注意**：性能测评时需将 `generation_kwargs` 中的 `ignore_eos` 设置为 `True`，以确保输出长度达到 `max_out_len` 限制；否则输出可能在到达限制长度前提前结束。 + +> 💡 关于 `model_dataset_combinations` 字段的更多用法（例如一对多、多对一等更复杂的配对），可参考 📚 [自定义模型与数据集组合](../../advanced_tutorials/run_custom_config.md#自定义模型与数据集组合)。 + +> 💡 关于合成数据集的更多分布类型（`uniform` / `gaussian` / `zipf`）与参数说明，可参考 📚 [随机合成数据集](../../advanced_tutorials/synthetic_dataset.md)。 + + ### 固定请求数测评 -当集规模过大，只想针数据对部分样本执行性能测试时，可使用 📚 [`--num-prompts`](../all_params/cli_args.md#性能测评参数) 参数指定读取的数据条数。示例如下： +当数据集规模过大，只想针对部分样本执行性能测试时，可使用以下两种方式控制读取的数据范围，二者作用一致，按使用习惯选择即可： + +- **基础方式**：通过命令行参数 📚 [`--num-prompts`](../all_params/cli_args.md#公共参数) 直接指定读取的数据条数，无需修改配置文件，使用最简单。 +- **进阶方式（功能更强大）**：在自定义配置文件中设置数据集的 `reader_cfg.test_range` 字段，支持更灵活的采样范围（如指定起始位置、自定义步长等），详细用法可参考 📚 [自定义配置文件](../../advanced_tutorials/run_custom_config.md)。 + +示例如下： + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +**方式一：基础方式 — 通过 `--num-prompts` 指定读取条数** + +完整样例请参考 [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +models = vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +执行命令（通过 `--num-prompts 1` 指定仅读取 1 条样本）： + +```bash +ais_bench ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py --mode perf --num-prompts 1 +``` + +**方式二：进阶方式 — 通过 `test_range` 灵活指定读取范围** + +如果需要更灵活的范围控制（如指定起始索引、自定义步长等），可在自定义配置文件中直接设置数据集的 `reader_cfg.test_range` 字段，无需通过命令行参数。完整样例请参考 [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py)： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.summarizers.perf.default_perf import summarizer + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# 关键：通过 reader_cfg.test_range 灵活控制采样范围 +# 例如：'[0:8]' 表示读取前 8 条样本；'[10:20]' 表示读取索引 10 到 20 的样本 +datasets[0]['reader_cfg']['test_range'] = '[0:8]' + +models = vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +执行命令（已在配置文件中指定 test_range，无需再传 `--num-prompts`）： + +```bash +ais_bench ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py --mode perf +``` + +::: +:::{tab-item} 备选：使用命令行参数 + ```bash ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt -m perf --num-prompts 1 ``` 上述命令仅对示例数据集中的第一条记录进行推理并测量性能。 -> ⚠️ 注意：当前数据集会按照默认队列顺序依次读取，不支持随机抽样或打乱顺序。 +::: +:::: + +> ⚠️ 注意：当前数据集会按照默认队列顺序依次读取，不支持随机抽样或打乱顺序。同时配置文件中设置 `reader_cfg.test_range` 与命令行 `--num-prompts` 时，命令行参数 `--num-prompts` 优先级更高。 + + +## 通过自定义配置文件实现 + +> 💡 上述所有功能场景（多任务测评、自定义序列长度、固定请求数、性能结果重计算等）均提供了两种启动方式（**⭐ 推荐：使用自定义配置文件**、**备选：使用命令行参数**）。自定义配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用。 + +本章节涉及的所有自定义配置文件样例已统一存放在 `ais_bench/configs/performance_benchmark/` 目录下，便于查阅与复用： + +| 文件名 | 对应场景 | +| --- | --- | +| [single_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/single_task_zh_cn.py) | 单任务测评 | +| [multi_task_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/multi_task_zh_cn.py) | 多任务测评 | +| [synthetic_gen_string_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/synthetic_gen_string_zh_cn.py) | 自定义序列长度测评 | +| [multi_task_synthetic_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/multi_task_synthetic_zh_cn.py) | 自定义序列的多任务组合测评 | +| [fixed_prompts_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/fixed_prompts_zh_cn.py) | 固定请求数测评 | +| [perf_recalculate_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py) | 性能结果重计算 | + +> 关于自定义配置文件语法的完整说明（包括可定义的顶层变量、字段详解、Python 高级用法等），请参考 📚 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)；其中"各场景自定义配置文件示例"章节还提供了 10 种典型场景的完整样例（如服务化性能测评、合成数据集性能测评、稳态性能测评、多轮对话性能测评、裁判模型测评、自定义数据集测评等）。 ## 其他功能场景 ### 投机推理指标采集 @@ -428,6 +799,63 @@ graph LR; 执行流程的每个环节是独立解耦的，计算和汇总可以基于性能采样的结果反复执行。如果直接打印出的性能数据不包含相关维度的数据（例如缺少percentage 95%的数据），就需要做一些配置修改来重计算，具体操作如下。 假设上次执行性能测评的命令是： + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +```bash +ais_bench ais_bench/configs/performance_benchmark/single_task_zh_cn.py --mode perf +``` +打印出的`Performance Parameters`表格如下所示： +```bash +[2025-11-06 11:11:33,463] [ais_bench] [INFO] Performance Results of task: vllm-api-general-stream/gsm8k: +╒══════════════════════════╤═════════╤═════════════════╤════════════════╤═════════════════╤═════════════════╤═════════════════╤════════════════╤═════════════════╤══════╕ +│ Performance Parameters │ Stage │ Average │ Min │ Max │ Median │ P75 │ P90 │ P99 │ N │ +╞══════════════════════════╪═════════╪═════════════════╪════════════════╪═════════════════╪═════════════════╪═════════════════╪════════════════╪═════════════════╪══════╡ +│ E2EL │ total │ 2753.3518 ms │ 2189.5185 ms │ 3339.4463 ms │ 2755.8153 ms │ 3039.7431 ms │ 3219.6642 ms │ 3313.0408 ms │ 1319 │ +...... +``` + +如果想知道"P95"维度的性能数据，需要在自定义配置文件（如 [perf_recalculate_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py)）中修改 `summarizer` 的 `stats_list` 字段，完整样例如下： + +```python +from mmengine.config import read_base +from ais_bench.benchmark.summarizers import DefaultPerfSummarizer +from ais_bench.benchmark.calculators import DefaultPerfMetricCalculator +from ais_bench.benchmark.partitioners import NaivePartitioner +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask + +with read_base(): + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + +# 关键：自定义结果呈现任务中的 stats_list，调整要呈现的性能维度 +summarizer = dict( + attr="performance", + type=DefaultPerfSummarizer, + calculator=dict( + type=DefaultPerfMetricCalculator, + stats_list=["Average", "Min", "Max", "Median", "P75", "P90", "P95", "P99"], + ) +) + +models = vllm_api_stream_chat +# ...其余参数配置详见配置文件 +``` + +其中`stats_list`中最多同时承载8个性能维度的数据。 + +修改完毕后可以执行如下命令重计算性能指标（`--mode perf_viz --pressure --reuse` 是公共参数，使用自定义配置文件时仍可通过命令行追加）： + +```bash +## 注意必须指定 --mode perf_viz 以触发重计算 +ais_bench ais_bench/configs/performance_benchmark/perf_recalculate_zh_cn.py --mode perf_viz --pressure --debug --reuse 20250628_151326 +``` + +::: +:::{tab-item} 备选：使用命令行参数 + ```bash ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --mode perf ``` @@ -441,7 +869,7 @@ ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_cha ...... ``` -如果想知道“P95”维度的性能数据，需要修改`--summarizer`对应的默认结果呈现任务default_perf对应的配置文件内容，default_perf的路径通过`--search`命令查询： +如果想知道"P95"维度的性能数据，需要修改`--summarizer`对应的默认结果呈现任务default_perf对应的配置文件内容，default_perf的路径通过`--search`命令查询： ```bash ╒══════════════╤══════════════╤═══════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ │ Task Type │ Task Name │ Config File Path │ @@ -473,6 +901,8 @@ summarizer = dict( ## 注意必须指定--summarizer default_perf ais_bench --models vllm_api_stream_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer default_perf --mode perf_viz --pressure --debug --reuse 20250628_151326 ``` +::: +:::: 性能结果打屏如下： ```bash [2025-11-06 11:11:33,463] [ais_bench] [INFO] Performance Results of task: vllm-api-general-stream/gsm8k: diff --git a/docs/source_zh_cn/best_practices/practice_ascend.md b/docs/source_zh_cn/best_practices/practice_ascend.md index b828c444..8f3547d8 100644 --- a/docs/source_zh_cn/best_practices/practice_ascend.md +++ b/docs/source_zh_cn/best_practices/practice_ascend.md @@ -1,6 +1,8 @@ # 基于昇腾800I-A2测评DeepSeek-R1数学能力，100%论文复现 ### 复现使用的aisbench评测工具版本 本文复现使用aisbench测评工具版本为[v3.0-20250331](https://github.com/AISBench/benchmark/releases/tag/v3.0-20250331) + +> 💡 本文档中的测评命令均可以通过 [自定义配置文件方式](../advanced_tutorials/run_custom_config.md) 实现，将模型、数据集、summarizer 等配置写入一个 Python 文件，一次编写、多次复用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。详见 [自定义配置文件运行AISBench](../advanced_tutorials/run_custom_config.md)。 ### 一 背景与目标 #### 1. 1复现意义 diff --git a/docs/source_zh_cn/best_practices/practice_nvidia.md b/docs/source_zh_cn/best_practices/practice_nvidia.md index 9ecb9e56..85cb56f0 100644 --- a/docs/source_zh_cn/best_practices/practice_nvidia.md +++ b/docs/source_zh_cn/best_practices/practice_nvidia.md @@ -1,6 +1,8 @@ # 基于英伟达A100加速卡测评DeepSeek-R1-Distill-Qwen-14B的数学能力，100%论文复现 ### 复现使用的aisbench评测工具版本 本文复现使用aisbench测评工具版本为[v3.0-20250412](https://github.com/AISBench/benchmark/releases/tag/v3.0-20250412) + +> 💡 本文档中的测评命令均可以通过 [自定义配置文件方式](../advanced_tutorials/run_custom_config.md) 实现，将模型、数据集、summarizer 等配置写入一个 Python 文件，一次编写、多次复用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。详见 [自定义配置文件运行AISBench](../advanced_tutorials/run_custom_config.md)。 ### 一 背景与目标 #### 1. 1 复现意义 @@ -267,7 +269,7 @@ math_datasets = [ ``` # 确保处于源码最外层路径your/work/dir/benchmark下 -vim ais_bench/benchmark/configs/models/hf_model/hf_chat_model.py +vim ais_bench/benchmark/configs/models/hf_models/hf_chat_model.py ``` 推理后端配置文件内容修改如下: ```py diff --git a/docs/source_zh_cn/best_practices/replicate_llm_datasets_accuracy.md b/docs/source_zh_cn/best_practices/replicate_llm_datasets_accuracy.md index 46570f54..53085f0f 100644 --- a/docs/source_zh_cn/best_practices/replicate_llm_datasets_accuracy.md +++ b/docs/source_zh_cn/best_practices/replicate_llm_datasets_accuracy.md @@ -1,4 +1,7 @@ # 复现大语言模型（LLM）论文（技术报告）中的数据集测评结果（以DeepSeek R1使用的GPQA数据集为例） + +> 💡 本文档中的测评命令均可以通过 [自定义配置文件方式](../advanced_tutorials/run_custom_config.md) 实现，将模型、数据集、summarizer 等配置写入一个 Python 文件，一次编写、多次复用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。详见 [自定义配置文件运行AISBench](../advanced_tutorials/run_custom_config.md)。 + ## 前言-方法论 如果想要通过AISBench测评工具复现论文精度，需要对齐模型的技术报告或论文中对此数据集的测试方法，在评测工具这边需要对齐的如下： **模型相关配置**： diff --git a/docs/source_zh_cn/conf.py b/docs/source_zh_cn/conf.py index 1ecc6687..28d3a189 100644 --- a/docs/source_zh_cn/conf.py +++ b/docs/source_zh_cn/conf.py @@ -37,6 +37,7 @@ 'sphinx.ext.imgconverter', # 支持图片格式转换 'sphinx.ext.mathjax', # 支持数学公式 'sphinx.ext.viewcode', # 查看代码源文件 + 'sphinx_design', # 支持 tab-set、card 等 UI 组件 ] # 4. 若使用 Markdown，需指定源文件后缀 @@ -60,6 +61,7 @@ 'dollarmath', # 支持 $ 分隔的数学公式 'html_admonition', # 支持 HTML 警告框 'replacements', # 支持文本替换 + 'colon_fence', # 支持 ::: 栅栏指令（用于 tab-set 等 sphinx_design 组件） ] # （可选）配置 Mermaid 输出格式 diff --git a/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md b/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md index 85bc1a04..fa7da7b3 100644 --- a/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md +++ b/docs/source_zh_cn/extended_benchmark/agent/harbor_bench.md @@ -163,6 +163,8 @@ AISBench修改的数据集获取链接：https://github.com/AISBench/terminal-be 在 AISBench 工具根目录下修改 `ais_bench/configs/agent_example/harbor_terminal_bench_2_task.py`： +> 💡 上述 `harbor_terminal_bench_2_task.py` 即为 [自定义配置文件方式](../../advanced_tutorials/run_custom_config.md) 的具体应用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。你可以参考此示例文件自行编写满足特定需求的配置文件。详见 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)。 + ```python models = [ dict( diff --git a/docs/source_zh_cn/extended_benchmark/agent/swe_bench.md b/docs/source_zh_cn/extended_benchmark/agent/swe_bench.md index 731113de..7c1915b1 100644 --- a/docs/source_zh_cn/extended_benchmark/agent/swe_bench.md +++ b/docs/source_zh_cn/extended_benchmark/agent/swe_bench.md @@ -21,6 +21,8 @@ SWE-bench是一个基准测试，用于评估大语言模型在从GitHub收集 - `mini_swe_agent_swe_bench_multilingual.py`：SWE-bench Multilingual（`SWE-bench/SWE-bench_Multilingual`），包含多语言 issue 描述的数据集。 - `mini_swe_agent_swe_bench_multilingual_mini.py`：SWE-bench Multilingual Mini（**15**/**30**/**60** 条），AISBench官方构造的 Multilingual 子集，用于显著降低评测成本；子集筛选/构造方式见数据集卡与构造仓库：`https://modelers.cn/datasets/AISBench/SWE-Bench_Multilingual_mini`、`https://github.com/AISBench/datasets/tree/main/mini_datasets/swe_bench_multiligual_mini`。 +> 💡 上述示例配置文件即为 [自定义配置文件方式](../../advanced_tutorials/run_custom_config.md) 的具体应用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。你可以参考这些示例文件自行编写满足特定需求的配置文件。详见 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)。 + ## 2. 前置依赖 diff --git a/docs/source_zh_cn/extended_benchmark/agent/swe_bench_pro.md b/docs/source_zh_cn/extended_benchmark/agent/swe_bench_pro.md index 853d6b77..51b4e115 100644 --- a/docs/source_zh_cn/extended_benchmark/agent/swe_bench_pro.md +++ b/docs/source_zh_cn/extended_benchmark/agent/swe_bench_pro.md @@ -19,6 +19,8 @@ SWE-Bench Pro 是一个用于评估大语言模型在长时域软件工程任务 - `mini_swe_agent_swe_bench_pro_mini.py`：SWE-bench Pro Mini，适合先跑通流程/快速迭代。 - `mini_swe_agent_swe_bench_pro_full.py`：SWE-bench Pro Full，完整测试集。 +> 💡 上述示例配置文件即为 [自定义配置文件方式](../../advanced_tutorials/run_custom_config.md) 的具体应用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。你可以参考这些示例文件自行编写满足特定需求的配置文件。详见 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)。 + ## 2. 前置依赖 运行前请确保以下依赖可用： diff --git a/docs/source_zh_cn/extended_benchmark/agent/tau2_bench.md b/docs/source_zh_cn/extended_benchmark/agent/tau2_bench.md index 5378163e..97a3454d 100644 --- a/docs/source_zh_cn/extended_benchmark/agent/tau2_bench.md +++ b/docs/source_zh_cn/extended_benchmark/agent/tau2_bench.md @@ -64,6 +64,8 @@ ### 3. 配置τ²-Bench任务的自定义配置文件 1. 在AISBench工具根目录下修改`ais_bench/configs/agent_example/tau2_bench_task.py`中必要的配置（主要是配置被测推理服务和模拟用户的推理服务的信息） + +> 💡 上述 `tau2_bench_task.py` 即为 [自定义配置文件方式](../../advanced_tutorials/run_custom_config.md) 的具体应用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。你可以参考此示例文件自行编写满足特定需求的配置文件。详见 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)。 ```python # ...... models = [ diff --git a/docs/source_zh_cn/extended_benchmark/lmm_generate/gedit_bench.md b/docs/source_zh_cn/extended_benchmark/lmm_generate/gedit_bench.md index 80e6ca37..24a4cabc 100644 --- a/docs/source_zh_cn/extended_benchmark/lmm_generate/gedit_bench.md +++ b/docs/source_zh_cn/extended_benchmark/lmm_generate/gedit_bench.md @@ -89,6 +89,8 @@ pip install yunchang==0.6.0 #### 测评配置准备 在容器中`${PATH_TO_WORKSPACE}/benchmark/ais_bench/configs/lmm_example`目录下，打开`multi_device_run_qwen_image_edit.py`文件，编辑如下内容设置模型配置： + +> 💡 上述 `multi_device_run_qwen_image_edit.py` 即为 [自定义配置文件方式](../../advanced_tutorials/run_custom_config.md) 的具体应用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。你可以参考此示例文件自行编写满足特定需求的配置文件。详见 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)。 ```python # ...... # ====== User configuration parameters ========= @@ -104,7 +106,7 @@ device_list = [0] # [0, 1, 2, 3] 修改成实际可用的NPU设备ID列表，不 ```bash ais_bench --datasets gedit_gen_0_shot_llmjudge --search ``` -编辑`gedit_gen_0_shot_llmjudge.py`文件中裁判模型相关的配置，裁判模型的配置与常规API模型配置相同（可以参考快速入门中相关配置教程[模型配置介绍](../../get_started/quick_start.md#任务对应配置文件修改)），只是在`judge_model`字段中： +编辑`gedit_gen_0_shot_llmjudge.py`文件中裁判模型相关的配置，裁判模型的配置与常规API模型配置相同（可以参考快速入门中相关配置教程[模型配置介绍](../../get_started/quick_start.md#启动测评两种方式任选其一)），只是在`judge_model`字段中： ```python # ...... judge_model=dict( diff --git a/docs/source_zh_cn/extended_benchmark/lmm_generate/vbench.md b/docs/source_zh_cn/extended_benchmark/lmm_generate/vbench.md index 0664cfa5..5f98daa8 100644 --- a/docs/source_zh_cn/extended_benchmark/lmm_generate/vbench.md +++ b/docs/source_zh_cn/extended_benchmark/lmm_generate/vbench.md @@ -2,7 +2,9 @@ **VBench**（VBench: Comprehensive Benchmark Suite for Video Generative Models）是面向视频生成模型的评测基准套件，围绕主体一致性、运动平滑度、时序闪烁、空间关系等与感知相关的指标组织评测（官方 Standard 套件为 **16 个维度**），并提供与各维度匹配的 prompt、流程与校验方式。 -AISBench **已适配 VBench 1.0**。仓库目录 `ais_bench/configs/vbench_examples/` 下放的是 **独立配置文件** 示例，在 **GPU** 或 **NPU** 上对**生成视频**做质量/语义类维度测评。**当前 AISBench 不包含多模态视频生成**，请先完成视频生成后再进行测评(Standard模式参考[数据集生成](#数据集生成)章节)。 +AISBench **已适配 VBench 1.0**。仓库目录 `ais_bench/configs/vbench_examples/` 下放的是 **独立配置文件** 示例，在 **GPU** 或 **NPU** 上对**生成视频**做质量/语义类维度测评。**当前 AISBench 不包含多模态视频生成**，请先完成视频生成后再进行测评(Standard模式参考[数据集生成](#推理结果视频生成)章节)。 + +> 💡 上述 `vbench_examples/` 下的示例配置文件即为 [自定义配置文件方式](../../advanced_tutorials/run_custom_config.md) 的具体应用。配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法。你可以参考这些示例文件自行编写满足特定需求的配置文件。详见 [自定义配置文件运行AISBench](../../advanced_tutorials/run_custom_config.md)。 ## 目录 @@ -11,7 +13,7 @@ AISBench **已适配 VBench 1.0**。仓库目录 `ais_bench/configs/vbench_examp - [配置与输出](#配置与输出) - [评分汇总（Quality / Semantic / Total）](#评分汇总quality--semantic--total) - [Prompt Suite（官方 prompt 结构）](#prompt-suite官方-prompt-结构) -- [数据集生成](#数据集生成) +- [数据集生成](#推理结果视频生成) - [采样伪代码（参考官方）](#采样伪代码参考官方) - [格式要求](#格式要求) - [VBench-1.0-mini（AISBench 官方采样子集）](#vbench-10-miniaisbench-官方采样子集) @@ -45,7 +47,7 @@ python3 setup.py install --user ## 快速开始 1. **准备视频目录** - Standard / Custom 均需在对应配置中将 `DATA_PATH` 设为生成视频的根目录（绝对或相对路径）。也可复制配置文件后改 `DATA_PATH`，再执行 `ais_bench --mode eval`。（视频采样说明参考：[数据集生成](#数据集生成)） + Standard / Custom 均需在对应配置中将 `DATA_PATH` 设为生成视频的根目录（绝对或相对路径）。也可复制配置文件后改 `DATA_PATH`，再执行 `ais_bench --mode eval`。（视频采样说明参考：[数据集生成](#推理结果视频生成)） 2. **下载第三方依赖本地缓存** VBench 会加载多种小模型权重用于视频生成质量评测，建议提前手动下载，默认测评过程中会自动下载相关依赖，但存在下载失败导致测评任务失败；细节见 [`vbench_cache_dependencies.md`](./vbench_cache_dependencies.md)。 diff --git a/docs/source_zh_cn/faqs/error_codes.md b/docs/source_zh_cn/faqs/error_codes.md index 4ca738a6..c2dcc33b 100644 --- a/docs/source_zh_cn/faqs/error_codes.md +++ b/docs/source_zh_cn/faqs/error_codes.md @@ -99,7 +99,7 @@ Location: /usr/local/lib/python3.10/dist-packages ### 错误描述 使用[随机合成数据集](../advanced_tutorials/synthetic_dataset.md)`tokenid`场景下，模型配置文件必须指定tokenizer路径。 ### 解决办法 -假设ais_bench评测工具命令为`ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_tokenid --mode perf`，那么`vllm_api_stream_chat.py`（配置文件路径检索方式参考[任务对应配置文件修改](../get_started/quick_start.md#任务对应配置文件修改)）配置文件中`models`中所有的`path`参数须传入tokenizer路径（一般就是模型权重文件夹路径）。 +假设ais_bench评测工具命令为`ais_bench --models vllm_api_stream_chat --datasets synthetic_gen_tokenid --mode perf`，那么`vllm_api_stream_chat.py`（配置文件路径检索方式参考[任务对应配置文件修改](../get_started/quick_start.md#启动测评两种方式任选其一)）配置文件中`models`中所有的`path`参数须传入tokenizer路径（一般就是模型权重文件夹路径）。 ```python # ...... models = dict( diff --git a/docs/source_zh_cn/get_started/quick_start.md b/docs/source_zh_cn/get_started/quick_start.md index 8c28e1b5..bb10ed62 100644 --- a/docs/source_zh_cn/get_started/quick_start.md +++ b/docs/source_zh_cn/get_started/quick_start.md @@ -1,37 +1,117 @@ # 快速入门 -## 命令含义 -AISBench命令执行的单个或多个评测任务是由模型任务（单个或多个）、数据集任务（单个或多个）和结果呈现任务（单个）的组合定义的，AISBench的其他命令行则规定了评测任务的场景（精度评测场景、性能评测场景等）。以如下AISBench命令为例： + +## 运行命令前置准备 + +- 需要准备支持`v1/chat/completions`子服务的推理服务，可以参考🔗 [VLLM启动OpenAI 兼容服务器](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server)启动推理服务 +- 需要准备gsm8k数据集，可以从🔗 [opencompass + 提供的gsm8k数据集压缩包](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip)下载。将解压后的`gsm8k/`文件夹部署到AISBench评测工具根路径下的`ais_bench/datasets`文件夹下。 + +## 启动测评（两种方式任选其一） + +| ⭐ 推荐：使用自定义配置文件 | 备选：使用命令行参数(原快速入门方式) | +| :--- | :--- | +| 修改一个文件，集中管理所有配置，在任意路径写配置 | 通过 `--models` `--datasets` 参数指定 | +| 一次编写，多次复用 | 每次运行需输入完整命令 | +| 支持 Python 全部语法，灵活扩展 | 仅支持笛卡尔积组合 | + +::::{tab-set} +:::{tab-item} ⭐ 推荐：使用自定义配置文件 + +AISBench 提供了预置的自定义配置文件 [model_api_test_zh_cn.py](https://github.com/AISBench/benchmark/tree/master/ais_bench/configs/model_api_test_zh_cn.py)，将常见的推理服务化测试配置（模型选择、服务地址、端口、生成参数等）集中在一个文件中，无需分别查找和修改多个配置文件。该文件本质上是 Python 脚本，支持所有 Python 语法，你可以自由扩展。 + +打开 `ais_bench/configs/model_api_test_zh_cn.py`，根据实际情况修改以下配置（如果是`pip3 install ais_bench_benchmark`方式直接安装工具，可以在任意路径自行创建`model_api_test_zh_cn.py`，将以下配置内容写入该文件）： + +```python +from mmengine.config import read_base + +with read_base(): +# 模型任务，选择其中一个，其他模型任务参考：https://ais-bench-benchmark-rf.readthedocs.io/zh-cn/latest/base_tutorials/all_params/models.html 获取更多模型任务 + # vllm_api_general 是基础模型，仅支持文本生成 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general import models as vllm_api_general + # vllm_api_general_chat 是对话模型，支持对话 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_chat import models as vllm_api_general_chat + # vllm_api_stream_chat 是流式对话模型，支持流式对话 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_stream_chat import models as vllm_api_stream_chat + # vllm_api_general_stream 是流式模型，支持流式生成 + from ais_bench.benchmark.configs.models.vllm_api.vllm_api_general_stream import models as vllm_api_general_stream + +# 数据集任务，参考：https://ais-bench-benchmark-rf.readthedocs.io/zh-cn/latest/get_started/datasets.html 获取更多数据集任务 + from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets + +models = vllm_api_general_chat + +models[0]["path"] = "" # 指定模型序列化词表文件的绝对路径（精度测试场景一般不需要配置） +models[0]["model"] = "" # 指定服务端加载的模型名称，根据 VLLM 推理服务实际拉取的模型名称配置（配置为空字符串则自动获取） +models[0]["request_rate"] = 0 # 请求发送频率：每 1/request_rate 秒向服务端发送 1 条请求；小于 0.001 时一次性发送所有请求 +models[0]["api_key"] = "" # 自定义 API key，默认为空字符串 +models[0]["host_ip"] = "localhost" # 指定推理服务的 IP +models[0]["host_port"] = 8080 # 指定推理服务的端口 +models[0]["url"] = "" # 自定义访问推理服务的 URL 路径（当基础 URL 不是 http://host_ip:host_port 的组合时需要配置；配置后 host_ip 和 host_port 将被忽略） +models[0]["max_out_len"] = 512 # 推理服务输出的最大 token 数 +models[0]["batch_size"] = 1 # 发送请求的最大并发数 +models[0]["trust_remote_code"] = False # tokenizer 是否信任远程代码，默认为 False +models[0]["generation_kwargs"] = dict( # 模型推理参数，参考 VLLM 文档配置；AISBench 评测工具不做处理，直接附加到发送的请求中 + temperature=0.01, + ignore_eos=False, +) + +# datasets[0]["path"] = ais_bench/datasets/gsm8k # 指定数据集目录的绝对路径（精度测试场景需要配置） + +work_dir = 'outputs/default/' # 指定任务结果和日志的保存工作目录（默认为 outputs/default/） + +``` + +> 💡 配置文件中已预置了常用模型类型的导入（`vllm_api_general`、`vllm_api_general_chat`、`vllm_api_stream_chat`、`vllm_api_general_stream`），只需取消/修改注释即可切换。更多自定义配置文件的用法请参考 📚 [自定义配置文件运行AISBench](../advanced_tutorials/run_custom_config.md)。 + +数据集任务的选取、准备和使用参考如下步骤： + +1. 在📚 [开源数据集](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/get_started/datasets.html#id3)内选取数据集任务 +2. 进入数据的 📚 [详细介绍/数据集部署](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README.md#数据集部署)准备数据集 +3. 参考📚 [详细介绍/可用数据集任务](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README.md#可用数据集任务)选取可用数据集任务，并将对应的任务导入方式（例如`from ais_bench.benchmark.configs.datasets.demo.demo_gsm8k_gen_4_shot_cot_chat_prompt import gsm8k_datasets as datasets`）复制到自定义配置文件中 + +修改好配置文件后，执行如下命令启动服务化精度评测： + +```bash +ais_bench ais_bench/configs/model_api_test_zh_cn.py +``` + +::: +:::{tab-item} 备选：使用命令行参数 + +如果你更习惯使用命令行参数方式，AISBench 同样支持通过 `--models`、`--datasets`、`--summarizer` 参数直接指定任务。以下是与上述自定义配置文件方式**执行效果完全相同**的命令行方式。 + +AISBench命令执行的单个或多个评测任务是由模型任务（单个或多个）、数据集任务（单个或多个）和结果呈现任务（单个）的组合定义的。以如下AISBench命令为例： + ```shell ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --summarizer example ``` + 此命令没有指定其他命令行，默认是一个精度评测场景的任务，其中： -- `--models`指定了模型任务，即`vllm_api_general_chat`模型任务。 +- `--models`指定了模型任务，即`vllm_api_general_chat`模型任务。 - `--datasets`指定了数据集任务，即`demo_gsm8k_gen_4_shot_cot_chat_prompt`数据集任务。 +- `--summarizer`指定了结果呈现任务，即`example`结果呈现任务(不指定`--summarizer`精度评测场景默认使用`example`任务)，一般使用默认，不需要在命令行中指定。 -- `--summarizer`指定了结果呈现任务，即`example`结果呈现任务(不指定`--summarizer`精度评测场景默认使用`example`任务)，一般使用默认，不需要在命令行中指定，后续命令不指定。 - -## 任务含义查询(可选) -所选模型任务`vllm_api_general_chat`、数据集任务`demo_gsm8k_gen_4_shot_cot_chat_prompt`和结果呈现任务`example`的具体信息(简介，使用约束等)可以分别从如下链接中查询含义： -- `--models`: 📚 [服务化推理后端](../base_tutorials/all_params/models.md#服务化推理后端) +多任务测评请参考：📚 精度场景的[多任务测评](../base_tutorials/scenes_intro/accuracy_benchmark.md#多任务测评) 和 性能场景的[多任务测评](../base_tutorials/scenes_intro/performance_benchmark.md#多任务测评)。 -- `--datasets`: 📚 [开源数据集](../get_started/datasets.md#开源数据集) → 📚 [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README.md) +如需自行组合测评任务，实现更灵活的测评方式，可参考：📚 [自定义配置文件运行AISBench](../advanced_tutorials/run_custom_config.md#自定义配置文件运行AISBench)。 -- `--summarizer`: 📚 [结果汇总任务](../base_tutorials/all_params/summarizer.md) +所选模型任务`vllm_api_general_chat`、数据集任务`demo_gsm8k_gen_4_shot_cot_chat_prompt`和结果呈现任务`example`的具体信息(简介，使用约束等)可以分别从如下链接中查询含义： -## 运行命令前置准备 -- `--models`: 使用`vllm_api_general_chat`模型任务，需要准备支持`v1/chat/completions`子服务的推理服务，可以参考🔗 [VLLM启动OpenAI 兼容服务器](https://docs.vllm.com.cn/en/latest/getting_started/quickstart.html#openai-compatible-server)启动推理服务 -- `--datasets`: 使用`demo_gsm8k_gen_4_shot_cot_chat_prompt`数据集任务，需要准备gsm8k数据集，可以从🔗 [opencompass -提供的gsm8k数据集压缩包](http://opencompass.oss-cn-shanghai.aliyuncs.com/datasets/data/gsm8k.zip)下载。将解压后的`gsm8k/`文件夹部署到AISBench评测工具根路径下的`ais_bench/datasets`文件夹下。 +- `--models`: 📚 [服务化推理后端](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/all_params/models.html#id2) +- `--datasets`: 📚 [开源数据集](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/get_started/datasets.html#id3) → 📚 [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/demo/README.md) +- `--summarizer`: 📚 [结果汇总任务](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/all_params/summarizer.html) -## 任务对应配置文件修改 每个模型任务、数据集任务和结果呈现任务都对应一个配置文件，运行命令前需要修改这些配置文件的内容。这些配置文件路径可以通过在原有AISBench命令基础上加上`--search`来查询，例如： + ```shell ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --search ``` + > ⚠️ **注意**： 执行带search命令会打印出任务对应的配置文件的绝对路径。 执行查询命令可以得到如下查询结果： + ```shell ╒══════════════╤═══════════════════════════════════════╤════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╕ │ Task Type │ Task Name │ Config File Path │ @@ -43,9 +123,10 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch ``` -- 快速入门中数据集任务配置文件`demo_gsm8k_gen_4_shot_cot_chat_prompt.py`不需要做额外修改，数据集任务配置文件内容介绍可参考📚 [配置开源数据集](../get_started/datasets.md#配置开源数据集) +- 快速入门中数据集任务配置文件`demo_gsm8k_gen_4_shot_cot_chat_prompt.py`不需要做额外修改，数据集任务配置文件内容介绍可参考📚 [配置开源数据集](https://ais-bench-benchmark.readthedocs.io/zh-cn/latest/base_tutorials/all_params/datasets.html#id6) 模型配置文件`vllm_api_general_chat.py`中包含了模型运行相关的配置内容，是需要依据实际情况修改的。快速入门中需要修改的内容用注释标明。 + ```python from ais_bench.benchmark.models import VLLMCustomAPIChat @@ -63,7 +144,7 @@ models = [ api_key="", # 自定义API key，默认是空字符串 host_ip="localhost", # 指定推理服务的IP host_port=8080, # 指定推理服务的端口 - url="", # 自定义访问推理服务的URL路径(当base url不是http://host_ip:host_port的组合时需要配置，配置后host_ip和host_port将被忽略) + url="", # 自定义访问推理服务的URL路径(当base url不是http://host_ip:host_port的组合时需要配置, 配置后host_ip和host_port会被忽略) max_out_len=512, # 推理服务输出的token的最大数量 batch_size=1, # 请求发送的最大并发数 trust_remote_code=False, # tokenizer是否信任远程代码，默认False; @@ -75,13 +156,19 @@ models = [ ] ``` -## 执行命令 修改好配置文件后，执行命令启动服务化精度评测： + ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt ``` + +::: +:::: + ## 查看任务执行细节 -执行AISBench命令后，正在执行的任务状态会在命令行实时刷新的看板上显示（键盘按"P"键可以停止刷新，用于复制看板信息，再按"P"可以继续刷新），例如： + +执行AISBench命令后，任务管理界面会在命令行实时刷新显示任务执行状态（键盘按"P"键可以暂停/恢复刷新，用于复制看板信息，再按"P"键可以继续刷新）。任务管理界面支持同时监控多个任务的详细执行状态，包括任务名称、进度、时间成本、状态、日志路径、扩展参数等信息，例如： + ``` Base path of result&log : outputs/default/20250628_151326 Task Progress Table (Updated at: 2025-11-06 10:08:21) @@ -97,47 +184,48 @@ Press Up/Down arrow to page, 'P' to PAUZE/RESUME screen refresh, 'Ctrl + C' to ``` 任务执行的细节日志会不断落盘在默认的输出路径，这个输出路径在实时刷新的看板上显示，即`Log Path`。`Log Path`（`logs/infer/vllm-api-general-chat/demo_gsm8k.out`）是在`Base path`（`outputs/default/20250628_151326`）下的路径，以上述的看板信息为例，任务执行的详细日志路径为： + ```shell # {Base path}/{Log Path} outputs/default/20250628_151326/logs/infer/vllm-api-general-chat/demo_gsm8k.out ``` > 💡 如果希望执行过程中将详细日志直接打印，执行命令时可以加上 `--debug`: -`ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --debug` - - - +> `ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --debug` `Base path`（`outputs/default/20250628_151326`）下包含了所有任务的执行细节，命令执行结束后所有的执行细节如下： + ```shell 20250628_151326/ ├── configs # 模型任务、数据集任务和结构呈现任务对应的配置文件合成的一个配置 -│   └── 20250628_151326_29317.py +│ └── 20250628_151326_29317.py ├── logs # 执行过程中日志，命令中如果加--debug，不会有过程日志落盘（都直接打印出来了） -│   ├── eval -│   │   └── vllm-api-general-chat -│   │   └── demo_gsm8k.out # 基于predictions/文件夹下的推理结果的精度评测过程的日志 -│   └── infer -│   └── vllm-api-general-chat -│   └── demo_gsm8k.out # 推理过程日志 +│ ├── eval +│ │ └── vllm-api-general-chat +│ │ └── demo_gsm8k.out # 基于predictions/文件夹下的推理结果的精度评测过程的日志 +│ └── infer +│ └── vllm-api-general-chat +│ └── demo_gsm8k.out # 推理过程日志 ├── predictions -│   └── vllm-api-general-chat -│   └── demo_gsm8k.json # 推理结果（推理服务返回的所有输出） +│ └── vllm-api-general-chat +│ └── demo_gsm8k.json # 推理结果（推理服务返回的所有输出） ├── results -│   └── vllm-api-general-chat -│   └── demo_gsm8k.json # 精度评测计算的原始分数 +│ └── vllm-api-general-chat +│ └── demo_gsm8k.json # 精度评测计算的原始分数 └── summary ├── summary_20250628_151326.csv # 最终精度分数呈现（表格格式） ├── summary_20250628_151326.md # 最终精度分数呈现（markdown格式） └── summary_20250628_151326.txt # # 最终精度分数呈现（文本格式） ``` -> ⚠️ **注意**： 不同评测场景落盘任务执行细节内容不同，具体请参考具体评测场景的指南。 +> ⚠️ **注意**： 不同评测场景落盘任务执行细节内容不同，具体请参考具体评测场景的指南。 ### 输出结果 + 因为只有8条数据，会很快跑出结果，结果显示的示例如下 + ```bash dataset version metric mode vllm_api_general_chat ----------------------- -------- -------- ----- ---------------------- demo_gsm8k 401e4c accuracy gen 62.50 -``` \ No newline at end of file +``` diff --git a/docs/source_zh_cn/index.rst b/docs/source_zh_cn/index.rst index 0f4d08ab..b1526c75 100644 --- a/docs/source_zh_cn/index.rst +++ b/docs/source_zh_cn/index.rst @@ -21,7 +21,7 @@ AISBench Benchmark 是基于 `OpenCompass ` 将引导你完成基本的精度评测配置和运行。 * :doc:`数据集准备指南 ` 将帮助你了解支持的数据集及其准备方法。 * 基础教程部分将介绍 :doc:`评测场景介绍 ` 、:doc:`评测结果说明 ` 以及 :doc:`详细参数说明 ` 等内容，帮助你更好地理解主要的评测场景的使用。 -* 如果想要更深入地了解 AISBench 评测工具的高级用法，可以参考 :doc:`进阶教程 `。 +* 如果想要更深入地了解 AISBench 评测工具的高级用法，可以参考 :doc:`进阶教程 `。**强烈推荐**阅读 :doc:`自定义配置文件运行AISBench `，配置文件本质上是 Python 脚本，支持循环、条件判断、列表推导等所有 Python 语法，可将模型、数据集、summarizer 等配置写入一个文件，一次编写、多次复用，覆盖几乎所有评测场景。 * 你可以参考 :doc:`最佳实践` 部分，了解在不同场景下使用 AISBench 评测工具的最佳实践。 * 最后，你可以参考 :doc:`常见问题 ` 部分，解决在使用 AISBench 评测工具过程中遇到的问题。 diff --git a/plugin_examples/README.md b/plugin_examples/README.md index 62df6a05..618fbc6a 100644 --- a/plugin_examples/README.md +++ b/plugin_examples/README.md @@ -72,8 +72,8 @@ from mmengine.config import read_base from ais_bench_plugin_example_pkg.models import ExampleModel # 导入样例中自定义的模型运行类 from ais_bench_plugin_example_pkg.clients import ExampleClient # 导入样例中自定义的请求客户端类 from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask from ais_bench.benchmark.utils.model_postprocessors import extract_non_reasoning_content with read_base(): @@ -109,9 +109,9 @@ models = [ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/example_model/' ``` diff --git a/plugin_examples/config_example/perf_example.py b/plugin_examples/config_example/perf_example.py index 30f416b5..6914043d 100644 --- a/plugin_examples/config_example/perf_example.py +++ b/plugin_examples/config_example/perf_example.py @@ -2,8 +2,8 @@ from ais_bench_plugin_example_pkg.models import ExampleModel # 导入样例中自定义的模型运行类 from ais_bench_plugin_example_pkg.clients import ExampleClient # 导入样例中自定义的请求客户端类 from ais_bench.benchmark.partitioners import NaivePartitioner -from ais_bench.benchmark.runners.local_api import LocalAPIRunner -from ais_bench.benchmark.tasks import OpenICLInferTask +from ais_bench.benchmark.runners.local import LocalRunner +from ais_bench.benchmark.tasks import OpenICLApiInferTask from ais_bench.benchmark.utils.model_postprocessors import extract_non_reasoning_content with read_base(): @@ -39,8 +39,8 @@ infer = dict(partitioner=dict(type=NaivePartitioner), runner=dict( - type=LocalAPIRunner, + type=LocalRunner, max_num_workers=2, - task=dict(type=OpenICLInferTask)), ) + task=dict(type=OpenICLApiInferTask)), ) work_dir = 'outputs/example_model/' diff --git a/tests/UT/cli/test_config_manager.py b/tests/UT/cli/test_config_manager.py index aea26c5c..5b9943d0 100644 --- a/tests/UT/cli/test_config_manager.py +++ b/tests/UT/cli/test_config_manager.py @@ -117,40 +117,6 @@ def test_check_dataset_missing_required_field(self): # 当前实现仅要求datasets中包含type和abbr，缺少eval_cfg不应抛错 checker.check() - def test_check_missing_summarizer(self): - """测试缺少summarizer配置""" - invalid_config = { - 'models': [{'type': 'test_model', 'abbr': 'test', 'attr': {}}], - 'datasets': [{'type': 'test_dataset', 'abbr': 'test', 'reader_cfg': {}, 'infer_cfg': {}, 'eval_cfg': {}}] - } - checker = CustomConfigChecker(invalid_config, self.file_path) - with self.assertRaises(AISBenchConfigError) as cm: - checker.check() - self.assertEqual(cm.exception.error_code_str, TMAN_CODES.CFG_CONTENT_MISS_REQUIRED_PARAM.full_code) - - def test_check_summarizer_not_dict(self): - """测试summarizer不是字典类型""" - invalid_config = { - 'models': [{'type': 'test_model', 'abbr': 'test', 'attr': {}}], - 'datasets': [{'type': 'test_dataset', 'abbr': 'test', 'reader_cfg': {}, 'infer_cfg': {}, 'eval_cfg': {}}], - 'summarizer': 'test_summarizer' # 应该是字典 - } - checker = CustomConfigChecker(invalid_config, self.file_path) - with self.assertRaises(AISBenchConfigError) as cm: - checker.check() - self.assertEqual(cm.exception.error_code_str, TMAN_CODES.TYPE_ERROR_IN_CFG_PARAM.full_code) - - def test_check_summarizer_missing_required_field(self): - """测试summarizer缺少必需字段""" - invalid_config = { - 'models': [{'type': 'test_model', 'abbr': 'test', 'attr': {}}], - 'datasets': [{'type': 'test_dataset', 'abbr': 'test', 'reader_cfg': {}, 'infer_cfg': {}, 'eval_cfg': {}}], - 'summarizer': {} # 缺少attr字段 - } - checker = CustomConfigChecker(invalid_config, self.file_path) - with self.assertRaises(AISBenchConfigError) as cm: - checker.check() - self.assertEqual(cm.exception.error_code_str, TMAN_CODES.CFG_CONTENT_MISS_REQUIRED_PARAM.full_code) class TestConfigManager(unittest.TestCase): def setUp(self): From 662de5c70661aa8ac2364a8e77c3bc9e51a72acf Mon Sep 17 00:00:00 2001 From: Bo Lee Date: Wed, 26 Aug 2026 15:32:15 +0800 Subject: [PATCH 09/25] docs: remove response anomaly design document from docs tree (#486) * docs: remove response anomaly design document from docs tree * docs: complete response anomaly detection coverage in user docs - Fix execution-order description: detection is serially bound to the inference stage, not parallel with Eval (cli_args/mode, zh+en) - Add a dedicated detection_status table (completed/skipped/unavailable/ failed) with troubleshooting hints - Document result schema fields (anomaly_type_name, uuid) and clarify that anomaly results are an independent audit and never change accuracy/perf metrics - Correct --reuse inheritance: matched by id+uuid, only completed cases are inherited; skipped/failed/unavailable are re-detected - Document msprobe_config_path (threshold tuning entry) in model config examples and parameter tables - Describe payload archive layout (zst shards + manifest), empty-manifest semantics, and stale build-dir cleanup - Add detection log path and status file location for troubleshooting - Add vLLM return_token_ids request parameters and version requirement - Note model_name fallback to abbr with degradation warning * docs: add on-disk layout tree for response anomaly outputs Add a directory tree of covering all detection-related artifacts with per-path explanations (zh+en): - predictions: lightweight inference results without payload - response_anomaly//.jsonl: per-Case detection results - payload_staging: transient during inference, cleaned after detection - payload archive: zst shards + manifest, retention-mode semantics, empty-manifest directory meaning - response_anomaly_config: auto-generated msProbe config layout (config.yaml thresholds, mtype_config.json, token2category map), non-overwrite and multi-model merge behavior - logs/response_anomaly: detection log path * docs: unify response_anomaly example comments for msprobe paths Give msprobe_mtype_path and msprobe_token2category_dir the same inline-comment style as msprobe_config_path (optional marker + purpose description) in both zh/en cli_args.md and models.md, instead of bare placeholder paths. * docs: use double quotes for msprobe path placeholders in examples * docs: align model_name fallback description with fail-fast behavior Update the model_name resolution description in zh/en cli_args.md and models.md to match the behavior change in the companion fix PR: - model_name falls back to the model directory basename (de-facto model name, same default as the config generator), never to the model abbr - when neither model_name nor a model directory is available (explicit msprobe paths only), startup fails fast with guidance instead of silently running detection with a wrong model name * docs: rephrase model_name default as taking name from model path Drop the negative 'abbr is never a fallback' phrasing (an artifact of the old behavior) and state the default positively: the model name is taken from the model path. * docs: response anomaly switch is command-line only Align docs with the behavior change in the companion fix PR: - --response-anomaly is the only enable switch; omitting it disables detection (no --no-response-anomaly form) - remove the config-file enabled=True examples; the config-file response_anomaly entry now documents only non-switch settings (payload_retention / payload_storage) - drop the 'command line overrides the config file' precedence note * docs: drop response_anomaly.enabled mentions from docs The enabled key is not a supported config-file switch, but the code only warns for the top-level block (model-level enabled keys are silently unused), so do not advertise 'config enabled is ignored with a warning' behavior. State only the positive CLI rule: --response-anomaly enables detection, omitting it disables. --- .../base_tutorials/all_params/cli_args.md | 69 +++- .../base_tutorials/all_params/mode.md | 2 +- .../base_tutorials/all_params/models.md | 7 +- .../scenes_intro/accuracy_benchmark.md | 3 +- .../base_tutorials/all_params/cli_args.md | 69 +++- .../base_tutorials/all_params/mode.md | 2 +- .../base_tutorials/all_params/models.md | 7 +- .../scenes_intro/accuracy_benchmark.md | 2 + ...41\345\235\227\350\256\276\350\256\241.md" | 372 ------------------ 9 files changed, 134 insertions(+), 399 deletions(-) delete mode 100644 "docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index bb92aa6f..b0c8828f 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -37,7 +37,7 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | | `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. Note: In performance evaluation scenarios, an excessively high concurrency may cause resource contention among different processes, leading to inaccurate test results. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | -| `--response-anomaly` / `--no-response-anomaly` | Enables or disables msProbe response anomaly detection. The command-line value overrides `response_anomaly.enabled` in the config file. Detection runs in a thread in parallel with Eval; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | +| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it disables detection (there is no `--no-response-anomaly` form). Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | | `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | @@ -76,11 +76,17 @@ The currently supported parameter configurations are as follows: Response anomaly detection currently supports only the vLLM Chat API model configurations `vllm_api_general_chat`, `vllm_api_stream_chat`, and `vllm_api_stream_chat_multiturn`. Other model backends are not supported yet. -Add a `response_anomaly` entry to the top-level config file to enable detection; it can also be overridden with `--response-anomaly`: +The **feature switch is command-line only**: add `--response-anomaly` to the command to enable detection (omit it to disable). The `response_anomaly` entry in the config file carries only non-switch settings (`payload_retention`, `payload_storage`, etc.): ```python response_anomaly = dict( - enabled=True, + payload_retention='anomalies', # all | anomalies | none + payload_storage=dict( + format='jsonl', + compression='zstd', + compression_level=3, + rows_per_shard=2000, + ), ) ``` @@ -94,14 +100,15 @@ models = [ response_anomaly=dict( model_name="", # Model name, for example Qwen3-30B-A3B model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs - msprobe_mtype_path='/path/to/mtype_config.json', - msprobe_token2category_dir='/path/to/token2category/', + msprobe_config_path="", # Optional; msProbe algorithm-threshold config.yaml path for manual threshold tuning + msprobe_mtype_path="", # Optional; msProbe mtype_config.json path mapping model names to BOS/EOS token ids + msprobe_token2category_dir="", # Optional; msProbe token2category directory holding per-model token-id-to-character-category maps ), ), ] ``` -When `msprobe_mtype_path` / `msprobe_token2category_dir` are not provided, the default files inside the msProbe package are used; when `model_path` is configured, configs are auto-generated into `/response_anomaly_config//`. They can also be generated manually: +When `msprobe_mtype_path` / `msprobe_token2category_dir` are not provided, the default files inside the msProbe package are used; when `model_path` is configured, configs are auto-generated into `/response_anomaly_config//` (auto-generation never overwrites an existing `config.yaml`, so manually tuned thresholds are preserved). They can also be generated manually: ```bash ais_bench-gen-response-anomaly-config \ @@ -110,11 +117,23 @@ ais_bench-gen-response-anomaly-config \ --output-dir ./msprobe_configs ``` -When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the service inference requests; the value is constrained by the detection algorithm and cannot be configured externally. During inference, the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start. After inference finishes, the detection thread streams and decompresses the staging data and calls msProbe; detection results are written to `response_anomaly//.jsonl`. Each Case contains `is_anomaly`, `anomaly_type` (0: normal, 1: rare character, 2: garbled, 3: repetition, 4: NaN value) and `detection_status`. After detection, the staging data is retained or cleaned according to `payload_retention`. The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. +When `model_name` is not configured explicitly, the **model name is taken from the model path** (`model_path`, or the model `path` field; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`), matching the config generator's default. When neither `model_name` nor a model path is available (e.g. only the explicit msProbe resource paths are configured), the task fails fast at startup and asks for an explicit `model_name`, instead of silently running detection with a wrong model name. + +When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the service inference requests; the value is constrained by the detection algorithm and cannot be configured externally. For vLLM backends, `return_token_ids=True` and `return_tokens_as_token_ids=True` are also appended to obtain token ids; if the server version is too old to support these parameters, requests may fail — upgrade vLLM in that case. During inference, the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start. After inference finishes, the detection thread streams and decompresses the staging data and calls msProbe; detection results are written to `response_anomaly//.jsonl`. Each Case contains `id`, `uuid`, `is_anomaly`, `anomaly_type` (0: normal, 1: rare character, 2: garbled, 3: repetition, 4: NaN value), `anomaly_type_name` (the type-name string such as `normal`/`garbled`/`repetition`, which is more convenient for statistics), and `detection_status`. After detection, the staging data is retained or cleaned according to `payload_retention`. The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. + +The `detection_status` values in the detection results are: + +| Status | Meaning | Troubleshooting | +| --- | --- | --- | +| `completed` | msProbe was invoked and returned a detection result | Nothing to do | +| `skipped` | The inference response did not carry token ids or top-k logprobs | Check whether the service supports and returns `logprobs` / `top_logprobs` / token id fields | +| `unavailable` | `mindstudio-probe` (the response_anomaly extra) is not installed | Install the optional dependency per the [Installation Guide](../../get_started/install.md) and re-run | +| `failed` | An exception occurred during invocation or input conversion | Check the `reason` field of the Case result (error type and summary) and the detection logs | + +**Anomaly detection results do not affect the original evaluation metrics**: anomalous Cases are not rewritten as inference failures; accuracy/performance metrics are computed as usual, and anomaly information is an independent audit result. Detection-specific logs are located at `/logs/response_anomaly//.out`; detection progress and per-type statistics can also be found in the `/status_tmp/tmp_ResponseAnomaly.json` status file. ```python response_anomaly = dict( - enabled=True, payload_retention='anomalies', # all | anomalies | none payload_storage=dict( format='jsonl', @@ -127,4 +146,36 @@ response_anomaly = dict( `all` keeps every payload and atomically promotes the staging data to the official archive after detection without re-compressing; `anomalies` keeps only detected anomalies plus detection-failed/unavailable Cases; `none` keeps no payload. All three modes keep the standalone detection results. `--reuse` must keep the retention policy of the original work directory. Results are written to disk in batches, and the status is refreshed at most once per second; msProbe token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. -Detection runs through msProbe's `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)`; all three file paths can be configured in AISBench. Install the optional dependencies first: `pip install 'ais-bench-benchmark[response_anomaly]'`. During installation, pip downloads and builds the pinned msProbe source from GitCode, so the environment needs Git and network access. The service response must contain `token_ids` (or `tokens`) and `topk_logprobs`; Cases missing these fields are recorded with a `skipped` status. `model_name` must be consistent with msProbe's `mtype_config.json` and the token-category mapping. When using `--reuse`, existing detection results are inherited by Case id, and completed Cases are not re-detected. +Files produced by detection are laid out under `` as follows: + +```text +/ +├── predictions//.jsonl # Inference results (lightweight, no token/logprob payload) +├── response_anomaly/ +│ └── / +│ ├── .jsonl # Detection results, one Case per line +│ ├── payload_staging// # Transient staging during inference; cleaned after detection +│ │ └── part-*.jsonl.zst +│ └── payload// # Payload archive; absent when payload_retention is none +│ ├── payload_manifest.json # Archive manifest (per-shard rows, sizes, sha256) +│ └── part-*.jsonl.zst # Compressed payload shards (at most rows_per_shard Cases each) +├── response_anomaly_config/ # Present only when auto-generated via model_path +│ └── / +│ ├── configs/ +│ │ ├── config.yaml # Detection algorithm thresholds (never overwritten once present) +│ │ └── mtype_config.json # Model name to BOS/EOS token id mapping +│ └── token2category/ +│ └── _.json # Token id to character-category mapping +└── logs/ + └── response_anomaly//.out # Detection-specific log +``` + +Path-by-path notes: + +- **Detection results** `response_anomaly//.jsonl`: one Case per line; field semantics are described in the `detection_status` table and the Case field notes above. +- **Payload archive** `response_anomaly//payload//`: `all` keeps every Case, `anomalies` keeps only anomalous plus detection-failed/unavailable Cases, and `none` keeps nothing (the directory does not exist). To read the data, decompress the `part-*.jsonl.zst` shards with zstandard and parse each line as JSON; `payload_manifest.json` records per-shard row counts, sizes, and sha256 checksums for integrity verification. Note: under `anomalies`, even when there is nothing to retain, an archive directory containing only an empty manifest is still published to indicate the archiving flow completed successfully — it is not a leftover file. +- **Transient files**: `payload_staging/` receives payload records during inference and is cleaned automatically after detection; `..payload-build-*` build directories left by an interrupted detection are cleaned automatically when the next detection starts; `status_tmp/tmp_ResponseAnomaly.json` is a runtime status file (detection progress and per-type statistics) removed together with the status directory when the workflow ends. +- **Auto-generated msProbe configs** `response_anomaly_config//`: generated only when `model_path` is configured and explicit mtype/token2category paths are absent. An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. +- **Detection log** `logs/response_anomaly//.out`: records the detection run for the model/dataset group, including detector initialization failures and per-Case failure reasons. + +Detection runs through msProbe's `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)`; all three file paths can be configured in AISBench. Install the optional dependencies first: `pip install 'ais-bench-benchmark[response_anomaly]'`. During installation, pip downloads and builds the pinned msProbe source from GitCode, so the environment needs Git and network access. The service response must contain `token_ids` (or `tokens`) and `topk_logprobs`; Cases missing these fields are recorded with a `skipped` status. `model_name` must be consistent with msProbe's `mtype_config.json` and the token-category mapping. When using `--reuse`, existing detection results are inherited by matching both the Case `id` and `uuid` (a changed `uuid` means the Case was re-inferred, so it never gets a stale result); Cases with `completed` status are not re-detected, while Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume. diff --git a/docs/source_en/base_tutorials/all_params/mode.md b/docs/source_en/base_tutorials/all_params/mode.md index 750b6aff..6e6f4e6f 100644 --- a/docs/source_en/base_tutorials/all_params/mode.md +++ b/docs/source_en/base_tutorials/all_params/mode.md @@ -108,7 +108,7 @@ outputs/default/ ### Response Anomaly Detection Mode Support (Optional) -msProbe response anomaly detection (`--response-anomaly`) is only supported in the `all`, `infer`, and `infer_judge` generation chains: the detection thread starts after inference finishes and runs in parallel with the Judge / Eval / Summary workflow, and the workflow waits for detection to complete before exiting. The `perf` and `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do **not** support this feature; enabling it raises an explicit error during config initialization. This feature requires the service model to return token ids and top-k logprobs; see [Response Anomaly Detection Configuration](./cli_args.md#response-anomaly-detection-configuration) for details. +msProbe response anomaly detection (`--response-anomaly`) is only supported in the `all`, `infer`, and `infer_judge` generation chains: detection starts after the inference stage finishes and is **serially bound to the inference stage** — the workflow waits for detection to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages, guaranteeing that detection results and payload archives are on disk when the inference stage exits. The `perf` and `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do **not** support this feature; enabling it raises an explicit error during config initialization. This feature requires the service model to return token ids and top-k logprobs; see [Response Anomaly Detection Configuration](./cli_args.md#response-anomaly-detection-configuration) for details. ## Performance Evaluation Scenarios ### Perf Mode diff --git a/docs/source_en/base_tutorials/all_params/models.md b/docs/source_en/base_tutorials/all_params/models.md index 29a00077..0f5a5d34 100644 --- a/docs/source_en/base_tutorials/all_params/models.md +++ b/docs/source_en/base_tutorials/all_params/models.md @@ -62,8 +62,9 @@ models = [ # Equivalent to the `models` imported via `from ais_bench.benchmark.c response_anomaly = dict( # Optional; model-level config for msProbe response anomaly detection model_name="", # Model name, for example Qwen3-30B-A3B model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs - msprobe_mtype_path='/path/to/mtype_config.json', - msprobe_token2category_dir='/path/to/token2category/', + msprobe_config_path="", # Optional; algorithm-threshold config.yaml path for manual threshold tuning + msprobe_mtype_path="", # Optional; mtype_config.json path mapping model names to BOS/EOS token ids + msprobe_token2category_dir="", # Optional; token2category directory holding per-model token-id-to-character-category maps ) ) ] @@ -95,7 +96,7 @@ The description of configurable parameters for the service-oriented inference ba | `generation_kwargs` | Dict | Configuration of inference generation parameters, depending on the specific service-oriented backend and interface type. Note: Currently, multi-sampling parameters such as `best_of` and `n` are not supported, but multiple independent inferences can be performed using the `num_return_sequences` parameter (for details, refer to 🔗 [the role of `num_return_sequences` in the Text Generation Documentation](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences\(int,)). Supports configuring `logprobs` / `top_logprobs` parameters to enable token probability information collection; see 🔗[Logprobs Collection and Analysis](../../advanced_tutorials/logprobs_collection.md) | | `returns_tool_calls` | Bool | Controls the extraction method of function call information. When set to `True`, the system extracts function call information from the `tool_calls` field of the API response; when set to `False`, the system parses function call information from the `content` field | | `pred_postprocessor` | Dict | Post-processing configuration for model output results. It is used to format, clean, or convert the original model output to meet the requirements of specific evaluation tasks | -| `response_anomaly` | Dict | Optional; model-level config for msProbe response anomaly detection, including `model_name` (must match the name in msProbe's mtype_config.json), `model_path` (local model directory, optional, used to auto-generate configs), `msprobe_mtype_path`, and `msprobe_token2category_dir`. When mtype/token-category paths are not provided, the default files inside the msProbe package are used | +| `response_anomaly` | Dict | Optional; model-level config for msProbe response anomaly detection, including `model_name` (must match the name in msProbe's mtype_config.json; when unset, the model name is taken from the model path; startup fails when neither `model_name` nor a model path is available), `model_path` (local model directory, optional, used to auto-generate configs), `msprobe_config_path` (algorithm-threshold config.yaml path, optional, for manual tuning; auto-generation never overwrites an existing file), `msprobe_mtype_path`, and `msprobe_token2category_dir`. When mtype/token-category paths are not provided, the default files inside the msProbe package are used | **Precautions**: diff --git a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md index b465318c..6be83033 100644 --- a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -322,6 +322,8 @@ After the resumption is completed, the accuracy results of all requests will be > ⚠️ Note: Resumption after interruption and retesting of failed cases may change the order of requests, which may cause slight fluctuations in results. +> 💡 When [response anomaly detection](../all_params/cli_args.md#response-anomaly-detection-configuration) is enabled, resumption also inherits existing detection results: Cases with `completed` status are not re-detected by msProbe, while Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume; existing anomaly counts are accumulated into the final statistics. + 💡[Multi-Task Evaluation](#multi-task-evaluation) also supports resumption after interruption and retesting of failed cases for all or part of the tasks. ::::{tab-set} @@ -371,7 +373,6 @@ ais_bench ais_bench/configs/accuracy_benchmark/multi_task_resume_partial_en.py - ::: :::{tab-item} Alternative: Using Command-Line Parameters - ```bash ais_bench --models vllm_api_general_chat vllm_api_stream_chat --datasets gsm8k_gen_4_shot_cot_str aime2024_gen_0_shot_chat_prompt ``` diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index 94458464..4dfd735d 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -36,7 +36,7 @@ ais_bench [OPTIONS] | `--num-prompts` | 指定数据集测评条数（按照数据集顺序选取），需传入正整数，超过数据集条数或默认情况下表示对全量数据集进行测评。 | `--num-prompts 500` | | `--max-num-workers` | 并行任务数，范围 `[1, CPU 核数]`，默认 `1`。在指定`--debug`时配置无效，所有任务串行执行。注意：性能测评场景下，并发数过高可能会导致不同进程出现资源抢占，导致测试结果失真。 | `--max-num-workers 2` | |`--num-warmups`|发送请求前预热次数，按照数据集顺序选取数据进行测试，大概num-warmups大于数据集条数时，会循环发送数据集中数据。默认 `1`；若设为0，则不预热。如果warmup阶段所有请求失败，后续推理任务将不会执行。| `--num-warmups 10` | -| `--response-anomaly` / `--no-response-anomaly` | 开启或关闭 msProbe 推理响应异常检测。命令行配置优先于配置文件中的 `response_anomaly.enabled`。检测在线程中与 Eval 并行运行；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。 | `--response-anomaly` | +| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加则关闭（无 `--no-response-anomaly` 形态）。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。 | `--response-anomaly` | | `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。命令行配置优先于配置文件，默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | ### 精度测评参数 @@ -60,11 +60,17 @@ ais_bench [OPTIONS] 当前响应异常检测仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 -在总配置文件中增加 `response_anomaly` 可启用检测，也可通过 `--response-anomaly` 覆盖： +异常检测的**功能开关仅支持命令行**：在命令中增加 `--response-anomaly` 开启（不增加即关闭）。配置文件中的 `response_anomaly` 仅用于非开关类配置（`payload_retention`、`payload_storage` 等）： ```python response_anomaly = dict( - enabled=True, + payload_retention='anomalies', # all | anomalies | none + payload_storage=dict( + format='jsonl', + compression='zstd', + compression_level=3, + rows_per_shard=2000, + ), ) ``` @@ -78,14 +84,15 @@ models = [ response_anomaly=dict( model_name="", # 填写模型名称，如 Qwen3-30B-A3B model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 - msprobe_mtype_path='/path/to/mtype_config.json', - msprobe_token2category_dir='/path/to/token2category/', + msprobe_config_path="", # 可选，msProbe 算法阈值配置 config.yaml 路径，用于手工调优检测阈值 + msprobe_mtype_path="", # 可选，msProbe 模型名与 BOS/EOS token id 映射文件 mtype_config.json 路径 + msprobe_token2category_dir="", # 可选，msProbe token2category 目录，存放各模型的 token id 到字符类别映射 ), ), ] ``` -未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时回退到 msProbe 包内默认文件；配置了 `model_path` 时会自动生成到 `/response_anomaly_config/<模型 abbr>/`。也可手动生成： +未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时回退到 msProbe 包内默认文件；配置了 `model_path` 时会自动生成到 `/response_anomaly_config/<模型 abbr>/`（自动生成不会覆盖已存在的 `config.yaml`，便于保留手工调优的阈值）。也可手动生成： ```bash ais_bench-gen-response-anomaly-config \ @@ -94,11 +101,23 @@ ais_bench-gen-response-anomaly-config \ --output-dir ./msprobe_configs ``` -启用后，AISBench 会在服务推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`；该值由检测算法约束，不支持外部配置。推理阶段将完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果。推理结束后，检测线程流式解压 staging 数据并调用 msProbe，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`；每个 Case 包含 `is_anomaly`、`anomaly_type`（0：正常，1：生僻字，2：乱码，3：重复，4：NaN Value）和 `detection_status`。检测完成后按 `payload_retention` 保留或清理 staging。状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 +`model_name` 未显式配置时自动取模型路径（`model_path` 或模型 `path` 字段）中的**模型名称**（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`），与配置生成工具的默认取值一致。当既未配置 `model_name`、也没有可用的模型路径（如仅显式配置 msprobe 三件套路径）时，任务启动即报错，要求显式配置 `model_name`，避免以错误的模型名静默运行导致检测失效。 + +启用后，AISBench 会在服务推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`；该值由检测算法约束，不支持外部配置。对 vLLM 后端还会追加 `return_token_ids=True` 与 `return_tokens_as_token_ids=True` 以获取 token id，服务端版本过低不支持这些参数时请求可能失败，需升级 vLLM。推理阶段将完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果。推理结束后，检测线程流式解压 staging 数据并调用 msProbe，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`；每个 Case 包含 `id`、`uuid`、`is_anomaly`、`anomaly_type`（0：正常，1：生僻字，2：乱码，3：重复，4：NaN Value）、`anomaly_type_name`（类型名字符串，如 `normal`/`garbled`/`repetition`，统计时更常用）和 `detection_status`。检测完成后按 `payload_retention` 保留或清理 staging。状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 + +检测结果中的 `detection_status` 取值如下： + +| 状态 | 含义 | 排查建议 | +| --- | --- | --- | +| `completed` | 已调用 msProbe 并得到检测结果 | 无需处理 | +| `skipped` | 推理响应未携带 token id 或 top-k logprobs | 检查服务端是否支持并返回 `logprobs` / `top_logprobs` / token id 字段 | +| `unavailable` | 未安装 `mindstudio-probe`（response_anomaly extra） | 参考[安装指南](../../get_started/install.md)安装可选依赖后重跑 | +| `failed` | 调用或输入转换发生异常 | 查看该 Case 结果中的 `reason` 字段（保存错误类型与摘要）及检测日志 | + +**异常检测结果不影响原有评测指标**：异常 Case 不会被改写为推理失败，精度/性能指标照常计算，异常信息是独立的审计结果。检测专属日志位于 `/logs/response_anomaly/<模型>/<数据集>.out`，检测进度与类型统计也可在 `/status_tmp/tmp_ResponseAnomaly.json` 状态文件中查看。 ```python response_anomaly = dict( - enabled=True, payload_retention='anomalies', # all | anomalies | none payload_storage=dict( format='jsonl', @@ -111,7 +130,39 @@ response_anomaly = dict( `all` 保存全部 payload，检测后直接将 staging 原子转为正式归档，不会二次压缩；`anomalies` 只保存已检出异常以及检测失败/不可用 Case；`none` 不保存 payload。三种模式都保留独立检测结果。`--reuse` 必须沿用原工作目录的保留策略。检测结果按批写盘，状态最多每秒刷新一次；msProbe token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 -检测通过 msProbe 的 `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)` 完成，三个文件路径均可由 AISBench 配置。请先安装 AISBench 的可选依赖：`pip install 'ais-bench-benchmark[response_anomaly]'`。安装过程中 pip 会从 GitCode 下载并构建已固定提交的 msProbe 源码，因此安装环境需要 Git 和网络访问。服务响应必须包含 `token_ids`（或 `tokens`）和 `topk_logprobs`；缺少这些字段的 Case 会以 `skipped` 状态落盘。`model_name` 需与 msProbe 的 `mtype_config.json` 配置以及 token 分类映射保持一致。使用 `--reuse` 时，已有检测结果按 Case id 继承，已完成 Case 不会重复检测。 +检测相关文件在 `` 下的落盘结构如下： + +```text +/ +├── predictions/<模型 abbr>/<数据集 abbr>.jsonl # 推理结果（轻量，不含 token/logprobs payload） +├── response_anomaly/ +│ └── <模型 abbr>/ +│ ├── <数据集 abbr>.jsonl # 检测结果，每行一个 Case +│ ├── payload_staging/<数据集 abbr>/ # 推理期间的临时存放区，检测完成后自动清理 +│ │ └── part-*.jsonl.zst +│ └── payload/<数据集 abbr>/ # payload 归档；payload_retention 为 none 时不存在 +│ ├── payload_manifest.json # 归档清单（分片行数、大小、sha256） +│ └── part-*.jsonl.zst # 压缩 payload 分片（每片最多 rows_per_shard 条 Case） +├── response_anomaly_config/ # 仅配置 model_path 自动生成时存在 +│ └── <模型 abbr>/ +│ ├── configs/ +│ │ ├── config.yaml # 检测算法阈值配置（已存在时不覆盖） +│ │ └── mtype_config.json # 模型名与 BOS/EOS token id 映射 +│ └── token2category/ +│ └── <模型名>_<词表大小>.json # token id 到字符类别映射 +└── logs/ + └── response_anomaly/<模型 abbr>/<数据集 abbr>.out # 检测专属日志 +``` + +各路径说明： + +- **检测结果** `response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`：每行一个 Case 的检测结果，字段含义见上文 `detection_status` 表与 Case 字段说明。 +- **payload 归档** `response_anomaly/<模型 abbr>/payload/<数据集 abbr>/`：`all` 保留全部 Case，`anomalies` 只保留异常及检测失败/不可用 Case，`none` 不保留（目录不存在）。读取时用 zstandard 解压 `part-*.jsonl.zst` 分片后逐行解析 JSON；`payload_manifest.json` 记录每个分片的行数、大小与 sha256 校验值，可用于完整性校验。注意：`anomalies` 模式下即使无任何需保留的 Case，仍会发布一个仅含空 manifest 的归档目录，表示归档流程已成功完成，不是残留文件。 +- **临时文件**：`payload_staging/` 在推理期间逐条接收 payload 写入，检测完成后自动清理；检测中断后残留的 `.<数据集>.payload-build-*` 构建目录会在下次检测启动时自动清理；`status_tmp/tmp_ResponseAnomaly.json` 为运行期状态文件（检测进度与类型统计），工作流结束后随状态目录一并清理。 +- **自动生成的 msProbe 配置** `response_anomaly_config/<模型 abbr>/`：仅当配置了 `model_path` 且未显式提供 mtype/token2category 路径时生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 +- **检测日志** `logs/response_anomaly/<模型 abbr>/<数据集 abbr>.out`：记录对应模型/数据集组的检测过程，含检测器初始化失败与单 Case 失败的具体原因。 + +检测通过 msProbe 的 `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)` 完成，三个文件路径均可由 AISBench 配置。请先安装 AISBench 的可选依赖：`pip install 'ais-bench-benchmark[response_anomaly]'`。安装过程中 pip 会从 GitCode 下载并构建已固定提交的 msProbe 源码，因此安装环境需要 Git 和网络访问。服务响应必须包含 `token_ids`（或 `tokens`）和 `topk_logprobs`；缺少这些字段的 Case 会以 `skipped` 状态落盘。`model_name` 需与 msProbe 的 `mtype_config.json` 配置以及 token 分类映射保持一致。使用 `--reuse` 时，已有检测结果按 Case 的 `id` + `uuid` 双键匹配继承（`uuid` 变化说明该 Case 已重新推理，不会错挂旧结果），`completed` 状态的 Case 不会重复检测；`skipped` / `failed` / `unavailable` 状态的 Case 会在续跑中重新检测。 部分全局常量不区分任务类型，推荐保持默认；如需自定义，可编辑常量文件：[`global_consts.py`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/global_consts.py)配置。 当前支持的参数配置如下： diff --git a/docs/source_zh_cn/base_tutorials/all_params/mode.md b/docs/source_zh_cn/base_tutorials/all_params/mode.md index d9266a05..fcd25206 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/mode.md +++ b/docs/source_zh_cn/base_tutorials/all_params/mode.md @@ -107,7 +107,7 @@ outputs/default/ ### 响应异常检测模式支持（可选） -msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测线程在推理完成后启动，与 Judge / Eval / 汇总流程并行执行，工作流退出前会等待检测完成。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，具体配置请参考 [推理响应异常检测配置](./cli_args.md#推理响应异常检测配置)。 +msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测在推理阶段结束后启动，并**串行绑定在推理阶段内**——工作流会等待检测完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，具体配置请参考 [推理响应异常检测配置](./cli_args.md#推理响应异常检测配置)。 ## 性能评测场景 ### perf模式 diff --git a/docs/source_zh_cn/base_tutorials/all_params/models.md b/docs/source_zh_cn/base_tutorials/all_params/models.md index 80ca3cd5..04cc082d 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/models.md +++ b/docs/source_zh_cn/base_tutorials/all_params/models.md @@ -58,8 +58,9 @@ models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.c response_anomaly = dict( # 可选，msProbe 推理响应异常检测的模型级配置 model_name="", # 填写模型名称，如 Qwen3-30B-A3B model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 - msprobe_mtype_path='/path/to/mtype_config.json', - msprobe_token2category_dir='/path/to/token2category/', + msprobe_config_path="", # 可选，算法阈值 config.yaml 路径，用于手工调优检测阈值 + msprobe_mtype_path="", # 可选，模型名与 BOS/EOS token id 映射文件 mtype_config.json 路径 + msprobe_token2category_dir="", # 可选，token2category 目录，存放各模型的 token id 到字符类别映射 ) ) ] @@ -90,7 +91,7 @@ models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.c | `generation_kwargs` | Dict | 推理生成参数配置，依赖具体的服务化后端和接口类型。注意：当前不支持 `best_of` 和 `n` 等多次采样参数，但支持通过`num_return_sequences`参数进行多次独立推理(具体请参考🔗[Text Generation 文档](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences)中`num_return_sequences`的作用)。支持配置 `logprobs` / `top_logprobs` 参数开启 token 概率信息采集，详见 🔗[Logprobs 采集与分析](../../advanced_tutorials/logprobs_collection.md) | | `returns_tool_calls` | Bool | 控制函数调用信息的提取方式。当设置为True时，系统从API响应的`tool_calls`字段中提取函数调用信息；当设置为False时，系统从`content`字段中解析函数调用信息 | | `pred_postprocessor` | Dict | 模型输出结果的后处理配置。用于对原始模型输出进行格式化、清理或转换，以满足特定评估任务的要求 | -| `response_anomaly` | Dict | 可选，msProbe 推理响应异常检测的模型级配置，包含 `model_name`（与 msProbe 的 mtype_config.json 名称一致）、`model_path`（本地模型目录，用于自动生成配置，可选）、`msprobe_mtype_path`、`msprobe_token2category_dir`。未提供 mtype/token 分类路径时回退到 msProbe 包内默认文件 | +| `response_anomaly` | Dict | 可选，msProbe 推理响应异常检测的模型级配置，包含 `model_name`（与 msProbe 的 mtype_config.json 名称一致；未配置时自动取模型路径中的模型名称；既无 `model_name` 也无模型路径时启动报错）、`model_path`（本地模型目录，用于自动生成配置，可选）、`msprobe_config_path`（算法阈值 config.yaml 路径，可选，用于手工调优；自动生成不会覆盖已存在的文件）、`msprobe_mtype_path`、`msprobe_token2category_dir`。未提供 mtype/token 分类路径时回退到 msProbe 包内默认文件 | **注意事项：** - 响应异常检测当前仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 diff --git a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md index 5a8f9c8c..2386e7e3 100644 --- a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -324,6 +324,8 @@ ais_bench --models vllm_api_general --datasets gsm8k_gen --reuse 20250628_151326 > ⚠️ 注意：中断续测与失败重测可能改变请求顺序，可能引发结果微小波动。 +> 💡 启用了[推理响应异常检测](../all_params/cli_args.md#推理响应异常检测配置)时，续测同样继承已有检测结果：`completed` 状态的 Case 不会重复调用 msProbe，`skipped` / `failed` / `unavailable` 状态的 Case 会重新检测；已有异常计数累加进最终统计。 + 💡[多任务测评](#多任务测评) 也支持全量和部分任务的中断续测 & 失败用例重测。 ::::{tab-set} diff --git "a/docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" "b/docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" deleted file mode 100644 index 5892e599..00000000 --- "a/docs/source_zh_cn/design/AISBench\346\216\250\347\220\206\345\223\215\345\272\224\345\274\202\345\270\270\346\243\200\346\265\213\346\250\241\345\235\227\350\256\276\350\256\241.md" +++ /dev/null @@ -1,372 +0,0 @@ -# AISBench 推理响应异常检测模块设计 - -## 1. 概述 - -### 1.1 背景 - -大模型推理服务可能出现生僻字、乱码、重复输出以及 logprob 为 `NaN` 或 `Inf` 等输出异常。AISBench 在完成服务模型推理后，基于推理响应中的 token id 与 top-k logprobs 调用 msProbe 的 Response Anomaly 能力，对每个 Case 进行异常检测。 - -本模块不实现或改写检测算法；异常判定统一由已安装的官方 `mindstudio-probe` 包提供的 `msprobe.response_anomaly.detector.ILLDetector` 完成。 - -### 1.2 目标 - -1. 通过总配置与命令行开关启用或关闭响应异常检测。 -2. 推理完成后以后台线程执行检测，并与后续 Eval、汇总阶段并行。 -3. 输出 Case 级异常状态、异常类型与检测执行状态。 -4. 将检测明细落盘，并在任务状态面板显示检测进度及类型统计。 -5. 在 `--reuse` 中断续推场景继承已有异常检测结果和统计数量。 -6. 将 msProbe 作为可选依赖；未安装时不影响普通 AISBench 推理与评测。 - -### 1.3 非目标 - -- 不在 AISBench 内部复制、修改或维护 msProbe 的异常检测算法。 -- 不在 AISBench 内部复制或维护 token 分类生成算法；AISBench 仅提供包装工具调用 msProbe 官方生成器，并将产物输出到用户指定目录。 -- 不将异常 Case 自动改写为推理失败，也不改变原有评测指标；异常信息是独立审计结果。 -- 当前仅覆盖通过 `BaseAPIModel` 的服务模型生成链路。 -- 不支持性能模式（`perf`）与 Agent 测评链路（SWE-bench / SWE-bench Pro / BFCL / agent_example 等）；在这些场景启用会直接报错，避免静默空转或改变 Agent 请求参数。 - -## 2. 依赖与前置条件 - -### 2.1 软件依赖 - -响应异常检测通过可选 extra 引入官方包： - -```bash -pip install 'ais-bench-benchmark[response_anomaly]' -``` - -依赖定义位于 [requirements/response_anomaly.txt](../../../requirements/response_anomaly.txt)。安装 `response_anomaly` extra 时，pip 会从 GitCode 下载并构建固定提交 `3de412d71d6566a62c28b9131f9969930628d87f` 的官方 msProbe 源码。AISBench 正常安装不强制安装该依赖；安装环境需要 Git 与 GitCode 网络访问。 - -### 2.2 服务响应要求 - -当前仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 - -服务端必须在响应中返回： - -- 生成 token 序列：`token_ids` 或 `tokens` -- 每个生成 token 对应的 top-k logprobs：`topk_logprobs` - -AISBench 启用功能时会向服务请求参数补充： - -```python -logprobs=True -top_logprobs=20 -``` - -`top_logprobs` 由检测算法固定为 `20`，不作为外部配置项开放。 - -服务适配器将上述字段提取为 `response_anomaly_payload`，输出处理器立即将其分流到独立的 ZSTD 压缩 JSONL staging，prediction 不保存完整 payload。检测线程流式解压 staging；服务未提供必要字段时，Case 仍正常评测，但检测结果记录为 `skipped`。 - -### 2.3 msProbe 模型配置要求 - -msProbe 检测依赖三个文件： - -| 文件 | 作用 | -| --- | --- | -| `config.yaml` | 检测算法阈值配置。 | -| `mtype_config.json` | 模型名与 BOS/EOS token id 映射，用于交叉验证模型。 | -| `token2category/<模型名>_<词表大小>.json` | token id 到字符类别映射，用于生僻字和乱码检测。 | - -三个文件均可通过 AISBench 配置指定路径；未指定时回退到 msProbe 安装包内默认文件。对于 msProbe 未内置的模型，可使用 AISBench 提供的包装工具调用 msProbe 官方 `gen_model_config.py` 生成： - -```bash -ais_bench-gen-response-anomaly-config \ - --model-path /home/Qwen3-30B-A3B \ - --model-name Qwen3-30B-A3B \ - --output-dir ./msprobe_configs -``` - -产物布局： - -```text -./msprobe_configs/ -├── configs/ -│ ├── config.yaml -│ └── mtype_config.json -└── token2category/ - └── qwen3-30b-a3b_151643.json -``` - -`mtype_config.json` 支持多模型合并，多次运行不会互相覆盖；`config.yaml` 已存在时不会被覆盖，便于用户手工调阈值。 - -## 3. 总体设计 - -### 3.1 模块关系 - -```mermaid -flowchart LR - CLI[CLI / 总配置] --> CM[ConfigManager] - CM --> API[BaseAPIModel] - API --> PRED[轻量预测 JSONL] - API --> PAYLOAD[独立 JSONL.ZST staging] - PAYLOAD --> COORD[ResponseAnomalyCoordinator] - COORD --> MSP[msprobe.ILLDetector] - MSP --> RESULT[异常结果 JSONL] - COORD --> STATUS[状态文件] - STATUS --> BOARD[任务状态面板] - PRED --> EVAL[Eval] - EVAL --> SUMMARY[评测汇总] -``` - -### 3.2 执行时序 - -```mermaid -sequenceDiagram - participant I as Infer - participant P as Predictions JSONL - participant C as ResponseAnomalyCoordinator - participant M as msProbe - participant E as Eval / Summary - participant R as Response Anomaly JSONL - - I->>P: 写入轻量推理 Case - I->>C: token/logprobs 写入独立 JSONL.ZST staging - I->>C: 推理阶段结束后启动后台线程 - par 异常检测 - C->>M: ILLDetector.run(topk_logprobs, tokens, model_configs) - M-->>C: [is_ill, ill_type] - C->>R: 写入 Case 检测结果 - and 正常评测 - I->>E: 进入 Judge / Eval / Summary - end - C-->>E: 工作流收尾前 join -``` - -### 3.3 关键模块 - -| 模块 | 文件 | 职责 | -| --- | --- | --- | -| CLI 开关 | [ais_bench/benchmark/cli/argument_parser.py](../../../ais_bench/benchmark/cli/argument_parser.py) | 提供 `--response-anomaly` 和 `--no-response-anomaly`。 | -| 配置归一化 | [ais_bench/benchmark/cli/config_manager.py](../../../ais_bench/benchmark/cli/config_manager.py) | 合并 CLI / 总配置，注入服务端 logprobs 请求参数。 | -| 工作流协调 | [ais_bench/benchmark/cli/workers.py](../../../ais_bench/benchmark/cli/workers.py) | 推理结束后启动检测线程；工作流结束前等待线程完成。 | -| 响应采集 | [ais_bench/benchmark/models/api_models/base_api.py](../../../ais_bench/benchmark/models/api_models/base_api.py) | 从流式或非流式服务响应中提取 token 与 top-k logprobs。 | -| Case 分流 | [ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py](../../../ais_bench/benchmark/openicl/icl_inferencer/output_handler/base_handler.py) | 将 `response_anomaly_payload` 写入独立 JSONL.ZST staging，并从 prediction Case 移除。 | -| 检测协调器 | [ais_bench/benchmark/utils/response_anomaly.py](../../../ais_bench/benchmark/utils/response_anomaly.py) | 读预测、合并模型级配置、按需生成 msProbe 配置、初始化检测器、落盘、恢复与状态上报。 | -| 配置生成工具 | [ais_bench/tools/response_anomaly/gen_model_config.py](../../../ais_bench/tools/response_anomaly/gen_model_config.py) | 包装 msProbe 官方生成器，输出到用户目录并合并 mtype 配置。 | -| 面板展示 | [ais_bench/benchmark/runners/base.py](../../../ais_bench/benchmark/runners/base.py) | 动态读取 `ResponseAnomaly` 状态并展示。 | - -## 4. 配置设计 - -### 4.1 总配置与模型级配置 - -运行级开关与公共参数放在全局 `response_anomaly`，模型相关配置放在各模型的 `response_anomaly` 中： - -异常检测配置不会预置在通用模型配置模板中。需要启用该功能时，请在实际使用的模型配置文件中找到 `models` 列表里的目标模型，并在该模型的 `dict` 内添加 `response_anomaly`；它与 `generation_kwargs`、`pred_postprocessor` 等模型字段同级。未启用异常检测时无需添加。 - -```python -response_anomaly = dict( - enabled=True, - payload_retention='anomalies', # all | anomalies | none - payload_storage=dict( - format='jsonl', - compression='zstd', - compression_level=3, - rows_per_shard=2000, - ), - msprobe_config_path='/path/to/config.yaml', # 可选，算法阈值配置 -) - -models = [ - dict( - abbr='qwen3-30b', - attr='service', - response_anomaly=dict( - model_name="", # 填写模型名称，如 Qwen3-30B-A3B - model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 - msprobe_mtype_path='/path/to/mtype_config.json', - msprobe_token2category_dir='/path/to/token2category/', - ), - ), -] -``` - -| 配置项 | 层级 | 类型 | 默认值 | 说明 | -| --- | --- | --- | --- | --- | -| `enabled` | 全局 | bool | `False` | 是否启动异常检测。 | -| `payload_retention` | 全局 | str | `anomalies` | `all` 保存全部 payload；`anomalies` 仅保存异常及检测失败/不可用 payload；`none` 不保存 payload。 | -| `payload_storage.compression_level` | 全局 | int | `3` | ZSTD 压缩级别，范围 1-22。 | -| `payload_storage.rows_per_shard` | 全局 | int | `2000` | 每个 `.jsonl.zst` 分片的最大 Case 数。 | -| `model_name` | 模型 | str | 模型 `abbr` | msProbe 模型名称，应与其 `mtype_config.json` 及 token 分类映射一致。 | -| `model_path` | 模型 | str | 无 | 本地模型目录；配置后且未指定 mtype/token2category 时自动生成。 | -| `msprobe_config_path` | 全局/模型 | str | msProbe 包内默认 | 算法阈值 `config.yaml` 路径。 | -| `msprobe_mtype_path` | 模型 | str | msProbe 包内默认 | `mtype_config.json` 路径。 | -| `msprobe_token2category_dir` | 模型 | str | msProbe 包内默认 | `token2category/` 目录路径。 | - -当 `model_path` 已配置且未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时，AISBench 在检测启动前自动调用配置生成工具，输出到 `/response_anomaly_config/<模型 abbr>/`。 - -### 4.2 命令行优先级 - -- `--response-anomaly`：强制启用。 -- `--no-response-anomaly`：强制关闭。 -- `--response-anomaly-payload-retention {all,anomalies,none}`：覆盖 payload 保存模式，但不隐式开启异常检测。 -- 未传命令行参数：采用 `response_anomaly.enabled`。 - -命令行优先级高于总配置；未指定 payload 保存模式时默认使用 `anomalies`。 - -## 5. 数据与接口设计 - -### 5.1 msProbe 调用接口 - -```python -from msprobe.response_anomaly.detector import ILLDetector - -detector = ILLDetector( - config_path, - mtype_path, - tk2cat_path, -) -result = detector.run([topk_logprobs], [tokens], [model_name]) -``` - -AISBench 直接使用 `ILLDetector`，以便传入用户配置的三个文件路径；每个模型组只初始化一次检测器，避免逐 Case 重复加载配置。输入、输出与 msProbe 保持一致： - -| 参数 | 类型 | 说明 | -| --- | --- | --- | -| `topk_logprobs` | `List[List[Dict[int, float]]]` | 每个请求中每个 token 的候选 token id 与 logprob。 | -| `tokens` | `List[List[int]]` | 每个请求的生成 token id 序列。 | -| `model_configs` | `List[Any]` | 每个请求对应的模型名称或 msProbe 支持的模型配置。 | -| 返回值 | `List[List[Any]]` | 格式为 `[[is_ill, ill_type], ...]`。 | - -异常类型约定：`0` 正常、`1` 生僻字、`2` 乱码、`3` 重复、`4` NaN Value。 - -### 5.2 预测 Case 扩展字段 - -当服务返回必要信息时，原始预测 JSONL 会增加： - -```json -{ - "id": 12, - "uuid": "...", - "success": true, - "prediction": "...", - "response_anomaly_payload": { - "tokens": [151643, 123, 456], - "topk_logprobs": [ - {"151643": -0.01, "12": -5.2}, - {"123": -0.12, "88": -3.5} - ] - } -} -``` - -该字段仅作为检测输入保留，Eval 不读取该字段，因此不改变原有指标计算。 - -### 5.3 异常结果 Schema - -异常检测结果写入： - -```text -/response_anomaly//.jsonl -``` - -每行对应一个 Case： - -```json -{ - "id": 12, - "uuid": "...", - "is_anomaly": true, - "anomaly_type": 3, - "anomaly_type_name": "repetition", - "detection_status": "completed" -} -``` - -`detection_status` 的取值： - -| 状态 | 含义 | -| --- | --- | -| `completed` | 已调用 msProbe 并得到检测结果。 | -| `skipped` | 推理响应未携带 token 或 top-k logprobs。 | -| `unavailable` | 未安装 `mindstudio-probe`。 | -| `failed` | 调用或输入转换发生异常，`reason` 保存错误摘要。 | - -## 6. 并发、状态与恢复设计 - -### 6.1 并发策略 - -检测线程由 `ResponseAnomalyCoordinator` 管理： - -1. Infer runner 全部任务完成后启动一个后台线程。 -2. 线程逐个读取预测 Case 并调用 msProbe。 -3. Infer worker 在启动检测线程后串行等待其完成：专属检测状态面板在推理面板之后、评测面板之前渲染，检测结束并打屏最终状态后才进入 Eval / JudgeInfer / AccViz。 - -检测不参与模型请求链路，不增加单 Case 推理请求的同步等待时间；检测串行绑定在推理阶段内完成，保证 Infer 阶段退出时检测结果与 payload 归档均已落盘。 - -同一个 `work_dir` 在任一时刻只允许一个 AISBench 任务读写。协调器的线程状态检查仅防止单进程内重复启动，不提供跨进程互斥；并行执行任务时必须使用不同的 `work_dir`，`--reuse` 也应在原任务退出后串行执行。 - -### 6.2 状态面板 - -协调器将状态写入： - -```text -/status_tmp/tmp_ResponseAnomaly.json -``` - -状态字段包括： - -- `finish_count`：已处理 Case 数量。 -- `total_count`：预测文件中的 Case 总数。 -- `progress_description`：检测中或检测完成。 -- `other_kwargs`：按 `anomaly_type_name` 聚合的数量。 - -检测状态由独立的专属面板渲染（评测面板不再混入检测行，保证两类状态分开放置）：`ResponseAnomaly` 状态文件使用原子替换写入，普通 runner 清理临时目录时会保留该文件；检测启动后 Infer worker 会拉起一个独立监控进程展示检测进度，检测完成后打出最终状态表并统一清理状态目录。 - -### 6.3 中断续推 - -检测开始前，协调器读取已存在的异常结果 JSONL： - -1. 以 `id` 建立已处理 Case 集合。 -2. 已存在的结果不重复调用 msProbe。 -3. 将既有结果的异常类型累加到实时统计。 -4. 仅处理预测 JSONL 中未记录的 Case。 - -因此，结合 `--reuse` 继续执行时，已完成 Case 的异常数量和检测结果均被继承。 - -当 `payload_retention='anomalies'` 且没有需要保留的 Case 时，仍会发布一个只包含空 manifest 的 payload 目录。该目录表示归档流程已经成功完成，并用于后续无操作续跑判断，不属于残留文件。 - -## 7. 异常处理 - -| 场景 | 行为 | -| --- | --- | -| 未安装 msProbe | Case 结果写为 `unavailable`，普通推理与 Eval 不失败。 | -| 没有 token/logprobs | Case 结果写为 `skipped`。 | -| msProbe 抛出异常 | Case 结果写为 `failed`，保留异常类型与消息到 `reason`。 | -| 单个 Case 检测失败 | 继续处理后续 Case。 | -| 结果文件已存在 | 通过文件锁追加写入，避免并发写入冲突。 | -| 推理结果文件不存在 | 该模型/数据集组合按空输入处理，不产生 Case 检测结果。 | - -## 8. 测试设计 - -### 8.1 单元测试 - -[tests/UT/utils/test_response_anomaly.py](../../../tests/UT/utils/test_response_anomaly.py) 覆盖： - -- `msprobe.response_anomaly.detector.ILLDetector` 的初始化与调用参数，以及自定义三个配置文件路径的传递。 -- 官方接口返回的异常标志及类型向 Case 结果的映射。 -- 缺少 `response_anomaly_payload` 时的 `skipped` 分支。 - -### 8.2 集成测试 - -集成环境应满足： - -1. 安装 `ais-bench-benchmark[response_anomaly]`。 -2. 准备 msProbe 支持的模型名称及 token 分类文件。 -3. 使用能返回 `token_ids` 和 `topk_logprobs` 的兼容推理服务。 -4. 执行 `--mode all --response-anomaly`，校验预测、评测、异常结果与状态统计。 - -### 8.3 回归验收 - -- 功能关闭时：预测与 Eval 行为应与未接入模块前一致。 -- msProbe 不可用时：不影响普通推理与评测，异常结果明确标记不可用。 -- `--reuse` 时：重复执行不应新增相同 `id` 的异常结果。 -- 已知异常样本：msProbe 返回的类型与落盘类型一致。 - -## 9. 安全、兼容性与限制 - -- 当前仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置。 -- 预测 JSONL 中的 token/logprob 可能增加落盘体积，应只在开关启用时请求和保存。 -- 当前服务响应字段兼容根节点或首个 `choices` 节点；新的服务协议需要在 `BaseAPIModel` 扩展提取逻辑。 -- 仅支持 `all` / `infer` / `infer_judge` 普通生成链路；性能模式与 Agent 测评模式不支持。 -- msProbe 依赖已锁定到验证通过的 Git commit；升级时应同步更新依赖声明、模型资源兼容性验证与回归结果。 -- 该模块依赖 msProbe 提供的模型资源。模型未在其 `mtype_config.json` 或 token 分类映射中配置时，部分检测能力可能无法生效。 From 3586e4d03c7ba9cab6ce4db24683806131aab9a3 Mon Sep 17 00:00:00 2001 From: Bo Lee Date: Wed, 26 Aug 2026 15:32:54 +0800 Subject: [PATCH 10/25] fix: response anomaly model_name fallback and CLI-only enable switch (#487) * fix: derive response anomaly model_name from model path instead of abbr When response_anomaly.model_name was not configured, ConfigManager fell back to the model 'abbr'. The abbr is a task label (e.g. 'vllm-api-general-chat') unrelated to the served model, so msProbe received a model name that would silently miss the keys in mtype_config.json and token2category. In the explicit-msprobe-paths setup this produced degraded detection results ( Cases still marked 'completed' while rare-character/garbled checks never matched); in the auto-generation setup it keyed the generated configs by a meaningless name that breaks later reuse with real model-name configs. New behavior: - model_name resolution order: model-level > global > basename of model 'path'/model_path (the de-facto model name, same default the config generator uses); the abbr is never used as a fallback - When no reliable name can be derived (no model_name and no model directory), fail fast at config initialization with guidance, instead of silently proceeding with a wrong name - The detector coordinator warning no longer claims an 'abbr fallback' that never actually happened (it passes None to the detector); it now truthfully states the detector runs without a model name Tests: add basename-fallback and explicit-paths-require-model_name cases; update explicit-paths and nonexistent-paths cases for the new validation. * feat: make --response-anomaly a command-line-only enable switch The response anomaly enable switch was previously accepted from both the CLI (--response-anomaly / --no-response-anomaly) and the config file (response_anomaly.enabled), with the CLI taking precedence. This made the single most important toggle of the feature a mixed-mode setting and left a silent enable path in config files. New behavior: - --response-anomaly is the only way to enable detection - --no-response-anomaly is removed: detection is simply disabled when the flag is absent - response_anomaly.enabled in the config file is no longer supported: it is dropped with a warning pointing to the command-line switch, so a stale config value can never silently enable detection - all non-switch response_anomaly settings (payload_retention, payload_storage, model-level msprobe resources) keep working as config-file options, and --response-anomaly-payload-retention still overrides the config from the CLI The enabled value is now taken strictly from the parsed CLI boolean; non-boolean values (missing attribute, mocks) resolve to disabled. Tests: add cases for config-file enabled being ignored and for the CLI switch as the sole enable path. * fix: fall back model_name to global model_path before model path field The global response_anomaly block (top-level, sibling of 'models') can carry model_path, but the model_name fallback only looked at the model-level model_path and then the model 'path' field. When a user accidentally placed model_path in the top-level block, model_name ended up being derived from the model 'path' field (wrong model name) or raising when that field was empty, even though a valid model_path was available in the global block. Fix the fallback chain to prefer the most specific source: model-level model_name (explicit) -> model-level model_path basename -> global model_name -> global model_path basename -> model 'path' field basename Tests: add cases for global-model_path fallback and model-level-beats-global priority. * refactor: drop 'abbr is not a fallback' explanation from model_name error Remove the 'The model abbr is a task label unrelated to the served model and is not used as a fallback' sentence from the model_name validation error. The negative framing is an artifact of the old abbr-fallback behavior and adds no information for a user who just hit the error; the remaining message states the requirement and how to fix it, matching the positive wording already used in the docs. --- ais_bench/benchmark/cli/argument_parser.py | 10 +- ais_bench/benchmark/cli/config_manager.py | 78 ++++++++- ais_bench/benchmark/utils/response_anomaly.py | 10 +- tests/UT/cli/test_config_manager.py | 150 +++++++++++++++++- 4 files changed, 233 insertions(+), 15 deletions(-) diff --git a/ais_bench/benchmark/cli/argument_parser.py b/ais_bench/benchmark/cli/argument_parser.py index 687d502a..f43b0251 100644 --- a/ais_bench/benchmark/cli/argument_parser.py +++ b/ais_bench/benchmark/cli/argument_parser.py @@ -124,10 +124,12 @@ def _base_parser(self): ) parser.add_argument( '--response-anomaly', - action=argparse.BooleanOptionalAction, - default=None, - help='Enable or disable msProbe response anomaly detection. ' - 'The command-line value overrides response_anomaly.enabled in the config file. ' + action='store_true', + default=False, + help='Enable msProbe response anomaly detection. This command-line ' + 'switch is the only way to enable the feature; detection is disabled ' + 'when the flag is absent (response_anomaly.enabled in the config ' + 'file is not supported and is ignored). ' 'Only supported in all/infer/infer_judge modes; perf and Agent modes are unsupported.' ) parser.add_argument( diff --git a/ais_bench/benchmark/cli/config_manager.py b/ais_bench/benchmark/cli/config_manager.py index 8e5d2f09..f8e0d5ea 100644 --- a/ais_bench/benchmark/cli/config_manager.py +++ b/ais_bench/benchmark/cli/config_manager.py @@ -161,10 +161,21 @@ def _normalize_response_anomaly_global_config(self): """Apply CLI overrides and defaults to the global anomaly config.""" raw_anomaly_cfg = self.cfg.get('response_anomaly') or {} global_cfg = dict(raw_anomaly_cfg) if isinstance(raw_anomaly_cfg, dict) else {} - cli_enabled = getattr(self.args, 'response_anomaly', None) - if isinstance(cli_enabled, bool): - global_cfg['enabled'] = cli_enabled - global_cfg.setdefault('enabled', False) + # The enabled switch is command-line only (--response-anomaly); an + # 'enabled' key in the config file is not a supported enable path. + # Warn and drop it so it can never silently enable detection. + if 'enabled' in global_cfg: + self.logger.warning( + "response_anomaly.enabled in the config file is not " + "supported; use the --response-anomaly command-line switch " + "to enable response anomaly detection. The configured " + "value is ignored." + ) + global_cfg.pop('enabled') + # Strictly a real boolean from the CLI; anything else (missing + # attribute, mocks) means the switch was not passed -> disabled. + cli_enabled = getattr(self.args, 'response_anomaly', False) + global_cfg['enabled'] = cli_enabled if isinstance(cli_enabled, bool) else False configured_top_logprobs = global_cfg.pop('top_logprobs', None) global_cfg.setdefault('msprobe_config_path', None) cli_payload_retention = getattr( @@ -286,6 +297,7 @@ def _init_response_anomaly_model(self, model_cfg, global_cfg): self._validate_response_anomaly_model_resources( model_cfg, model_anomaly_cfg ) + self._validate_response_anomaly_model_name(model_cfg, model_anomaly_cfg) self._validate_response_anomaly_resource_paths( model_cfg, model_anomaly_cfg ) @@ -297,10 +309,25 @@ def _merge_response_anomaly_model_config(self, model_cfg, global_cfg): model_anomaly_cfg = dict(model_cfg.get('response_anomaly') or {}) configured_top_logprobs = model_anomaly_cfg.pop('top_logprobs', None) self._validate_response_anomaly_top_logprobs(configured_top_logprobs) - model_anomaly_cfg.setdefault( - 'model_name', - global_cfg.get('model_name') or model_cfg.get('abbr'), - ) + # NOTE: model_name intentionally has no model-abbr fallback. The abbr + # is a task label (e.g. 'vllm-api-general-chat') unrelated to the + # served model, so it would silently miss the keys in msProbe's + # mtype_config.json and token2category. When model_name is not set, + # derive it from the most specific source available, in order (each + # path resolved to its directory basename, the same default the + # config generator uses): explicit model_name is handled above by the + # ``not model_anomaly_cfg.get('model_name')`` guard; then model-level + # model_path, global model_name, global model_path, and finally the + # model 'path' field. + if not model_anomaly_cfg.get('model_name'): + fallback = ( + self._model_name_from_path(model_anomaly_cfg.get('model_path')) + or global_cfg.get('model_name') + or self._model_name_from_path(global_cfg.get('model_path')) + or self._model_name_from_path(model_cfg.get('path')) + ) + if fallback: + model_anomaly_cfg['model_name'] = fallback for key in ( 'model_path', 'msprobe_config_path', @@ -311,6 +338,41 @@ def _merge_response_anomaly_model_config(self, model_cfg, global_cfg): model_anomaly_cfg[key] = global_cfg.get(key) return model_anomaly_cfg + @staticmethod + def _model_name_from_path(model_path): + """Derive the de-facto model name from the model directory basename.""" + if not model_path: + return None + basename = osp.basename(osp.normpath(str(model_path).strip())) + return basename or None + + @staticmethod + def _validate_response_anomaly_model_name(model_cfg, model_anomaly_cfg): + """Fail fast when no reliable msProbe model name can be determined.""" + if model_anomaly_cfg.get('model_name'): + return + has_explicit_resources = bool( + model_anomaly_cfg.get('msprobe_mtype_path') + and model_anomaly_cfg.get('msprobe_token2category_dir') + ) + raise AISBenchConfigError( + TMAN_CODES.UNKNOWN_ERROR, + f"response_anomaly is enabled for model " + f"'{model_cfg.get('abbr', '')}' but response_anomaly.model_name " + "is not set and cannot be inferred from a model directory. " + + ( + "Since explicit msProbe resource paths are configured, set " + "response_anomaly.model_name to the key used in " + "msprobe_mtype_path and in the token2category file names." + if has_explicit_resources + else "Set response_anomaly.model_name explicitly (it must " + "match the keys in msProbe's mtype_config.json and the " + "token2category file names), or set model " + "'path'/model_path to the local model directory so the " + "name can be derived from its basename." + ), + ) + @staticmethod def _resolve_response_anomaly_model_path(model_cfg, model_anomaly_cfg): """Use the model tokenizer path when no explicit model path is set.""" diff --git a/ais_bench/benchmark/utils/response_anomaly.py b/ais_bench/benchmark/utils/response_anomaly.py index 01162c53..2e7ff99e 100644 --- a/ais_bench/benchmark/utils/response_anomaly.py +++ b/ais_bench/benchmark/utils/response_anomaly.py @@ -453,9 +453,15 @@ def _prepare_detection_context( len(inherited), ) if not model_name_warned and not anomaly_cfg.get("model_name"): + # ConfigManager normally resolves model_name (explicit value, or + # the model_path basename) before the task runs, so this branch + # only triggers on manually built configs. The detector receives + # None as the model name in that case; warn truthfully instead + # of claiming an abbr fallback that never happens. self.logger.warning( - "response_anomaly.model_name is not set; falling back to model " - "abbr '%s'. msProbe model matching may be degraded.", + "response_anomaly.model_name is not set; msProbe will be " + "called without a model name and model matching may be " + "degraded. Set response_anomaly.model_name for model '%s'.", task.model_cfg.get("abbr"), ) model_name_warned = True diff --git a/tests/UT/cli/test_config_manager.py b/tests/UT/cli/test_config_manager.py index 5b9943d0..7fdea83c 100644 --- a/tests/UT/cli/test_config_manager.py +++ b/tests/UT/cli/test_config_manager.py @@ -700,6 +700,43 @@ def test_response_anomaly_rejected_for_agent_model(self): with self.assertRaises(AISBenchConfigError): config_manager._init_response_anomaly_config() + def test_response_anomaly_config_enabled_key_is_ignored(self): + """配置文件中的 response_anomaly.enabled 不再生效：开关仅支持命令行。""" + self.args.mode = 'all' + self.args.response_anomaly = False # 命令行未传 --response-anomaly + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'response_anomaly': {'enabled': True}, # 配置文件写 enabled=True + 'models': [self._service_model()], + 'datasets': [{'abbr': 'dataset'}], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + # 配置文件的 enabled 被忽略，最终以命令行为准：未启用 + self.assertFalse(config_manager.cfg['response_anomaly']['enabled']) + # 未启用时不注入 logprobs 请求参数 + generation_kwargs = config_manager.cfg['models'][0]['generation_kwargs'] + self.assertNotIn('response_anomaly_enabled', generation_kwargs) + + def test_response_anomaly_cli_switch_enables_detection(self): + """仅命令行 --response-anomaly 能开启检测（无 --no- 关闭形态）。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [self._service_model()], + 'datasets': [{'abbr': 'dataset'}], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + self.assertTrue(config_manager.cfg['response_anomaly']['enabled']) + generation_kwargs = config_manager.cfg['models'][0]['generation_kwargs'] + self.assertTrue(generation_kwargs['response_anomaly_enabled']) + def test_response_anomaly_injects_request_kwargs(self): """启用响应异常检测时为 service 模型注入 logprobs 与内部开关。""" self.args.mode = 'all' @@ -1009,7 +1046,7 @@ def test_response_anomaly_rejects_missing_model_resources(self): self.assertIn('ais_bench-gen-response-anomaly-config', message) def test_response_anomaly_accepts_explicit_msprobe_paths(self): - """显式配置 mtype + token2category（真实存在）时无需 model_path。""" + """显式配置 mtype + token2category（真实存在）+ model_name 时无需 model_path。""" import os as _os mtype_path = _os.path.join(self.tokenizer_dir, 'mtype_config.json') @@ -1025,6 +1062,7 @@ def test_response_anomaly_accepts_explicit_msprobe_paths(self): self._service_model( path='', response_anomaly={ + 'model_name': 'Qwen3-30B-A3B', 'msprobe_mtype_path': mtype_path, 'msprobe_token2category_dir': tk2cat_dir, }, @@ -1038,8 +1076,117 @@ def test_response_anomaly_accepts_explicit_msprobe_paths(self): model_anomaly_cfg = config_manager.cfg['models'][0]['response_anomaly'] self.assertEqual(model_anomaly_cfg['msprobe_mtype_path'], mtype_path) + self.assertEqual(model_anomaly_cfg['model_name'], 'Qwen3-30B-A3B') self.assertIsNone(model_anomaly_cfg['model_path']) + def test_response_anomaly_model_name_falls_back_to_path_basename(self): + """未配置 model_name 时回退 model_path 目录名，而不是模型 abbr。""" + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + self._service_model( + abbr='vllm-api-general-chat', + response_anomaly={}, + ) + ], + 'datasets': [], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + model_anomaly_cfg = config_manager.cfg['models'][0]['response_anomaly'] + self.assertEqual( + model_anomaly_cfg['model_name'], + os.path.basename(os.path.normpath(self.tokenizer_dir)), + ) + self.assertNotEqual(model_anomaly_cfg['model_name'], 'vllm-api-general-chat') + + def test_response_anomaly_model_name_falls_back_to_global_model_path(self): + """顶层全局块 model_path（未填 model_name）也参与 model_name 推导。""" + import os as _os + + global_model_dir = _os.path.join(self.tokenizer_dir, 'Qwen3-30B-A3B') + _os.makedirs(global_model_dir, exist_ok=True) + + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'response_anomaly': {'model_path': global_model_dir}, + 'models': [ + self._service_model(path='', response_anomaly={}), + ], + 'datasets': [], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + model_anomaly_cfg = config_manager.cfg['models'][0]['response_anomaly'] + self.assertEqual(model_anomaly_cfg['model_name'], 'Qwen3-30B-A3B') + + def test_response_anomaly_model_level_model_path_beats_global(self): + """模型级 model_path 优先于全局块 model_path 推导 model_name。""" + import os as _os + + global_dir = _os.path.join(self.tokenizer_dir, 'GlobalModel') + model_dir = _os.path.join(self.tokenizer_dir, 'ModelLevelModel') + _os.makedirs(global_dir, exist_ok=True) + _os.makedirs(model_dir, exist_ok=True) + + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'response_anomaly': {'model_path': global_dir}, + 'models': [ + self._service_model( + path='', + response_anomaly={'model_path': model_dir}, + ), + ], + 'datasets': [], + 'cli_args': {}, + } + + config_manager._init_response_anomaly_config() + + model_anomaly_cfg = config_manager.cfg['models'][0]['response_anomaly'] + self.assertEqual(model_anomaly_cfg['model_name'], 'ModelLevelModel') + + def test_response_anomaly_explicit_paths_require_model_name(self): + """显式 msprobe 路径但缺 model_name 时应启动报错，而非静默用 abbr 匹配失败。""" + import os as _os + + mtype_path = _os.path.join(self.tokenizer_dir, 'mtype_config.json') + tk2cat_dir = _os.path.join(self.tokenizer_dir, 'token2category') + open(mtype_path, 'w').close() + _os.makedirs(tk2cat_dir, exist_ok=True) + + self.args.mode = 'all' + self.args.response_anomaly = True + config_manager = ConfigManager(self.args) + config_manager.cfg = { + 'models': [ + self._service_model( + path='', + response_anomaly={ + 'msprobe_mtype_path': mtype_path, + 'msprobe_token2category_dir': tk2cat_dir, + }, + ) + ], + 'datasets': [], + 'cli_args': {}, + } + + with self.assertRaises(AISBenchConfigError) as cm: + config_manager._init_response_anomaly_config() + self.assertIn('model_name', str(cm.exception)) + def test_response_anomaly_rejects_nonexistent_msprobe_paths(self): """显式配置的 msProbe 路径不存在时启动即报错，而不是运行期全 failed。""" self.args.mode = 'all' @@ -1050,6 +1197,7 @@ def test_response_anomaly_rejects_nonexistent_msprobe_paths(self): self._service_model( path='', response_anomaly={ + 'model_name': 'Qwen3-30B-A3B', 'msprobe_mtype_path': '/workspace/missing/mtype_config.json', 'msprobe_token2category_dir': '/workspace/missing/token2category', }, From 38d5ecc799148ea3e1c0d8dde8ed1effdecafc28 Mon Sep 17 00:00:00 2001 From: Hanye <1037452625@qq.com> Date: Thu, 27 Aug 2026 09:24:20 +0800 Subject: [PATCH 11/25] [feature] Support api models useful cmd (#492) * add mini dataset doc * review fix * multilingual mini fix * add upload_pip.sh * add testpypi only * add testpypi only * add testpypi only * add testpypi only * add testpypi only * add testpypi only * update install docs * simple cmd for api * add UT * add UT * add docs --------- Co-authored-by: SJTUyh --- README.md | 8 + README_en.md | 9 + ais_bench/benchmark/cli/argument_parser.py | 41 ++++ ais_bench/benchmark/cli/config_manager.py | 53 ++++++ .../base_tutorials/all_params/cli_args.md | 25 +++ docs/source_en/get_started/quick_start.md | 8 + .../base_tutorials/all_params/cli_args.md | 25 +++ docs/source_zh_cn/get_started/quick_start.md | 8 + tests/UT/cli/test_argument_parser.py | 70 +++++++ tests/UT/cli/test_config_manager.py | 176 ++++++++++++++++++ 10 files changed, 423 insertions(+) diff --git a/README.md b/README.md index 9ae700a1..ca814112 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,14 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch 模型配置文件`vllm_api_general_chat.py`中包含了模型运行相关的配置内容，是需要依据实际情况修改的。快速入门中需要修改的内容用注释标明。 +> 💡 **提示**：模型配置中的部分参数（如 `host_ip`、`host_port`、`model`、`url`、`max_out_len`、`generation_kwargs` 等）无需修改配置文件，可直接通过命令行覆盖，例如： +> +> ```bash +> ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --host-ip 127.0.0.1 --host-port 8000 +> ``` +> +> 命令行显式指定的参数会覆盖本次执行的所有模型配置中对应字段；仅覆盖配置中**已存在的字段**，未指定的参数保持配置文件原值。更多可覆盖参数及覆盖范围说明请参考 📚 [用户配置参数 - API 模型通用覆盖参数](./docs/source_zh_cn/base_tutorials/all_params/cli_args.md#api-模型通用覆盖参数)。 + ```python from ais_bench.benchmark.models import VLLMCustomAPIChat diff --git a/README_en.md b/README_en.md index da38148c..dc4d6992 100644 --- a/README_en.md +++ b/README_en.md @@ -282,6 +282,15 @@ After executing the query command, you will get the following query results: - The dataset task configuration file `demo_gsm8k_gen_4_shot_cot_chat_prompt.py` in the quick start does not require additional modifications. For an introduction to the content of the dataset task configuration file, please refer to 📚 [Configure Open-Source Datasets](https://ais-bench-benchmark.readthedocs.io/en/latest/base_tutorials/all_params/datasets.html#id6) The model configuration file `vllm_api_general_chat.py` contains configuration content related to model operation and needs to be modified according to actual conditions. The content that needs to be modified in the quick start is marked with comments. + +> 💡 **Tip**: Some parameters in the model config above (e.g. `host_ip`, `host_port`, `model`, `url`, `max_out_len`, `generation_kwargs`, etc.) can be overridden directly on the command line without editing the config file. For example: +> +> ```bash +> ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --host-ip 127.0.0.1 --host-port 8000 +> ``` +> +> An explicitly specified parameter overrides the corresponding field in **all executed model configs**; only fields **already present** in the config are overridden, and unspecified parameters keep their config-file values. For the full overridable parameter list and coverage notes, refer to 📚 [User Configuration Parameters - API Model Common Override Parameters](./docs/source_en/base_tutorials/all_params/cli_args.md#api-model-common-override-parameters). + ```python from ais_bench.benchmark.models import VLLMCustomAPIChat diff --git a/ais_bench/benchmark/cli/argument_parser.py b/ais_bench/benchmark/cli/argument_parser.py index f43b0251..04280024 100644 --- a/ais_bench/benchmark/cli/argument_parser.py +++ b/ais_bench/benchmark/cli/argument_parser.py @@ -1,4 +1,5 @@ import argparse +import json from ais_bench.benchmark.cli.utils import ( get_current_time_str, validate_max_workers, @@ -18,6 +19,7 @@ def __init__(self): self._perf_parser() self._accuracy_parser() self._custom_dataset_parser() + self._api_model_parser() def parse_args(self): args = self.parser.parse_args() @@ -194,3 +196,42 @@ def _custom_dataset_parser(self): type=str, choices=['gen']) + def _api_model_parser(self): + """API 模型通用参数，覆盖执行的所有模型配置（mindie_api/tgi_api/triton_api/vllm_api）。 + + All args default to None so that they only take effect when explicitly + specified on the command line. + """ + parser = self.parser.add_argument_group('api_model_args') + parser.add_argument('--path', type=str, default=None, + help='Path override for all API model configs') + parser.add_argument('--model-name', type=str, default=None, + help='Model name override. Written to the `model` or `model_name` ' + 'field depending on the model type. Named --model-name to avoid ' + 'confusion with --models') + parser.add_argument('--request-rate', type=float, default=None, + help='Request rate override in the range [0, 64000]') + parser.add_argument('--retry', type=int, default=None, + help='Retry count override in the range [0, 1000]') + parser.add_argument('--api-key', type=str, default=None, + help='API key override') + parser.add_argument('--host-ip', type=str, default=None, + help='Host ip override') + parser.add_argument('--host-port', type=int, default=None, + help='Host port override in the range (0, 65536)') + parser.add_argument('--url', type=str, default=None, + help='Endpoint url override') + parser.add_argument('--max-out-len', type=int, default=None, + help='Max output length override') + parser.add_argument('--batch-size', type=int, default=None, + help='Batch size override in the range (0, 100000]') + parser.add_argument( + '--trust-remote-code', + action=argparse.BooleanOptionalAction, + default=None, + help='Trust remote code override (--trust-remote-code / --no-trust-remote-code)' + ) + parser.add_argument('--generation-kwargs', type=json.loads, default=None, + help='Generation kwargs override as a JSON object, ' + 'e.g. \'{"temperature": 0.01, "ignore_eos": false}\'') + diff --git a/ais_bench/benchmark/cli/config_manager.py b/ais_bench/benchmark/cli/config_manager.py index f8e0d5ea..319c62c5 100644 --- a/ais_bench/benchmark/cli/config_manager.py +++ b/ais_bench/benchmark/cli/config_manager.py @@ -1,5 +1,6 @@ import os import os.path as osp +import inspect import tabulate from mmengine.config import Config @@ -671,6 +672,8 @@ def _get_config_from_arg(self): config = try_fill_in_custom_cfgs(config) CustomConfigChecker(config, self.args.config).check() config.merge_from_dict(dict(cli_args = vars(self.args))) + if config.get('models'): + self._apply_cli_api_model_overrides(config['models']) return config models = self._load_models_config() @@ -749,8 +752,58 @@ def _load_models_config(self): if 'models' not in cfg: raise AISBenchConfigError(TMAN_CODES.CFG_CONTENT_MISS_REQUIRED_PARAM, f"Config file {model[1]} does not contain 'models' param") models += cfg['models'] + self._apply_cli_api_model_overrides(models) return models + def _resolve_model_field_name(self, model_cfg): + """Return the model-name key ('model'/'model_name') accepted by the model + class, or None if the type accepts neither. + + Which field name a config can carry depends on its `type` (the model + class), not on the config file itself. + """ + try: + params = inspect.signature(model_cfg["type"].__init__).parameters + except (TypeError, ValueError): + return None + for key in ("model", "model_name"): + if key in params: + return key + return None + + def _apply_cli_api_model_overrides(self, models): + """Override each model config with API model args explicitly given on the CLI. + + Only fields already present in a model config are overwritten (no new + keys are injected), so that classes not supporting a given param (e.g. + MindieStreamApi) are not passed unexpected keywords. The model-name field + is resolved by the model `type` signature. + """ + fields = ["path", "request_rate", "retry", "api_key", "host_ip", + "host_port", "url", "max_out_len", "batch_size", + "trust_remote_code", "generation_kwargs"] + for model_cfg in models: + # 1) model/model_name depends on the type: overwrite if accepted, + # otherwise warn and skip. + model_val = getattr(self.args, "model_name", None) + if model_val is not None: + target_key = self._resolve_model_field_name(model_cfg) + if target_key is not None: + model_cfg[target_key] = model_val + else: + type_name = getattr(model_cfg.get("type"), "__name__", model_cfg.get("type")) + self.logger.warning( + 'CLI --model-name=%s is ignored: model type %s accepts ' + 'neither model nor model_name', + model_val, type_name, + ) + # 2) Other common fields: only overwrite existing keys. + for field in fields: + cli_val = getattr(self.args, field, None) + if cli_val is None or field not in model_cfg: + continue + model_cfg[field] = cli_val + def _load_summarizers_config(self): # parse summarizer args summarizer_arg = self.args.summarizer if self.args.summarizer is not None else 'example' diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index b0c8828f..57a48709 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -40,6 +40,31 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it disables detection (there is no `--no-response-anomaly` form). Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | | `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | +### API Model Common Override Parameters + +Applicable to service-oriented inference backends (API models such as vLLM, Triton, MindIE, TGI, etc.), used to directly override common fields in the model configuration via the command line without modifying the model configuration files. + +> ⚠️ **Coverage Notes**: +> - Only fields **already present** in the model config are overridden; no new keys are added (so model classes that do not support a given field do not receive unexpected keywords, preserving backward compatibility). +> - An explicitly specified parameter overrides the corresponding field in **all executed model configs** (effective across multiple model tasks in the same command). +> - Parameters not explicitly specified are ignored (default `None`), keeping the original values in the config files. +> - The model-name field is written to `model` or `model_name` depending on the model `type` constructor signature: VLLM classes use `model`, Triton uses `model_name`; when the type accepts neither (e.g. MindIE, TGI), a warning is printed and the value is skipped. + +| Parameter | Description | Example | +| ---- | ---- | ---- | +| `--path` | Overrides the `path` field (Tokenizer/model vocabulary path) | `--path /weight/Qwen` | +| `--model-name` | Overrides the model name, written to `model` or `model_name` based on the model `type` (VLLM→`model`, Triton→`model_name`; warning+skip for MindIE/TGI) | `--model-name Qwen` | +| `--request-rate` | Overrides `request_rate` (request sending rate) | `--request-rate 10` | +| `--retry` | Overrides `retry` (max retries per request) | `--retry 3` | +| `--api-key` | Overrides `api_key` (custom API key) | `--api-key sk-xxx` | +| `--host-ip` | Overrides `host_ip` (inference service IP) | `--host-ip 127.0.0.1` | +| `--host-port` | Overrides `host_port` (inference service port) | `--host-port 8000` | +| `--url` | Overrides `url` (custom URL path for the inference service) | `--url http://x.x.x.x:8000/v1` | +| `--max-out-len` | Overrides `max_out_len` (max output tokens) | `--max-out-len 1024` | +| `--batch-size` | Overrides `batch_size` (max request concurrency) | `--batch-size 4` | +| `--trust-remote-code` | Overrides `trust_remote_code` (whether the tokenizer trusts remote code); supports `--trust-remote-code` / `--no-trust-remote-code` | `--no-trust-remote-code` | +| `--generation-kwargs` | Overrides `generation_kwargs` (generation parameters) as a JSON object, **replacing** the config dict entirely | `--generation-kwargs '{"temperature": 0.5}'` | + ### Accuracy Evaluation Parameters Valid only when the mode is `all`, `infer`, `eval`, or `viz`. diff --git a/docs/source_en/get_started/quick_start.md b/docs/source_en/get_started/quick_start.md index e5b8f0a7..894c738b 100644 --- a/docs/source_en/get_started/quick_start.md +++ b/docs/source_en/get_started/quick_start.md @@ -125,6 +125,14 @@ Executing the query command will yield the following results: The model configuration file `vllm_api_general_chat.py` contains configuration content related to model operation and needs to be modified according to the actual situation. The content that needs to be modified in the quick start is marked with comments. +> 💡 **Tip**: Some parameters in the model config above (e.g. `host_ip`, `host_port`, `model`, `url`, `max_out_len`, `generation_kwargs`, etc.) can be overridden directly on the command line without editing the config file. For example: +> +> ```bash +> ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --host-ip 127.0.0.1 --host-port 8000 +> ``` +> +> An explicitly specified parameter overrides the corresponding field in **all executed model configs**; only fields **already present** in the config are overridden, and unspecified parameters keep their config-file values. For the full overridable parameter list and coverage notes, refer to 📚 [User Configuration Parameters - API Model Common Override Parameters](../base_tutorials/all_params/cli_args.md#api-model-common-override-parameters). + ```python from ais_bench.benchmark.models import VLLMCustomAPIChat diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index 4dfd735d..7d616d32 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -39,6 +39,31 @@ ais_bench [OPTIONS] | `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加则关闭（无 `--no-response-anomaly` 形态）。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。 | `--response-anomaly` | | `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。命令行配置优先于配置文件，默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | +### API 模型通用覆盖参数 + +适用于服务化推理后端（API 模型，如 vLLM、Triton、MindIE、TGI 等），用于在不修改模型配置文件的前提下，通过命令行直接覆盖模型配置中的常用字段。 + +> ⚠️ **覆盖范围说明**： +> - 仅覆盖模型配置中**已存在的字段**，不新增字段（避免向不支持某字段的模型类传入多余参数，保证向后兼容）。 +> - 命令行显式指定的参数会覆盖**执行的所有模型配置**中对应的字段（同一命令下多个模型任务均生效）。 +> - 未显式指定的参数不生效（默认 `None`），配置文件中保持原值。 +> - `model` / `model_name` 字段按模型 `type` 的构造签名自动选择写入目标：VLLM 系列写入 `model`，Triton 写入 `model_name`；模型类型两者都不接收（如 MindIE、TGI）时打印 warning 并跳过。 + +| 参数 | 说明 | 示例 | +| ---- | ---- | ---- | +| `--path` | 覆盖模型配置的 `path` 字段（Tokenizer/模型序列化词表文件路径） | `--path /weight/Qwen` | +| `--model-name` | 覆盖模型名。按模型 `type` 自动写入 `model` 或 `model_name` 字段（VLLM→`model`，Triton→`model_name`；MindIE/TGI 不接收时 warning 跳过） | `--model-name Qwen` | +| `--request-rate` | 覆盖 `request_rate`（请求发送速率） | `--request-rate 10` | +| `--retry` | 覆盖 `retry`（每个请求最大重试次数） | `--retry 3` | +| `--api-key` | 覆盖 `api_key`（自定义 API key） | `--api-key sk-xxx` | +| `--host-ip` | 覆盖 `host_ip`（推理服务 IP） | `--host-ip 127.0.0.1` | +| `--host-port` | 覆盖 `host_port`（推理服务端口） | `--host-port 8000` | +| `--url` | 覆盖 `url`（自定义访问推理服务的 URL 路径） | `--url http://x.x.x.x:8000/v1` | +| `--max-out-len` | 覆盖 `max_out_len`（推理输出最大 token 数） | `--max-out-len 1024` | +| `--batch-size` | 覆盖 `batch_size`（请求最大并发数） | `--batch-size 4` | +| `--trust-remote-code` | 覆盖 `trust_remote_code`（tokenizer 是否信任远程代码），支持 `--trust-remote-code` / `--no-trust-remote-code` 两种形态 | `--no-trust-remote-code` | +| `--generation-kwargs` | 覆盖 `generation_kwargs`（推理生成参数），以 JSON 对象形式传入，**整体替换**配置中的 dict | `--generation-kwargs '{"temperature": 0.5}'` | + ### 精度测评参数 仅在模式为 `all、infer、eval` 或 `viz` 时有效。 | 参数| 说明 | 示例| diff --git a/docs/source_zh_cn/get_started/quick_start.md b/docs/source_zh_cn/get_started/quick_start.md index bb10ed62..ebf1f825 100644 --- a/docs/source_zh_cn/get_started/quick_start.md +++ b/docs/source_zh_cn/get_started/quick_start.md @@ -127,6 +127,14 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch 模型配置文件`vllm_api_general_chat.py`中包含了模型运行相关的配置内容，是需要依据实际情况修改的。快速入门中需要修改的内容用注释标明。 +> 💡 **提示**：模型配置中的部分参数（如 `host_ip`、`host_port`、`model`、`url`、`max_out_len`、`generation_kwargs` 等）无需修改配置文件，可直接通过命令行覆盖，例如： +> +> ```bash +> ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --host-ip 127.0.0.1 --host-port 8000 +> ``` +> +> 命令行显式指定的参数会覆盖本次执行的所有模型配置中对应字段；仅覆盖配置中**已存在的字段**，未指定的参数保持配置文件原值。更多可覆盖参数及覆盖范围说明请参考 📚 [用户配置参数 - API 模型通用覆盖参数](../base_tutorials/all_params/cli_args.md#api-模型通用覆盖参数)。 + ```python from ais_bench.benchmark.models import VLLMCustomAPIChat diff --git a/tests/UT/cli/test_argument_parser.py b/tests/UT/cli/test_argument_parser.py index d07c977a..0e8e3a77 100644 --- a/tests/UT/cli/test_argument_parser.py +++ b/tests/UT/cli/test_argument_parser.py @@ -217,6 +217,72 @@ def test_parse_args_mcq_and_gen_options(self, mock_get_current_time_str): self.assertEqual(args.custom_dataset_data_type, 'qa') self.assertEqual(args.custom_dataset_infer_method, 'gen') + @patch('ais_bench.benchmark.cli.argument_parser.get_current_time_str') + def test_parse_args_api_model_options(self, mock_get_current_time_str): + """测试API模型通用覆盖选项参数解析""" + mock_get_current_time_str.return_value = "20230516_144254" + + sys.argv = [ + 'benchmark.py', + '--path', '/tmp/model', + '--model-name', 'Qwen', + '--request-rate', '10', + '--retry', '3', + '--api-key', 'sk-test', + '--host-ip', '127.0.0.1', + '--host-port', '8000', + '--url', 'http://example.com/v1', + '--max-out-len', '256', + '--batch-size', '4', + '--trust-remote-code', + '--generation-kwargs', '{"temperature": 0.5, "ignore_eos": false}', + ] + + parser = ArgumentParser() + args = parser.parse_args() + + self.assertEqual(args.path, '/tmp/model') + self.assertEqual(args.model_name, 'Qwen') + self.assertEqual(args.request_rate, 10.0) + self.assertEqual(args.retry, 3) + self.assertEqual(args.api_key, 'sk-test') + self.assertEqual(args.host_ip, '127.0.0.1') + self.assertEqual(args.host_port, 8000) + self.assertEqual(args.url, 'http://example.com/v1') + self.assertEqual(args.max_out_len, 256) + self.assertEqual(args.batch_size, 4) + self.assertTrue(args.trust_remote_code) + self.assertEqual(args.generation_kwargs, + {'temperature': 0.5, 'ignore_eos': False}) + + @patch('ais_bench.benchmark.cli.argument_parser.get_current_time_str') + def test_parse_args_api_model_options_default_none(self, mock_get_current_time_str): + """未显式指定时 api_model 覆盖参数默认均为 None""" + mock_get_current_time_str.return_value = "20230516_144254" + + sys.argv = ['benchmark.py'] + args = ArgumentParser().parse_args() + + for attr in ('path', 'model_name', 'request_rate', 'retry', 'api_key', + 'host_ip', 'host_port', 'url', 'max_out_len', 'batch_size', + 'trust_remote_code', 'generation_kwargs'): + self.assertIsNone(getattr(args, attr)) + + @patch('ais_bench.benchmark.cli.argument_parser.get_current_time_str') + def test_parse_args_api_model_no_trust_remote_code(self, mock_get_current_time_str): + """测试 --no-trust-remote-code 关闭信任远程代码""" + mock_get_current_time_str.return_value = "20230516_144254" + + sys.argv = ['benchmark.py', '--no-trust-remote-code'] + args = ArgumentParser().parse_args() + self.assertFalse(args.trust_remote_code) + + def test_parse_args_api_model_invalid_generation_kwargs_json(self): + """非法 JSON 的 --generation-kwargs 应导致解析失败""" + sys.argv = ['benchmark.py', '--generation-kwargs', '{bad json'] + with self.assertRaises(SystemExit): + ArgumentParser().parse_args() + def test_init_method_creates_parser(self): """测试初始化方法创建了解析器并添加了所有参数组""" parser = ArgumentParser() @@ -233,6 +299,10 @@ def test_init_method_creates_parser(self): self.assertTrue(hasattr(args, 'merge_ds')) # accuracy_args self.assertTrue(hasattr(args, 'pressure')) # perf_args self.assertTrue(hasattr(args, 'custom_dataset_path')) # custom_dataset_args + self.assertTrue(hasattr(args, 'model_name')) # api_model_args + self.assertTrue(hasattr(args, 'host_port')) # api_model_args + self.assertTrue(hasattr(args, 'generation_kwargs')) # api_model_args + self.assertTrue(hasattr(args, 'trust_remote_code')) # api_model_args if __name__ == '__main__': diff --git a/tests/UT/cli/test_config_manager.py b/tests/UT/cli/test_config_manager.py index 7fdea83c..4e0f062e 100644 --- a/tests/UT/cli/test_config_manager.py +++ b/tests/UT/cli/test_config_manager.py @@ -6,6 +6,7 @@ from ais_bench.benchmark.cli.config_manager import CustomConfigChecker, ConfigManager from ais_bench.benchmark.models import VLLMCustomAPI, VLLMCustomAPIChat +from ais_bench.benchmark.models import TritonCustomAPI, MindieStreamApi, TGICustomAPI from ais_bench.benchmark.utils.logging.exceptions import CommandError, AISBenchConfigError from ais_bench.benchmark.utils.logging.error_codes import TMAN_CODES @@ -143,6 +144,20 @@ def setUp(self): self.args.custom_dataset_meta_path = None self.args.response_anomaly_payload_retention = None + # api_model_args 覆盖参数默认 None（未显式指定则不覆盖） + self.args.path = None + self.args.model_name = None + self.args.request_rate = None + self.args.retry = None + self.args.api_key = None + self.args.host_ip = None + self.args.host_port = None + self.args.url = None + self.args.max_out_len = None + self.args.batch_size = None + self.args.trust_remote_code = None + self.args.generation_kwargs = None + # Local tokenizer directory consumed by response anomaly model-path # fallback tests. self.tokenizer_dir = tempfile.mkdtemp() @@ -189,6 +204,7 @@ def test_get_config_from_arg_with_config_file(self, mock_checker, mock_fill_in, """测试从配置文件获取配置""" # 配置模拟返回值 mock_config = mock.MagicMock() + mock_config.get.return_value = None # 无 models，跳过 CLI 覆盖逻辑 mock_fromfile.return_value = mock_config mock_fill_in.return_value = mock_config @@ -312,6 +328,166 @@ def test_load_models_config_missing_models_param(self, mock_fromfile, mock_match config_manager._load_models_config() self.assertEqual(cm.exception.error_code_str, TMAN_CODES.CFG_CONTENT_MISS_REQUIRED_PARAM.full_code) + def test_resolve_model_field_name_vllm(self): + """VLLMCustomAPI 系列应写入 model 字段""" + config_manager = ConfigManager(self.args) + self.assertEqual( + config_manager._resolve_model_field_name({'type': VLLMCustomAPI}), + 'model', + ) + self.assertEqual( + config_manager._resolve_model_field_name({'type': VLLMCustomAPIChat}), + 'model', + ) + + def test_resolve_model_field_name_triton(self): + """TritonCustomAPI 应写入 model_name 字段""" + config_manager = ConfigManager(self.args) + self.assertEqual( + config_manager._resolve_model_field_name({'type': TritonCustomAPI}), + 'model_name', + ) + + def test_resolve_model_field_name_accepts_neither(self): + """MindieStreamApi/TGICustomAPI 不接收 model/model_name""" + config_manager = ConfigManager(self.args) + self.assertIsNone( + config_manager._resolve_model_field_name({'type': MindieStreamApi}) + ) + self.assertIsNone( + config_manager._resolve_model_field_name({'type': TGICustomAPI}) + ) + + def test_resolve_model_field_name_unresolvable_type(self): + """type 无法解析签名时返回 None""" + config_manager = ConfigManager(self.args) + self.assertIsNone( + config_manager._resolve_model_field_name({'type': 'NotAClass'}) + ) + + def test_apply_cli_api_model_overrides_vllm_model(self): + """--model-name 作用于 vllm 类型时写入 model 字段""" + self.args.model_name = 'Qwen' + self.args.host_port = 8000 + self.args.max_out_len = 256 + config_manager = ConfigManager(self.args) + model = { + 'type': VLLMCustomAPI, + 'model': '', + 'host_port': 8080, + 'max_out_len': 512, + } + config_manager._apply_cli_api_model_overrides([model]) + + self.assertEqual(model['model'], 'Qwen') + self.assertEqual(model['host_port'], 8000) + self.assertEqual(model['max_out_len'], 256) + self.assertNotIn('model_name', model) + + def test_apply_cli_api_model_overrides_triton_model_name(self): + """--model-name 作用于 triton 类型时写入 model_name 字段""" + self.args.model_name = 'Qwen' + config_manager = ConfigManager(self.args) + model = {'type': TritonCustomAPI, 'model_name': ''} + config_manager._apply_cli_api_model_overrides([model]) + + self.assertEqual(model['model_name'], 'Qwen') + self.assertNotIn('model', model) + + def test_apply_cli_api_model_overrides_ignores_unsupported_type(self): + """类型不接收 model/model_name 时告警且不新增字段""" + self.args.model_name = 'Qwen' + config_manager = ConfigManager(self.args) + config_manager.logger = mock.MagicMock() + model = {'type': MindieStreamApi, 'path': ''} + config_manager._apply_cli_api_model_overrides([model]) + + config_manager.logger.warning.assert_called_once() + self.assertNotIn('model', model) + self.assertNotIn('model_name', model) + + def test_apply_cli_api_model_overrides_no_cli_values(self): + """CLI 未显式指定时不修改模型配置""" + config_manager = ConfigManager(self.args) + model = { + 'type': VLLMCustomAPI, + 'model': 'default', + 'host_port': 8080, + } + config_manager._apply_cli_api_model_overrides([model]) + + self.assertEqual(model['model'], 'default') + self.assertEqual(model['host_port'], 8080) + + def test_apply_cli_api_model_overrides_skips_missing_fields(self): + """只覆盖配置中已存在的字段，缺失字段不新增""" + self.args.api_key = 'sk-test' + self.args.url = 'http://example.com/v1' + config_manager = ConfigManager(self.args) + model = {'type': VLLMCustomAPI, 'url': ''} # 无 api_key 字段 + config_manager._apply_cli_api_model_overrides([model]) + + self.assertEqual(model['url'], 'http://example.com/v1') + self.assertNotIn('api_key', model) + + def test_apply_cli_api_model_overrides_generation_kwargs(self): + """--generation-kwargs 整体替换配置中的 generation_kwargs""" + self.args.generation_kwargs = {'temperature': 0.5} + config_manager = ConfigManager(self.args) + model = { + 'type': VLLMCustomAPI, + 'generation_kwargs': {'temperature': 0.01, 'ignore_eos': False}, + } + config_manager._apply_cli_api_model_overrides([model]) + + self.assertEqual(model['generation_kwargs'], {'temperature': 0.5}) + + def test_apply_cli_api_model_overrides_trust_remote_code_false(self): + """--no-trust-remote-code（False）也应覆盖 True 的配置值""" + self.args.trust_remote_code = False + config_manager = ConfigManager(self.args) + model = {'type': VLLMCustomAPI, 'trust_remote_code': True} + config_manager._apply_cli_api_model_overrides([model]) + + self.assertFalse(model['trust_remote_code']) + + @mock.patch('ais_bench.benchmark.cli.config_manager.ConfigManager._apply_cli_api_model_overrides') + @mock.patch('ais_bench.benchmark.cli.config_manager.match_cfg_file') + @mock.patch('ais_bench.benchmark.cli.config_manager.Config.fromfile') + def test_load_models_config_applies_cli_overrides(self, mock_fromfile, mock_match_cfg_file, mock_apply): + """--models 入口加载模型后应调用 CLI 覆盖逻辑""" + mock_model_file = ('test_model', os.path.join(self.args.config_dir, 'models', 'test_model.py')) + mock_match_cfg_file.return_value = [mock_model_file] + mock_fromfile.return_value = { + 'models': [{'type': VLLMCustomAPI, 'model': ''}], + } + self.args.models = ['test_model'] + + config_manager = ConfigManager(self.args) + result = config_manager._load_models_config() + + mock_apply.assert_called_once_with(result) + + @mock.patch('ais_bench.benchmark.cli.config_manager.ConfigManager._apply_cli_api_model_overrides') + @mock.patch('ais_bench.benchmark.cli.config_manager.Config.fromfile') + @mock.patch('ais_bench.benchmark.cli.config_manager.try_fill_in_custom_cfgs') + @mock.patch('ais_bench.benchmark.cli.config_manager.CustomConfigChecker') + def test_get_config_from_arg_config_file_applies_cli_overrides(self, mock_checker, mock_fill_in, mock_fromfile, mock_apply): + """--config 入口的 models 也应被 CLI 覆盖逻辑处理""" + models = [{'type': VLLMCustomAPI, 'model': ''}] + mock_config = mock.MagicMock() + mock_config.get.return_value = models + mock_config.__getitem__.return_value = models + mock_fromfile.return_value = mock_config + mock_fill_in.return_value = mock_config + self.args.config = os.path.join(self.args.config_dir, 'test_config.py') + + config_manager = ConfigManager(self.args) + result = config_manager._get_config_from_arg() + + mock_apply.assert_called_once_with(models) + self.assertEqual(result, mock_config) + @mock.patch('ais_bench.benchmark.cli.config_manager.match_cfg_file') @mock.patch('ais_bench.benchmark.cli.config_manager.make_custom_dataset_config') def test_load_datasets_config_custom_dataset(self, mock_make_config, mock_match_cfg_file): From d694ddc4ec94c0a7ce88a59344e6540dc23d210c Mon Sep 17 00:00:00 2001 From: Jianxin Date: Thu, 27 Aug 2026 10:35:52 +0800 Subject: [PATCH 12/25] =?UTF-8?q?=E3=80=90feature=E3=80=91Add=20corpusQA?= =?UTF-8?q?=201M=20dataset=20(#495)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 数据集适配 * 提示词等完全对齐 * 修复已知环境变量问题 * 修复数据集加载问题 * 适配chat接口 * 优化裁判模型输入的提取逻辑：仅提取content部分，不带reasoning部分 * 补充资料 * 补充ut用例 * 资料补充 配套文件导入方式 --- .../configs/datasets/corpusqa/README.md | 27 ++ .../configs/datasets/corpusqa/README_en.md | 27 ++ .../datasets/corpusqa/corpusqa_1m_gen.py | 102 ++++++ ais_bench/benchmark/datasets/__init__.py | 1 + ais_bench/benchmark/datasets/corpusqa.py | 295 ++++++++++++++++++ .../gen_inferencer_output_handler.py | 9 + docs/source_en/get_started/datasets.md | 1 + docs/source_zh_cn/get_started/datasets.md | 1 + tests/UT/datasets/test_corpusqa.py | 282 +++++++++++++++++ 9 files changed, 745 insertions(+) create mode 100644 ais_bench/benchmark/configs/datasets/corpusqa/README.md create mode 100644 ais_bench/benchmark/configs/datasets/corpusqa/README_en.md create mode 100644 ais_bench/benchmark/configs/datasets/corpusqa/corpusqa_1m_gen.py create mode 100644 ais_bench/benchmark/datasets/corpusqa.py create mode 100644 tests/UT/datasets/test_corpusqa.py diff --git a/ais_bench/benchmark/configs/datasets/corpusqa/README.md b/ais_bench/benchmark/configs/datasets/corpusqa/README.md new file mode 100644 index 00000000..84d6b361 --- /dev/null +++ b/ais_bench/benchmark/configs/datasets/corpusqa/README.md @@ -0,0 +1,27 @@ +# CorpusQA +中文 | [English](README_en.md) + +## 数据集简介 + +CorpusQA 是一个用于评估语言模型百万级（1M）长上下文问答能力的基准测试，包含金融（中/英）、教育、房地产四大领域。每条样本包含原始多轮对话 `prompt`、`question` 与参考答案 `answer`。评测遵循官方协议：模型根据原始 prompt 生成答案后，由 LLM 裁判（ORM，Output Reward Model）判断生成答案与参考答案是否等价，输出 `[[YES]]` 或 `[[NO]]`，准确率 = 正确样本数 / 总样本数。 + +> 🔗 数据集主页链接: [https://github.com/Tongyi-Zhiwen/CorpusQA](https://github.com/Tongyi-Zhiwen/CorpusQA) + + +## 数据集部署 + +- 可从数据集主页 🔗 [https://github.com/Tongyi-Zhiwen/CorpusQA](https://github.com/Tongyi-Zhiwen/CorpusQA) 获取 `1m_4domains.jsonl` 数据文件。 +- 建议将数据文件部署在 `{tool_root_path}/ais_bench/datasets/CorpusQA/1m_4domains.jsonl` 路径下，与配置文件 `corpusqa_1m_gen.py` 中的 `path` 字段保持一致。 + +- 在 `{tool_root_path}/ais_bench/datasets/` 目录下检查目录结构。如果目录结构如下所示，则数据集部署成功： + ``` + CorpusQA/ + └── 1m_4domains.jsonl + ``` + + +## 可用数据集任务 + +| 任务名称 | 简介 | 评估指标 | Few-Shot | Prompt 格式 | 配套文件导入方式 | 对应源码配置文件路径 | +| --- | --- | --- | --- | --- | --- | --- | +| corpusqa_1m_gen | CorpusQA 1M 长上下文问答数据集 | 准确率 (accuracy，基于 ORM LLM 裁判) | 0-shot | 对话格式（原始多轮 prompt） | `from ais_bench.benchmark.configs.datasets.corpusqa.corpusqa_1m_gen import corpusqa_1m_datasets as datasets` | corpusqa_1m_gen.py | diff --git a/ais_bench/benchmark/configs/datasets/corpusqa/README_en.md b/ais_bench/benchmark/configs/datasets/corpusqa/README_en.md new file mode 100644 index 00000000..a27004c3 --- /dev/null +++ b/ais_bench/benchmark/configs/datasets/corpusqa/README_en.md @@ -0,0 +1,27 @@ +# CorpusQA +[中文](README.md) | English + +## Dataset Introduction + +CorpusQA is a benchmark for evaluating million-token (1M) long-context question-answering capabilities of language models, covering four domains: finance (Chinese/English), education, and real estate. Each sample contains a raw multi-turn `prompt`, a `question`, and a reference `answer`. The evaluation follows the official protocol: after the model generates an answer from the raw prompt, an LLM judge (ORM, Output Reward Model) decides whether the generated answer is equivalent to the reference answer, outputting `[[YES]]` or `[[NO]]`. Accuracy = number of correct samples / total samples. + +> 🔗 Dataset Homepage Link: [https://github.com/Tongyi-Zhiwen/CorpusQA](https://github.com/Tongyi-Zhiwen/CorpusQA) + + +## Dataset Deployment + +- Obtain the `1m_4domains.jsonl` data file from the dataset homepage: 🔗 [https://github.com/Tongyi-Zhiwen/CorpusQA](https://github.com/Tongyi-Zhiwen/CorpusQA). +- It is recommended to deploy the data file at `{tool_root_path}/ais_bench/datasets/CorpusQA/1m_4domains.jsonl`, consistent with the `path` field in the configuration file `corpusqa_1m_gen.py`. + +- Check the directory structure under `{tool_root_path}/ais_bench/datasets/`. If the directory structure is as shown below, the dataset has been deployed successfully: + ``` + CorpusQA/ + └── 1m_4domains.jsonl + ``` + + +## Available Dataset Tasks + +| Task Name | Introduction | Evaluation Metric | Few-Shot | Prompt Format | Import Statement | Corresponding Source Code Configuration File Path | +| --- | --- | --- | --- | --- | --- | --- | +| corpusqa_1m_gen | CorpusQA 1M long-context Q&A dataset | Accuracy (based on ORM LLM judge) | 0-shot | Chat format (raw multi-turn prompt) | `from ais_bench.benchmark.configs.datasets.corpusqa.corpusqa_1m_gen import corpusqa_1m_datasets as datasets` | corpusqa_1m_gen.py | diff --git a/ais_bench/benchmark/configs/datasets/corpusqa/corpusqa_1m_gen.py b/ais_bench/benchmark/configs/datasets/corpusqa/corpusqa_1m_gen.py new file mode 100644 index 00000000..2fd08906 --- /dev/null +++ b/ais_bench/benchmark/configs/datasets/corpusqa/corpusqa_1m_gen.py @@ -0,0 +1,102 @@ +from ais_bench.benchmark.datasets.corpusqa import ( + CORPUSQA_JUDGE_SYSTEM_PROMPT, + CORPUSQA_JUDGE_USER_TEMPLATE, + CorpusQADataset, + CorpusQAEvaluator, + CorpusQAJGDataset, + CorpusQAPromptTemplate, +) +from ais_bench.benchmark.models import VLLMCustomAPIChat +from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer +from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate +from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever + +# --------------------------------------------------------------------------- +# CorpusQA 1M inference configuration +# --------------------------------------------------------------------------- +# The raw multi-turn ``prompt`` field is forwarded unchanged through the +# CorpusQAPromptTemplate, faithfully reproducing the official prompt. + +corpusqa_reader_cfg = dict( + input_columns=['prompt'], + output_column='answer', +) + +corpusqa_infer_cfg = dict( + prompt_template=dict(type=CorpusQAPromptTemplate), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +# --------------------------------------------------------------------------- +# Judge model configuration (LLM-judge / ORM, following the official script) +# --------------------------------------------------------------------------- +# The judge is served through a DashScope-compatible endpoint. Adjust the +# ``url`` / ``host_ip`` / ``host_port`` / ``model`` fields below to point at +# your judge service. + + +corpusqa_judge_infer_cfg = dict( + judge_reader_cfg=dict( + input_columns=['question', 'answer', 'model_answer'], + output_column='model_pred_uuid', + ), + judge_model=dict( + attr='service', + type=VLLMCustomAPIChat, + abbr='dashscope-orm', + path='', + model='', + stream=False, + request_rate=0, + use_timestamp=False, + retry=2, + api_key='', + host_ip='localhost', + host_port=8005, + max_out_len=2048, + batch_size=8, + trust_remote_code=False, + generation_kwargs=dict( + temperature=0.0, + ), + ), + judge_dataset_type=CorpusQAJGDataset, + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict(role='SYSTEM', prompt=CORPUSQA_JUDGE_SYSTEM_PROMPT), + ], + round=[ + dict(role='HUMAN', prompt=CORPUSQA_JUDGE_USER_TEMPLATE), + ], + ), + ), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer), +) + +# --------------------------------------------------------------------------- +# Evaluation configuration +# --------------------------------------------------------------------------- + +corpusqa_eval_cfg = dict( + evaluator=dict(type=CorpusQAEvaluator), +) + +# --------------------------------------------------------------------------- +# Dataset definitions +# --------------------------------------------------------------------------- + +corpusqa_1m_datasets = [ + dict( + abbr='corpusqa_1m', + type=CorpusQADataset, + path='ais_bench/datasets/CorpusQA/1m_4domains.jsonl', + reader_cfg=corpusqa_reader_cfg, + infer_cfg=corpusqa_infer_cfg, + judge_infer_cfg=corpusqa_judge_infer_cfg, + eval_cfg=corpusqa_eval_cfg, + ) +] diff --git a/ais_bench/benchmark/datasets/__init__.py b/ais_bench/benchmark/datasets/__init__.py index 4098e8cc..d112206a 100644 --- a/ais_bench/benchmark/datasets/__init__.py +++ b/ais_bench/benchmark/datasets/__init__.py @@ -68,4 +68,5 @@ from ais_bench.benchmark.datasets.realworldqa import * # noqa: F401, F403 from ais_bench.benchmark.datasets.oneig import * # noqa: F401, F403 from ais_bench.benchmark.datasets.geometry3k import * # noqa: F401, F403 +from ais_bench.benchmark.datasets.corpusqa import * # noqa: F401, F403 diff --git a/ais_bench/benchmark/datasets/corpusqa.py b/ais_bench/benchmark/datasets/corpusqa.py new file mode 100644 index 00000000..f4a8d293 --- /dev/null +++ b/ais_bench/benchmark/datasets/corpusqa.py @@ -0,0 +1,295 @@ +"""CorpusQA dataset integration for aisbench. + +CorpusQA (https://github.com/Tongyi-Zhiwen/CorpusQA) is a long-context +question-answering benchmark that stresses 1M-token context windows. Each +sample in the JSONL file contains: + +- ``id``: unique sample id +- ``prompt``: raw multi-turn chat messages (list of ``{role, content}``) +- ``question``: the question text +- ``answer``: the reference answer + +Reproduction follows the official CorpusQA evaluation protocol: the model +generates an answer from the raw prompt, then an LLM judge (ORM - Output +Reward Model) decides whether the generated answer is equivalent to the +reference answer, outputting ``[[YES]]`` or ``[[NO]]``. +""" + +import json +import os +import re +from typing import Any, Dict, List, Union + +from datasets import Dataset + +from ais_bench.benchmark.datasets.base import BaseDataset +from ais_bench.benchmark.datasets.utils.datasets import get_data_path +from ais_bench.benchmark.datasets.utils.llm_judge import LLMJudgeDataset +from ais_bench.benchmark.openicl.icl_evaluator.icl_base_evaluator import BaseEvaluator +from ais_bench.benchmark.openicl.icl_prompt_template.icl_prompt_template_base import BasePromptTemplate +from ais_bench.benchmark.registry import ICL_EVALUATORS, ICL_PROMPT_TEMPLATES, LOAD_DATASET +from ais_bench.benchmark.utils.logging import AISLogger + +logger = AISLogger() + + +# --------------------------------------------------------------------------- +# Official CorpusQA ORM (Output Reward Model) judge prompt +# Byte-identical to the official eval.py: +# - system message: GENERAL_ORM_PROMPT +# - user message: ORM_USER_TEMPLATE.format(problem=question, answer_1=..., answer_2=...) +# --------------------------------------------------------------------------- +CORPUSQA_JUDGE_SYSTEM_PROMPT = """You are an expert in verifying if two answers are the same. +Your input is a problem and two answers, Answer 1 and Answer 2. You need to check if they are equivalent. +Your task is to determine if two answers are equivalent, without attempting to solve the original problem. +Compare the answers to verify they represent identical values or meaning, even when written in different forms or notations. + +Your output must follow the following format: +1) Provide an explanation for why the answers are equivalent or not. +2) Then provide your final answer in the form of: [[YES]] or [[NO]] +""" + +# Leading/trailing newlines preserved exactly as in the official template. +CORPUSQA_JUDGE_USER_TEMPLATE = """ +Problem: {question} +Answer 1: {model_answer} +Answer 2: {answer} +""" + + +@ICL_PROMPT_TEMPLATES.register_module("corpusqa_prompt") +class CorpusQAPromptTemplate(BasePromptTemplate): + """Pass-through template for CorpusQA raw multi-turn prompts. + + CorpusQA stores the conversation as a list of messages + (``[{role, content, ...}]``). This template converts that list into a + :class:`PromptList` so that the chat API model forwards the messages + unchanged, preserving the original prompt exactly as released. + """ + + def __init__( + self, + template: Union[Dict, str] = "", + ice_token: str = None, + sep_token: str = None, + ) -> None: + super().__init__(template=template or "", ice_token=ice_token, sep_token=sep_token) + + def generate_item( + self, + entry: Dict, + output_field=None, + output_field_replace_token: str = "", + ice_field_replace_token: str = "", + ): + messages = entry.get("prompt", []) + if isinstance(messages, str): + # Some subsets may provide a plain text prompt. + return messages + if isinstance(messages, dict): + messages = [messages] + + # Reuse the framework's standard section emission (``_encode_template``), + # exactly like the built-in PromptTemplate used by other gen datasets + # (e.g. gsm8k_gen_4_shot_cot_chat_prompt). System messages live in the + # ``begin`` section; user/assistant turns in the ``round`` section. + template = {} + system_items = [] + round_items = [] + for msg in messages: + if not isinstance(msg, dict): + msg = {"role": "user", "content": str(msg)} + # Map the released OpenAI-style roles onto the internal roles the + # API chat model understands (see ROLE_MAP in vllm_custom_api_chat): + # system -> SYSTEM (begin section) + # user / assistant -> HUMAN / BOT (round section) + raw_role = msg.get("role", "user") + if raw_role == "system": + role = "SYSTEM" + elif raw_role == "assistant": + role = "BOT" + else: + role = "HUMAN" + item = {"role": role, "prompt": msg.get("content", "")} + for key, value in msg.items(): + if key not in ("role", "content"): + item[key] = value + if role == "SYSTEM": + system_items.append(item) + else: + round_items.append(item) + if system_items: + template["begin"] = system_items + template["round"] = round_items + return self._encode_template(template, ice=False) + + +@LOAD_DATASET.register_module() +class CorpusQADataset(BaseDataset): + """CorpusQA dataset. + + Streams the JSONL file line by line so that the 1M-token / large sample + file can be loaded without exhausting memory, and keeps the raw + multi-turn ``prompt`` field untouched for faithful reproduction. + """ + + @staticmethod + def load(path: str, **kwargs) -> Dataset: + path = get_data_path(path) + if not os.path.exists(path): + raise FileNotFoundError(f"CorpusQA dataset file not found: {path}") + + dataset = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as exc: + logger.warning(f"Failed to parse line, skipped: {exc}") + continue + if "prompt" not in item or "answer" not in item: + logger.warning("Line misses 'prompt' or 'answer' field, skipped.") + continue + # Answers may be str, int/float or the empty list ``[]`` (see + # the official README). Arrow cannot build a column mixing + # these types, so normalise non-str answers to their str() + # rendering -- this keeps the judge prompt byte-identical to + # the official ``str.format`` output (e.g. ``Answer 2: []``). + answer = item["answer"] + if not isinstance(answer, str): + answer = str(answer) + dataset.append( + { + "id": item.get("id", len(dataset)), + "prompt": item["prompt"], + "question": item.get("question", ""), + "answer": answer, + } + ) + logger.info(f"Loaded {len(dataset)} samples from CorpusQA dataset: {path}") + return Dataset.from_list(dataset) + + +class CorpusQAJGDataset(LLMJudgeDataset): + """CorpusQA judge dataset. + + Follows the HLE / AA-LCR pattern: subclasses :class:`LLMJudgeDataset` + and merges the model predictions into the original dataset items so the + judge prompt can reference ``question``, ``answer`` and ``model_answer``. + """ + + @staticmethod + def _extract_answer(response: str) -> str: + """Mirror the official ``extract_answer`` helper. + + The official script runs this extraction on the model's *final + response only* (the API ``content`` field). aisbench's saved + ``prediction`` concatenates ``reasoning_content + "\\n\\n" + + content`` (see ``Output.get_prediction``), so a first-match + ``re.search`` can hit a draft ``The answer is: ...`` line inside + the reasoning, and the judge would compare a reasoning fragment + against the gold answer. Taking the last occurrence preserves the + official semantics because the model's final answer line is the + last ``The answer is:`` in the concatenated text. + """ + if not isinstance(response, str): + response = str(response) + matches = re.findall(r"The answer is: (.*)", response) + if matches: + return matches[-1].strip() + return response.strip() + + def _modify_dataset_item(self, dataset_item, pred_item): + super()._modify_dataset_item(dataset_item, pred_item) + # Prefer the reasoning-free final content when the prediction file + # provides it (see GenInferencerOutputHandler); fall back to the + # concatenated prediction for result files produced before that + # field existed. + raw_answer = pred_item.get("content") + if raw_answer is None: + raw_answer = dataset_item.get("model_answer") + if raw_answer is not None: + dataset_item["model_answer"] = self._extract_answer(str(raw_answer)) + return dataset_item + + def _get_dataset_class(self): + return CorpusQADataset + + +# --------------------------------------------------------------------------- +# Evaluator +# --------------------------------------------------------------------------- +@ICL_EVALUATORS.register_module() +class CorpusQAEvaluator(BaseEvaluator): + """CorpusQA ORM evaluator. + + Parses the judge model outputs (expected to contain ``[[YES]]`` or + ``[[NO]]``) and computes accuracy, mirroring the official CorpusQA + evaluation script. + """ + + def __init__(self): + super().__init__() + + def score(self, predictions: List[str], references: List[Any]) -> Dict[str, Any]: + if len(predictions) != len(references): + return { + "error": ( + "predictions and references have different length: " + f"len(predictions)={len(predictions)}, " + f"len(references)={len(references)}" + ) + } + + details = [] + correct = 0 + total = 0 + for index, (judge_output, ref) in enumerate(zip(predictions, references)): + is_correct = self._is_correct(judge_output) + if is_correct is None: + logger.warning( + f"Judge output {index} does not contain [[YES]]/[[NO]], " + "treated as incorrect." + ) + is_correct = False + if is_correct: + correct += 1 + total += 1 + details.append( + { + "id": index, + "judge_output": judge_output, + "answer": ref, + "correct": is_correct, + } + ) + + return { + "accuracy": 100.0 * correct / total if total else 0.0, + "num_correct": correct, + "num_total": total, + "details": details, + } + + @staticmethod + def _is_correct(judge_output: str): + """Mirror the official parsing rule on the judge's *final verdict*. + + The official script parses ``[[YES]]``/``[[NO]]`` from the judge's + final content only. aisbench's saved ``prediction`` concatenates + ``reasoning_content + "\\n\\n" + content``, and a thinking judge + often mentions both markers while reasoning before settling on + one, which breaks the official any-marker substring rule (e.g. + reasoning about ``[[NO]]`` then concluding ``[[YES]]`` is scored + as incorrect). Taking the last marker matches the official + verdict because the final content comes after the reasoning in the + concatenated text. + """ + if not judge_output: + return None + markers = re.findall(r"\[\[(YES|NO)\]\]", judge_output) + if markers: + return markers[-1] == "YES" + return None diff --git a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py index 58f58d23..50b1f009 100644 --- a/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py +++ b/ais_bench/benchmark/openicl/icl_inferencer/output_handler/gen_inferencer_output_handler.py @@ -77,6 +77,15 @@ def get_prediction_result( if output.origin_logprobs: result_data["origin_logprobs"] = output.origin_logprobs + # When the model emits reasoning, ``prediction`` concatenates + # ``reasoning_content + "\n\n" + content`` (see Output.get_prediction). + # Keep the reasoning-free final content separately so downstream + # judge datasets (e.g. CorpusQA) can evaluate exactly the answer + # the official evaluation scripts would see (the API ``content`` + # field). + if isinstance(output, Output) and output.reasoning_content: + result_data["content"] = output.content + if gold: result_data["gold"] = gold return result_data \ No newline at end of file diff --git a/docs/source_en/get_started/datasets.md b/docs/source_en/get_started/datasets.md index bd91f423..d9c8383a 100644 --- a/docs/source_en/get_started/datasets.md +++ b/docs/source_en/get_started/datasets.md @@ -60,6 +60,7 @@ Open-source datasets refer to widely used, publicly accessible datasets in the c | dapo-math-17k | Mathematical Reasoning (RL Evaluation) | [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/dapo_math/README_en.md) | | ifbench | Instruction Following Evaluation | [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/ifbench/README_en.md) | | aa_lcr | Long Context Retrieval & Reasoning | [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aa_lcr/README_en.md) | +| corpusqa | Long Context Q&A (1M tokens) | [Detailed Introduction](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/corpusqa/README_en.md) | ### Multimodal Datasets diff --git a/docs/source_zh_cn/get_started/datasets.md b/docs/source_zh_cn/get_started/datasets.md index ebb39d9a..66c687d7 100644 --- a/docs/source_zh_cn/get_started/datasets.md +++ b/docs/source_zh_cn/get_started/datasets.md @@ -60,6 +60,7 @@ AISBench Benchmark当前支持的数据集类型如下： | dapo-math-17k | 数学推理（RL评估） | [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/dapo_math/README.md) | | ifbench | 指令遵循评估 | [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/ifbench/README.md) | | aa_lcr | 长上下文检索与推理 | [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/aa_lcr/README.md) | +| corpusqa | 长上下文问答（1M token） | [详细介绍](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/configs/datasets/corpusqa/README.md) | ### 多模态类数据集 diff --git a/tests/UT/datasets/test_corpusqa.py b/tests/UT/datasets/test_corpusqa.py new file mode 100644 index 00000000..4bca145b --- /dev/null +++ b/tests/UT/datasets/test_corpusqa.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- +"""Unit tests for the CorpusQA dataset adaptation. + +Covers: +- ``CorpusQADataset.load``: streaming JSONL loading plus the answer-field + type normalisation that fixes Arrow build errors (str / int / float / + empty-list answers must all become str). +- ``CorpusQAPromptTemplate.generate_item``: raw multi-turn prompt -> + framework PromptList with ``begin``/``round`` section markers and + OpenAI-style role mapping (system -> SYSTEM, user -> HUMAN, + assistant -> BOT). +- ``CorpusQAJGDataset._extract_answer`` / ``_modify_dataset_item``: the + official ``The answer is: ...`` extraction used before the ORM judge. +- ``CorpusQAEvaluator``: ``[[YES]]``/``[[NO]]`` parsing and accuracy + computation, mirroring the official ORM evaluation. +""" + +import sys +import os +import json +from unittest.mock import patch, mock_open + +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) + +from ais_bench.benchmark.datasets.corpusqa import ( + CORPUSQA_JUDGE_SYSTEM_PROMPT, + CORPUSQA_JUDGE_USER_TEMPLATE, + CorpusQADataset, + CorpusQAEvaluator, + CorpusQAJGDataset, + CorpusQAPromptTemplate, +) + + +def _make_item(sample_id=0, answer="ans"): + return { + "id": sample_id, + "prompt": [{"role": "user", "content": f"question {sample_id}"}], + "question": f"question {sample_id}", + "answer": answer, + } + + +class TestCorpusQADataset: + def _load(self, lines, path="/fake/corpusqa.jsonl"): + read_data = "\n".join(lines) + "\n" if lines else "" + m = mock_open(read_data=read_data) + with patch("ais_bench.benchmark.datasets.corpusqa.get_data_path", return_value=path): + with patch("os.path.exists", return_value=True): + with patch("builtins.open", m): + return CorpusQADataset.load(path) + + def test_load_normalises_mixed_answer_types(self): + """混合类型 answer（str/int/float/空列表）必须被统一为 str，否则 Arrow 建表报错""" + lines = [ + json.dumps(_make_item(0, "string answer")), + json.dumps(_make_item(1, 42)), + json.dumps(_make_item(2, 3.14)), + json.dumps(_make_item(3, [])), + ] + ds = self._load(lines) + assert len(ds) == 4 + answers = list(ds["answer"]) + assert answers == ["string answer", "42", "3.14", "[]"] + # 所有 answer 都是 str，与官方 ``Answer 2: {}`` 的 str.format 输出保持一致 + assert all(isinstance(a, str) for a in answers) + + def test_load_keeps_raw_prompt_and_fields(self): + """原始多轮 prompt 原样保留，id/question/answer 字段完整""" + item = _make_item(7, "ans7") + ds = self._load([json.dumps(item)]) + assert ds[0]["id"] == 7 + assert ds[0]["question"] == "question 7" + assert ds[0]["prompt"] == [{"role": "user", "content": "question 7"}] + assert ds[0]["answer"] == "ans7" + + def test_load_skips_invalid_json_line(self): + """无法解析的行应被跳过""" + lines = [ + json.dumps(_make_item(0, "a")), + "not a json line", + json.dumps(_make_item(1, "b")), + ] + ds = self._load(lines) + assert len(ds) == 2 + + def test_load_skips_line_missing_prompt_or_answer(self): + """缺少 prompt 或 answer 字段的行应被跳过""" + lines = [ + json.dumps({"id": 0, "question": "no prompt"}), + json.dumps({"id": 1, "prompt": [{"role": "user", "content": "q"}]}), + json.dumps(_make_item(2, "ok")), + ] + ds = self._load(lines) + assert len(ds) == 1 + assert ds[0]["id"] == 2 + + def test_load_default_id(self): + """缺少 id 时使用当前行号""" + lines = [ + json.dumps({"prompt": [{"role": "user", "content": "q"}], "answer": "a"}), + ] + ds = self._load(lines) + assert ds[0]["id"] == 0 + + def test_load_file_not_found(self): + """文件不存在时抛出 FileNotFoundError""" + with patch("ais_bench.benchmark.datasets.corpusqa.get_data_path", return_value="/nope.jsonl"): + with patch("os.path.exists", return_value=False): + with pytest.raises(FileNotFoundError): + CorpusQADataset.load("/nope.jsonl") + + +class TestCorpusQAPromptTemplate: + def _prompt(self, messages): + return CorpusQAPromptTemplate(template="").generate_item({"prompt": messages}) + + def test_str_prompt_passthrough(self): + """字符串 prompt 原样返回""" + out = self._prompt("plain text prompt") + assert out == "plain text prompt" + + def test_dict_prompt_wrapped_in_list(self): + """单个 dict prompt 被包装成列表并生成对话模板""" + out = self._prompt({"role": "user", "content": "hi"}) + assert isinstance(out, list) + # 只有 user 消息：无 system 段，直接 round 段 + assert out == [ + {"section": "round", "pos": "begin"}, + {"role": "HUMAN", "prompt": "hi"}, + {"section": "round", "pos": "end"}, + ] + + def test_multiturn_role_mapping_and_sections(self): + """system/user/assistant 映射为 SYSTEM/HUMAN/BOT，system 进 begin 段，其余进 round 段""" + messages = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + ] + out = self._prompt(messages) + assert out == [ + {"section": "begin", "pos": "begin"}, + {"role": "SYSTEM", "prompt": "sys"}, + {"section": "begin", "pos": "end"}, + {"section": "round", "pos": "begin"}, + {"role": "HUMAN", "prompt": "u1"}, + {"role": "BOT", "prompt": "a1"}, + {"role": "HUMAN", "prompt": "u2"}, + {"section": "round", "pos": "end"}, + ] + + def test_message_without_role_defaults_to_user(self): + """缺少 role 的消息默认按 user 处理""" + out = self._prompt([{"content": "no role"}]) + assert {"role": "HUMAN", "prompt": "no role"} in out + + def test_non_dict_message_converted(self): + """列表中的非 dict 消息被转为 user 消息""" + out = self._prompt(["just a string"]) + assert {"role": "HUMAN", "prompt": "just a string"} in out + + def test_extra_keys_preserved(self): + """消息中的额外字段被保留""" + out = self._prompt([{"role": "user", "content": "q", "extra": 1}]) + assert {"role": "HUMAN", "prompt": "q", "extra": 1} in out + + +class TestCorpusQAJGDataset: + def test_extract_answer_found(self): + """提取 ``The answer is: X`` 中的 X""" + assert CorpusQAJGDataset._extract_answer("some text\nThe answer is: 42") == "42" + + def test_extract_answer_last_match_wins(self): + """多次出现时取最后一次（推理内容可能包含草稿答案）""" + response = "The answer is: draft\nreasoning\n\nThe answer is: 42" + assert CorpusQAJGDataset._extract_answer(response) == "42" + + def test_extract_answer_no_match(self): + """无匹配时返回去除首尾空白的完整内容""" + assert CorpusQAJGDataset._extract_answer(" no marker here ") == "no marker here" + + def test_extract_answer_non_str(self): + """非字符串输入转为字符串处理""" + assert CorpusQAJGDataset._extract_answer(123) == "123" + + def test_modify_dataset_item_prefers_content(self): + """存在 content 字段时，基于它做答案提取（推理+最终答案场景）""" + ds = CorpusQAJGDataset.__new__(CorpusQAJGDataset) + dataset_item = {} + pred_item = { + "prediction": "reasoning\nThe answer is: gold", + "content": "reasoning\nThe answer is: gold", + } + ds._modify_dataset_item(dataset_item, pred_item) + assert dataset_item["model_answer"] == "gold" + + def test_modify_dataset_item_fallback_to_model_answer(self): + """无 content 字段时回退到已有 model_answer 并重新提取""" + ds = CorpusQAJGDataset.__new__(CorpusQAJGDataset) + dataset_item = {"model_answer": "xxx The answer is: fallback"} + pred_item = {"prediction": "xxx The answer is: fallback"} + ds._modify_dataset_item(dataset_item, pred_item) + assert dataset_item["model_answer"] == "fallback" + + +class TestCorpusQAEvaluator: + def test_score_all_correct(self): + """全部 [[YES]] -> accuracy 100""" + eva = CorpusQAEvaluator() + out = eva.score(["[[YES]]", "[[YES]]"], ["a", "b"]) + assert out["accuracy"] == 100.0 + assert out["num_correct"] == 2 + assert out["num_total"] == 2 + + def test_score_mixed(self): + """混合 YES/NO -> 正确计算 accuracy""" + eva = CorpusQAEvaluator() + out = eva.score(["[[YES]]", "[[NO]]", "[[YES]]"], ["a", "b", "c"]) + assert out["accuracy"] == pytest.approx(100 * 2 / 3) + assert out["details"][0]["correct"] is True + assert out["details"][1]["correct"] is False + assert out["details"][2]["correct"] is True + + def test_score_no_marker_treated_incorrect(self): + """judge 输出不含标记时视为错误""" + eva = CorpusQAEvaluator() + out = eva.score(["no marker here"], ["a"]) + assert out["num_correct"] == 0 + assert out["details"][0]["correct"] is False + + def test_score_length_mismatch(self): + """预测与参考答案数量不一致时返回 error""" + eva = CorpusQAEvaluator() + out = eva.score(["[[YES]]"], ["a", "b"]) + assert "error" in out + assert "different length" in out["error"] + + def test_score_empty(self): + """空输入 -> accuracy 0""" + eva = CorpusQAEvaluator() + out = eva.score([], []) + assert out["accuracy"] == 0.0 + + def test_is_correct_yes(self): + assert CorpusQAEvaluator._is_correct("explanation\n[[YES]]") is True + + def test_is_correct_no(self): + assert CorpusQAEvaluator._is_correct("explanation\n[[NO]]") is False + + def test_is_correct_last_marker_wins(self): + """推理中同时出现两种标记时，以最后一个为准（官方取最终结论）""" + assert CorpusQAEvaluator._is_correct("[[NO]] ... conclusion [[YES]]") is True + assert CorpusQAEvaluator._is_correct("[[YES]] ... conclusion [[NO]]") is False + + def test_is_correct_empty_or_no_marker(self): + assert CorpusQAEvaluator._is_correct("") is None + assert CorpusQAEvaluator._is_correct(None) is None + assert CorpusQAEvaluator._is_correct("no markers") is None + + +class TestCorpusQAJudgePrompts: + def test_system_prompt_non_empty(self): + """官方 ORM system prompt 非空且要求输出 [[YES]]/[[NO]]""" + assert "[[YES]]" in CORPUSQA_JUDGE_SYSTEM_PROMPT + assert "[[NO]]" in CORPUSQA_JUDGE_SYSTEM_PROMPT + + def test_user_template_format(self): + """官方 user template 包含 question/model_answer/answer 占位符""" + rendered = CORPUSQA_JUDGE_USER_TEMPLATE.format( + question="q", model_answer="ma", answer="a" + ) + assert "Problem: q" in rendered + assert "Answer 1: ma" in rendered + assert "Answer 2: a" in rendered + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) From caeab013f27868a0c9aea65ff56875c4e2a26aca Mon Sep 17 00:00:00 2001 From: jschen069 <3563624058@qq.com> Date: Thu, 27 Aug 2026 12:00:54 +0800 Subject: [PATCH 13/25] generate/analysis prefix cache (#489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 初版代码 * fix prefix cache dataset and runtime edge cases * 修复bug * preserve requests artifact field order * complete prefix cache detailed requirements * 修复运行bug * 添加readme解释 * 文档说明 * 打印日志 * 添加注释 * 修改前缀方法 * 生成前缀缓存数据集：仅保留 inspect/prepare/validate 离线功能 * 添加需求设计 * 添加修改 * 同步写日志 * 添加修改cli * 删除文档 * 删除文档 * 删除文档 * 添加文档说明 * 门禁新增 prefix_cache 插件测试并补充用例提升覆盖率 - 门禁在 UT 之后执行 plugins/prefix_cache/tests，按 80/60 阈值校验覆盖率 - 新增 test_scenario/test_cli_flow/test_artifacts 用例，总覆盖率 77.92% -> 92.66% - 修复 test_cli.py 中 execution_timestamp 传参位置错误 Co-Authored-By: Claude * 冒烟测试增加 prefix_cache 变更触发条件 Co-Authored-By: Claude --------- Co-authored-by: Claude --- .github/workflows/run-ut-on-pr-py.yml | 52 ++ .../workflows/run_smoke_test_pr_and_daily.yml | 1 + .../advanced_tutorials/prefix_cache.md | 343 ++++++++ docs/source_en/index.rst | 3 +- .../advanced_tutorials/prefix_cache.md | 343 ++++++++ docs/source_zh_cn/index.rst | 3 +- plugins/prefix_cache/README.md | 480 ++++++++++++ .../ais_bench_prefix_cache/__init__.py | 3 + .../ais_bench_prefix_cache/artifacts.py | 128 +++ .../ais_bench_prefix_cache/cli.py | 262 +++++++ .../ais_bench_prefix_cache/errors.py | 14 + .../ais_bench_prefix_cache/generation.py | 740 ++++++++++++++++++ .../ais_bench_prefix_cache/pipeline.py | 502 ++++++++++++ .../ais_bench_prefix_cache/scenario.py | 402 ++++++++++ .../config_examples/scenario.example.json | 40 + .../config_examples/scenario.example.md | 731 +++++++++++++++++ plugins/prefix_cache/setup.py | 19 + plugins/prefix_cache/tests/__init__.py | 1 + plugins/prefix_cache/tests/test_artifacts.py | 117 +++ plugins/prefix_cache/tests/test_cli.py | 106 +++ plugins/prefix_cache/tests/test_cli_flow.py | 290 +++++++ plugins/prefix_cache/tests/test_core.py | 321 ++++++++ plugins/prefix_cache/tests/test_pipeline.py | 175 +++++ plugins/prefix_cache/tests/test_scenario.py | 381 +++++++++ 24 files changed, 5455 insertions(+), 2 deletions(-) create mode 100644 docs/source_en/advanced_tutorials/prefix_cache.md create mode 100644 docs/source_zh_cn/advanced_tutorials/prefix_cache.md create mode 100644 plugins/prefix_cache/README.md create mode 100644 plugins/prefix_cache/ais_bench_prefix_cache/__init__.py create mode 100644 plugins/prefix_cache/ais_bench_prefix_cache/artifacts.py create mode 100644 plugins/prefix_cache/ais_bench_prefix_cache/cli.py create mode 100644 plugins/prefix_cache/ais_bench_prefix_cache/errors.py create mode 100644 plugins/prefix_cache/ais_bench_prefix_cache/generation.py create mode 100644 plugins/prefix_cache/ais_bench_prefix_cache/pipeline.py create mode 100644 plugins/prefix_cache/ais_bench_prefix_cache/scenario.py create mode 100644 plugins/prefix_cache/config_examples/scenario.example.json create mode 100644 plugins/prefix_cache/config_examples/scenario.example.md create mode 100644 plugins/prefix_cache/setup.py create mode 100644 plugins/prefix_cache/tests/__init__.py create mode 100644 plugins/prefix_cache/tests/test_artifacts.py create mode 100644 plugins/prefix_cache/tests/test_cli.py create mode 100644 plugins/prefix_cache/tests/test_cli_flow.py create mode 100644 plugins/prefix_cache/tests/test_core.py create mode 100644 plugins/prefix_cache/tests/test_pipeline.py create mode 100644 plugins/prefix_cache/tests/test_scenario.py diff --git a/.github/workflows/run-ut-on-pr-py.yml b/.github/workflows/run-ut-on-pr-py.yml index a6011936..22082feb 100644 --- a/.github/workflows/run-ut-on-pr-py.yml +++ b/.github/workflows/run-ut-on-pr-py.yml @@ -5,6 +5,7 @@ on: paths: - 'ais_bench/**' - 'tests/**' + - 'plugins/**' workflow_dispatch: schedule: - cron: '0 1 * * *' @@ -29,11 +30,18 @@ jobs: pip3 install -r requirements/extra.txt -i https://repo.huaweicloud.com/repository/pypi/simple --trusted-host repo.huaweicloud.com pip3 install -r requirements/datasets/bfcl_dependencies.txt --no-deps -i https://repo.huaweicloud.com/repository/pypi/simple --trusted-host repo.huaweicloud.com pip3 install -r requirements/datasets/geometry3k.txt -i https://repo.huaweicloud.com/repository/pypi/simple --trusted-host repo.huaweicloud.com + # 安装 prefix_cache 插件（--no-deps：ais-bench-benchmark 已由上一步安装， + # 插件测试通过注入 FakeTokenizer 不依赖 transformers） + pip3 install -e plugins/prefix_cache --no-deps -i https://repo.huaweicloud.com/repository/pypi/simple --trusted-host repo.huaweicloud.com --use-pep517 - name: Run Test run: | pip3 install pytest pytest-cov pytest-xdist -i https://repo.huaweicloud.com/repository/pypi/simple --trusted-host repo.huaweicloud.com python3 tests/run_tests.py tests/UT -p 16 + - name: Run Prefix Cache Test + run: | + python3 tests/run_tests.py plugins/prefix_cache/tests --source-dirs plugins/prefix_cache/ais_bench_prefix_cache --output test_reports/prefix_cache + - name: Check Result run: | cur_dir=$(pwd) @@ -74,6 +82,43 @@ jobs: else echo "[INFO] Code branch coverage is higher than 60.0%, current coverage is ${branch_coverage}%. " fi + pc_summary_path=${cur_dir}/test_reports/prefix_cache/test_summary.txt + echo "pc_summary_path: ${pc_summary_path}" + if [ -f ${pc_summary_path} ]; then + echo "[INFO] prefix cache test_summary.txt exists. " + else + echo "[ERROR] prefix cache test_summary.txt not exists. " + exit 1 + fi + grep -q "Failed: 0" ${pc_summary_path} + if [ $? -ne 0 ]; then + pc_failed_count=$(cat ${pc_summary_path} | grep "Failed:" | awk '{print $2}') + echo "[ERROR] Not all prefix cache test case passed, failed count is ${pc_failed_count}. " + exit 1 + else + echo "[INFO] All prefix cache test case passed. " + fi + pc_total_coverage=$(cat ${pc_summary_path} | grep "Total coverage" | cut -d' ' -f3 | tr -d '%') + if python3 -c "exit(0 if float('$pc_total_coverage') < 80 else 1)"; then + echo "[ERROR] Prefix cache code total coverage is lower than 80.0%, current coverage is ${pc_total_coverage}%. " + exit 1 + else + echo "[INFO] Prefix cache code total coverage is higher than 80.0%, current coverage is ${pc_total_coverage}%. " + fi + pc_line_coverage=$(cat ${pc_summary_path} | grep "Line coverage" | cut -d' ' -f3 | tr -d '%') + if python3 -c "exit(0 if float('$pc_line_coverage') < 80 else 1)"; then + echo "[ERROR] Prefix cache code line coverage is lower than 80.0%, current coverage is ${pc_line_coverage}%. " + exit 1 + else + echo "[INFO] Prefix cache code line coverage is higher than 80.0%, current coverage is ${pc_line_coverage}%. " + fi + pc_branch_coverage=$(cat ${pc_summary_path} | grep "Branch coverage" | cut -d' ' -f3 | tr -d '%') + if python3 -c "exit(0 if float('$pc_branch_coverage') < 60 else 1)"; then + echo "[ERROR] Prefix cache code branch coverage is lower than 60.0%, current coverage is ${pc_branch_coverage}%. " + exit 1 + else + echo "[INFO] Prefix cache code branch coverage is higher than 60.0%, current coverage is ${pc_branch_coverage}%. " + fi - name: Uninstall benchmark if: always() run: | @@ -84,3 +129,10 @@ jobs: else echo "[INFO] Uninstall benchmark success. " fi + pip3 uninstall ais-bench-prefix-cache -y + if [ $? -ne 0 ]; then + echo "[ERROR] Uninstall prefix cache plugin failed. " + exit 1 + else + echo "[INFO] Uninstall prefix cache plugin success. " + fi diff --git a/.github/workflows/run_smoke_test_pr_and_daily.yml b/.github/workflows/run_smoke_test_pr_and_daily.yml index 34dd4e03..9989110e 100644 --- a/.github/workflows/run_smoke_test_pr_and_daily.yml +++ b/.github/workflows/run_smoke_test_pr_and_daily.yml @@ -8,6 +8,7 @@ on: paths: - 'ais_bench/benchmark/**' - 'smoke_tests/**' + - 'plugins/prefix_cache/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/docs/source_en/advanced_tutorials/prefix_cache.md b/docs/source_en/advanced_tutorials/prefix_cache.md new file mode 100644 index 00000000..462360d8 --- /dev/null +++ b/docs/source_en/advanced_tutorials/prefix_cache.md @@ -0,0 +1,343 @@ +# Prefix Cache Dataset Generation and Theoretical Hit-Rate Analysis + +## Overview + +The AISBench Prefix Cache plugin generates datasets with controlled shared prefixes and calculates their theoretical Prefix Cache hit rate before requests are sent. It helps evaluate how input lengths, shared-prefix ratios, Prefix Groups, request ordering, and multiple DP ranks behind one endpoint affect cache hits. + +The current plugin provides three offline commands: + +- `inspect`: preview the scenario, reachable range, and length distributions; +- `prepare`: generate formal requests, a Manifest, and theoretical analysis; +- `validate`: detect modified, truncated, reordered, or inconsistent artifacts. + +This branch does not connect to vLLM, send formal requests, or collect online Prometheus metrics. It produces auditable data that can be used by a later AISBench benchmark workflow. + +--- + +## Prerequisites + +1. **Python 3.10 or later**. +2. **An AISBench checkout whose dependencies can be imported**. +3. **The same tokenizer as the target vLLM server**. A mismatch changes token counts, Block boundaries, and the theoretical hit rate. +4. **A GSM8K JSONL corpus**. Every non-empty line must be a JSON object containing the text field selected by `corpus.field`, which defaults to `question`. +5. **The correct Prefix Cache Block size**. `tokenizer.block_size` must match the target server. + +--- + +## Installation + +The following commands assume that the current directory is the AISBench repository root: + +```shell +python -m venv .venv +source .venv/bin/activate +python -m pip install -e . +python -m pip install -e ./plugins/prefix_cache +ais-bench-prefix-cache --help +``` + +The editable (`-e`) installs normally make source changes available without reinstalling the packages. + +--- + +## Quick Start + +Copy the example Scenario: + +```shell +cp ./plugins/prefix_cache/config_examples/scenario.example.json ./scenario.json +``` + +At minimum, review `tokenizer.path`, `tokenizer.block_size`, and `corpus.path`. When modeling cold multi-DP routing or producing a warmup plan, also set `service.dp_size` to the number of DP ranks on the target server. + +A minimal example is shown below: + +```json +{ + "schema_version": "1.0", + "run": { + "run_id": "gsm8k-prefix-cache-60", + "random_seed": 42, + "output_dir": "./outputs/gsm8k-prefix-cache-60" + }, + "tokenizer": { + "path": "/path/to/tokenizer", + "block_size": 16 + }, + "corpus": { + "path": "./GSM8K.jsonl", + "field": "question", + "selection": {"mode": "random"} + }, + "requests": { + "count": 100, + "input_length": {"mode": "fixed", "value": 1024}, + "output_length": {"mode": "fixed", "value": 32} + }, + "prefix_cache": { + "mode": "warmup", + "target_hit_rate": 0.6, + "seed_blocks": 1, + "groups": {"count": 1, "assignment": {"mode": "uniform"}}, + "order": {"strategy": "interleave"} + }, + "service": {"dp_size": 2} +} +``` + +Run the commands in order: + +```shell +ais-bench-prefix-cache inspect --scenario ./scenario.json +ais-bench-prefix-cache prepare --scenario ./scenario.json +ais-bench-prefix-cache validate --manifest \ + ./outputs/gsm8k-prefix-cache-60_/result/gsm8k-prefix-cache-60_.manifest.json +``` + +--- + +## How It Works + +```mermaid +flowchart LR + S[Scenario] --> I[inspect preview] + I --> P[prepare prompts] + P --> G[Shared prefix] + P --> U[Globally unique seed] + P --> N[Natural GSM8K suffix] + G --> T[Order-aware watermark simulation] + U --> T + N --> T + T --> A[full / requests / Manifest / analysis] + A --> V[validate artifacts] +``` + +Every formal request consists of three regions: + +```text +shared prefix + globally unique seed + natural GSM8K suffix +``` + +- The shared prefix is aligned to `block_size` and is the main source of theoretical hits. +- The seed length is `seed_blocks × block_size`. It is globally unique for every request, preventing accidental sharing beyond the intended prefix. +- The natural suffix is selected, concatenated, and truncated from GSM8K questions so the non-shared region remains natural-language content. + +The plugin solves for the shared-prefix length of every request from the target global hit rate, then simulates cache watermarks in final request order. + +--- + +## Core Scenario Configuration + +The complete field-by-field reference is stored in the repository at: + +```text +plugins/prefix_cache/config_examples/scenario.example.md +``` + +### Complete Field Index + +| Configuration path | Allowed fields | +|---|---| +| Top level | `schema_version`, `run`, `tokenizer`, `corpus`, `requests`, `prefix_cache`, `service`, `validation`, `aisbench` | +| `run` | `run_id`, `random_seed`, `output_dir`, `overwrite` | +| `tokenizer` | `path`, `block_size`, `revision`, `trust_remote_code` | +| `corpus` | `path`, `field`, `selection` | +| `corpus.selection` | `mode`, `values`, `indices`, `question_sha256` | +| `requests` | `count`, `input_length`, `output_length` | +| `requests.input_length` | `mode`, `value`, `values`, `ranges`, `min`, `max`, `mean`, `std`, `path`; each range item only permits `min`, `max`, and `count` | +| `requests.output_length` | `mode`, `value`, `min`, `max`, `mean`, `std`, `path` | +| `prefix_cache` | `mode`, `target_hit_rate`, `seed_blocks`, `minimum_non_shared_length`, `groups`, `order` | +| `prefix_cache.groups` | `count`, `assignment`, `overrides` | +| `prefix_cache.groups.assignment` | `mode`, `exponent`, `weights` | +| `groups.overrides.group-N` | `input_length`, `output_length`, `corpus_selection` | +| `prefix_cache.order` | `strategy` | +| `service` | `inference_url`, `metrics_url`, `reset_url`, `model`, `dp_size`, `assume_empty_cache`, `engine_label_map`, `timeout_seconds`, `api_key` | +| `validation` | `target_warning_pp`, `actual_warning_pp` | +| `aisbench` | `config`, `work_dir`, `extra_args`; unused by the current offline workflow | + +Unknown Scenario fields are rejected. `dp_size` is the only `service` field used in offline calculations. The effective `inference_url`, `metrics_url`, and `model` values only need to remain non-empty; the other service fields and the entire `aisbench` section are compatibility placeholders. + +### Input and Output Lengths + +`requests.input_length` supports: + +- `fixed`: one fixed length; +- `explicit`: an explicit list of lengths; +- `range`: sampling from one or more inclusive ranges; +- `truncated_normal`: a bounded normal distribution; +- `csv`: values from `input_prompt_tokens`, `content_tokens`, or `input_tokens`. + +`requests.output_length` supports: + +- `fixed`; +- `uniform`; +- `truncated_normal`; +- `csv`, using an `output_tokens` column. + +All lengths must be positive integers. Global explicit lists, range counts, and CSV row counts must equal `requests.count`. A group override must instead produce exactly the number of requests assigned to that group. + +### GSM8K Selection + +`corpus.selection.mode` supports: + +- `random`: deterministic shuffling based on `run.random_seed`; +- `indices`: zero-based GSM8K line numbers; +- `question_sha256`: SHA-256 of normalized question text; +- `mixed`: append `indices` first and `question_sha256` second. + +When fewer records are specified than required, the selected sequence is reused cyclically. Both mixed-mode lists cannot be empty. + +### Prefix Groups + +`prefix_cache.groups.assignment.mode` supports: + +- `uniform`: distribute requests as evenly as possible; +- `zipf`: use `exponent` to control hotspot concentration; +- `weights`: provide relative group weights in `weights`. + +Each Prefix Group has its own canonical prefix, cache watermark, and theoretical statistics. `groups.overrides.group-N` can override input lengths, output lengths, and corpus selection for one group. + +### Request Ordering + +`prefix_cache.order.strategy` supports: + +- `sequential`; +- `within_group_shuffle`; +- `interleave`; +- `global_shuffle`; +- `input_len_asc`. + +The theoretical hit rate is always recomputed using the final reordered request sequence. + +--- + +## Cold and Warmup Modes + +### cold + +- Every `(group_id, dp_rank)` lane starts from an empty cache watermark. +- Requests in a group are routed round-robin to DP ranks in group occurrence order. +- `full.jsonl` records `dp_rank` and `lane_sequence`. +- Each lane is simulated independently and then aggregated using token weighting. + +### warmup + +- One warmup item is generated for every `Prefix Group × DP rank`. +- The plan is written to `warmup.plan` in the Manifest. +- Warmup requests are not written to `requests.jsonl` and are excluded from the formal request count and theoretical denominator. +- The current plugin generates the plan but does not send warmup requests. + +--- + +## Theoretical Hit Rate and Reachability + +For an independent cache lane, if the watermark before a request is `watermark` and the request has `shared_prefix_tokens`, the theoretical hit count is: + +```text +hit_tokens = min(shared_prefix_tokens, watermark) +watermark_after = max(watermark, shared_prefix_tokens) +``` + +The global result is token weighted: + +```text +global_hit_rate = sum(theoretical_hit_tokens) / sum(actual_input_tokens) +``` + +The plugin reports: + +- `requested_target_hit_rate`: the requested Scenario target; +- `effective_target_hit_rate`: the nearest reachable target selected by the solver; +- `theoretical_hit_rate`: the value simulated in final request order; +- `reachable_min` and `reachable_max`: the theoretical range under current constraints; +- `target_reachable`: whether the requested target falls within that range. + +Block alignment, unique seeds, natural suffixes, Prefix Groups, ordering, and cold DP lanes can all make a target unreachable. + +--- + +## Output Layout and Timestamps + +Timestamps use `_YYYYMMDD_HHMMSS`. In the recommended workflow, `inspect` creates the timestamp and reuse pointer, and the following `prepare` reuses it: + +```text +outputs/gsm8k-prefix-cache-60_20260825_123456/ +├── log/ +│ ├── gsm8k-prefix-cache-60_20260825_123456.inspect.log +│ ├── gsm8k-prefix-cache-60_20260825_123456.prepare.log +│ └── gsm8k-prefix-cache-60_20260825_123456.validate.log +└── result/ + ├── gsm8k-prefix-cache-60_20260825_123456.full.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.requests.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.manifest.json + └── gsm8k-prefix-cache-60_20260825_123456.analysis.json +``` + +An `.inspect.json` pointer is also written beside the base output directory. It currently matches only the base `run_id`, `output_dir`, and directory validity; it does not compare the Scenario hash. Run `inspect` again after changing other Scenario parameters when a fresh output directory is required. + +--- + +## Artifacts + +| Artifact | Purpose | +|---|---| +| `full.jsonl` | Complete audit rows: group, DP lane, input lengths, shared prefix, unique seed, GSM8K sources, theoretical watermark, and collision state. | +| `requests.jsonl` | Minimal AISBench requests. Each row contains exactly `question`, `answer`, and `max_tokens`. | +| `manifest.json` | Effective configuration, input hashes, tokenizer fingerprint, length distributions, reachable ranges, groups, DP, warmup plan, and artifact hashes. | +| `analysis.json` | Requested/effective/theoretical rates, differences, validation state, group/DP theory, and warnings. | + +The plaintext `service.api_key` is not stored in the Manifest; only `api_key_configured` is recorded. + +Fixed field index: + +- `requests.jsonl`: `question`, `answer`, `max_tokens`; +- `full.jsonl`: `request_id`, `sequence_index`, `group_id`, `occurrence_index_within_group`, `dp_rank`, `lane_sequence`, `target_input_tokens`, `actual_input_tokens`, `max_tokens`, `shared_prefix_tokens`, `seed_tokens`, `natural_suffix_tokens`, `question`, `answer`, `gsm_indices`, `gsm_hashes`, `canonical_prefix_sha256`, `seed_sha256`, `request_random_seed`, `watermark_before`, `theoretical_hit_tokens`, `watermark_after`, `theoretical_hit_rate`, `divergence_block_sha256`, `divergence_unique`, `collision_status`; +- Manifest top level: `schema_version`, `plugin_version`, `run_id`, `scenario_path`, `scenario_sha256`, `effective_config`, `effective_config_sha256`, `corpus_sha256`, `tokenizer`, `requests`, `prefix_cache`, `groups`, `dp`, `warmup`, `divergence`, `artifacts`; +- `analysis.json`: `schema_version`, `run_id`, `status`, `requested_target_hit_rate`, `effective_target_hit_rate`, `theoretical_hit_rate`, `target_difference_pp`, `target_signed_difference_pp`, `target_absolute_difference_pp`, `validation`, `theory`, `warnings`; +- `inspect` stdout: `run_id`, `mode`, `requested_target_hit_rate`, `effective_target_hit_rate`, `theoretical_hit_rate`, `reachable_min`, `reachable_max`, `target_reachable`, `group_reachability`, `groups`, `input_tokens`, `output_tokens`, `dp_route_counts`, `sends_requests`, `log`. + +See the plugin README and complete Scenario reference for field types and nested semantics. + +--- + +## Warnings and Exit Codes + +| Warning | Condition | +|---|---| +| `TARGET_UNREACHABLE` | The requested target is outside `[reachable_min, reachable_max]`. | +| `TARGET_DEVIATION` | The absolute difference between theory and target exceeds `validation.target_warning_pp`. | + +These warnings only change the displayed validation state to `PASS_WITH_WARNING`. `warning_only=true` and `affects_exit_code=false`, so they do not change an otherwise successful exit code. Scenario, generation, or artifact validation errors return a non-zero exit code. + +--- + +## Troubleshooting + +### Why does the theoretical rate not exactly match the target? + +Shared prefixes must be Block aligned, and space must remain for the unique seed and natural suffix. Cold mode also pays for initial misses and is constrained by request order, groups, and DP-lane watermarks. Run `inspect` first and check `reachable_min`, `reachable_max`, and `target_reachable`. + +### Why is warmup excluded from formal statistics? + +Warmup exists only to establish cache state. Counting it in request volume, throughput, latency, or hit rate would mix setup cost into the formal benchmark window. + +### Why does prepare report that artifacts already exist? + +`prepare` may have reused an inspect timestamp that already contains formal artifacts. Run `inspect` again to obtain a new timestamp. Use the following command only when intentionally rebuilding the same directory: + +```shell +ais-bench-prefix-cache prepare --scenario ./scenario.json --overwrite +``` + +### Why does tokenizer round-trip validation fail? + +The plugin requires canonical prefixes, seeds, and final prompts to survive tokenizer encode/decode round trips. Verify that the tokenizer files are complete, `trust_remote_code` is configured correctly, and the tokenizer version matches the target service. + +--- + +## Current Scope + +- Supports data planning for multiple DP ranks behind one HTTP endpoint. +- Does not support multiple independent inference-server instances. +- Does not execute online warmup, formal benchmarks, or Prometheus collection. +- The complete configuration and JSON contracts are defined by `plugins/prefix_cache/README.md` and `plugins/prefix_cache/config_examples/scenario.example.md`. diff --git a/docs/source_en/index.rst b/docs/source_en/index.rst index 26377c84..f4b36c86 100644 --- a/docs/source_en/index.rst +++ b/docs/source_en/index.rst @@ -56,6 +56,7 @@ To help you quickly get started with AISBench Benchmark Tool, we recommend learn advanced_tutorials/custom_dataset advanced_tutorials/judge_model_evaluate advanced_tutorials/spec_decode + advanced_tutorials/prefix_cache .. toctree:: :maxdepth: 2 @@ -107,4 +108,4 @@ To help you quickly get started with AISBench Benchmark Tool, we recommend learn :caption: 🏷️ Others :hidden: - others/others \ No newline at end of file + others/others diff --git a/docs/source_zh_cn/advanced_tutorials/prefix_cache.md b/docs/source_zh_cn/advanced_tutorials/prefix_cache.md new file mode 100644 index 00000000..11c39531 --- /dev/null +++ b/docs/source_zh_cn/advanced_tutorials/prefix_cache.md @@ -0,0 +1,343 @@ +# Prefix Cache 数据集生成与理论命中率分析 + +## 概述 + +AISBench Prefix Cache 插件用于离线构造具有可控公共前缀的数据集，并在发送请求前计算理论 Prefix Cache 命中率。它适用于验证不同输入长度、公共前缀比例、Prefix Group、请求顺序以及单入口多 DP 对缓存命中率的影响。 + +当前插件提供三个离线命令： + +- `inspect`：预览场景、可达范围和长度分布； +- `prepare`：生成正式请求、Manifest 和理论分析； +- `validate`：校验已有产物是否被修改、截断或换序。 + +本分支不连接 vLLM、不发送正式请求，也不采集在线 Prometheus 指标。它生成可供后续 AISBench 压测使用的数据和审计信息。 + +--- + +## 前置条件 + +1. **Python 3.10 或更高版本**。 +2. **可正常使用的 AISBench 仓库及依赖**。 +3. **与目标 vLLM 服务一致的 tokenizer**。tokenizer 不一致会造成 token 长度、Block 边界和理论命中率偏差。 +4. **GSM8K JSONL 语料**。每个非空行必须是 JSON 对象，并包含 Scenario 中 `corpus.field` 指定的文本字段，默认是 `question`。 +5. **正确的 Prefix Cache Block 大小**。`tokenizer.block_size` 必须与目标服务实际值一致。 + +--- + +## 安装 + +以下命令假设当前目录是 AISBench 仓库根目录： + +```shell +python -m venv .venv +source .venv/bin/activate +python -m pip install -e . +python -m pip install -e ./plugins/prefix_cache +ais-bench-prefix-cache --help +``` + +`-e` 表示 editable 安装，修改当前仓库源码后通常不需要重新安装。 + +--- + +## 快速使用 + +复制示例 Scenario： + +```shell +cp ./plugins/prefix_cache/config_examples/scenario.example.json ./scenario.json +``` + +至少检查 `tokenizer.path`、`tokenizer.block_size` 和 `corpus.path`。需要模拟 cold 多 DP 路由或生成 warmup 计划时，还应让 `service.dp_size` 与目标服务一致。 + +一个最小示例： + +```json +{ + "schema_version": "1.0", + "run": { + "run_id": "gsm8k-prefix-cache-60", + "random_seed": 42, + "output_dir": "./outputs/gsm8k-prefix-cache-60" + }, + "tokenizer": { + "path": "/path/to/tokenizer", + "block_size": 16 + }, + "corpus": { + "path": "./GSM8K.jsonl", + "field": "question", + "selection": {"mode": "random"} + }, + "requests": { + "count": 100, + "input_length": {"mode": "fixed", "value": 1024}, + "output_length": {"mode": "fixed", "value": 32} + }, + "prefix_cache": { + "mode": "warmup", + "target_hit_rate": 0.6, + "seed_blocks": 1, + "groups": {"count": 1, "assignment": {"mode": "uniform"}}, + "order": {"strategy": "interleave"} + }, + "service": {"dp_size": 2} +} +``` + +依次执行： + +```shell +ais-bench-prefix-cache inspect --scenario ./scenario.json +ais-bench-prefix-cache prepare --scenario ./scenario.json +ais-bench-prefix-cache validate --manifest \ + ./outputs/gsm8k-prefix-cache-60_<时间戳>/result/gsm8k-prefix-cache-60_<时间戳>.manifest.json +``` + +--- + +## 工作原理 + +```mermaid +flowchart LR + S[Scenario] --> I[inspect 预览] + I --> P[prepare 构造 Prompt] + P --> G[公共前缀] + P --> U[全局唯一 Seed] + P --> N[GSM8K 自然后缀] + G --> T[顺序感知理论水位模拟] + U --> T + N --> T + T --> A[full / requests / Manifest / analysis] + A --> V[validate 完整性校验] +``` + +每条正式请求由三部分构成： + +```text +公共前缀 + 全局唯一 seed + GSM8K 自然后缀 +``` + +- 公共前缀按 `block_size` 对齐，是理论命中的主要来源； +- seed 长度为 `seed_blocks × block_size`，每条请求全局唯一，防止公共前缀之后继续误共享； +- 自然后缀从 GSM8K 问题中选择、拼接并截断，使非共享区保持自然语言形态。 + +插件根据目标全局命中率反求每条请求的公共前缀长度，并按照最终请求顺序模拟缓存水位。 + +--- + +## Scenario 核心配置 + +完整逐字段参考位于仓库中的： + +```text +plugins/prefix_cache/config_examples/scenario.example.md +``` + +### 完整字段索引 + +| 配置路径 | 允许字段 | +|---|---| +| 顶层 | `schema_version`、`run`、`tokenizer`、`corpus`、`requests`、`prefix_cache`、`service`、`validation`、`aisbench` | +| `run` | `run_id`、`random_seed`、`output_dir`、`overwrite` | +| `tokenizer` | `path`、`block_size`、`revision`、`trust_remote_code` | +| `corpus` | `path`、`field`、`selection` | +| `corpus.selection` | `mode`、`values`、`indices`、`question_sha256` | +| `requests` | `count`、`input_length`、`output_length` | +| `requests.input_length` | `mode`、`value`、`values`、`ranges`、`min`、`max`、`mean`、`std`、`path`；range 项只允许 `min`、`max`、`count` | +| `requests.output_length` | `mode`、`value`、`min`、`max`、`mean`、`std`、`path` | +| `prefix_cache` | `mode`、`target_hit_rate`、`seed_blocks`、`minimum_non_shared_length`、`groups`、`order` | +| `prefix_cache.groups` | `count`、`assignment`、`overrides` | +| `prefix_cache.groups.assignment` | `mode`、`exponent`、`weights` | +| `groups.overrides.group-N` | `input_length`、`output_length`、`corpus_selection` | +| `prefix_cache.order` | `strategy` | +| `service` | `inference_url`、`metrics_url`、`reset_url`、`model`、`dp_size`、`assume_empty_cache`、`engine_label_map`、`timeout_seconds`、`api_key` | +| `validation` | `target_warning_pp`、`actual_warning_pp` | +| `aisbench` | `config`、`work_dir`、`extra_args`；当前离线流程不消费 | + +Scenario 会拒绝白名单之外的字段。`service` 中当前真正参与离线计算的是 `dp_size`；`inference_url`、`metrics_url` 和 `model` 只要求最终有效值非空，其余服务字段以及整个 `aisbench` 段仅兼容保留。 + +### 输入和输出长度 + +`requests.input_length` 支持： + +- `fixed`：固定长度； +- `explicit`：显式长度列表； +- `range`：一个或多个闭区间采样； +- `truncated_normal`：截断正态分布； +- `csv`：从 CSV 的 `input_prompt_tokens`、`content_tokens` 或 `input_tokens` 列读取。 + +`requests.output_length` 支持： + +- `fixed`； +- `uniform`； +- `truncated_normal`； +- `csv`，列名必须为 `output_tokens`。 + +所有长度必须为正整数。全局显式列表、range 计数和 CSV 行数必须等于 `requests.count`；组级覆盖时必须等于该组实际请求数。 + +### GSM8K 样本选择 + +`corpus.selection.mode` 支持： + +- `random`：根据 `run.random_seed` 确定性打乱； +- `indices`：按 GSM8K 零基行号选择； +- `question_sha256`：按规范化 question 的 SHA-256 选择； +- `mixed`：先加入 `indices`，再加入 `question_sha256`。 + +指定样本不足时会按已选顺序循环复用。mixed 模式的两个列表不能同时为空。 + +### Prefix Group + +`prefix_cache.groups.assignment.mode` 支持： + +- `uniform`：尽量均匀分配； +- `zipf`：使用 `exponent` 控制热点集中程度； +- `weights`：通过 `weights` 提供每组相对权重。 + +每个 Prefix Group 独立生成 canonical 前缀、维护缓存水位并统计理论命中率。`groups.overrides.group-N` 可以独立覆盖输入长度、输出长度和语料选择方式。 + +### 请求顺序 + +`prefix_cache.order.strategy` 支持： + +- `sequential`； +- `within_group_shuffle`； +- `interleave`； +- `global_shuffle`； +- `input_len_asc`。 + +理论命中率始终按重排后的最终发送顺序计算。 + +--- + +## cold 与 warmup + +### cold + +- 每个 `(group_id, dp_rank)` lane 从零缓存水位开始； +- 同一组的正式请求按组内出现顺序 round-robin 路由到各 DP rank； +- `full.jsonl` 记录 `dp_rank` 和 `lane_sequence`； +- 理论命中率按每个 lane 独立模拟后进行 token 加权汇总。 + +### warmup + +- 为每个 `Prefix Group × DP rank` 生成一条预热计划； +- 预热计划写入 Manifest 的 `warmup.plan`； +- warmup 请求不写入 `requests.jsonl`，不进入正式请求数量和理论统计分母； +- 当前插件只生成预热计划，不实际发送预热请求。 + +--- + +## 理论命中率和可达性 + +对于某个独立缓存 lane，请求到达前水位为 `watermark`，请求共享前缀为 `shared_prefix_tokens`，理论命中 token 为： + +```text +hit_tokens = min(shared_prefix_tokens, watermark) +watermark_after = max(watermark, shared_prefix_tokens) +``` + +全局命中率使用 token 加权口径： + +```text +global_hit_rate = sum(theoretical_hit_tokens) / sum(actual_input_tokens) +``` + +插件同时输出： + +- `requested_target_hit_rate`：Scenario 请求目标； +- `effective_target_hit_rate`：求解器选择的最近可达目标； +- `theoretical_hit_rate`：按最终顺序模拟的理论值； +- `reachable_min`、`reachable_max`：当前约束下的理论范围； +- `target_reachable`：请求目标是否位于可达范围内。 + +Block 对齐、唯一 seed、自然后缀、Prefix Group、顺序和 cold DP lane 都可能使某个目标不可达。 + +--- + +## 输出目录和时间戳 + +时间戳格式为 `_YYYYMMDD_HHMMSS`。推荐工作流中，`inspect` 创建时间戳和复用指针，紧接着的 `prepare` 复用该时间戳： + +```text +outputs/gsm8k-prefix-cache-60_20260825_123456/ +├── log/ +│ ├── gsm8k-prefix-cache-60_20260825_123456.inspect.log +│ ├── gsm8k-prefix-cache-60_20260825_123456.prepare.log +│ └── gsm8k-prefix-cache-60_20260825_123456.validate.log +└── result/ + ├── gsm8k-prefix-cache-60_20260825_123456.full.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.requests.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.manifest.json + └── gsm8k-prefix-cache-60_20260825_123456.analysis.json +``` + +基础输出目录旁还会生成 `.inspect.json`。当前指针只匹配基础 `run_id`、`output_dir` 和目录有效性，不比较 Scenario 哈希；修改其他参数后需要新目录时，应重新执行 `inspect`。 + +--- + +## 产物说明 + +| 产物 | 作用 | +|---|---| +| `full.jsonl` | 完整审计数据，包括组、DP lane、输入长度、公共前缀、唯一 seed、GSM8K 来源、理论水位和碰撞状态。 | +| `requests.jsonl` | 最小 AISBench 请求，每行严格为 `question`、`answer`、`max_tokens`。 | +| `manifest.json` | 有效配置、输入哈希、tokenizer 指纹、长度分布、可达范围、组、DP、warmup 和产物哈希。 | +| `analysis.json` | requested/effective/theoretical 命中率、偏差、验证状态、分组/分 DP 理论统计和 warnings。 | + +`service.api_key` 明文不会写入 Manifest，只记录 `api_key_configured`。 + +固定字段索引： + +- `requests.jsonl`：`question`、`answer`、`max_tokens`； +- `full.jsonl`：`request_id`、`sequence_index`、`group_id`、`occurrence_index_within_group`、`dp_rank`、`lane_sequence`、`target_input_tokens`、`actual_input_tokens`、`max_tokens`、`shared_prefix_tokens`、`seed_tokens`、`natural_suffix_tokens`、`question`、`answer`、`gsm_indices`、`gsm_hashes`、`canonical_prefix_sha256`、`seed_sha256`、`request_random_seed`、`watermark_before`、`theoretical_hit_tokens`、`watermark_after`、`theoretical_hit_rate`、`divergence_block_sha256`、`divergence_unique`、`collision_status`； +- Manifest 顶层：`schema_version`、`plugin_version`、`run_id`、`scenario_path`、`scenario_sha256`、`effective_config`、`effective_config_sha256`、`corpus_sha256`、`tokenizer`、`requests`、`prefix_cache`、`groups`、`dp`、`warmup`、`divergence`、`artifacts`； +- `analysis.json`：`schema_version`、`run_id`、`status`、`requested_target_hit_rate`、`effective_target_hit_rate`、`theoretical_hit_rate`、`target_difference_pp`、`target_signed_difference_pp`、`target_absolute_difference_pp`、`validation`、`theory`、`warnings`； +- inspect stdout：`run_id`、`mode`、`requested_target_hit_rate`、`effective_target_hit_rate`、`theoretical_hit_rate`、`reachable_min`、`reachable_max`、`target_reachable`、`group_reachability`、`groups`、`input_tokens`、`output_tokens`、`dp_route_counts`、`sends_requests`、`log`。 + +各字段类型和嵌套含义以插件 README 与 Scenario 完整字段说明为准。 + +--- + +## 告警与退出码 + +| 告警 | 条件 | +|---|---| +| `TARGET_UNREACHABLE` | 请求目标不在 `[reachable_min, reachable_max]` 内。 | +| `TARGET_DEVIATION` | 理论值与请求目标的绝对差超过 `validation.target_warning_pp`。 | + +这些告警只把展示状态改为 `PASS_WITH_WARNING`；`warning_only=true`、`affects_exit_code=false`，不会改变成功退出码。只有 Scenario、生成或产物校验错误才返回非零退出码。 + +--- + +## 常见问题 + +### 为什么理论命中率没有精确等于目标？ + +公共前缀必须按 Block 对齐，同时还要为唯一 seed 和自然后缀预留空间。cold 模式还受首次 miss、请求顺序、组和 DP lane 水位约束。请先运行 `inspect`，检查 `reachable_min`、`reachable_max` 和 `target_reachable`。 + +### 为什么 warmup 不进入正式统计？ + +warmup 只负责建立缓存。如果计入正式请求数、吞吐、时延或命中率，结果会混入准备阶段成本。 + +### 为什么 prepare 报同名文件已存在？ + +prepare 可能复用了已有正式产物的 inspect 时间戳。重新执行 `inspect` 可获得新时间戳；只有明确要重建同一目录时才使用： + +```shell +ais-bench-prefix-cache prepare --scenario ./scenario.json --overwrite +``` + +### 为什么 tokenizer round-trip 失败？ + +插件要求 canonical 前缀、seed 和最终 prompt 在 tokenizer 编解码后保持一致。请确认 tokenizer 文件完整、`trust_remote_code` 设置正确，并与目标服务使用同一 tokenizer 版本。 + +--- + +## 当前范围 + +- 支持单个 HTTP 入口对应的多 DP 数据规划； +- 不支持多个独立推理服务实例； +- 不执行在线 warmup、正式压测或 Prometheus 指标采集； +- 详细配置和全部 JSON 字段契约以 `plugins/prefix_cache/README.md` 与 `plugins/prefix_cache/config_examples/scenario.example.md` 为准。 diff --git a/docs/source_zh_cn/index.rst b/docs/source_zh_cn/index.rst index b1526c75..f98e0ab2 100644 --- a/docs/source_zh_cn/index.rst +++ b/docs/source_zh_cn/index.rst @@ -56,6 +56,7 @@ AISBench Benchmark 是基于 `OpenCompass .inspect.log`； +- 成功后在基础输出目录旁写入 `.inspect.json` 指针，记录本次时间戳，供下一次匹配的 `prepare` 复用同一目录； +- 输出的 JSON 摘要包含 `log` 字段，可直接定位该日志。 + +### 3.4 `prepare`：生成正式数据产物 + +```bash +ais-bench-prefix-cache prepare --scenario ./scenario.json +``` + +作用：根据 Scenario 确定性生成并校验四个文件： + +- `result/.full.jsonl`； +- `result/.requests.jsonl`； +- `result/.manifest.json`； +- `result/.analysis.json`。 + +执行时会先显示 prompt 生成进度，且每成功生成一条 prompt 增加 1： + +```text +Generate prompts [###############---------------] 50/100 50% +Generate prompts [##############################] 100/100 100% +{"full":"...","requests":"...","manifest":"...","analysis":"...","log":"..."} +``` + +进度写入 stderr，最后一行结果 JSON 写入 stdout，方便脚本继续解析。 + +时间戳采用 `_YYYYMMDD_HHMMSS`。单独执行 `prepare` 且没有可复用 inspect 指针时，会生成新时间戳；如果此前成功执行过匹配的 `inspect`，`prepare` 会复用 inspect 的时间戳，使 inspect、prepare、validate 的日志和正式产物位于同一个时间戳目录。 + +例如配置为： + +```text +run_id: gsm8k-prefix-cache-60 +output_dir: ./outputs/gsm8k-prefix-cache-60 +``` + +本次实际目录可能为： + +```text +./outputs/gsm8k-prefix-cache-60_20260825_123456/ +├── log/ +│ └── gsm8k-prefix-cache-60_20260825_123456.prepare.log +└── result/ + ├── gsm8k-prefix-cache-60_20260825_123456.full.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.requests.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.manifest.json + └── gsm8k-prefix-cache-60_20260825_123456.analysis.json +``` + +因此正常工作流不需要手动修改 `run_id` 或 `output_dir`。inspect 指针位于基础输出目录旁，例如 `./outputs/gsm8k-prefix-cache-60.inspect.json`，字段为 `schema_version`、`timestamp`、`run_id`、`output_dir` 和 `output_dir_with_timestamp`。 + +只有当指针版本、基础 `run_id`、基础 `output_dir`、时间戳格式和对应目录都有效时才会复用。当前实现不比较 Scenario 内容哈希；修改其他 Scenario 参数后如需确保使用新目录，应重新执行 `inspect` 生成新指针，或删除旧的 `.inspect.json` 指针后再执行 `prepare`。 + +默认不覆盖同名文件。确定需要重建时使用： + +```bash +ais-bench-prefix-cache prepare --scenario ./scenario.json --overwrite +``` + +`--overwrite` 只覆盖本次时间戳目录内该 run 对应的四个确定文件，不会清理整个输出目录。若 `prepare` 复用了已经存在正式产物的 inspect 时间戳，则默认会因同名文件失败；此时应重新执行 `inspect` 获得新时间戳，只有明确要重建同一目录时才使用 `--overwrite`。 + +### 3.5 `validate`：校验已有产物 + +```bash +ais-bench-prefix-cache validate --manifest ./outputs/gsm8k-prefix-cache-60_<时间戳>/result/gsm8k-prefix-cache-60_<时间戳>.manifest.json +``` + +作用：不生成数据、不访问 vLLM，只检查： + +- Manifest、full 和 requests 行数是否一致； +- `sequence_index` 是否连续； +- requests 是否严格只含 `question`、`answer`、`max_tokens`； +- requests 与 full 是否逐行对应； +- full 和 requests 的 SHA-256 是否匹配 Manifest。 + +它用于发现文件被手工编辑、截断、换序或使用了错误版本。 + +与 `inspect`/`prepare` 一样，validate 的详细日志写入 Manifest 所在时间戳输出目录的 `log/.validate.log`，终端只打印校验结果 JSON。 + +## 4. 推荐工作流 + +```bash +ais-bench-prefix-cache inspect --scenario ./scenario.json +ais-bench-prefix-cache prepare --scenario ./scenario.json +ais-bench-prefix-cache validate --manifest +``` + +这样可以在任何实际压测前人工审计数据。该顺序下 `prepare` 会复用刚刚 `inspect` 的时间戳；`prepare` 遇到同名产物时默认拒绝覆盖。 + +## 5. cold 与 warmup + +### cold + +- 每个 `(group_id, dp_rank)` 从零水位开始； +- 同一组的请求按组内 round-robin 定向 DP； +- 插件保证同一 lane 内请求顺序； +- 可输出严格的分 DP 理论命中率。 + +### warmup + +- 对每个 Prefix Group、每个 DP rank 分别预热； +- warmup 不进入 requests JSONL、理论分母或正式指标增量； +- 全局理论命中率有效，分 DP 主要展示实际指标。 + +> 本分支只负责生成 cold / warmup 模式下的数据与预热计划，不实际执行预热请求。预热计划落在 `result/.manifest.json` 的 `warmup.plan` 字段。 + +## 6. 分层产物 + +所有正式数据产物位于实际时间戳输出目录的 `result/` 下，详细日志位于同级 `log/` 下。 + +### `.full.jsonl` + +完整审计数据，每行固定包含以下字段： + +| 字段 | 含义 | +|---|---| +| `request_id` | 稳定请求 ID。 | +| `sequence_index` | 最终发送顺序中的零基序号。 | +| `group_id` | 所属 Prefix Group。 | +| `occurrence_index_within_group` | 该请求在组内第几次出现。 | +| `dp_rank` | cold 模式的目标 DP rank；warmup 正式请求为 `null`。 | +| `lane_sequence` | cold `(group_id, dp_rank)` lane 内序号；warmup 为 `null`。 | +| `target_input_tokens` | 配置要求的输入长度。 | +| `actual_input_tokens` | tokenizer 重编码后的实际输入长度。 | +| `max_tokens` | 最大输出 token 数。 | +| `shared_prefix_tokens` | 本请求使用的公共前缀 token 数。 | +| `seed_tokens` | 全局唯一 seed 的 token 数。 | +| `natural_suffix_tokens` | seed 后 GSM8K 自然后缀的 token 数。 | +| `question` | 最终完整 prompt。 | +| `answer` | AISBench 兼容占位值，当前固定为 `"none"`。 | +| `gsm_indices` | 本请求自然后缀使用的 GSM8K 零基行号。 | +| `gsm_hashes` | 对应规范化 question 的 SHA-256。 | +| `canonical_prefix_sha256` | 所属组 canonical 前缀指纹。 | +| `seed_sha256` | 本请求唯一 seed token 序列指纹。 | +| `request_random_seed` | 实际参与本请求 seed 构造的确定性随机种子。 | +| `watermark_before` | 请求到达前所在缓存 lane 的理论水位。 | +| `theoretical_hit_tokens` | 本请求理论命中 token 数。 | +| `watermark_after` | 请求完成后的理论水位。 | +| `theoretical_hit_rate` | `theoretical_hit_tokens / actual_input_tokens`。 | +| `divergence_block_sha256` | 差异块指纹，当前等于 `seed_sha256`。 | +| `divergence_unique` | 差异块是否通过全局唯一性检查。 | +| `collision_status` | 碰撞检查状态，成功产物为 `"pass"`。 | + +### `.requests.jsonl` + +最小输入，每行字段顺序严格为 `question`、`answer`、`max_tokens`： + +- `question`：最终完整 prompt； +- `answer`：当前固定为 `"none"`； +- `max_tokens`：该请求最大输出 token 数。 + +DP 路由等字段只存在于 full 文件，不污染通用请求格式。 + +### `.manifest.json` + +复现和校验入口。顶层字段如下： + +| 字段 | 含义 | +|---|---| +| `schema_version`、`plugin_version` | Manifest 契约版本和插件版本。 | +| `run_id` | 已追加执行时间戳的运行 ID。 | +| `scenario_path`、`scenario_sha256` | Scenario 绝对路径及原文件 SHA-256。 | +| `effective_config`、`effective_config_sha256` | 补齐默认值、解析路径后的有效配置及其指纹。 | +| `corpus_sha256` | GSM8K 文件 SHA-256。 | +| `tokenizer` | tokenizer 来源、类、词表、特殊 token、Block 和指纹。 | +| `requests` | 请求数、总输入 token、输入/输出长度摘要。 | +| `prefix_cache` | 模式、目标、理论值、可达区间、调整原因和验证状态。 | +| `groups` | 各组 canonical 来源、最大前缀、可达区间和理论命中率。 | +| `dp` | DP 数量及 cold 路由策略。 | +| `warmup` | 是否启用及逐组逐 DP 的预热计划。 | +| `divergence` | 唯一差异块策略、数量和碰撞状态。 | +| `artifacts` | full、requests、analysis 的名称、路径、行数、大小和哈希。 | + +重要嵌套字段： + +- `tokenizer`：`path`、`revision`、`class`、`vocab_size`、`special_token_ids`、`block_size`、`fingerprint_sha256`； +- `requests`：`count`、`total_input_tokens`、`input_length_summary`、`output_length_summary`；每个 summary 包含 `min`、`max`、`mean`、`p50`、`p90`、`p95`、`p99`、`bins`，每个 bin 包含 `min`、`max`、`count`； +- `prefix_cache`：`mode`、`requested_target_hit_rate`、`effective_target_hit_rate`、`theoretical_hit_rate`、`reachable_min`、`reachable_max`、`target_reachable`、`minimum_non_shared_length`、`adjusted`、`reason`、`validation_status`、`target_signed_difference_pp`、`target_absolute_difference_pp`； +- `groups.`：`canonical_prefix_sha256`、`canonical_prefix_tokens`、`max_shared_prefix_tokens`、`gsm_indices`、`gsm_question_sha256`、`reachable_min`、`reachable_max`、`theoretical_hit_rate`； +- `dp`：`size`、`cold_route_strategy`；warmup 模式的路由策略为 `null`； +- `warmup`：`enabled`、`plan`；plan 每项包含 `request_id`、`group_id`、`dp_rank`、`prompt`、`input_tokens`、`shared_prefix_tokens`、`max_tokens`、`included_in_formal_statistics`； +- `divergence`：`strategy`、`unique_request_blocks`、`request_count`、`collision_status`； +- `artifacts.full/requests`：`name`、`path`、`rows`、`bytes`、`sha256`；`artifacts.analysis` 包含 `name`、`path`、`bytes`、`sha256_at_prepare`。 + +`api_key` 明文不会写入 Manifest；`effective_config.service` 中改为布尔字段 `api_key_configured`。 + +### `.analysis.json` + +固定字段为： + +- `schema_version`、`run_id`、`status`； +- `requested_target_hit_rate`、`effective_target_hit_rate`、`theoretical_hit_rate`； +- `target_difference_pp`、`target_signed_difference_pp`、`target_absolute_difference_pp`，其中 `target_difference_pp` 当前等于绝对偏差； +- `validation`：`status`、`target_reachable`、`warning_only`、`affects_exit_code`； +- `theory`：`input_tokens`、`hit_tokens`、`groups`、`dp`；每个组或 DP 统计包含 `input_tokens`、`hit_tokens`、`hit_rate`； +- `warnings`：零个或多个告警。`TARGET_UNREACHABLE` 包含 requested target 和可达上下界，`TARGET_DEVIATION` 包含 `difference_pp`。 + +成功生成时 `status="prepared"`；偏差告警只改变 `validation.status` 的展示值，不改变成功退出码。 + +### `inspect`、指针和 CLI 返回字段 + +`inspect` 终端 JSON 包含：`run_id`、`mode`、`requested_target_hit_rate`、`effective_target_hit_rate`、`theoretical_hit_rate`、`reachable_min`、`reachable_max`、`target_reachable`、`group_reachability`、`groups`、`input_tokens`、`output_tokens`、`dp_route_counts`、`sends_requests`、`log`。其中 `sends_requests` 固定为 `false`。 + +`.inspect.json` 包含：`schema_version`、`timestamp`、`run_id`、`output_dir`、`output_dir_with_timestamp`。 + +CLI 最后一段 JSON 的固定字段为： + +- `prepare`：`full`、`requests`、`manifest`、`analysis`、`log`； +- `inspect`：上述 inspect 摘要和 `log`； +- `validate`：`ok`、`rows`、`run_id`。validate 会写日志，但返回 JSON 当前不包含 `log`。 + +## 7. 退出码 + +- 理论与目标差异超过 `target_warning_pp`：`TARGET_DEVIATION`； +- 目标超出可达区间：`TARGET_UNREACHABLE`； +- 两者始终只告警，不改变原本成功的退出码； +- 配置错误、产物损坏会返回非零退出码。 + +## 8. 常见问题 + +### 目标命中率为什么不完全相等？ + +公共前缀按 `block_size` 对齐，cold 还受顺序、组、DP 路由和缓存水位约束。插件选择最接近的可达结果，并记录 requested、effective、theoretical 和偏差原因。 + +如果多个 Prefix Group 选择了相同的首个 GSM8K 样本，插件会先尝试轮换组内样本；所有轮换仍碰撞时才使用确定性的组标记兜底，避免小语料或重复 indices 让整个 prepare 直接失败。 + +### warmup 为什么不进入正式统计？ + +warmup 只负责建立缓存。如果计入请求数、吞吐、时延或命中率，正式结果会混入准备阶段成本。 + +### 修改 Scenario 后为什么通常不再需要手动改 run_id？ + +单独执行 `prepare` 时会使用新的秒级时间戳；执行推荐的 `inspect → prepare` 工作流时，prepare 会复用 inspect 时间戳。两种方式都会把同一时间戳同时追加到 `run_id` 和 `output_dir`，因此不需要手动改名。`--overwrite` 仅用于明确重建同一时间戳目录。 diff --git a/plugins/prefix_cache/ais_bench_prefix_cache/__init__.py b/plugins/prefix_cache/ais_bench_prefix_cache/__init__.py new file mode 100644 index 00000000..47de88d2 --- /dev/null +++ b/plugins/prefix_cache/ais_bench_prefix_cache/__init__.py @@ -0,0 +1,3 @@ +"""AISBench Prefix Cache plugin.""" + +__version__ = "0.1.2" diff --git a/plugins/prefix_cache/ais_bench_prefix_cache/artifacts.py b/plugins/prefix_cache/ais_bench_prefix_cache/artifacts.py new file mode 100644 index 00000000..ec10cd92 --- /dev/null +++ b/plugins/prefix_cache/ais_bench_prefix_cache/artifacts.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import hashlib +import json +import logging +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +from .errors import ArtifactValidationError + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ArtifactPaths: + """一次运行产出的四个工件文件的路径集合。""" + full: Path # .full.jsonl：每条请求的完整审计记录 + requests: Path # .requests.jsonl：发给推理服务的请求本体 + manifest: Path # .manifest.json：运行元信息与配置指纹 + analysis: Path # .analysis.json：汇总/校验结果 + + +def sha256_file(path: Path) -> str: + """分块计算文件的 SHA-256 摘要（用于工件一致性校验）。""" + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + result = digest.hexdigest() + logger.info("[artifacts] sha256_file path=%s sha256=%s", path, result) + return result + + +def _atomic_text(path: Path, text: str, overwrite: bool) -> None: + """原子写文件：先写临时文件再 os.replace，避免半截文件。 + + 已存在且 overwrite=False 时拒绝覆盖，防止误破坏历史工件。 + """ + path.parent.mkdir(parents=True, exist_ok=True) + logger.info("[artifacts] _atomic_text path=%s exists=%s overwrite=%s text_bytes=%d", path, path.exists(), overwrite, len(text.encode("utf-8"))) + if path.exists() and not overwrite: + raise ArtifactValidationError(f"refusing to overwrite existing artifact: {path}") + temp = path.with_name(f".{path.name}.tmp-{os.getpid()}") + try: + temp.write_text(text, encoding="utf-8", newline="\n") + os.replace(temp, path) + finally: + if temp.exists(): + temp.unlink() + logger.info("[artifacts] _atomic_text written path=%s", path) + + +def write_json(path: Path, value: dict[str, Any], overwrite: bool) -> None: + """把字典以带缩进、键排序的 JSON 形式原子写入。""" + logger.info("[artifacts] write_json path=%s overwrite=%s keys=%s", path, overwrite, sorted(value)) + _atomic_text(path, json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", overwrite) + + +def write_jsonl(path: Path, rows: Iterable[dict[str, Any]], overwrite: bool) -> int: + """把多行记录写成 JSONL（每行一个对象），返回写入行数。""" + materialized = list(rows) + logger.info("[artifacts] write_jsonl path=%s overwrite=%s rows=%d", path, overwrite, len(materialized)) + # Preserve insertion order: requests.jsonl has a documented public field + # order (question, answer, max_tokens). + text = "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in materialized) + _atomic_text(path, text, overwrite) + return len(materialized) + + +def read_jsonl(path: Path) -> list[dict[str, Any]]: + """逐行读取 JSONL 文件并反序列化为字典列表。""" + try: + rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + except (OSError, json.JSONDecodeError) as exc: + raise ArtifactValidationError(f"cannot read JSONL {path}: {exc}") from exc + logger.info("[artifacts] read_jsonl path=%s rows=%d", path, len(rows)) + return rows + + +def artifact_paths(output_dir: Path, run_id: str) -> ArtifactPaths: + """在 output_dir/result 下按 run_id 拼接四个工件文件路径。""" + result_dir = output_dir / "result" + paths = ArtifactPaths( + result_dir / f"{run_id}.full.jsonl", + result_dir / f"{run_id}.requests.jsonl", + result_dir / f"{run_id}.manifest.json", + result_dir / f"{run_id}.analysis.json", + ) + logger.info("[artifacts] artifact_paths output_dir=%s run_id=%s full=%s requests=%s manifest=%s analysis=%s", output_dir, run_id, paths.full, paths.requests, paths.manifest, paths.analysis) + return paths + + +def validate_artifacts(manifest_path: Path) -> dict[str, Any]: + """按 manifest 记录校验全部工件：行数、字段、顺序与 SHA-256 指纹一致。""" + logger.info("[artifacts] validate_artifacts manifest_path=%s", manifest_path) + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ArtifactValidationError(f"cannot read Manifest {manifest_path}: {exc}") from exc + base = manifest_path.parent + files = manifest.get("artifacts", {}) + full_path = base / files["full"]["name"] + requests_path = base / files["requests"]["name"] + logger.info("[artifacts] validate_artifacts run_id=%s artifacts=%s", manifest["run_id"], files) + full_rows = read_jsonl(full_path) + request_rows = read_jsonl(requests_path) + logger.info("[artifacts] validate_artifacts full_rows=%d request_rows=%d expected_count=%s", len(full_rows), len(request_rows), manifest["requests"]["count"]) + if len(full_rows) != len(request_rows) or len(full_rows) != manifest["requests"]["count"]: + raise ArtifactValidationError("artifact row counts do not match") + for index, (full, request) in enumerate(zip(full_rows, request_rows)): + # 校验顺序一致、requests 字段集合严格为 {question, answer, max_tokens} 且与 full 对齐。 + if full["sequence_index"] != index: + raise ArtifactValidationError(f"full row {index} has invalid sequence_index") + if set(request) != {"question", "answer", "max_tokens"}: + raise ArtifactValidationError(f"requests row {index} has unexpected fields") + if any(request[key] != full[key] for key in request): + raise ArtifactValidationError(f"requests row {index} differs from full row") + for key, path in (("full", full_path), ("requests", requests_path)): + expected = files[key]["sha256"] + actual = sha256_file(path) + logger.info("[artifacts] validate_artifacts %s expected_sha256=%s actual_sha256=%s match=%s", key, expected, actual, actual == expected) + if actual != expected: + raise ArtifactValidationError(f"{key} SHA-256 mismatch") + result = {"ok": True, "rows": len(full_rows), "run_id": manifest["run_id"]} + logger.info("[artifacts] validate_artifacts result=%s", result) + return result diff --git a/plugins/prefix_cache/ais_bench_prefix_cache/cli.py b/plugins/prefix_cache/ais_bench_prefix_cache/cli.py new file mode 100644 index 00000000..d2f8d81b --- /dev/null +++ b/plugins/prefix_cache/ais_bench_prefix_cache/cli.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import argparse +import json +import logging +import sys +from pathlib import Path +from typing import TextIO + +from .artifacts import validate_artifacts +from .errors import PrefixCacheError +from .pipeline import inspect_scenario, prepare_scenario +from .scenario import Scenario, load_scenario, new_execution_timestamp, with_execution_timestamp + +# Parent logger name shared by all module loggers (ais_bench_prefix_cache.*). +PLUGIN_LOG_NAME = "ais_bench_prefix_cache" + +LOG_NORMAL_FORMAT = "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s" + +# 显式挂在 PLUGIN_LOG_NAME 之下（不用 __name__）：python -m 运行时 __name__ 会变成 +# "__main__"，导致日志绕过插件 logger 直接传播到 root。 +logger = logging.getLogger(f"{PLUGIN_LOG_NAME}.cli") + + +class PromptProgress: + """Render prompt-generation progress to a text stream without touching stdout.""" + + def __init__(self, stream: TextIO | None = None, width: int = 30): + self.stream = stream if stream is not None else sys.stderr + self.width = max(1, width) + self._active = False + self._completed = False + + def update(self, completed: int, total: int) -> None: + if total < 1: + return + completed = min(max(0, completed), total) + filled = self.width * completed // total + percent = 100 * completed // total + bar = "#" * filled + "-" * (self.width - filled) + end = "\n" if completed == total else "\r" + self.stream.write(f"\rGenerate prompts [{bar}] {completed}/{total} {percent:3d}%{end}") + self.stream.flush() + self._active = completed < total + self._completed = completed == total + + def close(self) -> None: + """Terminate an unfinished progress line before another stderr message.""" + if self._active and not self._completed: + self.stream.write("\n") + self.stream.flush() + self._active = False + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行参数解析器：prepare / inspect / validate 三个离线子命令。""" + parser = argparse.ArgumentParser(prog="ais-bench-prefix-cache") + sub = parser.add_subparsers(dest="command", required=True) + for name in ("prepare", "inspect"): + # prepare 生成请求工件；inspect 只预览不发请求（共用 --scenario）。 + item = sub.add_parser(name) + item.add_argument("--scenario", required=True, type=Path) + prepare = sub.choices["prepare"] + prepare.add_argument("--overwrite", action="store_true") + validate = sub.add_parser("validate") + validate.add_argument("--manifest", required=True, type=Path) + return parser + + +def _resolve_log_file( + command: str, + scenario_path: Path | None = None, + manifest_path: Path | None = None, + execution_timestamp: str | None = None, +) -> Path | None: + """Resolve a per-command log under the run output directory's log/ layer. + + prepare / inspect 从 scenario 解析 output_dir 与 run_id（prepare 优先复用 + 最近一次成功 inspect 的时间戳目录，见 _reusable_inspect_timestamp）； + validate 从 manifest 的 run_id 与 effective_config.run.output_dir 解析。 + + Falls back to console-only logging when the config cannot be loaded + or the output directory is not writable (the real error surfaces in + the normal command flow). + """ + if command == "validate": + if manifest_path is None: + return None + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + run_id = manifest["run_id"] + output_dir = Path(manifest["effective_config"]["run"]["output_dir"]) + log_file = output_dir / "log" / f"{run_id}.validate.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + return log_file + except (KeyError, TypeError, OSError, json.JSONDecodeError): + return None + if scenario_path is None: + return None + try: + scenario = load_scenario(scenario_path) + if execution_timestamp is not None: + scenario = with_execution_timestamp(scenario, execution_timestamp) + log_file = scenario.output_dir / "log" / f"{scenario.run_id}.{command}.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + return log_file + except (PrefixCacheError, OSError): + return None + + +def _inspect_pointer_path(output_dir: Path) -> Path: + """每个基础 output_dir 一个指针文件，记录最近一次成功 inspect 的时间戳。""" + return output_dir.with_name(f"{output_dir.name}.inspect.json") + + +def _reusable_inspect_timestamp(scenario: Scenario) -> str | None: + """若存在与当前场景匹配的 inspect 指针且其时间戳目录还在，返回可复用的时间戳。 + + 指针记录的 run_id / output_dir 必须与当前场景一致，时间戳格式合法， + 且对应的时间戳目录仍然存在，否则视为不可复用（返回 None）。 + """ + pointer = _inspect_pointer_path(scenario.output_dir) + try: + record = json.loads(pointer.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if record.get("schema_version") != "1.0": + return None + if record.get("run_id") != scenario.run_id or record.get("output_dir") != str(scenario.output_dir): + return None + timestamp = record.get("timestamp") + if not isinstance(timestamp, str): + return None + try: + stamped = with_execution_timestamp(scenario, timestamp) + except PrefixCacheError: + return None + if not stamped.output_dir.is_dir(): + return None + return timestamp + + +def _persist_inspect_pointer(scenario_path: Path, log_file: Path, timestamp: str) -> None: + """写入 inspect 复用指针，供后续 prepare 复用同一时间戳目录。 + + Best-effort：任何持久化失败只记日志，不影响 inspect 命令本身的结果。 + """ + try: + base_scenario = load_scenario(scenario_path) + except PrefixCacheError as exc: + logger.warning("[cli] cannot load scenario for inspect pointer: %s", exc) + return + run_dir = log_file.parent.parent + pointer = _inspect_pointer_path(base_scenario.output_dir) + record = { + "schema_version": "1.0", + "timestamp": timestamp, + "run_id": base_scenario.run_id, + "output_dir": str(base_scenario.output_dir), + "output_dir_with_timestamp": str(run_dir), + } + try: + pointer.write_text(json.dumps(record, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") + logger.info("[cli] inspect persisted pointer=%s record=%s", pointer, record) + except OSError as exc: + logger.warning("[cli] cannot persist inspect pointer: %s", exc) + + +def _install_logger(log_file: Path | None) -> None: + """安装插件自身的 logger handler，不依赖 ais_bench 的 AISLogger。 + + 解析到 .log 文件时日志只写入文件、不在终端打印；否则回退为仅控制台输出， + 真实的错误信息在正常命令流程中抛出。 + """ + plugin_logger = logging.getLogger(PLUGIN_LOG_NAME) + for existing in plugin_logger.handlers: + existing.close() + plugin_logger.handlers.clear() + plugin_logger.propagate = False + plugin_logger.setLevel(logging.INFO) + if log_file is not None: + handler: logging.Handler = logging.FileHandler(log_file, mode="w") + else: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(LOG_NORMAL_FORMAT)) + plugin_logger.addHandler(handler) + + +def main(argv: list[str] | None = None) -> int: + """CLI 主入口：分发到对应子命令并统一处理错误码。""" + args = build_parser().parse_args(argv) + # inspect 每次生成新时间戳目录；prepare 优先复用最近一次成功 inspect 的 + # 时间戳目录，让 inspect / prepare / validate 的 .log 与 .json 落在同一目录。 + execution_timestamp: str | None = None + reused_inspect_timestamp = False + if args.command == "inspect": + execution_timestamp = new_execution_timestamp() + elif args.command == "prepare": + try: + reusable = _reusable_inspect_timestamp(load_scenario(args.scenario)) + except PrefixCacheError: + reusable = None + if reusable is not None: + execution_timestamp = reusable + reused_inspect_timestamp = True + else: + execution_timestamp = new_execution_timestamp() + log_file = _resolve_log_file( + args.command, + scenario_path=getattr(args, "scenario", None), + manifest_path=getattr(args, "manifest", None), + execution_timestamp=execution_timestamp, + ) + # 安装插件自身的 logger（日志只缓存到 .log 文件，不在终端打印）。 + _install_logger(log_file) + logger.info("[cli] command=%s args=%s log_file=%s reused_inspect_timestamp=%s", args.command, vars(args), log_file, reused_inspect_timestamp) + progress = PromptProgress() if args.command == "prepare" else None + try: + if args.command == "prepare": + logger.info("[cli] prepare scenario=%s overwrite=%s", args.scenario, args.overwrite) + paths = prepare_scenario( + args.scenario, + overwrite=args.overwrite, + progress=progress.update, + execution_timestamp=execution_timestamp, + ) + result = {key: str(value) for key, value in paths.__dict__.items()} + if log_file is not None: + result["log"] = str(log_file) + logger.info("[cli] prepare_scenario returned paths=%s", result) + print(json.dumps(result, ensure_ascii=False)) + elif args.command == "validate": + logger.info("[cli] validate manifest=%s", args.manifest) + result = validate_artifacts(args.manifest) + logger.info("[cli] validate_artifacts returned result=%s", result) + print(json.dumps(result, ensure_ascii=False)) + elif args.command == "inspect": + logger.info("[cli] inspect scenario=%s", args.scenario) + result = inspect_scenario(args.scenario) + if log_file is not None: + result["log"] = str(log_file) + # 写入复用指针，供后续 prepare/validate 复用同一时间戳目录。 + _persist_inspect_pointer(args.scenario, log_file, execution_timestamp) + logger.info("[cli] inspect_scenario returned result=%s", json.dumps(result, ensure_ascii=False)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + except PrefixCacheError as exc: + # 业务错误统一以 ERROR 输出并返回退出码 2，便于脚本判断。 + if progress is not None: + progress.close() + logger.warning("[cli] PrefixCacheError: %s", exc) + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +def console_main() -> None: + """控制台入口：把 main 的返回码作为进程退出码。""" + raise SystemExit(main()) + + +if __name__ == "__main__": + console_main() diff --git a/plugins/prefix_cache/ais_bench_prefix_cache/errors.py b/plugins/prefix_cache/ais_bench_prefix_cache/errors.py new file mode 100644 index 00000000..e49895a0 --- /dev/null +++ b/plugins/prefix_cache/ais_bench_prefix_cache/errors.py @@ -0,0 +1,14 @@ +class PrefixCacheError(Exception): + """Base user-facing plugin error.""" + + +class ScenarioValidationError(PrefixCacheError): + """Invalid scenario or source data.""" + + +class ArtifactValidationError(PrefixCacheError): + """Generated artifact is incomplete or inconsistent.""" + + +class PromptRoundTripError(ArtifactValidationError): + """Composed prompt tokens do not survive decode/re-encode.""" diff --git a/plugins/prefix_cache/ais_bench_prefix_cache/generation.py b/plugins/prefix_cache/ais_bench_prefix_cache/generation.py new file mode 100644 index 00000000..5a657294 --- /dev/null +++ b/plugins/prefix_cache/ais_bench_prefix_cache/generation.py @@ -0,0 +1,740 @@ +from __future__ import annotations + +import csv +import hashlib +import itertools +import json +import logging +import math +import random +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Any, Iterable, Protocol, Sequence + +from .errors import ArtifactValidationError, PromptRoundTripError, ScenarioValidationError + +logger = logging.getLogger(__name__) + + +class TokenizerLike(Protocol): + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: ... + def decode(self, token_ids: Sequence[int], skip_special_tokens: bool = False) -> str: ... + + +@dataclass(frozen=True) +class GSMRecord: + """语料（GSM8K）中的一条记录。""" + line_index: int # 源文件行号 + question: str # 规范化后的题干文本 + question_sha256: str # 题干的 SHA-256，用于去重/溯源 + + +@dataclass(frozen=True) +class CanonicalPrefix: + """某个 Prefix Group 的规范化共享前缀（由语料重复拼接并截断到所需长度）。""" + group_id: str + text: str # 前缀文本 + token_ids: tuple[int, ...] # 前缀 token 序列 + sha256: str # 前缀指纹 + gsm_indices: tuple[int, ...] # 来源语料行号 + gsm_hashes: tuple[str, ...] # 来源语料 hash + + +@dataclass(frozen=True) +class RequestPlan: + """一条请求的完整生成计划（构造 prompt 与模拟命中率的载体）。""" + request_id: str + sequence_index: int # 全局序号 + group_id: str # 所属 Prefix Group + occurrence_index_within_group: int # 组内出现次序 + dp_rank: int | None # cold 模式下路由到的 DP 卡 + lane_sequence: int | None # 该 lane 内的序号（用于串行放行） + target_input_tokens: int + actual_input_tokens: int # 实际构造出的输入 token 数 + max_tokens: int + shared_prefix_tokens: int # 共享前缀长度（求解器的核心决策量） + seed_tokens: int # 唯一 seed 块长度 + natural_suffix_tokens: int # 自然后缀长度 + question: str = "" + answer: str = "none" + gsm_indices: tuple[int, ...] = () + gsm_hashes: tuple[str, ...] = () + canonical_prefix_sha256: str = "" + seed_sha256: str = "" + request_random_seed: int = 0 + watermark_before: int = 0 # 模拟：请求到达前缓存水位 + theoretical_hit_tokens: int = 0 # 模拟：理论命中 token 数 + watermark_after: int = 0 # 模拟：请求后缓存水位 + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class TheorySummary: + """缓存水位模拟的汇总结果。""" + rows: tuple[RequestPlan, ...] + total_input_tokens: int + total_hit_tokens: int + global_hit_rate: float + group_stats: dict[str, dict[str, float | int]] + dp_stats: dict[int, dict[str, float | int]] + + +@dataclass(frozen=True) +class SolveResult: + """共享前缀长度求解的结果与可达性/偏差诊断。""" + shared_prefix_tokens: tuple[int, ...] # 每条请求的共享前缀长度（核心产物） + requested_hit_tokens: int # 目标命中 token 数 + effective_hit_tokens: int # 实际达到的命中 token 数 + effective_hit_rate: float + min_reachable_rate: float # 理论最低可达命中率（全不共享） + max_reachable_rate: float # 理论最高可达命中率（全共享） + target_reachable: bool # 目标命中率是否落在可达区间 + group_reachability: dict[str, dict[str, float]] # 各组的可达区间 + adjusted: bool # 是否因约束无法精确命中目标 + reason: str | None # 无法精确命中的原因 + + +def normalize_question(value: str) -> str: + """规范化题干：合并空白、去首尾空格。""" + return " ".join(value.strip().split()) + + +def load_gsm8k(path: Path, field: str = "question") -> list[GSMRecord]: + """逐行读取 GSM8K JSONL 语料并构造 GSMRecord 列表，校验空内容。""" + logger.info("[gen] load_gsm8k path=%s field=%s", path, field) + records: list[GSMRecord] = [] + try: + lines = path.read_text(encoding="utf-8-sig").splitlines() + except OSError as exc: + raise ScenarioValidationError(f"cannot read GSM8K {path}: {exc}") from exc + logger.info("[gen] load_gsm8k lines=%d", len(lines)) + for line_index, line in enumerate(lines): + try: + raw = json.loads(line) + question = normalize_question(raw[field]) + except (json.JSONDecodeError, KeyError, TypeError, AttributeError) as exc: + raise ScenarioValidationError(f"GSM8K line {line_index} is invalid: {exc}") from exc + if not question: + raise ScenarioValidationError(f"GSM8K line {line_index} has empty {field}") + digest = hashlib.sha256(question.encode("utf-8")).hexdigest() + records.append(GSMRecord(line_index, question, digest)) + if not records: + raise ScenarioValidationError("GSM8K corpus is empty") + logger.info("[gen] load_gsm8k records=%d first_line_index=%d first_sha256=%s", len(records), records[0].line_index, records[0].question_sha256) + return records + + +def select_gsm8k(records: Sequence[GSMRecord], config: dict[str, Any], count: int, seed: int) -> list[GSMRecord]: + """按 selection 配置（random/indices/question_sha256/mixed）选出 count 条语料。""" + logger.info("[gen] select_gsm8k mode=%s config=%s count=%d seed=%d", config["mode"], config, count, seed) + mode = config["mode"] + by_index = {item.line_index: item for item in records} + by_hash: dict[str, list[GSMRecord]] = {} + for item in records: + by_hash.setdefault(item.question_sha256, []).append(item) + if mode == "random": + rng = random.Random(seed) + selected: list[GSMRecord] = [] + while len(selected) < count: + cycle = list(records) + rng.shuffle(cycle) + selected.extend(cycle) + result = selected[:count] + logger.info("[gen] select_gsm8k mode=random selected=%d line_indices=%s", len(result), [item.line_index for item in result]) + return result + values = config.get("values") + if values is None: + values = config.get("indices" if mode == "indices" else "question_sha256", []) + logger.info("[gen] select_gsm8k values=%s", values) + if mode == "indices": + try: + selected = [by_index[int(value)] for value in values] + except (KeyError, ValueError, TypeError) as exc: + raise ScenarioValidationError(f"specified GSM8K index does not exist: {exc}") from exc + elif mode == "question_sha256": + selected = [] + for value in values: + matches = by_hash.get(str(value), []) + if len(matches) != 1: + raise ScenarioValidationError(f"GSM8K hash must resolve uniquely: {value}") + selected.append(matches[0]) + else: + # mixed 模式：合并 indices 与 question_sha256 两部分的选择结果。 + selected = [] + index_values = config.get("indices", []) + hash_values = config.get("question_sha256", []) + if index_values: + selected.extend(select_gsm8k(records, {"mode": "indices", "values": index_values}, len(index_values), seed)) + if hash_values: + selected.extend(select_gsm8k(records, {"mode": "question_sha256", "values": hash_values}, len(hash_values), seed)) + if not selected: + raise ScenarioValidationError("specified GSM8K selection is empty") + # 不足 count 时循环复用已选中的记录。 + result = [selected[i % len(selected)] for i in range(count)] + logger.info("[gen] select_gsm8k selected=%d line_indices=%s", len(result), [item.line_index for item in result]) + return result + + +def _csv_values(path: str, aliases: Sequence[str]) -> list[int]: + """从 CSV 读取指定别名列的整数值（用于显式长度配置）。""" + logger.info("[gen] _csv_values path=%s aliases=%s", path, list(aliases)) + with Path(path).open(encoding="utf-8-sig", newline="") as source: + rows = list(csv.DictReader(source)) + fieldnames = rows[0].keys() if rows else [] + column = next((name for name in aliases if name in fieldnames), None) + if column is None: + raise ScenarioValidationError(f"CSV requires one of columns {list(aliases)}") + values = [int(row[column]) for row in rows] + if not values or any(value < 1 for value in values): + raise ScenarioValidationError(f"CSV column {column} must contain positive integers") + logger.info("[gen] _csv_values rows=%d column=%s values=%s", len(rows), column, values) + return values + + +def _log_lengths(label: str, values: list[int]) -> list[int]: + """带日志返回长度序列（供各生成函数复用）。""" + logger.info( + "[gen] %s count=%d min=%d max=%d mean=%.2f values=%s", + label, len(values), min(values), max(values), sum(values) / len(values), values, + ) + return values + + +def build_input_lengths(config: dict[str, Any], count: int, seed: int) -> list[int]: + """按输入长度配置生成 count 条请求的输入长度序列。""" + mode = config["mode"] + logger.info("[gen] build_input_lengths mode=%s config=%s count=%d seed=%d", mode, config, count, seed) + if mode == "fixed": + return _log_lengths("build_input_lengths", [int(config["value"])] * count) + if mode == "explicit": + values = [int(value) for value in config["values"]] + if len(values) != count: + raise ScenarioValidationError("explicit input length count must equal requests.count") + return _log_lengths("build_input_lengths", values) + if mode == "csv": + values = _csv_values(config["path"], ("input_prompt_tokens", "content_tokens", "input_tokens")) + if len(values) != count: + raise ScenarioValidationError("input CSV row count must equal requests.count") + return _log_lengths("build_input_lengths", values) + if mode == "range": + rng = random.Random(seed) + return _log_lengths("build_input_lengths", [rng.randint(int(item["min"]), int(item["max"])) for item in config["ranges"] for _ in range(int(item["count"]))]) + return _truncated_normal_values(config, count, seed, "build_input_lengths") + + +def _truncated_normal_values(config: dict[str, Any], count: int, seed: int, label: str = "_truncated_normal_values") -> list[int]: + """从截断正态分布采样 count 个整数值（min<=x<=max），失败时抛校验错误。""" + logger.info("[gen] %s config=%s count=%d seed=%d", label, config, count, seed) + low, high = int(config["min"]), int(config["max"]) + if low == high: + return _log_lengths(label, [low] * count) + mean = float(config.get("mean", (low + high) / 2)) + std = float(config.get("std", max(1.0, (high - low) / 4))) + logger.info("[gen] %s low=%d high=%d mean=%.2f std=%.2f", label, low, high, mean, std) + rng = random.Random(seed) + values: list[int] = [] + attempts = 0 + while len(values) < count and attempts < max(1000, count * 100): + value = int(round(rng.gauss(mean, std))) + if low <= value <= high: + values.append(value) + attempts += 1 + if len(values) != count: + raise ScenarioValidationError("truncated_normal could not produce enough values") + logger.info("[gen] %s attempts=%d produced=%d", label, attempts, len(values)) + return _log_lengths(label, values) + + +def build_output_lengths(config: dict[str, Any], count: int, seed: int) -> list[int]: + """按输出长度配置生成 count 条请求的输出（max_tokens）长度序列。""" + mode = config["mode"] + logger.info("[gen] build_output_lengths mode=%s config=%s count=%d seed=%d", mode, config, count, seed) + if mode == "fixed": + return _log_lengths("build_output_lengths", [int(config["value"])] * count) + if mode == "csv": + values = _csv_values(config["path"], ("output_tokens",)) + if len(values) != count: + raise ScenarioValidationError("output CSV row count must equal requests.count") + return _log_lengths("build_output_lengths", values) + low, high = int(config["min"]), int(config["max"]) + rng = random.Random(seed) + if mode == "uniform": + return _log_lengths("build_output_lengths", [rng.randint(low, high) for _ in range(count)]) + return _truncated_normal_values(config, count, seed, "build_output_lengths") + + +def assign_groups(count: int, config: dict[str, Any], seed: int) -> list[str]: + """按权重把 count 条请求分配到各 Prefix Group，返回每条请求的 group_id。 + + 支持 uniform / zipf / weights 三种分配模式；用 Largest Remainder 法 + 把配额小数部分均摊到各分组，保证总数恰为 count。 + """ + logger.info("[gen] assign_groups count=%d config=%s seed=%d", count, config, seed) + group_count = int(config["count"]) + assignment = config["assignment"] + mode = assignment["mode"] + if mode == "uniform": + weights = [1.0] * group_count + elif mode == "zipf": + exponent = float(assignment.get("exponent", 1.0)) + if exponent <= 0: + raise ScenarioValidationError("zipf exponent must be positive") + weights = [1 / ((index + 1) ** exponent) for index in range(group_count)] + else: + weights = [float(value) for value in assignment.get("weights", [])] + if len(weights) != group_count or any(value < 0 for value in weights) or sum(weights) <= 0: + raise ScenarioValidationError("explicit group weights must match group count and sum positive") + logger.info("[gen] assign_groups group_count=%d mode=%s weights=%s", group_count, mode, weights) + total = sum(weights) + quotas = [count * value / total for value in weights] + allocations = [math.floor(value) for value in quotas] + remaining = count - sum(allocations) + logger.info("[gen] assign_groups quotas=%s allocations=%s remaining=%d", quotas, allocations, remaining) + # 按小数余量从大到小把剩余配额依次补齐到各分组。 + order = sorted(range(group_count), key=lambda index: (-(quotas[index] - allocations[index]), index)) + for index in order[:remaining]: + allocations[index] += 1 + groups = [f"group-{index}" for index, amount in enumerate(allocations) for _ in range(amount)] + if mode == "zipf": + # zipf 模式下打乱顺序，避免长尾集中在头部。 + random.Random(seed).shuffle(groups) + logger.info("[gen] assign_groups groups=%s distribution=%s", groups, {group: groups.count(group) for group in sorted(set(groups))}) + return groups + + +def order_indices(group_ids: Sequence[str], strategy: str, seed: int, input_lengths: Sequence[int] | None = None) -> list[int]: + """生成请求的发送顺序排列，控制 Prefix Cache 命中特性。 + + 策略：sequential（原序）/ global_shuffle（全局乱序）/ within_group_shuffle + （组内乱序）/ interleave（组间交错）/ input_len_asc（组内按输入长度升序）。 + """ + logger.info("[gen] order_indices count=%d strategy=%s seed=%d input_lengths_provided=%s", len(group_ids), strategy, seed, input_lengths is not None) + indices = list(range(len(group_ids))) + rng = random.Random(seed) + if strategy == "sequential": + logger.info("[gen] order_indices strategy=sequential result=%s", indices) + return indices + if strategy == "global_shuffle": + rng.shuffle(indices) + logger.info("[gen] order_indices strategy=global_shuffle result=%s", indices) + return indices + # 按分组把请求装进桶，再按策略重排桶内顺序。 + buckets: dict[str, list[int]] = {} + for index, group in enumerate(group_ids): + buckets.setdefault(group, []).append(index) + logger.info("[gen] order_indices buckets=%s", {group: len(members) for group, members in buckets.items()}) + if strategy == "input_len_asc": + if input_lengths is None or len(input_lengths) != len(group_ids): + raise ScenarioValidationError("input_len_asc requires one input length per request") + for group in buckets: + buckets[group].sort(key=lambda index: (int(input_lengths[index]), index)) + if strategy == "within_group_shuffle": + result: list[int] = [] + for group in sorted(buckets): + rng.shuffle(buckets[group]) + result.extend(buckets[group]) + logger.info("[gen] order_indices strategy=within_group_shuffle result=%s", result) + return result + # interleave：用 zip_longest 把各组的请求轮流取出，实现交错。 + result = [] + for row in itertools.zip_longest(*(buckets[group] for group in sorted(buckets))): + result.extend(index for index in row if index is not None) + logger.info("[gen] order_indices strategy=interleave result=%s", result) + return result + + +def assign_cold_routes(group_ids: Sequence[str], dp_size: int, explicit: Sequence[int] | None = None) -> tuple[list[int], list[int]]: + """cold 模式：为每条请求分派 DP rank 与 lane 序号。 + + 默认按组内出现次序轮转到各 DP 卡（group_round_robin）；显式 routes 可覆盖。 + lane 序列用于让同一 (group, rank) 上的请求按序发送。 + """ + logger.info("[gen] assign_cold_routes count=%d dp_size=%d explicit=%s", len(group_ids), dp_size, explicit) + if explicit is not None: + if len(explicit) != len(group_ids) or any(rank < 0 or rank >= dp_size for rank in explicit): + raise ScenarioValidationError("explicit DP routes are invalid") + ranks = list(explicit) + else: + # 每个组内第 k 条请求路由到 rank = k % dp_size。 + seen: dict[str, int] = {} + ranks = [] + for group in group_ids: + occurrence = seen.get(group, 0) + ranks.append(occurrence % dp_size) + seen[group] = occurrence + 1 + lane_seen: dict[tuple[str, int], int] = {} + lane_sequences = [] + for group, rank in zip(group_ids, ranks): + lane = (group, rank) + lane_sequences.append(lane_seen.get(lane, 0)) + lane_seen[lane] = lane_sequences[-1] + 1 + logger.info("[gen] assign_cold_routes ranks=%s lane_sequences=%s", ranks, lane_sequences) + return ranks, lane_sequences + + +def simulate_theory(plans: Sequence[RequestPlan], mode: str, warmup_watermarks: dict[str, int] | None = None, verbose: bool = True) -> TheorySummary: + """按缓存水位模拟理论命中率。 + + 对每条请求维护其缓存键（组 或 组×rank）的水位；命中 = min(共享前缀, 既有水位)， + 随后水位提升到该请求的共享前缀长度。最终汇总全局/分组/DP 的命中统计。 + """ + if verbose: + logger.info("[gen] simulate_theory plans=%d mode=%s warmup_watermarks=%s", len(plans), mode, warmup_watermarks) + watermarks: dict[object, int] = {} + if mode == "warmup": + watermarks.update(warmup_watermarks or {}) + rows: list[RequestPlan] = [] + group_totals: dict[str, list[int]] = {} + dp_totals: dict[int, list[int]] = {} + for plan in plans: + key: object = plan.group_id if mode == "warmup" else (plan.group_id, plan.dp_rank or 0) + before = watermarks.get(key, 0) + hit = min(plan.shared_prefix_tokens, before) + after = max(before, plan.shared_prefix_tokens) + watermarks[key] = after + row = replace(plan, watermark_before=before, theoretical_hit_tokens=hit, watermark_after=after) + rows.append(row) + if verbose: + logger.info("[gen] simulate_theory request_id=%s key=%s watermark_before=%d hit=%d watermark_after=%d", plan.request_id, key, before, hit, after) + group_totals.setdefault(plan.group_id, [0, 0]) + group_totals[plan.group_id][0] += plan.actual_input_tokens + group_totals[plan.group_id][1] += hit + if plan.dp_rank is not None: + dp_totals.setdefault(plan.dp_rank, [0, 0]) + dp_totals[plan.dp_rank][0] += plan.actual_input_tokens + dp_totals[plan.dp_rank][1] += hit + total_input = sum(row.actual_input_tokens for row in rows) + total_hit = sum(row.theoretical_hit_tokens for row in rows) + global_rate = total_hit / total_input if total_input else 0.0 + stats = lambda values: {"input_tokens": values[0], "hit_tokens": values[1], "hit_rate": values[1] / values[0] if values[0] else 0.0} + group_stats = {key: stats(value) for key, value in group_totals.items()} + dp_stats = {key: stats(value) for key, value in dp_totals.items()} + if verbose: + logger.info("[gen] simulate_theory total_input_tokens=%d total_hit_tokens=%d global_hit_rate=%.4f", total_input, total_hit, global_rate) + logger.info("[gen] simulate_theory group_stats=%s dp_stats=%s", group_stats, dp_stats) + return TheorySummary(tuple(rows), total_input, total_hit, global_rate, group_stats, dp_stats) + + +def _plans_for_prefixes(input_lengths: Sequence[int], output_lengths: Sequence[int], group_ids: Sequence[str], ranks: Sequence[int | None], lane_sequences: Sequence[int | None], prefixes: Sequence[int]) -> list[RequestPlan]: + """按给定的一组共享前缀长度构造临时 RequestPlan 列表（供求解器打分用）。""" + occurrences: dict[str, int] = {} + plans = [] + for index, (length, out_len, group, rank, lane_seq, prefix) in enumerate(zip(input_lengths, output_lengths, group_ids, ranks, lane_sequences, prefixes)): + occurrence = occurrences.get(group, 0) + occurrences[group] = occurrence + 1 + plans.append(RequestPlan(f"request-{index:08d}", index, group, occurrence, rank, lane_seq, length, length, out_len, prefix, 0, length - prefix)) + return plans + + +def solve_prefix_lengths(input_lengths: Sequence[int], output_lengths: Sequence[int], group_ids: Sequence[str], ranks: Sequence[int | None], lane_sequences: Sequence[int | None], block_size: int, minimum_non_shared_tokens: int, mode: str, target_hit_rate: float) -> SolveResult: + """求解每条请求的共享前缀长度（shared_prefix_tokens）。 + + 目标：让整体理论命中率尽量逼近配置的 target_hit_rate。由于共享前缀必须按 + block_size 对齐、且必须为每条请求保留 minimum_non_shared_tokens 的非共享区 + （seed + 自然后缀），再叠加 KV cache 水位约束，目标值不一定能精确达到。 + 求解器先计算可达性区间（min/max），再将目标钳制到最近的 Block 对齐命中量： + warmup 按请求容量分配，cold 按 (Prefix Group, DP rank) lane 线性构造精确解。 + """ + logger.info("[gen] solve_prefix_lengths requests=%d block_size=%d minimum_non_shared_tokens=%d mode=%s target_hit_rate=%.4f", len(input_lengths), block_size, minimum_non_shared_tokens, mode, target_hit_rate) + logger.info("[gen] solve_prefix_lengths input_lengths=%s output_lengths=%s group_ids=%s ranks=%s lane_sequences=%s", list(input_lengths), list(output_lengths), list(group_ids), list(ranks), list(lane_sequences)) + # 每条请求的共享前缀长度候选集：必须是 block_size 的整数倍（前缀要按 block 对齐 + # 才能命中缓存），且最大只能取到 (length - minimum_non_shared_tokens) 向下对齐的值， + # 从而保证每条请求都留足非共享区（seed + 自然后缀）。 + candidates = [list(range(0, max(0, ((length - minimum_non_shared_tokens) // block_size) * block_size) + 1, block_size)) for length in input_lengths] + logger.info("[gen] solve_prefix_lengths candidates=%s", candidates) + total_input = sum(input_lengths) + # 把目标命中率换算成目标命中 token 总数（四舍五入），作为搜索的靶心。 + target_tokens = int(total_input * target_hit_rate + 0.5) + logger.info("[gen] solve_prefix_lengths total_input=%d target_tokens=%d", total_input, target_tokens) + + def score(prefixes: Sequence[int]) -> tuple[int, int]: + """给出一组共享前缀长度，模拟缓存水位后返回 (命中误差, 命中token数)。""" + # 按候选前缀构造临时请求计划，并模拟真实缓存水位下的理论命中 token 数。 + plans = _plans_for_prefixes(input_lengths, output_lengths, group_ids, ranks, lane_sequences, prefixes) + # warmup 模式下用各组前缀最大值作为预热水位，模拟缓存已被写满。 + warm = {group: max((prefix for prefix, current in zip(prefixes, group_ids) if current == group), default=0) for group in set(group_ids)} if mode == "warmup" else None + hit = simulate_theory(plans, mode, warm, verbose=False).total_hit_tokens + return abs(hit - target_tokens), hit + + # 先评估可达性上下界。旧实现先做局部搜索、最后才计算边界，导致目标高于 + # reachable_max 时仍可能停在次优局部解。边界现在是求解输入而非事后报告。 + zero_prefixes = [0] * len(candidates) + zero_plans = _plans_for_prefixes(input_lengths, output_lengths, group_ids, ranks, lane_sequences, zero_prefixes) + zero_warm = {group: 0 for group in set(group_ids)} if mode == "warmup" else None + zero_theory = simulate_theory(zero_plans, mode, zero_warm) + zero_hit = zero_theory.total_hit_tokens + max_prefixes = [values[-1] for values in candidates] + max_plans = _plans_for_prefixes(input_lengths, output_lengths, group_ids, ranks, lane_sequences, max_prefixes) + max_warm = { + group: max((prefix for prefix, current in zip(max_prefixes, group_ids) if current == group), default=0) + for group in set(group_ids) + } if mode == "warmup" else None + max_theory = simulate_theory(max_plans, mode, max_warm) + max_hit = max_theory.total_hit_tokens + + # 所有共享前缀和理论命中量都是 block_size 的整数倍。按 Block 单位选取距 + # target_tokens 最近、且落在可达边界内的目标命中量。 + max_hit_units = max_hit // block_size + lower_units = max(0, min(max_hit_units, target_tokens // block_size)) + upper_units = max(0, min(max_hit_units, lower_units + 1)) + desired_hit_units = min( + {lower_units, upper_units, 0, max_hit_units}, + key=lambda units: (abs(units * block_size - target_tokens), units), + ) + desired_hit_tokens = desired_hit_units * block_size + + prefixes = [0] * len(candidates) + caps = [values[-1] // block_size for values in candidates] + remaining_units = desired_hit_units + if mode == "warmup": + # warmup 后每个前缀 token 都命中；因此把目标 Block 数按请求容量依次分配即可。 + for index, cap in enumerate(caps): + assigned = min(cap, remaining_units) + prefixes[index] = assigned * block_size + remaining_units -= assigned + else: + # cold 模式按 (group, DP rank) 独立维护水位。对任一 lane： + # lane_hit = sum(prefix_i) - max(prefix_i) + # 选容量最大的请求作为 anchor；把所需 hit Block 分配给其余请求，再让 + # anchor 等于这些请求中的最大前缀。于是 anchor 的前缀恰好抵消 max 项， + # lane_hit 精确等于已分配 Block 数。每个 0..lane_max 区间都可构造，无需爬山。 + lanes: dict[tuple[str, int], list[int]] = {} + for index, (group, rank) in enumerate(zip(group_ids, ranks)): + lanes.setdefault((group, int(rank or 0)), []).append(index) + for lane_indices in lanes.values(): + anchor = max(lane_indices, key=lambda index: (caps[index], -index)) + lane_capacity = sum(caps[index] for index in lane_indices if index != anchor) + lane_units = min(lane_capacity, remaining_units) + lane_remaining = lane_units + for index in lane_indices: + if index == anchor: + continue + assigned = min(caps[index], lane_remaining) + prefixes[index] = assigned * block_size + lane_remaining -= assigned + if lane_remaining: + raise ArtifactValidationError("cold Prefix Cache lane construction did not consume its target") + prefixes[anchor] = max((prefixes[index] for index in lane_indices if index != anchor), default=0) + remaining_units -= lane_units + if remaining_units: + raise ArtifactValidationError("Prefix Cache solver could not construct the selected reachable target") + + best_error, best_hit = score(prefixes) + if best_hit != desired_hit_tokens: + raise ArtifactValidationError( + f"Prefix Cache solver constructed {best_hit} hit tokens; expected {desired_hit_tokens}" + ) + logger.info( + "[gen] solve_prefix_lengths strategy=exact_lane_construction desired_hit=%d chosen_prefixes=%s best_error=%d best_hit=%d", + desired_hit_tokens, prefixes, best_error, best_hit, + ) + # 实际/最低/最高可达的命中率。 + effective_rate = best_hit / total_input if total_input else 0.0 + min_rate = zero_hit / total_input if total_input else 0.0 + max_rate = max_hit / total_input if total_input else 0.0 + logger.info("[gen] solve_prefix_lengths zero_hit=%d zero_rate=%.4f max_hit=%d max_rate=%.4f effective_hit=%d effective_rate=%.4f", zero_hit, min_rate, max_hit, max_rate, best_hit, effective_rate) + # 按组给出可达命中率区间，便于定位是哪个组导致目标不可达。 + group_reachability = { + group: { + "min_reachable_rate": float(zero_theory.group_stats[group]["hit_rate"]), + "max_reachable_rate": float(max_theory.group_stats[group]["hit_rate"]), + } + for group in sorted(set(group_ids)) + } + # 目标命中率落在 [min_rate, max_rate] 区间内即为可达。 + target_reachable = min_rate <= target_hit_rate <= max_rate + # 无法精确命中目标时标记 adjusted，并区分目标越界与单纯 Block 对齐残差。 + adjusted = best_hit != target_tokens + if target_tokens > max_hit: + reason = "target exceeds maximum reachable hit rate" + elif target_tokens < zero_hit: + reason = "target is below minimum reachable hit rate" + elif adjusted: + reason = "block alignment prevents an exact target hit count" + else: + reason = None + logger.info("[gen] solve_prefix_lengths group_reachability=%s target_reachable=%s adjusted=%s reason=%s", group_reachability, target_reachable, adjusted, reason) + return SolveResult( + tuple(prefixes), target_tokens, best_hit, effective_rate, min_rate, max_rate, + target_reachable, group_reachability, adjusted, + reason, + ) + + +def _safe_token_text(tokenizer: TokenizerLike, token_id: int, special: set[int]) -> str | None: + """判定某个 token 是否是"边界安全"的：单独 decode 且前后加字都不改变编码。""" + if token_id in special: + return None + text = tokenizer.decode([token_id], skip_special_tokens=False) + if not text: + return None + # 单独编回、左侧拼接、右侧拼接都必须保持该 token 不变。 + if tokenizer.encode(text, add_special_tokens=False) != [token_id]: + return None + if tokenizer.encode("X" + text, add_special_tokens=False)[-1:] != [token_id]: + return None + if tokenizer.encode(text + "X", add_special_tokens=False)[:1] != [token_id]: + return None + return text + + +def find_boundary_safe_token_ids(tokenizer: TokenizerLike, minimum: int) -> list[int]: + # Prefer space-prefixed tokens: in BPE tokenizers they cannot merge with + # preceding text, so seeds built from them stay stable at every junction. + vocab_size = len(tokenizer) # type: ignore[arg-type] + logger.info("[gen] find_boundary_safe_token_ids minimum=%d vocab_size=%d", minimum, vocab_size) + special = set(getattr(tokenizer, "all_special_ids", [])) + preferred: list[int] = [] + fallback: list[int] = [] + for token_id in range(vocab_size): + text = _safe_token_text(tokenizer, token_id, special) + if text is None: + continue + # 优先收集空格开头的 token（BPE 下与前置文本不会合并，seed 更稳定）。 + if text.startswith(" "): + preferred.append(token_id) + if len(preferred) >= minimum: + logger.info("[gen] find_boundary_safe_token_ids preferred=%s", preferred) + return preferred + else: + fallback.append(token_id) + combined = preferred + fallback + if len(combined) < minimum: + raise ArtifactValidationError(f"tokenizer has only {len(combined)} boundary-safe tokens; need {minimum}") + result = combined[:minimum] + logger.info("[gen] find_boundary_safe_token_ids preferred=%d fallback=%d result=%s", len(preferred), len(fallback), result) + return result + + +def _seed_round_trips(tokenizer: TokenizerLike, seed: Sequence[int]) -> bool: + """校验 seed token 序列 decode 后再 encode 能原样恢复（round-trip 安全）。""" + text = tokenizer.decode(seed, skip_special_tokens=False) + return tokenizer.encode(text, add_special_tokens=False) == list(seed) + + +def build_unique_seed(tokenizer: TokenizerLike | None, safe_ids: Sequence[int], request_id: str, seed_length: int, random_seed: int, exclude: set[tuple[int, ...]] | None = None) -> tuple[int, ...]: + """构造一个全局唯一且 round-trip 安全的 seed token 序列（长度 seed_length）。 + + 用 SHA-256 派生的字节流从 safe_ids 中抽样；若与已用 seed 重复或无法 + round-trip，则换 nonce 重试。 + """ + logger.info("[gen] build_unique_seed request_id=%s seed_length=%d random_seed=%d safe_ids=%d exclude=%d", request_id, seed_length, random_seed, len(safe_ids), len(exclude) if exclude else 0) + if seed_length < 1 or len(safe_ids) < 2: + raise ArtifactValidationError("seed generation requires positive length and at least two safe tokens") + used = exclude if exclude is not None else set() + for nonce in range(4096): + digest = hashlib.sha256(f"{random_seed}:{request_id}:{nonce}".encode()).digest() + stream = itertools.cycle(digest) + seed = tuple(safe_ids[next(stream) % len(safe_ids)] for _ in range(seed_length)) + if seed in used: + logger.info("[gen] build_unique_seed retry request_id=%s nonce=%d reason=duplicate_seed", request_id, nonce) + continue + if tokenizer is not None and not _seed_round_trips(tokenizer, seed): + logger.info("[gen] build_unique_seed retry request_id=%s nonce=%d reason=round_trip_failure", request_id, nonce) + continue + logger.info("[gen] build_unique_seed request_id=%s nonce=%d seed=%s", request_id, nonce, seed) + return seed + raise ArtifactValidationError(f"unable to construct a unique round-trip-safe seed for {request_id}") + + +def build_unique_seed_tokens(safe_ids: Sequence[int], request_ids: Sequence[str], seed_length: int, random_seed: int, tokenizer: TokenizerLike | None = None) -> dict[str, tuple[int, ...]]: + """为一批 request_id 批量构造互不重复的唯一 seed，返回 {request_id: seed}。""" + logger.info("[gen] build_unique_seed_tokens request_ids=%d seed_length=%d random_seed=%d", len(request_ids), seed_length, random_seed) + result: dict[str, tuple[int, ...]] = {} + used: set[tuple[int, ...]] = set() + for request_id in request_ids: + seed = build_unique_seed(tokenizer, safe_ids, request_id, seed_length, random_seed, used) + used.add(seed) + result[request_id] = seed + logger.info("[gen] build_unique_seed_tokens result keys=%d", len(result)) + return result + + +def _repeat_tokens(records: Sequence[GSMRecord], tokenizer: TokenizerLike, target: int) -> tuple[list[int], tuple[int, ...], tuple[str, ...]]: + """循环拼接语料记录直至 token 数达到 target，返回 tokens 及来源索引/hash。""" + logger.info("[gen] _repeat_tokens records=%d target=%d", len(records), target) + tokens: list[int] = [] + indices: list[int] = [] + hashes: list[str] = [] + for record in itertools.cycle(records): + piece = tokenizer.encode((" " if tokens else "") + record.question, add_special_tokens=False) + if not piece: + continue + tokens.extend(piece) + indices.append(record.line_index) + hashes.append(record.question_sha256) + if len(tokens) >= target: + logger.info("[gen] _repeat_tokens result tokens=%d indices=%d hashes=%d", len(tokens[:target]), len(indices), len(hashes)) + return tokens[:target], tuple(indices), tuple(hashes) + raise ArtifactValidationError("cannot build tokens from empty GSM8K records") + + +def build_canonical_prefixes(tokenizer: TokenizerLike, group_sources: dict[str, Sequence[GSMRecord]], max_lengths: dict[str, int], block_size: int) -> dict[str, CanonicalPrefix]: + """为每个 Prefix Group 构造 canonical 前缀（语料重复拼接至组内最大共享长度）。 + + 各组的首个 block 必须互不相同，否则无法区分组；冲突时用确定性组标记兜底。 + """ + logger.info("[gen] build_canonical_prefixes groups=%s max_lengths=%s block_size=%d", sorted(group_sources), max_lengths, block_size) + result: dict[str, CanonicalPrefix] = {} + first_blocks: set[tuple[int, ...]] = set() + for group_position, group in enumerate(sorted(group_sources)): + source_records = list(group_sources[group]) + if not source_records: + raise ArtifactValidationError(f"canonical prefix source is empty for {group}") + token_ids = indices = hashes = None + # 尝试不同旋转起点，找到首个 block 不与其他组冲突的版本。 + for offset in range(len(source_records)): + rotated = source_records[offset:] + source_records[:offset] + candidate_tokens, candidate_indices, candidate_hashes = _repeat_tokens( + rotated, tokenizer, max(max_lengths[group], block_size) + ) + if tuple(candidate_tokens[:block_size]) not in first_blocks: + token_ids, indices, hashes = candidate_tokens, candidate_indices, candidate_hashes + logger.info("[gen] build_canonical_prefixes group=%s accepted rotation offset=%d", group, offset) + break + if token_ids is None: + # Explicitly duplicated corpus selections can make every source rotation + # identical. Add a deterministic group marker only in that collision case + # so one bad override cannot abort the whole dataset generation. + logger.info("[gen] build_canonical_prefixes group=%s all rotations collide -> adding deterministic marker", group) + marker = tokenizer.encode(f"{group_position} prefix-cache-group-{group} ", add_special_tokens=False) + source_tokens, source_indices, source_hashes = _repeat_tokens( + source_records, tokenizer, max(max_lengths[group], block_size) + ) + token_ids = marker + source_tokens + indices, hashes = source_indices, source_hashes + if tuple(token_ids[:block_size]) in first_blocks: + raise ArtifactValidationError(f"canonical prefixes collide in first block for {group} after deterministic fallback") + first_block = tuple(token_ids[:block_size]) + first_blocks.add(first_block) + text = tokenizer.decode(token_ids, skip_special_tokens=False) + actual = tokenizer.encode(text, add_special_tokens=False) + # 校验前缀 decode/re-encode 后不变（round-trip 安全）。 + if actual[:max_lengths[group]] != token_ids[:max_lengths[group]]: + raise ArtifactValidationError(f"canonical prefix does not round-trip for {group}") + digest = hashlib.sha256(bytes(str(token_ids), "utf-8")).hexdigest() + result[group] = CanonicalPrefix(group, text, tuple(token_ids), digest, indices, hashes) + logger.info("[gen] build_canonical_prefixes group=%s tokens=%d text_len=%d sha256=%s gsm_indices=%s", group, len(token_ids), len(text), digest, indices) + return result + + +def build_prompt(tokenizer: TokenizerLike, canonical: CanonicalPrefix, shared_prefix_tokens: int, seed: Sequence[int], suffix_records: Sequence[GSMRecord], target_tokens: int) -> tuple[str, tuple[int, ...], tuple[int, ...], tuple[str, ...]]: + """按 共享前缀 + 唯一seed + 自然后缀 拼接出目标长度的 prompt 文本。 + + 返回 (文本, 实际token, 后缀来源索引, 后缀来源hash)，并校验 round-trip。 + """ + logger.info("[gen] build_prompt group=%s shared_prefix_tokens=%d seed_len=%d target_tokens=%d suffix_records=%d", canonical.group_id, shared_prefix_tokens, len(seed), target_tokens, len(suffix_records)) + suffix_len = target_tokens - shared_prefix_tokens - len(seed) + logger.info("[gen] build_prompt suffix_len=%d", suffix_len) + if suffix_len < 0: + raise ArtifactValidationError("prefix and seed exceed target input length") + suffix, indices, hashes = _repeat_tokens(suffix_records, tokenizer, suffix_len) if suffix_len else ([], (), ()) + expected = list(canonical.token_ids[:shared_prefix_tokens]) + list(seed) + suffix + text = tokenizer.decode(expected, skip_special_tokens=False) + actual = tokenizer.encode(text, add_special_tokens=False) + logger.info("[gen] build_prompt group=%s expected_tokens=%d actual_tokens=%d text_len=%d suffix_indices=%d", canonical.group_id, len(expected), len(actual), len(text), len(indices)) + if actual != expected: + raise PromptRoundTripError("prompt token layout changed after decode/re-encode") + return text, tuple(actual), indices, hashes diff --git a/plugins/prefix_cache/ais_bench_prefix_cache/pipeline.py b/plugins/prefix_cache/ais_bench_prefix_cache/pipeline.py new file mode 100644 index 00000000..b79b441d --- /dev/null +++ b/plugins/prefix_cache/ais_bench_prefix_cache/pipeline.py @@ -0,0 +1,502 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import logging +import math +import tempfile +from dataclasses import replace +from pathlib import Path +from typing import Any, Callable + +from . import __version__ +from .artifacts import ArtifactPaths, artifact_paths, read_jsonl, sha256_file, validate_artifacts, write_json, write_jsonl +from .errors import ArtifactValidationError, PromptRoundTripError +from .generation import ( + RequestPlan, + assign_cold_routes, + assign_groups, + build_canonical_prefixes, + build_input_lengths, + build_output_lengths, + build_prompt, + build_unique_seed, + build_unique_seed_tokens, + find_boundary_safe_token_ids, + load_gsm8k, + order_indices, + select_gsm8k, + simulate_theory, + solve_prefix_lengths, +) +from .scenario import Scenario, load_scenario, new_execution_timestamp, with_execution_timestamp + +logger = logging.getLogger(__name__) + + +def _tokenizer_loader(scenario: Scenario): + """按场景配置加载 HuggingFace AutoTokenizer，作为默认 tokenizer 加载器。""" + try: + from transformers import AutoTokenizer + except ImportError as exc: + raise ArtifactValidationError("transformers is required to load the configured tokenizer") from exc + cfg = scenario.section("tokenizer") + logger.info("[prepare] _tokenizer_loader path=%s revision=%s trust_remote_code=%s", cfg["path"], cfg.get("revision"), cfg.get("trust_remote_code", False)) + return AutoTokenizer.from_pretrained(cfg["path"], revision=cfg.get("revision"), trust_remote_code=cfg.get("trust_remote_code", False)) + + +def _sha256_json(value: Any) -> str: + """对任意 JSON 可序列化值做规范化后计算 SHA-256（键排序、紧凑分隔符）。""" + encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _request_random_seed(global_seed: int, request_id: str) -> int: + """由全局种子 + 请求 id 派生每条请求的确定性随机种子（可复现）。""" + digest = hashlib.sha256(f"{global_seed}:{request_id}".encode("utf-8")).digest() + result = int.from_bytes(digest[:8], "big") + logger.info("[prepare] _request_random_seed global_seed=%d request_id=%s -> %d", global_seed, request_id, result) + return result + + +def _percentile(sorted_values: list[int], percentile: float) -> float: + """对已排序序列做线性插值分位数（支持单元素序列）。""" + if len(sorted_values) == 1: + return float(sorted_values[0]) + position = (len(sorted_values) - 1) * percentile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return float(sorted_values[lower]) + fraction = position - lower + return sorted_values[lower] * (1 - fraction) + sorted_values[upper] * fraction + + +def _length_summary(values: list[int]) -> dict[str, Any]: + """生成长度分布的摘要：min/max/mean/分位数，并分为最多 10 桶直方图。 + + 每个 bin 的 min/max 是该桶内实际取值的最小/最大值（count == 1 时二者相等）； + 桶按固定宽度 [bin_low, bin_high] 划分（最多 10 桶），仅输出非空桶。 + """ + logger.info("[prepare] _length_summary values_count=%d", len(values)) + ordered = sorted(values) + low, high = ordered[0], ordered[-1] + width = max(1, math.ceil((high - low + 1) / 10)) + bins: dict[tuple[int, int], dict[str, int]] = {} + for value in ordered: + bin_low = low + ((value - low) // width) * width + bounds = (bin_low, min(high, bin_low + width - 1)) + slot = bins.get(bounds) + if slot is None: + bins[bounds] = slot = {"min": value, "max": value, "count": 0} + slot["min"] = min(slot["min"], value) + slot["max"] = max(slot["max"], value) + slot["count"] += 1 + result = { + "min": low, + "max": high, + "mean": sum(ordered) / len(ordered), + "p50": _percentile(ordered, 0.50), + "p90": _percentile(ordered, 0.90), + "p95": _percentile(ordered, 0.95), + "p99": _percentile(ordered, 0.99), + "bins": [ + {"min": slot["min"], "max": slot["max"], "count": slot["count"]} + for _, slot in sorted(bins.items()) + ], + } + logger.info("[prepare] _length_summary result=%s", result) + return result + + +def _tokenizer_manifest(tokenizer: Any, effective: dict[str, Any], block_size: int) -> dict[str, Any]: + """生成 tokenizer 的指纹信息（路径/类/词表/特殊 id），用于工件溯源与一致性校验。""" + special_ids = sorted(int(value) for value in getattr(tokenizer, "all_special_ids", [])) + fingerprint_source = { + "path": effective["tokenizer"]["path"], + "revision": effective["tokenizer"].get("revision"), + "class": f"{tokenizer.__class__.__module__}.{tokenizer.__class__.__qualname__}", + "vocab_size": len(tokenizer), + "special_token_ids": special_ids, + } + logger.info("[prepare] _tokenizer_manifest tokenizer_class=%s block_size=%d fingerprint_source=%s", fingerprint_source["class"], block_size, fingerprint_source) + result = fingerprint_source | { + "block_size": block_size, + "fingerprint_sha256": _sha256_json(fingerprint_source), + } + logger.info("[prepare] _tokenizer_manifest result=%s", result) + return result + + +def _build_prompt_with_seed_retry(tokenizer: Any, canonical: Any, prefix_len: int, seeds: dict[str, tuple[int, ...]], request_id: str, rotated_pool: list[Any], target_tokens: int, safe_ids: list[int], seed_length: int, random_seed: int): + """构造 prompt；若 round-trip 失败则换一个唯一 seed 重试，最多 64 次。""" + logger.info("[prepare] _build_prompt_with_seed_retry request_id=%s prefix_len=%d target_tokens=%d seed_length=%d random_seed=%d rotated_pool=%d", request_id, prefix_len, target_tokens, seed_length, random_seed, len(rotated_pool)) + for attempt in range(64): + try: + result = build_prompt(tokenizer, canonical, prefix_len, seeds[request_id], rotated_pool, target_tokens) + logger.info("[prepare] _build_prompt_with_seed_retry request_id=%s attempt=%d ok", request_id, attempt) + return result + except PromptRoundTripError: + # seed 可能导致 decode/re-encode 后 token 布局变化，重新生成一个不重复的 seed。 + logger.info("[prepare] _build_prompt_with_seed_retry request_id=%s attempt=%d PromptRoundTripError -> regenerating seed", request_id, attempt) + seeds[request_id] = build_unique_seed(tokenizer, safe_ids, request_id, seed_length, random_seed + attempt * 10007 + 1, set(seeds.values())) + raise ArtifactValidationError(f"unable to construct a round-trip-safe prompt for {request_id}") + + +def prepare_scenario( + path: Path | str, + overwrite: bool | None = None, + tokenizer_loader: Callable[[Scenario], Any] | None = None, + progress: Callable[[int, int], None] | None = None, + execution_timestamp: str | None = None, +) -> ArtifactPaths: + """准备阶段：从场景配置生成全部请求工件（full/requests/manifest/analysis）。 + + 流程：解析配置 → 生成输入/输出长度、分组 → 应用 group override → 排序 → + cold 路由 → 求解共享前缀 → 构造 canonical 前缀与每条 prompt → 理论命中率模拟 → + 落盘工件并校验。整个过程不发任何网络请求。 + """ + logger.info("[prepare] prepare_scenario path=%s overwrite=%s tokenizer_loader=%s", path, overwrite, tokenizer_loader) + scenario = with_execution_timestamp(load_scenario(path), execution_timestamp or new_execution_timestamp()) + logger.info("[prepare] scenario run_id=%s random_seed=%d cache_mode=%s dp_size=%d block_size=%d output_dir=%s source_path=%s", scenario.run_id, scenario.random_seed, scenario.cache_mode, scenario.dp_size, scenario.block_size, scenario.output_dir, scenario.source_path) + effective = scenario.to_effective_dict() + logger.info("[prepare] effective keys=%s", sorted(effective)) + overwrite = effective["run"].get("overwrite", False) if overwrite is None else overwrite + request_cfg = effective["requests"] + pc_cfg = effective["prefix_cache"] + corpus_cfg = effective["corpus"] + logger.info("[prepare] overwrite=%s request_cfg=%s", overwrite, request_cfg) + logger.info("[prepare] pc_cfg=%s", pc_cfg) + logger.info("[prepare] corpus_cfg=%s", corpus_cfg) + count = request_cfg["count"] + seed = scenario.random_seed + logger.info("[prepare] count=%d seed=%d", count, seed) + # 阶段1：按配置生成输入/输出长度序列，并按权重把请求分配到各 Prefix Group。 + input_lengths = build_input_lengths(request_cfg["input_length"], count, seed) + output_lengths = build_output_lengths(request_cfg["output_length"], count, seed + 1) + groups = assign_groups(count, pc_cfg["groups"], seed + 2) + logger.info("[prepare] input_lengths=%s", input_lengths) + logger.info("[prepare] output_lengths=%s", output_lengths) + logger.info("[prepare] groups=%s distribution=%s", groups, {group: groups.count(group) for group in sorted(set(groups))}) + records = load_gsm8k(Path(corpus_cfg["path"]), corpus_cfg["field"]) + logger.info("[prepare] records=%d", len(records)) + # 阶段2：应用每个组的 override（独立输入/输出长度、语料选择），构造各组的语料池。 + overrides = pc_cfg["groups"].get("overrides", {}) + logger.info("[prepare] overrides=%s", overrides) + group_pools: dict[str, list[Any]] = {} + for group_index, group in enumerate(sorted(set(groups))): + group_positions = [index for index, value in enumerate(groups) if value == group] + logger.info("[prepare] group=%s group_index=%d positions_count=%d positions=%s", group, group_index, len(group_positions), group_positions) + override = overrides.get(group, {}) + logger.info("[prepare] group=%s override=%s", group, override) + if "input_length" in override: + values = build_input_lengths(override["input_length"], len(group_positions), seed + 100 + group_index) + if len(values) != len(group_positions): + raise ArtifactValidationError(f"{group} input_length generated {len(values)} values; expected {len(group_positions)}") + for position, value in zip(group_positions, values): + input_lengths[position] = value + logger.info("[prepare] group=%s input_length override values=%s -> input_lengths=%s", group, values, input_lengths) + if "output_length" in override: + values = build_output_lengths(override["output_length"], len(group_positions), seed + 200 + group_index) + for position, value in zip(group_positions, values): + output_lengths[position] = value + logger.info("[prepare] group=%s output_length override values=%s -> output_lengths=%s", group, values, output_lengths) + selection = override.get("corpus_selection", corpus_cfg["selection"]) + logger.info("[prepare] group=%s selection=%s", group, selection) + group_pools[group] = select_gsm8k(records, selection, max(2, len(group_positions)), seed + 300 + group_index) + logger.info("[prepare] group_pools sizes=%s", {group: len(pool) for group, pool in group_pools.items()}) + # 阶段3：按配置策略重排请求顺序，长度/分组随之对齐。 + ordering = order_indices(groups, pc_cfg["order"]["strategy"], seed + 4, input_lengths) + logger.info("[prepare] ordering=%s", ordering) + input_lengths = [input_lengths[index] for index in ordering] + output_lengths = [output_lengths[index] for index in ordering] + groups = [groups[index] for index in ordering] + logger.info("[prepare] after ordering input_lengths=%s", input_lengths) + logger.info("[prepare] after ordering output_lengths=%s", output_lengths) + logger.info("[prepare] after ordering groups=%s", groups) + # 阶段4：cold 模式给每条请求分派 DP rank 与 lane；warmup 模式无需路由。 + if scenario.cache_mode == "cold": + ranks_raw, lane_raw = assign_cold_routes(groups, scenario.dp_size) + ranks: list[int | None] = ranks_raw + lanes: list[int | None] = lane_raw + logger.info("[prepare] cache_mode=cold dp_size=%d ranks=%s lanes=%s", scenario.dp_size, ranks, lanes) + else: + ranks = [None] * count + lanes = [None] * count + logger.info("[prepare] cache_mode=%s -> ranks=lanes=None for all %d requests", scenario.cache_mode, count) + seed_length = scenario.block_size * pc_cfg["seed_blocks"] + minimum_non_shared_length = pc_cfg["minimum_non_shared_length"] + logger.info("[prepare] seed_length=%d minimum_non_shared_length=%d", seed_length, minimum_non_shared_length) + # 阶段5：求解每条请求的共享前缀长度，使理论命中率逼近目标。 + solve = solve_prefix_lengths(input_lengths, output_lengths, groups, ranks, lanes, scenario.block_size, minimum_non_shared_length, scenario.cache_mode, pc_cfg["target_hit_rate"]) + logger.info("[prepare] solve shared_prefix_tokens=%s", solve.shared_prefix_tokens) + logger.info("[prepare] solve requested_hit_tokens=%d effective_hit_tokens=%d effective_hit_rate=%.4f min_reachable_rate=%.4f max_reachable_rate=%.4f target_reachable=%s adjusted=%s reason=%s", solve.requested_hit_tokens, solve.effective_hit_tokens, solve.effective_hit_rate, solve.min_reachable_rate, solve.max_reachable_rate, solve.target_reachable, solve.adjusted, solve.reason) + max_by_group = {group: max((prefix for prefix, current in zip(solve.shared_prefix_tokens, groups) if current == group), default=0) for group in sorted(set(groups))} + group_sources = {group: group_pools[group] for group in sorted(set(groups))} + logger.info("[prepare] max_by_group=%s", max_by_group) + logger.info("[prepare] group_sources sizes=%s", {group: len(pool) for group, pool in group_sources.items()}) + # 阶段6：加载 tokenizer，为每个组构造 canonical 前缀并选出边界安全的 seed token。 + tokenizer = (tokenizer_loader or _tokenizer_loader)(scenario) + logger.info("[prepare] tokenizer=%s vocab_size=%d", f"{tokenizer.__class__.__module__}.{tokenizer.__class__.__qualname__}", len(tokenizer)) + canonical = build_canonical_prefixes(tokenizer, group_sources, max_by_group, scenario.block_size) + logger.info("[prepare] canonical=%s", {group: {"sha256": item.sha256, "tokens": len(item.token_ids), "gsm_indices": list(item.gsm_indices), "gsm_hashes": list(item.gsm_hashes)} for group, item in canonical.items()}) + safe_ids = find_boundary_safe_token_ids(tokenizer, max(2, min(64, len(tokenizer)))) + logger.info("[prepare] safe_ids count=%d safe_ids=%s", len(safe_ids), safe_ids) + # 阶段7：为每条请求生成唯一 seed，再按 shared_prefix + seed + 自然后缀拼 prompt。 + request_ids = [f"request-{index:08d}" for index in range(count)] + request_random_seeds = { + request_id: _request_random_seed(seed + 5, request_id) + for request_id in request_ids + } + logger.info("[prepare] request_ids count=%d first=%s last=%s", len(request_ids), request_ids[0], request_ids[-1]) + logger.info("[prepare] request_random_seeds=%s", request_random_seeds) + seeds: dict[str, tuple[int, ...]] = {} + used_seeds: set[tuple[int, ...]] = set() + for request_id in request_ids: + request_seed = build_unique_seed( + tokenizer, safe_ids, request_id, seed_length, + request_random_seeds[request_id], used_seeds, + ) + used_seeds.add(request_seed) + seeds[request_id] = request_seed + logger.info("[prepare] request_id=%s seed=%s used_seeds=%d", request_id, request_seed, len(used_seeds)) + plans: list[RequestPlan] = [] + occurrences: dict[str, int] = {} + if progress is not None: + progress(0, count) + for index, request_id in enumerate(request_ids): + group = groups[index] + occurrence = occurrences.get(group, 0) + occurrences[group] = occurrence + 1 + prefix_len = solve.shared_prefix_tokens[index] + pool = group_pools[group] + # 组内循环轮换语料池，保证不同请求的后缀不同但前缀可共享。 + rotated_pool = pool[occurrence % len(pool):] + pool[:occurrence % len(pool)] + logger.info("[prepare] build plan index=%d request_id=%s group=%s occurrence=%d prefix_len=%d pool_size=%d rotation_offset=%d", index, request_id, group, occurrence, prefix_len, len(pool), occurrence % len(pool)) + text, tokens, suffix_indices, suffix_hashes = _build_prompt_with_seed_retry(tokenizer, canonical[group], prefix_len, seeds, request_id, rotated_pool, input_lengths[index], safe_ids, seed_length, request_random_seeds[request_id]) + seed_hash = hashlib.sha256(str(seeds[request_id]).encode()).hexdigest() + plan = RequestPlan( + request_id, index, group, occurrence, ranks[index], lanes[index], input_lengths[index], len(tokens), output_lengths[index], prefix_len, seed_length, len(tokens) - prefix_len - seed_length, + text, "none", suffix_indices, suffix_hashes, canonical[group].sha256, seed_hash, + request_random_seed=request_random_seeds[request_id], + ) + plans.append(plan) + logger.info("[prepare] plan request_id=%s sequence_index=%d group=%s occurrence=%d dp_rank=%s lane=%s target_input_tokens=%d actual_input_tokens=%d max_tokens=%d shared_prefix_tokens=%d seed_tokens=%d natural_suffix_tokens=%d seed_sha256=%s", plan.request_id, plan.sequence_index, plan.group_id, plan.occurrence_index_within_group, plan.dp_rank, plan.lane_sequence, plan.target_input_tokens, plan.actual_input_tokens, plan.max_tokens, plan.shared_prefix_tokens, plan.seed_tokens, plan.natural_suffix_tokens, plan.seed_sha256) + if progress is not None: + progress(index + 1, count) + warm_watermarks = max_by_group if scenario.cache_mode == "warmup" else None + logger.info("[prepare] warm_watermarks=%s", warm_watermarks) + # 阶段8：按缓存水位模拟理论命中率，并生成 full/requests 行。 + theory = simulate_theory(plans, scenario.cache_mode, warm_watermarks) + logger.info("[prepare] theory total_input_tokens=%d total_hit_tokens=%d global_hit_rate=%.4f", theory.total_input_tokens, theory.total_hit_tokens, theory.global_hit_rate) + logger.info("[prepare] theory group_stats=%s dp_stats=%s", theory.group_stats, theory.dp_stats) + full_rows = [ + row.to_dict() | { + "theoretical_hit_rate": row.theoretical_hit_tokens / row.actual_input_tokens if row.actual_input_tokens else 0.0, + "divergence_block_sha256": row.seed_sha256, + "divergence_unique": True, + "collision_status": "pass", + } + for row in theory.rows + ] + request_rows = [{"question": row.question, "answer": row.answer, "max_tokens": row.max_tokens} for row in theory.rows] + logger.info("[prepare] full_rows=%d request_rows=%d first_full_row=%s", len(full_rows), len(request_rows), full_rows[0]) + paths = artifact_paths(scenario.output_dir, scenario.run_id) + logger.info("[prepare] paths full=%s requests=%s manifest=%s analysis=%s", paths.full, paths.requests, paths.manifest, paths.analysis) + # 阶段9：落盘 full/requests 工件，并在 warmup 模式下生成预热计划。 + write_jsonl(paths.full, full_rows, overwrite) + write_jsonl(paths.requests, request_rows, overwrite) + warmup_plan = [] + if scenario.cache_mode == "warmup": + # warmup 模式：为每个 (group, rank) 生成一条预热请求，把缓存前缀写满。 + warm_ids = [f"warmup:{group}:{rank}" for group in sorted(canonical) for rank in range(scenario.dp_size)] + logger.info("[prepare] warmup warm_ids=%s", warm_ids) + warm_seeds = build_unique_seed_tokens(safe_ids, warm_ids, seed_length, seed + 6, tokenizer) + logger.info("[prepare] warmup warm_seeds=%s", warm_seeds) + for group in sorted(canonical): + for rank in range(scenario.dp_size): + request_id = f"warmup:{group}:{rank}" + prompt, tokens, _, _ = _build_prompt_with_seed_retry(tokenizer, canonical[group], max_by_group[group], warm_seeds, request_id, [], max_by_group[group] + seed_length, safe_ids, seed_length, seed + 6) + warmup_plan.append({"request_id": request_id, "group_id": group, "dp_rank": rank, "prompt": prompt, "input_tokens": len(tokens), "shared_prefix_tokens": max_by_group[group], "max_tokens": 1, "included_in_formal_statistics": False}) + logger.info("[prepare] warmup plan item=%s", warmup_plan[-1]) + # 阶段10：生成告警（目标不可达 / 命中率偏差）并写 analysis.json。 + signed_difference_pp = (theory.global_hit_rate - pc_cfg["target_hit_rate"]) * 100 + absolute_difference_pp = abs(signed_difference_pp) + logger.info("[prepare] signed_difference_pp=%.4f absolute_difference_pp=%.4f", signed_difference_pp, absolute_difference_pp) + warnings = [] + if not solve.target_reachable: + warnings.append({ + "code": "TARGET_UNREACHABLE", + "requested_target_hit_rate": pc_cfg["target_hit_rate"], + "reachable_min": solve.min_reachable_rate, + "reachable_max": solve.max_reachable_rate, + }) + if absolute_difference_pp > effective["validation"]["target_warning_pp"]: + warnings.append({"code": "TARGET_DEVIATION", "difference_pp": absolute_difference_pp}) + validation_status = "PASS" if not warnings else "PASS_WITH_WARNING" + logger.info("[prepare] warnings=%s validation_status=%s", warnings, validation_status) + analysis = { + "schema_version": "1.0", + "run_id": scenario.run_id, + "status": "prepared", + "requested_target_hit_rate": pc_cfg["target_hit_rate"], + "effective_target_hit_rate": solve.effective_hit_rate, + "theoretical_hit_rate": theory.global_hit_rate, + "target_difference_pp": absolute_difference_pp, + "target_signed_difference_pp": signed_difference_pp, + "target_absolute_difference_pp": absolute_difference_pp, + "validation": { + "status": validation_status, + "target_reachable": solve.target_reachable, + "warning_only": True, + "affects_exit_code": False, + }, + "theory": {"input_tokens": theory.total_input_tokens, "hit_tokens": theory.total_hit_tokens, "groups": theory.group_stats, "dp": theory.dp_stats}, + "warnings": warnings, + } + logger.info("[prepare] analysis=%s", analysis) + write_json(paths.analysis, analysis, overwrite) + manifest_effective = copy.deepcopy(effective) + configured_api_key = bool(manifest_effective["service"].pop("api_key", "")) + manifest_effective["service"]["api_key_configured"] = configured_api_key + logger.info("[prepare] configured_api_key=%s", configured_api_key) + manifest = { + "schema_version": "1.0", + "plugin_version": __version__, + "run_id": scenario.run_id, + "scenario_path": str(scenario.source_path), + "scenario_sha256": sha256_file(scenario.source_path), + "effective_config": manifest_effective, + "effective_config_sha256": _sha256_json(manifest_effective), + "corpus_sha256": sha256_file(Path(corpus_cfg["path"])), + "tokenizer": _tokenizer_manifest(tokenizer, effective, scenario.block_size), + "requests": { + "count": count, + "total_input_tokens": theory.total_input_tokens, + "input_length_summary": _length_summary(input_lengths), + "output_length_summary": _length_summary(output_lengths), + }, + "prefix_cache": { + "mode": scenario.cache_mode, + "requested_target_hit_rate": pc_cfg["target_hit_rate"], + "effective_target_hit_rate": solve.effective_hit_rate, + "theoretical_hit_rate": theory.global_hit_rate, + "reachable_min": solve.min_reachable_rate, + "reachable_max": solve.max_reachable_rate, + "target_reachable": solve.target_reachable, + "minimum_non_shared_length": minimum_non_shared_length, + "adjusted": solve.adjusted, + "reason": solve.reason, + "validation_status": validation_status, + "target_signed_difference_pp": signed_difference_pp, + "target_absolute_difference_pp": absolute_difference_pp, + }, + "groups": { + group: { + "canonical_prefix_sha256": item.sha256, + "canonical_prefix_tokens": len(item.token_ids), + "max_shared_prefix_tokens": max_by_group[group], + "gsm_indices": list(item.gsm_indices), + "gsm_question_sha256": list(item.gsm_hashes), + "reachable_min": solve.group_reachability[group]["min_reachable_rate"], + "reachable_max": solve.group_reachability[group]["max_reachable_rate"], + "theoretical_hit_rate": theory.group_stats[group]["hit_rate"], + } + for group, item in canonical.items() + }, + "dp": {"size": scenario.dp_size, "cold_route_strategy": "group_round_robin" if scenario.cache_mode == "cold" else None}, + "warmup": {"enabled": scenario.cache_mode == "warmup", "plan": warmup_plan}, + "divergence": { + "strategy": "globally_unique_seed_block", + "unique_request_blocks": len({row.seed_sha256 for row in theory.rows}), + "request_count": count, + "collision_status": "pass", + }, + "artifacts": { + "full": {"name": paths.full.name, "path": str(paths.full.resolve()), "rows": count, "bytes": paths.full.stat().st_size, "sha256": sha256_file(paths.full)}, + "requests": {"name": paths.requests.name, "path": str(paths.requests.resolve()), "rows": count, "bytes": paths.requests.stat().st_size, "sha256": sha256_file(paths.requests)}, + "analysis": {"name": paths.analysis.name, "path": str(paths.analysis.resolve()), "bytes": paths.analysis.stat().st_size, "sha256_at_prepare": sha256_file(paths.analysis)}, + }, + } + logger.info("[prepare] manifest=%s", json.dumps(manifest, ensure_ascii=False)) + write_json(paths.manifest, manifest, overwrite) + validate_artifacts(paths.manifest) + logger.info("[prepare] prepare_scenario done paths=%s", {key: str(value) for key, value in paths.__dict__.items()}) + return paths + + +def inspect_scenario(path: Path | str, tokenizer_loader: Callable[[Scenario], Any] | None = None) -> dict[str, Any]: + """Generate a read-only summary in a temporary directory without sending requests. + + 只预览场景：在临时目录中改 run_id/output_dir 后复用 prepare 流程生成工件， + 但只返回统计摘要（分组分布、长度分布、可达命中率等），不发任何真实请求。 + """ + logger.info("[inspect] inspect_scenario path=%s tokenizer_loader=%s", path, tokenizer_loader) + scenario = load_scenario(path) + logger.info("[inspect] scenario run_id=%s cache_mode=%s dp_size=%d block_size=%d source_path=%s", scenario.run_id, scenario.cache_mode, scenario.dp_size, scenario.block_size, scenario.source_path) + effective = scenario.to_effective_dict() + logger.info("[inspect] effective keys=%s", sorted(effective)) + # tokenizer 相对路径且场景目录下存在本地副本时，改用本地路径，避免远程加载。 + tokenizer_path = Path(effective["tokenizer"]["path"]) + local_tokenizer = scenario.source_path.parent / tokenizer_path + logger.info("[inspect] tokenizer_path=%s local_tokenizer=%s", tokenizer_path, local_tokenizer) + if not tokenizer_path.is_absolute() and local_tokenizer.exists(): + effective["tokenizer"]["path"] = str(local_tokenizer.resolve()) + logger.info("[inspect] tokenizer path is relative and local copy exists -> resolved tokenizer path=%s", effective["tokenizer"]["path"]) + else: + logger.info("[inspect] tokenizer path kept as-is: is_absolute=%s local_exists=%s", tokenizer_path.is_absolute(), local_tokenizer.exists()) + with tempfile.TemporaryDirectory(prefix="aisbench-prefix-cache-inspect-") as folder: + logger.info("[inspect] temporary folder=%s", folder) + root = Path(folder) + # 改写 run_id/output_dir，让 prepare 的产物落到临时目录且不覆盖任何真实工件。 + effective["run"]["run_id"] = "inspect" + effective["run"]["output_dir"] = str(root / "artifacts") + effective["run"]["overwrite"] = False + logger.info("[inspect] effective run overrides run_id=%s output_dir=%s overwrite=%s", effective["run"]["run_id"], effective["run"]["output_dir"], effective["run"]["overwrite"]) + temporary_scenario = root / "scenario.json" + temporary_scenario.write_text(json.dumps(effective, ensure_ascii=False), encoding="utf-8") + logger.info("[inspect] temporary_scenario=%s", temporary_scenario) + paths = prepare_scenario(temporary_scenario, tokenizer_loader=tokenizer_loader) + logger.info("[inspect] prepare_scenario paths=%s", {key: str(value) for key, value in paths.__dict__.items()}) + manifest = json.loads(paths.manifest.read_text(encoding="utf-8")) + logger.info("[inspect] manifest keys=%s run_id=%s plugin_version=%s", sorted(manifest), manifest["run_id"], manifest["plugin_version"]) + rows = read_jsonl(paths.full) + logger.info("[inspect] rows=%d first_row=%s", len(rows), rows[0] if rows else None) + group_counts: dict[str, int] = {} + dp_counts: dict[str, int] = {} + for row in rows: + # 统计各分组与各 DP rank 的请求数量。 + group_counts[row["group_id"]] = group_counts.get(row["group_id"], 0) + 1 + if row["dp_rank"] is not None: + key = str(row["dp_rank"]) + dp_counts[key] = dp_counts.get(key, 0) + 1 + logger.info("[inspect] group_counts=%s", group_counts) + logger.info("[inspect] dp_counts=%s", dp_counts) + input_lengths = [int(row["actual_input_tokens"]) for row in rows] + output_lengths = [int(row["max_tokens"]) for row in rows] + prefix = manifest["prefix_cache"] + logger.info("[inspect] input_lengths=%s total=%d", input_lengths, sum(input_lengths)) + logger.info("[inspect] output_lengths=%s total=%d", output_lengths, sum(output_lengths)) + logger.info("[inspect] prefix manifest=%s", prefix) + result = { + "run_id": scenario.run_id, + "mode": prefix["mode"], + "requested_target_hit_rate": prefix["requested_target_hit_rate"], + "effective_target_hit_rate": prefix["effective_target_hit_rate"], + "theoretical_hit_rate": prefix["theoretical_hit_rate"], + "reachable_min": prefix["reachable_min"], + "reachable_max": prefix["reachable_max"], + "target_reachable": prefix["target_reachable"], + "group_reachability": { + group: {"reachable_min": value["reachable_min"], "reachable_max": value["reachable_max"]} + for group, value in manifest["groups"].items() + }, + "groups": group_counts, + "input_tokens": manifest["requests"]["input_length_summary"] | {"total": sum(input_lengths)}, + "output_tokens": manifest["requests"]["output_length_summary"] | {"total": sum(output_lengths)}, + "dp_route_counts": dp_counts, + "sends_requests": False, + } + logger.info("[inspect] result=%s", json.dumps(result, ensure_ascii=False)) + return result diff --git a/plugins/prefix_cache/ais_bench_prefix_cache/scenario.py b/plugins/prefix_cache/ais_bench_prefix_cache/scenario.py new file mode 100644 index 00000000..58edc43c --- /dev/null +++ b/plugins/prefix_cache/ais_bench_prefix_cache/scenario.py @@ -0,0 +1,402 @@ +from __future__ import annotations + +import copy +import csv +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from .errors import ScenarioValidationError + + +_ALLOWED = { + "": {"schema_version", "run", "tokenizer", "corpus", "requests", "prefix_cache", "service", "validation", "aisbench"}, + "run": {"run_id", "random_seed", "output_dir", "overwrite"}, + "tokenizer": {"path", "block_size", "revision", "trust_remote_code"}, + "corpus": {"path", "field", "selection"}, + "corpus.selection": {"mode", "values", "indices", "question_sha256"}, + "requests": {"count", "input_length", "output_length"}, + "requests.input_length": {"mode", "value", "values", "ranges", "min", "max", "mean", "std", "path"}, + "requests.output_length": {"mode", "value", "min", "max", "mean", "std", "path"}, + "prefix_cache": {"mode", "target_hit_rate", "seed_blocks", "minimum_non_shared_length", "groups", "order"}, + "prefix_cache.groups": {"count", "assignment", "overrides"}, + "prefix_cache.groups.assignment": {"mode", "exponent", "weights"}, + "prefix_cache.order": {"strategy"}, + "service": {"inference_url", "metrics_url", "reset_url", "model", "dp_size", "assume_empty_cache", "engine_label_map", "timeout_seconds", "api_key"}, + "validation": {"target_warning_pp", "actual_warning_pp"}, + "aisbench": {"config", "work_dir", "extra_args"}, +} + +_MODES = { + "input": {"fixed", "explicit", "range", "truncated_normal", "csv"}, + "output": {"fixed", "uniform", "truncated_normal", "csv"}, + "selection": {"random", "indices", "question_sha256", "mixed"}, + "assignment": {"uniform", "zipf", "weights"}, + "order": {"sequential", "within_group_shuffle", "interleave", "global_shuffle", "input_len_asc"}, + "cache": {"cold", "warmup"}, +} + + +def _require_dict(value: Any, path: str) -> dict[str, Any]: + """校验 value 必须是 dict（JSON 对象），否则抛校验错误，原样返回。""" + if not isinstance(value, dict): + raise ScenarioValidationError(f"{path or 'scenario'} must be an object") + return value + + +def _strict_keys(value: dict[str, Any], path: str) -> None: + """递归校验 dict 的键是否都在白名单 _ALLOWED 内，拒绝未知字段。""" + allowed = _ALLOWED.get(path) + if allowed is not None: + unknown = sorted(set(value) - allowed) + if unknown: + prefix = f"{path}." if path else "" + raise ScenarioValidationError(f"unknown field: {prefix}{unknown[0]}") + for key, child in value.items(): + child_path = f"{path}.{key}" if path else key + if child_path in _ALLOWED: + _strict_keys(_require_dict(child, child_path), child_path) + + +def _positive(value: Any, path: str) -> int: + """校验 value 是正整数（排除 bool），返回原值。""" + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ScenarioValidationError(f"{path} must be a positive integer") + return value + + +def _mode(section: dict[str, Any], allowed: set[str], path: str) -> str: + """校验 section 的 mode 取值必须在 allowed 集合内，返回 mode。""" + value = section.get("mode") + if value not in allowed: + raise ScenarioValidationError(f"{path}.mode must be one of {sorted(allowed)}") + return value + + +def _validate_input_config(config: dict[str, Any], path: str, base: Path, expected_count: int | None) -> None: + """按 mode 校验输入长度配置（fixed/explicit/range/truncated_normal/csv），并解析 csv 路径。""" + mode = _mode(config, _MODES["input"], path) + unknown = set(config) - {"mode", "value", "values", "ranges", "min", "max", "mean", "std", "path"} + if unknown: + raise ScenarioValidationError(f"unknown field: {path}.{sorted(unknown)[0]}") + if mode == "fixed": + _positive(config.get("value"), f"{path}.value") + elif mode == "explicit": + # 显式列表：逐项校验为正整数，且数量须等于请求总数。 + values = config.get("values") + if not isinstance(values, list) or not values: + raise ScenarioValidationError(f"{path}.values must be a non-empty list") + for index, value in enumerate(values): + _positive(value, f"{path}.values[{index}]") + if expected_count is not None and len(values) != expected_count: + raise ScenarioValidationError(f"{path}.values length must equal expected request count") + elif mode == "range": + # 区间抽样：每段校验 min/max/count，且总数须等于请求总数。 + ranges = config.get("ranges") + if not isinstance(ranges, list) or not ranges: + raise ScenarioValidationError(f"{path}.ranges must be a non-empty list") + total = 0 + for index, item in enumerate(ranges): + if not isinstance(item, dict) or set(item) - {"min", "max", "count"}: + raise ScenarioValidationError(f"{path}.ranges[{index}] has invalid fields") + low = _positive(item.get("min"), f"{path}.ranges[{index}].min") + high = _positive(item.get("max"), f"{path}.ranges[{index}].max") + if high < low: + raise ScenarioValidationError(f"{path}.ranges[{index}].max must be >= min") + total += _positive(item.get("count"), f"{path}.ranges[{index}].count") + if expected_count is not None and total != expected_count: + raise ScenarioValidationError(f"{path} range counts must equal expected request count") + elif mode == "truncated_normal": + # 截断正态：校验 min/max 区间与 std>0。 + low = _positive(config.get("min"), f"{path}.min") + high = _positive(config.get("max"), f"{path}.max") + if high < low: + raise ScenarioValidationError(f"{path}.max must be >= min") + if "std" in config and float(config["std"]) <= 0: + raise ScenarioValidationError(f"{path}.std must be positive") + else: + # csv 模式：要求 path 非空并解析为绝对路径。 + if not isinstance(config.get("path"), str) or not config["path"]: + raise ScenarioValidationError(f"{path}.path must be a non-empty string") + config["path"] = _resolve_path(base, config["path"]) + + +def _validate_output_config(config: dict[str, Any], path: str, base: Path) -> None: + """按 mode 校验输出长度配置（fixed/uniform/truncated_normal/csv），并解析 csv 路径。""" + mode = _mode(config, _MODES["output"], path) + unknown = set(config) - {"mode", "value", "min", "max", "mean", "std", "path"} + if unknown: + raise ScenarioValidationError(f"unknown field: {path}.{sorted(unknown)[0]}") + if mode == "fixed": + _positive(config.get("value"), f"{path}.value") + elif mode in {"uniform", "truncated_normal"}: + low = _positive(config.get("min"), f"{path}.min") + high = _positive(config.get("max"), f"{path}.max") + if high < low: + raise ScenarioValidationError(f"{path}.max must be >= min") + if mode == "truncated_normal" and "std" in config and float(config["std"]) <= 0: + raise ScenarioValidationError(f"{path}.std must be positive") + else: + if not isinstance(config.get("path"), str) or not config["path"]: + raise ScenarioValidationError(f"{path}.path must be a non-empty string") + config["path"] = _resolve_path(base, config["path"]) + + +def _minimum_input_tokens(config: dict[str, Any], path: str) -> int: + """计算输入长度配置能产生的最小 token 数（用于校验非共享区约束）。""" + mode = config["mode"] + if mode == "fixed": + return int(config["value"]) + if mode == "explicit": + return min(int(value) for value in config["values"]) + if mode == "range": + return min(int(item["min"]) for item in config["ranges"]) + if mode == "truncated_normal": + return int(config["min"]) + # csv 模式：读取文件并取 input 长度列的最小值。 + try: + with Path(config["path"]).open(encoding="utf-8-sig", newline="") as source: + rows = list(csv.DictReader(source)) + except OSError as exc: + raise ScenarioValidationError(f"{path} CSV cannot be read: {exc}") from exc + aliases = ("input_prompt_tokens", "content_tokens", "input_tokens") + if not rows: + raise ScenarioValidationError(f"{path} CSV must contain at least one data row") + column = next((name for name in aliases if name in rows[0]), None) + if column is None: + raise ScenarioValidationError(f"{path} CSV requires one of columns {list(aliases)}") + try: + return min(int(row[column]) for row in rows) + except (KeyError, TypeError, ValueError) as exc: + raise ScenarioValidationError(f"{path} CSV contains an invalid input length: {exc}") from exc + + +@dataclass(frozen=True) +class Scenario: + """校验通过后的场景对象：保存源文件路径与规范化后的有效配置。""" + + source_path: Path # 场景 JSON 文件绝对路径 + data: dict[str, Any] # 规范化后的完整配置（含默认值） + + @property + def run_id(self) -> str: + return self.data["run"]["run_id"] + + @property + def random_seed(self) -> int: + return self.data["run"]["random_seed"] + + @property + def output_dir(self) -> Path: + return Path(self.data["run"]["output_dir"]) + + @property + def block_size(self) -> int: + return self.data["tokenizer"]["block_size"] + + @property + def cache_mode(self) -> str: + return self.data["prefix_cache"]["mode"] + + @property + def dp_size(self) -> int: + return self.data["service"]["dp_size"] + + def section(self, name: str) -> dict[str, Any]: + """按段名取配置，如 scenario.section("service")。""" + return self.data[name] + + def to_effective_dict(self) -> dict[str, Any]: + """返回一份深拷贝的有效配置，调用方可安全修改而不影响内部数据。""" + return copy.deepcopy(self.data) + + +def new_execution_timestamp() -> str: + """Return a filename-safe local timestamp with second resolution.""" + return datetime.now().strftime("%Y%m%d_%H%M%S") + + +def with_execution_timestamp(scenario: Scenario, timestamp: str) -> Scenario: + """Append one execution timestamp to both run_id and output_dir.""" + if len(timestamp) != 15 or timestamp[8] != "_" or not (timestamp[:8] + timestamp[9:]).isdigit(): + raise ScenarioValidationError("execution timestamp must use YYYYMMDD_HHMMSS") + data = scenario.to_effective_dict() + data["run"]["run_id"] = f"{scenario.run_id}_{timestamp}" + output_dir = scenario.output_dir + if not output_dir.name: + raise ScenarioValidationError("run.output_dir must have a final directory name") + data["run"]["output_dir"] = str(output_dir.parent / f"{output_dir.name}_{timestamp}") + return Scenario(scenario.source_path, data) + + +def _resolve_path(base: Path, value: str) -> str: + """把配置里的路径解析为绝对路径：相对路径以 base 为基准。""" + path = Path(value) + return str((base / path).resolve() if not path.is_absolute() else path.resolve()) + + +def _validate(raw: dict[str, Any], source: Path) -> dict[str, Any]: + """对原始场景 dict 做完整语义校验与默认值填充，返回规范化副本。 + + 校验缺失/未知字段、类型与取值约束、路径解析、prefix cache 相关的一致性 + （如非共享区下限、分组覆盖 id、cold 多 DP 的地址要求），并原地补默认值。 + """ + _strict_keys(raw, "") + # 深拷贝后再修改，避免污染调用方的原始数据。 + data = copy.deepcopy(raw) + # Scenario 允许省略配置值；默认值与 config_examples/scenario.example.json + # 保持一致。多态配置（range/csv 等）只在整个配置缺失或 fixed 模式下 + # 填默认，避免给其他 mode 注入不合法的 fixed 字段。 + data.setdefault("schema_version", "1.0") + data.setdefault("run", {}) + data.setdefault("tokenizer", {}) + data.setdefault("corpus", {}) + data.setdefault("requests", {}) + data.setdefault("prefix_cache", {}) + data.setdefault("service", {}) + data.setdefault("validation", {}) + data.setdefault("aisbench", {}) + run = _require_dict(data["run"], "run") + tokenizer = _require_dict(data["tokenizer"], "tokenizer") + corpus = _require_dict(data["corpus"], "corpus") + requests = _require_dict(data["requests"], "requests") + pc = _require_dict(data["prefix_cache"], "prefix_cache") + service = _require_dict(data["service"], "service") + run.setdefault("run_id", "gsm8k-prefix-cache-60") + run.setdefault("random_seed", 42) + run.setdefault("output_dir", "./outputs/gsm8k-prefix-cache-60") + tokenizer.setdefault("path", "/home/weights/Qwen3.6-27B") + tokenizer.setdefault("block_size", 16) + corpus.setdefault("path", "./GSM8K.jsonl") + requests.setdefault("count", 100) + input_cfg = requests.setdefault("input_length", {"mode": "fixed", "value": 1024}) + if isinstance(input_cfg, dict): + input_cfg.setdefault("mode", "fixed") + if input_cfg["mode"] == "fixed": + input_cfg.setdefault("value", 1024) + output_cfg = requests.setdefault("output_length", {"mode": "fixed", "value": 32}) + if isinstance(output_cfg, dict): + output_cfg.setdefault("mode", "fixed") + if output_cfg["mode"] == "fixed": + output_cfg.setdefault("value", 32) + pc.setdefault("mode", "warmup") + pc.setdefault("target_hit_rate", 0.6) + pc.setdefault("seed_blocks", 1) + groups_cfg = pc.setdefault("groups", {"count": 1, "assignment": {"mode": "uniform"}}) + if isinstance(groups_cfg, dict): + groups_cfg.setdefault("count", 1) + assignment_cfg = groups_cfg.setdefault("assignment", {"mode": "uniform"}) + if isinstance(assignment_cfg, dict): + assignment_cfg.setdefault("mode", "uniform") + order_cfg = pc.setdefault("order", {"strategy": "interleave"}) + if isinstance(order_cfg, dict): + order_cfg.setdefault("strategy", "interleave") + service.setdefault("inference_url", "http://127.0.0.1:8000/v1/completions") + service.setdefault("metrics_url", "http://127.0.0.1:8000/metrics") + service.setdefault("reset_url", "http://127.0.0.1:8000/reset_prefix_cache") + service.setdefault("model", "model-name") + service.setdefault("dp_size", 2) + service.setdefault("assume_empty_cache", False) + if data["schema_version"] != "1.0": + raise ScenarioValidationError("schema_version must be '1.0'") + if not isinstance(run.get("run_id"), str) or not run["run_id"].strip(): + raise ScenarioValidationError("run.run_id must be a non-empty string") + if isinstance(run.get("random_seed"), bool) or not isinstance(run.get("random_seed"), int): + raise ScenarioValidationError("run.random_seed must be an integer") + run.setdefault("overwrite", False) + run["output_dir"] = _resolve_path(source.parent, run["output_dir"]) + tokenizer["block_size"] = _positive(tokenizer.get("block_size"), "tokenizer.block_size") + tokenizer.setdefault("revision", None) + tokenizer.setdefault("trust_remote_code", False) + corpus.setdefault("field", "question") + corpus["path"] = _resolve_path(source.parent, corpus["path"]) + selection = corpus.setdefault("selection", {"mode": "random"}) + if isinstance(selection, dict): + selection.setdefault("mode", "random") + _mode(selection, _MODES["selection"], "corpus.selection") + count = _positive(requests.get("count"), "requests.count") + input_cfg = _require_dict(input_cfg, "requests.input_length") + output_cfg = _require_dict(output_cfg, "requests.output_length") + _validate_input_config(input_cfg, "requests.input_length", source.parent, count) + _validate_output_config(output_cfg, "requests.output_length", source.parent) + cache_mode = _mode(pc, _MODES["cache"], "prefix_cache") + target = pc.get("target_hit_rate") + if isinstance(target, bool) or not isinstance(target, (int, float)) or not 0 <= target <= 1: + raise ScenarioValidationError("prefix_cache.target_hit_rate must be in [0, 1]") + pc["seed_blocks"] = _positive(pc.get("seed_blocks", 1), "prefix_cache.seed_blocks") + # 非共享区下限 = seed 长度，且须保证输入长度能容纳该非共享区。 + seed_tokens = tokenizer["block_size"] * pc["seed_blocks"] + pc["minimum_non_shared_length"] = _positive( + pc.get("minimum_non_shared_length", seed_tokens), + "prefix_cache.minimum_non_shared_length", + ) + if pc["minimum_non_shared_length"] < seed_tokens: + raise ScenarioValidationError( + f"prefix_cache.minimum_non_shared_length must be at least seed length {seed_tokens}" + ) + reserved_tokens = pc["minimum_non_shared_length"] + if _minimum_input_tokens(input_cfg, "requests.input_length") < reserved_tokens: + raise ScenarioValidationError( + f"requests.input_length must be at least {reserved_tokens} tokens to contain the configured non-shared region" + ) + groups = _require_dict(pc.get("groups"), "prefix_cache.groups") + groups["count"] = _positive(groups.get("count"), "prefix_cache.groups.count") + assignment = groups.setdefault("assignment", {"mode": "uniform"}) + _mode(assignment, _MODES["assignment"], "prefix_cache.groups.assignment") + overrides = groups.setdefault("overrides", {}) + if not isinstance(overrides, dict): + raise ScenarioValidationError("prefix_cache.groups.overrides must be an object") + for group_id, override in overrides.items(): + # 校验 override 的 id 必须是合法 group-N 且未越界，再校验其字段。 + expected_group_id = group_id.startswith("group-") and group_id[6:].isdigit() and int(group_id[6:]) < groups["count"] + if not expected_group_id: + raise ScenarioValidationError(f"invalid Prefix Group override id: {group_id}") + if not isinstance(override, dict): + raise ScenarioValidationError(f"prefix_cache.groups.overrides.{group_id} must be an object") + unknown = set(override) - {"input_length", "output_length", "corpus_selection"} + if unknown: + raise ScenarioValidationError(f"unknown field: prefix_cache.groups.overrides.{group_id}.{sorted(unknown)[0]}") + if "input_length" in override: + _validate_input_config(override["input_length"], f"prefix_cache.groups.overrides.{group_id}.input_length", source.parent, None) + if _minimum_input_tokens(override["input_length"], f"prefix_cache.groups.overrides.{group_id}.input_length") < reserved_tokens: + raise ScenarioValidationError( + f"prefix_cache.groups.overrides.{group_id}.input_length must be at least {reserved_tokens} tokens to contain the configured non-shared region" + ) + if "output_length" in override: + _validate_output_config(override["output_length"], f"prefix_cache.groups.overrides.{group_id}.output_length", source.parent) + if "corpus_selection" in override: + _mode(override["corpus_selection"], _MODES["selection"], f"prefix_cache.groups.overrides.{group_id}.corpus_selection") + order = pc.setdefault("order", {"strategy": "interleave"}) + if order.get("strategy") not in _MODES["order"]: + raise ScenarioValidationError(f"prefix_cache.order.strategy must be one of {sorted(_MODES['order'])}") + service["dp_size"] = _positive(service.get("dp_size", 1), "service.dp_size") + service.setdefault("reset_url", None) + service.setdefault("assume_empty_cache", False) + service.setdefault("engine_label_map", {}) + service.setdefault("timeout_seconds", 30) + service.setdefault("api_key", "") + for field in ("inference_url", "metrics_url", "model"): + if not isinstance(service.get(field), str) or not service[field]: + raise ScenarioValidationError(f"service.{field} must be a non-empty string") + validation = _require_dict(data["validation"], "validation") + validation.setdefault("target_warning_pp", 1.0) + validation.setdefault("actual_warning_pp", 5.0) + # cold 多 DP 必须显式提供推理地址，否则无法路由。 + if cache_mode == "cold" and service["dp_size"] > 1 and not service["inference_url"]: + raise ScenarioValidationError("cold multi-DP requires inference_url") + return data + + +def load_scenario(path: Path | str) -> Scenario: + """读取并解析场景 JSON 文件，校验后返回 Scenario 对象。 + + 任何读取/解析/校验失败都会以 ScenarioValidationError 形式抛出。 + """ + source = Path(path).resolve() + try: + raw = json.loads(source.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ScenarioValidationError(f"cannot read scenario {source}: {exc}") from exc + return Scenario(source, _validate(_require_dict(raw, "scenario"), source)) diff --git a/plugins/prefix_cache/config_examples/scenario.example.json b/plugins/prefix_cache/config_examples/scenario.example.json new file mode 100644 index 00000000..ee2470e3 --- /dev/null +++ b/plugins/prefix_cache/config_examples/scenario.example.json @@ -0,0 +1,40 @@ +{ + "schema_version": "1.0", + "run": { + "run_id": "gsm8k-prefix-cache-60", + "random_seed": 42, + "output_dir": "./outputs/gsm8k-prefix-cache-60" + }, + "tokenizer": { + "path": "/home/weights/Qwen3.6-27B", + "block_size": 16, + "trust_remote_code": false + }, + "corpus": { + "path": "./GSM8K.jsonl", + "field": "question", + "selection": {"mode": "random"} + }, + "requests": { + "count": 100, + "input_length": {"mode": "fixed", "value": 1024}, + "output_length": {"mode": "fixed", "value": 32} + }, + "prefix_cache": { + "mode": "warmup", + "target_hit_rate": 0.6, + "seed_blocks": 1, + "minimum_non_shared_length": 16, + "groups": {"count": 1, "assignment": {"mode": "uniform"}}, + "order": {"strategy": "interleave"} + }, + "service": { + "inference_url": "http://127.0.0.1:8000/v1/completions", + "metrics_url": "http://127.0.0.1:8000/metrics", + "reset_url": "http://127.0.0.1:8000/reset_prefix_cache", + "model": "model-name", + "dp_size": 2, + "assume_empty_cache": false + }, + "validation": {"target_warning_pp": 1.0, "actual_warning_pp": 5.0} +} diff --git a/plugins/prefix_cache/config_examples/scenario.example.md b/plugins/prefix_cache/config_examples/scenario.example.md new file mode 100644 index 00000000..2f06d5ae --- /dev/null +++ b/plugins/prefix_cache/config_examples/scenario.example.md @@ -0,0 +1,731 @@ +# Scenario 配置参数说明 + +本文逐字段解释 [scenario.example.json](scenario.example.json)，并补充示例中没有展开的可选参数和模式。 + +## 1. 基本规则 + +- 配置是严格 JSON，不能写注释、尾随逗号或未支持的字段； +- 相对路径以 Scenario JSON 所在目录为基准； +- 比例使用 `0.0～1.0`，例如 `0.6` 表示 60%； +- token 长度以 `tokenizer.path` 加载出的 tokenizer 编码结果为准； +- 示例中的 `{}` 和 `[]` 只是 JSON 对象与列表，不是参数。 +- 所有示例字段都可省略；源码默认值与当前 `scenario.example.json` 一致。未知字段仍会被严格拒绝。 + +## 2. 顶层字段 + +| 字段 | 必填 | 作用 | +|---|---:|---| +| `schema_version` | 否 | 配置契约版本，默认 `"1.0"`，当前只能为该值。 | +| `run` | 否 | 运行标识、随机种子和产物目录。 | +| `tokenizer` | 否 | token 计算和 Block 对齐。 | +| `corpus` | 否 | GSM8K 来源及样本选择方式。 | +| `requests` | 否 | 正式请求数量和输入/输出长度。 | +| `prefix_cache` | 否 | 缓存模式、目标命中率、组和顺序。 | +| `service` | 否 | 校验契约保留的服务段；`dp_size` 用于 cold DP 路由。 | +| `validation` | 否 | 偏差告警阈值。 | +| `aisbench` | 否 | 后续 AISBench 在线流程的兼容保留段；当前离线分支不消费。 | + +嵌套对象同样采用严格字段白名单： + +- `corpus.selection`：`mode`、`values`、`indices`、`question_sha256`； +- `requests.input_length`：`mode`、`value`、`values`、`ranges`、`min`、`max`、`mean`、`std`、`path`；每个 range 项只允许 `min`、`max`、`count`； +- `requests.output_length`：`mode`、`value`、`min`、`max`、`mean`、`std`、`path`； +- `prefix_cache.groups`：`count`、`assignment`、`overrides`；assignment 内允许 `mode`、`exponent`、`weights`； +- `prefix_cache.order`（即 `order` 对象）：`strategy`； +- 其余对象的允许字段由下文对应字段表完整列出。 + +## 3. `schema_version` + +```json +"schema_version": "1.0" +``` + +用于防止插件按错误结构解释配置。当前其他版本会直接失败。 + +## 4. `run` + +```json +"run": { + "run_id": "gsm8k-prefix-cache-60", + "random_seed": 42, + "output_dir": "./outputs/gsm8k-prefix-cache-60" +} +``` + +| 字段 | 必填 | 默认值 | 作用 | +|---|---:|---|---| +| `run_id` | 否 | `"gsm8k-prefix-cache-60"` | 基础运行 ID。执行时追加时间戳，并作为四类产物的文件名前缀。prepare 可复用最近一次匹配 inspect 的时间戳。 | +| `random_seed` | 否 | `42` | 控制 GSM8K 随机选择、长度采样、组分配、顺序和唯一 seed。相同输入与配置应生成相同内容。 | +| `output_dir` | 否 | `"./outputs/gsm8k-prefix-cache-60"` | 基础产物目录。执行时在最后一级目录名后追加与 run ID 相同的时间戳；prepare 可复用 inspect 已创建的目录。 | +| `overwrite` | 否 | `false` | 兼容保留字段。`prepare` 默认拒绝覆盖同名产物，重建使用 `prepare --overwrite`。 | + +假设执行时间戳为 `20260825_123456`，示例会生成： + +```text +outputs/gsm8k-prefix-cache-60_20260825_123456/ +├── log/gsm8k-prefix-cache-60_20260825_123456.prepare.log +└── result/ + ├── gsm8k-prefix-cache-60_20260825_123456.full.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.requests.jsonl + ├── gsm8k-prefix-cache-60_20260825_123456.manifest.json + └── gsm8k-prefix-cache-60_20260825_123456.analysis.json +``` + +时间戳采用 `_YYYYMMDD_HHMMSS`。`inspect` 每次创建新时间戳，并在基础输出目录旁写入 `.inspect.json`；后续 `prepare` 在指针匹配且时间戳目录仍存在时复用该时间戳，否则创建新时间戳。指针只比较基础 `run_id` 和 `output_dir`，不比较 Scenario 哈希；修改其他参数后若要求使用新目录，应重新执行 `inspect`，或删除旧指针后再执行 `prepare`。 + +## 5. `tokenizer` + +```json +"tokenizer": { + "path": "/home/weights/Qwen3.6-27B", + "block_size": 16, + "trust_remote_code": false +} +``` + +| 字段 | 必填 | 默认值 | 作用 | +|---|---:|---|---| +| `path` | 否 | `"/home/weights/Qwen3.6-27B"` | 传给 `AutoTokenizer.from_pretrained` 的本地目录或 Hugging Face 标识。必须与 vLLM 服务端 tokenizer 一致。 | +| `block_size` | 否 | `16` | Prefix Cache Block 的 token 数。公共前缀和 seed 按它对齐，必须与服务端实际值一致。 | +| `revision` | 否 | `null` | tokenizer 的分支、tag 或 commit，用于固定版本。 | +| `trust_remote_code` | 否 | `false` | 是否执行模型仓库的自定义 tokenizer 代码，只应对可信仓库启用。 | + +若 `block_size=16`、`seed_blocks=1`，每条请求会在公共前缀和自然后缀之间插入 16-token 唯一 seed。 + +## 6. `corpus` + +```json +"corpus": { + "path": "./GSM8K.jsonl", + "field": "question", + "selection": {"mode": "random"} +} +``` + +| 字段 | 必填 | 默认值 | 作用 | +|---|---:|---|---| +| `path` | 否 | `"./GSM8K.jsonl"` | GSM8K JSONL 路径，每个非空行必须是 JSON 对象。 | +| `field` | 否 | `"question"` | 读取自然语言问题的字段，只使用该字段，不拼接标准答案。 | +| `selection` | 否 | `{"mode":"random"}` | 为 canonical 前缀和自然后缀选择样本。 | + +问题文本会先去除首尾空白，并把连续空白折叠成一个空格。`question_sha256` 基于规范化后的 UTF-8 文本。 + +### 6.1 `selection.mode=random` + +```json +"selection": {"mode": "random"} +``` + +按 `random_seed` 确定性打乱。所需数量超过语料行数时开始新的打乱周期。 + +### 6.2 `selection.mode=indices` + +```json +"selection": {"mode": "indices", "values": [0, 15, 72]} +``` + +- 使用零基行号，`0` 是第一行； +- `values` 也可写成 `indices`； +- 列表不足时循环复用； +- 任一行号不存在会失败。 + +### 6.3 `selection.mode=question_sha256` + +```json +"selection": { + "mode": "question_sha256", + "values": ["规范化问题文本的64位SHA-256"] +} +``` + +`values` 也可写成 `question_sha256`。每个哈希必须唯一匹配一条语料；零匹配或多匹配都会失败。 + +### 6.4 `selection.mode=mixed` + +```json +"selection": { + "mode": "mixed", + "indices": [0, 15], + "question_sha256": ["某个问题的SHA-256"] +} +``` + +先加入行号样本，再加入哈希样本，适合同时固定位置和内容身份。 +如果两类列表合计样本数小于实际需要数量，插件会按合并后的顺序循环复用；如需避免复用，请提供足够多的指定样本。 +`indices` 与 `question_sha256` 不能同时为空，否则报 `specified GSM8K selection is empty`。 + +## 7. `requests` + +```json +"requests": { + "count": 100, + "input_length": {"mode": "fixed", "value": 1024}, + "output_length": {"mode": "fixed", "value": 32} +} +``` + +### 7.1 `count` + +正式请求总数，默认 `100`，必须是正整数。warmup 请求不计入该数量，也不写入 requests JSONL。 + +### 7.2 `input_length` + +定义每条正式请求的目标输入 token 总数： + +```text +公共前缀 + 全局唯一 seed + GSM8K 自然后缀 +``` + +整个字段省略时默认 `{"mode":"fixed","value":1024}`；fixed 模式省略 `value` 时也默认 1024。 + +#### 固定长度 + +```json +"input_length": {"mode": "fixed", "value": 1024} +``` + +所有请求都是 1024 token，`value` 必须为正整数。 + +#### 闭区间采样 + +```json +"input_length": { + "mode": "range", + "ranges": [ + {"min": 512, "max": 1024, "count": 80}, + {"min": 2048, "max": 4096, "count": 20} + ] +} +``` + +- `min`、`max` 均包含； +- 每个 `count` 表示该区间生成的请求数； +- 所有 `count` 之和必须等于 `requests.count`； +- 采样由 `random_seed` 决定。 + +#### 显式长度列表 + +```json +"input_length": {"mode": "explicit", "values": [512, 768, 1024, 2048]} +``` + +`values` 必须全部是正整数，元素个数必须等于对应范围内的请求数。全局配置时等于 `requests.count`；组级覆盖时等于该组实际请求数。 + +#### 截断正态分布 + +```json +"input_length": { + "mode": "truncated_normal", + "min": 512, + "max": 2048, + "mean": 1024, + "std": 256 +} +``` + +只接受 `[min,max]` 内的整数采样；`mean` 默认取区间中点，`std` 默认按区间宽度推导且显式值必须大于 0。相同 `random_seed` 产生相同长度序列。 + +#### CSV 指定 + +```json +"input_length": {"mode": "csv", "path": "./input_lengths.csv"} +``` + +CSV 行数必须等于 `requests.count`，并包含以下任一正整数列： + +- `input_prompt_tokens`； +- `content_tokens`； +- `input_tokens`。 + +### 7.3 `output_length` + +该值写入 requests JSONL 的 `max_tokens`。 + +整个字段省略时默认 `{"mode":"fixed","value":32}`；fixed 模式省略 `value` 时也默认 32。 + +#### 固定值 + +```json +"output_length": {"mode": "fixed", "value": 32} +``` + +`value` 必须为正整数。 + +#### 均匀分布 + +```json +"output_length": {"mode": "uniform", "min": 16, "max": 64} +``` + +`min`、`max` 必须是正整数且 `max >= min`；在包含上下界的整数区间均匀采样。 + +#### 截断正态分布 + +```json +"output_length": { + "mode": "truncated_normal", + "min": 16, + "max": 128, + "mean": 64, + "std": 16 +} +``` + +- 只保留 `[min,max]` 内的整数； +- `min`、`max` 必须是正整数且 `max >= min`； +- `mean` 省略时取区间中点； +- `std` 省略时按区间宽度推导，显式值必须大于 0； +- `min=max` 时直接返回固定值。 + +#### CSV 指定 + +```json +"output_length": {"mode": "csv", "path": "./output_lengths.csv"} +``` + +CSV 必须包含正整数 `output_tokens` 列，行数等于 `requests.count`。 + +## 8. `prefix_cache` + +```json +"prefix_cache": { + "mode": "warmup", + "target_hit_rate": 0.6, + "seed_blocks": 1, + "minimum_non_shared_length": 16, + "groups": { + "count": 1, + "assignment": {"mode": "uniform"} + }, + "order": {"strategy": "interleave"} +} +``` + +### 8.1 `mode` + +- `cold`：正式请求按 `(Prefix Group, DP rank)` lane 路由，理论命中率按 lane 从零水位模拟； +- `warmup`：为每个 `Prefix Group × DP rank` 生成预热计划（写入 Manifest 的 `warmup.plan`），正式请求本身不固定 DP。 + +本分支只生成数据与预热计划，不实际执行预热请求。warmup 请求不进入正式请求数或理论分母。 +省略时默认 `warmup`。 + +### 8.2 `target_hit_rate` + +期望的全局 token 加权命中率，范围 `[0,1]`。它是求解器的主目标，不等于简单地把每条请求的固定百分比设成前缀。 +省略时默认 `0.6`。 + +求解会考虑 Block 对齐、请求顺序、Prefix Group、水位和 cold DP 路由。目标不可精确达到时，采用最接近的可达值并记录 requested/effective/theoretical 及原因。 + +### 8.3 `seed_blocks` + +唯一 seed 的 Block 数，默认 `1`，必须为正整数： + +```text +seed token 数 = seed_blocks × tokenizer.block_size +``` + +seed 位于公共前缀和自然后缀之间，所有正式请求全局唯一，防止请求在公共前缀之后继续意外共享。输入长度必须能容纳 seed。 + +插件在加载 Scenario 时就会检查 fixed/explicit/range/truncated_normal/CSV 输入长度的最小值是否能容纳非共享区；不足时会在生成数据前直接报配置错误。 + +### 8.4 `minimum_non_shared_length` + +每条正式请求至少预留多少个非共享 token，默认等于 `seed_blocks × block_size`，并且不能小于唯一 seed 长度。 + +```text +公共前缀最大长度 = 按 Block 向下对齐(input_length - minimum_non_shared_length) +``` + +当该值大于 seed 长度时，多出的空间由 GSM8K 自然后缀填充。它用于保证公共前缀之后不仅有全局唯一 seed，还能保留指定规模的自然差异内容。 + +### 8.5 `groups.count` + +Prefix Group 数量，默认 `1`。插件生成 `group-0`、`group-1` 等 ID。每个组独立生成 canonical 前缀、维护水位、统计理论命中率，并在 warmup 时逐 DP 预热。 + +### 8.6 `groups.assignment` + +整个 `assignment` 省略时默认 `{"mode":"uniform"}`。 + +均匀分配： + +```json +"assignment": {"mode": "uniform"} +``` + +请求尽量平均分组，余数按稳定组序分配。 + +Zipf 分配： + +```json +"assignment": {"mode": "zipf", "exponent": 1.0} +``` + +热度与 `1/rank^exponent` 成正比；`exponent` 必须大于 0，越大越集中于热点组。 + +显式权重： + +```json +"assignment": { + "mode": "weights", + "weights": [0.5, 0.3, 0.15, 0.05] +} +``` + +权重数量必须等于 `groups.count`，不能为负且总和大于 0；无需预先归一化。 + +### 8.7 `groups.overrides` + +可按组覆盖全局设置；省略时默认 `{}`： + +```json +"groups": { + "count": 4, + "assignment": {"mode": "uniform"}, + "overrides": { + "group-0": { + "input_length": {"mode": "fixed", "value": 2048}, + "output_length": {"mode": "fixed", "value": 64}, + "corpus_selection": {"mode": "indices", "values": [0, 1, 2]} + } + } +} +``` + +- ID 必须是有效的 `group-0` 到 `group-(count-1)`； +- `input_length`、`output_length` 支持对应的全部全局模式； +- `corpus_selection` 支持 random/indices/question_sha256/mixed； +- 组级 range/CSV 生成数量必须等于实际分到该组的请求数。 + +### 8.8 `order.strategy` + +- `sequential`：保持目标分配阶段顺序； +- `within_group_shuffle`：各组内部打乱，再按组输出； +- `interleave`：各组轮转交错，默认，适合多租户流量； +- `global_shuffle`：全局确定性打乱。 +- `input_len_asc`：每个 Prefix Group 内按输入长度从短到长排列，再按组轮转交错；相同长度保持原始稳定顺序。 + +理论水位总是按最终发送顺序重新模拟。 + +## 9. `service` + +```json +"service": { + "inference_url": "http://127.0.0.1:8000/v1/completions", + "metrics_url": "http://127.0.0.1:8000/metrics", + "reset_url": "http://127.0.0.1:8000/reset_prefix_cache", + "model": "model-name", + "dp_size": 2, + "assume_empty_cache": false +} +``` + +| 字段 | 必填 | 默认值 | 作用 | +|---|---:|---|---| +| `inference_url` | 否 | `"http://127.0.0.1:8000/v1/completions"` | 后续在线流程使用的 vLLM Completions API 地址。当前离线生成只校验最终有效值为非空字符串。 | +| `metrics_url` | 否 | `"http://127.0.0.1:8000/metrics"` | Prometheus 地址，仅作非空校验字段。 | +| `reset_url` | 否 | `"http://127.0.0.1:8000/reset_prefix_cache"` | Prefix Cache reset 地址，仅作可选配置。 | +| `model` | 否 | `"model-name"` | 后续在线流程兼容字段。当前只做非空校验并保留在 effective config，不写入 `requests.jsonl` 或 warmup 请求。 | +| `dp_size` | 否 | `2` | 单入口内部 DP rank 数，必须为正整数。**离线用于 cold 模式的 DP 路由**与 warmup 预热计划。 | +| `assume_empty_cache` | 否 | `false` | 仅作可选配置，离线不消费。 | +| `engine_label_map` | 否 | `{}` | 仅作可选配置，离线不消费。 | +| `timeout_seconds` | 否 | `30` | 仅作可选配置，离线不消费。 | +| `api_key` | 否 | `""` | 仅作可选配置。Manifest 不保存明文，只记录是否配置；Scenario 文件本身仍需限制权限。 | + +> 本分支不访问任何服务地址。`inference_url`、`metrics_url`、`model` 都有默认值，用户无需显式填写，但最终有效值必须为非空字符串；其余服务字段仅兼容保留。当前真正参与离线计算的是 `dp_size`。 + +## 10. `validation` + +```json +"validation": { + "target_warning_pp": 1.0, + "actual_warning_pp": 5.0 +} +``` + +| 字段 | 默认值 | 作用 | +|---|---:|---| +| `target_warning_pp` | `1.0` | 理论值与请求目标相差超过多少百分点时记录 `TARGET_DEVIATION`。 | +| `actual_warning_pp` | `5.0` | 实际值与理论值相差超过多少百分点时记录 `ACTUAL_DEVIATION`。**在线流程**使用；本离线分支保留该字段但不消费。 | + +单位是百分点（pp），不是相对百分比。例如 60% 与 58.5% 相差 1.5 pp。两种偏差始终只 warning，不改变原本成功的退出码。 + +分析产物同时记录带符号偏差、绝对偏差、目标是否在全局可达范围内，以及 `PASS`/`PASS_WITH_WARNING` 展示状态。该状态只用于展示，不控制退出码。 + +## 11. `aisbench` + +```json +"aisbench": { + "config": "./configs/prefix_cache.py", + "work_dir": "./work_dirs/prefix_cache", + "extra_args": ["--debug"] +} +``` + +| 字段 | 必填 | 默认值 | 当前用途 | +|---|---:|---|---| +| `config` | 否 | 无 | AISBench 在线配置兼容字段；离线 `inspect/prepare/validate` 不消费。 | +| `work_dir` | 否 | 无 | AISBench 在线工作目录兼容字段；离线流程不消费。 | +| `extra_args` | 否 | 无 | AISBench 在线附加参数兼容字段；离线流程不消费。 | + +整个 `aisbench` 段省略时默认 `{}`。源码当前只限制该对象允许出现以上三个键，不校验三个值的类型，也不会据此启动 AISBench；配置这些字段不会改变本分支的离线产物。 + +## 12. 原示例最终表示的场景 + +- 生成 100 条正式请求； +- 输入长度固定为 1024 token； +- 每条最多输出 32 token； +- 创建 1 个 uniform 组； +- 目标全局命中率为 60%； +- 使用一个 16-token Block 作为全局唯一 seed； +- 每条请求至少保留 16 token 非共享区； +- 请求按组交错排列； +- 使用 warmup 模式； +- 单个 vLLM HTTP 入口内部有 2 个 DP rank（cold 路由 / warmup 计划使用）； +- 该组分别在 DP 0、DP 1 生成预热计划，共 2 条不进入正式统计的 warmup 请求； +- 理论/目标超过 1 pp 时只告警（`actual_warning_pp` 字段为在线流程保留）。 + +## 13. 建议检查顺序 + +```bash +ais-bench-prefix-cache inspect --scenario ./scenario.json +ais-bench-prefix-cache prepare --scenario ./scenario.json +ais-bench-prefix-cache validate --manifest +``` + +重点检查： + +- `requested_target_hit_rate`：请求目标； +- `effective_target_hit_rate`：求解器选择的可达目标； +- `theoretical_hit_rate`：按最终顺序模拟的理论值； +- `reachable_min/max`：当前长度、Block、分组和路由下的范围； +- `target_reachable`：目标是否落在全局最大/最小可达区间； +- `groups`：请求分布与 canonical 前缀； +- `warmup.plan`：是否覆盖每个 Prefix Group × DP rank； +- `warnings`：目标偏差或目标不可达。 + +Manifest 还会记录输入/输出长度的 min/max/mean/P50/P90/P95/P99 与分桶计数、各组 reachable min/max、每条请求的确定性 `request_random_seed`，以及唯一差异块的碰撞检查状态。 + +## 14. CLI 行为和返回字段 + +### 14.1 `inspect` + +`inspect` 会加载 tokenizer 和 GSM8K，在临时目录复用完整 prepare 流程计算可达范围，但不发送请求，也不在正式 `result/` 目录保留四类数据产物。 + +- 每次执行生成新时间戳； +- 日志写入 `output_dir_时间戳/log/.inspect.log`； +- 成功后写入 `<基础output_dir>.inspect.json`； +- stdout JSON 包含 `log` 路径。 + +### 14.2 `prepare` + +`prepare` 优先复用最近一次有效 inspect 指针的时间戳；没有有效指针时生成新时间戳。生成 prompt 时进度条写入 stderr，每完成一条 prompt 增加 1；最后一行 stdout JSON 固定包含： + +| 字段 | 含义 | +|---|---| +| `full` | full JSONL 路径。 | +| `requests` | 最小 requests JSONL 路径。 | +| `manifest` | Manifest JSON 路径。 | +| `analysis` | 理论分析 JSON 路径。 | +| `log` | prepare 日志路径；只有日志文件成功解析和创建时出现。 | + +`--overwrite` 只允许覆盖当前时间戳目录内上述四个固定产物，不会删除整个输出目录。 + +### 14.3 `validate` + +`validate` 不生成新数据，检查行数、字段集合、顺序对应关系及 full/requests SHA-256。stdout 固定返回： + +| 字段 | 含义 | +|---|---| +| `ok` | 校验是否通过；成功时为 `true`。 | +| `rows` | 通过校验的正式请求行数。 | +| `run_id` | Manifest 中的运行 ID。 | + +validate 日志写入 Manifest 对应时间戳目录的 `log/.validate.log`，但当前返回 JSON 不包含 `log` 字段。 + +正常成功退出码为 `0`；Scenario、生成或产物校验错误返回 `2`。目标不可达和命中率偏差始终只是 warning，不改变成功退出码。 + +## 15. 请求产物字段 + +### 15.1 `.requests.jsonl` + +每行严格只包含以下三个字段，且落盘顺序固定： + +| 字段 | 类型 | 含义 | +|---|---|---| +| `question` | string | 最终完整 prompt。 | +| `answer` | string | AISBench 兼容占位值，当前固定为 `"none"`。 | +| `max_tokens` | integer | 最大输出 token 数，来自 `requests.output_length` 或组级覆盖。 | + +### 15.2 `.full.jsonl` + +每行固定包含 26 个审计字段： + +| 字段 | 类型 | 含义 | +|---|---|---| +| `request_id` | string | 稳定请求 ID，例如 `request-00000000`。 | +| `sequence_index` | integer | 最终发送顺序中的零基序号，必须连续。 | +| `group_id` | string | 所属 Prefix Group。 | +| `occurrence_index_within_group` | integer | 该请求在组内的出现序号。 | +| `dp_rank` | integer/null | cold 模式的目标 DP rank；warmup 正式请求为 `null`。 | +| `lane_sequence` | integer/null | cold `(group_id, dp_rank)` lane 内序号；warmup 为 `null`。 | +| `target_input_tokens` | integer | 长度配置要求的输入 token 数。 | +| `actual_input_tokens` | integer | prompt 经 tokenizer 重编码后的实际 token 数。 | +| `max_tokens` | integer | 最大输出 token 数。 | +| `shared_prefix_tokens` | integer | 求解器为该请求选择的公共前缀长度。 | +| `seed_tokens` | integer | 全局唯一 seed 的 token 数。 | +| `natural_suffix_tokens` | integer | seed 后 GSM8K 自然后缀的 token 数。 | +| `question` | string | 最终完整 prompt。 | +| `answer` | string | 当前固定为 `"none"`。 | +| `gsm_indices` | array[integer] | 本请求自然后缀使用的 GSM8K 零基行号。 | +| `gsm_hashes` | array[string] | 对应规范化 GSM8K question 的 SHA-256。 | +| `canonical_prefix_sha256` | string | 所属组 canonical 前缀指纹。 | +| `seed_sha256` | string | 本请求唯一 seed token 序列指纹。 | +| `request_random_seed` | integer | 实际参与该请求 seed 构造的确定性随机种子。 | +| `watermark_before` | integer | 请求到达前所在缓存 lane 的理论水位。 | +| `theoretical_hit_tokens` | integer | 本请求理论命中 token 数。 | +| `watermark_after` | integer | 请求完成后的理论水位。 | +| `theoretical_hit_rate` | number | `theoretical_hit_tokens / actual_input_tokens`。 | +| `divergence_block_sha256` | string | 差异块指纹，当前等于 `seed_sha256`。 | +| `divergence_unique` | boolean | 差异块是否通过全局唯一性检查。 | +| `collision_status` | string | 碰撞检查状态，成功产物为 `"pass"`。 | + +## 16. Manifest 完整字段 + +Manifest 顶层字段： + +| 字段 | 含义 | +|---|---| +| `schema_version` | Manifest 契约版本，当前为 `"1.0"`。 | +| `plugin_version` | 生成产物的插件版本。 | +| `run_id` | 已追加执行时间戳的运行 ID。 | +| `scenario_path` | 原 Scenario 绝对路径。 | +| `scenario_sha256` | 原 Scenario 文件 SHA-256。 | +| `effective_config` | 补齐默认值、解析路径并追加时间戳后的有效配置。 | +| `effective_config_sha256` | 有效配置的规范化 JSON 指纹。 | +| `corpus_sha256` | GSM8K 文件 SHA-256。 | +| `tokenizer` | tokenizer 身份和 Block 信息。 | +| `requests` | 请求数量、总 token 和长度分布。 | +| `prefix_cache` | 目标、可达范围、理论值和验证结论。 | +| `groups` | 各 Prefix Group 的 canonical、来源和理论统计。 | +| `dp` | DP 数量与 cold 路由策略。 | +| `warmup` | warmup 开关和预热计划。 | +| `divergence` | 全局唯一差异块审计。 | +| `artifacts` | 产物路径、大小、行数和哈希。 | + +### 16.1 `tokenizer` + +| 字段 | 含义 | +|---|---| +| `path`、`revision` | tokenizer 来源和固定版本。 | +| `class` | 实际加载的 tokenizer Python 类。 | +| `vocab_size` | tokenizer 词表大小。 | +| `special_token_ids` | 特殊 token ID 列表。 | +| `block_size` | Prefix Cache Block token 数。 | +| `fingerprint_sha256` | path/revision/class/vocab/special IDs 的规范化指纹。 | + +### 16.2 `requests` + +- `count`：正式请求数； +- `total_input_tokens`：所有正式请求实际输入 token 总和； +- `input_length_summary`、`output_length_summary`：输入/输出长度摘要。 + +每个长度摘要包含 `min`、`max`、`mean`、`p50`、`p90`、`p95`、`p99`、`bins`。`bins` 最多十个非空桶；每项包含该桶内实际观测的 `min`、`max` 和 `count`。 + +### 16.3 `prefix_cache` + +| 字段 | 含义 | +|---|---| +| `mode` | `cold` 或 `warmup`。 | +| `requested_target_hit_rate` | Scenario 请求的目标命中率。 | +| `effective_target_hit_rate` | 求解器选择的最近可达目标。 | +| `theoretical_hit_rate` | 按最终顺序模拟得到的理论值。 | +| `reachable_min`、`reachable_max` | 当前约束下全局理论可达范围。 | +| `target_reachable` | 请求目标是否位于可达范围内。 | +| `minimum_non_shared_length` | 每条请求预留的最小非共享 token 数。 | +| `adjusted` | 求解目标是否因约束被调整。 | +| `reason` | 调整原因；无需调整时可为 `null`。 | +| `validation_status` | `PASS` 或 `PASS_WITH_WARNING`。 | +| `target_signed_difference_pp` | `theoretical - requested` 的带符号百分点差。 | +| `target_absolute_difference_pp` | 上述差值的绝对值。 | + +### 16.4 `groups.` + +- `canonical_prefix_sha256`、`canonical_prefix_tokens`：canonical 前缀指纹和总 token 数； +- `max_shared_prefix_tokens`：该组正式请求使用的最大公共前缀长度； +- `gsm_indices`、`gsm_question_sha256`：canonical 前缀语料来源； +- `reachable_min`、`reachable_max`：该组理论可达范围； +- `theoretical_hit_rate`：该组 token 加权理论命中率。 + +### 16.5 `dp`、`warmup`、`divergence` + +- `dp.size`：DP 数；`cold_route_strategy`：cold 时为 `"group_round_robin"`，warmup 时为 `null`； +- `warmup.enabled`：是否启用；`warmup.plan`：预热项列表； +- 每个 warmup 项包含 `request_id`、`group_id`、`dp_rank`、`prompt`、`input_tokens`、`shared_prefix_tokens`、`max_tokens`、`included_in_formal_statistics`；最后一个字段固定为 `false`； +- `divergence.strategy`：当前为 `"globally_unique_seed_block"`； +- `unique_request_blocks`、`request_count`、`collision_status`：唯一 seed 数、请求数和碰撞检查结论。 + +### 16.6 `artifacts` 和密钥处理 + +- `artifacts.full`、`artifacts.requests`：`name`、`path`、`rows`、`bytes`、`sha256`； +- `artifacts.analysis`：`name`、`path`、`bytes`、`sha256_at_prepare`。 + +Manifest 不保存 `service.api_key` 明文；它会被替换为 `effective_config.service.api_key_configured` 布尔值。 + +## 17. `analysis.json` 完整字段 + +| 字段 | 含义 | +|---|---| +| `schema_version` | 分析契约版本。 | +| `run_id` | 已追加时间戳的运行 ID。 | +| `status` | 成功 prepare 时为 `"prepared"`。 | +| `requested_target_hit_rate` | Scenario 请求目标。 | +| `effective_target_hit_rate` | 最近可达目标。 | +| `theoretical_hit_rate` | 最终顺序理论值。 | +| `target_difference_pp` | 当前等于目标绝对偏差。 | +| `target_signed_difference_pp` | `theoretical - requested` 的带符号百分点差。 | +| `target_absolute_difference_pp` | 目标绝对偏差。 | +| `validation` | 展示状态和可达性。 | +| `theory` | 全局、分组和分 DP 理论 token 统计。 | +| `warnings` | 目标不可达或偏差告警列表。 | + +`validation` 包含 `status`、`target_reachable`、`warning_only`、`affects_exit_code`。后两项固定为 `true`、`false`，表示告警不影响成功退出码。 + +`theory` 包含 `input_tokens`、`hit_tokens`、`groups`、`dp`；每个组或 DP 值包含 `input_tokens`、`hit_tokens`、`hit_rate`。warmup 正式请求没有固定 `dp_rank`，所以 `theory.dp` 可以为空对象。 + +`warnings` 可能包含： + +- `TARGET_UNREACHABLE`：`code`、`requested_target_hit_rate`、`reachable_min`、`reachable_max`； +- `TARGET_DEVIATION`：`code`、`difference_pp`。 + +`actual_warning_pp` 供未来在线实际值分析使用，因此当前离线 analysis 不会生成 `ACTUAL_DEVIATION`。 + +## 18. `inspect` 摘要与复用指针字段 + +inspect stdout JSON 字段： + +| 字段 | 含义 | +|---|---| +| `run_id`、`mode` | 基础运行 ID 和缓存模式。 | +| `requested_target_hit_rate` | Scenario 请求目标。 | +| `effective_target_hit_rate` | 求解器选择的可达目标。 | +| `theoretical_hit_rate` | 临时构造数据的理论值。 | +| `reachable_min`、`reachable_max` | 全局可达范围。 | +| `target_reachable` | 请求目标是否可达。 | +| `group_reachability` | 每组的 `reachable_min`、`reachable_max`。 | +| `groups` | 每组正式请求数量。 | +| `input_tokens`、`output_tokens` | 长度摘要，并额外包含 `total`。 | +| `dp_route_counts` | cold 下各 DP rank 请求数；warmup 通常为空对象。 | +| `sends_requests` | 固定为 `false`，表示不访问推理服务。 | +| `log` | inspect 日志路径。 | + +`<基础output_dir>.inspect.json` 字段： + +| 字段 | 含义 | +|---|---| +| `schema_version` | 指针契约版本，当前为 `"1.0"`。 | +| `timestamp` | 可复用时间戳，格式 `YYYYMMDD_HHMMSS`。 | +| `run_id` | 基础运行 ID，不含时间戳。 | +| `output_dir` | 基础输出目录。 | +| `output_dir_with_timestamp` | inspect 日志所在的时间戳目录。 | + +prepare 复用前会检查指针版本、基础 run/output、时间戳格式及时间戳目录是否存在；不会比较 Scenario SHA-256。 diff --git a/plugins/prefix_cache/setup.py b/plugins/prefix_cache/setup.py new file mode 100644 index 00000000..694c4f9d --- /dev/null +++ b/plugins/prefix_cache/setup.py @@ -0,0 +1,19 @@ +from setuptools import find_packages, setup + + +setup( + name="ais-bench-prefix-cache", + version="0.1.2", + description="Prefix Cache dataset generation and offline validation for AISBench", + packages=find_packages(), + python_requires=">=3.10", + install_requires=[ + "ais-bench-benchmark", + "transformers", + ], + entry_points={ + "console_scripts": [ + "ais-bench-prefix-cache = ais_bench_prefix_cache.cli:console_main", + ], + }, +) diff --git a/plugins/prefix_cache/tests/__init__.py b/plugins/prefix_cache/tests/__init__.py new file mode 100644 index 00000000..2c07de8d --- /dev/null +++ b/plugins/prefix_cache/tests/__init__.py @@ -0,0 +1 @@ +"""Prefix Cache plugin tests.""" diff --git a/plugins/prefix_cache/tests/test_artifacts.py b/plugins/prefix_cache/tests/test_artifacts.py new file mode 100644 index 00000000..99b23a98 --- /dev/null +++ b/plugins/prefix_cache/tests/test_artifacts.py @@ -0,0 +1,117 @@ +import hashlib +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from ais_bench_prefix_cache.artifacts import read_jsonl, sha256_file, validate_artifacts, write_json +from ais_bench_prefix_cache.errors import ArtifactValidationError + + +class AtomicWriteTest(unittest.TestCase): + def test_refuses_to_overwrite_existing_artifact(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "x.json" + write_json(path, {"a": 1}, overwrite=False) + with self.assertRaisesRegex(ArtifactValidationError, "refusing to overwrite"): + write_json(path, {"a": 2}, overwrite=False) + + def test_temp_file_is_cleaned_up_when_replace_fails(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "x.json" + temp = path.with_name(f".{path.name}.tmp-{os.getpid()}") + with patch("os.replace", side_effect=OSError("boom")): + with self.assertRaises(OSError): + write_json(path, {"a": 1}, overwrite=True) + self.assertFalse(temp.exists()) + + +class ReadJsonlTest(unittest.TestCase): + def test_unreadable_file_rejected(self): + with self.assertRaisesRegex(ArtifactValidationError, "cannot read JSONL"): + read_jsonl(Path("/nonexistent/x.jsonl")) + + def test_invalid_line_rejected(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "x.jsonl" + path.write_text("not json\n", encoding="utf-8") + with self.assertRaisesRegex(ArtifactValidationError, "cannot read JSONL"): + read_jsonl(path) + + +class ValidateArtifactsTest(unittest.TestCase): + def _write_artifacts(self, folder, full_row, request_row, count=None, full_sha=None, request_sha=None) -> Path: + """构造 manifest 与两个工件文件，返回 manifest 路径。""" + root = Path(folder) + result_dir = root / "result" + result_dir.mkdir() + full_path = result_dir / "x.full.jsonl" + request_path = result_dir / "x.requests.jsonl" + full_path.write_text(json.dumps(full_row) + "\n", encoding="utf-8") + # requests 行要求严格为 {question, answer, max_tokens}，剥离 sequence_index。 + request_row = {key: value for key, value in request_row.items() if key != "sequence_index"} + request_path.write_text(json.dumps(request_row) + "\n", encoding="utf-8") + manifest = { + "run_id": "pc-test", + "requests": {"count": count if count is not None else 1}, + "artifacts": { + "full": {"name": "x.full.jsonl", "sha256": full_sha or sha256_file(full_path)}, + "requests": {"name": "x.requests.jsonl", "sha256": request_sha or sha256_file(request_path)}, + }, + } + manifest_path = result_dir / "x.manifest.json" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + return manifest_path + + def _row(self, sequence_index=0, question="q", answer="a", max_tokens=2): + return {"sequence_index": sequence_index, "question": question, "answer": answer, "max_tokens": max_tokens} + + def test_unreadable_manifest_rejected(self): + with self.assertRaisesRegex(ArtifactValidationError, "cannot read Manifest"): + validate_artifacts(Path("/nonexistent/x.manifest.json")) + + def test_row_count_mismatch_rejected(self): + with tempfile.TemporaryDirectory() as folder: + manifest_path = self._write_artifacts(folder, self._row(), self._row(), count=5) + with self.assertRaisesRegex(ArtifactValidationError, "row counts do not match"): + validate_artifacts(manifest_path) + + def test_invalid_sequence_index_rejected(self): + with tempfile.TemporaryDirectory() as folder: + manifest_path = self._write_artifacts(folder, self._row(sequence_index=3), self._row()) + with self.assertRaisesRegex(ArtifactValidationError, "invalid sequence_index"): + validate_artifacts(manifest_path) + + def test_unexpected_request_fields_rejected(self): + with tempfile.TemporaryDirectory() as folder: + manifest_path = self._write_artifacts( + folder, self._row(), {"question": "q", "answer": "a", "max_tokens": 2, "bogus": 1} + ) + with self.assertRaisesRegex(ArtifactValidationError, "unexpected fields"): + validate_artifacts(manifest_path) + + def test_request_differing_from_full_rejected(self): + with tempfile.TemporaryDirectory() as folder: + manifest_path = self._write_artifacts(folder, self._row(question="q1"), self._row(question="q2")) + with self.assertRaisesRegex(ArtifactValidationError, "differs from full row"): + validate_artifacts(manifest_path) + + def test_sha256_mismatch_rejected(self): + with tempfile.TemporaryDirectory() as folder: + manifest_path = self._write_artifacts(folder, self._row(), self._row(), full_sha="0" * 64) + with self.assertRaisesRegex(ArtifactValidationError, "SHA-256 mismatch"): + validate_artifacts(manifest_path) + + def test_valid_artifacts_pass(self): + with tempfile.TemporaryDirectory() as folder: + manifest_path = self._write_artifacts(folder, self._row(), self._row()) + result = validate_artifacts(manifest_path) + self.assertTrue(result["ok"]) + self.assertEqual(result["rows"], 1) + self.assertEqual(result["run_id"], "pc-test") + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/prefix_cache/tests/test_cli.py b/plugins/prefix_cache/tests/test_cli.py new file mode 100644 index 00000000..d128eb7e --- /dev/null +++ b/plugins/prefix_cache/tests/test_cli.py @@ -0,0 +1,106 @@ +import json +import io +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +from ais_bench_prefix_cache.artifacts import ArtifactPaths +from ais_bench_prefix_cache.cli import PromptProgress, _resolve_log_file, main +from tests.test_pipeline import write_case + + +class CLITest(unittest.TestCase): + def test_log_file_is_nested_under_log_directory(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + for command in ("prepare", "inspect"): + with self.subTest(command=command): + log_file = _resolve_log_file(command, scenario, execution_timestamp="20260825_123456") + self.assertEqual( + log_file, + root / "out_20260825_123456" / "log" / f"pc-test_20260825_123456.{command}.log", + ) + self.assertTrue(log_file.parent.is_dir()) + + def test_prompt_progress_renders_zero_updates_and_completion(self): + stream = io.StringIO() + progress = PromptProgress(stream=stream, width=10) + progress.update(0, 4) + progress.update(1, 4) + progress.update(4, 4) + output = stream.getvalue() + self.assertIn("Generate prompts", output) + self.assertIn("0/4", output) + self.assertIn("1/4", output) + self.assertIn("4/4", output) + self.assertIn("100%", output) + self.assertTrue(output.endswith("\n")) + + def test_prepare_cli_keeps_progress_on_stderr_and_result_on_stdout(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + timestamp = "20260825_123456" + result_dir = root / f"out_{timestamp}" / "result" + paths = ArtifactPaths( + result_dir / f"pc-test_{timestamp}.full.jsonl", + result_dir / f"pc-test_{timestamp}.requests.jsonl", + result_dir / f"pc-test_{timestamp}.manifest.json", + result_dir / f"pc-test_{timestamp}.analysis.json", + ) + + def fake_prepare(path, overwrite, progress, execution_timestamp): + self.assertEqual(path, scenario) + self.assertEqual(execution_timestamp, timestamp) + for completed in range(3): + progress(completed, 2) + return paths + + stdout = io.StringIO() + stderr = io.StringIO() + with ( + patch("ais_bench_prefix_cache.cli.new_execution_timestamp", return_value=timestamp), + patch("ais_bench_prefix_cache.cli.prepare_scenario", side_effect=fake_prepare), + patch("ais_bench_prefix_cache.cli._install_logger"), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + self.assertEqual(main(["prepare", "--scenario", str(scenario)]), 0) + + output = json.loads(stdout.getvalue()) + self.assertEqual(output["manifest"], str(paths.manifest)) + self.assertEqual( + output["log"], + str(root / f"out_{timestamp}" / "log" / f"pc-test_{timestamp}.prepare.log"), + ) + self.assertIn("Generate prompts", stderr.getvalue()) + self.assertIn("2/2", stderr.getvalue()) + + def test_inspect_cli_returns_timestamped_log_path(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + timestamp = "20260825_123456" + summary = {"run_id": "pc-test", "sends_requests": False} + stdout = io.StringIO() + with ( + patch("ais_bench_prefix_cache.cli.new_execution_timestamp", return_value=timestamp), + patch("ais_bench_prefix_cache.cli.inspect_scenario", return_value=summary), + patch("ais_bench_prefix_cache.cli._install_logger"), + redirect_stdout(stdout), + ): + self.assertEqual(main(["inspect", "--scenario", str(scenario)]), 0) + + output = json.loads(stdout.getvalue()) + self.assertEqual(output["run_id"], "pc-test") + self.assertEqual( + output["log"], + str(root / f"out_{timestamp}" / "log" / f"pc-test_{timestamp}.inspect.log"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/prefix_cache/tests/test_cli_flow.py b/plugins/prefix_cache/tests/test_cli_flow.py new file mode 100644 index 00000000..8e0afebd --- /dev/null +++ b/plugins/prefix_cache/tests/test_cli_flow.py @@ -0,0 +1,290 @@ +import io +import json +import logging +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +from ais_bench_prefix_cache.artifacts import ArtifactPaths +from ais_bench_prefix_cache.cli import ( + PromptProgress, + _install_logger, + _persist_inspect_pointer, + _reusable_inspect_timestamp, + _resolve_log_file, + console_main, + main, +) +from ais_bench_prefix_cache.errors import PrefixCacheError +from ais_bench_prefix_cache.scenario import load_scenario +from tests.test_pipeline import write_case + + +class PromptProgressTest(unittest.TestCase): + def test_update_ignores_non_positive_total(self): + stream = io.StringIO() + progress = PromptProgress(stream=stream, width=10) + progress.update(1, 0) + self.assertEqual(stream.getvalue(), "") + + def test_close_terminates_unfinished_line(self): + stream = io.StringIO() + progress = PromptProgress(stream=stream, width=10) + progress.update(1, 4) + progress.close() + self.assertTrue(stream.getvalue().endswith("\n")) + + +class LogResolverTest(unittest.TestCase): + def test_validate_resolves_log_from_manifest(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + out_dir = root / "out" + manifest_path = root / "m.json" + manifest_path.write_text( + json.dumps({"run_id": "pc-test", "effective_config": {"run": {"output_dir": str(out_dir)}}}), + encoding="utf-8", + ) + log_file = _resolve_log_file("validate", manifest_path=manifest_path) + self.assertEqual(log_file, out_dir / "log" / "pc-test.validate.log") + self.assertTrue(log_file.parent.is_dir()) + + def test_validate_returns_none_for_missing_or_bad_manifest(self): + self.assertIsNone(_resolve_log_file("validate")) + with tempfile.TemporaryDirectory() as folder: + manifest_path = Path(folder) / "m.json" + manifest_path.write_text("not json", encoding="utf-8") + self.assertIsNone(_resolve_log_file("validate", manifest_path=manifest_path)) + + def test_returns_none_without_scenario(self): + self.assertIsNone(_resolve_log_file("prepare")) + + def test_returns_none_for_invalid_scenario(self): + with tempfile.TemporaryDirectory() as folder: + self.assertIsNone(_resolve_log_file("prepare", scenario_path=Path(folder) / "missing.json")) + + +class ReusableTimestampTest(unittest.TestCase): + def _pointer(self, root, **overrides): + record = { + "schema_version": "1.0", + "timestamp": "20260825_123456", + "run_id": "pc-test", + "output_dir": str(root / "out"), + } + record.update(overrides) + pointer = root / "out.inspect.json" + pointer.write_text(json.dumps(record), encoding="utf-8") + return pointer + + def test_none_without_pointer(self): + with tempfile.TemporaryDirectory() as folder: + scenario = load_scenario(write_case(Path(folder))) + self.assertIsNone(_reusable_inspect_timestamp(scenario)) + + def test_none_when_schema_version_mismatch(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = load_scenario(write_case(root)) + self._pointer(root, schema_version="2.0") + self.assertIsNone(_reusable_inspect_timestamp(scenario)) + + def test_none_when_run_id_mismatch(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = load_scenario(write_case(root)) + self._pointer(root, run_id="other") + self.assertIsNone(_reusable_inspect_timestamp(scenario)) + + def test_none_when_timestamp_malformed(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = load_scenario(write_case(root)) + self._pointer(root, timestamp="bad") + self.assertIsNone(_reusable_inspect_timestamp(scenario)) + + def test_none_when_timestamp_not_a_string(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = load_scenario(write_case(root)) + self._pointer(root, timestamp=123) + self.assertIsNone(_reusable_inspect_timestamp(scenario)) + + def test_none_when_stamped_dir_missing(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = load_scenario(write_case(root)) + self._pointer(root) + self.assertIsNone(_reusable_inspect_timestamp(scenario)) + + def test_reuses_valid_pointer(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = load_scenario(write_case(root)) + self._pointer(root) + (root / "out_20260825_123456").mkdir(parents=True) + self.assertEqual(_reusable_inspect_timestamp(scenario), "20260825_123456") + + +class PersistPointerTest(unittest.TestCase): + def test_load_failure_is_best_effort(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + _persist_inspect_pointer(root / "missing.json", root / "x.log", "20260825_123456") + self.assertFalse((root / "out.inspect.json").exists()) + + def test_write_failure_is_best_effort(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + with patch("pathlib.Path.write_text", side_effect=OSError("boom")): + _persist_inspect_pointer(scenario, root / "out_ts" / "log" / "x.log", "20260825_123456") + + +class InstallLoggerTest(unittest.TestCase): + def test_console_fallback_when_no_log_file(self): + _install_logger(None) + plugin_logger = logging.getLogger("ais_bench_prefix_cache") + self.assertIsInstance(plugin_logger.handlers[0], logging.StreamHandler) + self.assertFalse(plugin_logger.propagate) + + def test_file_handler_when_log_file_given(self): + with tempfile.TemporaryDirectory() as folder: + log_path = Path(folder) / "x.log" + _install_logger(log_path) + plugin_logger = logging.getLogger("ais_bench_prefix_cache") + self.assertIsInstance(plugin_logger.handlers[0], logging.FileHandler) + # 清理 FileHandler，避免句柄泄漏影响后续用例。 + _install_logger(None) + + +class MainFlowTest(unittest.TestCase): + def _fake_paths(self, scenario: Path) -> ArtifactPaths: + result_dir = scenario.parent / "result" + return ArtifactPaths( + result_dir / "x.full.jsonl", + result_dir / "x.requests.jsonl", + result_dir / "x.manifest.json", + result_dir / "x.analysis.json", + ) + + def test_validate_prints_result(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + manifest_path = root / "m.json" + manifest_path.write_text( + json.dumps({"run_id": "pc-test", "effective_config": {"run": {"output_dir": str(root / "out")}}}), + encoding="utf-8", + ) + stdout = io.StringIO() + with ( + patch("ais_bench_prefix_cache.cli.validate_artifacts", return_value={"ok": True, "rows": 0, "run_id": "pc-test"}), + redirect_stdout(stdout), + ): + self.assertEqual(main(["validate", "--manifest", str(manifest_path)]), 0) + output = json.loads(stdout.getvalue()) + self.assertTrue(output["ok"]) + + def test_validate_error_returns_two(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + manifest_path = root / "m.json" + manifest_path.write_text( + json.dumps({"run_id": "pc-test", "effective_config": {"run": {"output_dir": str(root / "out")}}}), + encoding="utf-8", + ) + stderr = io.StringIO() + with ( + patch("ais_bench_prefix_cache.cli.validate_artifacts", side_effect=PrefixCacheError("bad manifest")), + redirect_stderr(stderr), + ): + self.assertEqual(main(["validate", "--manifest", str(manifest_path)]), 2) + self.assertIn("ERROR: bad manifest", stderr.getvalue()) + + def test_prepare_error_closes_progress_and_returns_two(self): + with tempfile.TemporaryDirectory() as folder: + scenario = write_case(Path(folder)) + stderr = io.StringIO() + with ( + patch("ais_bench_prefix_cache.cli.new_execution_timestamp", return_value="20260825_123456"), + patch("ais_bench_prefix_cache.cli.prepare_scenario", side_effect=PrefixCacheError("boom")), + redirect_stderr(stderr), + ): + self.assertEqual(main(["prepare", "--scenario", str(scenario)]), 2) + self.assertIn("ERROR: boom", stderr.getvalue()) + + def test_prepare_reuses_inspect_timestamp(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + (root / "out_20260825_123456").mkdir(parents=True) + (root / "out.inspect.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "timestamp": "20260825_123456", + "run_id": "pc-test", + "output_dir": str(root / "out"), + } + ), + encoding="utf-8", + ) + calls = [] + + def fake_prepare(path, overwrite, progress, execution_timestamp): + calls.append((path, overwrite, execution_timestamp)) + return self._fake_paths(path) + + with ( + patch("ais_bench_prefix_cache.cli.new_execution_timestamp") as new_ts, + patch("ais_bench_prefix_cache.cli.prepare_scenario", side_effect=fake_prepare), + redirect_stdout(io.StringIO()), + ): + self.assertEqual(main(["prepare", "--scenario", str(scenario)]), 0) + new_ts.assert_not_called() + self.assertEqual(calls[0][2], "20260825_123456") + + def test_prepare_falls_back_to_new_timestamp_on_bad_scenario(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + bad_scenario = root / "bad.json" + bad_scenario.write_text("not json", encoding="utf-8") + stdout = io.StringIO() + with ( + patch("ais_bench_prefix_cache.cli.new_execution_timestamp", return_value="20260825_123456") as new_ts, + patch( + "ais_bench_prefix_cache.cli.prepare_scenario", + side_effect=lambda path, overwrite, progress, execution_timestamp: self._fake_paths(path), + ), + redirect_stdout(stdout), + ): + self.assertEqual(main(["prepare", "--scenario", str(bad_scenario)]), 0) + new_ts.assert_called_once() + output = json.loads(stdout.getvalue()) + self.assertNotIn("log", output) + + def test_inspect_without_log_file_omits_log_key(self): + with tempfile.TemporaryDirectory() as folder: + scenario = write_case(Path(folder)) + stdout = io.StringIO() + with ( + patch("ais_bench_prefix_cache.cli.new_execution_timestamp", return_value="20260825_123456"), + patch("ais_bench_prefix_cache.cli._resolve_log_file", return_value=None), + patch("ais_bench_prefix_cache.cli.inspect_scenario", return_value={"run_id": "pc-test", "sends_requests": False}), + redirect_stdout(stdout), + ): + self.assertEqual(main(["inspect", "--scenario", str(scenario)]), 0) + output = json.loads(stdout.getvalue()) + self.assertNotIn("log", output) + + def test_console_main_raises_system_exit(self): + with patch("ais_bench_prefix_cache.cli.main", return_value=3): + with self.assertRaises(SystemExit) as context: + console_main() + self.assertEqual(context.exception.code, 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/prefix_cache/tests/test_core.py b/plugins/prefix_cache/tests/test_core.py new file mode 100644 index 00000000..ad2098c2 --- /dev/null +++ b/plugins/prefix_cache/tests/test_core.py @@ -0,0 +1,321 @@ +import json +import itertools +import tempfile +import unittest +from pathlib import Path + +from ais_bench_prefix_cache.errors import ScenarioValidationError +from ais_bench_prefix_cache.generation import ( + RequestPlan, + assign_cold_routes, + assign_groups, + build_canonical_prefixes, + build_input_lengths, + build_output_lengths, + build_unique_seed, + build_unique_seed_tokens, + find_boundary_safe_token_ids, + GSMRecord, + load_gsm8k, + order_indices, + select_gsm8k, + simulate_theory, + solve_prefix_lengths, +) + + +class _FakeTokenizer: + """Ids 0..25 map to single letters; the pair 'ab' re-encodes as id 52.""" + + def __init__(self): + self.all_special_ids = [] + + def __len__(self): + return 64 + + def decode(self, token_ids, skip_special_tokens=False): + return "".join(chr(97 + (token_id % 26)) for token_id in token_ids) + + def encode(self, text, add_special_tokens=False): + ids = [ord(ch) - 97 for ch in text if "a" <= ch <= "z"] + out, i = [], 0 + while i < len(ids): + if ids[i] == 0 and i + 1 < len(ids) and ids[i + 1] == 1: + out.append(52) + i += 2 + else: + out.append(ids[i]) + i += 1 + return out +from ais_bench_prefix_cache.scenario import load_scenario + + +def scenario_dict(root: Path, mode: str = "cold", dp_size: int = 2) -> dict: + return { + "schema_version": "1.0", + "run": {"run_id": "pc-test", "random_seed": 42, "output_dir": str(root / "out")}, + "tokenizer": {"path": "fake", "block_size": 4}, + "corpus": {"path": str(root / "gsm.jsonl"), "field": "question", "selection": {"mode": "random"}}, + "requests": {"count": 8, "input_length": {"mode": "fixed", "value": 32}, "output_length": {"mode": "fixed", "value": 2}}, + "prefix_cache": {"mode": mode, "target_hit_rate": 0.5, "seed_blocks": 1, "groups": {"count": 2, "assignment": {"mode": "uniform"}}, "order": {"strategy": "interleave"}}, + "service": {"inference_url": "http://127.0.0.1:8000/v1/completions", "metrics_url": "http://127.0.0.1:8000/metrics", "reset_url": "http://127.0.0.1:8000/reset_prefix_cache", "model": "m", "dp_size": dp_size, "assume_empty_cache": False}, + "validation": {"target_warning_pp": 1.0, "actual_warning_pp": 5.0}, + } + + +class CoreTest(unittest.TestCase): + def test_omitted_values_use_current_example_defaults(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + path = root / "scenario.json" + path.write_text("{}", encoding="utf-8") + scenario = load_scenario(path) + effective = scenario.to_effective_dict() + self.assertEqual(effective["schema_version"], "1.0") + self.assertEqual(effective["run"]["run_id"], "gsm8k-prefix-cache-60") + self.assertEqual(effective["run"]["random_seed"], 42) + self.assertEqual(effective["run"]["output_dir"], str((root / "outputs/gsm8k-prefix-cache-60").resolve())) + self.assertEqual(effective["tokenizer"]["path"], "/home/weights/Qwen3.6-27B") + self.assertEqual(effective["tokenizer"]["block_size"], 16) + self.assertEqual(effective["corpus"]["path"], str((root / "GSM8K.jsonl").resolve())) + self.assertEqual(effective["corpus"]["selection"], {"mode": "random"}) + self.assertEqual(effective["requests"]["count"], 100) + self.assertEqual(effective["requests"]["input_length"], {"mode": "fixed", "value": 1024}) + self.assertEqual(effective["requests"]["output_length"], {"mode": "fixed", "value": 32}) + self.assertEqual(effective["prefix_cache"]["mode"], "warmup") + self.assertEqual(effective["prefix_cache"]["target_hit_rate"], 0.6) + self.assertEqual(effective["prefix_cache"]["minimum_non_shared_length"], 16) + self.assertEqual(effective["prefix_cache"]["groups"]["count"], 1) + self.assertEqual(effective["prefix_cache"]["groups"]["assignment"], {"mode": "uniform"}) + self.assertEqual(effective["service"]["dp_size"], 2) + self.assertEqual(effective["service"]["inference_url"], "http://127.0.0.1:8000/v1/completions") + self.assertEqual(effective["validation"], {"target_warning_pp": 1.0, "actual_warning_pp": 5.0}) + + def test_partially_empty_sections_receive_nested_defaults(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + path = root / "scenario.json" + path.write_text(json.dumps({ + "run": {}, + "tokenizer": {}, + "corpus": {"selection": {}}, + "requests": {"input_length": {}, "output_length": {}}, + "prefix_cache": {"groups": {"assignment": {}}, "order": {}}, + "service": {}, + "validation": {}, + }), encoding="utf-8") + effective = load_scenario(path).to_effective_dict() + self.assertEqual(effective["corpus"]["selection"]["mode"], "random") + self.assertEqual(effective["requests"]["input_length"], {"mode": "fixed", "value": 1024}) + self.assertEqual(effective["requests"]["output_length"], {"mode": "fixed", "value": 32}) + self.assertEqual(effective["prefix_cache"]["groups"]["count"], 1) + self.assertEqual(effective["prefix_cache"]["groups"]["assignment"]["mode"], "uniform") + self.assertEqual(effective["prefix_cache"]["order"]["strategy"], "interleave") + + def test_scenario_rejects_unknown_multi_instance_field(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + data = scenario_dict(root) + data["service"]["instances"] = ["forbidden"] + path = root / "scenario.json" + path.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(ScenarioValidationError, "service.instances"): + load_scenario(path) + + def test_lengths_are_deterministic(self): + ranges = {"mode": "range", "ranges": [{"min": 10, "max": 12, "count": 5}]} + self.assertEqual(build_input_lengths(ranges, 5, 7), build_input_lengths(ranges, 5, 7)) + normal = {"mode": "truncated_normal", "min": 2, "max": 8} + self.assertEqual(build_output_lengths(normal, 6, 9), build_output_lengths(normal, 6, 9)) + + def test_explicit_and_truncated_normal_input_lengths(self): + self.assertEqual( + build_input_lengths({"mode": "explicit", "values": [8, 12, 16]}, 3, 7), + [8, 12, 16], + ) + normal = {"mode": "truncated_normal", "min": 8, "max": 16, "mean": 12, "std": 2} + first = build_input_lengths(normal, 20, 9) + self.assertEqual(first, build_input_lengths(normal, 20, 9)) + self.assertTrue(all(8 <= value <= 16 for value in first)) + + def test_csv_lengths_and_specified_gsm_selection(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + (root / "lengths.csv").write_text( + "input_tokens,output_tokens\n16,2\n20,3\n", encoding="utf-8" + ) + self.assertEqual( + build_input_lengths({"mode": "csv", "path": str(root / "lengths.csv")}, 2, 1), + [16, 20], + ) + self.assertEqual( + build_output_lengths({"mode": "csv", "path": str(root / "lengths.csv")}, 2, 1), + [2, 3], + ) + corpus = root / "gsm.jsonl" + corpus.write_text( + "".join(json.dumps({"question": value}) + "\n" for value in ("alpha", "beta", "gamma")), + encoding="utf-8", + ) + records = load_gsm8k(corpus) + by_index = select_gsm8k(records, {"mode": "indices", "values": [2, 0]}, 3, 1) + self.assertEqual([row.line_index for row in by_index], [2, 0, 2]) + by_hash = select_gsm8k( + records, + {"mode": "question_sha256", "values": [records[1].question_sha256]}, + 2, + 1, + ) + self.assertEqual([row.line_index for row in by_hash], [1, 1]) + + def test_mixed_selection_allows_hashes_without_indices(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + corpus = root / "gsm.jsonl" + corpus.write_text( + "".join(json.dumps({"question": value}) + "\n" for value in ("alpha", "beta")), + encoding="utf-8", + ) + records = load_gsm8k(corpus) + selected = select_gsm8k( + records, + {"mode": "mixed", "indices": [], "question_sha256": [records[1].question_sha256]}, + 3, + 1, + ) + self.assertEqual([row.line_index for row in selected], [1, 1, 1]) + + def test_canonical_collision_uses_group_fallback(self): + class CharTokenizer: + def encode(self, text, add_special_tokens=False): + return [ord(char) for char in text] + + def decode(self, token_ids, skip_special_tokens=False): + return "".join(chr(token_id) for token_id in token_ids) + + record = GSMRecord(0, "same question", "hash") + canonical = build_canonical_prefixes( + CharTokenizer(), + {"group-0": [record], "group-1": [record]}, + {"group-0": 8, "group-1": 8}, + 4, + ) + self.assertNotEqual( + canonical["group-0"].token_ids[:4], canonical["group-1"].token_ids[:4] + ) + + def test_seed_capacity_is_rejected_during_scenario_validation(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + data = scenario_dict(root) + data["requests"]["input_length"] = {"mode": "fixed", "value": 2} + path = root / "scenario.json" + path.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(ScenarioValidationError, "at least 4 tokens"): + load_scenario(path) + + def test_seed_generation_round_trips(self): + tokenizer = _FakeTokenizer() + safe_ids = find_boundary_safe_token_ids(tokenizer, 8) + self.assertNotIn(52, safe_ids) + for index in range(20): + seed = build_unique_seed(tokenizer, safe_ids, f"r{index}", 4, 42) + self.assertEqual(tokenizer.encode(tokenizer.decode(seed), add_special_tokens=False), list(seed)) + + def test_groups_and_orders(self): + groups = assign_groups(10, {"count": 3, "assignment": {"mode": "weights", "weights": [0.5, 0.3, 0.2]}}, 42) + self.assertEqual([groups.count(f"group-{i}") for i in range(3)], [5, 3, 2]) + for strategy in ("sequential", "within_group_shuffle", "interleave", "global_shuffle"): + self.assertEqual(sorted(order_indices(groups, strategy, 42)), list(range(10))) + lengths = [30, 20, 10, 40, 15, 25] + grouped = ["g0", "g1", "g0", "g1", "g0", "g1"] + ordered = order_indices(grouped, "input_len_asc", 42, lengths) + for group in ("g0", "g1"): + selected = [lengths[index] for index in ordered if grouped[index] == group] + self.assertEqual(selected, sorted(selected)) + + def test_routes_and_dp_watermarks(self): + groups = ["g0", "g1", "g0", "g1", "g0", "g1", "g0", "g1"] + ranks, lanes = assign_cold_routes(groups, 2) + self.assertEqual(ranks, [0, 0, 1, 1, 0, 0, 1, 1]) + plans = [RequestPlan(f"r{i}", i, group, i // 2, rank, lane, 32, 32, 1, 16, 4, 12) for i, (group, rank, lane) in enumerate(zip(groups, ranks, lanes))] + theory = simulate_theory(plans, "cold") + self.assertEqual([row.theoretical_hit_tokens for row in theory.rows], [0, 0, 0, 0, 16, 16, 16, 16]) + + def test_unique_seed_and_solver(self): + seeds = build_unique_seed_tokens(list(range(32, 96)), [f"r{i}" for i in range(20)], 4, 42) + self.assertEqual(len(set(seeds.values())), 20) + groups = ["g0"] * 4 + ranks, lanes = assign_cold_routes(groups, 1) + result = solve_prefix_lengths([32] * 4, [1] * 4, groups, ranks, lanes, 4, 4, "cold", 0.5) + self.assertTrue(all(value % 4 == 0 for value in result.shared_prefix_tokens)) + self.assertLessEqual(result.effective_hit_rate, result.max_reachable_rate) + self.assertIn("g0", result.group_reachability) + + def test_solver_reserves_minimum_non_shared_length(self): + groups = ["g0", "g0"] + ranks, lanes = assign_cold_routes(groups, 1) + result = solve_prefix_lengths([20, 20], [1, 1], groups, ranks, lanes, 4, 8, "cold", 1.0) + self.assertTrue(all(prefix <= 12 for prefix in result.shared_prefix_tokens)) + self.assertFalse(result.target_reachable) + + def test_unreachable_large_cold_target_uses_exact_upper_boundary(self): + lengths = [ + 2504, 2150, 3174, 3051, 2962, 2619, 2467, 2404, 3776, 2178, + 2170, 2431, 2943, 3000, 2156, 2862, 3766, 2950, 3887, 3187, + ] + groups = ["g0"] * len(lengths) + ranks, lanes = assign_cold_routes(groups, 1) + result = solve_prefix_lengths( + lengths, [32] * len(lengths), groups, ranks, lanes, + 128, 128, "cold", 0.9, + ) + self.assertEqual(result.effective_hit_tokens, 48_896) + self.assertEqual(result.effective_hit_rate, result.max_reachable_rate) + self.assertFalse(result.target_reachable) + + def test_solver_matches_exhaustive_small_oracle(self): + lengths = [16, 20, 24] + outputs = [1, 1, 1] + groups = ["g0", "g0", "g0"] + ranks, lanes = assign_cold_routes(groups, 1) + for mode in ("cold", "warmup"): + for target in (0.2, 0.4, 0.6): + result = solve_prefix_lengths(lengths, outputs, groups, ranks if mode == "cold" else [None] * 3, lanes if mode == "cold" else [None] * 3, 4, 4, mode, target) + target_tokens = int(sum(lengths) * target + 0.5) + best_error = None + for prefixes in itertools.product(*(range(0, ((length - 4) // 4) * 4 + 1, 4) for length in lengths)): + plans = [RequestPlan(f"r{i}", i, "g0", i, ranks[i] if mode == "cold" else None, lanes[i] if mode == "cold" else None, lengths[i], lengths[i], 1, prefixes[i], 4, lengths[i] - prefixes[i] - 4) for i in range(3)] + warm = {"g0": max(prefixes)} if mode == "warmup" else None + hit = simulate_theory(plans, mode, warm).total_hit_tokens + error = abs(hit - target_tokens) + best_error = error if best_error is None else min(best_error, error) + self.assertEqual(abs(result.effective_hit_tokens - target_tokens), best_error, (mode, target, result)) + + def test_exact_cold_solver_matches_multi_lane_exhaustive_oracle(self): + lengths = [16, 20, 24, 28] + outputs = [1] * len(lengths) + groups = ["g0"] * len(lengths) + ranks = [0, 0, 1, 1] + lanes = [0, 1, 0, 1] + candidates = [range(0, ((length - 4) // 4) * 4 + 1, 4) for length in lengths] + for target in (0.0, 0.1, 0.35, 0.6, 0.95, 1.0): + result = solve_prefix_lengths(lengths, outputs, groups, ranks, lanes, 4, 4, "cold", target) + target_tokens = int(sum(lengths) * target + 0.5) + best_error = min( + abs( + simulate_theory([ + RequestPlan( + f"r{i}", i, "g0", i, ranks[i], lanes[i], lengths[i], lengths[i], + 1, prefixes[i], 4, lengths[i] - prefixes[i] - 4, + ) + for i in range(len(lengths)) + ], "cold").total_hit_tokens - target_tokens + ) + for prefixes in itertools.product(*candidates) + ) + self.assertEqual(abs(result.effective_hit_tokens - target_tokens), best_error) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/prefix_cache/tests/test_pipeline.py b/plugins/prefix_cache/tests/test_pipeline.py new file mode 100644 index 00000000..c0913306 --- /dev/null +++ b/plugins/prefix_cache/tests/test_pipeline.py @@ -0,0 +1,175 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from ais_bench_prefix_cache.artifacts import read_jsonl, sha256_file, validate_artifacts +from ais_bench_prefix_cache.pipeline import _length_summary, inspect_scenario, prepare_scenario +from tests.test_core import scenario_dict + + +class FakeTokenizer: + all_special_ids = list(range(32)) + + def __len__(self): + return 128 + + def encode(self, text, add_special_tokens=False): + return [ord(char) for char in text] + + def decode(self, token_ids, skip_special_tokens=False): + return "".join(chr(token_id) for token_id in token_ids) + + +def write_case(root: Path, mode: str = "cold") -> Path: + questions = ["alpha arithmetic question", "beta arithmetic question", "gamma arithmetic question", "delta arithmetic question"] + (root / "gsm.jsonl").write_text("".join(json.dumps({"question": value}) + "\n" for value in questions), encoding="utf-8") + data = scenario_dict(root, mode=mode) + path = root / "scenario.json" + path.write_text(json.dumps(data), encoding="utf-8") + return path + + +class PipelineTest(unittest.TestCase): + def test_four_artifacts_and_minimal_requests(self): + with tempfile.TemporaryDirectory() as folder: + scenario = write_case(Path(folder)) + paths = prepare_scenario(scenario, tokenizer_loader=lambda _: FakeTokenizer()) + self.assertTrue(all(path.exists() for path in paths.__dict__.values())) + self.assertTrue(all(path.parent.name == "result" for path in paths.__dict__.values())) + requests = read_jsonl(paths.requests) + self.assertEqual(set(requests[0]), {"question", "answer", "max_tokens"}) + first_line = paths.requests.read_text(encoding="utf-8").splitlines()[0] + self.assertEqual(list(json.loads(first_line)), ["question", "answer", "max_tokens"]) + self.assertTrue(validate_artifacts(paths.manifest)["ok"]) + + def test_prepare_reports_each_generated_prompt(self): + with tempfile.TemporaryDirectory() as folder: + scenario = write_case(Path(folder)) + events = [] + prepare_scenario( + scenario, + tokenizer_loader=lambda _: FakeTokenizer(), + progress=lambda completed, total: events.append((completed, total)), + ) + self.assertEqual(events, [(completed, 8) for completed in range(9)]) + + def test_prepare_appends_one_timestamp_to_run_id_and_output_dir(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + timestamp = "20260825_123456" + paths = prepare_scenario( + scenario, + tokenizer_loader=lambda _: FakeTokenizer(), + execution_timestamp=timestamp, + ) + expected_root = root / f"out_{timestamp}" + self.assertEqual(paths.manifest.parent, expected_root / "result") + self.assertEqual(paths.manifest.name, f"pc-test_{timestamp}.manifest.json") + manifest = json.loads(paths.manifest.read_text(encoding="utf-8")) + self.assertEqual(manifest["run_id"], f"pc-test_{timestamp}") + self.assertEqual(manifest["effective_config"]["run"]["output_dir"], str(expected_root)) + + def test_inspect_reports_reachability_without_persisting_run_artifacts(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + summary = inspect_scenario(scenario, tokenizer_loader=lambda _: FakeTokenizer()) + self.assertIn("reachable_min", summary) + self.assertIn("reachable_max", summary) + self.assertEqual(sum(summary["groups"].values()), 8) + self.assertFalse(summary["sends_requests"]) + self.assertFalse((root / "out").exists()) + + def test_deterministic_content_hashes(self): + hashes = [] + for _ in range(2): + with tempfile.TemporaryDirectory() as folder: + scenario = write_case(Path(folder)) + paths = prepare_scenario(scenario, tokenizer_loader=lambda _: FakeTokenizer()) + hashes.append((sha256_file(paths.full), sha256_file(paths.requests))) + self.assertEqual(hashes[0], hashes[1]) + + def test_manifest_does_not_persist_api_key(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + data = json.loads(scenario.read_text(encoding="utf-8")) + data["service"]["api_key"] = "do-not-write-this-secret" + scenario.write_text(json.dumps(data), encoding="utf-8") + paths = prepare_scenario(scenario, tokenizer_loader=lambda _: FakeTokenizer()) + manifest_text = paths.manifest.read_text(encoding="utf-8") + self.assertNotIn("do-not-write-this-secret", manifest_text) + manifest = json.loads(manifest_text) + self.assertTrue(manifest["effective_config"]["service"]["api_key_configured"]) + + def test_manifest_contains_detailed_audit_fields(self): + with tempfile.TemporaryDirectory() as folder: + scenario = write_case(Path(folder)) + paths = prepare_scenario(scenario, tokenizer_loader=lambda _: FakeTokenizer()) + manifest = json.loads(paths.manifest.read_text(encoding="utf-8")) + analysis = json.loads(paths.analysis.read_text(encoding="utf-8")) + rows = read_jsonl(paths.full) + self.assertIn("p99", manifest["requests"]["input_length_summary"]) + self.assertIn("bins", manifest["requests"]["input_length_summary"]) + self.assertIn("target_reachable", manifest["prefix_cache"]) + self.assertTrue(all("reachable_max" in group for group in manifest["groups"].values())) + self.assertEqual(manifest["divergence"]["collision_status"], "pass") + self.assertEqual(len({row["request_random_seed"] for row in rows}), len(rows)) + self.assertTrue(all(row["divergence_unique"] for row in rows)) + self.assertIn(analysis["validation"]["status"], {"PASS", "PASS_WITH_WARNING"}) + self.assertFalse(analysis["validation"]["affects_exit_code"]) + self.assertIn("target_signed_difference_pp", analysis) + + def test_length_summary_bins_report_actual_value_min_max(self): + # bins 中每项的 min/max 必须是桶内实际取值的最小/最大值， + # 而非桶边界：count == 1 时二者必须相等。 + values = [1024, 2048, 512, 768, 1024] + summary = _length_summary(values) + self.assertEqual( + summary["bins"], + [ + {"min": 512, "max": 512, "count": 1}, + {"min": 768, "max": 768, "count": 1}, + {"min": 1024, "max": 1024, "count": 2}, + {"min": 2048, "max": 2048, "count": 1}, + ], + ) + for entry in summary["bins"]: + if entry["count"] == 1: + self.assertEqual(entry["min"], entry["max"]) + self.assertEqual(sum(entry["count"] for entry in summary["bins"]), len(values)) + self.assertEqual(summary["min"], 512) + self.assertEqual(summary["max"], 2048) + + def test_warmup_manifest_has_every_group_rank(self): + with tempfile.TemporaryDirectory() as folder: + scenario = write_case(Path(folder), mode="warmup") + paths = prepare_scenario(scenario, tokenizer_loader=lambda _: FakeTokenizer()) + manifest = json.loads(paths.manifest.read_text(encoding="utf-8")) + pairs = {(row["group_id"], row["dp_rank"]) for row in manifest["warmup"]["plan"]} + self.assertEqual(pairs, {(f"group-{group}", rank) for group in range(2) for rank in range(2)}) + self.assertTrue(all(not row["included_in_formal_statistics"] for row in manifest["warmup"]["plan"])) + + def test_group_override_and_multi_sample_suffix(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + scenario = write_case(root) + data = json.loads(scenario.read_text(encoding="utf-8")) + data["prefix_cache"]["groups"]["overrides"] = { + "group-0": { + "input_length": {"mode": "fixed", "value": 80}, + "corpus_selection": {"mode": "indices", "values": [0, 1]}, + } + } + data["prefix_cache"]["target_hit_rate"] = 0.0 + scenario.write_text(json.dumps(data), encoding="utf-8") + paths = prepare_scenario(scenario, tokenizer_loader=lambda _: FakeTokenizer()) + rows = [row for row in read_jsonl(paths.full) if row["group_id"] == "group-0"] + self.assertTrue(all(row["actual_input_tokens"] == 80 for row in rows)) + self.assertTrue(any(len(set(row["gsm_indices"])) >= 2 for row in rows)) + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/prefix_cache/tests/test_scenario.py b/plugins/prefix_cache/tests/test_scenario.py new file mode 100644 index 00000000..222af643 --- /dev/null +++ b/plugins/prefix_cache/tests/test_scenario.py @@ -0,0 +1,381 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from ais_bench_prefix_cache.errors import ScenarioValidationError +from ais_bench_prefix_cache.scenario import ( + Scenario, + _minimum_input_tokens, + _mode, + _positive, + _require_dict, + _strict_keys, + _validate_input_config, + _validate_output_config, + load_scenario, + with_execution_timestamp, +) +from tests.test_core import scenario_dict + + +class ValidationHelpersTest(unittest.TestCase): + def test_require_dict_rejects_non_object(self): + with self.assertRaisesRegex(ScenarioValidationError, "must be an object"): + _require_dict([], "run") + + def test_strict_keys_rejects_unknown_top_level_and_nested(self): + with self.assertRaisesRegex(ScenarioValidationError, "unknown field: bogus"): + _strict_keys({"bogus": 1}, "") + with self.assertRaisesRegex(ScenarioValidationError, "unknown field: run.bogus"): + _strict_keys({"run": {"bogus": 1}}, "") + # 合法嵌套结构应静默通过。 + _strict_keys({"run": {"run_id": "x"}, "service": {"model": "m"}}, "") + + def test_positive_rejects_bool_zero_negative_and_non_int(self): + for value in (True, 0, -1, "3", 3.0): + with self.assertRaisesRegex(ScenarioValidationError, "positive integer"): + _positive(value, "x") + self.assertEqual(_positive(3, "x"), 3) + + def test_mode_rejects_unknown_mode(self): + with self.assertRaisesRegex(ScenarioValidationError, "mode must be one of"): + _mode({"mode": "bogus"}, {"fixed", "csv"}, "requests.input_length") + self.assertEqual(_mode({"mode": "csv"}, {"fixed", "csv"}, "p"), "csv") + + +class InputConfigTest(unittest.TestCase): + def test_explicit_mode_valid(self): + config = {"mode": "explicit", "values": [32, 32, 32, 32]} + _validate_input_config(config, "requests.input_length", Path("."), 4) + + def test_explicit_mode_rejects_bad_values(self): + with self.assertRaisesRegex(ScenarioValidationError, "non-empty list"): + _validate_input_config({"mode": "explicit", "values": []}, "p", Path("."), None) + with self.assertRaisesRegex(ScenarioValidationError, "non-empty list"): + _validate_input_config({"mode": "explicit", "values": "32"}, "p", Path("."), None) + with self.assertRaisesRegex(ScenarioValidationError, "positive integer"): + _validate_input_config({"mode": "explicit", "values": [32, -1]}, "p", Path("."), 2) + with self.assertRaisesRegex(ScenarioValidationError, "must equal expected request count"): + _validate_input_config({"mode": "explicit", "values": [32, 32]}, "p", Path("."), 3) + + def test_range_mode_valid(self): + config = { + "mode": "range", + "ranges": [ + {"min": 16, "max": 32, "count": 2}, + {"min": 64, "max": 64, "count": 2}, + ], + } + _validate_input_config(config, "p", Path("."), 4) + + def test_range_mode_rejects_bad_ranges(self): + with self.assertRaisesRegex(ScenarioValidationError, "non-empty list"): + _validate_input_config({"mode": "range", "ranges": []}, "p", Path("."), None) + with self.assertRaisesRegex(ScenarioValidationError, "invalid fields"): + _validate_input_config({"mode": "range", "ranges": ["x"]}, "p", Path("."), None) + with self.assertRaisesRegex(ScenarioValidationError, "invalid fields"): + _validate_input_config({"mode": "range", "ranges": [{"min": 1, "max": 2, "count": 1, "bogus": 1}]}, "p", Path("."), None) + with self.assertRaisesRegex(ScenarioValidationError, "must be >= min"): + _validate_input_config({"mode": "range", "ranges": [{"min": 8, "max": 4, "count": 1}]}, "p", Path("."), None) + with self.assertRaisesRegex(ScenarioValidationError, "must equal expected request count"): + _validate_input_config({"mode": "range", "ranges": [{"min": 8, "max": 16, "count": 2}]}, "p", Path("."), 3) + + def test_truncated_normal_valid(self): + _validate_input_config({"mode": "truncated_normal", "min": 16, "max": 64, "std": 8}, "p", Path("."), None) + # std 缺省时不校验，也合法。 + _validate_input_config({"mode": "truncated_normal", "min": 16, "max": 64}, "p", Path("."), None) + + def test_truncated_normal_rejects_bad_bounds(self): + with self.assertRaisesRegex(ScenarioValidationError, "must be >= min"): + _validate_input_config({"mode": "truncated_normal", "min": 64, "max": 16}, "p", Path("."), None) + with self.assertRaisesRegex(ScenarioValidationError, "std must be positive"): + _validate_input_config({"mode": "truncated_normal", "min": 16, "max": 64, "std": 0}, "p", Path("."), None) + + def test_csv_mode_resolves_path(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + config = {"mode": "csv", "path": "lens.csv"} + _validate_input_config(config, "p", root, None) + self.assertEqual(config["path"], str((root / "lens.csv").resolve())) + + def test_csv_mode_rejects_missing_path(self): + for path in (None, ""): + with self.assertRaisesRegex(ScenarioValidationError, "non-empty string"): + _validate_input_config({"mode": "csv", "path": path}, "p", Path("."), None) + + def test_unknown_field_rejected(self): + with self.assertRaisesRegex(ScenarioValidationError, "unknown field: p.bogus"): + _validate_input_config({"mode": "fixed", "value": 32, "bogus": 1}, "p", Path("."), None) + + +class OutputConfigTest(unittest.TestCase): + def test_uniform_valid(self): + _validate_output_config({"mode": "uniform", "min": 1, "max": 8}, "p", Path(".")) + + def test_uniform_rejects_max_below_min(self): + with self.assertRaisesRegex(ScenarioValidationError, "must be >= min"): + _validate_output_config({"mode": "uniform", "min": 8, "max": 1}, "p", Path(".")) + + def test_truncated_normal_rejects_non_positive_std(self): + with self.assertRaisesRegex(ScenarioValidationError, "std must be positive"): + _validate_output_config({"mode": "truncated_normal", "min": 1, "max": 8, "std": 0}, "p", Path(".")) + + def test_csv_mode_resolves_path(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + config = {"mode": "csv", "path": "lens.csv"} + _validate_output_config(config, "p", root) + self.assertEqual(config["path"], str((root / "lens.csv").resolve())) + + def test_csv_mode_rejects_missing_path(self): + with self.assertRaisesRegex(ScenarioValidationError, "non-empty string"): + _validate_output_config({"mode": "csv", "path": ""}, "p", Path(".")) + + def test_unknown_field_rejected(self): + with self.assertRaisesRegex(ScenarioValidationError, "unknown field: p.bogus"): + _validate_output_config({"mode": "uniform", "min": 1, "max": 8, "bogus": 1}, "p", Path(".")) + + +class MinimumInputTokensTest(unittest.TestCase): + def test_mode_dispatch(self): + self.assertEqual(_minimum_input_tokens({"mode": "fixed", "value": 32}, "p"), 32) + self.assertEqual(_minimum_input_tokens({"mode": "explicit", "values": [64, 32]}, "p"), 32) + self.assertEqual(_minimum_input_tokens({"mode": "range", "ranges": [{"min": 48}]}, "p"), 48) + self.assertEqual(_minimum_input_tokens({"mode": "truncated_normal", "min": 24}, "p"), 24) + + def test_csv_reads_min_of_aliases(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + for alias in ("input_prompt_tokens", "content_tokens", "input_tokens"): + csv_path = root / f"{alias}.csv" + csv_path.write_text(f"{alias}\n32\n48\n", encoding="utf-8") + self.assertEqual(_minimum_input_tokens({"mode": "csv", "path": str(csv_path)}, "p"), 32) + + def test_csv_handles_bom_header(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + csv_path = root / "lens.csv" + csv_path.write_text("input_tokens\n40\n", encoding="utf-8") + self.assertEqual(_minimum_input_tokens({"mode": "csv", "path": str(csv_path)}, "p"), 40) + + def test_csv_empty_rows_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + csv_path = root / "lens.csv" + csv_path.write_text("input_tokens\n", encoding="utf-8") + with self.assertRaisesRegex(ScenarioValidationError, "at least one data row"): + _minimum_input_tokens({"mode": "csv", "path": str(csv_path)}, "p") + + def test_csv_missing_column_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + csv_path = root / "lens.csv" + csv_path.write_text("bogus\n32\n", encoding="utf-8") + with self.assertRaisesRegex(ScenarioValidationError, "requires one of columns"): + _minimum_input_tokens({"mode": "csv", "path": str(csv_path)}, "p") + + def test_csv_invalid_value_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + csv_path = root / "lens.csv" + csv_path.write_text("input_tokens\nabc\n", encoding="utf-8") + with self.assertRaisesRegex(ScenarioValidationError, "invalid input length"): + _minimum_input_tokens({"mode": "csv", "path": str(csv_path)}, "p") + + def test_csv_unreadable_rejected(self): + with self.assertRaisesRegex(ScenarioValidationError, "cannot be read"): + _minimum_input_tokens({"mode": "csv", "path": "/nonexistent/lens.csv"}, "p") + + +class ScenarioObjectTest(unittest.TestCase): + def test_section_and_to_effective_dict_are_independent(self): + scenario = Scenario(Path("s.json"), {"run": {"run_id": "r"}}) + self.assertEqual(scenario.section("run")["run_id"], "r") + effective = scenario.to_effective_dict() + effective["run"]["run_id"] = "changed" + self.assertEqual(scenario.run_id, "r") + + def test_with_execution_timestamp_rejects_malformed_timestamps(self): + scenario = Scenario(Path("s.json"), {"run": {"run_id": "r", "output_dir": "/tmp/out"}}) + for timestamp in ("2026-08-25 12:00:00", "20260825123456", "abcd1234_5678"): + with self.assertRaisesRegex(ScenarioValidationError, "YYYYMMDD_HHMMSS"): + with_execution_timestamp(scenario, timestamp) + + def test_with_execution_timestamp_rejects_empty_output_dir_name(self): + scenario = Scenario(Path("s.json"), {"run": {"run_id": "r", "output_dir": "/"}}) + with self.assertRaisesRegex(ScenarioValidationError, "final directory name"): + with_execution_timestamp(scenario, "20260825_123456") + + +class LoadScenarioErrorTest(unittest.TestCase): + def _write(self, folder, content) -> Path: + path = Path(folder) / "scenario.json" + path.write_text(content if isinstance(content, str) else json.dumps(content), encoding="utf-8") + return path + + def _expect(self, content, pattern): + with tempfile.TemporaryDirectory() as folder: + with self.assertRaisesRegex(ScenarioValidationError, pattern): + load_scenario(self._write(folder, content)) + + def test_non_object_scenario_rejected(self): + self._expect("[1, 2]", "must be an object") + + def test_unknown_field_rejected(self): + self._expect({"run": {}, "bogus": 1}, "unknown field: bogus") + self._expect({"run": {"bogus": 1}}, "unknown field: run.bogus") + + def test_schema_version_rejected(self): + self._expect({"schema_version": "2.0"}, "schema_version") + + def test_run_id_rejected(self): + self._expect({"run": {"run_id": ""}}, "run.run_id") + self._expect({"run": {"run_id": 123}}, "run.run_id") + + def test_random_seed_rejected(self): + self._expect({"run": {"random_seed": True}}, "random_seed") + self._expect({"run": {"random_seed": "42"}}, "random_seed") + + def test_block_size_rejected(self): + self._expect({"tokenizer": {"block_size": 0}}, "block_size") + + def test_input_length_must_be_object(self): + self._expect({"requests": {"input_length": "x"}}, "input_length must be an object") + + def test_target_hit_rate_rejected(self): + self._expect({"prefix_cache": {"target_hit_rate": 1.5}}, "target_hit_rate") + self._expect({"prefix_cache": {"target_hit_rate": True}}, "target_hit_rate") + + def test_seed_blocks_rejected(self): + self._expect({"prefix_cache": {"seed_blocks": 0}}, "seed_blocks") + + def test_minimum_non_shared_below_seed_rejected(self): + self._expect({"prefix_cache": {"minimum_non_shared_length": 8}}, "non_shared_length") + + def test_input_too_small_for_reserved_region_rejected(self): + self._expect( + {"requests": {"input_length": {"mode": "fixed", "value": 8}}}, + "at least 16 tokens", + ) + + def test_groups_must_be_object(self): + self._expect({"prefix_cache": {"groups": "x"}}, "groups must be an object") + + def test_groups_count_rejected(self): + self._expect({"prefix_cache": {"groups": {"count": 0}}}, "groups.count") + + def test_assignment_mode_rejected(self): + self._expect( + {"prefix_cache": {"groups": {"assignment": {"mode": "bogus"}}}}, + "assignment.mode must be one of", + ) + + def test_overrides_must_be_object(self): + self._expect({"prefix_cache": {"groups": {"overrides": []}}}, "overrides must be an object") + + def test_override_invalid_group_id_rejected(self): + self._expect( + {"prefix_cache": {"groups": {"overrides": {"bad": {}}}}}, + "invalid Prefix Group override id", + ) + self._expect( + {"prefix_cache": {"groups": {"count": 2, "overrides": {"group-9": {}}}}}, + "invalid Prefix Group override id", + ) + + def test_override_must_be_object(self): + self._expect( + {"prefix_cache": {"groups": {"overrides": {"group-0": []}}}}, + "must be an object", + ) + + def test_override_unknown_field_rejected(self): + self._expect( + {"prefix_cache": {"groups": {"overrides": {"group-0": {"bogus": 1}}}}}, + "unknown field: prefix_cache.groups.overrides.group-0.bogus", + ) + + def test_override_input_length_too_small_rejected(self): + self._expect( + {"prefix_cache": {"groups": {"overrides": {"group-0": {"input_length": {"mode": "fixed", "value": 8}}}}}}, + "at least 16 tokens", + ) + + def test_override_corpus_selection_mode_rejected(self): + self._expect( + {"prefix_cache": {"groups": {"overrides": {"group-0": {"corpus_selection": {"mode": "bogus"}}}}}}, + "corpus_selection.mode must be one of", + ) + + def test_order_strategy_rejected(self): + self._expect({"prefix_cache": {"order": {"strategy": "bogus"}}}, "order.strategy") + + def test_service_fields_rejected(self): + for field in ("inference_url", "metrics_url", "model"): + self._expect({"service": {field: ""}}, f"service.{field}") + + def test_unreadable_file_rejected(self): + with self.assertRaisesRegex(ScenarioValidationError, "cannot read scenario"): + load_scenario(Path("/nonexistent/scenario.json")) + + def test_invalid_json_rejected(self): + self._expect("not json", "cannot read scenario") + + +class LoadScenarioMultimodeTest(unittest.TestCase): + def test_valid_explicit_zipf_scenario_with_overrides(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + data = scenario_dict(root) + data["requests"] = { + "count": 4, + "input_length": {"mode": "explicit", "values": [32, 32, 32, 32]}, + "output_length": {"mode": "uniform", "min": 1, "max": 8}, + } + data["corpus"]["selection"] = {"mode": "indices", "indices": [0, 1]} + data["prefix_cache"] = { + "mode": "cold", + "target_hit_rate": 0.5, + "seed_blocks": 1, + "groups": { + "count": 2, + "assignment": {"mode": "zipf", "exponent": 1.2}, + "overrides": { + "group-0": { + "input_length": {"mode": "fixed", "value": 32}, + "output_length": {"mode": "uniform", "min": 1, "max": 4}, + "corpus_selection": {"mode": "indices"}, + } + }, + }, + "order": {"strategy": "input_len_asc"}, + } + path = root / "scenario.json" + path.write_text(json.dumps(data), encoding="utf-8") + scenario = load_scenario(path) + # 多态配置不应被注入 fixed 模式的默认字段。 + self.assertNotIn("value", scenario.data["requests"]["input_length"]) + self.assertNotIn("value", scenario.data["requests"]["output_length"]) + self.assertEqual(scenario.data["prefix_cache"]["groups"]["assignment"]["mode"], "zipf") + self.assertEqual(scenario.data["prefix_cache"]["order"]["strategy"], "input_len_asc") + + def test_valid_csv_input_scenario(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + csv_path = root / "lens.csv" + csv_path.write_text("input_tokens\n32\n40\n", encoding="utf-8") + data = scenario_dict(root) + data["requests"] = { + "count": 8, + "input_length": {"mode": "csv", "path": str(csv_path)}, + "output_length": {"mode": "fixed", "value": 2}, + } + path = root / "scenario.json" + path.write_text(json.dumps(data), encoding="utf-8") + scenario = load_scenario(path) + self.assertEqual(scenario.data["requests"]["input_length"]["path"], str(csv_path.resolve())) + + +if __name__ == "__main__": + unittest.main() From 52204f0d15d2f66b2ea3160721670a3ff18a8249 Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:07:59 +0800 Subject: [PATCH 14/25] docs: remove redundant no-response-anomaly absence notes from cli args The parenthetical noting the absence of a --no-response-anomaly flag adds noise rather than information: omitting --response-anomaly already implies detection is disabled. --- docs/source_en/base_tutorials/all_params/cli_args.md | 2 +- docs/source_zh_cn/base_tutorials/all_params/cli_args.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index 57a48709..2d275218 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -37,7 +37,7 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | | `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. Note: In performance evaluation scenarios, an excessively high concurrency may cause resource contention among different processes, leading to inaccurate test results. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | -| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it disables detection (there is no `--no-response-anomaly` form). Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | +| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it disables detection. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | | `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | ### API Model Common Override Parameters diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index 7d616d32..77f4d95d 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -36,7 +36,7 @@ ais_bench [OPTIONS] | `--num-prompts` | 指定数据集测评条数（按照数据集顺序选取），需传入正整数，超过数据集条数或默认情况下表示对全量数据集进行测评。 | `--num-prompts 500` | | `--max-num-workers` | 并行任务数，范围 `[1, CPU 核数]`，默认 `1`。在指定`--debug`时配置无效，所有任务串行执行。注意：性能测评场景下，并发数过高可能会导致不同进程出现资源抢占，导致测试结果失真。 | `--max-num-workers 2` | |`--num-warmups`|发送请求前预热次数，按照数据集顺序选取数据进行测试，大概num-warmups大于数据集条数时，会循环发送数据集中数据。默认 `1`；若设为0，则不预热。如果warmup阶段所有请求失败，后续推理任务将不会执行。| `--num-warmups 10` | -| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加则关闭（无 `--no-response-anomaly` 形态）。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。 | `--response-anomaly` | +| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加则关闭。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。 | `--response-anomaly` | | `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。命令行配置优先于配置文件，默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | ### API 模型通用覆盖参数 From 755400bfbc8bf0d14e474aa0d8c90949159d953c Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:36:21 +0800 Subject: [PATCH 15/25] docs: move response anomaly detection guide into a dedicated advanced tutorial The response anomaly detection section was previously embedded in cli_args.md, where it split the Configuration Constant File Parameters heading from its content (zh) and mixed feature-level guidance into a parameter reference page. Move it into a standalone advanced tutorial page (zh/en) restructured as a complete guide from dependency installation to configuration, fix the orphaned heading, point the --response-anomaly CLI rows to the new page, update all cross links (models/mode/accuracy_benchmark), and register the new page in the toctree. --- .../response_anomaly_detection.md | 217 ++++++++++++++++++ .../base_tutorials/all_params/cli_args.md | 110 +-------- .../base_tutorials/all_params/mode.md | 2 +- .../base_tutorials/all_params/models.md | 2 +- .../scenes_intro/accuracy_benchmark.md | 2 +- docs/source_en/index.rst | 1 + .../response_anomaly_detection.md | 217 ++++++++++++++++++ .../base_tutorials/all_params/cli_args.md | 110 +-------- .../base_tutorials/all_params/mode.md | 2 +- .../base_tutorials/all_params/models.md | 2 +- .../scenes_intro/accuracy_benchmark.md | 2 +- docs/source_zh_cn/index.rst | 1 + 12 files changed, 444 insertions(+), 224 deletions(-) create mode 100644 docs/source_en/advanced_tutorials/response_anomaly_detection.md create mode 100644 docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md new file mode 100644 index 00000000..c9afdd11 --- /dev/null +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -0,0 +1,217 @@ +# Response Anomaly Detection + +## Overview + +AISBench integrates msProbe's `ILLDetector` to automatically detect generation anomalies in LLM responses while running inference evaluations. Detection results cover the following types: + +| `anomaly_type` | `anomaly_type_name` | Meaning | +| -------------- | ------------------- | ------- | +| 0 | `normal` | Normal | +| 1 | `rare_character` | Rare character | +| 2 | `garbled` | Garbled text | +| 3 | `repetition` | Repetition | +| 4 | `nan_value` | NaN value | + +**Anomaly detection results do not affect the original evaluation metrics**: anomalous Cases are not rewritten as inference failures; accuracy/performance metrics are computed as usual, and anomaly information is an independent audit result. + +Detection runs through msProbe's `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)`; all three file paths can be configured in AISBench (see [Configuration](#configuration)). + +--- + +## Prerequisites + +1. **Inference backend**: Response anomaly detection currently supports only the vLLM Chat API model configurations `vllm_api_general_chat`, `vllm_api_stream_chat`, and `vllm_api_stream_chat_multiturn`. Other model backends are not supported yet. +2. **Evaluation modes**: Only the `all`, `infer`, and `infer_judge` generation chains are supported; the `perf` / `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do not support this feature, and enabling it raises an explicit error during config initialization. +3. **Service requirements**: The service response must contain `token_ids` (or `tokens`) and `topk_logprobs` fields; Cases missing these fields are recorded with a `skipped` status. +4. **Optional dependency**: The `response_anomaly` extra must be installed (see the next section). + +> 💡 Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages, guaranteeing that detection results and payload archives are on disk when the inference stage exits. See the [mode documentation](../base_tutorials/all_params/mode.md) for the full mode support matrix. + +--- + +## Installing Dependencies + +Response anomaly detection relies on the optional package `mindstudio-probe`, installed through the AISBench extra: + +```bash +pip install 'ais-bench-benchmark[response_anomaly]' +``` + +During installation, pip downloads and builds the pinned msProbe source from GitCode, so the environment needs Git and network access. + +Without this dependency, evaluation still runs normally, but all Cases are marked with the `unavailable` status in the detection results (see [Detection Results](#detection-results)); re-run after installation to restore detection. + +--- + +## Quick Start + +**1. Add the `response_anomaly` field to the model config** (the minimal configuration only needs one of `model_name` or `model_path`): + +```python +models = [ + dict( + abbr='qwen3-30b', + attr='service', + response_anomaly=dict( + model_name='Qwen3-30B-A3B', # or provide only model_path; the model name is taken from it + ), + ), +] +``` + +**2. Run the evaluation with `--response-anomaly` on the command line**: + +```bash +ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --response-anomaly +``` + +> ⚠️ The **feature switch is command-line only**: add `--response-anomaly` to the command to enable detection (omit it to disable). The `response_anomaly` entry in the config file carries only non-switch settings (`payload_retention`, `payload_storage`, etc.). + +**3. Inspect the detection results**: after inference and detection finish, the results are written to `/response_anomaly//.jsonl`, one Case per line (see [Runtime Flow and On-Disk Layout](#runtime-flow-and-on-disk-layout) for the full layout). + +--- + +## Configuration + +### Global Configuration + +The global `response_anomaly` config controls the payload retention policy and storage format: + +```python +response_anomaly = dict( + payload_retention='anomalies', # all | anomalies | none + payload_storage=dict( + format='jsonl', + compression='zstd', + compression_level=3, + rows_per_shard=2000, + ), +) +``` + +`payload_retention` determines which payloads are kept after detection: + +| Value | Behavior | +| ----- | -------- | +| `all` | Keeps every payload and atomically promotes the staging data to the official archive after detection without re-compressing | +| `anomalies` (default) | Keeps only detected anomalies plus detection-failed/unavailable Cases | +| `none` | Keeps no payload | + +All three modes keep the standalone detection results. The command-line parameter `--response-anomaly-payload-retention` overrides the config file value. + +> 💡 Runtime details: results are written to disk in batches, and the status is refreshed at most once per second; msProbe token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. + +### Model-Level Configuration (msProbe) + +Model-specific msProbe configuration goes into the model config: + +```python +models = [ + dict( + abbr='qwen3-30b', + attr='service', + response_anomaly=dict( + model_name="", # Model name, for example Qwen3-30B-A3B + model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs + msprobe_config_path="", # Optional; msProbe algorithm-threshold config.yaml path for manual threshold tuning + msprobe_mtype_path="", # Optional; msProbe mtype_config.json path mapping model names to BOS/EOS token ids + msprobe_token2category_dir="", # Optional; msProbe token2category directory holding per-model token-id-to-character-category maps + ), + ), +] +``` + +**Rules for `model_name`**: when it is not configured explicitly, the **model name is taken from the model path** (`model_path`, or the model `path` field; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`), matching the config generator's default. When neither `model_name` nor a model path is available (e.g. only the explicit msProbe resource paths are configured), the task fails fast at startup and asks for an explicit `model_name`, instead of silently running detection with a wrong model name. + +`model_name` must be consistent with msProbe's `mtype_config.json` and the token-category mapping. + +When `msprobe_mtype_path` / `msprobe_token2category_dir` are not provided, the default files inside the msProbe package are used. + +### Auto-Generation and Manual Generation of msProbe Configs + +When `model_path` is configured, the msProbe configs are auto-generated into `/response_anomaly_config//` (auto-generation never overwrites an existing `config.yaml`, so manually tuned thresholds are preserved). They can also be generated manually: + +```bash +ais_bench-gen-response-anomaly-config \ + --model-path /home/Qwen3-30B-A3B \ + --model-name Qwen3-30B-A3B \ + --output-dir ./msprobe_configs +``` + +--- + +## Runtime Flow and On-Disk Layout + +### Automatic Request Parameter Injection + +When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the service inference requests; the value is constrained by the detection algorithm and cannot be configured externally. For vLLM backends, `return_token_ids=True` and `return_tokens_as_token_ids=True` are also appended to obtain token ids; if the server version is too old to support these parameters, requests may fail — upgrade vLLM in that case. + +### Detection Flow + +1. **Inference stage**: the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start; +2. **Detection stage**: after inference finishes, the detection thread streams and decompresses the staging data and calls msProbe; detection results are written to `response_anomaly//.jsonl`; +3. **Archive finalization**: after detection, the staging data is retained or cleaned according to `payload_retention`. + +The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. + +### On-Disk Layout + +Files produced by detection are laid out under `` as follows: + +```text +/ +├── predictions//.jsonl # Inference results (lightweight, no token/logprob payload) +├── response_anomaly/ +│ └── / +│ ├── .jsonl # Detection results, one Case per line +│ ├── payload_staging// # Transient staging during inference; cleaned after detection +│ │ └── part-*.jsonl.zst +│ └── payload// # Payload archive; absent when payload_retention is none +│ ├── payload_manifest.json # Archive manifest (per-shard rows, sizes, sha256) +│ └── part-*.jsonl.zst # Compressed payload shards (at most rows_per_shard Cases each) +├── response_anomaly_config/ # Present only when auto-generated via model_path +│ └── / +│ ├── configs/ +│ │ ├── config.yaml # Detection algorithm thresholds (never overwritten once present) +│ │ └── mtype_config.json # Model name to BOS/EOS token id mapping +│ └── token2category/ +│ └── _.json # Token id to character-category mapping +└── logs/ + └── response_anomaly//.out # Detection-specific log +``` + +Path-by-path notes: + +- **Detection results** `response_anomaly//.jsonl`: one Case per line; field semantics are described in [Detection Results](#detection-results). +- **Payload archive** `response_anomaly//payload//`: `all` keeps every Case, `anomalies` keeps only anomalous plus detection-failed/unavailable Cases, and `none` keeps nothing (the directory does not exist). To read the data, decompress the `part-*.jsonl.zst` shards with zstandard and parse each line as JSON; `payload_manifest.json` records per-shard row counts, sizes, and sha256 checksums for integrity verification. Note: under `anomalies`, even when there is nothing to retain, an archive directory containing only an empty manifest is still published to indicate the archiving flow completed successfully — it is not a leftover file. +- **Transient files**: `payload_staging/` receives payload records during inference and is cleaned automatically after detection; `..payload-build-*` build directories left by an interrupted detection are cleaned automatically when the next detection starts; `status_tmp/tmp_ResponseAnomaly.json` is a runtime status file (detection progress and per-type statistics) removed together with the status directory when the workflow ends. +- **Auto-generated msProbe configs** `response_anomaly_config//`: generated only when `model_path` is configured and explicit mtype/token2category paths are absent. An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. +- **Detection log** `logs/response_anomaly//.out`: records the detection run for the model/dataset group, including detector initialization failures and per-Case failure reasons. + +--- + +## Detection Results + +The detection results `response_anomaly//.jsonl` contain one Case per line with `id`, `uuid`, `is_anomaly`, `anomaly_type` (0: normal, 1: rare character, 2: garbled, 3: repetition, 4: NaN value), `anomaly_type_name` (the type-name string such as `normal`/`garbled`/`repetition`, which is more convenient for statistics), and `detection_status`. + +The `detection_status` values in the detection results are: + +| Status | Meaning | Troubleshooting | +| --- | --- | --- | +| `completed` | msProbe was invoked and returned a detection result | Nothing to do | +| `skipped` | The inference response did not carry token ids or top-k logprobs | Check whether the service supports and returns `logprobs` / `top_logprobs` / token id fields | +| `unavailable` | `mindstudio-probe` (the response_anomaly extra) is not installed | Install the optional dependency per the [Installation Guide](../get_started/install.md) and re-run | +| `failed` | An exception occurred during invocation or input conversion | Check the `reason` field of the Case result (error type and summary) and the detection logs | + +Detection-specific logs are located at `/logs/response_anomaly//.out`; detection progress and per-type statistics can also be found in the `/status_tmp/tmp_ResponseAnomaly.json` status file. + +--- + +## Resuming Runs + +When using `--reuse`, existing detection results are inherited by matching both the Case `id` and `uuid` (a changed `uuid` means the Case was re-inferred, so it never gets a stale result): + +- Cases with `completed` status are not re-detected; +- Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume. + +`--reuse` runs must keep the payload retention policy of the original work directory. diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index 2d275218..f8b23f26 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -37,7 +37,7 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | | `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. Note: In performance evaluation scenarios, an excessively high concurrency may cause resource contention among different processes, leading to inaccurate test results. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | -| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it disables detection. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. | `--response-anomaly` | +| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it disables detection. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. See 📚 [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for detailed usage. | `--response-anomaly` | | `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | ### API Model Common Override Parameters @@ -96,111 +96,3 @@ The currently supported parameter configurations are as follows: | `MAX_CHUNK_SIZE` | Maximum cache size for a single chunk returned by the streaming inference model backend. The default value is 65535 bytes (64KB). | `(0, 16777216]` (Unit: Byte) | | `REQUEST_TIME_OUT` | Timeout period for the client to wait for a response after sending a request. The default value is None, meaning infinite waiting (always waiting for the model to return results). | `None` or `>0` (Unit: seconds) | | `LOG_LEVEL` | Log level, optional values: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`. Default value: `INFO`. | `[DEBUG, INFO, WARNING, ERROR, CRITICAL]` | - -## Response Anomaly Detection Configuration - -Response anomaly detection currently supports only the vLLM Chat API model configurations `vllm_api_general_chat`, `vllm_api_stream_chat`, and `vllm_api_stream_chat_multiturn`. Other model backends are not supported yet. - -The **feature switch is command-line only**: add `--response-anomaly` to the command to enable detection (omit it to disable). The `response_anomaly` entry in the config file carries only non-switch settings (`payload_retention`, `payload_storage`, etc.): - -```python -response_anomaly = dict( - payload_retention='anomalies', # all | anomalies | none - payload_storage=dict( - format='jsonl', - compression='zstd', - compression_level=3, - rows_per_shard=2000, - ), -) -``` - -Model-specific msProbe configuration goes into the model config: - -```python -models = [ - dict( - abbr='qwen3-30b', - attr='service', - response_anomaly=dict( - model_name="", # Model name, for example Qwen3-30B-A3B - model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs - msprobe_config_path="", # Optional; msProbe algorithm-threshold config.yaml path for manual threshold tuning - msprobe_mtype_path="", # Optional; msProbe mtype_config.json path mapping model names to BOS/EOS token ids - msprobe_token2category_dir="", # Optional; msProbe token2category directory holding per-model token-id-to-character-category maps - ), - ), -] -``` - -When `msprobe_mtype_path` / `msprobe_token2category_dir` are not provided, the default files inside the msProbe package are used; when `model_path` is configured, configs are auto-generated into `/response_anomaly_config//` (auto-generation never overwrites an existing `config.yaml`, so manually tuned thresholds are preserved). They can also be generated manually: - -```bash -ais_bench-gen-response-anomaly-config \ - --model-path /home/Qwen3-30B-A3B \ - --model-name Qwen3-30B-A3B \ - --output-dir ./msprobe_configs -``` - -When `model_name` is not configured explicitly, the **model name is taken from the model path** (`model_path`, or the model `path` field; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`), matching the config generator's default. When neither `model_name` nor a model path is available (e.g. only the explicit msProbe resource paths are configured), the task fails fast at startup and asks for an explicit `model_name`, instead of silently running detection with a wrong model name. - -When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the service inference requests; the value is constrained by the detection algorithm and cannot be configured externally. For vLLM backends, `return_token_ids=True` and `return_tokens_as_token_ids=True` are also appended to obtain token ids; if the server version is too old to support these parameters, requests may fail — upgrade vLLM in that case. During inference, the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start. After inference finishes, the detection thread streams and decompresses the staging data and calls msProbe; detection results are written to `response_anomaly//.jsonl`. Each Case contains `id`, `uuid`, `is_anomaly`, `anomaly_type` (0: normal, 1: rare character, 2: garbled, 3: repetition, 4: NaN value), `anomaly_type_name` (the type-name string such as `normal`/`garbled`/`repetition`, which is more convenient for statistics), and `detection_status`. After detection, the staging data is retained or cleaned according to `payload_retention`. The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. - -The `detection_status` values in the detection results are: - -| Status | Meaning | Troubleshooting | -| --- | --- | --- | -| `completed` | msProbe was invoked and returned a detection result | Nothing to do | -| `skipped` | The inference response did not carry token ids or top-k logprobs | Check whether the service supports and returns `logprobs` / `top_logprobs` / token id fields | -| `unavailable` | `mindstudio-probe` (the response_anomaly extra) is not installed | Install the optional dependency per the [Installation Guide](../../get_started/install.md) and re-run | -| `failed` | An exception occurred during invocation or input conversion | Check the `reason` field of the Case result (error type and summary) and the detection logs | - -**Anomaly detection results do not affect the original evaluation metrics**: anomalous Cases are not rewritten as inference failures; accuracy/performance metrics are computed as usual, and anomaly information is an independent audit result. Detection-specific logs are located at `/logs/response_anomaly//.out`; detection progress and per-type statistics can also be found in the `/status_tmp/tmp_ResponseAnomaly.json` status file. - -```python -response_anomaly = dict( - payload_retention='anomalies', # all | anomalies | none - payload_storage=dict( - format='jsonl', - compression='zstd', - compression_level=3, - rows_per_shard=2000, - ), -) -``` - -`all` keeps every payload and atomically promotes the staging data to the official archive after detection without re-compressing; `anomalies` keeps only detected anomalies plus detection-failed/unavailable Cases; `none` keeps no payload. All three modes keep the standalone detection results. `--reuse` must keep the retention policy of the original work directory. Results are written to disk in batches, and the status is refreshed at most once per second; msProbe token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. - -Files produced by detection are laid out under `` as follows: - -```text -/ -├── predictions//.jsonl # Inference results (lightweight, no token/logprob payload) -├── response_anomaly/ -│ └── / -│ ├── .jsonl # Detection results, one Case per line -│ ├── payload_staging// # Transient staging during inference; cleaned after detection -│ │ └── part-*.jsonl.zst -│ └── payload// # Payload archive; absent when payload_retention is none -│ ├── payload_manifest.json # Archive manifest (per-shard rows, sizes, sha256) -│ └── part-*.jsonl.zst # Compressed payload shards (at most rows_per_shard Cases each) -├── response_anomaly_config/ # Present only when auto-generated via model_path -│ └── / -│ ├── configs/ -│ │ ├── config.yaml # Detection algorithm thresholds (never overwritten once present) -│ │ └── mtype_config.json # Model name to BOS/EOS token id mapping -│ └── token2category/ -│ └── _.json # Token id to character-category mapping -└── logs/ - └── response_anomaly//.out # Detection-specific log -``` - -Path-by-path notes: - -- **Detection results** `response_anomaly//.jsonl`: one Case per line; field semantics are described in the `detection_status` table and the Case field notes above. -- **Payload archive** `response_anomaly//payload//`: `all` keeps every Case, `anomalies` keeps only anomalous plus detection-failed/unavailable Cases, and `none` keeps nothing (the directory does not exist). To read the data, decompress the `part-*.jsonl.zst` shards with zstandard and parse each line as JSON; `payload_manifest.json` records per-shard row counts, sizes, and sha256 checksums for integrity verification. Note: under `anomalies`, even when there is nothing to retain, an archive directory containing only an empty manifest is still published to indicate the archiving flow completed successfully — it is not a leftover file. -- **Transient files**: `payload_staging/` receives payload records during inference and is cleaned automatically after detection; `..payload-build-*` build directories left by an interrupted detection are cleaned automatically when the next detection starts; `status_tmp/tmp_ResponseAnomaly.json` is a runtime status file (detection progress and per-type statistics) removed together with the status directory when the workflow ends. -- **Auto-generated msProbe configs** `response_anomaly_config//`: generated only when `model_path` is configured and explicit mtype/token2category paths are absent. An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. -- **Detection log** `logs/response_anomaly//.out`: records the detection run for the model/dataset group, including detector initialization failures and per-Case failure reasons. - -Detection runs through msProbe's `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)`; all three file paths can be configured in AISBench. Install the optional dependencies first: `pip install 'ais-bench-benchmark[response_anomaly]'`. During installation, pip downloads and builds the pinned msProbe source from GitCode, so the environment needs Git and network access. The service response must contain `token_ids` (or `tokens`) and `topk_logprobs`; Cases missing these fields are recorded with a `skipped` status. `model_name` must be consistent with msProbe's `mtype_config.json` and the token-category mapping. When using `--reuse`, existing detection results are inherited by matching both the Case `id` and `uuid` (a changed `uuid` means the Case was re-inferred, so it never gets a stale result); Cases with `completed` status are not re-detected, while Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume. diff --git a/docs/source_en/base_tutorials/all_params/mode.md b/docs/source_en/base_tutorials/all_params/mode.md index 6e6f4e6f..e0b01214 100644 --- a/docs/source_en/base_tutorials/all_params/mode.md +++ b/docs/source_en/base_tutorials/all_params/mode.md @@ -108,7 +108,7 @@ outputs/default/ ### Response Anomaly Detection Mode Support (Optional) -msProbe response anomaly detection (`--response-anomaly`) is only supported in the `all`, `infer`, and `infer_judge` generation chains: detection starts after the inference stage finishes and is **serially bound to the inference stage** — the workflow waits for detection to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages, guaranteeing that detection results and payload archives are on disk when the inference stage exits. The `perf` and `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do **not** support this feature; enabling it raises an explicit error during config initialization. This feature requires the service model to return token ids and top-k logprobs; see [Response Anomaly Detection Configuration](./cli_args.md#response-anomaly-detection-configuration) for details. +msProbe response anomaly detection (`--response-anomaly`) is only supported in the `all`, `infer`, and `infer_judge` generation chains: detection starts after the inference stage finishes and is **serially bound to the inference stage** — the workflow waits for detection to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages, guaranteeing that detection results and payload archives are on disk when the inference stage exits. The `perf` and `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do **not** support this feature; enabling it raises an explicit error during config initialization. This feature requires the service model to return token ids and top-k logprobs; see [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for details. ## Performance Evaluation Scenarios ### Perf Mode diff --git a/docs/source_en/base_tutorials/all_params/models.md b/docs/source_en/base_tutorials/all_params/models.md index 0f5a5d34..6f787b25 100644 --- a/docs/source_en/base_tutorials/all_params/models.md +++ b/docs/source_en/base_tutorials/all_params/models.md @@ -107,7 +107,7 @@ The description of configurable parameters for the service-oriented inference ba - Setting `batch_size` too large may result in high CPU usage. Please configure it reasonably based on hardware conditions. - The default service address used by the service-oriented inference evaluation API is `localhost:8080`. In actual use, you need to modify it to the IP and port of the service-oriented backend according to the actual deployment. - When using an IPv6 literal (such as `::1` or `2001:db8::1`) as `host_ip`, the tool will automatically wrap it in brackets in the generated URL (for example, `http://[2001:db8::1]:8080/`), so you do not need to manually add brackets in the configuration. -- When response anomaly detection (`response_anomaly`) is enabled, the service must return token ids and top-k logprobs; AISBench automatically adds `logprobs=True` and a fixed `top_logprobs=20` to the inference requests, and Cases whose responses lack these fields are marked as `skipped` in the detection results. See [Response Anomaly Detection Configuration](./cli_args.md#response-anomaly-detection-configuration) for details. +- When response anomaly detection (`response_anomaly`) is enabled, the service must return token ids and top-k logprobs; AISBench automatically adds `logprobs=True` and a fixed `top_logprobs=20` to the inference requests, and Cases whose responses lack these fields are marked as `skipped` in the detection results. See [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for details. ### Multi-LoRA Routing diff --git a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md index 6be83033..79f7a2df 100644 --- a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -322,7 +322,7 @@ After the resumption is completed, the accuracy results of all requests will be > ⚠️ Note: Resumption after interruption and retesting of failed cases may change the order of requests, which may cause slight fluctuations in results. -> 💡 When [response anomaly detection](../all_params/cli_args.md#response-anomaly-detection-configuration) is enabled, resumption also inherits existing detection results: Cases with `completed` status are not re-detected by msProbe, while Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume; existing anomaly counts are accumulated into the final statistics. +> 💡 When [response anomaly detection](../../advanced_tutorials/response_anomaly_detection.md) is enabled, resumption also inherits existing detection results: Cases with `completed` status are not re-detected by msProbe, while Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume; existing anomaly counts are accumulated into the final statistics. 💡[Multi-Task Evaluation](#multi-task-evaluation) also supports resumption after interruption and retesting of failed cases for all or part of the tasks. diff --git a/docs/source_en/index.rst b/docs/source_en/index.rst index f4b36c86..9d961965 100644 --- a/docs/source_en/index.rst +++ b/docs/source_en/index.rst @@ -57,6 +57,7 @@ To help you quickly get started with AISBench Benchmark Tool, we recommend learn advanced_tutorials/judge_model_evaluate advanced_tutorials/spec_decode advanced_tutorials/prefix_cache + advanced_tutorials/response_anomaly_detection .. toctree:: :maxdepth: 2 diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md new file mode 100644 index 00000000..7369d179 --- /dev/null +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -0,0 +1,217 @@ +# 推理响应异常检测 + +## 概述 + +AISBench 集成 msProbe 的 `ILLDetector`，支持在推理评测的同时自动检测大模型响应中的生成异常。检测结果覆盖以下类型： + +| `anomaly_type` | `anomaly_type_name` | 含义 | +| -------------- | ------------------- | ---- | +| 0 | `normal` | 正常 | +| 1 | `rare_character` | 生僻字 | +| 2 | `garbled` | 乱码 | +| 3 | `repetition` | 重复 | +| 4 | `nan_value` | NaN Value | + +**异常检测结果不影响原有评测指标**：异常 Case 不会被改写为推理失败，精度/性能指标照常计算，异常信息是独立的审计结果。 + +检测通过 msProbe 的 `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)` 完成，三个文件路径均可由 AISBench 配置（见[配置说明](#配置说明)）。 + +--- + +## 前置条件 + +1. **推理后端**：当前响应异常检测仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 +2. **评测模式**：仅支持 `all`、`infer`、`infer_judge` 普通生成链路；`perf` / `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路不支持，启用时会在配置初始化阶段显式报错。 +3. **服务端要求**：服务响应必须包含 `token_ids`（或 `tokens`）和 `topk_logprobs` 字段；缺少这些字段的 Case 会以 `skipped` 状态落盘。 +4. **可选依赖**：需安装 `response_anomaly` extra（见下节）。 + +> 💡 检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。模式支持范围详见[模式说明](../base_tutorials/all_params/mode.md)。 + +--- + +## 安装依赖 + +响应异常检测依赖可选包 `mindstudio-probe`，通过 AISBench 的 extra 安装： + +```bash +pip install 'ais-bench-benchmark[response_anomaly]' +``` + +安装过程中 pip 会从 GitCode 下载并构建已固定提交的 msProbe 源码，因此安装环境需要 Git 和网络访问。 + +未安装该依赖时评测流程仍可正常运行，但所有 Case 的检测结果会标记为 `unavailable` 状态（见[检测结果说明](#检测结果说明)）；安装后重跑即可恢复检测。 + +--- + +## 快速使用 + +**1. 在模型配置中添加 `response_anomaly` 字段**（最小配置只需提供 `model_name` 或 `model_path` 之一）： + +```python +models = [ + dict( + abbr='qwen3-30b', + attr='service', + response_anomaly=dict( + model_name='Qwen3-30B-A3B', # 或仅提供 model_path，自动取其中的模型名称 + ), + ), +] +``` + +**2. 在命令行增加 `--response-anomaly` 运行评测**： + +```bash +ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --response-anomaly +``` + +> ⚠️ 异常检测的**功能开关仅支持命令行**：在命令中增加 `--response-anomaly` 开启（不增加即关闭）；配置文件中的 `response_anomaly` 仅用于非开关类配置（`payload_retention`、`payload_storage` 等）。 + +**3. 查看检测结果**：推理与检测结束后，检测结果位于 `/response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`，每行一个 Case（完整落盘结构见[运行流程与落盘结构](#运行流程与落盘结构)）。 + +--- + +## 配置说明 + +### 全局配置 + +全局 `response_anomaly` 配置控制 payload 的保留策略与存储格式： + +```python +response_anomaly = dict( + payload_retention='anomalies', # all | anomalies | none + payload_storage=dict( + format='jsonl', + compression='zstd', + compression_level=3, + rows_per_shard=2000, + ), +) +``` + +`payload_retention` 决定检测完成后 payload 的保留范围： + +| 取值 | 行为 | +| ---- | ---- | +| `all` | 保存全部 payload，检测后直接将 staging 原子转为正式归档，不会二次压缩 | +| `anomalies`（默认） | 只保存已检出异常以及检测失败/不可用 Case | +| `none` | 不保存 payload | + +三种模式都保留独立检测结果。命令行参数 `--response-anomaly-payload-retention` 可覆盖配置文件取值。 + +> 💡 运行期细节：检测结果按批写盘，状态最多每秒刷新一次；msProbe token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 + +### 模型级配置（msProbe） + +模型相关的 msProbe 配置放在模型配置中： + +```python +models = [ + dict( + abbr='qwen3-30b', + attr='service', + response_anomaly=dict( + model_name="", # 填写模型名称，如 Qwen3-30B-A3B + model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 + msprobe_config_path="", # 可选，msProbe 算法阈值配置 config.yaml 路径，用于手工调优检测阈值 + msprobe_mtype_path="", # 可选，msProbe 模型名与 BOS/EOS token id 映射文件 mtype_config.json 路径 + msprobe_token2category_dir="", # 可选，msProbe token2category 目录，存放各模型的 token id 到字符类别映射 + ), + ), +] +``` + +**`model_name` 的取值规则**：未显式配置时自动取模型路径（`model_path` 或模型 `path` 字段）中的**模型名称**（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`），与配置生成工具的默认取值一致。当既未配置 `model_name`、也没有可用的模型路径（如仅显式配置 msProbe 三件套路径）时，任务启动即报错，要求显式配置 `model_name`，避免以错误的模型名静默运行导致检测失效。 + +`model_name` 需与 msProbe 的 `mtype_config.json` 配置以及 token 分类映射保持一致。 + +未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时回退到 msProbe 包内默认文件。 + +### msProbe 配置的自动生成与手动生成 + +配置了 `model_path` 时，msProbe 配置会自动生成到 `/response_anomaly_config/<模型 abbr>/`（自动生成不会覆盖已存在的 `config.yaml`，便于保留手工调优的阈值）。也可手动生成： + +```bash +ais_bench-gen-response-anomaly-config \ + --model-path /home/Qwen3-30B-A3B \ + --model-name Qwen3-30B-A3B \ + --output-dir ./msprobe_configs +``` + +--- + +## 运行流程与落盘结构 + +### 请求参数自动注入 + +启用后，AISBench 会在服务推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`；该值由检测算法约束，不支持外部配置。对 vLLM 后端还会追加 `return_token_ids=True` 与 `return_tokens_as_token_ids=True` 以获取 token id，服务端版本过低不支持这些参数时请求可能失败，需升级 vLLM。 + +### 检测流程 + +1. **推理阶段**：完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果； +2. **检测阶段**：推理结束后，检测线程流式解压 staging 数据并调用 msProbe，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`； +3. **归档收尾**：检测完成后按 `payload_retention` 保留或清理 staging。 + +状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 + +### 落盘结构 + +检测相关文件在 `` 下的落盘结构如下： + +```text +/ +├── predictions/<模型 abbr>/<数据集 abbr>.jsonl # 推理结果（轻量，不含 token/logprobs payload） +├── response_anomaly/ +│ └── <模型 abbr>/ +│ ├── <数据集 abbr>.jsonl # 检测结果，每行一个 Case +│ ├── payload_staging/<数据集 abbr>/ # 推理期间的临时存放区，检测完成后自动清理 +│ │ └── part-*.jsonl.zst +│ └── payload/<数据集 abbr>/ # payload 归档；payload_retention 为 none 时不存在 +│ ├── payload_manifest.json # 归档清单（分片行数、大小、sha256） +│ └── part-*.jsonl.zst # 压缩 payload 分片（每片最多 rows_per_shard 条 Case） +├── response_anomaly_config/ # 仅配置 model_path 自动生成时存在 +│ └── <模型 abbr>/ +│ ├── configs/ +│ │ ├── config.yaml # 检测算法阈值配置（已存在时不覆盖） +│ │ └── mtype_config.json # 模型名与 BOS/EOS token id 映射 +│ └── token2category/ +│ └── <模型名>_<词表大小>.json # token id 到字符类别映射 +└── logs/ + └── response_anomaly/<模型 abbr>/<数据集 abbr>.out # 检测专属日志 +``` + +各路径说明： + +- **检测结果** `response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`：每行一个 Case 的检测结果，字段含义见[检测结果说明](#检测结果说明)。 +- **payload 归档** `response_anomaly/<模型 abbr>/payload/<数据集 abbr>/`：`all` 保留全部 Case，`anomalies` 只保留异常及检测失败/不可用 Case，`none` 不保留（目录不存在）。读取时用 zstandard 解压 `part-*.jsonl.zst` 分片后逐行解析 JSON；`payload_manifest.json` 记录每个分片的行数、大小与 sha256 校验值，可用于完整性校验。注意：`anomalies` 模式下即使无任何需保留的 Case，仍会发布一个仅含空 manifest 的归档目录，表示归档流程已成功完成，不是残留文件。 +- **临时文件**：`payload_staging/` 在推理期间逐条接收 payload 写入，检测完成后自动清理；检测中断后残留的 `.<数据集>.payload-build-*` 构建目录会在下次检测启动时自动清理；`status_tmp/tmp_ResponseAnomaly.json` 为运行期状态文件（检测进度与类型统计），工作流结束后随状态目录一并清理。 +- **自动生成的 msProbe 配置** `response_anomaly_config/<模型 abbr>/`：仅当配置了 `model_path` 且未显式提供 mtype/token2category 路径时生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 +- **检测日志** `logs/response_anomaly/<模型 abbr>/<数据集 abbr>.out`：记录对应模型/数据集组的检测过程，含检测器初始化失败与单 Case 失败的具体原因。 + +--- + +## 检测结果说明 + +检测结果 `response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl` 每行一个 Case，包含 `id`、`uuid`、`is_anomaly`、`anomaly_type`（0：正常，1：生僻字，2：乱码，3：重复，4：NaN Value）、`anomaly_type_name`（类型名字符串，如 `normal`/`garbled`/`repetition`，统计时更常用）和 `detection_status`。 + +检测结果中的 `detection_status` 取值如下： + +| 状态 | 含义 | 排查建议 | +| --- | --- | --- | +| `completed` | 已调用 msProbe 并得到检测结果 | 无需处理 | +| `skipped` | 推理响应未携带 token id 或 top-k logprobs | 检查服务端是否支持并返回 `logprobs` / `top_logprobs` / token id 字段 | +| `unavailable` | 未安装 `mindstudio-probe`（response_anomaly extra） | 参考[安装指南](../get_started/install.md)安装可选依赖后重跑 | +| `failed` | 调用或输入转换发生异常 | 查看该 Case 结果中的 `reason` 字段（保存错误类型与摘要）及检测日志 | + +检测专属日志位于 `/logs/response_anomaly/<模型>/<数据集>.out`，检测进度与类型统计也可在 `/status_tmp/tmp_ResponseAnomaly.json` 状态文件中查看。 + +--- + +## 断点续跑 + +使用 `--reuse` 时，已有检测结果按 Case 的 `id` + `uuid` 双键匹配继承（`uuid` 变化说明该 Case 已重新推理，不会错挂旧结果）： + +- `completed` 状态的 Case 不会重复检测； +- `skipped` / `failed` / `unavailable` 状态的 Case 会在续跑中重新检测。 + +`--reuse` 续跑必须沿用原工作目录的 payload 保留策略。 diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index 77f4d95d..09c49c05 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -36,7 +36,7 @@ ais_bench [OPTIONS] | `--num-prompts` | 指定数据集测评条数（按照数据集顺序选取），需传入正整数，超过数据集条数或默认情况下表示对全量数据集进行测评。 | `--num-prompts 500` | | `--max-num-workers` | 并行任务数，范围 `[1, CPU 核数]`，默认 `1`。在指定`--debug`时配置无效，所有任务串行执行。注意：性能测评场景下，并发数过高可能会导致不同进程出现资源抢占，导致测试结果失真。 | `--max-num-workers 2` | |`--num-warmups`|发送请求前预热次数，按照数据集顺序选取数据进行测试，大概num-warmups大于数据集条数时，会循环发送数据集中数据。默认 `1`；若设为0，则不预热。如果warmup阶段所有请求失败，后续推理任务将不会执行。| `--num-warmups 10` | -| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加则关闭。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。 | `--response-anomaly` | +| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加则关闭。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。详细用法见 📚 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 | `--response-anomaly` | | `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。命令行配置优先于配置文件，默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | ### API 模型通用覆盖参数 @@ -81,114 +81,6 @@ ais_bench [OPTIONS] ## 配置常量文件参数 -## 推理响应异常检测配置 - -当前响应异常检测仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 - -异常检测的**功能开关仅支持命令行**：在命令中增加 `--response-anomaly` 开启（不增加即关闭）。配置文件中的 `response_anomaly` 仅用于非开关类配置（`payload_retention`、`payload_storage` 等）： - -```python -response_anomaly = dict( - payload_retention='anomalies', # all | anomalies | none - payload_storage=dict( - format='jsonl', - compression='zstd', - compression_level=3, - rows_per_shard=2000, - ), -) -``` - -模型相关的 msProbe 配置放在模型配置中： - -```python -models = [ - dict( - abbr='qwen3-30b', - attr='service', - response_anomaly=dict( - model_name="", # 填写模型名称，如 Qwen3-30B-A3B - model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 - msprobe_config_path="", # 可选，msProbe 算法阈值配置 config.yaml 路径，用于手工调优检测阈值 - msprobe_mtype_path="", # 可选，msProbe 模型名与 BOS/EOS token id 映射文件 mtype_config.json 路径 - msprobe_token2category_dir="", # 可选，msProbe token2category 目录，存放各模型的 token id 到字符类别映射 - ), - ), -] -``` - -未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时回退到 msProbe 包内默认文件；配置了 `model_path` 时会自动生成到 `/response_anomaly_config/<模型 abbr>/`（自动生成不会覆盖已存在的 `config.yaml`，便于保留手工调优的阈值）。也可手动生成： - -```bash -ais_bench-gen-response-anomaly-config \ - --model-path /home/Qwen3-30B-A3B \ - --model-name Qwen3-30B-A3B \ - --output-dir ./msprobe_configs -``` - -`model_name` 未显式配置时自动取模型路径（`model_path` 或模型 `path` 字段）中的**模型名称**（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`），与配置生成工具的默认取值一致。当既未配置 `model_name`、也没有可用的模型路径（如仅显式配置 msprobe 三件套路径）时，任务启动即报错，要求显式配置 `model_name`，避免以错误的模型名静默运行导致检测失效。 - -启用后，AISBench 会在服务推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`；该值由检测算法约束，不支持外部配置。对 vLLM 后端还会追加 `return_token_ids=True` 与 `return_tokens_as_token_ids=True` 以获取 token id，服务端版本过低不支持这些参数时请求可能失败，需升级 vLLM。推理阶段将完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果。推理结束后，检测线程流式解压 staging 数据并调用 msProbe，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`；每个 Case 包含 `id`、`uuid`、`is_anomaly`、`anomaly_type`（0：正常，1：生僻字，2：乱码，3：重复，4：NaN Value）、`anomaly_type_name`（类型名字符串，如 `normal`/`garbled`/`repetition`，统计时更常用）和 `detection_status`。检测完成后按 `payload_retention` 保留或清理 staging。状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 - -检测结果中的 `detection_status` 取值如下： - -| 状态 | 含义 | 排查建议 | -| --- | --- | --- | -| `completed` | 已调用 msProbe 并得到检测结果 | 无需处理 | -| `skipped` | 推理响应未携带 token id 或 top-k logprobs | 检查服务端是否支持并返回 `logprobs` / `top_logprobs` / token id 字段 | -| `unavailable` | 未安装 `mindstudio-probe`（response_anomaly extra） | 参考[安装指南](../../get_started/install.md)安装可选依赖后重跑 | -| `failed` | 调用或输入转换发生异常 | 查看该 Case 结果中的 `reason` 字段（保存错误类型与摘要）及检测日志 | - -**异常检测结果不影响原有评测指标**：异常 Case 不会被改写为推理失败，精度/性能指标照常计算，异常信息是独立的审计结果。检测专属日志位于 `/logs/response_anomaly/<模型>/<数据集>.out`，检测进度与类型统计也可在 `/status_tmp/tmp_ResponseAnomaly.json` 状态文件中查看。 - -```python -response_anomaly = dict( - payload_retention='anomalies', # all | anomalies | none - payload_storage=dict( - format='jsonl', - compression='zstd', - compression_level=3, - rows_per_shard=2000, - ), -) -``` - -`all` 保存全部 payload，检测后直接将 staging 原子转为正式归档，不会二次压缩；`anomalies` 只保存已检出异常以及检测失败/不可用 Case；`none` 不保存 payload。三种模式都保留独立检测结果。`--reuse` 必须沿用原工作目录的保留策略。检测结果按批写盘，状态最多每秒刷新一次；msProbe token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 - -检测相关文件在 `` 下的落盘结构如下： - -```text -/ -├── predictions/<模型 abbr>/<数据集 abbr>.jsonl # 推理结果（轻量，不含 token/logprobs payload） -├── response_anomaly/ -│ └── <模型 abbr>/ -│ ├── <数据集 abbr>.jsonl # 检测结果，每行一个 Case -│ ├── payload_staging/<数据集 abbr>/ # 推理期间的临时存放区，检测完成后自动清理 -│ │ └── part-*.jsonl.zst -│ └── payload/<数据集 abbr>/ # payload 归档；payload_retention 为 none 时不存在 -│ ├── payload_manifest.json # 归档清单（分片行数、大小、sha256） -│ └── part-*.jsonl.zst # 压缩 payload 分片（每片最多 rows_per_shard 条 Case） -├── response_anomaly_config/ # 仅配置 model_path 自动生成时存在 -│ └── <模型 abbr>/ -│ ├── configs/ -│ │ ├── config.yaml # 检测算法阈值配置（已存在时不覆盖） -│ │ └── mtype_config.json # 模型名与 BOS/EOS token id 映射 -│ └── token2category/ -│ └── <模型名>_<词表大小>.json # token id 到字符类别映射 -└── logs/ - └── response_anomaly/<模型 abbr>/<数据集 abbr>.out # 检测专属日志 -``` - -各路径说明： - -- **检测结果** `response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`：每行一个 Case 的检测结果，字段含义见上文 `detection_status` 表与 Case 字段说明。 -- **payload 归档** `response_anomaly/<模型 abbr>/payload/<数据集 abbr>/`：`all` 保留全部 Case，`anomalies` 只保留异常及检测失败/不可用 Case，`none` 不保留（目录不存在）。读取时用 zstandard 解压 `part-*.jsonl.zst` 分片后逐行解析 JSON；`payload_manifest.json` 记录每个分片的行数、大小与 sha256 校验值，可用于完整性校验。注意：`anomalies` 模式下即使无任何需保留的 Case，仍会发布一个仅含空 manifest 的归档目录，表示归档流程已成功完成，不是残留文件。 -- **临时文件**：`payload_staging/` 在推理期间逐条接收 payload 写入，检测完成后自动清理；检测中断后残留的 `.<数据集>.payload-build-*` 构建目录会在下次检测启动时自动清理；`status_tmp/tmp_ResponseAnomaly.json` 为运行期状态文件（检测进度与类型统计），工作流结束后随状态目录一并清理。 -- **自动生成的 msProbe 配置** `response_anomaly_config/<模型 abbr>/`：仅当配置了 `model_path` 且未显式提供 mtype/token2category 路径时生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 -- **检测日志** `logs/response_anomaly/<模型 abbr>/<数据集 abbr>.out`：记录对应模型/数据集组的检测过程，含检测器初始化失败与单 Case 失败的具体原因。 - -检测通过 msProbe 的 `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)` 完成，三个文件路径均可由 AISBench 配置。请先安装 AISBench 的可选依赖：`pip install 'ais-bench-benchmark[response_anomaly]'`。安装过程中 pip 会从 GitCode 下载并构建已固定提交的 msProbe 源码，因此安装环境需要 Git 和网络访问。服务响应必须包含 `token_ids`（或 `tokens`）和 `topk_logprobs`；缺少这些字段的 Case 会以 `skipped` 状态落盘。`model_name` 需与 msProbe 的 `mtype_config.json` 配置以及 token 分类映射保持一致。使用 `--reuse` 时，已有检测结果按 Case 的 `id` + `uuid` 双键匹配继承（`uuid` 变化说明该 Case 已重新推理，不会错挂旧结果），`completed` 状态的 Case 不会重复检测；`skipped` / `failed` / `unavailable` 状态的 Case 会在续跑中重新检测。 - 部分全局常量不区分任务类型，推荐保持默认；如需自定义，可编辑常量文件：[`global_consts.py`](https://github.com/AISBench/benchmark/tree/master/ais_bench/benchmark/global_consts.py)配置。 当前支持的参数配置如下： | 参数名| 说明| 取值范围 / 要求 | diff --git a/docs/source_zh_cn/base_tutorials/all_params/mode.md b/docs/source_zh_cn/base_tutorials/all_params/mode.md index fcd25206..00bc19f1 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/mode.md +++ b/docs/source_zh_cn/base_tutorials/all_params/mode.md @@ -107,7 +107,7 @@ outputs/default/ ### 响应异常检测模式支持（可选） -msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测在推理阶段结束后启动，并**串行绑定在推理阶段内**——工作流会等待检测完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，具体配置请参考 [推理响应异常检测配置](./cli_args.md#推理响应异常检测配置)。 +msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测在推理阶段结束后启动，并**串行绑定在推理阶段内**——工作流会等待检测完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，具体配置请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 ## 性能评测场景 ### perf模式 diff --git a/docs/source_zh_cn/base_tutorials/all_params/models.md b/docs/source_zh_cn/base_tutorials/all_params/models.md index 04cc082d..09c31866 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/models.md +++ b/docs/source_zh_cn/base_tutorials/all_params/models.md @@ -98,7 +98,7 @@ models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.c - `request_rate` 受硬件性能影响，可通过增加 📚 [WORKERS_NUM](./cli_args.md#配置常量文件参数) 提高并发能力。 - `request_rate` 功能可能被`traffic_cfg`项覆盖，具体原因请参考 🔗 [请求速率(RPS)分布控制及可视化说明中的参数解读章节](../../advanced_tutorials/rps_distribution.md#参数解读)。 - 当数据集含 timestamp 且模型配置中 **use_timestamp** 为 True 时，请求按 timestamp 发送，**request_rate** 与 **traffic_cfg** 将被忽略。 -- 使用响应异常检测（`response_anomaly`）时，服务端必须返回 token id 与 top-k logprobs；启用检测后 AISBench 会自动在推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`，服务响应缺少这些字段的 Case 检测结果标记为 `skipped`。详细配置请参考 [推理响应异常检测配置](./cli_args.md#推理响应异常检测配置)。 +- 使用响应异常检测（`response_anomaly`）时，服务端必须返回 token id 与 top-k logprobs；启用检测后 AISBench 会自动在推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`，服务响应缺少这些字段的 Case 检测结果标记为 `skipped`。详细配置请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 - `batch_size` 设置过大可能导致 CPU 占用过高，请根据硬件条件合理配置。 - 服务化推理评测 API 默认使用的服务地址为 `localhost:8080`。实际使用时需根据实际部署修改为服务化后端的 IP 和端口。 - 当使用 IPv6 字面量（如 `::1`、`2001:db8::1`）作为 `host_ip` 时，工具会在生成的访问 URL 中自动为其添加方括号（例如 `http://[2001:db8::1]:8080/`），无需在配置中手动编写方括号。 diff --git a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md index 2386e7e3..314cf7f1 100644 --- a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -324,7 +324,7 @@ ais_bench --models vllm_api_general --datasets gsm8k_gen --reuse 20250628_151326 > ⚠️ 注意：中断续测与失败重测可能改变请求顺序，可能引发结果微小波动。 -> 💡 启用了[推理响应异常检测](../all_params/cli_args.md#推理响应异常检测配置)时，续测同样继承已有检测结果：`completed` 状态的 Case 不会重复调用 msProbe，`skipped` / `failed` / `unavailable` 状态的 Case 会重新检测；已有异常计数累加进最终统计。 +> 💡 启用了[推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)时，续测同样继承已有检测结果：`completed` 状态的 Case 不会重复调用 msProbe，`skipped` / `failed` / `unavailable` 状态的 Case 会重新检测；已有异常计数累加进最终统计。 💡[多任务测评](#多任务测评) 也支持全量和部分任务的中断续测 & 失败用例重测。 diff --git a/docs/source_zh_cn/index.rst b/docs/source_zh_cn/index.rst index f98e0ab2..52b18215 100644 --- a/docs/source_zh_cn/index.rst +++ b/docs/source_zh_cn/index.rst @@ -57,6 +57,7 @@ AISBench Benchmark 是基于 `OpenCompass Date: Thu, 27 Aug 2026 20:45:39 +0800 Subject: [PATCH 16/25] docs: clarify response-anomaly switch stays off by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wording 不增加即关闭/不增加则关闭 (omitting it disables detection) reads like omitting the flag actively disables the feature; reword to 不增加默认不开启 (not enabled by default) and align the English wording accordingly. --- docs/source_en/advanced_tutorials/response_anomaly_detection.md | 2 +- docs/source_en/base_tutorials/all_params/cli_args.md | 2 +- .../advanced_tutorials/response_anomaly_detection.md | 2 +- docs/source_zh_cn/base_tutorials/all_params/cli_args.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index c9afdd11..f79f5cee 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -65,7 +65,7 @@ models = [ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --response-anomaly ``` -> ⚠️ The **feature switch is command-line only**: add `--response-anomaly` to the command to enable detection (omit it to disable). The `response_anomaly` entry in the config file carries only non-switch settings (`payload_retention`, `payload_storage`, etc.). +> ⚠️ The **feature switch is command-line only**: add `--response-anomaly` to the command to enable detection (omitting it leaves detection off by default). The `response_anomaly` entry in the config file carries only non-switch settings (`payload_retention`, `payload_storage`, etc.). **3. Inspect the detection results**: after inference and detection finish, the results are written to `/response_anomaly//.jsonl`, one Case per line (see [Runtime Flow and On-Disk Layout](#runtime-flow-and-on-disk-layout) for the full layout). diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index f8b23f26..20172eb5 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -37,7 +37,7 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | | `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. Note: In performance evaluation scenarios, an excessively high concurrency may cause resource contention among different processes, leading to inaccurate test results. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | -| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it disables detection. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. See 📚 [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for detailed usage. | `--response-anomaly` | +| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it leaves detection off by default. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. See 📚 [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for detailed usage. | `--response-anomaly` | | `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | ### API Model Common Override Parameters diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index 7369d179..7ed77302 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -65,7 +65,7 @@ models = [ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --response-anomaly ``` -> ⚠️ 异常检测的**功能开关仅支持命令行**：在命令中增加 `--response-anomaly` 开启（不增加即关闭）；配置文件中的 `response_anomaly` 仅用于非开关类配置（`payload_retention`、`payload_storage` 等）。 +> ⚠️ 异常检测的**功能开关仅支持命令行**：在命令中增加 `--response-anomaly` 开启（不增加默认不开启）；配置文件中的 `response_anomaly` 仅用于非开关类配置（`payload_retention`、`payload_storage` 等）。 **3. 查看检测结果**：推理与检测结束后，检测结果位于 `/response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`，每行一个 Case（完整落盘结构见[运行流程与落盘结构](#运行流程与落盘结构)）。 diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index 09c49c05..4ca93606 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -36,7 +36,7 @@ ais_bench [OPTIONS] | `--num-prompts` | 指定数据集测评条数（按照数据集顺序选取），需传入正整数，超过数据集条数或默认情况下表示对全量数据集进行测评。 | `--num-prompts 500` | | `--max-num-workers` | 并行任务数，范围 `[1, CPU 核数]`，默认 `1`。在指定`--debug`时配置无效，所有任务串行执行。注意：性能测评场景下，并发数过高可能会导致不同进程出现资源抢占，导致测试结果失真。 | `--max-num-workers 2` | |`--num-warmups`|发送请求前预热次数，按照数据集顺序选取数据进行测试，大概num-warmups大于数据集条数时，会循环发送数据集中数据。默认 `1`；若设为0，则不预热。如果warmup阶段所有请求失败，后续推理任务将不会执行。| `--num-warmups 10` | -| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加则关闭。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。详细用法见 📚 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 | `--response-anomaly` | +| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加默认不开启。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。详细用法见 📚 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 | `--response-anomaly` | | `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。命令行配置优先于配置文件，默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | ### API 模型通用覆盖参数 From 452ad1fded65762b4ce0f3d45192f2c3438a2aed Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Thu, 27 Aug 2026 21:02:19 +0800 Subject: [PATCH 17/25] docs: make response anomaly detection zero-config via two CLI switches The feature needs no config-file fields in practice: the model path field (always set for real runs) drives model-name derivation and msProbe config auto-generation, and payload retention is controlled by the --response-anomaly-payload-retention CLI switch. Drop the global and model-level response_anomaly config-block guidance from the tutorial, the models.md code sample and parameter table, and rework Quick Start around the two CLI switches. --- .../response_anomaly_detection.md | 99 ++++--------------- .../base_tutorials/all_params/cli_args.md | 4 +- .../base_tutorials/all_params/models.md | 10 +- .../response_anomaly_detection.md | 99 ++++--------------- .../base_tutorials/all_params/cli_args.md | 4 +- .../base_tutorials/all_params/mode.md | 2 +- .../base_tutorials/all_params/models.md | 10 +- 7 files changed, 43 insertions(+), 185 deletions(-) diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index f79f5cee..c59d2c9c 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -14,7 +14,7 @@ AISBench integrates msProbe's `ILLDetector` to automatically detect generation a **Anomaly detection results do not affect the original evaluation metrics**: anomalous Cases are not rewritten as inference failures; accuracy/performance metrics are computed as usual, and anomaly information is an independent audit result. -Detection runs through msProbe's `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)`; all three file paths can be configured in AISBench (see [Configuration](#configuration)). +The feature works out of the box with **no config-file changes**: add `--response-anomaly` to the command to enable detection, and control payload retention with `--response-anomaly-payload-retention` (see [Quick Start](#quick-start)). The model name and msProbe algorithm configs required by detection are derived and auto-generated by AISBench from the model `path` (local model directory). --- @@ -45,98 +45,33 @@ Without this dependency, evaluation still runs normally, but all Cases are marke ## Quick Start -**1. Add the `response_anomaly` field to the model config** (the minimal configuration only needs one of `model_name` or `model_path`): - -```python -models = [ - dict( - abbr='qwen3-30b', - attr='service', - response_anomaly=dict( - model_name='Qwen3-30B-A3B', # or provide only model_path; the model name is taken from it - ), - ), -] -``` - -**2. Run the evaluation with `--response-anomaly` on the command line**: +No config-file changes are needed — just add `--response-anomaly` to your evaluation command: ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --response-anomaly ``` -> ⚠️ The **feature switch is command-line only**: add `--response-anomaly` to the command to enable detection (omitting it leaves detection off by default). The `response_anomaly` entry in the config file carries only non-switch settings (`payload_retention`, `payload_storage`, etc.). - -**3. Inspect the detection results**: after inference and detection finish, the results are written to `/response_anomaly//.jsonl`, one Case per line (see [Runtime Flow and On-Disk Layout](#runtime-flow-and-on-disk-layout) for the full layout). - ---- - -## Configuration +> ⚠️ The **feature switch is command-line only**: add `--response-anomaly` to the command to enable detection (omitting it leaves detection off by default). -### Global Configuration +To adjust which payloads are kept after detection, add the second command-line switch `--response-anomaly-payload-retention`: -The global `response_anomaly` config controls the payload retention policy and storage format: - -```python -response_anomaly = dict( - payload_retention='anomalies', # all | anomalies | none - payload_storage=dict( - format='jsonl', - compression='zstd', - compression_level=3, - rows_per_shard=2000, - ), -) +```bash +ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt \ + --response-anomaly \ + --response-anomaly-payload-retention anomalies ``` -`payload_retention` determines which payloads are kept after detection: - | Value | Behavior | | ----- | -------- | | `all` | Keeps every payload and atomically promotes the staging data to the official archive after detection without re-compressing | | `anomalies` (default) | Keeps only detected anomalies plus detection-failed/unavailable Cases | | `none` | Keeps no payload | -All three modes keep the standalone detection results. The command-line parameter `--response-anomaly-payload-retention` overrides the config file value. +All three modes keep the standalone detection results. -> 💡 Runtime details: results are written to disk in batches, and the status is refreshed at most once per second; msProbe token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. +> 💡 **msProbe resources are prepared fully automatically**: the model name is taken from the basename of the model `path` (local model directory; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`); the msProbe threshold config, model mapping, and token-category vocabulary are auto-generated into `/response_anomaly_config//`, and an existing `config.yaml` is never overwritten, so manually tuned thresholds are preserved. If the model config provides no `path`, the task fails fast at startup with explicit guidance. -### Model-Level Configuration (msProbe) - -Model-specific msProbe configuration goes into the model config: - -```python -models = [ - dict( - abbr='qwen3-30b', - attr='service', - response_anomaly=dict( - model_name="", # Model name, for example Qwen3-30B-A3B - model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs - msprobe_config_path="", # Optional; msProbe algorithm-threshold config.yaml path for manual threshold tuning - msprobe_mtype_path="", # Optional; msProbe mtype_config.json path mapping model names to BOS/EOS token ids - msprobe_token2category_dir="", # Optional; msProbe token2category directory holding per-model token-id-to-character-category maps - ), - ), -] -``` - -**Rules for `model_name`**: when it is not configured explicitly, the **model name is taken from the model path** (`model_path`, or the model `path` field; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`), matching the config generator's default. When neither `model_name` nor a model path is available (e.g. only the explicit msProbe resource paths are configured), the task fails fast at startup and asks for an explicit `model_name`, instead of silently running detection with a wrong model name. - -`model_name` must be consistent with msProbe's `mtype_config.json` and the token-category mapping. - -When `msprobe_mtype_path` / `msprobe_token2category_dir` are not provided, the default files inside the msProbe package are used. - -### Auto-Generation and Manual Generation of msProbe Configs - -When `model_path` is configured, the msProbe configs are auto-generated into `/response_anomaly_config//` (auto-generation never overwrites an existing `config.yaml`, so manually tuned thresholds are preserved). They can also be generated manually: - -```bash -ais_bench-gen-response-anomaly-config \ - --model-path /home/Qwen3-30B-A3B \ - --model-name Qwen3-30B-A3B \ - --output-dir ./msprobe_configs -``` +**Inspect the detection results**: after inference and detection finish, the results are written to `/response_anomaly//.jsonl`, one Case per line (see [Runtime Flow and On-Disk Layout](#runtime-flow-and-on-disk-layout) for the full layout). --- @@ -150,10 +85,12 @@ When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the 1. **Inference stage**: the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start; 2. **Detection stage**: after inference finishes, the detection thread streams and decompresses the staging data and calls msProbe; detection results are written to `response_anomaly//.jsonl`; -3. **Archive finalization**: after detection, the staging data is retained or cleaned according to `payload_retention`. +3. **Archive finalization**: after detection, the staging data is retained or cleaned according to `--response-anomaly-payload-retention`. The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. +> 💡 Runtime details: results are written to disk in batches, and the status is refreshed at most once per second; msProbe token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. + ### On-Disk Layout Files produced by detection are laid out under `` as follows: @@ -166,10 +103,10 @@ Files produced by detection are laid out under `` as follows: │ ├── .jsonl # Detection results, one Case per line │ ├── payload_staging// # Transient staging during inference; cleaned after detection │ │ └── part-*.jsonl.zst -│ └── payload// # Payload archive; absent when payload_retention is none +│ └── payload// # Payload archive; absent when payload retention is none │ ├── payload_manifest.json # Archive manifest (per-shard rows, sizes, sha256) -│ └── part-*.jsonl.zst # Compressed payload shards (at most rows_per_shard Cases each) -├── response_anomaly_config/ # Present only when auto-generated via model_path +│ └── part-*.jsonl.zst # Compressed payload shards (at most 2000 Cases each) +├── response_anomaly_config/ # Auto-generated from the model path │ └── / │ ├── configs/ │ │ ├── config.yaml # Detection algorithm thresholds (never overwritten once present) @@ -185,7 +122,7 @@ Path-by-path notes: - **Detection results** `response_anomaly//.jsonl`: one Case per line; field semantics are described in [Detection Results](#detection-results). - **Payload archive** `response_anomaly//payload//`: `all` keeps every Case, `anomalies` keeps only anomalous plus detection-failed/unavailable Cases, and `none` keeps nothing (the directory does not exist). To read the data, decompress the `part-*.jsonl.zst` shards with zstandard and parse each line as JSON; `payload_manifest.json` records per-shard row counts, sizes, and sha256 checksums for integrity verification. Note: under `anomalies`, even when there is nothing to retain, an archive directory containing only an empty manifest is still published to indicate the archiving flow completed successfully — it is not a leftover file. - **Transient files**: `payload_staging/` receives payload records during inference and is cleaned automatically after detection; `..payload-build-*` build directories left by an interrupted detection are cleaned automatically when the next detection starts; `status_tmp/tmp_ResponseAnomaly.json` is a runtime status file (detection progress and per-type statistics) removed together with the status directory when the workflow ends. -- **Auto-generated msProbe configs** `response_anomaly_config//`: generated only when `model_path` is configured and explicit mtype/token2category paths are absent. An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. +- **Auto-generated msProbe configs** `response_anomaly_config//`: auto-generated from the model `path` (local model directory). An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. - **Detection log** `logs/response_anomaly//.out`: records the detection run for the model/dataset group, including detector initialization failures and per-Case failure reasons. --- diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index 20172eb5..1d8efc56 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -37,8 +37,8 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | | `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. Note: In performance evaluation scenarios, an excessively high concurrency may cause resource contention among different processes, leading to inaccurate test results. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | -| `--response-anomaly` | Enables msProbe response anomaly detection. This switch is **command-line only**: adding `--response-anomaly` to the command enables detection; omitting it leaves detection off by default. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. See 📚 [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for detailed usage. | `--response-anomaly` | -| `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. The command-line value overrides the config file; defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | +| `--response-anomaly` | Enables msProbe response anomaly detection with zero extra configuration: adding `--response-anomaly` to the command enables detection; omitting it leaves detection off by default. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. See 📚 [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for detailed usage. | `--response-anomaly` | +| `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. Defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | ### API Model Common Override Parameters diff --git a/docs/source_en/base_tutorials/all_params/models.md b/docs/source_en/base_tutorials/all_params/models.md index 6f787b25..04e1dad0 100644 --- a/docs/source_en/base_tutorials/all_params/models.md +++ b/docs/source_en/base_tutorials/all_params/models.md @@ -58,13 +58,6 @@ models = [ # Equivalent to the `models` imported via `from ais_bench.benchmark.c generation_kwargs = dict( # Model inference parameters, configured with reference to the VLLM documentation; the AISBench evaluation tool does not process these parameters and attaches them to the sent requests temperature = 0.01, ignore_eos=False, - ), - response_anomaly = dict( # Optional; model-level config for msProbe response anomaly detection - model_name="", # Model name, for example Qwen3-30B-A3B - model_path="", # Local model directory, for example /home/Qwen3-30B-A3B; optional, used to auto-generate configs - msprobe_config_path="", # Optional; algorithm-threshold config.yaml path for manual threshold tuning - msprobe_mtype_path="", # Optional; mtype_config.json path mapping model names to BOS/EOS token ids - msprobe_token2category_dir="", # Optional; token2category directory holding per-model token-id-to-character-category maps ) ) ] @@ -96,7 +89,6 @@ The description of configurable parameters for the service-oriented inference ba | `generation_kwargs` | Dict | Configuration of inference generation parameters, depending on the specific service-oriented backend and interface type. Note: Currently, multi-sampling parameters such as `best_of` and `n` are not supported, but multiple independent inferences can be performed using the `num_return_sequences` parameter (for details, refer to 🔗 [the role of `num_return_sequences` in the Text Generation Documentation](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences\(int,)). Supports configuring `logprobs` / `top_logprobs` parameters to enable token probability information collection; see 🔗[Logprobs Collection and Analysis](../../advanced_tutorials/logprobs_collection.md) | | `returns_tool_calls` | Bool | Controls the extraction method of function call information. When set to `True`, the system extracts function call information from the `tool_calls` field of the API response; when set to `False`, the system parses function call information from the `content` field | | `pred_postprocessor` | Dict | Post-processing configuration for model output results. It is used to format, clean, or convert the original model output to meet the requirements of specific evaluation tasks | -| `response_anomaly` | Dict | Optional; model-level config for msProbe response anomaly detection, including `model_name` (must match the name in msProbe's mtype_config.json; when unset, the model name is taken from the model path; startup fails when neither `model_name` nor a model path is available), `model_path` (local model directory, optional, used to auto-generate configs), `msprobe_config_path` (algorithm-threshold config.yaml path, optional, for manual tuning; auto-generation never overwrites an existing file), `msprobe_mtype_path`, and `msprobe_token2category_dir`. When mtype/token-category paths are not provided, the default files inside the msProbe package are used | **Precautions**: @@ -107,7 +99,7 @@ The description of configurable parameters for the service-oriented inference ba - Setting `batch_size` too large may result in high CPU usage. Please configure it reasonably based on hardware conditions. - The default service address used by the service-oriented inference evaluation API is `localhost:8080`. In actual use, you need to modify it to the IP and port of the service-oriented backend according to the actual deployment. - When using an IPv6 literal (such as `::1` or `2001:db8::1`) as `host_ip`, the tool will automatically wrap it in brackets in the generated URL (for example, `http://[2001:db8::1]:8080/`), so you do not need to manually add brackets in the configuration. -- When response anomaly detection (`response_anomaly`) is enabled, the service must return token ids and top-k logprobs; AISBench automatically adds `logprobs=True` and a fixed `top_logprobs=20` to the inference requests, and Cases whose responses lack these fields are marked as `skipped` in the detection results. See [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for details. +- When response anomaly detection (`--response-anomaly`) is enabled, the service must return token ids and top-k logprobs; AISBench automatically adds `logprobs=True` and a fixed `top_logprobs=20` to the inference requests, and Cases whose responses lack these fields are marked as `skipped` in the detection results. See [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for details. ### Multi-LoRA Routing diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index 7ed77302..77d31a67 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -14,7 +14,7 @@ AISBench 集成 msProbe 的 `ILLDetector`，支持在推理评测的同时自动 **异常检测结果不影响原有评测指标**：异常 Case 不会被改写为推理失败，精度/性能指标照常计算，异常信息是独立的审计结果。 -检测通过 msProbe 的 `ILLDetector(config_path, mtype_path, tk2cat_path).run(...)` 完成，三个文件路径均可由 AISBench 配置（见[配置说明](#配置说明)）。 +功能**开箱即用，无需修改配置文件**：命令行增加 `--response-anomaly` 即可开启检测，payload 保留模式通过 `--response-anomaly-payload-retention` 控制（见[快速使用](#快速使用)）。检测所需的模型名与 msProbe 算法配置由 AISBench 根据模型 `path`（本地模型目录）自动推导与生成。 --- @@ -45,98 +45,33 @@ pip install 'ais-bench-benchmark[response_anomaly]' ## 快速使用 -**1. 在模型配置中添加 `response_anomaly` 字段**（最小配置只需提供 `model_name` 或 `model_path` 之一）： - -```python -models = [ - dict( - abbr='qwen3-30b', - attr='service', - response_anomaly=dict( - model_name='Qwen3-30B-A3B', # 或仅提供 model_path，自动取其中的模型名称 - ), - ), -] -``` - -**2. 在命令行增加 `--response-anomaly` 运行评测**： +无需修改任何配置文件，在原有评测命令上增加 `--response-anomaly` 即可开启： ```bash ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt --response-anomaly ``` -> ⚠️ 异常检测的**功能开关仅支持命令行**：在命令中增加 `--response-anomaly` 开启（不增加默认不开启）；配置文件中的 `response_anomaly` 仅用于非开关类配置（`payload_retention`、`payload_storage` 等）。 - -**3. 查看检测结果**：推理与检测结束后，检测结果位于 `/response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`，每行一个 Case（完整落盘结构见[运行流程与落盘结构](#运行流程与落盘结构)）。 - ---- - -## 配置说明 +> ⚠️ 异常检测的**功能开关仅支持命令行**：在命令中增加 `--response-anomaly` 开启（不增加默认不开启）。 -### 全局配置 +如需调整检测完成后 payload 的保留范围，增加第二个命令行参数 `--response-anomaly-payload-retention`： -全局 `response_anomaly` 配置控制 payload 的保留策略与存储格式： - -```python -response_anomaly = dict( - payload_retention='anomalies', # all | anomalies | none - payload_storage=dict( - format='jsonl', - compression='zstd', - compression_level=3, - rows_per_shard=2000, - ), -) +```bash +ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_chat_prompt \ + --response-anomaly \ + --response-anomaly-payload-retention anomalies ``` -`payload_retention` 决定检测完成后 payload 的保留范围： - | 取值 | 行为 | | ---- | ---- | | `all` | 保存全部 payload，检测后直接将 staging 原子转为正式归档，不会二次压缩 | | `anomalies`（默认） | 只保存已检出异常以及检测失败/不可用 Case | | `none` | 不保存 payload | -三种模式都保留独立检测结果。命令行参数 `--response-anomaly-payload-retention` 可覆盖配置文件取值。 +三种模式都保留独立检测结果。 -> 💡 运行期细节：检测结果按批写盘，状态最多每秒刷新一次；msProbe token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 +> 💡 **msProbe 资源全自动准备**：模型名自动取模型 `path`（本地模型目录）的目录名（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`）；msProbe 阈值配置、模型映射与 token 分类词表自动生成到 `/response_anomaly_config/<模型 abbr>/`，已存在的 `config.yaml` 不会被覆盖，便于保留手工调优的阈值。若模型配置未提供 `path`，任务会在启动时报错并给出明确的解决指引。 -### 模型级配置（msProbe） - -模型相关的 msProbe 配置放在模型配置中： - -```python -models = [ - dict( - abbr='qwen3-30b', - attr='service', - response_anomaly=dict( - model_name="", # 填写模型名称，如 Qwen3-30B-A3B - model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 - msprobe_config_path="", # 可选，msProbe 算法阈值配置 config.yaml 路径，用于手工调优检测阈值 - msprobe_mtype_path="", # 可选，msProbe 模型名与 BOS/EOS token id 映射文件 mtype_config.json 路径 - msprobe_token2category_dir="", # 可选，msProbe token2category 目录，存放各模型的 token id 到字符类别映射 - ), - ), -] -``` - -**`model_name` 的取值规则**：未显式配置时自动取模型路径（`model_path` 或模型 `path` 字段）中的**模型名称**（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`），与配置生成工具的默认取值一致。当既未配置 `model_name`、也没有可用的模型路径（如仅显式配置 msProbe 三件套路径）时，任务启动即报错，要求显式配置 `model_name`，避免以错误的模型名静默运行导致检测失效。 - -`model_name` 需与 msProbe 的 `mtype_config.json` 配置以及 token 分类映射保持一致。 - -未提供 `msprobe_mtype_path` / `msprobe_token2category_dir` 时回退到 msProbe 包内默认文件。 - -### msProbe 配置的自动生成与手动生成 - -配置了 `model_path` 时，msProbe 配置会自动生成到 `/response_anomaly_config/<模型 abbr>/`（自动生成不会覆盖已存在的 `config.yaml`，便于保留手工调优的阈值）。也可手动生成： - -```bash -ais_bench-gen-response-anomaly-config \ - --model-path /home/Qwen3-30B-A3B \ - --model-name Qwen3-30B-A3B \ - --output-dir ./msprobe_configs -``` +**查看检测结果**：推理与检测结束后，检测结果位于 `/response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`，每行一个 Case（完整落盘结构见[运行流程与落盘结构](#运行流程与落盘结构)）。 --- @@ -150,10 +85,12 @@ ais_bench-gen-response-anomaly-config \ 1. **推理阶段**：完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果； 2. **检测阶段**：推理结束后，检测线程流式解压 staging 数据并调用 msProbe，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`； -3. **归档收尾**：检测完成后按 `payload_retention` 保留或清理 staging。 +3. **归档收尾**：检测完成后按 `--response-anomaly-payload-retention` 保留或清理 staging。 状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 +> 💡 运行期细节：检测结果按批写盘，状态最多每秒刷新一次；msProbe token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 + ### 落盘结构 检测相关文件在 `` 下的落盘结构如下： @@ -166,10 +103,10 @@ ais_bench-gen-response-anomaly-config \ │ ├── <数据集 abbr>.jsonl # 检测结果，每行一个 Case │ ├── payload_staging/<数据集 abbr>/ # 推理期间的临时存放区，检测完成后自动清理 │ │ └── part-*.jsonl.zst -│ └── payload/<数据集 abbr>/ # payload 归档；payload_retention 为 none 时不存在 +│ └── payload/<数据集 abbr>/ # payload 归档；payload 保留模式为 none 时不存在 │ ├── payload_manifest.json # 归档清单（分片行数、大小、sha256） -│ └── part-*.jsonl.zst # 压缩 payload 分片（每片最多 rows_per_shard 条 Case） -├── response_anomaly_config/ # 仅配置 model_path 自动生成时存在 +│ └── part-*.jsonl.zst # 压缩 payload 分片（每片最多 2000 条 Case） +├── response_anomaly_config/ # 由模型 path 自动生成 │ └── <模型 abbr>/ │ ├── configs/ │ │ ├── config.yaml # 检测算法阈值配置（已存在时不覆盖） @@ -185,7 +122,7 @@ ais_bench-gen-response-anomaly-config \ - **检测结果** `response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`：每行一个 Case 的检测结果，字段含义见[检测结果说明](#检测结果说明)。 - **payload 归档** `response_anomaly/<模型 abbr>/payload/<数据集 abbr>/`：`all` 保留全部 Case，`anomalies` 只保留异常及检测失败/不可用 Case，`none` 不保留（目录不存在）。读取时用 zstandard 解压 `part-*.jsonl.zst` 分片后逐行解析 JSON；`payload_manifest.json` 记录每个分片的行数、大小与 sha256 校验值，可用于完整性校验。注意：`anomalies` 模式下即使无任何需保留的 Case，仍会发布一个仅含空 manifest 的归档目录，表示归档流程已成功完成，不是残留文件。 - **临时文件**：`payload_staging/` 在推理期间逐条接收 payload 写入，检测完成后自动清理；检测中断后残留的 `.<数据集>.payload-build-*` 构建目录会在下次检测启动时自动清理；`status_tmp/tmp_ResponseAnomaly.json` 为运行期状态文件（检测进度与类型统计），工作流结束后随状态目录一并清理。 -- **自动生成的 msProbe 配置** `response_anomaly_config/<模型 abbr>/`：仅当配置了 `model_path` 且未显式提供 mtype/token2category 路径时生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 +- **自动生成的 msProbe 配置** `response_anomaly_config/<模型 abbr>/`：由模型 `path`（本地模型目录）自动生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 - **检测日志** `logs/response_anomaly/<模型 abbr>/<数据集 abbr>.out`：记录对应模型/数据集组的检测过程，含检测器初始化失败与单 Case 失败的具体原因。 --- diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index 4ca93606..3cabe7ac 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -36,8 +36,8 @@ ais_bench [OPTIONS] | `--num-prompts` | 指定数据集测评条数（按照数据集顺序选取），需传入正整数，超过数据集条数或默认情况下表示对全量数据集进行测评。 | `--num-prompts 500` | | `--max-num-workers` | 并行任务数，范围 `[1, CPU 核数]`，默认 `1`。在指定`--debug`时配置无效，所有任务串行执行。注意：性能测评场景下，并发数过高可能会导致不同进程出现资源抢占，导致测试结果失真。 | `--max-num-workers 2` | |`--num-warmups`|发送请求前预热次数，按照数据集顺序选取数据进行测试，大概num-warmups大于数据集条数时，会循环发送数据集中数据。默认 `1`；若设为0，则不预热。如果warmup阶段所有请求失败，后续推理任务将不会执行。| `--num-warmups 10` | -| `--response-anomaly` | 开启 msProbe 推理响应异常检测。该开关**仅支持命令行配置**：命令行增加 `--response-anomaly` 即开启，不增加默认不开启。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。详细用法见 📚 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 | `--response-anomaly` | -| `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。命令行配置优先于配置文件，默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | +| `--response-anomaly` | 开启 msProbe 推理响应异常检测，无需额外配置：命令行增加 `--response-anomaly` 即开启，不增加默认不开启。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。详细用法见 📚 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 | `--response-anomaly` | +| `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | ### API 模型通用覆盖参数 diff --git a/docs/source_zh_cn/base_tutorials/all_params/mode.md b/docs/source_zh_cn/base_tutorials/all_params/mode.md index 00bc19f1..d724152e 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/mode.md +++ b/docs/source_zh_cn/base_tutorials/all_params/mode.md @@ -107,7 +107,7 @@ outputs/default/ ### 响应异常检测模式支持（可选） -msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测在推理阶段结束后启动，并**串行绑定在推理阶段内**——工作流会等待检测完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，具体配置请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 +msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测在推理阶段结束后启动，并**串行绑定在推理阶段内**——工作流会等待检测完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，详细用法请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 ## 性能评测场景 ### perf模式 diff --git a/docs/source_zh_cn/base_tutorials/all_params/models.md b/docs/source_zh_cn/base_tutorials/all_params/models.md index 09c31866..ea57cc95 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/models.md +++ b/docs/source_zh_cn/base_tutorials/all_params/models.md @@ -54,13 +54,6 @@ models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.c generation_kwargs = dict( # 模型推理参数，参考VLLM文档配置，AISBench评测工具不做处理，在发送的请求中附带 temperature = 0.01, ignore_eos=False, - ), - response_anomaly = dict( # 可选，msProbe 推理响应异常检测的模型级配置 - model_name="", # 填写模型名称，如 Qwen3-30B-A3B - model_path="", # 填写本地模型目录，如 /home/Qwen3-30B-A3B；可选，用于自动生成配置 - msprobe_config_path="", # 可选，算法阈值 config.yaml 路径，用于手工调优检测阈值 - msprobe_mtype_path="", # 可选，模型名与 BOS/EOS token id 映射文件 mtype_config.json 路径 - msprobe_token2category_dir="", # 可选，token2category 目录，存放各模型的 token id 到字符类别映射 ) ) ] @@ -91,14 +84,13 @@ models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.c | `generation_kwargs` | Dict | 推理生成参数配置，依赖具体的服务化后端和接口类型。注意：当前不支持 `best_of` 和 `n` 等多次采样参数，但支持通过`num_return_sequences`参数进行多次独立推理(具体请参考🔗[Text Generation 文档](https://huggingface.co/docs/transformers/v4.18.0/en/main_classes/text_generation#transformers.generation_utils.GenerationMixin.generate.num_return_sequences)中`num_return_sequences`的作用)。支持配置 `logprobs` / `top_logprobs` 参数开启 token 概率信息采集，详见 🔗[Logprobs 采集与分析](../../advanced_tutorials/logprobs_collection.md) | | `returns_tool_calls` | Bool | 控制函数调用信息的提取方式。当设置为True时，系统从API响应的`tool_calls`字段中提取函数调用信息；当设置为False时，系统从`content`字段中解析函数调用信息 | | `pred_postprocessor` | Dict | 模型输出结果的后处理配置。用于对原始模型输出进行格式化、清理或转换，以满足特定评估任务的要求 | -| `response_anomaly` | Dict | 可选，msProbe 推理响应异常检测的模型级配置，包含 `model_name`（与 msProbe 的 mtype_config.json 名称一致；未配置时自动取模型路径中的模型名称；既无 `model_name` 也无模型路径时启动报错）、`model_path`（本地模型目录，用于自动生成配置，可选）、`msprobe_config_path`（算法阈值 config.yaml 路径，可选，用于手工调优；自动生成不会覆盖已存在的文件）、`msprobe_mtype_path`、`msprobe_token2category_dir`。未提供 mtype/token 分类路径时回退到 msProbe 包内默认文件 | **注意事项：** - 响应异常检测当前仅支持基于 vLLM Chat API 的 `vllm_api_general_chat`、`vllm_api_stream_chat` 和 `vllm_api_stream_chat_multiturn` 模型配置，其他模型后端暂不支持。 - `request_rate` 受硬件性能影响，可通过增加 📚 [WORKERS_NUM](./cli_args.md#配置常量文件参数) 提高并发能力。 - `request_rate` 功能可能被`traffic_cfg`项覆盖，具体原因请参考 🔗 [请求速率(RPS)分布控制及可视化说明中的参数解读章节](../../advanced_tutorials/rps_distribution.md#参数解读)。 - 当数据集含 timestamp 且模型配置中 **use_timestamp** 为 True 时，请求按 timestamp 发送，**request_rate** 与 **traffic_cfg** 将被忽略。 -- 使用响应异常检测（`response_anomaly`）时，服务端必须返回 token id 与 top-k logprobs；启用检测后 AISBench 会自动在推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`，服务响应缺少这些字段的 Case 检测结果标记为 `skipped`。详细配置请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 +- 使用响应异常检测（`--response-anomaly`）时，服务端必须返回 token id 与 top-k logprobs；启用检测后 AISBench 会自动在推理请求中补充 `logprobs=True` 与固定的 `top_logprobs=20`，服务响应缺少这些字段的 Case 检测结果标记为 `skipped`。详细用法请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 - `batch_size` 设置过大可能导致 CPU 占用过高，请根据硬件条件合理配置。 - 服务化推理评测 API 默认使用的服务地址为 `localhost:8080`。实际使用时需根据实际部署修改为服务化后端的 IP 和端口。 - 当使用 IPv6 字面量（如 `::1`、`2001:db8::1`）作为 `host_ip` 时，工具会在生成的访问 URL 中自动为其添加方括号（例如 `http://[2001:db8::1]:8080/`），无需在配置中手动编写方括号。 From 75a1a667412b27b9c374d0eaa4e5c1cc7582c104 Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 08:43:40 +0800 Subject: [PATCH 18/25] docs: drop internal msProbe branding from user-facing docs User-facing docs should not expose the internal detector implementation name. Replace msProbe/mindstudio-probe wording with neutral terms (built-in anomaly detection, detector source, detection configs) across the tutorial, cli_args/mode/accuracy_benchmark notes and the install guide, zh and en. The pip extra name response_anomaly remains the single user-facing handle for the dependency. --- .../response_anomaly_detection.md | 20 +++++++++---------- .../base_tutorials/all_params/cli_args.md | 2 +- .../base_tutorials/all_params/mode.md | 2 +- .../scenes_intro/accuracy_benchmark.md | 2 +- docs/source_en/get_started/install.md | 4 ++-- .../response_anomaly_detection.md | 20 +++++++++---------- .../base_tutorials/all_params/cli_args.md | 2 +- .../base_tutorials/all_params/mode.md | 2 +- .../scenes_intro/accuracy_benchmark.md | 2 +- docs/source_zh_cn/get_started/install.md | 4 ++-- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index c59d2c9c..4434f25f 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -2,7 +2,7 @@ ## Overview -AISBench integrates msProbe's `ILLDetector` to automatically detect generation anomalies in LLM responses while running inference evaluations. Detection results cover the following types: +AISBench has a built-in detector that automatically identifies generation anomalies in LLM responses while running inference evaluations. Detection results cover the following types: | `anomaly_type` | `anomaly_type_name` | Meaning | | -------------- | ------------------- | ------- | @@ -14,7 +14,7 @@ AISBench integrates msProbe's `ILLDetector` to automatically detect generation a **Anomaly detection results do not affect the original evaluation metrics**: anomalous Cases are not rewritten as inference failures; accuracy/performance metrics are computed as usual, and anomaly information is an independent audit result. -The feature works out of the box with **no config-file changes**: add `--response-anomaly` to the command to enable detection, and control payload retention with `--response-anomaly-payload-retention` (see [Quick Start](#quick-start)). The model name and msProbe algorithm configs required by detection are derived and auto-generated by AISBench from the model `path` (local model directory). +The feature works out of the box with **no config-file changes**: add `--response-anomaly` to the command to enable detection, and control payload retention with `--response-anomaly-payload-retention` (see [Quick Start](#quick-start)). The model name and algorithm configs required by detection are derived and auto-generated by AISBench from the model `path` (local model directory). --- @@ -31,13 +31,13 @@ The feature works out of the box with **no config-file changes**: add `--respons ## Installing Dependencies -Response anomaly detection relies on the optional package `mindstudio-probe`, installed through the AISBench extra: +Response anomaly detection relies on an optional dependency, installed through the AISBench extra: ```bash pip install 'ais-bench-benchmark[response_anomaly]' ``` -During installation, pip downloads and builds the pinned msProbe source from GitCode, so the environment needs Git and network access. +During installation, pip downloads and builds the pinned detector source from GitCode, so the environment needs Git and network access. Without this dependency, evaluation still runs normally, but all Cases are marked with the `unavailable` status in the detection results (see [Detection Results](#detection-results)); re-run after installation to restore detection. @@ -69,7 +69,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch All three modes keep the standalone detection results. -> 💡 **msProbe resources are prepared fully automatically**: the model name is taken from the basename of the model `path` (local model directory; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`); the msProbe threshold config, model mapping, and token-category vocabulary are auto-generated into `/response_anomaly_config//`, and an existing `config.yaml` is never overwritten, so manually tuned thresholds are preserved. If the model config provides no `path`, the task fails fast at startup with explicit guidance. +> 💡 **Detection resources are prepared fully automatically**: the model name is taken from the basename of the model `path` (local model directory; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`); the detection threshold config, model mapping, and token-category vocabulary are auto-generated into `/response_anomaly_config//`, and an existing `config.yaml` is never overwritten, so manually tuned thresholds are preserved. If the model config provides no `path`, the task fails fast at startup with explicit guidance. **Inspect the detection results**: after inference and detection finish, the results are written to `/response_anomaly//.jsonl`, one Case per line (see [Runtime Flow and On-Disk Layout](#runtime-flow-and-on-disk-layout) for the full layout). @@ -84,12 +84,12 @@ When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the ### Detection Flow 1. **Inference stage**: the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start; -2. **Detection stage**: after inference finishes, the detection thread streams and decompresses the staging data and calls msProbe; detection results are written to `response_anomaly//.jsonl`; +2. **Detection stage**: after inference finishes, the detection thread streams and decompresses the staging data and runs anomaly detection; detection results are written to `response_anomaly//.jsonl`; 3. **Archive finalization**: after detection, the staging data is retained or cleaned according to `--response-anomaly-payload-retention`. The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. -> 💡 Runtime details: results are written to disk in batches, and the status is refreshed at most once per second; msProbe token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. +> 💡 Runtime details: results are written to disk in batches, and the status is refreshed at most once per second; token-category maps are cached per model and EOS token to avoid re-parsing large JSON files for every Case. ### On-Disk Layout @@ -122,7 +122,7 @@ Path-by-path notes: - **Detection results** `response_anomaly//.jsonl`: one Case per line; field semantics are described in [Detection Results](#detection-results). - **Payload archive** `response_anomaly//payload//`: `all` keeps every Case, `anomalies` keeps only anomalous plus detection-failed/unavailable Cases, and `none` keeps nothing (the directory does not exist). To read the data, decompress the `part-*.jsonl.zst` shards with zstandard and parse each line as JSON; `payload_manifest.json` records per-shard row counts, sizes, and sha256 checksums for integrity verification. Note: under `anomalies`, even when there is nothing to retain, an archive directory containing only an empty manifest is still published to indicate the archiving flow completed successfully — it is not a leftover file. - **Transient files**: `payload_staging/` receives payload records during inference and is cleaned automatically after detection; `..payload-build-*` build directories left by an interrupted detection are cleaned automatically when the next detection starts; `status_tmp/tmp_ResponseAnomaly.json` is a runtime status file (detection progress and per-type statistics) removed together with the status directory when the workflow ends. -- **Auto-generated msProbe configs** `response_anomaly_config//`: auto-generated from the model `path` (local model directory). An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. +- **Auto-generated detection configs** `response_anomaly_config//`: auto-generated from the model `path` (local model directory). An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. - **Detection log** `logs/response_anomaly//.out`: records the detection run for the model/dataset group, including detector initialization failures and per-Case failure reasons. --- @@ -135,9 +135,9 @@ The `detection_status` values in the detection results are: | Status | Meaning | Troubleshooting | | --- | --- | --- | -| `completed` | msProbe was invoked and returned a detection result | Nothing to do | +| `completed` | The detector ran and returned a detection result | Nothing to do | | `skipped` | The inference response did not carry token ids or top-k logprobs | Check whether the service supports and returns `logprobs` / `top_logprobs` / token id fields | -| `unavailable` | `mindstudio-probe` (the response_anomaly extra) is not installed | Install the optional dependency per the [Installation Guide](../get_started/install.md) and re-run | +| `unavailable` | The response anomaly detection optional dependency (response_anomaly extra) is not installed | Install the optional dependency per the [Installation Guide](../get_started/install.md) and re-run | | `failed` | An exception occurred during invocation or input conversion | Check the `reason` field of the Case result (error type and summary) and the detection logs | Detection-specific logs are located at `/logs/response_anomaly//.out`; detection progress and per-type statistics can also be found in the `/status_tmp/tmp_ResponseAnomaly.json` status file. diff --git a/docs/source_en/base_tutorials/all_params/cli_args.md b/docs/source_en/base_tutorials/all_params/cli_args.md index 1d8efc56..f19b3102 100644 --- a/docs/source_en/base_tutorials/all_params/cli_args.md +++ b/docs/source_en/base_tutorials/all_params/cli_args.md @@ -37,7 +37,7 @@ Applicable to all modes and can be used in combination with accuracy or performa | `--num-prompts` | Specifies the number of test cases for the dataset (selected in dataset order). A positive integer must be passed. If the number exceeds the total number of cases in the dataset or no value is specified, the entire dataset is used for testing. | `--num-prompts 500` | | `--max-num-workers` | Number of parallel tasks, range: `[1, number of CPU cores]`; default value: `1`. Invalid when `--debug` is specified; all tasks are executed serially. Note: In performance evaluation scenarios, an excessively high concurrency may cause resource contention among different processes, leading to inaccurate test results. | `--max-num-workers 2` | | `--num-warmups` | Number of warm-up runs before sending requests. Data is selected in dataset order for testing. When `num-warmups` exceeds the number of dataset entries, data from the dataset will be sent in a loop. Default value: `1`; set to `0` to disable warm-up. If all requests fail during the warmup phase, subsequent inference tasks will not be executed. | `--num-warmups 10` | -| `--response-anomaly` | Enables msProbe response anomaly detection with zero extra configuration: adding `--response-anomaly` to the command enables detection; omitting it leaves detection off by default. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. See 📚 [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for detailed usage. | `--response-anomaly` | +| `--response-anomaly` | Enables response anomaly detection with zero extra configuration: adding `--response-anomaly` to the command enables detection; omitting it leaves detection off by default. Detection is serially bound to the inference stage: after inference finishes, the workflow starts detection and waits for it to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages; requires the service to return token ids and top-k logprobs. Only supported in `all`, `infer`, and `infer_judge` generation chains; performance mode and Agent evaluation modes are unsupported. See 📚 [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for detailed usage. | `--response-anomaly` | | `--response-anomaly-payload-retention` | Payload retention mode after anomaly detection: `all` keeps everything, `anomalies` keeps anomalous and detection-failed/unavailable Cases, `none` keeps nothing. Defaults to `anomalies`. | `--response-anomaly-payload-retention anomalies` | ### API Model Common Override Parameters diff --git a/docs/source_en/base_tutorials/all_params/mode.md b/docs/source_en/base_tutorials/all_params/mode.md index e0b01214..2ae58219 100644 --- a/docs/source_en/base_tutorials/all_params/mode.md +++ b/docs/source_en/base_tutorials/all_params/mode.md @@ -108,7 +108,7 @@ outputs/default/ ### Response Anomaly Detection Mode Support (Optional) -msProbe response anomaly detection (`--response-anomaly`) is only supported in the `all`, `infer`, and `infer_judge` generation chains: detection starts after the inference stage finishes and is **serially bound to the inference stage** — the workflow waits for detection to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages, guaranteeing that detection results and payload archives are on disk when the inference stage exits. The `perf` and `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do **not** support this feature; enabling it raises an explicit error during config initialization. This feature requires the service model to return token ids and top-k logprobs; see [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for details. +Response anomaly detection (`--response-anomaly`) is only supported in the `all`, `infer`, and `infer_judge` generation chains: detection starts after the inference stage finishes and is **serially bound to the inference stage** — the workflow waits for detection to complete (the dedicated status board prints the final result) before entering the subsequent Judge / Eval / Summary stages, guaranteeing that detection results and payload archives are on disk when the inference stage exits. The `perf` and `perf_viz` performance modes, as well as Agent / function-call and other custom chains, do **not** support this feature; enabling it raises an explicit error during config initialization. This feature requires the service model to return token ids and top-k logprobs; see [Response Anomaly Detection](../../advanced_tutorials/response_anomaly_detection.md) for details. ## Performance Evaluation Scenarios ### Perf Mode diff --git a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md index 79f7a2df..c6a2e952 100644 --- a/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_en/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -322,7 +322,7 @@ After the resumption is completed, the accuracy results of all requests will be > ⚠️ Note: Resumption after interruption and retesting of failed cases may change the order of requests, which may cause slight fluctuations in results. -> 💡 When [response anomaly detection](../../advanced_tutorials/response_anomaly_detection.md) is enabled, resumption also inherits existing detection results: Cases with `completed` status are not re-detected by msProbe, while Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume; existing anomaly counts are accumulated into the final statistics. +> 💡 When [response anomaly detection](../../advanced_tutorials/response_anomaly_detection.md) is enabled, resumption also inherits existing detection results: Cases with `completed` status are not re-detected, while Cases with `skipped` / `failed` / `unavailable` status are re-detected on resume; existing anomaly counts are accumulated into the final statistics. 💡[Multi-Task Evaluation](#multi-task-evaluation) also supports resumption after interruption and retesting of failed cases for all or part of the tasks. diff --git a/docs/source_en/get_started/install.md b/docs/source_en/get_started/install.md index b33fd33c..f9c4eb94 100644 --- a/docs/source_en/get_started/install.md +++ b/docs/source_en/get_started/install.md @@ -33,7 +33,7 @@ pip3 install -r requirements/extra.txt ⚙️ Response Anomaly Detection Support (Optional) -If you need to use msProbe response anomaly detection (`--response-anomaly`), install the additional dependencies: +If you need to use response anomaly detection (`--response-anomaly`), install the additional dependencies: ```shell pip3 install -r requirements/response_anomaly.txt ``` @@ -42,7 +42,7 @@ Or install them via the extra: pip3 install 'ais-bench-benchmark[response_anomaly]' ``` -**Note**: These dependencies include building the `mindstudio-probe` source code from a pinned commit on GitCode, so the installation environment needs Git and network access. Without them, the AISBench main workflow is not affected; the affected Cases are marked as `unavailable` in the detection results. +**Note**: These dependencies include building the detector source code from a pinned commit on GitCode, so the installation environment needs Git and network access. Without them, the AISBench main workflow is not affected; the affected Cases are marked as `unavailable` in the detection results. ⚙️ Huggingface Multi-modal Model / vLLM Multi-modal Offline Inference Support (Optional) diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index 77d31a67..a76b0bb9 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -2,7 +2,7 @@ ## 概述 -AISBench 集成 msProbe 的 `ILLDetector`，支持在推理评测的同时自动检测大模型响应中的生成异常。检测结果覆盖以下类型： +AISBench 内置异常检测能力，支持在推理评测的同时自动检测大模型响应中的生成异常。检测结果覆盖以下类型： | `anomaly_type` | `anomaly_type_name` | 含义 | | -------------- | ------------------- | ---- | @@ -14,7 +14,7 @@ AISBench 集成 msProbe 的 `ILLDetector`，支持在推理评测的同时自动 **异常检测结果不影响原有评测指标**：异常 Case 不会被改写为推理失败，精度/性能指标照常计算，异常信息是独立的审计结果。 -功能**开箱即用，无需修改配置文件**：命令行增加 `--response-anomaly` 即可开启检测，payload 保留模式通过 `--response-anomaly-payload-retention` 控制（见[快速使用](#快速使用)）。检测所需的模型名与 msProbe 算法配置由 AISBench 根据模型 `path`（本地模型目录）自动推导与生成。 +功能**开箱即用，无需修改配置文件**：命令行增加 `--response-anomaly` 即可开启检测，payload 保留模式通过 `--response-anomaly-payload-retention` 控制（见[快速使用](#快速使用)）。检测所需的模型名与算法配置由 AISBench 根据模型 `path`（本地模型目录）自动推导与生成。 --- @@ -31,13 +31,13 @@ AISBench 集成 msProbe 的 `ILLDetector`，支持在推理评测的同时自动 ## 安装依赖 -响应异常检测依赖可选包 `mindstudio-probe`，通过 AISBench 的 extra 安装： +该功能依赖可选组件，通过 AISBench 的 extra 安装： ```bash pip install 'ais-bench-benchmark[response_anomaly]' ``` -安装过程中 pip 会从 GitCode 下载并构建已固定提交的 msProbe 源码，因此安装环境需要 Git 和网络访问。 +安装过程中 pip 会从 GitCode 下载并构建已固定提交的检测器源码，因此安装环境需要 Git 和网络访问。 未安装该依赖时评测流程仍可正常运行，但所有 Case 的检测结果会标记为 `unavailable` 状态（见[检测结果说明](#检测结果说明)）；安装后重跑即可恢复检测。 @@ -69,7 +69,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch 三种模式都保留独立检测结果。 -> 💡 **msProbe 资源全自动准备**：模型名自动取模型 `path`（本地模型目录）的目录名（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`）；msProbe 阈值配置、模型映射与 token 分类词表自动生成到 `/response_anomaly_config/<模型 abbr>/`，已存在的 `config.yaml` 不会被覆盖，便于保留手工调优的阈值。若模型配置未提供 `path`，任务会在启动时报错并给出明确的解决指引。 +> 💡 **检测资源全自动准备**：模型名自动取模型 `path`（本地模型目录）的目录名（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`）；检测阈值配置、模型映射与 token 分类词表自动生成到 `/response_anomaly_config/<模型 abbr>/`，已存在的 `config.yaml` 不会被覆盖，便于保留手工调优的阈值。若模型配置未提供 `path`，任务会在启动时报错并给出明确的解决指引。 **查看检测结果**：推理与检测结束后，检测结果位于 `/response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`，每行一个 Case（完整落盘结构见[运行流程与落盘结构](#运行流程与落盘结构)）。 @@ -84,12 +84,12 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch ### 检测流程 1. **推理阶段**：完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果； -2. **检测阶段**：推理结束后，检测线程流式解压 staging 数据并调用 msProbe，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`； +2. **检测阶段**：推理结束后，检测线程流式解压 staging 数据并执行异常检测，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`； 3. **归档收尾**：检测完成后按 `--response-anomaly-payload-retention` 保留或清理 staging。 状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 -> 💡 运行期细节：检测结果按批写盘，状态最多每秒刷新一次；msProbe token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 +> 💡 运行期细节：检测结果按批写盘，状态最多每秒刷新一次；token 分类映射按模型和 EOS token 缓存，避免每个 Case 重复解析大 JSON 文件。 ### 落盘结构 @@ -122,7 +122,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch - **检测结果** `response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`：每行一个 Case 的检测结果，字段含义见[检测结果说明](#检测结果说明)。 - **payload 归档** `response_anomaly/<模型 abbr>/payload/<数据集 abbr>/`：`all` 保留全部 Case，`anomalies` 只保留异常及检测失败/不可用 Case，`none` 不保留（目录不存在）。读取时用 zstandard 解压 `part-*.jsonl.zst` 分片后逐行解析 JSON；`payload_manifest.json` 记录每个分片的行数、大小与 sha256 校验值，可用于完整性校验。注意：`anomalies` 模式下即使无任何需保留的 Case，仍会发布一个仅含空 manifest 的归档目录，表示归档流程已成功完成，不是残留文件。 - **临时文件**：`payload_staging/` 在推理期间逐条接收 payload 写入，检测完成后自动清理；检测中断后残留的 `.<数据集>.payload-build-*` 构建目录会在下次检测启动时自动清理；`status_tmp/tmp_ResponseAnomaly.json` 为运行期状态文件（检测进度与类型统计），工作流结束后随状态目录一并清理。 -- **自动生成的 msProbe 配置** `response_anomaly_config/<模型 abbr>/`：由模型 `path`（本地模型目录）自动生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 +- **自动生成的检测配置** `response_anomaly_config/<模型 abbr>/`：由模型 `path`（本地模型目录）自动生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 - **检测日志** `logs/response_anomaly/<模型 abbr>/<数据集 abbr>.out`：记录对应模型/数据集组的检测过程，含检测器初始化失败与单 Case 失败的具体原因。 --- @@ -135,9 +135,9 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch | 状态 | 含义 | 排查建议 | | --- | --- | --- | -| `completed` | 已调用 msProbe 并得到检测结果 | 无需处理 | +| `completed` | 检测器已执行并得到检测结果 | 无需处理 | | `skipped` | 推理响应未携带 token id 或 top-k logprobs | 检查服务端是否支持并返回 `logprobs` / `top_logprobs` / token id 字段 | -| `unavailable` | 未安装 `mindstudio-probe`（response_anomaly extra） | 参考[安装指南](../get_started/install.md)安装可选依赖后重跑 | +| `unavailable` | 未安装异常检测可选依赖（response_anomaly extra） | 参考[安装指南](../get_started/install.md)安装可选依赖后重跑 | | `failed` | 调用或输入转换发生异常 | 查看该 Case 结果中的 `reason` 字段（保存错误类型与摘要）及检测日志 | 检测专属日志位于 `/logs/response_anomaly/<模型>/<数据集>.out`，检测进度与类型统计也可在 `/status_tmp/tmp_ResponseAnomaly.json` 状态文件中查看。 diff --git a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md index 3cabe7ac..a6c229ca 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/cli_args.md +++ b/docs/source_zh_cn/base_tutorials/all_params/cli_args.md @@ -36,7 +36,7 @@ ais_bench [OPTIONS] | `--num-prompts` | 指定数据集测评条数（按照数据集顺序选取），需传入正整数，超过数据集条数或默认情况下表示对全量数据集进行测评。 | `--num-prompts 500` | | `--max-num-workers` | 并行任务数，范围 `[1, CPU 核数]`，默认 `1`。在指定`--debug`时配置无效，所有任务串行执行。注意：性能测评场景下，并发数过高可能会导致不同进程出现资源抢占，导致测试结果失真。 | `--max-num-workers 2` | |`--num-warmups`|发送请求前预热次数，按照数据集顺序选取数据进行测试，大概num-warmups大于数据集条数时，会循环发送数据集中数据。默认 `1`；若设为0，则不预热。如果warmup阶段所有请求失败，后续推理任务将不会执行。| `--num-warmups 10` | -| `--response-anomaly` | 开启 msProbe 推理响应异常检测，无需额外配置：命令行增加 `--response-anomaly` 即开启，不增加默认不开启。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。详细用法见 📚 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 | `--response-anomaly` | +| `--response-anomaly` | 开启推理响应异常检测，无需额外配置：命令行增加 `--response-anomaly` 即开启，不增加默认不开启。检测串行绑定在推理阶段内：推理结束后启动检测并等待其完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程；需服务返回 token id 与 top-k logprobs。仅支持 `all`、`infer`、`infer_judge` 普通生成链路，不支持性能模式与 Agent 测评模式。详细用法见 📚 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 | `--response-anomaly` | | `--response-anomaly-payload-retention` | 异常检测完成后的 payload 保存模式：`all` 保存全部，`anomalies` 保存异常及检测失败/不可用 Case，`none` 不保存。默认 `anomalies`。 | `--response-anomaly-payload-retention anomalies` | ### API 模型通用覆盖参数 diff --git a/docs/source_zh_cn/base_tutorials/all_params/mode.md b/docs/source_zh_cn/base_tutorials/all_params/mode.md index d724152e..1c1e29f0 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/mode.md +++ b/docs/source_zh_cn/base_tutorials/all_params/mode.md @@ -107,7 +107,7 @@ outputs/default/ ### 响应异常检测模式支持（可选） -msProbe 推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测在推理阶段结束后启动，并**串行绑定在推理阶段内**——工作流会等待检测完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，详细用法请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 +推理响应异常检测（`--response-anomaly`）仅支持 `all`、`infer`、`infer_judge` 普通生成链路：检测在推理阶段结束后启动，并**串行绑定在推理阶段内**——工作流会等待检测完成（专属状态面板打印最终结果后）才进入后续 Judge / Eval / 汇总流程，保证推理阶段退出时检测结果与 payload 归档均已落盘。`perf` 与 `perf_viz` 性能评测模式以及 Agent / 函数调用等自定义链路**不支持**该功能，启用时会在配置初始化阶段显式报错。使用该功能要求服务化模型返回 token id 与 top-k logprobs，详细用法请参考 [推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)。 ## 性能评测场景 ### perf模式 diff --git a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md index 314cf7f1..d17dfe8e 100644 --- a/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md +++ b/docs/source_zh_cn/base_tutorials/scenes_intro/accuracy_benchmark.md @@ -324,7 +324,7 @@ ais_bench --models vllm_api_general --datasets gsm8k_gen --reuse 20250628_151326 > ⚠️ 注意：中断续测与失败重测可能改变请求顺序，可能引发结果微小波动。 -> 💡 启用了[推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)时，续测同样继承已有检测结果：`completed` 状态的 Case 不会重复调用 msProbe，`skipped` / `failed` / `unavailable` 状态的 Case 会重新检测；已有异常计数累加进最终统计。 +> 💡 启用了[推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)时，续测同样继承已有检测结果：`completed` 状态的 Case 不会重复执行检测，`skipped` / `failed` / `unavailable` 状态的 Case 会重新检测；已有异常计数累加进最终统计。 💡[多任务测评](#多任务测评) 也支持全量和部分任务的中断续测 & 失败用例重测。 diff --git a/docs/source_zh_cn/get_started/install.md b/docs/source_zh_cn/get_started/install.md index a3118b4d..2e653697 100644 --- a/docs/source_zh_cn/get_started/install.md +++ b/docs/source_zh_cn/get_started/install.md @@ -33,7 +33,7 @@ pip3 install -r requirements/extra.txt ⚙️ 推理响应异常检测支持（可选） -若需使用 msProbe 推理响应异常检测（`--response-anomaly`），需额外安装相关依赖： +若需使用推理响应异常检测（`--response-anomaly`），需额外安装相关依赖： ```shell pip3 install -r requirements/response_anomaly.txt ``` @@ -42,7 +42,7 @@ pip3 install -r requirements/response_anomaly.txt pip3 install 'ais-bench-benchmark[response_anomaly]' ``` -**注意**：该依赖包含从 GitCode 下载并构建固定提交的 `mindstudio-probe` 源码，安装环境需要 Git 与网络访问。未安装该依赖不影响 AISBench 主流程，相关 Case 的检测结果会标记为 `unavailable`。 +**注意**：该依赖包含从 GitCode 下载并构建固定提交的检测器源码，安装环境需要 Git 与网络访问。未安装该依赖不影响 AISBench 主流程，相关 Case 的检测结果会标记为 `unavailable`。 ⚙️ Huggingface多模态模型/vllm多模态离线推理支持（可选） From 2b775dc61637caf903fc85225eb29b8d8aedbb59 Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 08:48:37 +0800 Subject: [PATCH 19/25] docs: remind users to keep the model name in path consistent with model repositories The path directory basename is the model-name source for detection; document that it must stay consistent with the model name on Hugging Face / ModelScope / Modelers and must not be renamed arbitrarily, in the models.md path row and the anomaly detection tutorial tip (zh/en). --- docs/source_en/advanced_tutorials/response_anomaly_detection.md | 2 +- docs/source_en/base_tutorials/all_params/models.md | 2 +- .../advanced_tutorials/response_anomaly_detection.md | 2 +- docs/source_zh_cn/base_tutorials/all_params/models.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index 4434f25f..98a5d28b 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -69,7 +69,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch All three modes keep the standalone detection results. -> 💡 **Detection resources are prepared fully automatically**: the model name is taken from the basename of the model `path` (local model directory; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`); the detection threshold config, model mapping, and token-category vocabulary are auto-generated into `/response_anomaly_config//`, and an existing `config.yaml` is never overwritten, so manually tuned thresholds are preserved. If the model config provides no `path`, the task fails fast at startup with explicit guidance. +> 💡 **Detection resources are prepared fully automatically**: the model name is taken from the basename of the model `path` (local model directory; e.g. `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`); the detection threshold config, model mapping, and token-category vocabulary are auto-generated into `/response_anomaly_config//`, and an existing `config.yaml` is never overwritten, so manually tuned thresholds are preserved. If the model config provides no `path`, the task fails fast at startup with explicit guidance. Note: the final directory name of `path` must stay consistent with the model name in model repositories such as Hugging Face, ModelScope, or Modelers; do not rename it arbitrarily, otherwise the model name may fail to be parsed and detection may not run correctly. **Inspect the detection results**: after inference and detection finish, the results are written to `/response_anomaly//.jsonl`, one Case per line (see [Runtime Flow and On-Disk Layout](#runtime-flow-and-on-disk-layout) for the full layout). diff --git a/docs/source_en/base_tutorials/all_params/models.md b/docs/source_en/base_tutorials/all_params/models.md index 04e1dad0..56fd9b71 100644 --- a/docs/source_en/base_tutorials/all_params/models.md +++ b/docs/source_en/base_tutorials/all_params/models.md @@ -71,7 +71,7 @@ The description of configurable parameters for the service-oriented inference ba | `attr` | String | Identifier for the inference backend type, fixed as `service` (service-oriented inference) or `local` (local model); cannot be customized | | `type` | Python Class | Class name of the API type, automatically associated by the system; no manual configuration is required by the user. Refer to [Service-Oriented Inference Backend](#service-oriented-inference-backend) | | `abbr` | String | Unique identifier for the service-oriented task, used to distinguish different tasks. It consists of English characters and hyphens, e.g., `vllm-api-general-chat` | -| `path` | String | Tokenizer path, usually the same as the model path. The Tokenizer is loaded using `AutoTokenizer.from_pretrained(path)`. Specify an accessible local path, e.g., `/weight/DeepSeek-R1` | +| `path` | String | Tokenizer path, usually the same as the model path. The Tokenizer is loaded using `AutoTokenizer.from_pretrained(path)`. Specify an accessible local path, e.g., `/weight/DeepSeek-R1`. The final directory name must stay consistent with the model name in model repositories such as Hugging Face, ModelScope, or Modelers; do not rename it arbitrarily, otherwise the model name may fail to be parsed (e.g., [response anomaly detection](../../advanced_tutorials/response_anomaly_detection.md) derives the model name from this path) | | `model` | String | Name of the model accessible on the server, which must be consistent with the name specified during service-oriented deployment | | `model_name` | String | Applicable only to Triton services. It is concatenated into the endpoint URI `/v2/models/{modelname}/{infer, generate, generate_stream}` and must be consistent with the name used during deployment | | `stream` | Boolean | API model inference interface type. The default is False, meaning a non-streaming interface. When True, it indicates a streaming interface (for details, refer to 🔗 [Service-Oriented Inference Backend](#service-oriented-inference-backend)) | diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index a76b0bb9..641a4a21 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -69,7 +69,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch 三种模式都保留独立检测结果。 -> 💡 **检测资源全自动准备**：模型名自动取模型 `path`（本地模型目录）的目录名（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`）；检测阈值配置、模型映射与 token 分类词表自动生成到 `/response_anomaly_config/<模型 abbr>/`，已存在的 `config.yaml` 不会被覆盖，便于保留手工调优的阈值。若模型配置未提供 `path`，任务会在启动时报错并给出明确的解决指引。 +> 💡 **检测资源全自动准备**：模型名自动取模型 `path`（本地模型目录）的目录名（如 `/home/Qwen3-30B-A3B` → `Qwen3-30B-A3B`）；检测阈值配置、模型映射与 token 分类词表自动生成到 `/response_anomaly_config/<模型 abbr>/`，已存在的 `config.yaml` 不会被覆盖，便于保留手工调优的阈值。若模型配置未提供 `path`，任务会在启动时报错并给出明确的解决指引。注意：`path` 末级目录名需与 Hugging Face、ModelScope、Modelers 等模型仓库中的模型名称保持一致，请勿随意改动，否则可能导致模型名称解析失败、检测无法正确执行。 **查看检测结果**：推理与检测结束后，检测结果位于 `/response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`，每行一个 Case（完整落盘结构见[运行流程与落盘结构](#运行流程与落盘结构)）。 diff --git a/docs/source_zh_cn/base_tutorials/all_params/models.md b/docs/source_zh_cn/base_tutorials/all_params/models.md index ea57cc95..eed506cd 100644 --- a/docs/source_zh_cn/base_tutorials/all_params/models.md +++ b/docs/source_zh_cn/base_tutorials/all_params/models.md @@ -66,7 +66,7 @@ models = [ # 相当于自定义配置文件中通过 `from ais_bench.benchmark.c | `attr` | String | 推理后端类型标识，固定为 `service`（服务化推理）或 `local`（本地模型），不可配置 | | `type` | Python Class | API 类型类名，由系统自动关联，用户无需手动配置，参考 [服务化推理后端](#服务化推理后端) | | `abbr` | String | 服务化任务的唯一标识，用于区分不同任务，英文字符与短横线组合，例如：`vllm-api-general-chat` | -| `path` | String | Tokenizer 路径，通常与模型路径相同，使用 `AutoTokenizer.from_pretrained(path)` 加载。指定可访问的本地路径，例如：`/weight/DeepSeek-R1` | +| `path` | String | Tokenizer 路径，通常与模型路径相同，使用 `AutoTokenizer.from_pretrained(path)` 加载。指定可访问的本地路径，例如：`/weight/DeepSeek-R1`。路径末级的目录名需与 Hugging Face、ModelScope、Modelers 等模型仓库中的模型名称保持一致，请勿随意改动，否则可能导致模型名称解析失败（如[推理响应异常检测](../../advanced_tutorials/response_anomaly_detection.md)依赖该路径解析模型名称） | | `model` | String | 服务端可访问的模型名称，必须与服务化部署时指定的名称一致 | | `model_name` | String | 仅适用于 Triton 服务，拼接为 endpoint 的 URI `/v2/models/{modelname}/{infer、generate、generate_stream}`，应与部署时名称一致 | | `stream` | Boolean | API模型推理接口类型，默认为False，表示非流式接口，当为True时表示流式接口（具体请参考🔗[服务化推理后端](#服务化推理后端)）| From a9f535577b7f4675acf5d2ae8d1f040e25470605 Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:00:23 +0800 Subject: [PATCH 20/25] docs: label the payload retention table with its parameter name The bare value/behavior table header did not identify which parameter the values belong to; label it with --response-anomaly-payload-retention (zh/en). --- docs/source_en/advanced_tutorials/response_anomaly_detection.md | 2 +- .../advanced_tutorials/response_anomaly_detection.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index 98a5d28b..d70da157 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -61,7 +61,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch --response-anomaly-payload-retention anomalies ``` -| Value | Behavior | +| `--response-anomaly-payload-retention` value | Behavior | | ----- | -------- | | `all` | Keeps every payload and atomically promotes the staging data to the official archive after detection without re-compressing | | `anomalies` (default) | Keeps only detected anomalies plus detection-failed/unavailable Cases | diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index 641a4a21..ab0b962d 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -61,7 +61,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch --response-anomaly-payload-retention anomalies ``` -| 取值 | 行为 | +| `--response-anomaly-payload-retention` 取值 | 行为 | | ---- | ---- | | `all` | 保存全部 payload，检测后直接将 staging 原子转为正式归档，不会二次压缩 | | `anomalies`（默认） | 只保存已检出异常以及检测失败/不可用 Case | From 58f380d94c2be8caace01d6d34f272bf939e47eb Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:02:37 +0800 Subject: [PATCH 21/25] docs: drop prediction wording from the anomaly detection flow The detection flow step describes payload staging only; the prediction-file behavior belongs to the main inference flow and is out of scope here (zh/en). --- docs/source_en/advanced_tutorials/response_anomaly_detection.md | 2 +- .../advanced_tutorials/response_anomaly_detection.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index d70da157..06fb59fc 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -83,7 +83,7 @@ When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the ### Detection Flow -1. **Inference stage**: the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`, and predictions only keep lightweight results from the start; +1. **Inference stage**: the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`; 2. **Detection stage**: after inference finishes, the detection thread streams and decompresses the staging data and runs anomaly detection; detection results are written to `response_anomaly//.jsonl`; 3. **Archive finalization**: after detection, the staging data is retained or cleaned according to `--response-anomaly-payload-retention`. diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index ab0b962d..324fc4b4 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -83,7 +83,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch ### 检测流程 -1. **推理阶段**：完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`，prediction 从一开始只保存轻量结果； +1. **推理阶段**：完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`； 2. **检测阶段**：推理结束后，检测线程流式解压 staging 数据并执行异常检测，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`； 3. **归档收尾**：检测完成后按 `--response-anomaly-payload-retention` 保留或清理 staging。 From b482f1b6224e08d3fbf1fc09c6eb8581c048f2f4 Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:04:04 +0800 Subject: [PATCH 22/25] docs: fix conjunction in the archive finalization step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retention and staging cleanup both happen during archive finalization; use and/及 instead of or (zh/en). --- docs/source_en/advanced_tutorials/response_anomaly_detection.md | 2 +- .../advanced_tutorials/response_anomaly_detection.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index 06fb59fc..d5bd0e7c 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -85,7 +85,7 @@ When enabled, AISBench adds `logprobs=True` and a fixed `top_logprobs=20` to the 1. **Inference stage**: the full payload is written directly to `response_anomaly//payload_staging//*.jsonl.zst`; 2. **Detection stage**: after inference finishes, the detection thread streams and decompresses the staging data and runs anomaly detection; detection results are written to `response_anomaly//.jsonl`; -3. **Archive finalization**: after detection, the staging data is retained or cleaned according to `--response-anomaly-payload-retention`. +3. **Archive finalization**: after detection, the staging data is retained and cleaned according to `--response-anomaly-payload-retention`. The status panel shows the config preparation, detector loading, streaming detection, and archive finalization stages. diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index 324fc4b4..80b3a503 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -85,7 +85,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch 1. **推理阶段**：完整 payload 直接写入 `response_anomaly/<模型>/payload_staging/<数据集>/*.jsonl.zst`； 2. **检测阶段**：推理结束后，检测线程流式解压 staging 数据并执行异常检测，检测结果写入 `response_anomaly/<模型>/<数据集>.jsonl`； -3. **归档收尾**：检测完成后按 `--response-anomaly-payload-retention` 保留或清理 staging。 +3. **归档收尾**：检测完成后按 `--response-anomaly-payload-retention` 保留及清理 staging。 状态面板会显示配置准备、检测器加载、流式检测和归档收尾阶段。 From 664f931ac5c76f1085bd824767949c4046674279 Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:18:24 +0800 Subject: [PATCH 23/25] fix: remove the empty payload_staging shell after detection Detection cleanup only removed the per-dataset staging subtree (or renamed it away in all mode), leaving an empty payload_staging directory under response_anomaly/. rmdir the staging root after the per-dataset cleanup; it only succeeds when the last dataset of the model finishes detection, so multi-dataset runs stay safe. Covered by a new UT; wording in the zh/en tutorial updated accordingly. --- ais_bench/benchmark/utils/response_anomaly.py | 4 +++ .../response_anomaly_detection.md | 4 +-- .../response_anomaly_detection.md | 4 +-- tests/UT/utils/test_response_anomaly.py | 32 +++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ais_bench/benchmark/utils/response_anomaly.py b/ais_bench/benchmark/utils/response_anomaly.py index 2e7ff99e..b506ec9b 100644 --- a/ais_bench/benchmark/utils/response_anomaly.py +++ b/ais_bench/benchmark/utils/response_anomaly.py @@ -868,6 +868,10 @@ def _finalize_group_payloads( shutil.rmtree(payload.payload_dir) if payload.source_dir.exists(): shutil.rmtree(payload.source_dir) + try: + payload.source_dir.parent.rmdir() + except OSError: + pass if any( "response_anomaly_payload" in prediction for prediction in task.predictions diff --git a/docs/source_en/advanced_tutorials/response_anomaly_detection.md b/docs/source_en/advanced_tutorials/response_anomaly_detection.md index d5bd0e7c..850282a4 100644 --- a/docs/source_en/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_en/advanced_tutorials/response_anomaly_detection.md @@ -101,7 +101,7 @@ Files produced by detection are laid out under `` as follows: ├── response_anomaly/ │ └── / │ ├── .jsonl # Detection results, one Case per line -│ ├── payload_staging// # Transient staging during inference; cleaned after detection +│ ├── payload_staging// # Transient staging during inference; removed (including the directory) after detection │ │ └── part-*.jsonl.zst │ └── payload// # Payload archive; absent when payload retention is none │ ├── payload_manifest.json # Archive manifest (per-shard rows, sizes, sha256) @@ -121,7 +121,7 @@ Path-by-path notes: - **Detection results** `response_anomaly//.jsonl`: one Case per line; field semantics are described in [Detection Results](#detection-results). - **Payload archive** `response_anomaly//payload//`: `all` keeps every Case, `anomalies` keeps only anomalous plus detection-failed/unavailable Cases, and `none` keeps nothing (the directory does not exist). To read the data, decompress the `part-*.jsonl.zst` shards with zstandard and parse each line as JSON; `payload_manifest.json` records per-shard row counts, sizes, and sha256 checksums for integrity verification. Note: under `anomalies`, even when there is nothing to retain, an archive directory containing only an empty manifest is still published to indicate the archiving flow completed successfully — it is not a leftover file. -- **Transient files**: `payload_staging/` receives payload records during inference and is cleaned automatically after detection; `..payload-build-*` build directories left by an interrupted detection are cleaned automatically when the next detection starts; `status_tmp/tmp_ResponseAnomaly.json` is a runtime status file (detection progress and per-type statistics) removed together with the status directory when the workflow ends. +- **Transient files**: `payload_staging/` receives payload records during inference and is removed automatically (including the directory) after detection; `..payload-build-*` build directories left by an interrupted detection are cleaned automatically when the next detection starts; `status_tmp/tmp_ResponseAnomaly.json` is a runtime status file (detection progress and per-type statistics) removed together with the status directory when the workflow ends. - **Auto-generated detection configs** `response_anomaly_config//`: auto-generated from the model `path` (local model directory). An existing `config.yaml` is never overwritten (manually tuned thresholds are preserved), and `mtype_config.json` supports multi-model merging across repeated generations. - **Detection log** `logs/response_anomaly//.out`: records the detection run for the model/dataset group, including detector initialization failures and per-Case failure reasons. diff --git a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md index 80b3a503..38ad6049 100644 --- a/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md +++ b/docs/source_zh_cn/advanced_tutorials/response_anomaly_detection.md @@ -101,7 +101,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch ├── response_anomaly/ │ └── <模型 abbr>/ │ ├── <数据集 abbr>.jsonl # 检测结果，每行一个 Case -│ ├── payload_staging/<数据集 abbr>/ # 推理期间的临时存放区，检测完成后自动清理 +│ ├── payload_staging/<数据集 abbr>/ # 推理期间的临时存放区，检测完成后连同目录一并清理 │ │ └── part-*.jsonl.zst │ └── payload/<数据集 abbr>/ # payload 归档；payload 保留模式为 none 时不存在 │ ├── payload_manifest.json # 归档清单（分片行数、大小、sha256） @@ -121,7 +121,7 @@ ais_bench --models vllm_api_general_chat --datasets demo_gsm8k_gen_4_shot_cot_ch - **检测结果** `response_anomaly/<模型 abbr>/<数据集 abbr>.jsonl`：每行一个 Case 的检测结果，字段含义见[检测结果说明](#检测结果说明)。 - **payload 归档** `response_anomaly/<模型 abbr>/payload/<数据集 abbr>/`：`all` 保留全部 Case，`anomalies` 只保留异常及检测失败/不可用 Case，`none` 不保留（目录不存在）。读取时用 zstandard 解压 `part-*.jsonl.zst` 分片后逐行解析 JSON；`payload_manifest.json` 记录每个分片的行数、大小与 sha256 校验值，可用于完整性校验。注意：`anomalies` 模式下即使无任何需保留的 Case，仍会发布一个仅含空 manifest 的归档目录，表示归档流程已成功完成，不是残留文件。 -- **临时文件**：`payload_staging/` 在推理期间逐条接收 payload 写入，检测完成后自动清理；检测中断后残留的 `.<数据集>.payload-build-*` 构建目录会在下次检测启动时自动清理；`status_tmp/tmp_ResponseAnomaly.json` 为运行期状态文件（检测进度与类型统计），工作流结束后随状态目录一并清理。 +- **临时文件**：`payload_staging/` 在推理期间逐条接收 payload 写入，检测完成后连同目录一并清理；检测中断后残留的 `.<数据集>.payload-build-*` 构建目录会在下次检测启动时自动清理；`status_tmp/tmp_ResponseAnomaly.json` 为运行期状态文件（检测进度与类型统计），工作流结束后随状态目录一并清理。 - **自动生成的检测配置** `response_anomaly_config/<模型 abbr>/`：由模型 `path`（本地模型目录）自动生成。`config.yaml` 已存在时不会被覆盖（保留手工调优的阈值）；`mtype_config.json` 支持多模型合并，多次生成不互相覆盖。 - **检测日志** `logs/response_anomaly/<模型 abbr>/<数据集 abbr>.out`：记录对应模型/数据集组的检测过程，含检测器初始化失败与单 Case 失败的具体原因。 diff --git a/tests/UT/utils/test_response_anomaly.py b/tests/UT/utils/test_response_anomaly.py index bcbdcf05..7886677d 100644 --- a/tests/UT/utils/test_response_anomaly.py +++ b/tests/UT/utils/test_response_anomaly.py @@ -799,6 +799,38 @@ def test_resume_backfills_only_inherited_payloads_missing_from_archive( assert len(archived_ids) == len(set(archived_ids)) +def test_detection_removes_payload_staging_shell(tmp_path, monkeypatch): + """检测收尾后 payload_staging 目录本体一并移除，不留空壳目录。""" + prediction_file = tmp_path / "predictions" / "modelA" / "ds.jsonl" + _write_jsonl( + prediction_file, + [{"data_abbr": "ds", "id": 1, "uuid": "u1", "prediction": "ok"}], + ) + source_dir = ( + tmp_path / "response_anomaly" / "modelA" / "payload_staging" / "ds" + ) + source_writer = ResponseAnomalyJsonlWriter(source_dir, 3, 10) + source_writer.write(_payload_record(1)) + source_writer.close(write_manifest=False) + + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + monkeypatch.setattr( + coordinator, + "_detect_case", + lambda prediction, anomaly_cfg, detector=None, init_error=None: ( + _completed_anomaly_result(prediction["id"]) + ), + ) + + coordinator._detect(_anomaly_cfg(tmp_path)) + + assert not source_dir.exists() + assert not source_dir.parent.exists() + + def test_resume_backfills_inherited_legacy_prediction_payload( tmp_path, monkeypatch ): From 878fd2711107cc053f923e697fb87738dfed018b Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:59:34 +0800 Subject: [PATCH 24/25] fix: drop redundant sha256 prefix in payload manifest checksum values The shard checksum value was formatted as sha256: while the field is already named sha256, duplicating the algorithm name. Store the bare hex digest in both manifest builders and strengthen the UT to compare against the hash recomputed from the shard file. --- ais_bench/benchmark/utils/response_anomaly_jsonl.py | 4 ++-- tests/UT/utils/test_response_anomaly_jsonl.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ais_bench/benchmark/utils/response_anomaly_jsonl.py b/ais_bench/benchmark/utils/response_anomaly_jsonl.py index 0ec13a92..e1a29542 100644 --- a/ais_bench/benchmark/utils/response_anomaly_jsonl.py +++ b/ais_bench/benchmark/utils/response_anomaly_jsonl.py @@ -115,7 +115,7 @@ def _close_shard(self) -> None: "file": final_path.name, "rows": self.shard_rows, "size_bytes": final_path.stat().st_size, - "sha256": f"sha256:{_sha256_file(final_path)}", + "sha256": _sha256_file(final_path), } ) self.shard_index += 1 @@ -186,7 +186,7 @@ def build_jsonl_zstd_manifest( "file": shard.name, "rows": rows, "size_bytes": shard.stat().st_size, - "sha256": f"sha256:{_sha256_file(shard)}", + "sha256": _sha256_file(shard), } ) manifest = { diff --git a/tests/UT/utils/test_response_anomaly_jsonl.py b/tests/UT/utils/test_response_anomaly_jsonl.py index 31957da5..939713c2 100644 --- a/tests/UT/utils/test_response_anomaly_jsonl.py +++ b/tests/UT/utils/test_response_anomaly_jsonl.py @@ -1,3 +1,4 @@ +import hashlib import json import pytest @@ -50,7 +51,9 @@ def test_jsonl_zstd_writer_round_trips_and_shards(tmp_path): assert len(shards) == 2 assert manifest["total_rows"] == 3 assert [item["rows"] for item in manifest["shards"]] == [2, 1] - assert manifest["shards"][0]["sha256"].startswith("sha256:") + assert manifest["shards"][0]["sha256"] == hashlib.sha256( + shards[0].read_bytes() + ).hexdigest() restored = [item for shard in shards for item in _read_shard(shard)] assert restored == records assert not list(tmp_path.glob("*.inprogress")) From 3434ef222b1162e070828356f6133ea7f2454371 Mon Sep 17 00:00:00 2001 From: Libotry <12138201+Libotry@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:17:19 +0800 Subject: [PATCH 25/25] fix: differentiate response anomaly summary log from per-task completion log The finalize step logged the aggregated summary with the exact same wording as the per-task completion line, making them indistinguishable in the console. The summary line now reads Response anomaly detection summary across N task(s). Per-task completion logs and dedicated task log files are unchanged; a new UT covers multi-dataset aggregation (summary totals) and per-task log isolation. --- ais_bench/benchmark/cli/workers.py | 3 +- tests/UT/utils/test_response_anomaly.py | 50 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/ais_bench/benchmark/cli/workers.py b/ais_bench/benchmark/cli/workers.py index afe9a212..09b4ca07 100644 --- a/ais_bench/benchmark/cli/workers.py +++ b/ais_bench/benchmark/cli/workers.py @@ -733,7 +733,8 @@ def _finalize_response_anomaly_detection( TasksMonitor.rm_tmp_files(work_dir) if coordinator.summary: logger.info( - "Response anomaly detection completed: %s", + "Response anomaly detection summary across %d task(s): %s", + len(coordinator.anomaly_report), coordinator.summary, ) for task_name, info in coordinator.anomaly_report.items(): diff --git a/tests/UT/utils/test_response_anomaly.py b/tests/UT/utils/test_response_anomaly.py index 7886677d..9094531a 100644 --- a/tests/UT/utils/test_response_anomaly.py +++ b/tests/UT/utils/test_response_anomaly.py @@ -632,6 +632,56 @@ def test_detect_writes_separate_status_and_log_for_each_dataset( ) +def test_detect_summary_aggregates_across_tasks(tmp_path, monkeypatch): + """多数据集检测：summary 为全量聚合，单任务日志只含本任务统计。""" + for dataset_abbr, token_sets in (("ds1", [[1]]), ("ds2", [[2], [1]])): + prediction_file = ( + tmp_path / "predictions" / "modelA" / f"{dataset_abbr}.jsonl" + ) + _write_jsonl( + prediction_file, + [ + { + "data_abbr": dataset_abbr, + "id": idx + 1, + "uuid": f"{dataset_abbr}-u{idx + 1}", + "prediction": "ok", + "response_anomaly_payload": { + "tokens": token_tokens, + "topk_logprobs": [ + {str(t): -0.1} for t in token_tokens + ], + }, + } + for idx, token_tokens in enumerate(token_sets) + ], + ) + cfg = { + "work_dir": str(tmp_path), + "models": [{"abbr": "modelA", "attr": "service"}], + "datasets": [{"abbr": "ds1"}, {"abbr": "ds2"}], + "response_anomaly": {}, + } + coordinator = ResponseAnomalyCoordinator() + monkeypatch.setattr( + coordinator, "_build_detector", lambda cfg: (TokenDetector(), None) + ) + + coordinator._detect(cfg) + + assert coordinator.summary == {"normal": 2, "rare_character": 1} + ds1_log = ( + tmp_path / "logs" / "response_anomaly" / "modelA" / "ds1.out" + ).read_text(encoding="utf-8") + ds2_log = ( + tmp_path / "logs" / "response_anomaly" / "modelA" / "ds2.out" + ).read_text(encoding="utf-8") + assert "Response anomaly detection completed: {'normal': 1}" in ds1_log + assert "'rare_character': 1" in ds2_log + assert "'normal': 1" in ds2_log + assert "'normal': 2" not in ds2_log + + @pytest.mark.parametrize( ("retention", "expected_ids"), [("all", [1, 2]), ("anomalies", [2]), ("none", [])],