From ab3739219e452b73496021d3503358e74f1ff0b7 Mon Sep 17 00:00:00 2001 From: jialin Date: Thu, 18 Jun 2026 15:42:58 +0800 Subject: [PATCH] chore: update vllm, sglang, preset argument options --- src/pages/llmodels/config/index.ts | 2 +- .../constants/backend-parameters/sglang.ts | 727 ++++++-- .../constants/backend-parameters/vllm.ts | 1502 ++++++++++++----- 3 files changed, 1663 insertions(+), 568 deletions(-) diff --git a/src/pages/llmodels/config/index.ts b/src/pages/llmodels/config/index.ts index 478015e3..17b3a4d6 100644 --- a/src/pages/llmodels/config/index.ts +++ b/src/pages/llmodels/config/index.ts @@ -455,7 +455,7 @@ export const getBackendParamsTips = (backend: string) => { return { backend: 'SGLang', releases: '', - link: 'https://docs.sglang.ai/advanced_features/server_arguments.html', + link: 'https://docs.sglang.io/docs/advanced_features/server_arguments', version: 'v0.5.4' }; } diff --git a/src/pages/llmodels/constants/backend-parameters/sglang.ts b/src/pages/llmodels/constants/backend-parameters/sglang.ts index f12238ac..f2958f14 100644 --- a/src/pages/llmodels/constants/backend-parameters/sglang.ts +++ b/src/pages/llmodels/constants/backend-parameters/sglang.ts @@ -1,5 +1,12 @@ import { BackendParameter } from './index'; +// Generated from the SGLang `launch_server` server arguments reference: +// https://docs.sglang.io/docs/advanced_features/server_arguments +// Grouped by the doc's sections. `options` lists the fixed choice values +// when the flag accepts a restricted set (sourced from the doc's Options column). +// Internal/experimental sections (Ktransformers, Diffusion LLM, Forward hooks, +// msProbe dump) and deprecated arguments are omitted. + const options: BackendParameter[] = [ // Model and tokenizer { @@ -45,8 +52,12 @@ const options: BackendParameter[] = [ 'gguf', 'bitsandbytes', 'layered', + 'flash_rl', 'remote', - 'remote_instance' + 'remote_instance', + 'fastsafetensors', + 'private', + 'runai_streamer' ] }, { @@ -85,11 +96,26 @@ const options: BackendParameter[] = [ options: [] }, // HTTP server + { + label: '--host', + value: '--host', + options: [] + }, { label: '--port', value: '--port', options: [] }, + { + label: '--fastapi-root-path', + value: '--fastapi-root-path', + options: [] + }, + { + label: '--grpc-mode', + value: '--grpc-mode', + options: [] + }, { label: '--skip-server-warmup', value: '--skip-server-warmup', @@ -105,6 +131,11 @@ const options: BackendParameter[] = [ value: '--nccl-port', options: [] }, + { + label: '--checkpoint-engine-wait-weights-before-ready', + value: '--checkpoint-engine-wait-weights-before-ready', + options: [] + }, // Quantization and data type { label: '--dtype', @@ -124,6 +155,7 @@ const options: BackendParameter[] = [ 'bitsandbytes', 'gguf', 'modelopt', + 'modelopt_fp8', 'modelopt_fp4', 'petit_nvfp4', 'w8a8_int8', @@ -131,7 +163,12 @@ const options: BackendParameter[] = [ 'moe_wna16', 'qoq', 'w4afp8', - 'mxfp4' + 'mxfp4', + 'mxfp8', + 'auto-round', + 'compressed-tensors', + 'modelslim', + 'quark_int4fp8_moe' ] }, { @@ -139,6 +176,16 @@ const options: BackendParameter[] = [ value: '--quantization-param-path', options: [] }, + { + label: '--kv-cache-dtype', + value: '--kv-cache-dtype', + options: ['auto', 'fp8_e5m2', 'fp8_e4m3', 'bf16', 'bfloat16', 'fp4_e2m1'] + }, + { + label: '--enable-fp32-lm-head', + value: '--enable-fp32-lm-head', + options: [] + }, { label: '--modelopt-quant', value: '--modelopt-quant', @@ -155,13 +202,23 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--kv-cache-dtype', - value: '--kv-cache-dtype', - options: ['auto', 'fp8_e5m2', 'fp8_e4m3'] + label: '--modelopt-export-path', + value: '--modelopt-export-path', + options: [] }, { - label: '--enable-fp32-lm-head', - value: '--enable-fp32-lm-head', + label: '--quantize-and-serve', + value: '--quantize-and-serve', + options: [] + }, + { + label: '--rl-quant-profile', + value: '--rl-quant-profile', + options: [] + }, + { + label: '--enable-quant-communications', + value: '--enable-quant-communications', options: [] }, // Memory and scheduling @@ -190,6 +247,16 @@ const options: BackendParameter[] = [ value: '--chunked-prefill-size', options: [] }, + { + label: '--prefill-max-requests', + value: '--prefill-max-requests', + options: [] + }, + { + label: '--enable-dynamic-chunking', + value: '--enable-dynamic-chunking', + options: [] + }, { label: '--max-prefill-tokens', value: '--max-prefill-tokens', @@ -198,13 +265,26 @@ const options: BackendParameter[] = [ { label: '--schedule-policy', value: '--schedule-policy', - options: ['lpm', 'random', 'fcfs', 'dfs-weight', 'lof', 'priority'] + options: [ + 'lpm', + 'random', + 'fcfs', + 'dfs-weight', + 'lof', + 'priority', + 'routing-key' + ] }, { label: '--enable-priority-scheduling', value: '--enable-priority-scheduling', options: [] }, + { + label: '--abort-on-priority-when-disabled', + value: '--abort-on-priority-when-disabled', + options: [] + }, { label: '--schedule-low-priority-values-first', value: '--schedule-low-priority-values-first', @@ -225,11 +305,6 @@ const options: BackendParameter[] = [ value: '--page-size', options: [] }, - { - label: '--hybrid-kvcache-ratio', - value: '--hybrid-kvcache-ratio', - options: [] - }, { label: '--swa-full-tokens-ratio', value: '--swa-full-tokens-ratio', @@ -240,22 +315,52 @@ const options: BackendParameter[] = [ value: '--disable-hybrid-swa-memory', options: [] }, + { + label: '--radix-eviction-policy', + value: '--radix-eviction-policy', + options: ['lru', 'lfu'] + }, + { + label: '--enable-prefill-delayer', + value: '--enable-prefill-delayer', + options: [] + }, + { + label: '--prefill-delayer-max-delay-passes', + value: '--prefill-delayer-max-delay-passes', + options: [] + }, + { + label: '--prefill-delayer-token-usage-low-watermark', + value: '--prefill-delayer-token-usage-low-watermark', + options: [] + }, + { + label: '--prefill-delayer-queue-min-ratio', + value: '--prefill-delayer-queue-min-ratio', + options: [] + }, + { + label: '--prefill-delayer-max-delay-ms', + value: '--prefill-delayer-max-delay-ms', + options: [] + }, + { + label: '--prefill-delayer-forward-passes-buckets', + value: '--prefill-delayer-forward-passes-buckets', + options: [] + }, + { + label: '--prefill-delayer-wait-seconds-buckets', + value: '--prefill-delayer-wait-seconds-buckets', + options: [] + }, // Runtime options { label: '--device', value: '--device', options: [] }, - { - label: '--elastic-ep-backend', - value: '--elastic-ep-backend', - options: [] - }, - { - label: '--mooncake-ib-device', - value: '--mooncake-ib-device', - options: [] - }, { label: '--tensor-parallel-size', value: '--tensor-parallel-size', @@ -276,19 +381,44 @@ const options: BackendParameter[] = [ value: '--pp-size', options: [] }, + { + label: '--attention-context-parallel-size', + value: '--attention-context-parallel-size', + options: [] + }, + { + label: '--attn-cp-size', + value: '--attn-cp-size', + options: [] + }, + { + label: '--moe-data-parallel-size', + value: '--moe-data-parallel-size', + options: [] + }, + { + label: '--moe-dp-size', + value: '--moe-dp-size', + options: [] + }, { label: '--pp-max-micro-batch-size', value: '--pp-max-micro-batch-size', options: [] }, + { + label: '--pp-async-batch-depth', + value: '--pp-async-batch-depth', + options: [] + }, { label: '--stream-interval', value: '--stream-interval', options: [] }, { - label: '--stream-output', - value: '--stream-output', + label: '--incremental-streaming-output', + value: '--incremental-streaming-output', options: [] }, { @@ -311,6 +441,11 @@ const options: BackendParameter[] = [ value: '--watchdog-timeout', options: [] }, + { + label: '--soft-watchdog-timeout', + value: '--soft-watchdog-timeout', + options: [] + }, { label: '--dist-timeout', value: '--dist-timeout', @@ -321,6 +456,11 @@ const options: BackendParameter[] = [ value: '--download-dir', options: [] }, + { + label: '--model-checksum', + value: '--model-checksum', + options: [] + }, { label: '--base-gpu-id', value: '--base-gpu-id', @@ -336,6 +476,11 @@ const options: BackendParameter[] = [ value: '--sleep-on-idle', options: [] }, + { + label: '--custom-sigquit-handler', + value: '--custom-sigquit-handler', + options: [] + }, // Logging { label: '--log-level', @@ -358,13 +503,23 @@ const options: BackendParameter[] = [ options: ['0', '1', '2', '3'] }, { - label: '--crash-dump-folder', - value: '--crash-dump-folder', + label: '--log-requests-format', + value: '--log-requests-format', + options: ['text', 'json'] + }, + { + label: '--log-requests-target', + value: '--log-requests-target', options: [] }, { - label: '--crash-on-nan', - value: '--crash-on-nan', + label: '--uvicorn-access-log-exclude-prefixes', + value: '--uvicorn-access-log-exclude-prefixes', + options: [] + }, + { + label: '--crash-dump-folder', + value: '--crash-dump-folder', options: [] }, { @@ -377,6 +532,11 @@ const options: BackendParameter[] = [ value: '--enable-metrics', options: [] }, + { + label: '--enable-mfu-metrics', + value: '--enable-mfu-metrics', + options: [] + }, { label: '--enable-metrics-for-all-schedulers', value: '--enable-metrics-for-all-schedulers', @@ -448,8 +608,19 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--oltp-traces-endpoint', - value: '--oltp-traces-endpoint', + label: '--otlp-traces-endpoint', + value: '--otlp-traces-endpoint', + options: [] + }, + // RequestMetricsExporter configuration + { + label: '--export-metrics-to-file', + value: '--export-metrics-to-file', + options: [] + }, + { + label: '--export-metrics-to-file-dir', + value: '--export-metrics-to-file-dir', options: [] }, // API related @@ -458,6 +629,11 @@ const options: BackendParameter[] = [ value: '--api-key', options: [] }, + { + label: '--admin-api-key', + value: '--admin-api-key', + options: [] + }, { label: '--served-model-name', value: '--served-model-name', @@ -473,6 +649,11 @@ const options: BackendParameter[] = [ value: '--chat-template', options: [] }, + { + label: '--hf-chat-template-name', + value: '--hf-chat-template-name', + options: [] + }, { label: '--completion-template', value: '--completion-template', @@ -510,6 +691,7 @@ const options: BackendParameter[] = [ 'deepseekv31', 'glm', 'glm45', + 'glm47', 'gpt-oss', 'kimi_k2', 'llama3', @@ -518,25 +700,26 @@ const options: BackendParameter[] = [ 'qwen', 'qwen25', 'qwen3_coder', - 'step3' + 'step3', + 'gigachat3' ] }, - { - label: '--sampling-defaults', - value: '--sampling-defaults', - options: ['openai', 'model'] - }, { label: '--tool-server', value: '--tool-server', options: [] }, + { + label: '--sampling-defaults', + value: '--sampling-defaults', + options: ['openai', 'model'] + }, + // Data parallelism { label: '--data-parallel-size', value: '--data-parallel-size', options: [] }, - // Data parallelism { label: '--dp-size', value: '--dp-size', @@ -545,17 +728,13 @@ const options: BackendParameter[] = [ { label: '--load-balance-method', value: '--load-balance-method', - options: ['round_robin', 'shortest_queue', 'minimum_tokens'] - }, - { - label: '--load-watch-interval', - value: '--load-watch-interval', - options: [] - }, - { - label: '--prefill-round-robin-balance', - value: '--prefill-round-robin-balance', - options: [] + options: [ + 'auto', + 'round_robin', + 'follow_bootstrap_room', + 'total_requests', + 'total_tokens' + ] }, // Multi-node distributed serving { @@ -595,6 +774,11 @@ const options: BackendParameter[] = [ value: '--enable-lora', options: [] }, + { + label: '--enable-lora-overlap-loading', + value: '--enable-lora-overlap-loading', + options: [] + }, { label: '--max-lora-rank', value: '--max-lora-rank', @@ -619,7 +803,7 @@ const options: BackendParameter[] = [ { label: '--lora-paths', value: '--lora-paths', - options: ['{"lora_name": str, "lora_path": str, "pinned": bool}'] + options: [] }, { label: '--max-loras-per-batch', @@ -639,14 +823,19 @@ const options: BackendParameter[] = [ { label: '--lora-backend', value: '--lora-backend', - options: ['triton', 'csgmv'] + options: ['triton', 'csgmv', 'ascend', 'torch_native'] }, { label: '--max-lora-chunk-size', value: '--max-lora-chunk-size', options: ['16', '32', '64', '128'] }, - // Kernel backend + { + label: '--lora-drain-wait-threshold', + value: '--lora-drain-wait-threshold', + options: [] + }, + // Kernel Backends (Attention, Sampling, Grammar, GEMM) { label: '--attention-backend', value: '--attention-backend', @@ -654,6 +843,7 @@ const options: BackendParameter[] = [ 'triton', 'torch_native', 'flex_attention', + 'dsa', 'nsa', 'cutlass_mla', 'fa3', @@ -676,6 +866,7 @@ const options: BackendParameter[] = [ 'triton', 'torch_native', 'flex_attention', + 'dsa', 'nsa', 'cutlass_mla', 'fa3', @@ -698,6 +889,7 @@ const options: BackendParameter[] = [ 'triton', 'torch_native', 'flex_attention', + 'dsa', 'nsa', 'cutlass_mla', 'fa3', @@ -716,7 +908,7 @@ const options: BackendParameter[] = [ { label: '--sampling-backend', value: '--sampling-backend', - options: ['flashinfer', 'pytorch'] + options: ['flashinfer', 'pytorch', 'ascend'] }, { label: '--grammar-backend', @@ -726,17 +918,66 @@ const options: BackendParameter[] = [ { label: '--mm-attention-backend', value: '--mm-attention-backend', - options: ['sdpa', 'fa3', 'triton_attn', 'ascend_attn'] + options: ['sdpa', 'fa3', 'fa4', 'triton_attn', 'ascend_attn', 'aiter_attn'] }, { - label: '--nsa-prefill', - value: '--nsa-prefill', - options: ['flashmla_sparse', 'flashmla_decode', 'fa3', 'tilelang', 'aiter'] + label: '--dsa-prefill-backend', + value: '--dsa-prefill-backend', + options: [ + 'flashmla_sparse', + 'flashmla_kv', + 'flashmla_auto', + 'fa3', + 'tilelang', + 'aiter', + 'trtllm' + ] }, { - label: '--nsa-decode', - value: '--nsa-decode', - options: ['flashmla_prefill', 'flashmla_kv', 'fa3', 'tilelang', 'aiter'] + label: '--dsa-decode-backend', + value: '--dsa-decode-backend', + options: [ + 'flashmla_sparse', + 'flashmla_kv', + 'fa3', + 'tilelang', + 'aiter', + 'trtllm' + ] + }, + { + label: '--dsa-topk-backend', + value: '--dsa-topk-backend', + options: ['sgl-kernel', 'torch', 'flashinfer'] + }, + { + label: '--fp8-gemm-backend', + value: '--fp8-gemm-backend', + options: [ + 'auto', + 'deep_gemm', + 'flashinfer_trtllm', + 'flashinfer_cutlass', + 'flashinfer_deepgemm', + 'cutlass', + 'triton', + 'aiter' + ] + }, + { + label: '--fp4-gemm-backend', + value: '--fp4-gemm-backend', + options: [ + 'auto', + 'flashinfer_cudnn', + 'flashinfer_cutlass', + 'flashinfer_trtllm' + ] + }, + { + label: '--disable-flashinfer-autotune', + value: '--disable-flashinfer-autotune', + options: [] }, // Speculative decoding { @@ -759,6 +1000,11 @@ const options: BackendParameter[] = [ value: '--speculative-draft-model-revision', options: [] }, + { + label: '--speculative-draft-load-format', + value: '--speculative-draft-load-format', + options: [] + }, { label: '--speculative-num-steps', value: '--speculative-num-steps', @@ -794,17 +1040,27 @@ const options: BackendParameter[] = [ value: '--speculative-attention-mode', options: ['prefill', 'decode'] }, + { + label: '--speculative-draft-attention-backend', + value: '--speculative-draft-attention-backend', + options: [] + }, + { + label: '--speculative-moe-runner-backend', + value: '--speculative-moe-runner-backend', + options: [] + }, + { + label: '--speculative-moe-a2a-backend', + value: '--speculative-moe-a2a-backend', + options: [] + }, + { + label: '--speculative-draft-model-quantization', + value: '--speculative-draft-model-quantization', + options: [] + }, // Ngram speculative decoding - { - label: '--speculative-ngram-min-match-window-size', - value: '--speculative-ngram-min-match-window-size', - options: [] - }, - { - label: '--speculative-ngram-max-match-window-size', - value: '--speculative-ngram-max-match-window-size', - options: [] - }, { label: '--speculative-ngram-min-bfs-breadth', value: '--speculative-ngram-min-bfs-breadth', @@ -821,8 +1077,8 @@ const options: BackendParameter[] = [ options: ['BFS', 'PROB'] }, { - label: '--speculative-ngram-branch-length', - value: '--speculative-ngram-branch-length', + label: '--speculative-ngram-max-trie-depth', + value: '--speculative-ngram-max-trie-depth', options: [] }, { @@ -830,7 +1086,13 @@ const options: BackendParameter[] = [ value: '--speculative-ngram-capacity', options: [] }, - // Expert parallelism + // Multi-layer Eagle speculative decoding + { + label: '--enable-multi-layer-eagle', + value: '--enable-multi-layer-eagle', + options: [] + }, + // MoE { label: '--expert-parallel-size', value: '--expert-parallel-size', @@ -849,7 +1111,7 @@ const options: BackendParameter[] = [ { label: '--moe-a2a-backend', value: '--moe-a2a-backend', - options: ['none', 'deepep'] + options: ['none', 'deepep', 'mooncake', 'mori', 'nixl', 'ascend_fuseep'] }, { label: '--moe-runner-backend', @@ -860,9 +1122,11 @@ const options: BackendParameter[] = [ 'triton', 'triton_kernel', 'flashinfer_trtllm', + 'flashinfer_trtllm_routed', 'flashinfer_cutlass', 'flashinfer_mxfp4', - 'flashinfer_cutedsl' + 'flashinfer_cutedsl', + 'cutlass' ] }, { @@ -875,11 +1139,21 @@ const options: BackendParameter[] = [ value: '--enable-flashinfer-allreduce-fusion', options: [] }, + { + label: '--enable-aiter-allreduce-fusion', + value: '--enable-aiter-allreduce-fusion', + options: [] + }, { label: '--deepep-mode', value: '--deepep-mode', options: ['normal', 'low_latency', 'auto'] }, + { + label: '--deepep-dispatcher-output-dtype', + value: '--deepep-dispatcher-output-dtype', + options: ['bf16', 'fp8', 'int8', 'nvfp4', 'auto'] + }, { label: '--ep-num-redundant-experts', value: '--ep-num-redundant-experts', @@ -945,6 +1219,31 @@ const options: BackendParameter[] = [ value: '--moe-dense-tp-size', options: [] }, + { + label: '--elastic-ep-backend', + value: '--elastic-ep-backend', + options: ['none', 'mooncake'] + }, + { + label: '--enable-elastic-expert-backup', + value: '--enable-elastic-expert-backup', + options: [] + }, + { + label: '--mooncake-ib-device', + value: '--mooncake-ib-device', + options: [] + }, + { + label: '--enable-deepep-waterfill', + value: '--enable-deepep-waterfill', + options: [] + }, + { + label: '--elastic-ep-rejoin', + value: '--elastic-ep-rejoin', + options: [] + }, // Mamba Cache { label: '--max-mamba-cache-size', @@ -954,17 +1253,21 @@ const options: BackendParameter[] = [ { label: '--mamba-ssm-dtype', value: '--mamba-ssm-dtype', - options: ['float32', 'bfloat16'] + options: ['float32', 'bfloat16', 'float16'] }, { label: '--mamba-full-memory-ratio', value: '--mamba-full-memory-ratio', options: [] }, - // Args for multi-item scoring { - label: '--multi-item-scoring-delimiter', - value: '--multi-item-scoring-delimiter', + label: '--mamba-scheduler-strategy', + value: '--mamba-scheduler-strategy', + options: ['auto', 'no_buffer', 'extra_buffer'] + }, + { + label: '--mamba-track-interval', + value: '--mamba-track-interval', options: [] }, // Hierarchical cache @@ -988,20 +1291,21 @@ const options: BackendParameter[] = [ value: '--hicache-write-policy', options: ['write_back', 'write_through', 'write_through_selective'] }, - { - label: '--radix-eviction-policy', - value: '--radix-eviction-policy', - options: ['lru', 'lfu'] - }, { label: '--hicache-io-backend', value: '--hicache-io-backend', - options: ['direct', 'kernel'] + options: ['direct', 'kernel', 'kernel_ascend'] }, { label: '--hicache-mem-layout', value: '--hicache-mem-layout', - options: ['layer_first', 'page_first', 'page_first_direct'] + options: [ + 'layer_first', + 'page_first', + 'page_first_direct', + 'page_first_kv_split', + 'page_head' + ] }, { label: '--hicache-storage-backend', @@ -1018,43 +1322,18 @@ const options: BackendParameter[] = [ value: '--hicache-storage-backend-extra-config', options: [] }, + // Hierarchical sparse attention + { + label: '--hierarchical-sparse-attention-extra-config', + value: '--hierarchical-sparse-attention-extra-config', + options: [] + }, // LMCache { label: '--enable-lmcache', value: '--enable-lmcache', options: [] }, - // Double Sparsity - { - label: '--enable-double-sparsity', - value: '--enable-double-sparsity', - options: [] - }, - { - label: '--ds-channel-config-path', - value: '--ds-channel-config-path', - options: [] - }, - { - label: '--ds-heavy-channel-num', - value: '--ds-heavy-channel-num', - options: [] - }, - { - label: '--ds-heavy-token-num', - value: '--ds-heavy-token-num', - options: [] - }, - { - label: '--ds-heavy-channel-type', - value: '--ds-heavy-channel-type', - options: [] - }, - { - label: '--ds-sparse-decode-threshold', - value: '--ds-sparse-decode-threshold', - options: [] - }, // Offloading { label: '--cpu-offload-gb', @@ -1081,6 +1360,12 @@ const options: BackendParameter[] = [ value: '--offload-mode', options: [] }, + // Args for multi-item scoring + { + label: '--multi-item-scoring-delimiter', + value: '--multi-item-scoring-delimiter', + options: [] + }, // Optimization/debug options { label: '--disable-radix-cache', @@ -1117,6 +1402,11 @@ const options: BackendParameter[] = [ value: '--enable-cudagraph-gc', options: [] }, + { + label: '--enable-layerwise-nvtx-marker', + value: '--enable-layerwise-nvtx-marker', + options: [] + }, { label: '--enable-nccl-nvls', value: '--enable-nccl-nvls', @@ -1137,6 +1427,11 @@ const options: BackendParameter[] = [ value: '--enable-tokenizer-batch-encode', options: [] }, + { + label: '--disable-tokenizer-batch-decode', + value: '--disable-tokenizer-batch-decode', + options: [] + }, { label: '--disable-outlines-disk-cache', value: '--disable-outlines-disk-cache', @@ -1198,8 +1493,18 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--enable-piecewise-cuda-graph', - value: '--enable-piecewise-cuda-graph', + label: '--enable-torch-compile-debug-mode', + value: '--enable-torch-compile-debug-mode', + options: [] + }, + { + label: '--disable-piecewise-cuda-graph', + value: '--disable-piecewise-cuda-graph', + options: [] + }, + { + label: '--enforce-piecewise-cuda-graph', + value: '--enforce-piecewise-cuda-graph', options: [] }, { @@ -1207,6 +1512,11 @@ const options: BackendParameter[] = [ value: '--piecewise-cuda-graph-tokens', options: [] }, + { + label: '--piecewise-cuda-graph-compiler', + value: '--piecewise-cuda-graph-compiler', + options: ['eager', 'inductor'] + }, { label: '--torch-compile-max-bs', value: '--torch-compile-max-bs', @@ -1267,6 +1577,11 @@ const options: BackendParameter[] = [ value: '--enable-weights-cpu-backup', options: [] }, + { + label: '--enable-draft-weights-cpu-backup', + value: '--enable-draft-weights-cpu-backup', + options: [] + }, { label: '--allow-auto-truncate', value: '--allow-auto-truncate', @@ -1307,6 +1622,11 @@ const options: BackendParameter[] = [ value: '--enable-return-hidden-states', options: [] }, + { + label: '--enable-return-routed-experts', + value: '--enable-return-routed-experts', + options: [] + }, { label: '--scheduler-recv-interval', value: '--scheduler-recv-interval', @@ -1317,22 +1637,42 @@ const options: BackendParameter[] = [ value: '--numa-node', options: [] }, - // Debug tensor dumps { - label: '--debug-tensor-dump-output-folder', - value: '--debug-tensor-dump-output-folder', + label: '--enable-deterministic-inference', + value: '--enable-deterministic-inference', options: [] }, { - label: '--debug-tensor-dump-input-file', - value: '--debug-tensor-dump-input-file', + label: '--rl-on-policy-target', + value: '--rl-on-policy-target', + options: ['fsdp'] + }, + { + label: '--enable-attn-tp-input-scattered', + value: '--enable-attn-tp-input-scattered', options: [] }, { - label: '--debug-tensor-dump-inject', - value: '--debug-tensor-dump-inject', + label: '--enable-dsa-prefill-context-parallel', + value: '--enable-dsa-prefill-context-parallel', options: [] }, + { + label: '--dsa-prefill-cp-mode', + value: '--dsa-prefill-cp-mode', + options: ['in-seq-split', 'round-robin-split'] + }, + { + label: '--enable-fused-qk-norm-rope', + value: '--enable-fused-qk-norm-rope', + options: [] + }, + { + label: '--enable-precise-embedding-interpolation', + value: '--enable-precise-embedding-interpolation', + options: [] + }, + // Dynamic batch tokenizer { label: '--enable-dynamic-batch-tokenizer', value: '--enable-dynamic-batch-tokenizer', @@ -1348,6 +1688,27 @@ const options: BackendParameter[] = [ value: '--dynamic-batch-tokenizer-batch-timeout', options: [] }, + // Debug tensor dumps + { + label: '--debug-tensor-dump-output-folder', + value: '--debug-tensor-dump-output-folder', + options: [] + }, + { + label: '--debug-tensor-dump-layers', + value: '--debug-tensor-dump-layers', + options: [] + }, + { + label: '--debug-tensor-dump-input-file', + value: '--debug-tensor-dump-input-file', + options: [] + }, + { + label: '--debug-tensor-dump-inject', + value: '--debug-tensor-dump-inject', + options: [] + }, // PD disaggregation { label: '--disaggregation-mode', @@ -1364,21 +1725,6 @@ const options: BackendParameter[] = [ value: '--disaggregation-bootstrap-port', options: [] }, - { - label: '--disaggregation-decode-tp', - value: '--disaggregation-decode-tp', - options: [] - }, - { - label: '--disaggregation-decode-dp', - value: '--disaggregation-decode-dp', - options: [] - }, - { - label: '--disaggregation-prefill-pp', - value: '--disaggregation-prefill-pp', - options: [] - }, { label: '--disaggregation-ib-device', value: '--disaggregation-ib-device', @@ -1399,6 +1745,27 @@ const options: BackendParameter[] = [ value: '--disaggregation-decode-polling-interval', options: [] }, + // Encode prefill disaggregation + { + label: '--encoder-only', + value: '--encoder-only', + options: [] + }, + { + label: '--language-only', + value: '--language-only', + options: [] + }, + { + label: '--encoder-transfer-backend', + value: '--encoder-transfer-backend', + options: ['zmq_to_scheduler', 'zmq_to_tokenizer', 'mooncake'] + }, + { + label: '--encoder-urls', + value: '--encoder-urls', + options: [] + }, // Custom weight loader { label: '--custom-weight-loader', @@ -1410,6 +1777,16 @@ const options: BackendParameter[] = [ value: '--weight-loader-disable-mmap', options: [] }, + { + label: '--weight-loader-prefetch-checkpoints', + value: '--weight-loader-prefetch-checkpoints', + options: [] + }, + { + label: '--weight-loader-prefetch-num-threads', + value: '--weight-loader-prefetch-num-threads', + options: [] + }, { label: '--remote-instance-weight-loader-seed-instance-ip', value: '--remote-instance-weight-loader-seed-instance-ip', @@ -1425,6 +1802,16 @@ const options: BackendParameter[] = [ value: '--remote-instance-weight-loader-send-weights-group-ports', options: [] }, + { + label: '--remote-instance-weight-loader-backend', + value: '--remote-instance-weight-loader-backend', + options: ['transfer_engine', 'nccl'] + }, + { + label: '--remote-instance-weight-loader-start-seed-via-transfer-engine', + value: '--remote-instance-weight-loader-start-seed-via-transfer-engine', + options: [] + }, // For PD-Multiplexing { label: '--enable-pdmux', @@ -1441,17 +1828,63 @@ const options: BackendParameter[] = [ value: '--sm-group-num', options: [] }, - // For deterministic inference - { - label: '--enable-deterministic-inference', - value: '--enable-deterministic-inference', - options: [] - }, // Configuration file support { label: '--config', value: '--config', options: [] + }, + // For Multi-Modal + { + label: '--mm-max-concurrent-calls', + value: '--mm-max-concurrent-calls', + options: [] + }, + { + label: '--mm-per-request-timeout', + value: '--mm-per-request-timeout', + options: [] + }, + { + label: '--enable-broadcast-mm-inputs-process', + value: '--enable-broadcast-mm-inputs-process', + options: [] + }, + { + label: '--mm-process-config', + value: '--mm-process-config', + options: [] + }, + { + label: '--mm-enable-dp-encoder', + value: '--mm-enable-dp-encoder', + options: [] + }, + { + label: '--limit-mm-data-per-request', + value: '--limit-mm-data-per-request', + options: [] + }, + { + label: '--enable-mm-global-cache', + value: '--enable-mm-global-cache', + options: [] + }, + // For checkpoint decryption + { + label: '--decrypted-config-file', + value: '--decrypted-config-file', + options: [] + }, + { + label: '--decrypted-draft-config-file', + value: '--decrypted-draft-config-file', + options: [] + }, + { + label: '--enable-prefix-mm-cache', + value: '--enable-prefix-mm-cache', + options: [] } ]; diff --git a/src/pages/llmodels/constants/backend-parameters/vllm.ts b/src/pages/llmodels/constants/backend-parameters/vllm.ts index a5b08dc6..5397f097 100644 --- a/src/pages/llmodels/constants/backend-parameters/vllm.ts +++ b/src/pages/llmodels/constants/backend-parameters/vllm.ts @@ -1,16 +1,234 @@ import { BackendParameter } from './index'; +// Generated from the vLLM `vllm serve` CLI reference: +// https://docs.vllm.ai/en/latest/cli/serve/ +// Grouped by the config sections in the docs. Long-form flags only +// (short aliases and `--no-*` negations are omitted). `options` lists the +// fixed choice values when the flag accepts a restricted set. const options: BackendParameter[] = [ + // General + { + label: '--headless', + value: '--headless', + options: [] + }, + { + label: '--api-server-count', + value: '--api-server-count', + options: [] + }, + { + label: '--config', + value: '--config', + options: [] + }, + { + label: '--grpc', + value: '--grpc', + options: [] + }, + { + label: '--disable-log-stats', + value: '--disable-log-stats', + options: [] + }, + { + label: '--aggregate-engine-logging', + value: '--aggregate-engine-logging', + options: [] + }, + { + label: '--fail-on-environ-validation', + value: '--fail-on-environ-validation', + options: [] + }, + { + label: '--shutdown-timeout', + value: '--shutdown-timeout', + options: [] + }, + { + label: '--gdn-prefill-backend', + value: '--gdn-prefill-backend', + options: ['flashinfer', 'triton', 'cutedsl'] + }, + { + label: '--enable-log-requests', + value: '--enable-log-requests', + options: [] + }, + + // Frontend + { + label: '--lora-modules', + value: '--lora-modules', + options: [] + }, + { + label: '--chat-template', + value: '--chat-template', + options: [] + }, + { + label: '--chat-template-content-format', + value: '--chat-template-content-format', + options: ['auto', 'openai', 'string'] + }, + { + label: '--trust-request-chat-template', + value: '--trust-request-chat-template', + options: [] + }, + { + label: '--default-chat-template-kwargs', + value: '--default-chat-template-kwargs', + options: [] + }, + { + label: '--response-role', + value: '--response-role', + options: [] + }, + { + label: '--return-tokens-as-token-ids', + value: '--return-tokens-as-token-ids', + options: [] + }, + { + label: '--enable-auto-tool-choice', + value: '--enable-auto-tool-choice', + options: [] + }, + { + label: '--exclude-tools-when-tool-choice-none', + value: '--exclude-tools-when-tool-choice-none', + options: [] + }, + { + label: '--tool-call-parser', + value: '--tool-call-parser', + options: [] + }, + { + label: '--tool-parser-plugin', + value: '--tool-parser-plugin', + options: [] + }, + { + label: '--tool-server', + value: '--tool-server', + options: [] + }, + { + label: '--log-config-file', + value: '--log-config-file', + options: [] + }, + { + label: '--max-log-len', + value: '--max-log-len', + options: [] + }, + { + label: '--enable-prompt-tokens-details', + value: '--enable-prompt-tokens-details', + options: [] + }, + { + label: '--enable-server-load-tracking', + value: '--enable-server-load-tracking', + options: [] + }, + { + label: '--enable-force-include-usage', + value: '--enable-force-include-usage', + options: [] + }, + { + label: '--enable-tokenizer-info-endpoint', + value: '--enable-tokenizer-info-endpoint', + options: [] + }, + { + label: '--enable-log-outputs', + value: '--enable-log-outputs', + options: [] + }, + { + label: '--enable-log-deltas', + value: '--enable-log-deltas', + options: [] + }, + { + label: '--log-error-stack', + value: '--log-error-stack', + options: [] + }, + { + label: '--tokens-only', + value: '--tokens-only', + options: [] + }, + { + label: '--fingerprint-mode', + value: '--fingerprint-mode', + options: ['custom', 'full', 'hash', 'none'] + }, + { + label: '--fingerprint-value', + value: '--fingerprint-value', + options: [] + }, + { + label: '--host', + value: '--host', + options: [] + }, + { + label: '--port', + value: '--port', + options: [] + }, + { + label: '--data-parallel-supervisor-port', + value: '--data-parallel-supervisor-port', + options: [] + }, + { + label: '--dp-supervisor-probe-interval-s', + value: '--dp-supervisor-probe-interval-s', + options: [] + }, + { + label: '--dp-supervisor-probe-timeout-s', + value: '--dp-supervisor-probe-timeout-s', + options: [] + }, + { + label: '--dp-supervisor-probe-failure-threshold', + value: '--dp-supervisor-probe-failure-threshold', + options: [] + }, + { + label: '--uds', + value: '--uds', + options: [] + }, { label: '--uvicorn-log-level', value: '--uvicorn-log-level', - options: ['debug', 'info', 'warning', 'error', 'critical', 'trace'] + options: ['critical', 'debug', 'error', 'info', 'trace', 'warning'] }, { label: '--disable-uvicorn-access-log', value: '--disable-uvicorn-access-log', options: [] }, + { + label: '--disable-access-log-for-endpoints', + value: '--disable-access-log-for-endpoints', + options: [] + }, { label: '--allow-credentials', value: '--allow-credentials', @@ -36,26 +254,6 @@ const options: BackendParameter[] = [ value: '--api-key', options: [] }, - { - label: '--lora-modules', - value: '--lora-modules', - options: [] - }, - { - label: '--prompt-adapters', - value: '--prompt-adapters', - options: [] - }, - { - label: '--chat-template', - value: '--chat-template', - options: [] - }, - { - label: '--response-role', - value: '--response-role', - options: [] - }, { label: '--ssl-keyfile', value: '--ssl-keyfile', @@ -67,8 +265,8 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--data-parallel-size-local', - value: '--data-parallel-size-local', + label: '--ssl-ca-certs', + value: '--ssl-ca-certs', options: [] }, { @@ -77,13 +275,13 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--ssl-ca-certs', - value: '--ssl-ca-certs', + label: '--ssl-cert-reqs', + value: '--ssl-cert-reqs', options: [] }, { - label: '--ssl-cert-reqs', - value: '--ssl-cert-reqs', + label: '--ssl-ciphers', + value: '--ssl-ciphers', options: [] }, { @@ -96,93 +294,91 @@ const options: BackendParameter[] = [ value: '--middleware', options: [] }, - { - label: '--return-tokens-as-token-ids', - value: '--return-tokens-as-token-ids', - options: [] - }, - { - label: '--disable-frontend-multiprocessing', - value: '--disable-frontend-multiprocessing', - options: [] - }, { label: '--enable-request-id-headers', value: '--enable-request-id-headers', options: [] }, { - label: '--enable-auto-tool-choice', - value: '--enable-auto-tool-choice', + label: '--disable-fastapi-docs', + value: '--disable-fastapi-docs', options: [] }, { - label: '--tool-call-parser', - value: '--tool-call-parser', - options: [ - 'phi4_mini_json', - 'llama3_json', - 'llama4_json', - 'pythonic', - 'jamba', - 'xlam', - 'llama4_pythonic', - 'hunyuan_a13b', - 'mistral', - 'deepseek_v3', - 'kimi_k2', - 'step3', - 'qwen3_coder', - 'hermes', - 'glm45', - 'granite', - 'minimax', - 'granite-20b-fc', - 'internlm' - ] - }, - { - label: '--task', - value: '--task', - options: [ - 'auto', - 'generate', - 'embedding', - 'embed', - 'classify', - 'score', - 'reward', - 'transcription' - ] - }, - { - label: '--allowed-local-media-path', - value: '--allowed-local-media-path', + label: '--h11-max-incomplete-event-size', + value: '--h11-max-incomplete-event-size', options: [] }, { - label: '--tool-parser-plugin', - value: '--tool-parser-plugin', + label: '--h11-max-header-count', + value: '--h11-max-header-count', options: [] }, + { + label: '--enable-offline-docs', + value: '--enable-offline-docs', + options: [] + }, + { + label: '--enable-flash-late-interaction', + value: '--enable-flash-late-interaction', + options: [] + }, + + // ModelConfig { label: '--model', value: '--model', options: [] }, + { + label: '--runner', + value: '--runner', + options: ['auto', 'draft', 'generate', 'pooling'] + }, + { + label: '--convert', + value: '--convert', + options: ['auto', 'classify', 'embed', 'none'] + }, { label: '--tokenizer', value: '--tokenizer', options: [] }, + { + label: '--tokenizer-mode', + value: '--tokenizer-mode', + options: ['auto', 'deepseek_v32', 'deepseek_v4', 'hf', 'mistral', 'slow'] + }, + { + label: '--trust-remote-code', + value: '--trust-remote-code', + options: [] + }, + { + label: '--dtype', + value: '--dtype', + options: ['auto', 'bfloat16', 'float', 'float16', 'float32', 'half'] + }, + { + label: '--seed', + value: '--seed', + options: [] + }, { label: '--hf-config-path', value: '--hf-config-path', options: [] }, { - label: '--skip-tokenizer-init', - value: '--skip-tokenizer-init', + label: '--allowed-local-media-path', + value: '--allowed-local-media-path', + options: [] + }, + { + label: '--allowed-media-domains', + value: '--allowed-media-domains', options: [] }, { @@ -201,66 +397,59 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--tokenizer-mode', - value: '--tokenizer-mode', - options: ['auto', 'slow', 'mistral', 'custom'] - }, - { - label: '--trust-remote-code', - value: '--trust-remote-code', + label: '--max-model-len', + value: '--max-model-len', options: [] }, { - label: '--download-dir', - value: '--download-dir', + label: '--quantization', + value: '--quantization', options: [] }, { - label: '--load-format', - value: '--load-format', + label: '--quantization-config', + value: '--quantization-config', + options: [] + }, + { + label: '--allow-deprecated-quantization', + value: '--allow-deprecated-quantization', + options: [] + }, + { + label: '--enforce-eager', + value: '--enforce-eager', + options: [] + }, + { + label: '--enable-return-routed-experts', + value: '--enable-return-routed-experts', + options: [] + }, + { + label: '--max-logprobs', + value: '--max-logprobs', + options: [] + }, + { + label: '--logprobs-mode', + value: '--logprobs-mode', options: [ - 'auto', - 'pt', - 'safetensors', - 'npcache', - 'dummy', - 'tensorizer', - 'sharded_state', - 'gguf', - 'bitsandbytes', - 'mistral', - 'runai_streamer', - 'fastsafetensors' + 'processed_logits', + 'processed_logprobs', + 'raw_logits', + 'raw_logprobs' ] }, { - label: '--chat-template-content-format', - value: '--chat-template-content-format', - options: ['auto', 'string', 'openai'] - }, - { - label: '--enable-reasoning', - value: '--enable-reasoning', + label: '--use-fp64-gumbel', + value: '--use-fp64-gumbel', options: [] }, { - label: '--reasoning-parser', - value: '--reasoning-parser', - options: [ - 'deepseek_r1', - 'glm45', - 'GptOss', - 'granite', - 'hunyuan_a13b', - 'mistral', - 'qwen3', - 'step3' - ] - }, - { - label: '--config-format', - value: '--config-format', - options: ['auto', 'hf', 'mistral'] + label: '--disable-sliding-window', + value: '--disable-sliding-window', + options: [] }, { label: '--disable-cascade-attn', @@ -268,59 +457,267 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--dtype', - value: '--dtype', - options: ['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'] - }, - { - label: '--kv-cache-dtype', - value: '--kv-cache-dtype', - options: ['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'] - }, - { - label: '--disable-chunked-mm-input', - value: '--disable-chunked-mm-input', + label: '--skip-tokenizer-init', + value: '--skip-tokenizer-init', options: [] }, { - label: 'DISABLE_CHUNKED_MM_INPUT', - value: 'DISABLE_CHUNKED_MM_INPUT', + label: '--enable-prompt-embeds', + value: '--enable-prompt-embeds', options: [] }, { - label: '--quantization-param-path', - value: '--quantization-param-path', + label: '--served-model-name', + value: '--served-model-name', options: [] }, { - label: '--max-model-len', - value: '--max-model-len', + label: '--config-format', + value: '--config-format', + options: ['auto', 'hf', 'mistral'] + }, + { + label: '--hf-token', + value: '--hf-token', options: [] }, { - label: '--guided-decoding-backend', - value: '--guided-decoding-backend', - options: ['outlines', 'lm-format-enforcer', 'xgrammar'] + label: '--hf-overrides', + value: '--hf-overrides', + options: [] }, { - label: '--logits-processor-pattern', - value: '--logits-processor-pattern', + label: '--pooler-config', + value: '--pooler-config', + options: [] + }, + { + label: '--generation-config', + value: '--generation-config', + options: [] + }, + { + label: '--override-generation-config', + value: '--override-generation-config', + options: [] + }, + { + label: '--enable-sleep-mode', + value: '--enable-sleep-mode', + options: [] + }, + { + label: '--enable-cumem-allocator', + value: '--enable-cumem-allocator', options: [] }, - { label: '--model-impl', value: '--model-impl', - options: ['auto', 'vllm', 'transformers'] + options: ['auto', 'terratorch', 'transformers', 'vllm'] }, + { + label: '--override-attention-dtype', + value: '--override-attention-dtype', + options: [] + }, + { + label: '--logits-processors', + value: '--logits-processors', + options: [] + }, + { + label: '--io-processor-plugin', + value: '--io-processor-plugin', + options: [] + }, + { + label: '--renderer-num-workers', + value: '--renderer-num-workers', + options: [] + }, + + // LoadConfig + { + label: '--load-format', + value: '--load-format', + options: [ + 'auto', + 'pt', + 'safetensors', + 'instanttensor', + 'npcache', + 'dummy', + 'tensorizer', + 'runai_streamer', + 'runai_streamer_sharded', + 'bitsandbytes', + 'sharded_state', + 'mistral', + 'modelexpress' + ] + }, + { + label: '--download-dir', + value: '--download-dir', + options: [] + }, + { + label: '--safetensors-load-strategy', + value: '--safetensors-load-strategy', + options: ['eager', 'lazy', 'prefetch', 'torchao', 'None'] + }, + { + label: '--safetensors-prefetch-num-threads', + value: '--safetensors-prefetch-num-threads', + options: [] + }, + { + label: '--safetensors-prefetch-block-size', + value: '--safetensors-prefetch-block-size', + options: [] + }, + { + label: '--model-loader-extra-config', + value: '--model-loader-extra-config', + options: [] + }, + { + label: '--ignore-patterns', + value: '--ignore-patterns', + options: [] + }, + { + label: '--use-tqdm-on-load', + value: '--use-tqdm-on-load', + options: [] + }, + { + label: '--pt-load-map-location', + value: '--pt-load-map-location', + options: [] + }, + + // AttentionConfig + { + label: '--attention-backend', + value: '--attention-backend', + options: [] + }, + + // MambaConfig + { + label: '--mamba-backend', + value: '--mamba-backend', + options: [] + }, + { + label: '--enable-mamba-cache-stochastic-rounding', + value: '--enable-mamba-cache-stochastic-rounding', + options: [] + }, + { + label: '--mamba-cache-philox-rounds', + value: '--mamba-cache-philox-rounds', + options: [] + }, + + // StructuredOutputsConfig + { + label: '--reasoning-parser', + value: '--reasoning-parser', + options: [] + }, + { + label: '--reasoning-parser-plugin', + value: '--reasoning-parser-plugin', + options: [] + }, + + // ParallelConfig { label: '--distributed-executor-backend', value: '--distributed-executor-backend', - options: ['ray', 'mp', 'uni', 'external_launcher'] + options: ['external_launcher', 'mp', 'ray', 'uni'] }, { - label: '--worker-use-ray', - value: '--worker-use-ray', + label: '--pipeline-parallel-size', + value: '--pipeline-parallel-size', + options: [] + }, + { + label: '--master-addr', + value: '--master-addr', + options: [] + }, + { + label: '--master-port', + value: '--master-port', + options: [] + }, + { + label: '--nnodes', + value: '--nnodes', + options: [] + }, + { + label: '--node-rank', + value: '--node-rank', + options: [] + }, + { + label: '--distributed-timeout-seconds', + value: '--distributed-timeout-seconds', + options: [] + }, + { + label: '--cpu-distributed-timeout-seconds', + value: '--cpu-distributed-timeout-seconds', + options: [] + }, + { + label: '--numa-bind', + value: '--numa-bind', + options: [] + }, + { + label: '--numa-bind-nodes', + value: '--numa-bind-nodes', + options: [] + }, + { + label: '--numa-bind-cpus', + value: '--numa-bind-cpus', + options: [] + }, + { + label: '--tensor-parallel-size', + value: '--tensor-parallel-size', + options: [] + }, + { + label: '--decode-context-parallel-size', + value: '--decode-context-parallel-size', + options: [] + }, + { + label: '--dcp-comm-backend', + value: '--dcp-comm-backend', + options: ['a2a', 'ag_rs'] + }, + { + label: '--dcp-kv-cache-interleave-size', + value: '--dcp-kv-cache-interleave-size', + options: [] + }, + { + label: '--cp-kv-cache-interleave-size', + value: '--cp-kv-cache-interleave-size', + options: [] + }, + { + label: '--prefill-context-parallel-size', + value: '--prefill-context-parallel-size', options: [] }, { @@ -328,21 +725,124 @@ const options: BackendParameter[] = [ value: '--data-parallel-size', options: [] }, + { + label: '--data-parallel-rank', + value: '--data-parallel-rank', + options: [] + }, + { + label: '--data-parallel-start-rank', + value: '--data-parallel-start-rank', + options: [] + }, + { + label: '--data-parallel-size-local', + value: '--data-parallel-size-local', + options: [] + }, + { + label: '--data-parallel-address', + value: '--data-parallel-address', + options: [] + }, + { + label: '--data-parallel-rpc-port', + value: '--data-parallel-rpc-port', + options: [] + }, + { + label: '--data-parallel-backend', + value: '--data-parallel-backend', + options: ['mp', 'ray'] + }, + { + label: '--data-parallel-hybrid-lb', + value: '--data-parallel-hybrid-lb', + options: [] + }, + { + label: '--data-parallel-external-lb', + value: '--data-parallel-external-lb', + options: [] + }, + { + label: '--data-parallel-multi-port-external-lb', + value: '--data-parallel-multi-port-external-lb', + options: [] + }, { label: '--enable-expert-parallel', value: '--enable-expert-parallel', options: [] }, { - label: '--pipeline-parallel-size', - value: '--pipeline-parallel-size', + label: '--enable-ep-weight-filter', + value: '--enable-ep-weight-filter', options: [] }, { - label: '--tensor-parallel-size', - value: '--tensor-parallel-size', + label: '--all2all-backend', + value: '--all2all-backend', + options: [ + 'allgather_reducescatter', + 'deepep_high_throughput', + 'deepep_low_latency', + 'deepep_v2', + 'flashinfer_all2allv', + 'flashinfer_nvlink_one_sided', + 'flashinfer_nvlink_two_sided', + 'mori_high_throughput', + 'mori_low_latency', + 'naive', + 'nixl_ep', + 'pplx' + ] + }, + { + label: '--enable-dbo', + value: '--enable-dbo', options: [] }, + { + label: '--ubatch-size', + value: '--ubatch-size', + options: [] + }, + { + label: '--enable-elastic-ep', + value: '--enable-elastic-ep', + options: [] + }, + { + label: '--dbo-decode-token-threshold', + value: '--dbo-decode-token-threshold', + options: [] + }, + { + label: '--dbo-prefill-token-threshold', + value: '--dbo-prefill-token-threshold', + options: [] + }, + { + label: '--disable-nccl-for-dp-synchronization', + value: '--disable-nccl-for-dp-synchronization', + options: [] + }, + { + label: '--enable-eplb', + value: '--enable-eplb', + options: [] + }, + { + label: '--eplb-config', + value: '--eplb-config', + options: [] + }, + { + label: '--expert-placement-strategy', + value: '--expert-placement-strategy', + options: ['linear', 'round_robin'] + }, { label: '--max-parallel-loading-workers', value: '--max-parallel-loading-workers', @@ -353,10 +853,63 @@ const options: BackendParameter[] = [ value: '--ray-workers-use-nsight', options: [] }, + { + label: '--disable-custom-all-reduce', + value: '--disable-custom-all-reduce', + options: [] + }, + { + label: '--worker-cls', + value: '--worker-cls', + options: [] + }, + { + label: '--worker-extension-cls', + value: '--worker-extension-cls', + options: [] + }, + + // CacheConfig { label: '--block-size', value: '--block-size', - options: ['8', '16', '32', '64', '128'] + options: ['1', '8', '16', '32', '64', '128'] + }, + { + label: '--gpu-memory-utilization', + value: '--gpu-memory-utilization', + options: [] + }, + { + label: '--kv-cache-memory-bytes', + value: '--kv-cache-memory-bytes', + options: [] + }, + { + label: '--kv-cache-dtype', + value: '--kv-cache-dtype', + options: [ + 'auto', + 'bfloat16', + 'float16', + 'fp8', + 'fp8_ds_mla', + 'fp8_e4m3', + 'fp8_e5m2', + 'fp8_inc', + 'fp8_per_token_head', + 'int8_per_token_head', + 'nvfp4', + 'turboquant_3bit_nc', + 'turboquant_4bit_nc', + 'turboquant_k3v4_nc', + 'turboquant_k8v4' + ] + }, + { + label: '--num-gpu-blocks-override', + value: '--num-gpu-blocks-override', + options: [] }, { label: '--enable-prefix-caching', @@ -366,182 +919,95 @@ const options: BackendParameter[] = [ { label: '--prefix-caching-hash-algo', value: '--prefix-caching-hash-algo', - options: ['builtin', 'sha256'] + options: ['sha256', 'sha256_cbor', 'xxhash', 'xxhash_cbor'] }, { - label: '--disable-sliding-window', - value: '--disable-sliding-window', + label: '--calculate-kv-scales', + value: '--calculate-kv-scales', options: [] }, { - label: '--use-v2-block-manager', - value: '--use-v2-block-manager', + label: '--kv-cache-dtype-skip-layers', + value: '--kv-cache-dtype-skip-layers', options: [] }, { - label: '--num-lookahead-slots', - value: '--num-lookahead-slots', + label: '--kv-sharing-fast-prefill', + value: '--kv-sharing-fast-prefill', options: [] }, { - label: '--seed', - value: '--seed', + label: '--mamba-cache-dtype', + value: '--mamba-cache-dtype', + options: ['auto', 'bfloat16', 'float16', 'float32'] + }, + { + label: '--mamba-ssm-cache-dtype', + value: '--mamba-ssm-cache-dtype', + options: ['auto', 'bfloat16', 'float16', 'float32'] + }, + { + label: '--mamba-block-size', + value: '--mamba-block-size', options: [] }, { - label: '--swap-space', - value: '--swap-space', + label: '--mamba-cache-mode', + value: '--mamba-cache-mode', + options: ['align', 'all', 'none'] + }, + { + label: '--kv-offloading-size', + value: '--kv-offloading-size', options: [] }, + { + label: '--kv-offloading-backend', + value: '--kv-offloading-backend', + options: ['lmcache', 'native'] + }, + + // OffloadConfig + { + label: '--offload-backend', + value: '--offload-backend', + options: ['auto', 'prefetch', 'uva'] + }, { label: '--cpu-offload-gb', value: '--cpu-offload-gb', options: [] }, { - label: '--gpu-memory-utilization', - value: '--gpu-memory-utilization', + label: '--cpu-offload-params', + value: '--cpu-offload-params', options: [] }, { - label: '--num-gpu-blocks-override', - value: '--num-gpu-blocks-override', + label: '--offload-group-size', + value: '--offload-group-size', options: [] }, { - label: '--max-num-batched-tokens', - value: '--max-num-batched-tokens', + label: '--offload-num-in-group', + value: '--offload-num-in-group', options: [] }, { - label: '--max-num-partial-prefills', - value: '--max-num-partial-prefills', + label: '--offload-prefetch-step', + value: '--offload-prefetch-step', + options: [] + }, + { + label: '--offload-params', + value: '--offload-params', options: [] }, + // MultiModalConfig { - label: '--max-long-partial-prefills', - value: '--max-long-partial-prefills', - options: [] - }, - - { - label: '--long-prefill-token-threshold', - value: '--long-prefill-token-threshold', - options: [] - }, - { - label: '--max-num-seqs', - value: '--max-num-seqs', - options: [] - }, - { - label: '--max-logprobs', - value: '--max-logprobs', - options: [] - }, - { - label: '--disable-log-stats', - value: '--disable-log-stats', - options: [] - }, - { - label: '--hf-overrides', - value: '--hf-overrides', - options: [] - }, - { - label: '--hf-token', - value: '--hf-token', - options: [] - }, - - { - label: 'HF_TOKEN', - value: 'HF_TOKEN', - options: [] - }, - { - label: '--disable-mm-preprocessor-cache', - value: '--disable-mm-preprocessor-cache', - options: [] - }, - { - label: '--quantization', - value: '--quantization', - options: [ - 'aqlm', - 'awq', - 'deepspeedfp', - 'tpu_int8', - 'fp8', - 'ptpc_fp8', - 'fbgemm_fp8', - 'modelopt', - 'nvfp4', - 'marlin', - 'gguf', - 'gptq_marlin_24', - 'gptq_marlin', - 'awq_marlin', - 'gptq', - 'compressed-tensors', - 'bitsandbytes', - 'qqq', - 'hqq', - 'experts_int8', - 'neuron_quant', - 'ipex', - 'quark', - 'moe_wna16', - 'torchao', - 'ascend', - 'None' - ] - }, - { - label: '--enable-lora-bias', - value: '--enable-lora-bias', - options: [] - }, - { - label: '--rope-theta', - value: '--rope-theta', - options: [] - }, - { - label: '--enforce-eager', - value: '--enforce-eager', - options: [] - }, - { - label: '--max-context-len-to-capture', - value: '--max-context-len-to-capture', - options: [] - }, - { - label: '--max-seq-len-to-capture', - value: '--max-seq-len-to-capture', - options: [] - }, - { - label: '--disable-custom-all-reduce', - value: '--disable-custom-all-reduce', - options: [] - }, - { - label: '--tokenizer-pool-size', - value: '--tokenizer-pool-size', - options: [] - }, - { - label: '--tokenizer-pool-type', - value: '--tokenizer-pool-type', - options: [] - }, - { - label: '--tokenizer-pool-extra-config', - value: '--tokenizer-pool-extra-config', + label: '--language-model-only', + value: '--language-model-only', options: [] }, { @@ -549,11 +1015,93 @@ const options: BackendParameter[] = [ value: '--limit-mm-per-prompt', options: [] }, + { + label: '--enable-mm-embeds', + value: '--enable-mm-embeds', + options: [] + }, + { + label: '--media-io-kwargs', + value: '--media-io-kwargs', + options: [] + }, { label: '--mm-processor-kwargs', value: '--mm-processor-kwargs', options: [] }, + { + label: '--mm-processor-cache-gb', + value: '--mm-processor-cache-gb', + options: [] + }, + { + label: '--mm-processor-cache-type', + value: '--mm-processor-cache-type', + options: ['lru', 'shm'] + }, + { + label: '--mm-shm-cache-max-object-size-mb', + value: '--mm-shm-cache-max-object-size-mb', + options: [] + }, + { + label: '--mm-encoder-only', + value: '--mm-encoder-only', + options: [] + }, + { + label: '--mm-encoder-tp-mode', + value: '--mm-encoder-tp-mode', + options: ['data', 'weights'] + }, + { + label: '--mm-encoder-attn-backend', + value: '--mm-encoder-attn-backend', + options: [] + }, + { + label: '--mm-encoder-attn-dtype', + value: '--mm-encoder-attn-dtype', + options: ['fp8', 'None'] + }, + { + label: '--mm-encoder-fp8-scale-path', + value: '--mm-encoder-fp8-scale-path', + options: [] + }, + { + label: '--mm-encoder-fp8-scale-save-path', + value: '--mm-encoder-fp8-scale-save-path', + options: [] + }, + { + label: '--mm-encoder-fp8-scale-save-margin', + value: '--mm-encoder-fp8-scale-save-margin', + options: [] + }, + { + label: '--interleave-mm-strings', + value: '--interleave-mm-strings', + options: [] + }, + { + label: '--skip-mm-profiling', + value: '--skip-mm-profiling', + options: [] + }, + { + label: '--video-pruning-rate', + value: '--video-pruning-rate', + options: [] + }, + { + label: '--mm-tensor-ipc', + value: '--mm-tensor-ipc', + options: ['direct_rpc', 'torch_shm'] + }, + + // LoRAConfig { label: '--enable-lora', value: '--enable-lora', @@ -567,21 +1115,16 @@ const options: BackendParameter[] = [ { label: '--max-lora-rank', value: '--max-lora-rank', - options: [] - }, - { - label: '--lora-extra-vocab-size', - value: '--lora-extra-vocab-size', - options: [] + options: ['1', '8', '16', '32', '64', '128', '256', '320', '512'] }, { label: '--lora-dtype', value: '--lora-dtype', - options: ['auto', 'float16', 'bfloat16'] + options: [] }, { - label: '--long-lora-scaling-factors', - value: '--long-lora-scaling-factors', + label: '--enable-tower-connector-lora', + value: '--enable-tower-connector-lora', options: [] }, { @@ -595,90 +1138,32 @@ const options: BackendParameter[] = [ options: [] }, { - label: '--enable-prompt-adapter', - value: '--enable-prompt-adapter', + label: '--lora-target-modules', + value: '--lora-target-modules', options: [] }, { - label: '--max-prompt-adapters', - value: '--max-prompt-adapters', + label: '--default-mm-loras', + value: '--default-mm-loras', options: [] }, { - label: '--max-prompt-adapter-token', - value: '--max-prompt-adapter-token', + label: '--specialize-active-lora', + value: '--specialize-active-lora', options: [] }, { - label: '--device', - value: '--device', - options: ['auto', 'cuda', 'neuron', 'cpu', 'tpu', 'xpu', 'hpu'] - }, - { - label: '--speculative-config', - value: '--speculative-config', - options: [] - }, - { - label: '--num-scheduler-steps', - value: '--num-scheduler-steps', - options: [] - }, - { - label: '--multi-step-stream-outputs', - value: '--multi-step-stream-outputs', - options: [] - }, - { - label: '--scheduler-delay-factor', - value: '--scheduler-delay-factor', - options: [] - }, - { - label: '--enable-chunked-prefill', - value: '--enable-chunked-prefill', - options: [] - }, - { - label: '--model-loader-extra-config', - value: '--model-loader-extra-config', - options: [] - }, - { - label: '--scheduler-cls', - value: '--scheduler-cls', - options: [] - }, - { - label: '--use-tqdm-on-load', - value: '--use-tqdm-on-load', - options: [] - }, - { - label: '--ignore-patterns', - value: '--ignore-patterns', + label: '--enable-mixed-moe-lora-format', + value: '--enable-mixed-moe-lora-format', options: [] }, + + // ObservabilityConfig { label: '--show-hidden-metrics-for-version', value: '--show-hidden-metrics-for-version', options: [] }, - { - label: '--preemption-mode', - value: '--preemption-mode', - options: [] - }, - { - label: '--served-model-name', - value: '--served-model-name', - options: [] - }, - { - label: '--qlora-adapter-name-or-path', - value: '--qlora-adapter-name-or-path', - options: [] - }, { label: '--otlp-traces-endpoint', value: '--otlp-traces-endpoint', @@ -687,57 +1172,68 @@ const options: BackendParameter[] = [ { label: '--collect-detailed-traces', value: '--collect-detailed-traces', + options: ['all', 'model', 'worker', 'None'] + }, + { + label: '--kv-cache-metrics', + value: '--kv-cache-metrics', options: [] }, { - label: '--disable-async-output-proc', - value: '--disable-async-output-proc', + label: '--kv-cache-metrics-sample', + value: '--kv-cache-metrics-sample', options: [] }, { - label: '--override-neuron-config', - value: '--override-neuron-config', + label: '--cudagraph-metrics', + value: '--cudagraph-metrics', options: [] }, { - label: '--compilation-config', - value: '--compilation-config', + label: '--enable-layerwise-nvtx-tracing', + value: '--enable-layerwise-nvtx-tracing', options: [] }, { - label: '--override-pooler-config', - value: '--override-pooler-config', + label: '--enable-mfu-metrics', + value: '--enable-mfu-metrics', options: [] }, { - label: '--kv-transfer-config', - value: '--kv-transfer-config', + label: '--enable-logging-iteration-details', + value: '--enable-logging-iteration-details', options: [] }, { - label: '--worker-cls', - value: '--worker-cls', - options: [] - }, - { - label: '--override-generation-config', - value: '--override-generation-config', - options: [] - }, - { - label: '--enable-sleep-mode', - value: '--enable-sleep-mode', + label: '--jit-monitor-verbose', + value: '--jit-monitor-verbose', options: [] }, + // SchedulerConfig { - label: '--calculate-kv-scales', - value: '--calculate-kv-scales', + label: '--max-num-batched-tokens', + value: '--max-num-batched-tokens', options: [] }, { - label: '--generation-config', - value: '--generation-config', + label: '--max-num-seqs', + value: '--max-num-seqs', + options: [] + }, + { + label: '--max-num-partial-prefills', + value: '--max-num-partial-prefills', + options: [] + }, + { + label: '--max-long-partial-prefills', + value: '--max-long-partial-prefills', + options: [] + }, + { + label: '--long-prefill-token-threshold', + value: '--long-prefill-token-threshold', options: [] }, { @@ -746,33 +1242,199 @@ const options: BackendParameter[] = [ options: ['fcfs', 'priority'] }, { - label: '--disable-log-requests', - value: '--disable-log-requests', + label: '--enable-chunked-prefill', + value: '--enable-chunked-prefill', options: [] }, { - label: '--additional-config', - value: '--additional-config', + label: '--disable-chunked-mm-input', + value: '--disable-chunked-mm-input', options: [] }, { - label: '--max-log-len', - value: '--max-log-len', + label: '--scheduler-cls', + value: '--scheduler-cls', options: [] }, { - label: '--disable-fastapi-docs', - value: '--disable-fastapi-docs', + label: '--scheduler-reserve-full-isl', + value: '--scheduler-reserve-full-isl', options: [] }, { - label: '--enable-prompt-tokens-details', - value: '--enable-prompt-tokens-details', + label: '--watermark', + value: '--watermark', options: [] }, { - label: '--enable-server-load-tracking', - value: '--enable-server-load-tracking', + label: '--prefill-schedule-interval', + value: '--prefill-schedule-interval', + options: [] + }, + { + label: '--disable-hybrid-kv-cache-manager', + value: '--disable-hybrid-kv-cache-manager', + options: [] + }, + { + label: '--async-scheduling', + value: '--async-scheduling', + options: [] + }, + { + label: '--stream-interval', + value: '--stream-interval', + options: [] + }, + + // CompilationConfig + { + label: '--cudagraph-capture-sizes', + value: '--cudagraph-capture-sizes', + options: [] + }, + { + label: '--max-cudagraph-capture-size', + value: '--max-cudagraph-capture-size', + options: [] + }, + + // KernelConfig + { + label: '--ir-op-priority', + value: '--ir-op-priority', + options: [] + }, + { + label: '--enable-flashinfer-autotune', + value: '--enable-flashinfer-autotune', + options: [] + }, + { + label: '--moe-backend', + value: '--moe-backend', + options: [ + 'aiter', + 'auto', + 'cutlass', + 'deep_gemm', + 'deep_gemm_mega_moe', + 'emulation', + 'flashinfer_b12x', + 'flashinfer_cutedsl', + 'flashinfer_cutlass', + 'flashinfer_trtllm', + 'flydsl', + 'humming', + 'marlin', + 'triton', + 'triton_unfused' + ] + }, + { + label: '--linear-backend', + value: '--linear-backend', + options: [ + 'aiter', + 'auto', + 'conch', + 'cutlass', + 'deep_gemm', + 'emulation', + 'exllama', + 'fbgemm', + 'flashinfer_b12x', + 'flashinfer_cudnn', + 'flashinfer_cutlass', + 'flashinfer_trtllm', + 'machete', + 'marlin', + 'torch', + 'triton' + ] + }, + + // VllmConfig + { + label: '--speculative-config', + value: '--speculative-config', + options: [] + }, + { + label: '--spec-method', + value: '--spec-method', + options: [ + 'custom_class', + 'deepseek_mtp', + 'dflash', + 'draft_model', + 'eagle', + 'eagle3', + 'ernie_mtp', + 'exaone4_5_mtp', + 'exaone_moe_mtp', + 'extract_hidden_states', + 'gemma4_mtp', + 'glm4_moe_lite_mtp', + 'glm4_moe_mtp', + 'glm_ocr_mtp', + 'hy_v3_mtp', + 'longcat_flash_mtp', + 'medusa', + 'mimo_mtp', + 'mimo_v2_mtp', + 'minimax_m3_mtp', + 'mlp_speculator', + 'mtp', + 'nemotron_h_mtp', + 'ngram', + 'ngram_gpu', + 'pangu_ultra_moe_mtp', + 'qwen3_5_mtp', + 'qwen3_next_mtp', + 'step3p5_mtp', + 'suffix', + 'None' + ] + }, + { + label: '--spec-model', + value: '--spec-model', + options: [] + }, + { + label: '--spec-tokens', + value: '--spec-tokens', + options: [] + }, + { + label: '--diffusion-config', + value: '--diffusion-config', + options: [] + }, + { + label: '--kv-transfer-config', + value: '--kv-transfer-config', + options: [] + }, + { + label: '--kv-events-config', + value: '--kv-events-config', + options: [] + }, + { + label: '--ec-transfer-config', + value: '--ec-transfer-config', + options: [] + }, + { + label: '--compilation-config', + value: '--compilation-config', + options: [] + }, + { + label: '--attention-config', + value: '--attention-config', options: [] } ];