import { BackendParameter } from './index'; // Generated from the SGLang `launch_server` server arguments reference: // https://docs.sglang.io/docs/advanced_features/server_arguments // Grouped by the doc's sections. `options` lists the fixed choice values // when the flag accepts a restricted set (sourced from the doc's Options column). // Internal/experimental sections (Ktransformers, Diffusion LLM, Forward hooks, // msProbe dump) and deprecated arguments are omitted. const options: BackendParameter[] = [ // Model and tokenizer { label: '--model-path', value: '--model-path', options: [] }, { label: '--model', value: '--model', options: [] }, { label: '--tokenizer-path', value: '--tokenizer-path', options: [] }, { label: '--tokenizer-mode', value: '--tokenizer-mode', options: ['auto', 'slow'] }, { label: '--tokenizer-worker-num', value: '--tokenizer-worker-num', options: [] }, { label: '--skip-tokenizer-init', value: '--skip-tokenizer-init', options: [] }, { label: '--load-format', value: '--load-format', options: [ 'auto', 'pt', 'safetensors', 'npcache', 'dummy', 'sharded_state', 'gguf', 'bitsandbytes', 'layered', 'flash_rl', 'remote', 'remote_instance', 'fastsafetensors', 'private', 'runai_streamer' ] }, { label: '--model-loader-extra-config', value: '--model-loader-extra-config', options: [] }, { label: '--trust-remote-code', value: '--trust-remote-code', options: [] }, { label: '--context-length', value: '--context-length', options: [] }, { label: '--is-embedding', value: '--is-embedding', options: [] }, { label: '--enable-multimodal', value: '--enable-multimodal', options: [] }, { label: '--revision', value: '--revision', options: [] }, { label: '--model-impl', value: '--model-impl', options: [] }, // HTTP server { label: '--host', value: '--host', options: [] }, { label: '--port', value: '--port', options: [] }, { label: '--fastapi-root-path', value: '--fastapi-root-path', options: [] }, { label: '--grpc-mode', value: '--grpc-mode', options: [] }, { label: '--skip-server-warmup', value: '--skip-server-warmup', options: [] }, { label: '--warmups', value: '--warmups', options: [] }, { label: '--nccl-port', value: '--nccl-port', options: [] }, { label: '--checkpoint-engine-wait-weights-before-ready', value: '--checkpoint-engine-wait-weights-before-ready', options: [] }, // Quantization and data type { label: '--dtype', value: '--dtype', options: ['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'] }, { label: '--quantization', value: '--quantization', options: [ 'awq', 'fp8', 'gptq', 'marlin', 'gptq_marlin', 'awq_marlin', 'bitsandbytes', 'gguf', 'modelopt', 'modelopt_fp8', 'modelopt_fp4', 'petit_nvfp4', 'w8a8_int8', 'w8a8_fp8', 'moe_wna16', 'qoq', 'w4afp8', 'mxfp4', 'mxfp8', 'auto-round', 'compressed-tensors', 'modelslim', 'quark_int4fp8_moe' ] }, { label: '--quantization-param-path', value: '--quantization-param-path', options: [] }, { label: '--kv-cache-dtype', value: '--kv-cache-dtype', options: ['auto', 'fp8_e5m2', 'fp8_e4m3', 'bf16', 'bfloat16', 'fp4_e2m1'] }, { label: '--enable-fp32-lm-head', value: '--enable-fp32-lm-head', options: [] }, { label: '--modelopt-quant', value: '--modelopt-quant', options: [] }, { label: '--modelopt-checkpoint-restore-path', value: '--modelopt-checkpoint-restore-path', options: [] }, { label: '--modelopt-checkpoint-save-path', value: '--modelopt-checkpoint-save-path', options: [] }, { label: '--modelopt-export-path', value: '--modelopt-export-path', options: [] }, { label: '--quantize-and-serve', value: '--quantize-and-serve', options: [] }, { label: '--rl-quant-profile', value: '--rl-quant-profile', options: [] }, { label: '--enable-quant-communications', value: '--enable-quant-communications', options: [] }, // Memory and scheduling { label: '--mem-fraction-static', value: '--mem-fraction-static', options: [] }, { label: '--max-running-requests', value: '--max-running-requests', options: [] }, { label: '--max-queued-requests', value: '--max-queued-requests', options: [] }, { label: '--max-total-tokens', value: '--max-total-tokens', options: [] }, { label: '--chunked-prefill-size', value: '--chunked-prefill-size', options: [] }, { label: '--prefill-max-requests', value: '--prefill-max-requests', options: [] }, { label: '--enable-dynamic-chunking', value: '--enable-dynamic-chunking', options: [] }, { label: '--max-prefill-tokens', value: '--max-prefill-tokens', options: [] }, { label: '--schedule-policy', value: '--schedule-policy', options: [ 'lpm', 'random', 'fcfs', 'dfs-weight', 'lof', 'priority', 'routing-key' ] }, { label: '--enable-priority-scheduling', value: '--enable-priority-scheduling', options: [] }, { label: '--abort-on-priority-when-disabled', value: '--abort-on-priority-when-disabled', options: [] }, { label: '--schedule-low-priority-values-first', value: '--schedule-low-priority-values-first', options: [] }, { label: '--priority-scheduling-preemption-threshold', value: '--priority-scheduling-preemption-threshold', options: [] }, { label: '--schedule-conservativeness', value: '--schedule-conservativeness', options: [] }, { label: '--page-size', value: '--page-size', options: [] }, { label: '--swa-full-tokens-ratio', value: '--swa-full-tokens-ratio', options: [] }, { label: '--disable-hybrid-swa-memory', value: '--disable-hybrid-swa-memory', options: [] }, { label: '--radix-eviction-policy', value: '--radix-eviction-policy', options: ['lru', 'lfu'] }, { label: '--enable-prefill-delayer', value: '--enable-prefill-delayer', options: [] }, { label: '--prefill-delayer-max-delay-passes', value: '--prefill-delayer-max-delay-passes', options: [] }, { label: '--prefill-delayer-token-usage-low-watermark', value: '--prefill-delayer-token-usage-low-watermark', options: [] }, { label: '--prefill-delayer-queue-min-ratio', value: '--prefill-delayer-queue-min-ratio', options: [] }, { label: '--prefill-delayer-max-delay-ms', value: '--prefill-delayer-max-delay-ms', options: [] }, { label: '--prefill-delayer-forward-passes-buckets', value: '--prefill-delayer-forward-passes-buckets', options: [] }, { label: '--prefill-delayer-wait-seconds-buckets', value: '--prefill-delayer-wait-seconds-buckets', options: [] }, // Runtime options { label: '--device', value: '--device', options: [] }, { label: '--tensor-parallel-size', value: '--tensor-parallel-size', options: [] }, { label: '--tp-size', value: '--tp-size', options: [] }, { label: '--pipeline-parallel-size', value: '--pipeline-parallel-size', options: [] }, { label: '--pp-size', value: '--pp-size', options: [] }, { label: '--attention-context-parallel-size', value: '--attention-context-parallel-size', options: [] }, { label: '--attn-cp-size', value: '--attn-cp-size', options: [] }, { label: '--moe-data-parallel-size', value: '--moe-data-parallel-size', options: [] }, { label: '--moe-dp-size', value: '--moe-dp-size', options: [] }, { label: '--pp-max-micro-batch-size', value: '--pp-max-micro-batch-size', options: [] }, { label: '--pp-async-batch-depth', value: '--pp-async-batch-depth', options: [] }, { label: '--stream-interval', value: '--stream-interval', options: [] }, { label: '--incremental-streaming-output', value: '--incremental-streaming-output', options: [] }, { label: '--random-seed', value: '--random-seed', options: [] }, { label: '--constrained-json-whitespace-pattern', value: '--constrained-json-whitespace-pattern', options: [] }, { label: '--constrained-json-disable-any-whitespace', value: '--constrained-json-disable-any-whitespace', options: [] }, { label: '--watchdog-timeout', value: '--watchdog-timeout', options: [] }, { label: '--soft-watchdog-timeout', value: '--soft-watchdog-timeout', options: [] }, { label: '--dist-timeout', value: '--dist-timeout', options: [] }, { label: '--download-dir', value: '--download-dir', options: [] }, { label: '--model-checksum', value: '--model-checksum', options: [] }, { label: '--base-gpu-id', value: '--base-gpu-id', options: [] }, { label: '--gpu-id-step', value: '--gpu-id-step', options: [] }, { label: '--sleep-on-idle', value: '--sleep-on-idle', options: [] }, { label: '--custom-sigquit-handler', value: '--custom-sigquit-handler', options: [] }, // Logging { label: '--log-level', value: '--log-level', options: [] }, { label: '--log-level-http', value: '--log-level-http', options: [] }, { label: '--log-requests', value: '--log-requests', options: [] }, { label: '--log-requests-level', value: '--log-requests-level', options: ['0', '1', '2', '3'] }, { label: '--log-requests-format', value: '--log-requests-format', options: ['text', 'json'] }, { label: '--log-requests-target', value: '--log-requests-target', options: [] }, { label: '--uvicorn-access-log-exclude-prefixes', value: '--uvicorn-access-log-exclude-prefixes', options: [] }, { label: '--crash-dump-folder', value: '--crash-dump-folder', options: [] }, { label: '--show-time-cost', value: '--show-time-cost', options: [] }, { label: '--enable-metrics', value: '--enable-metrics', options: [] }, { label: '--enable-mfu-metrics', value: '--enable-mfu-metrics', options: [] }, { label: '--enable-metrics-for-all-schedulers', value: '--enable-metrics-for-all-schedulers', options: [] }, { label: '--tokenizer-metrics-custom-labels-header', value: '--tokenizer-metrics-custom-labels-header', options: [] }, { label: '--tokenizer-metrics-allowed-custom-labels', value: '--tokenizer-metrics-allowed-custom-labels', options: [] }, { label: '--bucket-time-to-first-token', value: '--bucket-time-to-first-token', options: [] }, { label: '--bucket-inter-token-latency', value: '--bucket-inter-token-latency', options: [] }, { label: '--bucket-e2e-request-latency', value: '--bucket-e2e-request-latency', options: [] }, { label: '--collect-tokens-histogram', value: '--collect-tokens-histogram', options: [] }, { label: '--prompt-tokens-buckets', value: '--prompt-tokens-buckets', options: [] }, { label: '--generation-tokens-buckets', value: '--generation-tokens-buckets', options: [] }, { label: '--gc-warning-threshold-secs', value: '--gc-warning-threshold-secs', options: [] }, { label: '--decode-log-interval', value: '--decode-log-interval', options: [] }, { label: '--enable-request-time-stats-logging', value: '--enable-request-time-stats-logging', options: [] }, { label: '--kv-events-config', value: '--kv-events-config', options: [] }, { label: '--enable-trace', value: '--enable-trace', options: [] }, { label: '--otlp-traces-endpoint', value: '--otlp-traces-endpoint', options: [] }, // RequestMetricsExporter configuration { label: '--export-metrics-to-file', value: '--export-metrics-to-file', options: [] }, { label: '--export-metrics-to-file-dir', value: '--export-metrics-to-file-dir', options: [] }, // API related { label: '--api-key', value: '--api-key', options: [] }, { label: '--admin-api-key', value: '--admin-api-key', options: [] }, { label: '--served-model-name', value: '--served-model-name', options: [] }, { label: '--weight-version', value: '--weight-version', options: [] }, { label: '--chat-template', value: '--chat-template', options: [] }, { label: '--hf-chat-template-name', value: '--hf-chat-template-name', options: [] }, { label: '--completion-template', value: '--completion-template', options: [] }, { label: '--file-storage-path', value: '--file-storage-path', options: [] }, { label: '--enable-cache-report', value: '--enable-cache-report', options: [] }, { label: '--reasoning-parser', value: '--reasoning-parser', options: [ 'deepseek-r1', 'deepseek-v3', 'glm45', 'gpt-oss', 'kimi', 'qwen3', 'qwen3-thinking', 'step3' ] }, { label: '--tool-call-parser', value: '--tool-call-parser', options: [ 'deepseekv3', 'deepseekv31', 'glm', 'glm45', 'glm47', 'gpt-oss', 'kimi_k2', 'llama3', 'mistral', 'pythonic', 'qwen', 'qwen25', 'qwen3_coder', 'step3', 'gigachat3' ] }, { label: '--tool-server', value: '--tool-server', options: [] }, { label: '--sampling-defaults', value: '--sampling-defaults', options: ['openai', 'model'] }, // Data parallelism { label: '--data-parallel-size', value: '--data-parallel-size', options: [] }, { label: '--dp-size', value: '--dp-size', options: [] }, { label: '--load-balance-method', value: '--load-balance-method', options: [ 'auto', 'round_robin', 'follow_bootstrap_room', 'total_requests', 'total_tokens' ] }, // Multi-node distributed serving { label: '--dist-init-addr', value: '--dist-init-addr', options: [] }, { label: '--nccl-init-addr', value: '--nccl-init-addr', options: [] }, { label: '--nnodes', value: '--nnodes', options: [] }, { label: '--node-rank', value: '--node-rank', options: [] }, // Model override args { label: '--json-model-override-args', value: '--json-model-override-args', options: [] }, { label: '--preferred-sampling-params', value: '--preferred-sampling-params', options: [] }, // LoRA { label: '--enable-lora', value: '--enable-lora', options: [] }, { label: '--enable-lora-overlap-loading', value: '--enable-lora-overlap-loading', options: [] }, { label: '--max-lora-rank', value: '--max-lora-rank', options: [] }, { label: '--lora-target-modules', value: '--lora-target-modules', options: [ 'q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'qkv_proj', 'gate_up_proj', 'all' ] }, { label: '--lora-paths', value: '--lora-paths', options: [] }, { label: '--max-loras-per-batch', value: '--max-loras-per-batch', options: [] }, { label: '--max-loaded-loras', value: '--max-loaded-loras', options: [] }, { label: '--lora-eviction-policy', value: '--lora-eviction-policy', options: ['lru', 'fifo'] }, { label: '--lora-backend', value: '--lora-backend', options: ['triton', 'csgmv', 'ascend', 'torch_native'] }, { label: '--max-lora-chunk-size', value: '--max-lora-chunk-size', options: ['16', '32', '64', '128'] }, { label: '--lora-drain-wait-threshold', value: '--lora-drain-wait-threshold', options: [] }, // Kernel Backends (Attention, Sampling, Grammar, GEMM) { label: '--attention-backend', value: '--attention-backend', options: [ 'triton', 'torch_native', 'flex_attention', 'dsa', 'nsa', 'cutlass_mla', 'fa3', 'fa4', 'flashinfer', 'flashmla', 'trtllm_mla', 'trtllm_mha', 'dual_chunk_flash_attn', 'aiter', 'wave', 'intel_amx', 'ascend' ] }, { label: '--prefill-attention-backend', value: '--prefill-attention-backend', options: [ 'triton', 'torch_native', 'flex_attention', 'dsa', 'nsa', 'cutlass_mla', 'fa3', 'fa4', 'flashinfer', 'flashmla', 'trtllm_mla', 'trtllm_mha', 'dual_chunk_flash_attn', 'aiter', 'wave', 'intel_amx', 'ascend' ] }, { label: '--decode-attention-backend', value: '--decode-attention-backend', options: [ 'triton', 'torch_native', 'flex_attention', 'dsa', 'nsa', 'cutlass_mla', 'fa3', 'fa4', 'flashinfer', 'flashmla', 'trtllm_mla', 'trtllm_mha', 'dual_chunk_flash_attn', 'aiter', 'wave', 'intel_amx', 'ascend' ] }, { label: '--sampling-backend', value: '--sampling-backend', options: ['flashinfer', 'pytorch', 'ascend'] }, { label: '--grammar-backend', value: '--grammar-backend', options: ['xgrammar', 'outlines', 'llguidance', 'none'] }, { label: '--mm-attention-backend', value: '--mm-attention-backend', options: ['sdpa', 'fa3', 'fa4', 'triton_attn', 'ascend_attn', 'aiter_attn'] }, { label: '--dsa-prefill-backend', value: '--dsa-prefill-backend', options: [ 'flashmla_sparse', 'flashmla_kv', 'flashmla_auto', 'fa3', 'tilelang', 'aiter', 'trtllm' ] }, { label: '--dsa-decode-backend', value: '--dsa-decode-backend', options: [ 'flashmla_sparse', 'flashmla_kv', 'fa3', 'tilelang', 'aiter', 'trtllm' ] }, { label: '--dsa-topk-backend', value: '--dsa-topk-backend', options: ['sgl-kernel', 'torch', 'flashinfer'] }, { label: '--fp8-gemm-backend', value: '--fp8-gemm-backend', options: [ 'auto', 'deep_gemm', 'flashinfer_trtllm', 'flashinfer_cutlass', 'flashinfer_deepgemm', 'cutlass', 'triton', 'aiter' ] }, { label: '--fp4-gemm-backend', value: '--fp4-gemm-backend', options: [ 'auto', 'flashinfer_cudnn', 'flashinfer_cutlass', 'flashinfer_trtllm' ] }, { label: '--disable-flashinfer-autotune', value: '--disable-flashinfer-autotune', options: [] }, // Speculative decoding { label: '--speculative-algorithm', value: '--speculative-algorithm', options: ['EAGLE', 'EAGLE3', 'NEXTN', 'STANDALONE', 'NGRAM'] }, { label: '--speculative-draft-model-path', value: '--speculative-draft-model-path', options: [] }, { label: '--speculative-draft-model', value: '--speculative-draft-model', options: [] }, { label: '--speculative-draft-model-revision', value: '--speculative-draft-model-revision', options: [] }, { label: '--speculative-draft-load-format', value: '--speculative-draft-load-format', options: [] }, { label: '--speculative-num-steps', value: '--speculative-num-steps', options: [] }, { label: '--speculative-eagle-topk', value: '--speculative-eagle-topk', options: [] }, { label: '--speculative-num-draft-tokens', value: '--speculative-num-draft-tokens', options: [] }, { label: '--speculative-accept-threshold-single', value: '--speculative-accept-threshold-single', options: [] }, { label: '--speculative-accept-threshold-acc', value: '--speculative-accept-threshold-acc', options: [] }, { label: '--speculative-token-map', value: '--speculative-token-map', options: [] }, { label: '--speculative-attention-mode', value: '--speculative-attention-mode', options: ['prefill', 'decode'] }, { label: '--speculative-draft-attention-backend', value: '--speculative-draft-attention-backend', options: [] }, { label: '--speculative-moe-runner-backend', value: '--speculative-moe-runner-backend', options: [] }, { label: '--speculative-moe-a2a-backend', value: '--speculative-moe-a2a-backend', options: [] }, { label: '--speculative-draft-model-quantization', value: '--speculative-draft-model-quantization', options: [] }, // Ngram speculative decoding { label: '--speculative-ngram-min-bfs-breadth', value: '--speculative-ngram-min-bfs-breadth', options: [] }, { label: '--speculative-ngram-max-bfs-breadth', value: '--speculative-ngram-max-bfs-breadth', options: [] }, { label: '--speculative-ngram-match-type', value: '--speculative-ngram-match-type', options: ['BFS', 'PROB'] }, { label: '--speculative-ngram-max-trie-depth', value: '--speculative-ngram-max-trie-depth', options: [] }, { label: '--speculative-ngram-capacity', value: '--speculative-ngram-capacity', options: [] }, // Multi-layer Eagle speculative decoding { label: '--enable-multi-layer-eagle', value: '--enable-multi-layer-eagle', options: [] }, // MoE { label: '--expert-parallel-size', value: '--expert-parallel-size', options: [] }, { label: '--ep-size', value: '--ep-size', options: [] }, { label: '--ep', value: '--ep', options: [] }, { label: '--moe-a2a-backend', value: '--moe-a2a-backend', options: ['none', 'deepep', 'mooncake', 'mori', 'nixl', 'ascend_fuseep'] }, { label: '--moe-runner-backend', value: '--moe-runner-backend', options: [ 'auto', 'deep_gemm', 'triton', 'triton_kernel', 'flashinfer_trtllm', 'flashinfer_trtllm_routed', 'flashinfer_cutlass', 'flashinfer_mxfp4', 'flashinfer_cutedsl', 'cutlass' ] }, { label: '--flashinfer-mxfp4-moe-precision', value: '--flashinfer-mxfp4-moe-precision', options: ['default', 'bf16'] }, { label: '--enable-flashinfer-allreduce-fusion', value: '--enable-flashinfer-allreduce-fusion', options: [] }, { label: '--enable-aiter-allreduce-fusion', value: '--enable-aiter-allreduce-fusion', options: [] }, { label: '--deepep-mode', value: '--deepep-mode', options: ['normal', 'low_latency', 'auto'] }, { label: '--deepep-dispatcher-output-dtype', value: '--deepep-dispatcher-output-dtype', options: ['bf16', 'fp8', 'int8', 'nvfp4', 'auto'] }, { label: '--ep-num-redundant-experts', value: '--ep-num-redundant-experts', options: [] }, { label: '--ep-dispatch-algorithm', value: '--ep-dispatch-algorithm', options: [] }, { label: '--init-expert-location', value: '--init-expert-location', options: [] }, { label: '--enable-eplb', value: '--enable-eplb', options: [] }, { label: '--eplb-algorithm', value: '--eplb-algorithm', options: [] }, { label: '--eplb-rebalance-num-iterations', value: '--eplb-rebalance-num-iterations', options: [] }, { label: '--eplb-rebalance-layers-per-chunk', value: '--eplb-rebalance-layers-per-chunk', options: [] }, { label: '--eplb-min-rebalancing-utilization-threshold', value: '--eplb-min-rebalancing-utilization-threshold', options: [] }, { label: '--expert-distribution-recorder-mode', value: '--expert-distribution-recorder-mode', options: [] }, { label: '--expert-distribution-recorder-buffer-size', value: '--expert-distribution-recorder-buffer-size', options: [] }, { label: '--enable-expert-distribution-metrics', value: '--enable-expert-distribution-metrics', options: [] }, { label: '--deepep-config', value: '--deepep-config', options: [] }, { label: '--moe-dense-tp-size', value: '--moe-dense-tp-size', options: [] }, { label: '--elastic-ep-backend', value: '--elastic-ep-backend', options: ['none', 'mooncake'] }, { label: '--enable-elastic-expert-backup', value: '--enable-elastic-expert-backup', options: [] }, { label: '--mooncake-ib-device', value: '--mooncake-ib-device', options: [] }, { label: '--enable-deepep-waterfill', value: '--enable-deepep-waterfill', options: [] }, { label: '--elastic-ep-rejoin', value: '--elastic-ep-rejoin', options: [] }, // Mamba Cache { label: '--max-mamba-cache-size', value: '--max-mamba-cache-size', options: [] }, { label: '--mamba-ssm-dtype', value: '--mamba-ssm-dtype', options: ['float32', 'bfloat16', 'float16'] }, { label: '--mamba-full-memory-ratio', value: '--mamba-full-memory-ratio', options: [] }, { label: '--mamba-scheduler-strategy', value: '--mamba-scheduler-strategy', options: ['auto', 'no_buffer', 'extra_buffer'] }, { label: '--mamba-track-interval', value: '--mamba-track-interval', options: [] }, // Hierarchical cache { label: '--enable-hierarchical-cache', value: '--enable-hierarchical-cache', options: [] }, { label: '--hicache-ratio', value: '--hicache-ratio', options: [] }, { label: '--hicache-size', value: '--hicache-size', options: [] }, { label: '--hicache-write-policy', value: '--hicache-write-policy', options: ['write_back', 'write_through', 'write_through_selective'] }, { label: '--hicache-io-backend', value: '--hicache-io-backend', options: ['direct', 'kernel', 'kernel_ascend'] }, { label: '--hicache-mem-layout', value: '--hicache-mem-layout', options: [ 'layer_first', 'page_first', 'page_first_direct', 'page_first_kv_split', 'page_head' ] }, { label: '--hicache-storage-backend', value: '--hicache-storage-backend', options: ['file', 'mooncake', 'hf3fs', 'nixl', 'aibrix', 'dynamic', 'eic'] }, { label: '--hicache-storage-prefetch-policy', value: '--hicache-storage-prefetch-policy', options: ['best_effort', 'wait_complete', 'timeout'] }, { label: '--hicache-storage-backend-extra-config', value: '--hicache-storage-backend-extra-config', options: [] }, // Hierarchical sparse attention { label: '--hierarchical-sparse-attention-extra-config', value: '--hierarchical-sparse-attention-extra-config', options: [] }, // LMCache { label: '--enable-lmcache', value: '--enable-lmcache', options: [] }, // Offloading { label: '--cpu-offload-gb', value: '--cpu-offload-gb', options: [] }, { label: '--offload-group-size', value: '--offload-group-size', options: [] }, { label: '--offload-num-in-group', value: '--offload-num-in-group', options: [] }, { label: '--offload-prefetch-step', value: '--offload-prefetch-step', options: [] }, { label: '--offload-mode', value: '--offload-mode', options: [] }, // Args for multi-item scoring { label: '--multi-item-scoring-delimiter', value: '--multi-item-scoring-delimiter', options: [] }, // Optimization/debug options { label: '--disable-radix-cache', value: '--disable-radix-cache', options: [] }, { label: '--cuda-graph-max-bs', value: '--cuda-graph-max-bs', options: [] }, { label: '--cuda-graph-bs', value: '--cuda-graph-bs', options: [] }, { label: '--disable-cuda-graph', value: '--disable-cuda-graph', options: [] }, { label: '--disable-cuda-graph-padding', value: '--disable-cuda-graph-padding', options: [] }, { label: '--enable-profile-cuda-graph', value: '--enable-profile-cuda-graph', options: [] }, { label: '--enable-cudagraph-gc', value: '--enable-cudagraph-gc', options: [] }, { label: '--enable-layerwise-nvtx-marker', value: '--enable-layerwise-nvtx-marker', options: [] }, { label: '--enable-nccl-nvls', value: '--enable-nccl-nvls', options: [] }, { label: '--enable-symm-mem', value: '--enable-symm-mem', options: [] }, { label: '--disable-flashinfer-cutlass-moe-fp4-allgather', value: '--disable-flashinfer-cutlass-moe-fp4-allgather', options: [] }, { label: '--enable-tokenizer-batch-encode', value: '--enable-tokenizer-batch-encode', options: [] }, { label: '--disable-tokenizer-batch-decode', value: '--disable-tokenizer-batch-decode', options: [] }, { label: '--disable-outlines-disk-cache', value: '--disable-outlines-disk-cache', options: [] }, { label: '--disable-custom-all-reduce', value: '--disable-custom-all-reduce', options: [] }, { label: '--enable-mscclpp', value: '--enable-mscclpp', options: [] }, { label: '--enable-torch-symm-mem', value: '--enable-torch-symm-mem', options: [] }, { label: '--disable-overlap-schedule', value: '--disable-overlap-schedule', options: [] }, { label: '--enable-mixed-chunk', value: '--enable-mixed-chunk', options: [] }, { label: '--enable-dp-attention', value: '--enable-dp-attention', options: [] }, { label: '--enable-dp-lm-head', value: '--enable-dp-lm-head', options: [] }, { label: '--enable-two-batch-overlap', value: '--enable-two-batch-overlap', options: [] }, { label: '--enable-single-batch-overlap', value: '--enable-single-batch-overlap', options: [] }, { label: '--tbo-token-distribution-threshold', value: '--tbo-token-distribution-threshold', options: [] }, { label: '--enable-torch-compile', value: '--enable-torch-compile', options: [] }, { label: '--enable-torch-compile-debug-mode', value: '--enable-torch-compile-debug-mode', options: [] }, { label: '--disable-piecewise-cuda-graph', value: '--disable-piecewise-cuda-graph', options: [] }, { label: '--enforce-piecewise-cuda-graph', value: '--enforce-piecewise-cuda-graph', options: [] }, { label: '--piecewise-cuda-graph-tokens', value: '--piecewise-cuda-graph-tokens', options: [] }, { label: '--piecewise-cuda-graph-compiler', value: '--piecewise-cuda-graph-compiler', options: ['eager', 'inductor'] }, { label: '--torch-compile-max-bs', value: '--torch-compile-max-bs', options: [] }, { label: '--piecewise-cuda-graph-max-tokens', value: '--piecewise-cuda-graph-max-tokens', options: [] }, { label: '--torchao-config', value: '--torchao-config', options: [] }, { label: '--enable-nan-detection', value: '--enable-nan-detection', options: [] }, { label: '--enable-p2p-check', value: '--enable-p2p-check', options: [] }, { label: '--triton-attention-reduce-in-fp32', value: '--triton-attention-reduce-in-fp32', options: [] }, { label: '--triton-attention-num-kv-splits', value: '--triton-attention-num-kv-splits', options: [] }, { label: '--triton-attention-split-tile-size', value: '--triton-attention-split-tile-size', options: [] }, { label: '--num-continuous-decode-steps', value: '--num-continuous-decode-steps', options: [] }, { label: '--delete-ckpt-after-loading', value: '--delete-ckpt-after-loading', options: [] }, { label: '--enable-memory-saver', value: '--enable-memory-saver', options: [] }, { label: '--enable-weights-cpu-backup', value: '--enable-weights-cpu-backup', options: [] }, { label: '--enable-draft-weights-cpu-backup', value: '--enable-draft-weights-cpu-backup', options: [] }, { label: '--allow-auto-truncate', value: '--allow-auto-truncate', options: [] }, { label: '--enable-custom-logit-processor', value: '--enable-custom-logit-processor', options: [] }, { label: '--flashinfer-mla-disable-ragged', value: '--flashinfer-mla-disable-ragged', options: [] }, { label: '--disable-shared-experts-fusion', value: '--disable-shared-experts-fusion', options: [] }, { label: '--disable-chunked-prefix-cache', value: '--disable-chunked-prefix-cache', options: [] }, { label: '--disable-fast-image-processor', value: '--disable-fast-image-processor', options: [] }, { label: '--keep-mm-feature-on-device', value: '--keep-mm-feature-on-device', options: [] }, { label: '--enable-return-hidden-states', value: '--enable-return-hidden-states', options: [] }, { label: '--enable-return-routed-experts', value: '--enable-return-routed-experts', options: [] }, { label: '--scheduler-recv-interval', value: '--scheduler-recv-interval', options: [] }, { label: '--numa-node', value: '--numa-node', options: [] }, { label: '--enable-deterministic-inference', value: '--enable-deterministic-inference', options: [] }, { label: '--rl-on-policy-target', value: '--rl-on-policy-target', options: ['fsdp'] }, { label: '--enable-attn-tp-input-scattered', value: '--enable-attn-tp-input-scattered', options: [] }, { label: '--enable-dsa-prefill-context-parallel', value: '--enable-dsa-prefill-context-parallel', options: [] }, { label: '--dsa-prefill-cp-mode', value: '--dsa-prefill-cp-mode', options: ['in-seq-split', 'round-robin-split'] }, { label: '--enable-fused-qk-norm-rope', value: '--enable-fused-qk-norm-rope', options: [] }, { label: '--enable-precise-embedding-interpolation', value: '--enable-precise-embedding-interpolation', options: [] }, // Dynamic batch tokenizer { label: '--enable-dynamic-batch-tokenizer', value: '--enable-dynamic-batch-tokenizer', options: [] }, { label: '--dynamic-batch-tokenizer-batch-size', value: '--dynamic-batch-tokenizer-batch-size', options: [] }, { label: '--dynamic-batch-tokenizer-batch-timeout', value: '--dynamic-batch-tokenizer-batch-timeout', options: [] }, // Debug tensor dumps { label: '--debug-tensor-dump-output-folder', value: '--debug-tensor-dump-output-folder', options: [] }, { label: '--debug-tensor-dump-layers', value: '--debug-tensor-dump-layers', options: [] }, { label: '--debug-tensor-dump-input-file', value: '--debug-tensor-dump-input-file', options: [] }, { label: '--debug-tensor-dump-inject', value: '--debug-tensor-dump-inject', options: [] }, // PD disaggregation { label: '--disaggregation-mode', value: '--disaggregation-mode', options: ['null', 'prefill', 'decode'] }, { label: '--disaggregation-transfer-backend', value: '--disaggregation-transfer-backend', options: ['mooncake', 'nixl', 'ascend', 'fake'] }, { label: '--disaggregation-bootstrap-port', value: '--disaggregation-bootstrap-port', options: [] }, { label: '--disaggregation-ib-device', value: '--disaggregation-ib-device', options: [] }, { label: '--disaggregation-decode-enable-offload-kvcache', value: '--disaggregation-decode-enable-offload-kvcache', options: [] }, { label: '--num-reserved-decode-tokens', value: '--num-reserved-decode-tokens', options: [] }, { label: '--disaggregation-decode-polling-interval', value: '--disaggregation-decode-polling-interval', options: [] }, // Encode prefill disaggregation { label: '--encoder-only', value: '--encoder-only', options: [] }, { label: '--language-only', value: '--language-only', options: [] }, { label: '--encoder-transfer-backend', value: '--encoder-transfer-backend', options: ['zmq_to_scheduler', 'zmq_to_tokenizer', 'mooncake'] }, { label: '--encoder-urls', value: '--encoder-urls', options: [] }, // Custom weight loader { label: '--custom-weight-loader', value: '--custom-weight-loader', options: [] }, { label: '--weight-loader-disable-mmap', value: '--weight-loader-disable-mmap', options: [] }, { label: '--weight-loader-prefetch-checkpoints', value: '--weight-loader-prefetch-checkpoints', options: [] }, { label: '--weight-loader-prefetch-num-threads', value: '--weight-loader-prefetch-num-threads', options: [] }, { label: '--remote-instance-weight-loader-seed-instance-ip', value: '--remote-instance-weight-loader-seed-instance-ip', options: [] }, { label: '--remote-instance-weight-loader-seed-instance-service-port', value: '--remote-instance-weight-loader-seed-instance-service-port', options: [] }, { label: '--remote-instance-weight-loader-send-weights-group-ports', value: '--remote-instance-weight-loader-send-weights-group-ports', options: [] }, { label: '--remote-instance-weight-loader-backend', value: '--remote-instance-weight-loader-backend', options: ['transfer_engine', 'nccl'] }, { label: '--remote-instance-weight-loader-start-seed-via-transfer-engine', value: '--remote-instance-weight-loader-start-seed-via-transfer-engine', options: [] }, // For PD-Multiplexing { label: '--enable-pdmux', value: '--enable-pdmux', options: [] }, { label: '--pdmux-config-path', value: '--pdmux-config-path', options: [] }, { label: '--sm-group-num', value: '--sm-group-num', options: [] }, // Configuration file support { label: '--config', value: '--config', options: [] }, // For Multi-Modal { label: '--mm-max-concurrent-calls', value: '--mm-max-concurrent-calls', options: [] }, { label: '--mm-per-request-timeout', value: '--mm-per-request-timeout', options: [] }, { label: '--enable-broadcast-mm-inputs-process', value: '--enable-broadcast-mm-inputs-process', options: [] }, { label: '--mm-process-config', value: '--mm-process-config', options: [] }, { label: '--mm-enable-dp-encoder', value: '--mm-enable-dp-encoder', options: [] }, { label: '--limit-mm-data-per-request', value: '--limit-mm-data-per-request', options: [] }, { label: '--enable-mm-global-cache', value: '--enable-mm-global-cache', options: [] }, // For checkpoint decryption { label: '--decrypted-config-file', value: '--decrypted-config-file', options: [] }, { label: '--decrypted-draft-config-file', value: '--decrypted-draft-config-file', options: [] }, { label: '--enable-prefix-mm-cache', value: '--enable-prefix-mm-cache', options: [] } ]; export default options;