diff --git a/src/pages/llmodels/config/vllm-config.ts b/src/pages/llmodels/config/vllm-config.ts index 1396455c..c62759a7 100644 --- a/src/pages/llmodels/config/vllm-config.ts +++ b/src/pages/llmodels/config/vllm-config.ts @@ -97,7 +97,34 @@ const options = [ { label: '--tool-call-parser', value: '--tool-call-parser', - options: ['mistral', 'hermes', 'internlm', 'llama3_json'] + options: [ + 'mistral', + 'hermes', + 'internlm', + 'jamba', + 'llama3_json', + 'granite-20b-fc', + 'granite', + 'pythonic' + ] + }, + { + label: '--task', + value: '--task', + options: [ + 'auto', + 'generate', + 'embedding', + 'embed', + 'classify', + 'score', + 'reward' + ] + }, + { + label: '--allowed-local-media-path', + value: '--allowed-local-media-path', + options: [] }, { label: '--tool-parser-plugin', @@ -162,9 +189,25 @@ const options = [ 'sharded_state', 'gguf', 'bitsandbytes', - 'mistral' + 'mistral', + 'runai_streamer' ] }, + { + label: '--chat-template-content-format', + value: '--chat-template-content-format', + options: ['auto', 'string', 'openai'] + }, + { + label: '--enable-reasoning', + value: '--enable-reasoning', + options: [] + }, + { + label: '--reasoning-parser', + value: '--reasoning-parser', + options: ['deepseek_r1'] + }, { label: '--config-format', value: '--config-format', @@ -193,12 +236,23 @@ const options = [ { label: '--guided-decoding-backend', value: '--guided-decoding-backend', - options: ['outlines', 'lm-format-enforcer'] + options: ['outlines', 'lm-format-enforcer', 'xgrammar'] + }, + { + label: '--logits-processor-pattern', + value: '--logits-processor-pattern', + options: [] + }, + + { + label: '--model-impl', + value: '--model-impl', + options: ['auto', 'vllm', 'transformers'] }, { label: '--distributed-executor-backend', value: '--distributed-executor-backend', - options: ['ray', 'mp'] + options: ['ray', 'mp', 'uni', 'external_launcher'] }, { label: '--worker-use-ray', @@ -228,7 +282,7 @@ const options = [ { label: '--block-size', value: '--block-size', - options: ['8', '16', '32'] + options: ['8', '16', '32', '64', '128'] }, { label: '--enable-prefix-caching', @@ -295,6 +349,16 @@ const options = [ value: '--disable-log-stats', options: [] }, + { + label: '--hf-overrides', + value: '--hf-overrides', + options: [] + }, + { + label: '--disable-mm-preprocessor-cache', + value: '--disable-mm-preprocessor-cache', + options: [] + }, { label: '--quantization', value: '--quantization', @@ -308,10 +372,13 @@ const options = [ 'modelopt', 'marlin', 'gguf', + 'hqq', 'gptq_marlin_24', 'gptq_marlin', 'awq_marlin', 'gptq', + 'quark', + 'moe_wna16', 'compressed-tensors', 'bitsandbytes', 'qqq', @@ -321,6 +388,11 @@ const options = [ 'None' ] }, + { + label: '--enable-lora-bias', + value: '--enable-lora-bias', + options: [] + }, { label: '--rope-scaling', value: '--rope-scaling', @@ -399,7 +471,7 @@ const options = [ { label: '--lora-dtype', value: '--lora-dtype', - options: ['auto', 'float16', 'bfloat16', 'float32'] + options: ['auto', 'float16', 'bfloat16'] }, { label: '--long-lora-scaling-factors', @@ -434,7 +506,7 @@ const options = [ { label: '--device', value: '--device', - options: ['auto', 'cuda', 'neuron', 'cpu', 'openvino', 'tpu', 'xpu'] + options: ['auto', 'cuda', 'neuron', 'cpu', 'openvino', 'tpu', 'xpu', 'hpu'] }, { label: '--num-scheduler-steps', @@ -481,9 +553,12 @@ const options = [ 'compressed-tensors', 'bitsandbytes', 'qqq', + 'hqq', 'experts_int8', 'neuron_quant', 'ipex', + 'quark', + 'moe_wna16', 'None' ] }, @@ -587,6 +662,47 @@ const options = [ value: '--override-neuron-config', options: [] }, + { + label: '--compilation-config', + value: '--compilation-config', + options: [] + }, + { + label: '--override-pooler-config', + value: '--override-pooler-config', + options: [] + }, + { + label: '--kv-transfer-config', + value: '--kv-transfer-config', + options: [] + }, + { + label: '--worker-cls', + value: '--worker-cls', + options: [] + }, + { + label: '--override-generation-config', + value: '--override-generation-config', + options: [] + }, + { + label: '--enable-sleep-mode', + value: '--enable-sleep-mode', + options: [] + }, + + { + label: '--calculate-kv-scales', + value: '--calculate-kv-scales', + options: [] + }, + { + label: '--generation-config', + value: '--generation-config', + options: [] + }, { label: '--scheduling-policy', value: '--scheduling-policy', @@ -606,6 +722,11 @@ const options = [ label: '--disable-fastapi-docs', value: '--disable-fastapi-docs', options: [] + }, + { + label: '--enable-prompt-tokens-details', + value: '--enable-prompt-tokens-details', + options: [] } ]; diff --git a/src/pages/playground/components/ground-left.tsx b/src/pages/playground/components/ground-left.tsx index f0363b44..9c01035e 100644 --- a/src/pages/playground/components/ground-left.tsx +++ b/src/pages/playground/components/ground-left.tsx @@ -52,6 +52,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { const currentMessageRef = useRef(null); const paramsRef = useRef(null); const messageListLengthCache = useRef(0); + const reasonContentRef = useRef(''); const { initialize, updateScrollerPosition } = useOverlayScroller(); const { initialize: innitializeParams } = useOverlayScroller(); @@ -79,6 +80,19 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { messageId.current = messageId.current + 1; }; + const formatContent = (data: { + content: string; + reasoningContent: string; + }) => { + if (data.reasoningContent && !data.content) { + return `${data.reasoningContent}${data.content}`; + } + if (data.reasoningContent && data.content) { + return `${data.reasoningContent}${data.content}`; + } + return data.content; + }; + const handleNewMessage = (message?: { role: string; content: string }) => { const newMessage = message || { role: @@ -102,6 +116,10 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { if (!chunk || !_.get(chunk, 'choices', []).length) { return; } + + reasonContentRef.current = + reasonContentRef.current + + _.get(chunk, 'choices.0.delta.reasoning_content', ''); contentRef.current = contentRef.current + _.get(chunk, 'choices.0.delta.content', ''); setMessageList([ @@ -109,7 +127,10 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { ...currentMessageRef.current, { role: Roles.Assistant, - content: contentRef.current, + content: formatContent({ + content: contentRef.current, + reasoningContent: reasonContentRef.current + }), uid: messageId.current } ]); @@ -139,6 +160,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { : []; contentRef.current = ''; + reasonContentRef.current = ''; setMessageList((pre) => { return [...pre, ...currentMessageRef.current]; }); diff --git a/src/pages/playground/components/multiple-chat/model-item.tsx b/src/pages/playground/components/multiple-chat/model-item.tsx index 227d21fb..e2bd181c 100644 --- a/src/pages/playground/components/multiple-chat/model-item.tsx +++ b/src/pages/playground/components/multiple-chat/model-item.tsx @@ -68,6 +68,7 @@ const ModelItem: React.FC = forwardRef( const currentMessageRef = useRef([]); const modelScrollRef = useRef(null); const messageListLengthCache = useRef(0); + const reasonContentRef = useRef(''); const { initialize, updateScrollerPosition } = useOverlayScroller(); @@ -87,6 +88,19 @@ const ModelItem: React.FC = forwardRef( setLoadingStatus(instanceId, false); }; + const formatContent = (data: { + content: string; + reasoningContent: string; + }) => { + if (data.reasoningContent && !data.content) { + return `${data.reasoningContent}${data.content}`; + } + if (data.reasoningContent && data.content) { + return `${data.reasoningContent}${data.content}`; + } + return data.content; + }; + const joinMessage = (chunk: any) => { setTokenResult({ ...(chunk?.usage ?? {}) @@ -96,6 +110,10 @@ const ModelItem: React.FC = forwardRef( return; } + reasonContentRef.current = + reasonContentRef.current + + _.get(chunk, 'choices.0.delta.reasoning_content', ''); + contentRef.current = contentRef.current + _.get(chunk, 'choices.0.delta.content', ''); @@ -104,7 +122,10 @@ const ModelItem: React.FC = forwardRef( ...currentMessageRef.current, { role: Roles.Assistant, - content: contentRef.current, + content: formatContent({ + content: contentRef.current, + reasoningContent: reasonContentRef.current + }), uid: messageId.current } ]); @@ -132,6 +153,7 @@ const ModelItem: React.FC = forwardRef( }); contentRef.current = ''; + reasonContentRef.current = ''; // ====== payload ================= const messageParams = [