From 56eba0d4fc601c83ba828a75952596191bc41d29 Mon Sep 17 00:00:00 2001 From: jialin Date: Fri, 17 Oct 2025 18:41:43 +0800 Subject: [PATCH] fix: default spec by mode --- src/pages/llmodels/apis/index.ts | 19 +-- .../components/deploy-builtin-modal.tsx | 116 +++++++++++---- src/pages/llmodels/config/form-context.ts | 2 + src/pages/llmodels/config/test.ts | 138 ++++++++++++++++++ src/pages/llmodels/config/types.ts | 15 ++ src/pages/llmodels/forms/performance.tsx | 39 ++--- 6 files changed, 269 insertions(+), 60 deletions(-) create mode 100644 src/pages/llmodels/config/test.ts diff --git a/src/pages/llmodels/apis/index.ts b/src/pages/llmodels/apis/index.ts index 348a1a07..fe1a5508 100644 --- a/src/pages/llmodels/apis/index.ts +++ b/src/pages/llmodels/apis/index.ts @@ -3,11 +3,11 @@ import { downloadFile, listFiles, listModels } from '@huggingface/hub'; import { PipelineType } from '@huggingface/tasks'; import { request } from '@umijs/max'; import qs from 'query-string'; +import testSpecs from '../config/test'; import { AccessControlFormData, BackendItem, CatalogItem, - CatalogSpec, DraftModelItem, EvaluateResult, EvaluateSpec, @@ -370,14 +370,15 @@ export async function queryCatalogItemSpec( params: { id: number }, options?: any ) { - return await request>( - `/model-sets/${params.id}/specs`, - { - method: 'GET', - ...options, - params - } - ); + // return await request>( + // `/model-sets/${params.id}/specs`, + // { + // method: 'GET', + // ...options, + // params + // } + // ); + return testSpecs; } export async function evaluationsModelSpec( diff --git a/src/pages/llmodels/components/deploy-builtin-modal.tsx b/src/pages/llmodels/components/deploy-builtin-modal.tsx index 7ff73833..154a983a 100644 --- a/src/pages/llmodels/components/deploy-builtin-modal.tsx +++ b/src/pages/llmodels/components/deploy-builtin-modal.tsx @@ -25,12 +25,28 @@ import { useCheckCompatibility } from '../hooks'; import useFormInitialValues from '../hooks/use-form-initial-values'; import CompatibilityAlert from './compatible-alert'; +const ModesMap: Record = { + latency: 'models.form.mode.latency', + reference: 'models.form.mode.reference', + throughput: 'models.form.mode.throughput' +}; + +const ModesTipsMap: Record = { + latency: 'models.form.mode.latency.tips', + reference: 'models.form.mode.reference.tips', + throughput: 'models.form.mode.throughput.tips' +}; + const pickFieldsFromSpec = [ - 'backend_version', - 'backend_parameters', 'env', 'size', - 'quantization' + 'source', + 'quantization', + 'backend_version', + 'backend_parameters', + 'backend', + 'extended_kv_cache', + 'speculative_config' ]; type AddModalProps = { @@ -115,6 +131,7 @@ const AddModal: React.FC = (props) => { const [quantizationOptions, setQuantizationOptions] = useState< Global.BaseOption[] >([]); + const [modeList, setModeList] = useState[]>([]); const sourceGroupMap = useRef({}); const axiosToken = useRef(null); const selectSpecRef = useRef({} as CatalogSpec); @@ -167,32 +184,37 @@ const AddModal: React.FC = (props) => { }; const getModelSpec = (data: { + mode?: string; backend: string; size: number; quantization: string; }) => { - const spec = _.find(specListRef.current, (item: CatalogSpec) => { - if (data.size && data.quantization) { - return ( - item.size === data.size && - item.backend === data.backend && - item.quantization === data.quantization - ); - } - if (data.size) { - return item.size === data.size && item.backend === data.backend; - } - if (data.quantization) { - return ( - item.quantization === data.quantization && - item.backend === data.backend - ); - } - return item.backend === data.backend; - }); - selectSpecRef.current = spec; + // const spec = _.find(specListRef.current, (item: CatalogSpec) => { + // if (data.size && data.quantization) { + // return ( + // item.size === data.size && + // item.backend === data.backend && + // item.quantization === data.quantization + // ); + // } + // if (data.size) { + // return item.size === data.size && item.backend === data.backend; + // } + // if (data.quantization) { + // return ( + // item.quantization === data.quantization && + // item.backend === data.backend + // ); + // } + // return item.backend === data.backend; + // }); + const defaultSpec = _.find( + specListRef.current, + (item: CatalogSpec) => item.mode === data.mode + ); + selectSpecRef.current = defaultSpec; return { - ..._.pick(spec, pickFieldsFromSpec), + ..._.pick(defaultSpec, pickFieldsFromSpec), categories: _.get(current, 'categories.0', null) }; }; @@ -242,6 +264,7 @@ const AddModal: React.FC = (props) => { }; }); const result = _.uniqBy(quantizationList, 'value'); + console.log('quantization options:', result); setQuantizationOptions(result); return result; }; @@ -376,6 +399,16 @@ const AddModal: React.FC = (props) => { } ); const groupList = _.groupBy(res.items, 'source'); + const modes = _.groupBy(res.items, 'mode'); + + const modeDataList = _.map(modes, (item: CatalogSpec, key: string) => { + return { + label: ModesMap[key] || key, + isBuiltIn: ModesMap[key] ? true : false, + value: key, + tips: ModesTipsMap[key] || '' + }; + }); sourceGroupMap.current = groupList; @@ -387,20 +420,21 @@ const AddModal: React.FC = (props) => { const list = _.sortBy(res.items, 'size'); + console.log('spec items:', res.items); hasF16Ref.current = _.some(res.items, (item: CatalogSpec) => { return AscendNPUQuant_F16.includes(_.toUpper(item.quantization)); }); - const defaultSpec = - _.find(list, (item: CatalogSpec) => { - return getDefaultQuant({ - category: _.get(current, 'categories.0', ''), - quantOption: item.quantization, - backend: item.backend - }); - }) || _.get(res.items, `0`, {}); + const defaultSpec = _.find( + list, + (item: CatalogSpec) => item.mode === modeDataList[0]?.value + ); + + console.log('default spec:', defaultSpec); selectSpecRef.current = defaultSpec; + + setModeList(modeDataList); setSourceList(sources); handleSetBackendOptions(); handleSetSizeOptions({ @@ -472,6 +506,22 @@ const AddModal: React.FC = (props) => { handleCheckFormData(); }; + const handleOnModeChange = (val: string) => { + const data = getModelSpec({ + mode: val, + backend: form.current.getFieldValue('backend'), + size: 0, + quantization: '' + }); + + console.log('mode change data:', data); + + form.current.setFieldsValue({ + ...data + }); + handleCheckFormData(); + }; + const handleOk = async (values: FormData) => { const data = { ..._.omit(selectSpecRef.current, ['name']), @@ -542,6 +592,8 @@ const AddModal: React.FC = (props) => { value={{ sizeOptions: [], quantizationOptions: [], + modeList: modeList, + onModeChange: handleOnModeChange, onSizeChange: handleOnSizeChange, onQuantizationChange: handleOnQuantizationChange }} diff --git a/src/pages/llmodels/config/form-context.ts b/src/pages/llmodels/config/form-context.ts index fe5e7903..fc1592d1 100644 --- a/src/pages/llmodels/config/form-context.ts +++ b/src/pages/llmodels/config/form-context.ts @@ -30,6 +30,8 @@ interface FormContextProps { interface CatalogFormContextProps { sizeOptions: Global.BaseOption[]; quantizationOptions: Global.BaseOption[]; + modeList: Global.BaseOption[]; + onModeChange: (val: string) => void; onSizeChange: (val: number) => void; onQuantizationChange: (val: string) => void; } diff --git a/src/pages/llmodels/config/test.ts b/src/pages/llmodels/config/test.ts new file mode 100644 index 00000000..f1f661d3 --- /dev/null +++ b/src/pages/llmodels/config/test.ts @@ -0,0 +1,138 @@ +export default { + items: [ + { + source: 'model_scope', + huggingface_repo_id: null, + huggingface_filename: null, + ollama_library_model_name: null, + model_scope_model_id: 'Qwen/Qwen3-30B-A3B-FP8', + model_scope_file_path: null, + local_path: null, + name: null, + description: null, + meta: {}, + replicas: 1, + ready_replicas: 0, + categories: [], + embedding_only: false, + image_only: false, + reranker: false, + speech_to_text: false, + text_to_speech: false, + placement_strategy: 'spread', + cpu_offloading: false, + distributed_inference_across_workers: false, + worker_selector: {}, + gpu_selector: null, + backend: 'SGLang', + backend_version: null, + backend_parameters: [ + '--reasoning-parser=qwen3', + '--tool-call-parser=qwen25' + ], + image_name: null, + run_command: null, + env: null, + restart_on_error: true, + distributable: false, + extended_kv_cache: null, + speculative_config: null, + quantization: 'FP8', + mode: 'throughput' + }, + { + source: 'model_scope', + huggingface_repo_id: null, + huggingface_filename: null, + ollama_library_model_name: null, + model_scope_model_id: 'Qwen/Qwen3-30B-A3B-FP8', + model_scope_file_path: null, + local_path: null, + name: null, + description: null, + meta: {}, + replicas: 1, + ready_replicas: 0, + categories: [], + embedding_only: false, + image_only: false, + reranker: false, + speech_to_text: false, + text_to_speech: false, + placement_strategy: 'spread', + cpu_offloading: false, + distributed_inference_across_workers: false, + worker_selector: {}, + gpu_selector: null, + backend: 'SGLang', + backend_version: null, + backend_parameters: [ + '--reasoning-parser=qwen3', + '--tool-call-parser=qwen25', + '--speculative-algorithm=EAGLE3', + '--speculative-draft-model-path=Tengyunw/qwen3_30b_moe_eagle3', + '--speculative-num-steps=6', + '--speculative-eagle-topk=10', + '--speculative-num-draft-tokens=32' + ], + image_name: null, + run_command: null, + env: null, + restart_on_error: true, + distributable: false, + extended_kv_cache: null, + speculative_config: { + enabled: true, + algorithm: 'eagle3', + draft_model_name: 'Qwen3-30B-A3B-EAGLE3', + num_draft_tokens: 8, + ngram_min_match_length: null, + ngram_max_match_length: null + }, + quantization: 'FP8', + mode: 'latency' + }, + { + source: 'model_scope', + huggingface_repo_id: null, + huggingface_filename: null, + ollama_library_model_name: null, + model_scope_model_id: 'Qwen/Qwen3-30B-A3B', + model_scope_file_path: null, + local_path: null, + name: null, + description: null, + meta: {}, + replicas: 1, + ready_replicas: 0, + categories: [], + embedding_only: false, + image_only: false, + reranker: false, + speech_to_text: false, + text_to_speech: false, + placement_strategy: 'spread', + cpu_offloading: false, + distributed_inference_across_workers: true, + worker_selector: {}, + gpu_selector: null, + backend: 'vLLM', + backend_version: null, + backend_parameters: [ + '--tool-call-parser=hermes', + '--enable-auto-tool-choice', + '--max-model-len=32768' + ], + image_name: null, + run_command: null, + env: null, + restart_on_error: true, + distributable: false, + extended_kv_cache: null, + speculative_config: null, + quantization: 'BF16', + mode: 'reference' + } + ], + pagination: { page: 1, perPage: 100, total: 3, totalPage: 1 } +}; diff --git a/src/pages/llmodels/config/types.ts b/src/pages/llmodels/config/types.ts index 72362def..8b8d54fa 100644 --- a/src/pages/llmodels/config/types.ts +++ b/src/pages/llmodels/config/types.ts @@ -198,12 +198,27 @@ export interface CatalogSpec { categories: any[]; placement_strategy: string; cpu_offloading: boolean; + mode: string; distributed_inference_across_workers: boolean; worker_selector: Record; gpu_selector: { gpu_ids: string[]; gpus_per_replica: number; }; + extended_kv_cache: { + enabled: boolean; + chunk_size: number; + max_local_cpu_size: number; + remote_url: string; + }; + speculative_config: { + enabled: boolean; + algorithm: string; + draft_model_name: string; + num_draft_tokens: number; + ngram_min_match_length: number; + ngram_max_match_length: number; + }; backend: string; backend_version: string; backend_parameters: any[]; diff --git a/src/pages/llmodels/forms/performance.tsx b/src/pages/llmodels/forms/performance.tsx index 6a0fbf09..c7dd5b8a 100644 --- a/src/pages/llmodels/forms/performance.tsx +++ b/src/pages/llmodels/forms/performance.tsx @@ -1,10 +1,11 @@ +import AutoTooltip from '@/components/auto-tooltip'; import SealSelect from '@/components/seal-form/seal-select'; import TooltipList from '@/components/tooltip-list'; import { useIntl } from '@umijs/max'; -import { Form } from 'antd'; +import { Form, Select } from 'antd'; import React from 'react'; import { deployFormKeyMap } from '../config'; -import { useFormContext } from '../config/form-context'; +import { useCatalogFormContext, useFormContext } from '../config/form-context'; import KVCacheForm from './kv-cache'; import SpeculativeDecode from './speculative-decode'; @@ -36,6 +37,7 @@ const Performance: React.FC = () => { const intl = useIntl(); const form = Form.useFormInstance(); const { formKey } = useFormContext(); + const { modeList, onModeChange } = useCatalogFormContext(); return ( <> @@ -43,25 +45,24 @@ const Performance: React.FC = () => { {formKey === deployFormKeyMap.catalog && ( } label={intl.formatMessage({ id: 'models.form.mode' })} - options={[ - { - label: intl.formatMessage({ - id: 'models.form.mode.throughput' - }), - value: 'throughput' - }, - { - label: intl.formatMessage({ id: 'models.form.mode.latency' }), - value: 'latency' - }, - { - label: intl.formatMessage({ id: 'models.form.mode.reference' }), - value: 'reference' - } - ]} - > + > + {modeList.map((item: any) => ( + + + {item.isBuiltIn + ? intl.formatMessage({ id: item.label }) + : item.label} + + + ))} + )}