From 1a92cc8543874d5fac0d946554596e576b23562d Mon Sep 17 00:00:00 2001 From: jialin Date: Fri, 31 Oct 2025 20:26:58 +0800 Subject: [PATCH] feat: search draft models from remote --- src/pages/llmodels/apis/index.ts | 6 +- src/pages/llmodels/config/index.ts | 3 +- src/pages/llmodels/forms/performance.tsx | 4 +- .../llmodels/forms/speculative-decode.tsx | 136 +++++++++++++++++- 4 files changed, 142 insertions(+), 7 deletions(-) diff --git a/src/pages/llmodels/apis/index.ts b/src/pages/llmodels/apis/index.ts index 348a1a07..02336cd9 100644 --- a/src/pages/llmodels/apis/index.ts +++ b/src/pages/llmodels/apis/index.ts @@ -255,20 +255,22 @@ export async function queryModelScopeModelFiles( // list models from huggingface export async function queryHuggingfaceModels( params: { + limit?: number; search: { query: string; - tags: string[]; + tags?: string[]; sort?: string; task?: PipelineType; }; }, options?: any ) { + console.log('params', params); const result = []; for await (const model of listModels({ ...params, ...options, - limit: 500, + limit: params.limit || 500, additionalFields: ['sha', 'tags'], fetch(_url: string, config: any) { const url = params.search.sort diff --git a/src/pages/llmodels/config/index.ts b/src/pages/llmodels/config/index.ts index c6c7285f..a6bdf342 100644 --- a/src/pages/llmodels/config/index.ts +++ b/src/pages/llmodels/config/index.ts @@ -349,7 +349,8 @@ export const DO_NOT_TRIGGER_CHECK_COMPATIBILITY = [ 'gpu_selector.gpu_ids', 'run_command', 'image_name', - 'extended_kv_cache.enabled' + 'extended_kv_cache.enabled', + 'speculative_config.draft_model' ]; // ignore to compare old and new data when these fields change in updating model diff --git a/src/pages/llmodels/forms/performance.tsx b/src/pages/llmodels/forms/performance.tsx index aa367749..39748319 100644 --- a/src/pages/llmodels/forms/performance.tsx +++ b/src/pages/llmodels/forms/performance.tsx @@ -3,7 +3,7 @@ import SealSelect from '@/components/seal-form/seal-select'; import { useIntl } from '@umijs/max'; import { Form, Select } from 'antd'; import React from 'react'; -import { deployFormKeyMap } from '../config'; +import { DeployFormKeyMap } from '../config'; import { useCatalogFormContext, useFormContext } from '../config/form-context'; import KVCacheForm from './kv-cache'; import SpeculativeDecode from './speculative-decode'; @@ -41,7 +41,7 @@ const Performance: React.FC = () => { return ( <>
- {formKey === deployFormKeyMap.catalog && ( + {formKey === DeployFormKeyMap.CATALOG && ( { const intl = useIntl(); + const { source } = useFormContext(); const { getRuleMessage } = useAppUtils(); const form = Form.useFormInstance(); const speculativeEnabled = Form.useWatch( @@ -26,9 +35,12 @@ const SpeculativeDecode = () => { ); const algorithm = Form.useWatch(['speculative_config', 'algorithm'], form); const [draftModelList, setDraftModelList] = useState< - Global.BaseOption[] + Global.BaseOption[] }>[] >([]); + const presetDraftModelListRef = useRef[]>([]); const speculativeConfigRef = useRef({}); + const axiosTokenRef = useRef(null); + const [loading, setLoading] = useState(false); const fetchDraftModels = async () => { const response = await queryDraftModelList({ @@ -39,9 +51,128 @@ const SpeculativeDecode = () => { label: item.name, value: item.name })); + presetDraftModelListRef.current = options; setDraftModelList(options); }; + const getHuggingfaceModels = async (query: string) => { + if (axiosTokenRef.current) { + axiosTokenRef.current.abort(); + } + axiosTokenRef.current = new AbortController(); + try { + const params = { + limit: 10, + search: { + query: query + } + }; + setLoading(true); + const data = await queryHuggingfaceModels(params, { + signal: axiosTokenRef.current.signal + }); + const list = _.map(data || [], (item: any) => { + return { + value: item.name, + label: item.name + }; + }); + + const catalogModelList = + presetDraftModelListRef.current.length > 0 + ? [ + { + label: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`, + title: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`, + options: presetDraftModelListRef.current || [] + } + ] + : []; + + setDraftModelList([ + ...catalogModelList, + { + label: `${intl.formatMessage({ id: 'models.form.source' })}: Hugging Face`, + title: `${intl.formatMessage({ id: 'models.form.source' })}: Hugging Face`, + options: list + } + ]); + } catch (error) { + setDraftModelList(presetDraftModelListRef.current); + } finally { + setLoading(false); + } + }; + + const getModelScopeModels = async (query: string) => { + if (axiosTokenRef.current) { + axiosTokenRef.current.abort(); + } + axiosTokenRef.current = new AbortController(); + try { + const params = { + Name: query, + PageSize: 10, + PageNumber: 1, + tasks: [] + }; + setLoading(true); + const data = await queryModelScopeModels(params, { + signal: axiosTokenRef.current.signal + }); + const list = _.map( + _.get(data, 'Data.Model.Models') || [], + (item: any) => { + return { + label: `${item.Path}/${item.Name}`, + value: `${item.Path}/${item.Name}` + }; + } + ); + + const catalogModelList = + presetDraftModelListRef.current.length > 0 + ? [ + { + label: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`, + title: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`, + options: presetDraftModelListRef.current || [] + } + ] + : []; + + setDraftModelList([ + ...catalogModelList, + { + label: `${intl.formatMessage({ id: 'models.form.source' })}: ModelScope`, + title: `${intl.formatMessage({ id: 'models.form.source' })}: ModelScope`, + options: list + } + ]); + } catch (error) { + setDraftModelList(presetDraftModelListRef.current); + } finally { + setLoading(false); + } + }; + + const handleOnSearch = async (value: string) => { + if (!value) { + setDraftModelList(presetDraftModelListRef.current); + return; + } + if (source === modelSourceMap.huggingface_value) { + await getHuggingfaceModels(value); + } else if (source === modelSourceMap.modelscope_value) { + await getModelScopeModels(value); + } + }; + + const { run: onSearch } = useDeferredRequest( + (value: string) => handleOnSearch(value), + 150 + ); + const handleSpeculativeEnabledChange = (e: any) => { if (e.target.checked) { form.setFieldValue('speculative_config', { @@ -129,6 +260,7 @@ const SpeculativeDecode = () => { id: 'models.form.draftModel.tips' })} options={draftModelList} + onSearch={onSearch} > )}