diff --git a/src/pages/llmodels/apis/index.ts b/src/pages/llmodels/apis/index.ts
index 348a1a07..02336cd9 100644
--- a/src/pages/llmodels/apis/index.ts
+++ b/src/pages/llmodels/apis/index.ts
@@ -255,20 +255,22 @@ export async function queryModelScopeModelFiles(
// list models from huggingface
export async function queryHuggingfaceModels(
params: {
+ limit?: number;
search: {
query: string;
- tags: string[];
+ tags?: string[];
sort?: string;
task?: PipelineType;
};
},
options?: any
) {
+ console.log('params', params);
const result = [];
for await (const model of listModels({
...params,
...options,
- limit: 500,
+ limit: params.limit || 500,
additionalFields: ['sha', 'tags'],
fetch(_url: string, config: any) {
const url = params.search.sort
diff --git a/src/pages/llmodels/config/index.ts b/src/pages/llmodels/config/index.ts
index c6c7285f..a6bdf342 100644
--- a/src/pages/llmodels/config/index.ts
+++ b/src/pages/llmodels/config/index.ts
@@ -349,7 +349,8 @@ export const DO_NOT_TRIGGER_CHECK_COMPATIBILITY = [
'gpu_selector.gpu_ids',
'run_command',
'image_name',
- 'extended_kv_cache.enabled'
+ 'extended_kv_cache.enabled',
+ 'speculative_config.draft_model'
];
// ignore to compare old and new data when these fields change in updating model
diff --git a/src/pages/llmodels/forms/performance.tsx b/src/pages/llmodels/forms/performance.tsx
index aa367749..39748319 100644
--- a/src/pages/llmodels/forms/performance.tsx
+++ b/src/pages/llmodels/forms/performance.tsx
@@ -3,7 +3,7 @@ import SealSelect from '@/components/seal-form/seal-select';
import { useIntl } from '@umijs/max';
import { Form, Select } from 'antd';
import React from 'react';
-import { deployFormKeyMap } from '../config';
+import { DeployFormKeyMap } from '../config';
import { useCatalogFormContext, useFormContext } from '../config/form-context';
import KVCacheForm from './kv-cache';
import SpeculativeDecode from './speculative-decode';
@@ -41,7 +41,7 @@ const Performance: React.FC = () => {
return (
<>
- {formKey === deployFormKeyMap.catalog && (
+ {formKey === DeployFormKeyMap.CATALOG && (
{
const intl = useIntl();
+ const { source } = useFormContext();
const { getRuleMessage } = useAppUtils();
const form = Form.useFormInstance();
const speculativeEnabled = Form.useWatch(
@@ -26,9 +35,12 @@ const SpeculativeDecode = () => {
);
const algorithm = Form.useWatch(['speculative_config', 'algorithm'], form);
const [draftModelList, setDraftModelList] = useState<
- Global.BaseOption[]
+ Global.BaseOption[] }>[]
>([]);
+ const presetDraftModelListRef = useRef[]>([]);
const speculativeConfigRef = useRef({});
+ const axiosTokenRef = useRef(null);
+ const [loading, setLoading] = useState(false);
const fetchDraftModels = async () => {
const response = await queryDraftModelList({
@@ -39,9 +51,128 @@ const SpeculativeDecode = () => {
label: item.name,
value: item.name
}));
+ presetDraftModelListRef.current = options;
setDraftModelList(options);
};
+ const getHuggingfaceModels = async (query: string) => {
+ if (axiosTokenRef.current) {
+ axiosTokenRef.current.abort();
+ }
+ axiosTokenRef.current = new AbortController();
+ try {
+ const params = {
+ limit: 10,
+ search: {
+ query: query
+ }
+ };
+ setLoading(true);
+ const data = await queryHuggingfaceModels(params, {
+ signal: axiosTokenRef.current.signal
+ });
+ const list = _.map(data || [], (item: any) => {
+ return {
+ value: item.name,
+ label: item.name
+ };
+ });
+
+ const catalogModelList =
+ presetDraftModelListRef.current.length > 0
+ ? [
+ {
+ label: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`,
+ title: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`,
+ options: presetDraftModelListRef.current || []
+ }
+ ]
+ : [];
+
+ setDraftModelList([
+ ...catalogModelList,
+ {
+ label: `${intl.formatMessage({ id: 'models.form.source' })}: Hugging Face`,
+ title: `${intl.formatMessage({ id: 'models.form.source' })}: Hugging Face`,
+ options: list
+ }
+ ]);
+ } catch (error) {
+ setDraftModelList(presetDraftModelListRef.current);
+ } finally {
+ setLoading(false);
+ }
+ };
+
+ const getModelScopeModels = async (query: string) => {
+ if (axiosTokenRef.current) {
+ axiosTokenRef.current.abort();
+ }
+ axiosTokenRef.current = new AbortController();
+ try {
+ const params = {
+ Name: query,
+ PageSize: 10,
+ PageNumber: 1,
+ tasks: []
+ };
+ setLoading(true);
+ const data = await queryModelScopeModels(params, {
+ signal: axiosTokenRef.current.signal
+ });
+ const list = _.map(
+ _.get(data, 'Data.Model.Models') || [],
+ (item: any) => {
+ return {
+ label: `${item.Path}/${item.Name}`,
+ value: `${item.Path}/${item.Name}`
+ };
+ }
+ );
+
+ const catalogModelList =
+ presetDraftModelListRef.current.length > 0
+ ? [
+ {
+ label: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`,
+ title: `${intl.formatMessage({ id: 'models.form.source' })}: ${intl.formatMessage({ id: 'menu.models.modelCatalog' })}`,
+ options: presetDraftModelListRef.current || []
+ }
+ ]
+ : [];
+
+ setDraftModelList([
+ ...catalogModelList,
+ {
+ label: `${intl.formatMessage({ id: 'models.form.source' })}: ModelScope`,
+ title: `${intl.formatMessage({ id: 'models.form.source' })}: ModelScope`,
+ options: list
+ }
+ ]);
+ } catch (error) {
+ setDraftModelList(presetDraftModelListRef.current);
+ } finally {
+ setLoading(false);
+ }
+ };
+
+ const handleOnSearch = async (value: string) => {
+ if (!value) {
+ setDraftModelList(presetDraftModelListRef.current);
+ return;
+ }
+ if (source === modelSourceMap.huggingface_value) {
+ await getHuggingfaceModels(value);
+ } else if (source === modelSourceMap.modelscope_value) {
+ await getModelScopeModels(value);
+ }
+ };
+
+ const { run: onSearch } = useDeferredRequest(
+ (value: string) => handleOnSearch(value),
+ 150
+ );
+
const handleSpeculativeEnabledChange = (e: any) => {
if (e.target.checked) {
form.setFieldValue('speculative_config', {
@@ -129,6 +260,7 @@ const SpeculativeDecode = () => {
id: 'models.form.draftModel.tips'
})}
options={draftModelList}
+ onSearch={onSearch}
>
)}