From 3613fb3f696b8559692ff3a922b06039c2de924f Mon Sep 17 00:00:00 2001 From: jialin Date: Fri, 17 Oct 2025 15:11:20 +0800 Subject: [PATCH] feat: add speculative decoding --- src/assets/styles/common.less | 4 + src/locales/en-US/models.ts | 8 +- src/locales/ja-JP/models.ts | 8 +- src/pages/llmodels/apis/index.ts | 10 ++ src/pages/llmodels/catalog.tsx | 24 +-- .../llmodels/components/catalog-item.tsx | 82 +++++----- .../components/deploy-builtin-modal.tsx | 4 +- src/pages/llmodels/config/form-context.ts | 12 ++ src/pages/llmodels/config/types.ts | 23 ++- src/pages/llmodels/forms/advance-config.tsx | 62 ++++---- src/pages/llmodels/forms/index.tsx | 9 ++ src/pages/llmodels/forms/kv-cache.tsx | 50 +++--- src/pages/llmodels/forms/performance.tsx | 62 +++++--- .../llmodels/forms/speculative-decode.tsx | 150 ++++++++++++++++++ src/pages/llmodels/style/catalog-item.less | 13 +- 15 files changed, 356 insertions(+), 165 deletions(-) create mode 100644 src/pages/llmodels/forms/speculative-decode.tsx diff --git a/src/assets/styles/common.less b/src/assets/styles/common.less index 60af24d8..6c22912e 100644 --- a/src/assets/styles/common.less +++ b/src/assets/styles/common.less @@ -169,6 +169,10 @@ gap: 8px; } +.gap-4 { + gap: 4px; +} + .gap-20 { gap: 20px; } diff --git a/src/locales/en-US/models.ts b/src/locales/en-US/models.ts index 5e01881b..0f74d1aa 100644 --- a/src/locales/en-US/models.ts +++ b/src/locales/en-US/models.ts @@ -218,5 +218,11 @@ export default { 'models.form.ramRatio.tips': 'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.', 'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`, - 'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.' + 'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.', + 'models.form.flavor.throughput.tips': + 'optimized for high throughput under high request concurrency.', + 'models.form.flavor.latency.tips': + 'optimized for low latency under low request concurrency.', + 'models.form.flavor.reference.tips': + 'the most compatible option with full precision.' }; diff --git a/src/locales/ja-JP/models.ts b/src/locales/ja-JP/models.ts index 72d38252..29f9d4f6 100644 --- a/src/locales/ja-JP/models.ts +++ b/src/locales/ja-JP/models.ts @@ -218,7 +218,13 @@ export default { 'models.form.ramRatio.tips': 'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.', 'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`, - 'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.' + 'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.', + 'models.form.flavor.throughput.tips': + 'optimized for high throughput under high request concurrency.', + 'models.form.flavor.latency.tips': + 'optimized for low latency under low request concurrency.', + 'models.form.flavor.reference.tips': + 'the most compatible option with full precision.' }; // ========== To-Do: Translate Keys (Remove After Translation) ========== diff --git a/src/pages/llmodels/apis/index.ts b/src/pages/llmodels/apis/index.ts index b3fdeabb..348a1a07 100644 --- a/src/pages/llmodels/apis/index.ts +++ b/src/pages/llmodels/apis/index.ts @@ -8,6 +8,7 @@ import { BackendItem, CatalogItem, CatalogSpec, + DraftModelItem, EvaluateResult, EvaluateSpec, FormData, @@ -27,6 +28,8 @@ export const BACKEND_LIST_API = '/inference-backends/list'; export const MY_MODELS_API = '/my-models'; +export const DRAFT_MODELS_API = '/draft-models'; + const setProxyUrl = (url: string) => { return `/proxy?url=${encodeURIComponent(url)}`; }; @@ -443,3 +446,10 @@ export async function queryMyModelDetail(id: number) { method: 'GET' }); } + +export async function queryDraftModelList(params?: Global.SearchParams) { + return request<{ items: DraftModelItem[] }>(DRAFT_MODELS_API, { + method: 'GET', + params + }); +} diff --git a/src/pages/llmodels/catalog.tsx b/src/pages/llmodels/catalog.tsx index eda06b3a..62fe13ec 100644 --- a/src/pages/llmodels/catalog.tsx +++ b/src/pages/llmodels/catalog.tsx @@ -10,7 +10,7 @@ import { SearchOutlined, SyncOutlined } from '@ant-design/icons'; import { PageContainer } from '@ant-design/pro-components'; import { useIntl, useNavigate } from '@umijs/max'; import { useMemoizedFn } from 'ahooks'; -import { Button, Input, Pagination, Space, message } from 'antd'; +import { Button, Input, Space, message } from 'antd'; import { useAtom } from 'jotai'; import _ from 'lodash'; import React, { useCallback, useEffect, useState } from 'react'; @@ -159,17 +159,6 @@ const Catalog: React.FC = () => { [openDeployModal] ); - const handleOnPageChange = useCallback( - (page: number, pageSize?: number) => { - setQueryParams({ - ...queryParams, - page, - perPage: pageSize || 10 - }); - }, - [queryParams] - ); - const handleSearch = (e: any) => { fetchData({ ...queryParams, @@ -312,17 +301,6 @@ const Catalog: React.FC = () => { subTitle={intl.formatMessage({ id: 'noresult.catalog.subTitle' })} > - - - = (props) => { height: 22 }} > - {sItem}B + {sItem.label} ); }, []); + const description = useMemo(() => { + return ( + + {data.description} + + ) + }} + > + {data.description} + + ); + }, [data.description]); + return (
= (props) => { onError={handleOnError} />
- - {data.name} - + {data.name} - - {data.description} - - ) - }} - > - {data.description} -
@@ -121,9 +124,10 @@ const CatalogItem: React.FC = (props) => { {data.categories.map((sItem, i) => { return ( {_.find(modelCategories, { value: sItem })?.label || sItem} @@ -147,22 +151,16 @@ const CatalogItem: React.FC = (props) => { ); })} - {data.sizes?.length > 0 && ( - <> - -
- -
- - )} + + + {data.activated_size + ? `${data.size}B-A${data.activated_size}B` + : `${data.size}B`} +
); }; -export default React.memo(CatalogItem); +export default CatalogItem; diff --git a/src/pages/llmodels/components/deploy-builtin-modal.tsx b/src/pages/llmodels/components/deploy-builtin-modal.tsx index e21d0c34..7ff73833 100644 --- a/src/pages/llmodels/components/deploy-builtin-modal.tsx +++ b/src/pages/llmodels/components/deploy-builtin-modal.tsx @@ -540,8 +540,8 @@ const AddModal: React.FC = (props) => { > ( {} as FormContextProps ); +/** + * Catalog form context + */ export const CatalogFormContext = React.createContext( {} as CatalogFormContextProps ); @@ -50,6 +53,10 @@ export const FormOuterContext = React.createContext( {} as FormOuterContextProps ); +/** + * Hooks to use the form context + */ + export const useFormContext = () => { const context = React.useContext(FormContext); if (!context) { @@ -68,6 +75,11 @@ export const useCatalogFormContext = () => { return context; }; +/** + * + * Hooks to use the outer form context + */ + export const useFormOuterContext = () => { const context = React.useContext(FormOuterContext); if (!context) { diff --git a/src/pages/llmodels/config/types.ts b/src/pages/llmodels/config/types.ts index 0aeeb866..72362def 100644 --- a/src/pages/llmodels/config/types.ts +++ b/src/pages/llmodels/config/types.ts @@ -82,6 +82,14 @@ export interface FormData { ram_ratio: number; ram_size: number; }; + speculative_config: { + enabled: boolean; + algorithm: string; + draft_model_name: string; + num_draft_tokens: number; + ngram_min_match_length: number; + ngram_max_match_length: number; + }; } interface ComputedResourceClaim { @@ -168,7 +176,8 @@ export interface CatalogItem { icon: string; categories: string[]; capabilities: string[]; - sizes: number[]; + size: number; + activated_size: number; licenses: string[]; release_date: string; } @@ -279,3 +288,15 @@ export interface BackendItem { is_deprecated: boolean; }[]; } + +export interface DraftModelItem { + source: string; + huggingface_repo_id: string; + huggingface_filename: string; + ollama_library_model_name: string; + model_scope_model_id: string; + model_scope_file_path: string; + local_path: string; + name: string; + algorithm: string; +} diff --git a/src/pages/llmodels/forms/advance-config.tsx b/src/pages/llmodels/forms/advance-config.tsx index f93b9900..6b15f244 100644 --- a/src/pages/llmodels/forms/advance-config.tsx +++ b/src/pages/llmodels/forms/advance-config.tsx @@ -81,41 +81,35 @@ const AdvanceConfig = () => { [backendOptionsMap.vllm, backendOptionsMap.ascendMindie].includes( backend ) && ( -
- - name="distributed_inference_across_workers" - valuePropName="checked" - style={{ padding: '0 10px', marginBottom: 0 }} - noStyle - > - - -
+ + name="distributed_inference_across_workers" + valuePropName="checked" + style={{ marginBottom: 8 }} + > + + )} -
- - name="restart_on_error" - valuePropName="checked" - style={{ padding: '0 10px', marginBottom: 0 }} - noStyle - > - - -
+ + name="restart_on_error" + valuePropName="checked" + style={{ marginBottom: 8 }} + > + + ); }; diff --git a/src/pages/llmodels/forms/index.tsx b/src/pages/llmodels/forms/index.tsx index 1d212fe1..da862489 100644 --- a/src/pages/llmodels/forms/index.tsx +++ b/src/pages/llmodels/forms/index.tsx @@ -366,12 +366,21 @@ const DataForm: React.FC = forwardRef((props, ref) => { categories: null, restart_on_error: true, distributed_inference_across_workers: true, + mode: 'throughput', extended_kv_cache: { enabled: false, chunk_size: null, ram_ratio: 1.2, ram_size: null }, + speculative_config: { + enabled: false, + algorithm: '', + draft_model_name: null, + num_draft_tokens: 3, + ngram_min_match_length: 1, + ngram_max_match_length: 10 + }, ...initialValues }} > diff --git a/src/pages/llmodels/forms/kv-cache.tsx b/src/pages/llmodels/forms/kv-cache.tsx index 754caa19..f16f1b70 100644 --- a/src/pages/llmodels/forms/kv-cache.tsx +++ b/src/pages/llmodels/forms/kv-cache.tsx @@ -49,32 +49,30 @@ const KVCacheForm = () => { return ( <> -
- - data-field="extended_kv_cache.enabled" - name={['extended_kv_cache', 'enabled']} - valuePropName="checked" - style={{ padding: '0 10px', marginBottom: 0 }} - extra={ - !builtInBackend && ( - - ) - } - > - - -
+ + data-field="extended_kv_cache.enabled" + name={['extended_kv_cache', 'enabled']} + valuePropName="checked" + style={{ marginBottom: 8 }} + extra={ + !builtInBackend && ( + + ) + } + > + + {kvCacheEnabled && ( <> name={['extended_kv_cache', 'ram_ratio']}> diff --git a/src/pages/llmodels/forms/performance.tsx b/src/pages/llmodels/forms/performance.tsx index 511f6938..9838d0e2 100644 --- a/src/pages/llmodels/forms/performance.tsx +++ b/src/pages/llmodels/forms/performance.tsx @@ -1,41 +1,51 @@ +import SealSelect from '@/components/seal-form/seal-select'; +import TooltipList from '@/components/tooltip-list'; import { useIntl } from '@umijs/max'; import { Form } from 'antd'; import React from 'react'; +import { deployFormKeyMap } from '../config'; +import { useFormContext } from '../config/form-context'; import KVCacheForm from './kv-cache'; +import SpeculativeDecode from './speculative-decode'; + +const flavorTipsList = [ + { + title: 'Throughput', + tips: 'models.form.flavor.throughput.tips' + }, + { + title: 'Latency', + tips: 'models.form.flavor.latency.tips' + }, + { + title: 'Reference', + tips: 'models.form.flavor.reference.tips' + } +]; const Performance: React.FC = () => { const intl = useIntl(); const form = Form.useFormInstance(); + const { formKey } = useFormContext(); return ( <> +
+ {formKey === deployFormKeyMap.catalog && ( + + } + label="Mode" + options={[ + { label: 'Throughput', value: 'throughput' }, + { label: 'Latency', value: 'latency' }, + { label: 'Reference', value: 'reference' } + ]} + > + + )} - {/*
- - name="optimize_long_prompt" - valuePropName="checked" - style={{ padding: '0 10px', marginBottom: 0 }} - noStyle - > - - -
-
- - name="enable_speculative_decoding" - valuePropName="checked" - style={{ padding: '0 10px', marginBottom: 0 }} - noStyle - > - - -
*/} + ); }; diff --git a/src/pages/llmodels/forms/speculative-decode.tsx b/src/pages/llmodels/forms/speculative-decode.tsx new file mode 100644 index 00000000..33b98e46 --- /dev/null +++ b/src/pages/llmodels/forms/speculative-decode.tsx @@ -0,0 +1,150 @@ +import CheckboxField from '@/components/seal-form/checkbox-field'; +import SealInputNumber from '@/components/seal-form/input-number'; +import SealInput from '@/components/seal-form/seal-input'; +import SealSelect from '@/components/seal-form/seal-select'; +import useAppUtils from '@/hooks/use-app-utils'; +import { useIntl } from '@umijs/max'; +import { Form } from 'antd'; +import { useEffect, useState } from 'react'; +import { queryDraftModelList } from '../apis'; +import { FormData } from '../config/types'; + +const AlgorithmMap = { + Eagle3: 'eagle3', + MTP: 'mtp', + Ngram: 'ngram' +}; + +const SpeculativeDecode = () => { + const intl = useIntl(); + const { getRuleMessage } = useAppUtils(); + const form = Form.useFormInstance(); + const speculativeEnabled = Form.useWatch( + ['speculative_config', 'enabled'], + form + ); + const algorithm = Form.useWatch(['speculative_config', 'algorithm'], form); + const [draftModelList, setDraftModelList] = useState< + Global.BaseOption[] + >([]); + + const fetchDraftModels = async () => { + const response = await queryDraftModelList({ + page: 1, + perPage: 100 + }); + const options = response.items.map((item) => ({ + label: item.name, + value: item.name + })); + setDraftModelList(options); + }; + + const handleSpeculativeEnabledChange = (e: any) => { + if (e.target.checked) { + form.setFieldValue('speculative_config', { + enabled: true, + algorithm: AlgorithmMap.Eagle3, + draft_model_name: null, + num_draft_tokens: 3, + ngram_min_match_length: 1, + ngram_max_match_length: 10 + }); + } + }; + + useEffect(() => { + if (algorithm === AlgorithmMap.Eagle3) { + fetchDraftModels(); + } + }, [algorithm]); + + return ( + <> + + name={['speculative_config', 'enabled']} + valuePropName="checked" + style={{ marginBottom: 8 }} + > + + + {speculativeEnabled && ( + <> + + name={['speculative_config', 'algorithm']} + rules={[ + { + required: true, + message: getRuleMessage('select', 'Algorithm', false) + } + ]} + > + + + {algorithm === AlgorithmMap.Eagle3 && ( + + name={['speculative_config', 'draft_model_name']} + rules={[ + { + required: true, + message: getRuleMessage('select', 'Draft Model', false) + } + ]} + > + + + )} + + name={['speculative_config', 'num_draft_tokens']} + > + + + {algorithm === AlgorithmMap.Ngram && ( + <> + + name={['speculative_config', 'ngram_min_match_length']} + > + + + + name={['speculative_config', 'ngram_max_match_length']} + > + + + + )} + + )} + + ); +}; + +export default SpeculativeDecode; diff --git a/src/pages/llmodels/style/catalog-item.less b/src/pages/llmodels/style/catalog-item.less index de110f6a..e0b28eef 100644 --- a/src/pages/llmodels/style/catalog-item.less +++ b/src/pages/llmodels/style/catalog-item.less @@ -1,10 +1,10 @@ .catalog-item { - height: 180px; + height: 150px; overflow: hidden; display: flex; padding: 16px 20px; flex-direction: column; - justify-content: flex-start; + justify-content: space-between; align-items: flex-start; border: 1px solid var(--ant-color-border); border-radius: var(--border-radius-base); @@ -12,15 +12,12 @@ width: 100%; .img { - width: 32px; display: flex; justify-content: center; align-items: center; - height: 100%; img { height: 32px; - width: 32px; object-fit: contain; } } @@ -29,7 +26,6 @@ display: flex; flex-direction: column; justify-content: flex-start; - flex: 1; width: 100%; } @@ -47,8 +43,7 @@ display: flex; align-items: center; justify-content: flex-start; - gap: 12px; - margin-bottom: 12px; + gap: 16px; .name { flex: 1; @@ -67,7 +62,7 @@ width: 100%; display: flex; flex-direction: column; - gap: 12px; + gap: 16px; justify-content: flex-start; align-items: flex-start; }