(
{} as FormOuterContextProps
);
+/**
+ * Hooks to use the form context
+ */
+
export const useFormContext = () => {
const context = React.useContext(FormContext);
if (!context) {
@@ -68,6 +75,11 @@ export const useCatalogFormContext = () => {
return context;
};
+/**
+ *
+ * Hooks to use the outer form context
+ */
+
export const useFormOuterContext = () => {
const context = React.useContext(FormOuterContext);
if (!context) {
diff --git a/src/pages/llmodels/config/types.ts b/src/pages/llmodels/config/types.ts
index 0aeeb866..72362def 100644
--- a/src/pages/llmodels/config/types.ts
+++ b/src/pages/llmodels/config/types.ts
@@ -82,6 +82,14 @@ export interface FormData {
ram_ratio: number;
ram_size: number;
};
+ speculative_config: {
+ enabled: boolean;
+ algorithm: string;
+ draft_model_name: string;
+ num_draft_tokens: number;
+ ngram_min_match_length: number;
+ ngram_max_match_length: number;
+ };
}
interface ComputedResourceClaim {
@@ -168,7 +176,8 @@ export interface CatalogItem {
icon: string;
categories: string[];
capabilities: string[];
- sizes: number[];
+ size: number;
+ activated_size: number;
licenses: string[];
release_date: string;
}
@@ -279,3 +288,15 @@ export interface BackendItem {
is_deprecated: boolean;
}[];
}
+
+export interface DraftModelItem {
+ source: string;
+ huggingface_repo_id: string;
+ huggingface_filename: string;
+ ollama_library_model_name: string;
+ model_scope_model_id: string;
+ model_scope_file_path: string;
+ local_path: string;
+ name: string;
+ algorithm: string;
+}
diff --git a/src/pages/llmodels/forms/advance-config.tsx b/src/pages/llmodels/forms/advance-config.tsx
index f93b9900..6b15f244 100644
--- a/src/pages/llmodels/forms/advance-config.tsx
+++ b/src/pages/llmodels/forms/advance-config.tsx
@@ -81,41 +81,35 @@ const AdvanceConfig = () => {
[backendOptionsMap.vllm, backendOptionsMap.ascendMindie].includes(
backend
) && (
-
-
- name="distributed_inference_across_workers"
- valuePropName="checked"
- style={{ padding: '0 10px', marginBottom: 0 }}
- noStyle
- >
-
-
-
+
+ name="distributed_inference_across_workers"
+ valuePropName="checked"
+ style={{ marginBottom: 8 }}
+ >
+
+
)}
-
-
- name="restart_on_error"
- valuePropName="checked"
- style={{ padding: '0 10px', marginBottom: 0 }}
- noStyle
- >
-
-
-
+
+ name="restart_on_error"
+ valuePropName="checked"
+ style={{ marginBottom: 8 }}
+ >
+
+
>
);
};
diff --git a/src/pages/llmodels/forms/index.tsx b/src/pages/llmodels/forms/index.tsx
index 1d212fe1..da862489 100644
--- a/src/pages/llmodels/forms/index.tsx
+++ b/src/pages/llmodels/forms/index.tsx
@@ -366,12 +366,21 @@ const DataForm: React.FC = forwardRef((props, ref) => {
categories: null,
restart_on_error: true,
distributed_inference_across_workers: true,
+ mode: 'throughput',
extended_kv_cache: {
enabled: false,
chunk_size: null,
ram_ratio: 1.2,
ram_size: null
},
+ speculative_config: {
+ enabled: false,
+ algorithm: '',
+ draft_model_name: null,
+ num_draft_tokens: 3,
+ ngram_min_match_length: 1,
+ ngram_max_match_length: 10
+ },
...initialValues
}}
>
diff --git a/src/pages/llmodels/forms/kv-cache.tsx b/src/pages/llmodels/forms/kv-cache.tsx
index 754caa19..f16f1b70 100644
--- a/src/pages/llmodels/forms/kv-cache.tsx
+++ b/src/pages/llmodels/forms/kv-cache.tsx
@@ -49,32 +49,30 @@ const KVCacheForm = () => {
return (
<>
-
-
- data-field="extended_kv_cache.enabled"
- name={['extended_kv_cache', 'enabled']}
- valuePropName="checked"
- style={{ padding: '0 10px', marginBottom: 0 }}
- extra={
- !builtInBackend && (
-
- )
- }
- >
-
-
-
+
+ data-field="extended_kv_cache.enabled"
+ name={['extended_kv_cache', 'enabled']}
+ valuePropName="checked"
+ style={{ marginBottom: 8 }}
+ extra={
+ !builtInBackend && (
+
+ )
+ }
+ >
+
+
{kvCacheEnabled && (
<>
name={['extended_kv_cache', 'ram_ratio']}>
diff --git a/src/pages/llmodels/forms/performance.tsx b/src/pages/llmodels/forms/performance.tsx
index 511f6938..9838d0e2 100644
--- a/src/pages/llmodels/forms/performance.tsx
+++ b/src/pages/llmodels/forms/performance.tsx
@@ -1,41 +1,51 @@
+import SealSelect from '@/components/seal-form/seal-select';
+import TooltipList from '@/components/tooltip-list';
import { useIntl } from '@umijs/max';
import { Form } from 'antd';
import React from 'react';
+import { deployFormKeyMap } from '../config';
+import { useFormContext } from '../config/form-context';
import KVCacheForm from './kv-cache';
+import SpeculativeDecode from './speculative-decode';
+
+const flavorTipsList = [
+ {
+ title: 'Throughput',
+ tips: 'models.form.flavor.throughput.tips'
+ },
+ {
+ title: 'Latency',
+ tips: 'models.form.flavor.latency.tips'
+ },
+ {
+ title: 'Reference',
+ tips: 'models.form.flavor.reference.tips'
+ }
+];
const Performance: React.FC = () => {
const intl = useIntl();
const form = Form.useFormInstance();
+ const { formKey } = useFormContext();
return (
<>
+
+ {formKey === deployFormKeyMap.catalog && (
+
+ }
+ label="Mode"
+ options={[
+ { label: 'Throughput', value: 'throughput' },
+ { label: 'Latency', value: 'latency' },
+ { label: 'Reference', value: 'reference' }
+ ]}
+ >
+
+ )}
- {/*
-
- name="optimize_long_prompt"
- valuePropName="checked"
- style={{ padding: '0 10px', marginBottom: 0 }}
- noStyle
- >
-
-
-
-
-
- name="enable_speculative_decoding"
- valuePropName="checked"
- style={{ padding: '0 10px', marginBottom: 0 }}
- noStyle
- >
-
-
- */}
+
>
);
};
diff --git a/src/pages/llmodels/forms/speculative-decode.tsx b/src/pages/llmodels/forms/speculative-decode.tsx
new file mode 100644
index 00000000..33b98e46
--- /dev/null
+++ b/src/pages/llmodels/forms/speculative-decode.tsx
@@ -0,0 +1,150 @@
+import CheckboxField from '@/components/seal-form/checkbox-field';
+import SealInputNumber from '@/components/seal-form/input-number';
+import SealInput from '@/components/seal-form/seal-input';
+import SealSelect from '@/components/seal-form/seal-select';
+import useAppUtils from '@/hooks/use-app-utils';
+import { useIntl } from '@umijs/max';
+import { Form } from 'antd';
+import { useEffect, useState } from 'react';
+import { queryDraftModelList } from '../apis';
+import { FormData } from '../config/types';
+
+const AlgorithmMap = {
+ Eagle3: 'eagle3',
+ MTP: 'mtp',
+ Ngram: 'ngram'
+};
+
+const SpeculativeDecode = () => {
+ const intl = useIntl();
+ const { getRuleMessage } = useAppUtils();
+ const form = Form.useFormInstance();
+ const speculativeEnabled = Form.useWatch(
+ ['speculative_config', 'enabled'],
+ form
+ );
+ const algorithm = Form.useWatch(['speculative_config', 'algorithm'], form);
+ const [draftModelList, setDraftModelList] = useState<
+ Global.BaseOption[]
+ >([]);
+
+ const fetchDraftModels = async () => {
+ const response = await queryDraftModelList({
+ page: 1,
+ perPage: 100
+ });
+ const options = response.items.map((item) => ({
+ label: item.name,
+ value: item.name
+ }));
+ setDraftModelList(options);
+ };
+
+ const handleSpeculativeEnabledChange = (e: any) => {
+ if (e.target.checked) {
+ form.setFieldValue('speculative_config', {
+ enabled: true,
+ algorithm: AlgorithmMap.Eagle3,
+ draft_model_name: null,
+ num_draft_tokens: 3,
+ ngram_min_match_length: 1,
+ ngram_max_match_length: 10
+ });
+ }
+ };
+
+ useEffect(() => {
+ if (algorithm === AlgorithmMap.Eagle3) {
+ fetchDraftModels();
+ }
+ }, [algorithm]);
+
+ return (
+ <>
+
+ name={['speculative_config', 'enabled']}
+ valuePropName="checked"
+ style={{ marginBottom: 8 }}
+ >
+
+
+ {speculativeEnabled && (
+ <>
+
+ name={['speculative_config', 'algorithm']}
+ rules={[
+ {
+ required: true,
+ message: getRuleMessage('select', 'Algorithm', false)
+ }
+ ]}
+ >
+
+
+ {algorithm === AlgorithmMap.Eagle3 && (
+
+ name={['speculative_config', 'draft_model_name']}
+ rules={[
+ {
+ required: true,
+ message: getRuleMessage('select', 'Draft Model', false)
+ }
+ ]}
+ >
+
+
+ )}
+
+ name={['speculative_config', 'num_draft_tokens']}
+ >
+
+
+ {algorithm === AlgorithmMap.Ngram && (
+ <>
+
+ name={['speculative_config', 'ngram_min_match_length']}
+ >
+
+
+
+ name={['speculative_config', 'ngram_max_match_length']}
+ >
+
+
+ >
+ )}
+ >
+ )}
+ >
+ );
+};
+
+export default SpeculativeDecode;
diff --git a/src/pages/llmodels/style/catalog-item.less b/src/pages/llmodels/style/catalog-item.less
index de110f6a..e0b28eef 100644
--- a/src/pages/llmodels/style/catalog-item.less
+++ b/src/pages/llmodels/style/catalog-item.less
@@ -1,10 +1,10 @@
.catalog-item {
- height: 180px;
+ height: 150px;
overflow: hidden;
display: flex;
padding: 16px 20px;
flex-direction: column;
- justify-content: flex-start;
+ justify-content: space-between;
align-items: flex-start;
border: 1px solid var(--ant-color-border);
border-radius: var(--border-radius-base);
@@ -12,15 +12,12 @@
width: 100%;
.img {
- width: 32px;
display: flex;
justify-content: center;
align-items: center;
- height: 100%;
img {
height: 32px;
- width: 32px;
object-fit: contain;
}
}
@@ -29,7 +26,6 @@
display: flex;
flex-direction: column;
justify-content: flex-start;
- flex: 1;
width: 100%;
}
@@ -47,8 +43,7 @@
display: flex;
align-items: center;
justify-content: flex-start;
- gap: 12px;
- margin-bottom: 12px;
+ gap: 16px;
.name {
flex: 1;
@@ -67,7 +62,7 @@
width: 100%;
display: flex;
flex-direction: column;
- gap: 12px;
+ gap: 16px;
justify-content: flex-start;
align-items: flex-start;
}