feat: add speculative decoding

src/pages/llmodels/forms/advance-config.tsx

@@ -81,41 +81,35 @@ const AdvanceConfig = () => {
         [backendOptionsMap.vllm, backendOptionsMap.ascendMindie].includes(
           backend
         ) && (
-          <div style={{ paddingBottom: 22 }}>
-            <Form.Item<FormData>
-              name="distributed_inference_across_workers"
-              valuePropName="checked"
-              style={{ padding: '0 10px', marginBottom: 0 }}
-              noStyle
-            >
-              <CheckboxField
-                description={intl.formatMessage({
-                  id: 'models.form.distribution.tips'
-                })}
-                label={intl.formatMessage({
-                  id: 'resources.form.enableDistributedInferenceAcrossWorkers'
-                })}
-              ></CheckboxField>
-            </Form.Item>
-          </div>
+          <Form.Item<FormData>
+            name="distributed_inference_across_workers"
+            valuePropName="checked"
+            style={{ marginBottom: 8 }}
+          >
+            <CheckboxField
+              description={intl.formatMessage({
+                id: 'models.form.distribution.tips'
+              })}
+              label={intl.formatMessage({
+                id: 'resources.form.enableDistributedInferenceAcrossWorkers'
+              })}
+            ></CheckboxField>
+          </Form.Item>
         )}
-      <div style={{ paddingBottom: 22 }}>
-        <Form.Item<FormData>
-          name="restart_on_error"
-          valuePropName="checked"
-          style={{ padding: '0 10px', marginBottom: 0 }}
-          noStyle
-        >
-          <CheckboxField
-            description={intl.formatMessage({
-              id: 'models.form.restart.onerror.tips'
-            })}
-            label={intl.formatMessage({
-              id: 'models.form.restart.onerror'
-            })}
-          ></CheckboxField>
-        </Form.Item>
-      </div>
+      <Form.Item<FormData>
+        name="restart_on_error"
+        valuePropName="checked"
+        style={{ marginBottom: 8 }}
+      >
+        <CheckboxField
+          description={intl.formatMessage({
+            id: 'models.form.restart.onerror.tips'
+          })}
+          label={intl.formatMessage({
+            id: 'models.form.restart.onerror'
+          })}
+        ></CheckboxField>
+      </Form.Item>
     </>
   );
 };

src/pages/llmodels/forms/index.tsx

@@ -366,12 +366,21 @@ const DataForm: React.FC<DataFormProps> = forwardRef((props, ref) => {
           categories: null,
           restart_on_error: true,
           distributed_inference_across_workers: true,
+          mode: 'throughput',
           extended_kv_cache: {
             enabled: false,
             chunk_size: null,
             ram_ratio: 1.2,
             ram_size: null
           },
+          speculative_config: {
+            enabled: false,
+            algorithm: '',
+            draft_model_name: null,
+            num_draft_tokens: 3,
+            ngram_min_match_length: 1,
+            ngram_max_match_length: 10
+          },
           ...initialValues
         }}
       >

src/pages/llmodels/forms/kv-cache.tsx

@@ -49,32 +49,30 @@ const KVCacheForm = () => {
 
   return (
     <>
-      <div style={{ paddingBottom: 22 }}>
-        <Form.Item<FormData>
-          data-field="extended_kv_cache.enabled"
-          name={['extended_kv_cache', 'enabled']}
-          valuePropName="checked"
-          style={{ padding: '0 10px', marginBottom: 0 }}
-          extra={
-            !builtInBackend && (
-              <span
-                dangerouslySetInnerHTML={{
-                  __html: intl.formatMessage({ id: 'models.form.kvCache.tips' })
-                }}
-              ></span>
-            )
-          }
-        >
-          <CheckboxField
-            description={intl.formatMessage({
-              id: 'models.form.kvCache.tips2'
-            })}
-            disabled={!builtInBackend}
-            onChange={handleOnChange}
-            label={intl.formatMessage({ id: 'models.form.extendedkvcache' })}
-          ></CheckboxField>
-        </Form.Item>
-      </div>
+      <Form.Item<FormData>
+        data-field="extended_kv_cache.enabled"
+        name={['extended_kv_cache', 'enabled']}
+        valuePropName="checked"
+        style={{ marginBottom: 8 }}
+        extra={
+          !builtInBackend && (
+            <span
+              dangerouslySetInnerHTML={{
+                __html: intl.formatMessage({ id: 'models.form.kvCache.tips' })
+              }}
+            ></span>
+          )
+        }
+      >
+        <CheckboxField
+          description={intl.formatMessage({
+            id: 'models.form.kvCache.tips2'
+          })}
+          disabled={!builtInBackend}
+          onChange={handleOnChange}
+          label={intl.formatMessage({ id: 'models.form.extendedkvcache' })}
+        ></CheckboxField>
+      </Form.Item>
       {kvCacheEnabled && (
         <>
           <Form.Item<FormData> name={['extended_kv_cache', 'ram_ratio']}>

src/pages/llmodels/forms/performance.tsx

@@ -1,41 +1,51 @@
+import SealSelect from '@/components/seal-form/seal-select';
+import TooltipList from '@/components/tooltip-list';
 import { useIntl } from '@umijs/max';
 import { Form } from 'antd';
 import React from 'react';
+import { deployFormKeyMap } from '../config';
+import { useFormContext } from '../config/form-context';
 import KVCacheForm from './kv-cache';
+import SpeculativeDecode from './speculative-decode';
+
+const flavorTipsList = [
+  {
+    title: 'Throughput',
+    tips: 'models.form.flavor.throughput.tips'
+  },
+  {
+    title: 'Latency',
+    tips: 'models.form.flavor.latency.tips'
+  },
+  {
+    title: 'Reference',
+    tips: 'models.form.flavor.reference.tips'
+  }
+];
 
 const Performance: React.FC = () => {
   const intl = useIntl();
   const form = Form.useFormInstance();
+  const { formKey } = useFormContext();
 
   return (
     <>
+      <div data-field="extended_kv_cache.enabled"></div>
+      {formKey === deployFormKeyMap.catalog && (
+        <Form.Item name="mode">
+          <SealSelect
+            description={<TooltipList list={flavorTipsList}></TooltipList>}
+            label="Mode"
+            options={[
+              { label: 'Throughput', value: 'throughput' },
+              { label: 'Latency', value: 'latency' },
+              { label: 'Reference', value: 'reference' }
+            ]}
+          ></SealSelect>
+        </Form.Item>
+      )}
       <KVCacheForm></KVCacheForm>
-      {/* <div style={{ paddingBottom: 22, paddingLeft: 10 }}>
-        <Form.Item<FormData>
-          name="optimize_long_prompt"
-          valuePropName="checked"
-          style={{ padding: '0 10px', marginBottom: 0 }}
-          noStyle
-        >
-          <CheckboxField
-            label={intl.formatMessage({ id: 'models.form.optimizeLongPrompt' })}
-          ></CheckboxField>
-        </Form.Item>
-      </div>
-      <div style={{ paddingBottom: 22, paddingLeft: 10 }}>
-        <Form.Item<FormData>
-          name="enable_speculative_decoding"
-          valuePropName="checked"
-          style={{ padding: '0 10px', marginBottom: 0 }}
-          noStyle
-        >
-          <CheckboxField
-            label={intl.formatMessage({
-              id: 'models.form.enableSpeculativeDecoding'
-            })}
-          ></CheckboxField>
-        </Form.Item>
-      </div> */}
+      <SpeculativeDecode></SpeculativeDecode>
     </>
   );
 };

src/pages/llmodels/forms/speculative-decode.tsx

@@ -0,0 +1,150 @@
+import CheckboxField from '@/components/seal-form/checkbox-field';
+import SealInputNumber from '@/components/seal-form/input-number';
+import SealInput from '@/components/seal-form/seal-input';
+import SealSelect from '@/components/seal-form/seal-select';
+import useAppUtils from '@/hooks/use-app-utils';
+import { useIntl } from '@umijs/max';
+import { Form } from 'antd';
+import { useEffect, useState } from 'react';
+import { queryDraftModelList } from '../apis';
+import { FormData } from '../config/types';
+
+const AlgorithmMap = {
+  Eagle3: 'eagle3',
+  MTP: 'mtp',
+  Ngram: 'ngram'
+};
+
+const SpeculativeDecode = () => {
+  const intl = useIntl();
+  const { getRuleMessage } = useAppUtils();
+  const form = Form.useFormInstance();
+  const speculativeEnabled = Form.useWatch(
+    ['speculative_config', 'enabled'],
+    form
+  );
+  const algorithm = Form.useWatch(['speculative_config', 'algorithm'], form);
+  const [draftModelList, setDraftModelList] = useState<
+    Global.BaseOption<string>[]
+  >([]);
+
+  const fetchDraftModels = async () => {
+    const response = await queryDraftModelList({
+      page: 1,
+      perPage: 100
+    });
+    const options = response.items.map((item) => ({
+      label: item.name,
+      value: item.name
+    }));
+    setDraftModelList(options);
+  };
+
+  const handleSpeculativeEnabledChange = (e: any) => {
+    if (e.target.checked) {
+      form.setFieldValue('speculative_config', {
+        enabled: true,
+        algorithm: AlgorithmMap.Eagle3,
+        draft_model_name: null,
+        num_draft_tokens: 3,
+        ngram_min_match_length: 1,
+        ngram_max_match_length: 10
+      });
+    }
+  };
+
+  useEffect(() => {
+    if (algorithm === AlgorithmMap.Eagle3) {
+      fetchDraftModels();
+    }
+  }, [algorithm]);
+
+  return (
+    <>
+      <Form.Item<FormData>
+        name={['speculative_config', 'enabled']}
+        valuePropName="checked"
+        style={{ marginBottom: 8 }}
+      >
+        <CheckboxField
+          label={'Enable Speculative Decoding'}
+          onChange={handleSpeculativeEnabledChange}
+        ></CheckboxField>
+      </Form.Item>
+      {speculativeEnabled && (
+        <>
+          <Form.Item<FormData>
+            name={['speculative_config', 'algorithm']}
+            rules={[
+              {
+                required: true,
+                message: getRuleMessage('select', 'Algorithm', false)
+              }
+            ]}
+          >
+            <SealSelect
+              required
+              label={'Algorithm'}
+              options={[
+                { label: 'Eagle3', value: AlgorithmMap.Eagle3 },
+                { label: 'MTP', value: AlgorithmMap.MTP },
+                { label: 'N-gram', value: AlgorithmMap.Ngram }
+              ]}
+            ></SealSelect>
+          </Form.Item>
+          {algorithm === AlgorithmMap.Eagle3 && (
+            <Form.Item<FormData>
+              name={['speculative_config', 'draft_model_name']}
+              rules={[
+                {
+                  required: true,
+                  message: getRuleMessage('select', 'Draft Model', false)
+                }
+              ]}
+            >
+              <SealSelect
+                required
+                label={'Draft Model'}
+                options={draftModelList}
+              ></SealSelect>
+            </Form.Item>
+          )}
+          <Form.Item<FormData>
+            name={['speculative_config', 'num_draft_tokens']}
+          >
+            <SealInputNumber
+              label={'Number of Draft Tokens'}
+              min={1}
+              step={1}
+              precision={0}
+            />
+          </Form.Item>
+          {algorithm === AlgorithmMap.Ngram && (
+            <>
+              <Form.Item<FormData>
+                name={['speculative_config', 'ngram_min_match_length']}
+              >
+                <SealInputNumber
+                  label={'Minimum N-gram Match Length'}
+                  min={1}
+                  step={1}
+                />
+              </Form.Item>
+              <Form.Item<FormData>
+                name={['speculative_config', 'ngram_max_match_length']}
+              >
+                <SealInput.Input
+                  label={'Maximum N-gram Match Length'}
+                  min={2}
+                  step={1}
+                />
+              </Form.Item>
+            </>
+          )}
+        </>
+      )}
+    </>
+  );
+};
+
+export default SpeculativeDecode;