feat: add speculative decoding

src/assets/styles/common.less

@@ -169,6 +169,10 @@
   gap: 8px;
 }
 
+.gap-4 {
+  gap: 4px;
+}
+
 .gap-20 {
   gap: 20px;
 }

src/locales/en-US/models.ts

@@ -218,5 +218,11 @@ export default {
   'models.form.ramRatio.tips':
     'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.',
   'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`,
-  'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.'
+  'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.',
+  'models.form.flavor.throughput.tips':
+    'optimized for high throughput under high request concurrency.',
+  'models.form.flavor.latency.tips':
+    'optimized for low latency under low request concurrency.',
+  'models.form.flavor.reference.tips':
+    'the most compatible option with full precision.'
 };

src/locales/ja-JP/models.ts

@@ -218,7 +218,13 @@ export default {
   'models.form.ramRatio.tips':
     'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.',
   'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`,
-  'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.'
+  'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk.',
+  'models.form.flavor.throughput.tips':
+    'optimized for high throughput under high request concurrency.',
+  'models.form.flavor.latency.tips':
+    'optimized for low latency under low request concurrency.',
+  'models.form.flavor.reference.tips':
+    'the most compatible option with full precision.'
 };
 
 // ========== To-Do: Translate Keys (Remove After Translation) ==========

src/pages/llmodels/apis/index.ts

@@ -8,6 +8,7 @@ import {
   BackendItem,
   CatalogItem,
   CatalogSpec,
+  DraftModelItem,
   EvaluateResult,
   EvaluateSpec,
   FormData,
@@ -27,6 +28,8 @@ export const BACKEND_LIST_API = '/inference-backends/list';
 
 export const MY_MODELS_API = '/my-models';
 
+export const DRAFT_MODELS_API = '/draft-models';
+
 const setProxyUrl = (url: string) => {
   return `/proxy?url=${encodeURIComponent(url)}`;
 };
@@ -443,3 +446,10 @@ export async function queryMyModelDetail(id: number) {
     method: 'GET'
   });
 }
+
+export async function queryDraftModelList(params?: Global.SearchParams) {
+  return request<{ items: DraftModelItem[] }>(DRAFT_MODELS_API, {
+    method: 'GET',
+    params
+  });
+}

src/pages/llmodels/catalog.tsx

@@ -10,7 +10,7 @@ import { SearchOutlined, SyncOutlined } from '@ant-design/icons';
 import { PageContainer } from '@ant-design/pro-components';
 import { useIntl, useNavigate } from '@umijs/max';
 import { useMemoizedFn } from 'ahooks';
-import { Button, Input, Pagination, Space, message } from 'antd';
+import { Button, Input, Space, message } from 'antd';
 import { useAtom } from 'jotai';
 import _ from 'lodash';
 import React, { useCallback, useEffect, useState } from 'react';
@@ -159,17 +159,6 @@ const Catalog: React.FC = () => {
     [openDeployModal]
   );
 
-  const handleOnPageChange = useCallback(
-    (page: number, pageSize?: number) => {
-      setQueryParams({
-        ...queryParams,
-        page,
-        perPage: pageSize || 10
-      });
-    },
-    [queryParams]
-  );
-
   const handleSearch = (e: any) => {
     fetchData({
       ...queryParams,
@@ -312,17 +301,6 @@ const Catalog: React.FC = () => {
           subTitle={intl.formatMessage({ id: 'noresult.catalog.subTitle' })}
         ></NoResult>
       </ScrollerContext.Provider>
-      <PageWrapper>
-        <Pagination
-          hideOnSinglePage={queryParams.perPage === 100}
-          align="end"
-          defaultCurrent={1}
-          total={dataSource.total}
-          pageSize={queryParams.perPage}
-          showSizeChanger
-          onChange={handleOnPageChange}
-        />
-      </PageWrapper>
       <DelopyBuiltInModal
         open={openDeployModal.show}
         action={PageAction.CREATE}

src/pages/llmodels/components/catalog-item.tsx

@@ -1,16 +1,16 @@
 import fallbackImg from '@/assets/images/img.png';
 import AutoTooltip from '@/components/auto-tooltip';
 import IconFont from '@/components/icon-font';
-import TagWrapper from '@/components/tags-wrapper';
 import ThemeTag from '@/components/tags-wrapper/theme-tag';
 import { useIntl } from '@umijs/max';
 import { Typography } from 'antd';
 import classNames from 'classnames';
 import _ from 'lodash';
-import React, { useCallback } from 'react';
+import React, { useCallback, useMemo } from 'react';
 import { modelCategories } from '../config';
 import { CatalogItem as CatalogItemType } from '../config/types';
 import '../style/catalog-item.less';
+import { categoryConfig } from './model-tag';
 
 const COLORS = ['blue', 'purple', 'orange'];
 interface CatalogItemProps {
@@ -47,11 +47,38 @@ const CatalogItem: React.FC<CatalogItemProps> = (props) => {
           height: 22
         }}
       >
-        {sItem}B
+        {sItem.label}
       </ThemeTag>
     );
   }, []);
 
+  const description = useMemo(() => {
+    return (
+      <Typography.Paragraph
+        className="desc"
+        ellipsis={{
+          rows: 2,
+          tooltip: (
+            <div
+              className="custome-scrollbar"
+              style={{
+                display: 'flex',
+                justifyContent: 'flex-start',
+                maxHeight: 300,
+                maxWidth: 300,
+                overflow: 'auto'
+              }}
+            >
+              {data.description}
+            </div>
+          )
+        }}
+      >
+        {data.description}
+      </Typography.Paragraph>
+    );
+  }, [data.description]);
+
   return (
     <div
       onClick={handleOnClick}
@@ -66,32 +93,8 @@ const CatalogItem: React.FC<CatalogItemProps> = (props) => {
               onError={handleOnError}
             />
           </div>
-          <AutoTooltip ghost style={{ flex: 1 }}>
+          <AutoTooltip ghost>{data.name}</AutoTooltip>
-            {data.name}
-          </AutoTooltip>
         </div>
-        <Typography.Paragraph
-          className="desc"
-          ellipsis={{
-            rows: 2,
-            tooltip: (
-              <div
-                className="custome-scrollbar"
-                style={{
-                  display: 'flex',
-                  justifyContent: 'flex-start',
-                  maxHeight: 300,
-                  maxWidth: 300,
-                  overflow: 'auto'
-                }}
-              >
-                {data.description}
-              </div>
-            )
-          }}
-        >
-          {data.description}
-        </Typography.Paragraph>
       </div>
       <div className="item-footer">
         <div className="update-time">
@@ -121,9 +124,10 @@ const CatalogItem: React.FC<CatalogItemProps> = (props) => {
           {data.categories.map((sItem, i) => {
             return (
               <ThemeTag
+                icon={categoryConfig[sItem]?.icon}
                 key={sItem}
                 className="tag-item"
-                color="blue"
+                color={categoryConfig[sItem]?.color || 'blue'}
                 opacity={0.7}
               >
                 {_.find(modelCategories, { value: sItem })?.label || sItem}
@@ -147,22 +151,16 @@ const CatalogItem: React.FC<CatalogItemProps> = (props) => {
                 </ThemeTag>
               );
             })}
-          {data.sizes?.length > 0 && (
+          <span className="dot"></span>
-            <>
+          <ThemeTag>
-              <span className="dot"></span>
+            {data.activated_size
-              <div className="box">
+              ? `${data.size}B-A${data.activated_size}B`
-                <TagWrapper
+              : `${data.size}B`}
-                  gap={8}
+          </ThemeTag>
-                  dataList={data.sizes}
-                  renderTag={renderTag}
-                ></TagWrapper>
-              </div>
-            </>
-          )}
         </div>
       </div>
     </div>
   );
 };
 
-export default React.memo(CatalogItem);
+export default CatalogItem;

src/pages/llmodels/components/deploy-builtin-modal.tsx

@@ -540,8 +540,8 @@ const AddModal: React.FC<AddModalProps> = (props) => {
     >
       <CatalogFormContext.Provider
         value={{
-          sizeOptions: sizeOptions,
+          sizeOptions: [],
-          quantizationOptions: quantizationOptions,
+          quantizationOptions: [],
           onSizeChange: handleOnSizeChange,
           onQuantizationChange: handleOnQuantizationChange
         }}

src/pages/llmodels/config/form-context.ts

@@ -42,6 +42,9 @@ export const FormContext = React.createContext<FormContextProps>(
   {} as FormContextProps
 );
 
+/**
+ * Catalog form context
+ */
 export const CatalogFormContext = React.createContext<CatalogFormContextProps>(
   {} as CatalogFormContextProps
 );
@@ -50,6 +53,10 @@ export const FormOuterContext = React.createContext<FormOuterContextProps>(
   {} as FormOuterContextProps
 );
 
+/**
+ * Hooks to use the form context
+ */
+
 export const useFormContext = () => {
   const context = React.useContext(FormContext);
   if (!context) {
@@ -68,6 +75,11 @@ export const useCatalogFormContext = () => {
   return context;
 };
 
+/**
+ *
+ * Hooks to use the outer form context
+ */
+
 export const useFormOuterContext = () => {
   const context = React.useContext(FormOuterContext);
   if (!context) {

src/pages/llmodels/config/types.ts

@@ -82,6 +82,14 @@ export interface FormData {
     ram_ratio: number;
     ram_size: number;
   };
+  speculative_config: {
+    enabled: boolean;
+    algorithm: string;
+    draft_model_name: string;
+    num_draft_tokens: number;
+    ngram_min_match_length: number;
+    ngram_max_match_length: number;
+  };
 }
 
 interface ComputedResourceClaim {
@@ -168,7 +176,8 @@ export interface CatalogItem {
   icon: string;
   categories: string[];
   capabilities: string[];
-  sizes: number[];
+  size: number;
+  activated_size: number;
   licenses: string[];
   release_date: string;
 }
@@ -279,3 +288,15 @@ export interface BackendItem {
     is_deprecated: boolean;
   }[];
 }
+
+export interface DraftModelItem {
+  source: string;
+  huggingface_repo_id: string;
+  huggingface_filename: string;
+  ollama_library_model_name: string;
+  model_scope_model_id: string;
+  model_scope_file_path: string;
+  local_path: string;
+  name: string;
+  algorithm: string;
+}

src/pages/llmodels/forms/advance-config.tsx

@@ -81,41 +81,35 @@ const AdvanceConfig = () => {
         [backendOptionsMap.vllm, backendOptionsMap.ascendMindie].includes(
           backend
         ) && (
-          <div style={{ paddingBottom: 22 }}>
+          <Form.Item<FormData>
-            <Form.Item<FormData>
+            name="distributed_inference_across_workers"
-              name="distributed_inference_across_workers"
+            valuePropName="checked"
-              valuePropName="checked"
+            style={{ marginBottom: 8 }}
-              style={{ padding: '0 10px', marginBottom: 0 }}
+          >
-              noStyle
+            <CheckboxField
-            >
+              description={intl.formatMessage({
-              <CheckboxField
+                id: 'models.form.distribution.tips'
-                description={intl.formatMessage({
+              })}
-                  id: 'models.form.distribution.tips'
+              label={intl.formatMessage({
-                })}
+                id: 'resources.form.enableDistributedInferenceAcrossWorkers'
-                label={intl.formatMessage({
+              })}
-                  id: 'resources.form.enableDistributedInferenceAcrossWorkers'
+            ></CheckboxField>
-                })}
+          </Form.Item>
-              ></CheckboxField>
-            </Form.Item>
-          </div>
         )}
-      <div style={{ paddingBottom: 22 }}>
+      <Form.Item<FormData>
-        <Form.Item<FormData>
+        name="restart_on_error"
-          name="restart_on_error"
+        valuePropName="checked"
-          valuePropName="checked"
+        style={{ marginBottom: 8 }}
-          style={{ padding: '0 10px', marginBottom: 0 }}
+      >
-          noStyle
+        <CheckboxField
-        >
+          description={intl.formatMessage({
-          <CheckboxField
+            id: 'models.form.restart.onerror.tips'
-            description={intl.formatMessage({
+          })}
-              id: 'models.form.restart.onerror.tips'
+          label={intl.formatMessage({
-            })}
+            id: 'models.form.restart.onerror'
-            label={intl.formatMessage({
+          })}
-              id: 'models.form.restart.onerror'
+        ></CheckboxField>
-            })}
+      </Form.Item>
-          ></CheckboxField>
-        </Form.Item>
-      </div>
     </>
   );
 };

src/pages/llmodels/forms/index.tsx

@@ -366,12 +366,21 @@ const DataForm: React.FC<DataFormProps> = forwardRef((props, ref) => {
           categories: null,
           restart_on_error: true,
           distributed_inference_across_workers: true,
+          mode: 'throughput',
           extended_kv_cache: {
             enabled: false,
             chunk_size: null,
             ram_ratio: 1.2,
             ram_size: null
           },
+          speculative_config: {
+            enabled: false,
+            algorithm: '',
+            draft_model_name: null,
+            num_draft_tokens: 3,
+            ngram_min_match_length: 1,
+            ngram_max_match_length: 10
+          },
           ...initialValues
         }}
       >

src/pages/llmodels/forms/kv-cache.tsx

@@ -49,32 +49,30 @@ const KVCacheForm = () => {
 
   return (
     <>
-      <div style={{ paddingBottom: 22 }}>
+      <Form.Item<FormData>
-        <Form.Item<FormData>
+        data-field="extended_kv_cache.enabled"
-          data-field="extended_kv_cache.enabled"
+        name={['extended_kv_cache', 'enabled']}
-          name={['extended_kv_cache', 'enabled']}
+        valuePropName="checked"
-          valuePropName="checked"
+        style={{ marginBottom: 8 }}
-          style={{ padding: '0 10px', marginBottom: 0 }}
+        extra={
-          extra={
+          !builtInBackend && (
-            !builtInBackend && (
+            <span
-              <span
+              dangerouslySetInnerHTML={{
-                dangerouslySetInnerHTML={{
+                __html: intl.formatMessage({ id: 'models.form.kvCache.tips' })
-                  __html: intl.formatMessage({ id: 'models.form.kvCache.tips' })
+              }}
-                }}
+            ></span>
-              ></span>
+          )
-            )
+        }
-          }
+      >
-        >
+        <CheckboxField
-          <CheckboxField
+          description={intl.formatMessage({
-            description={intl.formatMessage({
+            id: 'models.form.kvCache.tips2'
-              id: 'models.form.kvCache.tips2'
+          })}
-            })}
+          disabled={!builtInBackend}
-            disabled={!builtInBackend}
+          onChange={handleOnChange}
-            onChange={handleOnChange}
+          label={intl.formatMessage({ id: 'models.form.extendedkvcache' })}
-            label={intl.formatMessage({ id: 'models.form.extendedkvcache' })}
+        ></CheckboxField>
-          ></CheckboxField>
+      </Form.Item>
-        </Form.Item>
-      </div>
       {kvCacheEnabled && (
         <>
           <Form.Item<FormData> name={['extended_kv_cache', 'ram_ratio']}>

src/pages/llmodels/forms/performance.tsx

@@ -1,41 +1,51 @@
+import SealSelect from '@/components/seal-form/seal-select';
+import TooltipList from '@/components/tooltip-list';
 import { useIntl } from '@umijs/max';
 import { Form } from 'antd';
 import React from 'react';
+import { deployFormKeyMap } from '../config';
+import { useFormContext } from '../config/form-context';
 import KVCacheForm from './kv-cache';
+import SpeculativeDecode from './speculative-decode';
+
+const flavorTipsList = [
+  {
+    title: 'Throughput',
+    tips: 'models.form.flavor.throughput.tips'
+  },
+  {
+    title: 'Latency',
+    tips: 'models.form.flavor.latency.tips'
+  },
+  {
+    title: 'Reference',
+    tips: 'models.form.flavor.reference.tips'
+  }
+];
 
 const Performance: React.FC = () => {
   const intl = useIntl();
   const form = Form.useFormInstance();
+  const { formKey } = useFormContext();
 
   return (
     <>
+      <div data-field="extended_kv_cache.enabled"></div>
+      {formKey === deployFormKeyMap.catalog && (
+        <Form.Item name="mode">
+          <SealSelect
+            description={<TooltipList list={flavorTipsList}></TooltipList>}
+            label="Mode"
+            options={[
+              { label: 'Throughput', value: 'throughput' },
+              { label: 'Latency', value: 'latency' },
+              { label: 'Reference', value: 'reference' }
+            ]}
+          ></SealSelect>
+        </Form.Item>
+      )}
       <KVCacheForm></KVCacheForm>
-      {/* <div style={{ paddingBottom: 22, paddingLeft: 10 }}>
+      <SpeculativeDecode></SpeculativeDecode>
-        <Form.Item<FormData>
-          name="optimize_long_prompt"
-          valuePropName="checked"
-          style={{ padding: '0 10px', marginBottom: 0 }}
-          noStyle
-        >
-          <CheckboxField
-            label={intl.formatMessage({ id: 'models.form.optimizeLongPrompt' })}
-          ></CheckboxField>
-        </Form.Item>
-      </div>
-      <div style={{ paddingBottom: 22, paddingLeft: 10 }}>
-        <Form.Item<FormData>
-          name="enable_speculative_decoding"
-          valuePropName="checked"
-          style={{ padding: '0 10px', marginBottom: 0 }}
-          noStyle
-        >
-          <CheckboxField
-            label={intl.formatMessage({
-              id: 'models.form.enableSpeculativeDecoding'
-            })}
-          ></CheckboxField>
-        </Form.Item>
-      </div> */}
     </>
   );
 };

src/pages/llmodels/forms/speculative-decode.tsx

@@ -0,0 +1,150 @@
+import CheckboxField from '@/components/seal-form/checkbox-field';
+import SealInputNumber from '@/components/seal-form/input-number';
+import SealInput from '@/components/seal-form/seal-input';
+import SealSelect from '@/components/seal-form/seal-select';
+import useAppUtils from '@/hooks/use-app-utils';
+import { useIntl } from '@umijs/max';
+import { Form } from 'antd';
+import { useEffect, useState } from 'react';
+import { queryDraftModelList } from '../apis';
+import { FormData } from '../config/types';
+
+const AlgorithmMap = {
+  Eagle3: 'eagle3',
+  MTP: 'mtp',
+  Ngram: 'ngram'
+};
+
+const SpeculativeDecode = () => {
+  const intl = useIntl();
+  const { getRuleMessage } = useAppUtils();
+  const form = Form.useFormInstance();
+  const speculativeEnabled = Form.useWatch(
+    ['speculative_config', 'enabled'],
+    form
+  );
+  const algorithm = Form.useWatch(['speculative_config', 'algorithm'], form);
+  const [draftModelList, setDraftModelList] = useState<
+    Global.BaseOption<string>[]
+  >([]);
+
+  const fetchDraftModels = async () => {
+    const response = await queryDraftModelList({
+      page: 1,
+      perPage: 100
+    });
+    const options = response.items.map((item) => ({
+      label: item.name,
+      value: item.name
+    }));
+    setDraftModelList(options);
+  };
+
+  const handleSpeculativeEnabledChange = (e: any) => {
+    if (e.target.checked) {
+      form.setFieldValue('speculative_config', {
+        enabled: true,
+        algorithm: AlgorithmMap.Eagle3,
+        draft_model_name: null,
+        num_draft_tokens: 3,
+        ngram_min_match_length: 1,
+        ngram_max_match_length: 10
+      });
+    }
+  };
+
+  useEffect(() => {
+    if (algorithm === AlgorithmMap.Eagle3) {
+      fetchDraftModels();
+    }
+  }, [algorithm]);
+
+  return (
+    <>
+      <Form.Item<FormData>
+        name={['speculative_config', 'enabled']}
+        valuePropName="checked"
+        style={{ marginBottom: 8 }}
+      >
+        <CheckboxField
+          label={'Enable Speculative Decoding'}
+          onChange={handleSpeculativeEnabledChange}
+        ></CheckboxField>
+      </Form.Item>
+      {speculativeEnabled && (
+        <>
+          <Form.Item<FormData>
+            name={['speculative_config', 'algorithm']}
+            rules={[
+              {
+                required: true,
+                message: getRuleMessage('select', 'Algorithm', false)
+              }
+            ]}
+          >
+            <SealSelect
+              required
+              label={'Algorithm'}
+              options={[
+                { label: 'Eagle3', value: AlgorithmMap.Eagle3 },
+                { label: 'MTP', value: AlgorithmMap.MTP },
+                { label: 'N-gram', value: AlgorithmMap.Ngram }
+              ]}
+            ></SealSelect>
+          </Form.Item>
+          {algorithm === AlgorithmMap.Eagle3 && (
+            <Form.Item<FormData>
+              name={['speculative_config', 'draft_model_name']}
+              rules={[
+                {
+                  required: true,
+                  message: getRuleMessage('select', 'Draft Model', false)
+                }
+              ]}
+            >
+              <SealSelect
+                required
+                label={'Draft Model'}
+                options={draftModelList}
+              ></SealSelect>
+            </Form.Item>
+          )}
+          <Form.Item<FormData>
+            name={['speculative_config', 'num_draft_tokens']}
+          >
+            <SealInputNumber
+              label={'Number of Draft Tokens'}
+              min={1}
+              step={1}
+              precision={0}
+            />
+          </Form.Item>
+          {algorithm === AlgorithmMap.Ngram && (
+            <>
+              <Form.Item<FormData>
+                name={['speculative_config', 'ngram_min_match_length']}
+              >
+                <SealInputNumber
+                  label={'Minimum N-gram Match Length'}
+                  min={1}
+                  step={1}
+                />
+              </Form.Item>
+              <Form.Item<FormData>
+                name={['speculative_config', 'ngram_max_match_length']}
+              >
+                <SealInput.Input
+                  label={'Maximum N-gram Match Length'}
+                  min={2}
+                  step={1}
+                />
+              </Form.Item>
+            </>
+          )}
+        </>
+      )}
+    </>
+  );
+};
+
+export default SpeculativeDecode;

src/pages/llmodels/style/catalog-item.less

@@ -1,10 +1,10 @@
 .catalog-item {
-  height: 180px;
+  height: 150px;
   overflow: hidden;
   display: flex;
   padding: 16px 20px;
   flex-direction: column;
-  justify-content: flex-start;
+  justify-content: space-between;
   align-items: flex-start;
   border: 1px solid var(--ant-color-border);
   border-radius: var(--border-radius-base);
@@ -12,15 +12,12 @@
   width: 100%;
 
   .img {
-    width: 32px;
     display: flex;
     justify-content: center;
     align-items: center;
-    height: 100%;
 
     img {
       height: 32px;
-      width: 32px;
       object-fit: contain;
     }
   }
@@ -29,7 +26,6 @@
     display: flex;
     flex-direction: column;
     justify-content: flex-start;
-    flex: 1;
     width: 100%;
   }
 
@@ -47,8 +43,7 @@
     display: flex;
     align-items: center;
     justify-content: flex-start;
-    gap: 12px;
+    gap: 16px;
-    margin-bottom: 12px;
 
     .name {
       flex: 1;
@@ -67,7 +62,7 @@
     width: 100%;
     display: flex;
     flex-direction: column;
-    gap: 12px;
+    gap: 16px;
     justify-content: flex-start;
     align-items: flex-start;
   }