root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: kv cache support in builtin backend

Browse Source
This commit is contained in:
jialin
2025-10-27 14:13:47 +08:00
parent 6969cb0b9c
commit df65d3e2f6
15 changed files with 124 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/atoms/models.ts
+1
View File
@@ -30,6 +30,7 @@ export const backendOptionsAtom = atom<
label: string;
default_backend_param: string[];
default_version: string;
isBuiltIn: boolean;
versions: { label: string; value: string }[];
}[]
>([]);
src/components/seal-form/checkbox-field.tsx
+10 -8
View File
@@ -1,16 +1,18 @@
import { QuestionCircleOutlined } from '@ant-design/icons';
import { Checkbox, Tooltip } from 'antd';
import { Checkbox, CheckboxProps, Tooltip } from 'antd';
import { CheckboxChangeEvent } from 'antd/es/checkbox';
import React from 'react';
const CheckboxField: React.FC<{
description?: React.ReactNode;
label: React.ReactNode;
checked?: boolean;
onChange?: (e: CheckboxChangeEvent) => void;
}> = ({ description, label, checked, onChange }) => {
const CheckboxField: React.FC<
{
description?: React.ReactNode;
label: React.ReactNode;
checked?: boolean;
onChange?: (e: CheckboxChangeEvent) => void;
} & CheckboxProps
> = ({ description, label, checked, onChange, ...rest }) => {
return (
<Checkbox className="p-l-6" checked={checked} onChange={onChange}>
<Checkbox className="p-l-6" checked={checked} onChange={onChange} {...rest}>
<Tooltip title={description || false}>
<span style={{ color: 'var(--ant-color-text-tertiary)' }}>{label}</span>
{!!description && (
src/locales/en-US/models.ts
+7 -1
View File
@@ -187,6 +187,8 @@ export default {
'models.form.chunkSize': 'Size Of Cache Chunks',
'models.form.maxCPUSize': 'Maximum CPU Cache Size (GiB)',
'models.form.remoteURL': 'Remote Storage URL',
'models.form.remoteURL.tips':
'Refer to the <a href="https://docs.lmcache.ai/api_reference/configurations.html" target="_blank">configuration documentation</a> for details.',
'models.form.runCommandPlaceholder':
'e.g., vllm serve Qwen/Qwen2.5-1.5B-Instruct',
'models.accessSettings.public': 'Public',
@@ -206,5 +208,9 @@ export default {
'You can specify the exact number of GPUs per replica.',
'models.mymodels.status.inactive': 'Stopped',
'models.mymodels.status.degrade': 'Abnormal',
'models.mymodels.status.active': 'Active'
'models.mymodels.status.active': 'Active',
'models.form.kvCache.tips':
'Available only with built-in backends (vLLM / SGLang) — switch backend in <span class="bold-text">Advanced</span> to enable.',
'models.form.kvCache.tips2':
'KV cache is only supported when using built-in inference backends (vLLM or SGLang).'
};
src/locales/ja-JP/models.ts
+10 -2
View File
@@ -184,6 +184,8 @@ export default {
'models.form.chunkSize': 'Size Of Cache Chunks',
'models.form.maxCPUSize': 'Maximum CPU Cache Size (GiB)',
'models.form.remoteURL': 'Remote Storage URL',
'models.form.remoteURL.tips':
'Refer to the <a href="https://docs.lmcache.ai/api_reference/configurations.html" target="_blank">configuration documentation</a> for details.',
'models.form.runCommandPlaceholder':
'e.g., vllm serve Qwen/Qwen2.5-1.5B-Instruct',
'models.accessSettings.public': 'Public',
@@ -203,7 +205,11 @@ export default {
'You can specify the exact number of GPUs per replica.',
'models.mymodels.status.inactive': 'Stopped',
'models.mymodels.status.degrade': 'Abnormal',
'models.mymodels.status.active': 'Active'
'models.mymodels.status.active': 'Active',
'models.form.kvCache.tips':
'Available only with built-in backends (vLLM / SGLang) — switch backend in <span class="bold-text">Advanced</span> to enable.',
'models.form.kvCache.tips2':
'KV cache is only supported when using built-in inference backends (vLLM or SGLang).'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
@@ -247,5 +253,7 @@ export default {
// 39. 'models.mymodels.status.inactive': 'Stopped',
// 41. 'models.mymodels.status.degrade': 'Abnormal',
// 42. 'models.mymodels.status.active': 'Active'
// 43. 'models.form.remoteURL.tips': 'Refer to the <a href="https://docs.lmcache.ai/api_reference/configurations.html" target="_blank">configuration documentation</a> for details.',
// 44. 'models.form.kvCache.tips': 'Available only with built-in backends (vLLM / SGLang) — switch backend in <span class="bold-text">Advanced</span> to enable.'
// 45. 'models.form.kvCache.tips2': 'KV cache is only supported when using built-in inference backends (vLLM or SGLang).'
// ========== End of To-Do List ==========
src/locales/ru-RU/models.ts
+10 -1
View File
@@ -187,6 +187,8 @@ export default {
'models.form.chunkSize': 'Размер чанков кэша',
'models.form.maxCPUSize': 'Максимальный размер CPU кэша (ГиБ)',
'models.form.remoteURL': 'URL удаленного хранилища',
'models.form.remoteURL.tips':
'Refer to the <a href="https://docs.lmcache.ai/api_reference/configurations.html" target="_blank">configuration documentation</a> for details.',
'models.form.runCommandPlaceholder':
'напр., vllm serve Qwen/Qwen2.5-1.5B-Instruct',
'models.accessSettings.public': 'Публичный',
@@ -206,11 +208,18 @@ export default {
'Вы можете указать точное количество GPU на реплику.',
'models.mymodels.status.inactive': 'Stopped',
'models.mymodels.status.degrade': 'Abnormal',
'models.mymodels.status.active': 'Active'
'models.mymodels.status.active': 'Active',
'models.form.kvCache.tips':
'Available only with built-in backends (vLLM / SGLang) — switch backend in <span class="bold-text">Advanced</span> to enable.',
'models.form.kvCache.tips2':
'KV cache is only supported when using built-in inference backends (vLLM or SGLang).'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
// 1. 'models.mymodels.status.inactive': 'Stopped',
// 3. 'models.mymodels.status.degrade': 'Abnormal',
// 4. 'models.mymodels.status.active': 'Active'
// 5. 'models.form.remoteURL.tips': 'Refer to the <a href="https://docs.lmcache.ai/api_reference/configurations.html" target="_blank">configuration documentation</a> for details.',
// 6. 'models.form.kvCache.tips': 'Available only with built-in backends (vLLM / SGLang) — switch backend in <span class="bold-text">Advanced</span> to enable.'
// 7. 'models.form.kvCache.tips2': 'KV cache is only supported when using built-in inference backends (vLLM or SGLang).';
// ========== End of To-Do List ==========
src/locales/zh-CN/models.ts
+7 -1
View File
@@ -176,6 +176,8 @@ export default {
'models.form.chunkSize': '缓存分块大小',
'models.form.maxCPUSize': 'CPU 缓存最大占用 (GiB)',
'models.form.remoteURL': '远程存储地址',
'models.form.remoteURL.tips':
'参考 <a href="https://docs.lmcache.ai/api_reference/configurations.html" target="_blank">配置文档</a> 配置',
'models.form.runCommandPlaceholder':
'例如，vllm serve Qwen/Qwen2.5-1.5B-Instruct',
'models.accessSettings.public': '公开',
@@ -194,5 +196,9 @@ export default {
'models.form.gpusAllocationType.custom.tips': '您可以指定每个副本的 GPU 数量',
'models.mymodels.status.inactive': '已停止',
'models.mymodels.status.degrade': '异常',
'models.mymodels.status.active': '可用'
'models.mymodels.status.active': '可用',
'models.form.kvCache.tips':
'仅在内置后端（vLLM / SGLang）可用 —— 请在<span class="bold-text">高级</span>配置中切换后端以启用。',
'models.form.kvCache.tips2':
'仅在使用内置推理后端（vLLM 或 SGLang）时支持 KV 缓存。'
};
src/pages/backends/components/import-yaml.tsx
+2 -2
View File
@@ -101,7 +101,7 @@ const ImportYAML: React.FC<ImportYAMLProps> = forwardRef(
const renderHeader = () => {
return (
<Header>
<span className="title">YAML Editor</span>
<span className="title">YAML</span>
<Upload
name="file"
multiple={false}
@@ -110,7 +110,7 @@ const ImportYAML: React.FC<ImportYAMLProps> = forwardRef(
accept=".yaml,.yml,text/yaml,application/x-yaml"
>
<Button icon={<ImportOutlined />} type="text" size="small">
Import
{intl.formatMessage({ id: 'common.button.import' })}
</Button>
</Upload>
</Header>
src/pages/backends/config/index.ts
+5 -11
View File
@@ -2,6 +2,7 @@ import MindIELogo from '@/assets/logo/ascend.png';
import SGLangLogo from '@/assets/logo/sglang.png';
import vLLMLogo from '@/assets/logo/vllm.png';
import icons from '@/components/icon-font/icons';
import { backendOptionsMap } from '@/pages/llmodels/config/backend-parameters';
import jsYaml from 'js-yaml';
import { trim } from 'lodash';
@@ -16,18 +17,11 @@ const SealYamlType = new jsYaml.Type('!seal', {
const SEAL_SCHEMA = jsYaml.DEFAULT_SCHEMA.extend([SealYamlType]);
export const builtInBackends = {
SGLang: 'SGLang',
vLLM: 'vLLM',
MindIE: 'MindIE',
VoxBox: 'voxbox'
};
export const builtInBackendLogos: Record<string, string> = {
[builtInBackends.SGLang]: SGLangLogo,
[builtInBackends.vLLM]: vLLMLogo,
[builtInBackends.MindIE]: MindIELogo,
[builtInBackends.VoxBox]: ''
[backendOptionsMap.SGLang]: SGLangLogo,
[backendOptionsMap.vllm]: vLLMLogo,
[backendOptionsMap.ascendMindie]: MindIELogo,
[backendOptionsMap.voxBox]: ''
};
export const backendActions = [
src/pages/backends/config/schema/update-builtin.json
+2 -2
View File
@@ -23,7 +23,7 @@
"version_configs": {
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+-custom$": {
"^[a-zA-Z0-9._-]+$": {
"type": "object",
"properties": {
"image_name": {
@@ -44,7 +44,7 @@
}
},
"additionalProperties": false,
"description": "multiple backend versions ending with -custom"
"description": "multiple backend versions"
}
},
"required": ["version_configs"],
src/pages/llmodels/config/backend-parameters/index.ts
+4 -2
View File
@@ -7,7 +7,8 @@ export const backendOptionsMap = {
vllm: 'vLLM',
voxBox: 'VoxBox',
ascendMindie: 'MindIE',
custom: 'Custom'
custom: 'Custom',
SGLang: 'SGLang'
};
export interface BackendParameter {
@@ -36,5 +37,6 @@ export default {
[backendOptionsMap.vllm]: generateBackendParameters(vllmParameters),
[backendOptionsMap.ascendMindie]: generateBackendParameters(mindieParameters),
[backendOptionsMap.voxBox]: [],
[backendOptionsMap.custom]: []
[backendOptionsMap.custom]: [],
[backendOptionsMap.SGLang]: []
};
src/pages/llmodels/config/types.ts
+2
View File
@@ -260,6 +260,7 @@ export interface BackendOption {
label: string;
default_backend_param: string[];
default_version: string;
isBuiltIn: boolean;
versions: { label: string; value: string }[];
}
@@ -273,6 +274,7 @@ export interface BackendItem {
from_config: boolean;
default_version: string;
default_backend_param: string[];
is_built_in: boolean;
versions: {
version: string;
is_deprecated: boolean;
src/pages/llmodels/forms/index.tsx
+15
View File
@@ -184,11 +184,26 @@ const DataForm: React.FC<DataFormProps> = forwardRef((props, ref) => {
};
};
const updateKVCacheConfig = (backend: string, option: BackendOption) => {
if (
!option.isBuiltIn &&
[backendOptionsMap.SGLang, backendOptionsMap.vllm].includes(backend)
) {
return {
extended_kv_cache: {
enabled: false
}
};
}
return {};
};
const handleBackendChange = async (val: string, option: BackendOption) => {
form.setFieldsValue({
env: null,
backend_version: option.default_version || '',
backend_parameters: option.default_backend_param || [],
...updateKVCacheConfig(val, option),
...updateGPUSelector(val)
});
onBackendChange?.(val);
src/pages/llmodels/forms/kv-cache.tsx
+39 -1
View File
@@ -3,14 +3,17 @@ import SealInputNumber from '@/components/seal-form/input-number';
import SealInput from '@/components/seal-form/seal-input';
import { useIntl } from '@umijs/max';
import { Form } from 'antd';
import { useMemo } from 'react';
import { backendOptionsMap } from '../config/backend-parameters';
import { useFormContext } from '../config/form-context';
import { FormData } from '../config/types';
const KVCacheForm = () => {
const intl = useIntl();
const form = Form.useFormInstance();
const { onValuesChange } = useFormContext();
const { onValuesChange, backendOptions } = useFormContext();
const kvCacheEnabled = Form.useWatch(['extended_kv_cache', 'enabled'], form);
const backend = Form.useWatch('backend', form);
const handleOnChange = async (e: any) => {
if (e.target.checked) {
@@ -29,6 +32,19 @@ const KVCacheForm = () => {
onValuesChange?.({}, form.getFieldsValue());
};
const builtInBackend = useMemo(() => {
const currentBackend = backendOptions.find(
(item) => item.value === backend
);
return (
currentBackend?.isBuiltIn &&
[backendOptionsMap.SGLang, backendOptionsMap.vllm].includes(
backend as string
)
);
}, [backend, backendOptions]);
return (
<>
<div style={{ paddingBottom: 22 }}>
@@ -37,8 +53,21 @@ const KVCacheForm = () => {
name={['extended_kv_cache', 'enabled']}
valuePropName="checked"
style={{ padding: '0 10px', marginBottom: 0 }}
extra={
!builtInBackend && (
<span
dangerouslySetInnerHTML={{
__html: intl.formatMessage({ id: 'models.form.kvCache.tips' })
}}
></span>
)
}
>
<CheckboxField
description={intl.formatMessage({
id: 'models.form.kvCache.tips2'
})}
disabled={!builtInBackend}
onChange={handleOnChange}
label={intl.formatMessage({ id: 'models.form.extendedkvcache' })}
></CheckboxField>
@@ -65,6 +94,15 @@ const KVCacheForm = () => {
</Form.Item>
<Form.Item<FormData> name={['extended_kv_cache', 'remote_url']}>
<SealInput.Input
description={
<span
dangerouslySetInnerHTML={{
__html: intl.formatMessage({
id: 'models.form.remoteURL.tips'
})
}}
></span>
}
label={intl.formatMessage({ id: 'models.form.remoteURL' })}
min={0}
step={1}
src/pages/llmodels/hooks/use-query-backends.ts
+1
View File
@@ -14,6 +14,7 @@ export default function useQueryBackends() {
label: item.backend_name,
default_backend_param: item.default_backend_param || [],
default_version: item.default_version,
isBuiltIn: item.is_built_in,
versions: (item.versions || []).map((vItem) => ({
label: vItem.version,
value: vItem.version