diff --git a/src/locales/en-US/models.ts b/src/locales/en-US/models.ts
index cf75b857..3859b87f 100644
--- a/src/locales/en-US/models.ts
+++ b/src/locales/en-US/models.ts
@@ -212,5 +212,12 @@ export default {
'Available only with built-in backends (vLLM / SGLang) — switch backend in Advanced to enable.',
'models.form.kvCache.tips2':
'KV cache is only supported when using built-in inference backends (vLLM or SGLang).',
- 'models.form.scheduling': 'Scheduling'
+ 'models.form.scheduling': 'Scheduling',
+ 'models.form.ramRatio': 'RAM-to-VRAM Ratio',
+ 'models.form.ramSize': 'Maximum RAM Size (GiB)',
+ 'models.form.ramRatio.tips':
+ 'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.',
+ 'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`,
+ 'models.form.chunkSize.tips':
+ 'Number of tokens per KV cache chunk. A larger chunk size may improve throughput but increase memory usage.'
};
diff --git a/src/locales/ja-JP/models.ts b/src/locales/ja-JP/models.ts
index fe761991..9d8b80f6 100644
--- a/src/locales/ja-JP/models.ts
+++ b/src/locales/ja-JP/models.ts
@@ -212,7 +212,14 @@ export default {
'Available only with built-in backends (vLLM / SGLang) — switch backend in Advanced to enable.',
'models.form.kvCache.tips2':
'KV cache is only supported when using built-in inference backends (vLLM or SGLang).',
- 'models.form.scheduling': 'Scheduling'
+ 'models.form.scheduling': 'Scheduling',
+ 'models.form.ramRatio': 'RAM-to-VRAM Ratio',
+ 'models.form.ramSize': 'Maximum RAM Size (GiB)',
+ 'models.form.ramRatio.tips':
+ 'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.',
+ 'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`,
+ 'models.form.chunkSize.tips':
+ 'Number of tokens per KV cache chunk. A larger chunk size may improve throughput but increase memory usage.'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
@@ -259,5 +266,10 @@ export default {
// 43. 'models.form.remoteURL.tips': 'Refer to the configuration documentation for details.',
// 44. 'models.form.kvCache.tips': 'Available only with built-in backends (vLLM / SGLang) — switch backend in Advanced to enable.'
// 45. 'models.form.kvCache.tips2': 'KV cache is only supported when using built-in inference backends (vLLM or SGLang).',
-// 46. 'models.form.scheduling': 'Scheduling'
+// 46. 'models.form.scheduling': 'Scheduling',
+// 47. 'models.form.ramRatio': 'RAM-to-VRAM Ratio',
+// 48. 'models.form.ramSize': 'Maximum RAM Size (GiB)',
+// 49. 'models.form.ramRatio.tips': 'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.',
+// 50. 'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`,
+// 51. 'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk. A larger chunk size may improve throughput but increase memory usage.'
// ========== End of To-Do List ==========
diff --git a/src/locales/ru-RU/models.ts b/src/locales/ru-RU/models.ts
index 8603be60..bc2833f2 100644
--- a/src/locales/ru-RU/models.ts
+++ b/src/locales/ru-RU/models.ts
@@ -212,7 +212,14 @@ export default {
'Available only with built-in backends (vLLM / SGLang) — switch backend in Advanced to enable.',
'models.form.kvCache.tips2':
'KV cache is only supported when using built-in inference backends (vLLM or SGLang).',
- 'models.form.scheduling': 'Scheduling'
+ 'models.form.scheduling': 'Scheduling',
+ 'models.form.ramRatio': 'RAM-to-VRAM Ratio',
+ 'models.form.ramSize': 'Maximum RAM Size (GiB)',
+ 'models.form.ramRatio.tips':
+ 'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.',
+ 'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`,
+ 'models.form.chunkSize.tips':
+ 'Number of tokens per KV cache chunk. A larger chunk size may improve throughput but increase memory usage.'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
@@ -222,5 +229,10 @@ export default {
// 5. 'models.form.remoteURL.tips': 'Refer to the configuration documentation for details.',
// 6. 'models.form.kvCache.tips': 'Available only with built-in backends (vLLM / SGLang) — switch backend in Advanced to enable.'
// 7. 'models.form.kvCache.tips2': 'KV cache is only supported when using built-in inference backends (vLLM or SGLang).';
-// 8. 'models.form.scheduling': 'Scheduling'
+// 8. 'models.form.scheduling': 'Scheduling',
+// 9. 'models.form.ramRatio': 'RAM-to-VRAM Ratio',
+// 10. 'models.form.ramSize': 'Maximum RAM Size (GiB)',
+// 11. 'models.form.ramRatio.tips': 'Ratio of system RAM to GPU VRAM used for KV cache. For example, 2.0 means the cache in RAM can be twice as large as the GPU VRAM.',
+// 12. 'models.form.ramSize.tips': `Maximum size of the KV cache stored in system memory (GiB). If set, this value overrides "{content}".`,
+// 13. 'models.form.chunkSize.tips': 'Number of tokens per KV cache chunk. A larger chunk size may improve throughput but increase memory usage.'
// ========== End of To-Do List ==========
diff --git a/src/locales/zh-CN/models.ts b/src/locales/zh-CN/models.ts
index fa4c32da..423df50a 100644
--- a/src/locales/zh-CN/models.ts
+++ b/src/locales/zh-CN/models.ts
@@ -201,5 +201,12 @@ export default {
'仅在内置后端(vLLM / SGLang)可用 —— 请在高级配置中切换后端以启用。',
'models.form.kvCache.tips2':
'仅在使用内置推理后端(vLLM 或 SGLang)时支持 KV 缓存。',
- 'models.form.scheduling': '调度'
+ 'models.form.scheduling': '调度',
+ 'models.form.ramRatio': '内存与显存比例',
+ 'models.form.ramSize': '内存最大占用 (GiB)',
+ 'models.form.ramRatio.tips':
+ 'KV 缓存在系统内存与 GPU 显存之间的比例。例如设置为 2.0 表示系统内存中可缓存的数据量是显存的两倍。',
+ 'models.form.ramSize.tips': `KV 缓存在系统内存中的最大值。当设置该值时,将覆盖 "{content}" 的配置。`,
+ 'models.form.chunkSize.tips':
+ '每个 KV 缓存块包含的 token 数量。数值越大可提升吞吐量,但也会增加内存占用。'
};
diff --git a/src/pages/llmodels/config/types.ts b/src/pages/llmodels/config/types.ts
index bdb479d8..0aeeb866 100644
--- a/src/pages/llmodels/config/types.ts
+++ b/src/pages/llmodels/config/types.ts
@@ -79,8 +79,8 @@ export interface FormData {
extended_kv_cache: {
enabled: boolean;
chunk_size: number;
- max_local_cpu_size: number;
- remote_url: string;
+ ram_ratio: number;
+ ram_size: number;
};
}
diff --git a/src/pages/llmodels/forms/index.tsx b/src/pages/llmodels/forms/index.tsx
index bda7cea9..bd5202d9 100644
--- a/src/pages/llmodels/forms/index.tsx
+++ b/src/pages/llmodels/forms/index.tsx
@@ -368,9 +368,9 @@ const DataForm: React.FC = forwardRef((props, ref) => {
distributed_inference_across_workers: true,
extended_kv_cache: {
enabled: false,
- chunk_size: 256,
- max_local_cpu_size: 10,
- remote_url: ''
+ chunk_size: null,
+ ram_ratio: 1.2,
+ ram_size: null
},
...initialValues
}}
diff --git a/src/pages/llmodels/forms/kv-cache.tsx b/src/pages/llmodels/forms/kv-cache.tsx
index 41aff1f0..a8e8feda 100644
--- a/src/pages/llmodels/forms/kv-cache.tsx
+++ b/src/pages/llmodels/forms/kv-cache.tsx
@@ -1,6 +1,5 @@
import CheckboxField from '@/components/seal-form/checkbox-field';
import SealInputNumber from '@/components/seal-form/input-number';
-import SealInput from '@/components/seal-form/seal-input';
import { useIntl } from '@umijs/max';
import { Form } from 'antd';
import { useMemo } from 'react';
@@ -21,9 +20,9 @@ const KVCacheForm = () => {
form.setFieldsValue({
extended_kv_cache: {
enabled: true,
- chunk_size: extendedKVCache?.chunk_size || 256,
- max_local_cpu_size: extendedKVCache?.max_local_cpu_size || 10,
- remote_url: extendedKVCache?.remote_url || ''
+ chunk_size: extendedKVCache?.chunk_size,
+ ram_ratio: extendedKVCache?.ram_ratio || 1.2,
+ ram_size: extendedKVCache?.ram_size
}
});
}
@@ -76,11 +75,26 @@ const KVCacheForm = () => {
{kvCacheEnabled && (
<>
-
- name={['extended_kv_cache', 'max_local_cpu_size']}
- >
+ name={['extended_kv_cache', 'ram_ratio']}>
+
+ name={['extended_kv_cache', 'ram_size']}>
+ {
name={['extended_kv_cache', 'chunk_size']}>
- name={['extended_kv_cache', 'remote_url']}>
-
- }
- label={intl.formatMessage({ id: 'models.form.remoteURL' })}
- min={0}
- step={1}
- placeholder="protocol://host:port"
- />
-
>
)}
>