From 12707793b101711e70529807486583c64967746e Mon Sep 17 00:00:00 2001 From: jialin Date: Mon, 10 Mar 2025 15:05:21 +0800 Subject: [PATCH] chore: display vllm distributed info --- .../seal-table/styles/row-children.less | 3 +- src/locales/en-US/models.ts | 4 +- src/locales/ru-RU/models.ts | 9 +- src/locales/zh-CN/models.ts | 4 +- .../llmodels/components/instance-item.tsx | 252 ++++++++++-------- src/pages/llmodels/config/types.ts | 15 +- 6 files changed, 162 insertions(+), 125 deletions(-) diff --git a/src/components/seal-table/styles/row-children.less b/src/components/seal-table/styles/row-children.less index 43fd8f86..575bcc45 100644 --- a/src/components/seal-table/styles/row-children.less +++ b/src/components/seal-table/styles/row-children.less @@ -2,7 +2,8 @@ position: relative; display: flex; align-items: center; - height: 54px; + min-height: 54px; + padding-block: 5px; border-radius: var(--border-radius-mdium); transition: all 0.2s ease; diff --git a/src/locales/en-US/models.ts b/src/locales/en-US/models.ts index 915ecc08..19ae8b24 100644 --- a/src/locales/en-US/models.ts +++ b/src/locales/en-US/models.ts @@ -106,5 +106,7 @@ export default { 'Multiple replicas enable load balancing for { api } inference requests.', 'models.table.list.empty': 'No Models yet!', 'models.table.list.getStart': - 'Get started with DeepSeek-R1-Distill-Qwen-1.5B' + 'Get started with DeepSeek-R1-Distill-Qwen-1.5B', + 'models.table.llamaAcrossworker': 'Llama-box Across Workers', + 'models.table.vllmAcrossworker': 'vLLM Across Workers' }; diff --git a/src/locales/ru-RU/models.ts b/src/locales/ru-RU/models.ts index 83a32167..7362cb20 100644 --- a/src/locales/ru-RU/models.ts +++ b/src/locales/ru-RU/models.ts @@ -85,8 +85,7 @@ export default { 'models.form.gpuselector': 'Селектор GPU', 'models.form.backend.llamabox': 'Для моделей формата GGUF. Поддержка Linux, macOS и Windows.', - 'models.form.backend.vllm': - 'Для моделей не-GGUF формата. Только x86 Linux.', + 'models.form.backend.vllm': 'Для моделей не-GGUF формата. Только x86 Linux.', 'models.form.backend.voxbox': 'Для аудиомоделей не-GGUF формата.', 'models.form.search.gguftips': 'Для воркеров на macOS/Windows отметьте GGUF (для аудиомоделей снимите).', @@ -106,5 +105,9 @@ export default { 'Несколько реплик обеспечивают балансировку нагрузки для { api } запросов.', 'models.table.list.empty': 'Модели отсутствуют!', 'models.table.list.getStart': - 'Начните работу с DeepSeek-R1-Distill-Qwen-1.5B' + 'Начните работу с DeepSeek-R1-Distill-Qwen-1.5B', + 'models.table.llamaAcrossworker': + 'TODO: Translate key "models.table.llamaAcrossworker"', + 'models.table.vllmAcrossworker': + 'TODO: Translate key "models.table.vllmAcrossworker"' }; diff --git a/src/locales/zh-CN/models.ts b/src/locales/zh-CN/models.ts index 41476075..d4a3a604 100644 --- a/src/locales/zh-CN/models.ts +++ b/src/locales/zh-CN/models.ts @@ -102,5 +102,7 @@ export default { 'models.form.replicas.tips': '多副本数实现 { api } 接口推理请求的负载均衡', 'models.table.list.empty': '暂无已部署模型', 'models.table.list.getStart': - '一键部署DeepSeek-R1-Distill-Qwen-1.5B立即使用！' + '一键部署DeepSeek-R1-Distill-Qwen-1.5B立即使用！', + 'models.table.llamaAcrossworker': 'Llama-box 跨节点', + 'models.table.vllmAcrossworker': 'vLLM 跨节点' }; diff --git a/src/pages/llmodels/components/instance-item.tsx b/src/pages/llmodels/components/instance-item.tsx index 10af42c3..d5636c06 100644 --- a/src/pages/llmodels/components/instance-item.tsx +++ b/src/pages/llmodels/components/instance-item.tsx @@ -88,11 +88,6 @@ const distributeCols = [ return row.port ? `${row.worker_ip}:${row.port}` : row.worker_ip; } }, - { - title: 'models.form.backend', - locale: true, - key: 'backend' - }, { title: 'models.table.gpuindex', locale: true, @@ -278,38 +273,143 @@ const InstanceItem: React.FC = ({ ); }, [modelData, instanceData, intl]); - const renderDistributionInfo = useMemo(() => { - const rpcServerList = instanceData.distributed_servers?.rpc_servers || []; - const list = _.map(rpcServerList, (item: any) => { - const data = _.find(workerList, { id: item.worker_id }); - return { - worker_name: data?.name, - worker_ip: data?.ip, - port: '', - ram: convertFileSize(item.computed_resource_claim?.ram, 0), - gpu_index: displayGPUs(item.computed_resource_claim?.vram || {}) - }; - }); + const renderDistributedServer = useCallback( + (severList: any[]) => { + const list = _.map(severList, (item: any) => { + const data = _.find(workerList, { id: item.worker_id }); + return { + worker_name: data?.name, + worker_ip: data?.ip, + port: '', + ram: convertFileSize(item.computed_resource_claim?.ram, 0), + gpu_index: displayGPUs(item.computed_resource_claim?.vram || {}) + }; + }); - const mainWorker = [ - { - worker_name: `${instanceData.worker_name} (main)`, - worker_ip: `${instanceData.worker_ip}`, - port: '', - ram: convertFileSize(instanceData.computed_resource_claim?.ram, 0), - gpu_index: displayGPUs(instanceData.computed_resource_claim?.vram || {}) + const mainWorker = [ + { + worker_name: `${instanceData.worker_name} (main)`, + worker_ip: `${instanceData.worker_ip}`, + port: '', + ram: convertFileSize(instanceData.computed_resource_claim?.ram, 0), + gpu_index: displayGPUs( + instanceData.computed_resource_claim?.vram || {} + ) + } + ]; + + return ( +
+ +
+ ); + }, + [workerList, instanceData, intl] + ); + + const renderDistributionInfo = useCallback( + (severList: any[], label: string) => { + if (!severList.length) { + return null; } - ]; + return ( + + + + {intl.formatMessage({ + id: label + })} + + + ); + }, + [renderDistributedServer] + ); + const renderOffloadInfo = useMemo(() => { + const total_layers = instanceData.computed_resource_claim?.total_layers; + const offload_layers = instanceData.computed_resource_claim?.offload_layers; + if (total_layers === offload_layers || !offload_layers || !total_layers) { + return null; + } return ( -
- -
+ + + CPU:{' '} + {_.subtract( + instanceData.computed_resource_claim?.total_layers, + instanceData.computed_resource_claim?.offload_layers + ) || 0}{' '} + {intl.formatMessage({ + id: 'models.table.layers' + })} + + + + GPU: {instanceData.computed_resource_claim?.offload_layers}{' '} + {intl.formatMessage({ + id: 'models.table.layers' + })} + + + } + > + + + {intl.formatMessage({ + id: 'models.table.cpuoffload' + })} + + ); - }, [workerList, instanceData, intl]); + }, [ + instanceData.computed_resource_claim?.total_layers, + instanceData.computed_resource_claim?.offload_layers + ]); const handleOnSelect = useCallback( (val: string) => { @@ -366,88 +466,14 @@ const InstanceItem: React.FC = ({ }} className="flex align-center" > - {instanceData.computed_resource_claim?.total_layers !== - instanceData.computed_resource_claim?.offload_layers && ( - - - CPU:{' '} - {_.subtract( - instanceData.computed_resource_claim?.total_layers, - instanceData.computed_resource_claim?.offload_layers - ) || 0}{' '} - {intl.formatMessage({ - id: 'models.table.layers' - })} - - - - GPU:{' '} - {instanceData.computed_resource_claim?.offload_layers}{' '} - {intl.formatMessage({ - id: 'models.table.layers' - })} - - - } - > - - - {intl.formatMessage({ - id: 'models.table.cpuoffload' - })} - - + {renderOffloadInfo} + {renderDistributionInfo( + instanceData.distributed_servers?.rpc_servers || [], + 'models.table.llamaAcrossworker' )} - {instanceData?.distributed_servers?.rpc_servers?.length && ( - - - - {intl.formatMessage({ - id: 'models.table.acrossworker' - })} - - + {renderDistributionInfo( + instanceData.distributed_servers?.ray_actors || [], + 'models.table.vllmAcrossworker' )} diff --git a/src/pages/llmodels/config/types.ts b/src/pages/llmodels/config/types.ts index 77e4e19d..c093de79 100644 --- a/src/pages/llmodels/config/types.ts +++ b/src/pages/llmodels/config/types.ts @@ -66,6 +66,13 @@ interface ComputedResourceClaim { ram: number; vram: Record; } + +interface DistributedServerItem { + pid: number; + port: number; + worker_id: string; + computed_resource_claim: ComputedResourceClaim; +} export interface ModelInstanceListItem { backend?: string; backend_version?: string; @@ -74,12 +81,8 @@ export interface ModelInstanceListItem { huggingface_filename: string; ollama_library_model_name: string; distributed_servers?: { - rpc_servers: { - pid: number; - port: number; - worker_id: string; - computed_resource_claim: ComputedResourceClaim; - }[]; + rpc_servers: DistributedServerItem[]; + ray_actors: DistributedServerItem[]; }; computed_resource_claim?: ComputedResourceClaim; s3_address: string;