From 864cf76c074449523dd2d22b3895b00e4cc600e3 Mon Sep 17 00:00:00 2001 From: jialin Date: Thu, 13 Nov 2025 20:59:56 +0800 Subject: [PATCH] feat: worker maintenance --- src/global.less | 1 + src/layouts/index.tsx | 25 ++-- src/locales/en-US/resources.ts | 10 +- src/locales/ja-JP/resources.ts | 18 ++- src/locales/ru-RU/resources.ts | 17 ++- src/locales/zh-CN/resources.ts | 9 +- .../components/add-worker-command.tsx | 5 +- .../components/add-worker-step.tsx | 1 - src/pages/resources/components/workers.tsx | 15 ++ src/pages/resources/config/gpu-driver.ts | 77 +++------- src/pages/resources/config/index.ts | 9 +- src/pages/resources/config/types.ts | 4 + .../resources/hooks/use-worker-columns.tsx | 20 ++- .../hooks/use-worker-maintenance.tsx | 132 ++++++++++++++++++ 14 files changed, 264 insertions(+), 79 deletions(-) create mode 100644 src/pages/resources/hooks/use-worker-maintenance.tsx diff --git a/src/global.less b/src/global.less index e8bf87f6..90de4304 100644 --- a/src/global.less +++ b/src/global.less @@ -235,6 +235,7 @@ body { .ant-pro-sider-logo-collapsed { padding-left: 12px; + cursor: e-resize; .collapse-wrap { display: none; diff --git a/src/layouts/index.tsx b/src/layouts/index.tsx index e57e1eb6..da931c6e 100644 --- a/src/layouts/index.tsx +++ b/src/layouts/index.tsx @@ -285,22 +285,21 @@ export default (props: any) => { {logo}
diff --git a/src/locales/en-US/resources.ts b/src/locales/en-US/resources.ts index 0e3359a5..71a7907e 100644 --- a/src/locales/en-US/resources.ts +++ b/src/locales/en-US/resources.ts @@ -96,5 +96,13 @@ export default { 'resources.model.instance': 'Model Instance', 'resources.worker.download.privatekey': 'Download Private Key', 'resources.modelfiles.form.exsting': 'Downloaded', - 'resources.modelfiles.form.added': 'Added' + 'resources.modelfiles.form.added': 'Added', + 'resources.worker.maintenance.title': 'System Maintenance', + 'resources.worker.maintenance.enable': 'Enter Maintenance Mode', + 'resources.worker.maintenance.disable': 'Exit Maintenance Mode', + 'resources.worker.maintenance.remark': 'Remark', + 'resources.worker.maintenance.remark.rules': + 'Please enter maintenance remarks', + 'resources.worker.maintenance.tips': + 'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.' }; diff --git a/src/locales/ja-JP/resources.ts b/src/locales/ja-JP/resources.ts index 263f7b92..861c08c0 100644 --- a/src/locales/ja-JP/resources.ts +++ b/src/locales/ja-JP/resources.ts @@ -97,7 +97,15 @@ export default { 'resources.worker.download.privatekey': 'Download Private Key', 'resources.worker': 'Worker', 'resources.modelfiles.form.exsting': 'Downloaded', - 'resources.modelfiles.form.added': 'Added' + 'resources.modelfiles.form.added': 'Added', + 'resources.worker.maintenance.title': 'System Maintenance', + 'resources.worker.maintenance.enable': 'Enter Maintenance Mode', + 'resources.worker.maintenance.disable': 'Exit Maintenance Mode', + 'resources.worker.maintenance.remark': 'Remark', + 'resources.worker.maintenance.remark.rules': + 'Please enter maintenance remarks', + 'resources.worker.maintenance.tips': + 'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.' }; // ========== To-Do: Translate Keys (Remove After Translation) ========== @@ -117,5 +125,11 @@ export default { // 14. 'resources.worker.download.privatekey': 'Download Private Key', // 15. 'resources.worker': 'Worker', // 16. 'resources.modelfiles.form.exsting': 'Downloaded', -// 17. 'resources.modelfiles.form.added': 'Added' +// 17. 'resources.modelfiles.form.added': 'Added', +// 18. 'resources.worker.maintenance.title': 'System Maintenance', +// 19. 'resources.worker.maintenance.enable': 'Enter Maintenance Mode', +// 20. 'resources.worker.maintenance.disable': 'Exit Maintenance Mode', +// 21. 'resources.worker.maintenance.remark': 'Remark', +// 22. 'resources.worker.maintenance.remark.rules': 'Please enter maintenance remarks', +// 23. 'resources.worker.maintenance.tips': 'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.' // ========== End of To-Do List ========== diff --git a/src/locales/ru-RU/resources.ts b/src/locales/ru-RU/resources.ts index edd332e0..acfa5ada 100644 --- a/src/locales/ru-RU/resources.ts +++ b/src/locales/ru-RU/resources.ts @@ -95,9 +95,22 @@ export default { 'resources.worker.download.privatekey': 'Скачать приватный ключ', 'resources.worker': 'Рабочий узел', 'resources.modelfiles.form.exsting': 'Загружено', - 'resources.modelfiles.form.added': 'Добавлено' + 'resources.modelfiles.form.added': 'Добавлено', + 'resources.worker.maintenance.title': 'System Maintenance', + 'resources.worker.maintenance.enable': 'Enter Maintenance Mode', + 'resources.worker.maintenance.disable': 'Exit Maintenance Mode', + 'resources.worker.maintenance.remark': 'Remark', + 'resources.worker.maintenance.remark.rules': + 'Please enter maintenance remarks', + 'resources.worker.maintenance.tips': + 'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.' }; // ========== To-Do: Translate Keys (Remove After Translation) ========== - +// 2. 'resources.worker.maintenance.title': 'System Maintenance', +// 3. 'resources.worker.maintenance.enable': 'Enter Maintenance Mode', +// 4. 'resources.worker.maintenance.disable': 'Exit Maintenance Mode', +// 5. 'resources.worker.maintenance.remark': 'Remark', +// 6. 'resources.worker.maintenance.remark.rules': 'Please enter maintenance remarks', +// 7. 'resources.worker.maintenance.tips': 'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.' // ========== End of To-Do List ========== diff --git a/src/locales/zh-CN/resources.ts b/src/locales/zh-CN/resources.ts index 3bdda76b..515f3be9 100644 --- a/src/locales/zh-CN/resources.ts +++ b/src/locales/zh-CN/resources.ts @@ -94,5 +94,12 @@ export default { 'resources.model.instance': '模型实例', 'resources.worker.download.privatekey': '下载私钥', 'resources.modelfiles.form.exsting': '已下载', - 'resources.modelfiles.form.added': '已添加' + 'resources.modelfiles.form.added': '已添加', + 'resources.worker.maintenance.title': '系统维护', + 'resources.worker.maintenance.enable': '开启维护', + 'resources.worker.maintenance.disable': '停止维护', + 'resources.worker.maintenance.remark': '备注', + 'resources.worker.maintenance.remark.rules': '请输入维护备注', + 'resources.worker.maintenance.tips': + '启用维护后，节点将停止调度新的模型实例部署任务，正在运行的实例不会受到影响。' }; diff --git a/src/pages/cluster-management/components/add-worker-command.tsx b/src/pages/cluster-management/components/add-worker-command.tsx index cbac73cf..78594d62 100644 --- a/src/pages/cluster-management/components/add-worker-command.tsx +++ b/src/pages/cluster-management/components/add-worker-command.tsx @@ -4,6 +4,7 @@ import React from 'react'; type ViewModalProps = { currentGPU?: string; + workerIP?: string; registrationInfo: { token: string; image: string; @@ -13,6 +14,7 @@ type ViewModalProps = { const AddWorkerCommand: React.FC = ({ registrationInfo, + workerIP, currentGPU }) => { const code = React.useMemo(() => { @@ -21,10 +23,11 @@ const AddWorkerCommand: React.FC = ({ gpu: currentGPU || '', server: registrationInfo.server_url || origin, tag: '', + workerIP: '${WORKER_IP}', image: registrationInfo.image, token: registrationInfo.token || '${token}' }); - }, [registrationInfo, currentGPU]); + }, [registrationInfo, currentGPU, workerIP]); return ( = ({ const [currentGPU, setCurrentGPU] = React.useState( GPUDriverMap.NVIDIA ); - React.useState(GPUDriverMap.NVIDIA); const [workerCommand, setWorkerCommand] = React.useState>( { label: 'NVIDIA', diff --git a/src/pages/resources/components/workers.tsx b/src/pages/resources/components/workers.tsx index 9376c748..20b21c11 100644 --- a/src/pages/resources/components/workers.tsx +++ b/src/pages/resources/components/workers.tsx @@ -20,6 +20,7 @@ import { } from '../apis'; import { ListItem } from '../config/types'; import useWorkerColumns from '../hooks/use-worker-columns'; +import useWorkerMaintenance from '../hooks/use-worker-maintenance'; import UpdateLabels from './update-labels'; import WorkerDetailModal from './worker-detail-modal'; @@ -46,6 +47,8 @@ const Workers: React.FC = () => { API: WORKERS_API }); const [, setClusterSession] = useAtom(clusterSessionAtom); + const { MaintenanceModal, handleStopMaintenance, setOpenStatus } = + useWorkerMaintenance({ fetchData }); const navigate = useNavigate(); const intl = useIntl(); @@ -154,6 +157,17 @@ const Workers: React.FC = () => { name: record.name }); } + + if (val === 'star_maintenance') { + setOpenStatus({ + open: true, + currentData: record + }); + } + + if (val === 'stop_maintenance') { + handleStopMaintenance(record); + } }); const handleAddWorker = () => { @@ -253,6 +267,7 @@ const Workers: React.FC = () => { setWorkerDetailStatus({ currentData: null, open: false }) } /> + {MaintenanceModal} ); diff --git a/src/pages/resources/config/gpu-driver.ts b/src/pages/resources/config/gpu-driver.ts index c07d6b16..65197f08 100644 --- a/src/pages/resources/config/gpu-driver.ts +++ b/src/pages/resources/config/gpu-driver.ts @@ -134,25 +134,19 @@ const registerWorker = (params: { token: string; image: string; gpu: string; + workerIP?: string; }) => { const config = GPUsConfigs[params.gpu]; - return `CONTAINER_NAME="gpustack-worker" \\ -WORKER_PORT="80" \\ -WORKER_IP="" \\ -sudo docker run -d --name \${CONTAINER_NAME} \\ + return `sudo docker run -d --name gpustack-worker \\ --restart=unless-stopped \\ --privileged \\ - --publish \${WORKER_PORT}:\${WORKER_PORT} \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\ --volume /var/run/docker.sock:/var/run/docker.sock \\ --volume gpustack-data:/var/lib/gpustack \\ --runtime ${config.runtime} \\ ${params.image} \\ --server-url ${params.server} \\ --token ${params.token} \\ - --worker-port \${WORKER_PORT} \\ - --advertise-address \${WORKER_IP}`; + ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim(); }; // avaliable for Ascend @@ -162,17 +156,13 @@ const registerAscendWorker = (params: { token: string; image: string; gpu: string; + workerIP?: string; }) => { const config = GPUsConfigs[params.gpu]; - return `CONTAINER_NAME="gpustack-worker" \\ -WORKER_PORT="80" \\ -WORKER_IP="" \\ -sudo docker run -d --name \${CONTAINER_NAME} \\ + return `sudo docker run -d --name gpustack-worker \\ --restart=unless-stopped \\ --privileged \\ - --publish \${WORKER_PORT}:\${WORKER_PORT} \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\ + --network=host \\ --env "ASCEND_VISIBLE_DEVICES=$(npu-smi info -m | tail -n 1 | awk '{print $1}') \\ --volume /var/run/docker.sock:/var/run/docker.sock \\ --volume gpustack-data:/var/lib/gpustack \\ @@ -180,8 +170,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\ ${params.image} \\ --server-url ${params.server} \\ --token ${params.token} \\ - --worker-port \${WORKER_PORT} \\ - --advertise-address \${WORKER_IP}`; + ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim(); }; const registerHygonWorker = (params: { @@ -190,17 +179,13 @@ const registerHygonWorker = (params: { token: string; image: string; gpu: string; + workerIP?: string; }) => { const config = GPUsConfigs[params.gpu]; - return `CONTAINER_NAME="gpustack-worker" \\ -WORKER_PORT="80" \\ -WORKER_IP="" \\ -sudo docker run -d --name \${CONTAINER_NAME} \\ + return `sudo docker run -d --name gpustack-worker \\ --restart=unless-stopped \\ --privileged \\ - --publish \${WORKER_PORT}:\${WORKER_PORT} \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\ + --network=host \\ --env ROCM_PATH=/opt/dtk \\ --env ROCM_SMI_LIB_PATH=/opt/hyhal/lib \\ --volume /var/run/docker.sock:/var/run/docker.sock \\ @@ -210,8 +195,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\ ${params.image} \\ --server-url ${params.server} \\ --token ${params.token} \\ - --worker-port \${WORKER_PORT} \\ - --advertise-address \${WORKER_IP}`; + ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim(); }; const registerIluvatarWorker = (params: { @@ -220,17 +204,13 @@ const registerIluvatarWorker = (params: { token: string; image: string; gpu: string; + workerIP?: string; }) => { const config = GPUsConfigs[params.gpu]; - return `CONTAINER_NAME="gpustack-worker" \\ -WORKER_PORT="80" \\ -WORKER_IP="" \\ -sudo docker run -d --name \${CONTAINER_NAME} \\ + return `sudo docker run -d --name gpustack-worker \\ --restart=unless-stopped \\ --privileged \\ - --publish \${WORKER_PORT}:\${WORKER_PORT} \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\ + --network=host \\ --volume /var/run/docker.sock:/var/run/docker.sock \\ --volume gpustack-data:/var/lib/gpustack \\ --volume /lib/modules:/lib/modules:ro \\ @@ -240,8 +220,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\ ${params.image} \\ --server-url ${params.server} \\ --token ${params.token} \\ - --worker-port \${WORKER_PORT} \\ - --advertise-address \${WORKER_IP}`; + ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim(); }; const registerMetaXWorker = (params: { @@ -250,17 +229,13 @@ const registerMetaXWorker = (params: { token: string; image: string; gpu: string; + workerIP?: string; }) => { const config = GPUsConfigs[params.gpu]; - return `CONTAINER_NAME="gpustack-worker" \\ -WORKER_PORT="80" \\ -WORKER_IP="" \\ -sudo docker run -d --name \${CONTAINER_NAME} \\ + return `sudo docker run -d --name gpustack-worker \\ --restart=unless-stopped \\ --privileged \\ - --publish \${WORKER_PORT}:\${WORKER_PORT} \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\ + --network=host \\ --volume /var/run/docker.sock:/var/run/docker.sock \\ --volume gpustack-data:/var/lib/gpustack \\ --volume /opt/mxdriver:/opt/mxdriver:ro \\ @@ -268,8 +243,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\ ${params.image} \\ --server-url ${params.server} \\ --token ${params.token} \\ - --worker-port \${WORKER_PORT} \\ - --advertise-address \${WORKER_IP}`; + ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim(); }; const registerCambriconWorker = (params: { @@ -278,17 +252,13 @@ const registerCambriconWorker = (params: { token: string; image: string; gpu: string; + workerIP?: string; }) => { const config = GPUsConfigs[params.gpu]; - return `CONTAINER_NAME="gpustack-worker" \\ -WORKER_PORT="80" \\ -WORKER_IP="" \\ -sudo docker run -d --name \${CONTAINER_NAME} \\ + return `sudo docker run -d --name gpustack-worker \\ --restart=unless-stopped \\ --privileged \\ - --publish \${WORKER_PORT}:\${WORKER_PORT} \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\ - --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\ + --network=host \\ --volume /var/run/docker.sock:/var/run/docker.sock \\ --volume gpustack-data:/var/lib/gpustack \\ --volume /usr/local/neuware:/usr/local/neuware:ro \\ @@ -296,8 +266,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\ ${params.image} \\ --server-url ${params.server} \\ --token ${params.token} \\ - --worker-port \${WORKER_PORT} \\ - --advertise-address \${WORKER_IP}`; + ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim(); }; export const registerAddWokerCommandMap = { diff --git a/src/pages/resources/config/index.ts b/src/pages/resources/config/index.ts index e635047a..0fce6d88 100644 --- a/src/pages/resources/config/index.ts +++ b/src/pages/resources/config/index.ts @@ -14,7 +14,8 @@ export const WorkerStatusMap = { deleting: 'deleting', error: 'error', pending: 'pending', - initializing: 'initializing' + initializing: 'initializing', + maintenance: 'maintenance' }; export const WorkerStatusMapValue = { @@ -25,7 +26,8 @@ export const WorkerStatusMapValue = { [WorkerStatusMap.deleting]: 'Deleting', [WorkerStatusMap.error]: 'Error', [WorkerStatusMap.initializing]: 'Initializing', - [WorkerStatusMap.pending]: 'Pending' + [WorkerStatusMap.pending]: 'Pending', + [WorkerStatusMap.maintenance]: 'Maintenance' }; export const status: any = { @@ -36,7 +38,8 @@ export const status: any = { [WorkerStatusMap.deleting]: StatusMaps.transitioning, [WorkerStatusMap.error]: StatusMaps.error, [WorkerStatusMap.pending]: StatusMaps.transitioning, - [WorkerStatusMap.initializing]: StatusMaps.transitioning + [WorkerStatusMap.initializing]: StatusMaps.transitioning, + [WorkerStatusMap.maintenance]: StatusMaps.warning }; export const checkEnvCommand = (gpu: string) => { return { diff --git a/src/pages/resources/config/types.ts b/src/pages/resources/config/types.ts index 10f46340..793dacdb 100644 --- a/src/pages/resources/config/types.ts +++ b/src/pages/resources/config/types.ts @@ -68,6 +68,10 @@ export interface ListItem { state_message: string; ssh_key_id: string; provision_progress: string; + maintenance: { + enabled: boolean; + message: string; + }; status: { cpu: { total: number; diff --git a/src/pages/resources/hooks/use-worker-columns.tsx b/src/pages/resources/hooks/use-worker-columns.tsx index 531b4072..672a7f3d 100644 --- a/src/pages/resources/hooks/use-worker-columns.tsx +++ b/src/pages/resources/hooks/use-worker-columns.tsx @@ -9,7 +9,9 @@ import { DeleteOutlined, DownloadOutlined, EditOutlined, - InfoCircleOutlined + InfoCircleOutlined, + SafetyOutlined, + ToolOutlined } from '@ant-design/icons'; import { useIntl } from '@umijs/max'; import { Tooltip } from 'antd'; @@ -31,6 +33,16 @@ const ActionList = [ key: 'download_ssh_key', icon: }, + { + label: 'resources.worker.maintenance.enable', + key: 'star_maintenance', + icon: + }, + { + label: 'resources.worker.maintenance.disable', + key: 'stop_maintenance', + icon: + }, // { // label: 'common.button.logs', // locale: false, @@ -51,6 +63,12 @@ const setActions = (row: ListItem) => { if (action.key === 'download_ssh_key') { return !!row.ssh_key_id; } + if (action.key === 'star_maintenance') { + return !row.maintenance?.enabled; + } + if (action.key === 'stop_maintenance') { + return row.maintenance?.enabled; + } return true; }); }; diff --git a/src/pages/resources/hooks/use-worker-maintenance.tsx b/src/pages/resources/hooks/use-worker-maintenance.tsx new file mode 100644 index 00000000..ffbea2b5 --- /dev/null +++ b/src/pages/resources/hooks/use-worker-maintenance.tsx @@ -0,0 +1,132 @@ +import ModalFooter from '@/components/modal-footer'; +import ScrollerModal from '@/components/scroller-modal'; +import SealInput from '@/components/seal-form/seal-input'; +import SealTextArea from '@/components/seal-form/seal-textarea'; +import { useIntl } from '@umijs/max'; +import { Form, message } from 'antd'; +import React from 'react'; +import { updateWorker } from '../apis'; +import { ListItem } from '../config/types'; + +const useWorkerMaintenance = ({ fetchData }: { fetchData: () => void }) => { + const intl = useIntl(); + const [form] = Form.useForm(); + const [openStatus, setOpenStatus] = React.useState<{ + open: boolean; + currentData: ListItem; + }>({ + open: false, + currentData: {} as ListItem + }); + + const handleSumit = () => { + form.submit(); + }; + + const onCancel = () => { + setOpenStatus({ + open: false, + currentData: {} as ListItem + }); + }; + + const onOk = async (values: any) => { + try { + await updateWorker(openStatus.currentData.id, { + name: openStatus.currentData.name, + labels: openStatus.currentData.labels, + maintenance: { + enabled: true, + message: values.remark + } + }); + message.success(intl.formatMessage({ id: 'common.message.success' })); + fetchData(); + onCancel(); + } catch (error) {} + }; + + const handleStopMaintenance = async (row: ListItem) => { + try { + await updateWorker(row.id, { + name: row.name, + labels: row.labels, + maintenance: { + enabled: false, + message: '' + } + }); + message.success(intl.formatMessage({ id: 'common.message.success' })); + fetchData(); + onCancel(); + } catch (error) {} + }; + + const MaintenanceModal = ( + + } + > +
+ + + + + + +
+ + ); + + return { + MaintenanceModal, + handleStopMaintenance, + setOpenStatus + }; +}; + +export default useWorkerMaintenance;