feat: worker maintenance

src/global.less

@@ -235,6 +235,7 @@ body {
 
     .ant-pro-sider-logo-collapsed {
       padding-left: 12px;
+      cursor: e-resize;
 
       .collapse-wrap {
         display: none;

src/layouts/index.tsx

@@ -285,22 +285,21 @@ export default (props: any) => {
         {logo}
         <div className="collapse-wrap" onClick={handleToggleCollapse}>
           <Button
-            style={{ marginRight: collapsed ? 0 : -14, border: 'none' }}
+            style={{
+              marginRight: collapsed ? 0 : -14,
+              border: 'none',
+              cursor: 'w-resize'
+            }}
             size="small"
             type={collapsed ? 'default' : 'text'}
           >
-            <>
+            <IconFont
-              <IconFont
+              type={collapsed ? 'icon-expand-left' : 'icon-expand-right'}
-                type="icon-expand-left"
+              className="font-size-18 text-secondary"
-                className="font-size-18 text-secondary"
+              style={{
-                style={{ display: collapsed ? 'block' : 'none' }}
+                display: 'block'
-              />
+              }}
-              <IconFont
+            />
-                type="icon-expand-right"
-                className="font-size-18 text-secondary"
-                style={{ display: !collapsed ? 'block' : 'none' }}
-              />
-            </>
           </Button>
         </div>
       </>

src/locales/en-US/resources.ts

@@ -96,5 +96,13 @@ export default {
   'resources.model.instance': 'Model Instance',
   'resources.worker.download.privatekey': 'Download Private Key',
   'resources.modelfiles.form.exsting': 'Downloaded',
-  'resources.modelfiles.form.added': 'Added'
+  'resources.modelfiles.form.added': 'Added',
+  'resources.worker.maintenance.title': 'System Maintenance',
+  'resources.worker.maintenance.enable': 'Enter Maintenance Mode',
+  'resources.worker.maintenance.disable': 'Exit Maintenance Mode',
+  'resources.worker.maintenance.remark': 'Remark',
+  'resources.worker.maintenance.remark.rules':
+    'Please enter maintenance remarks',
+  'resources.worker.maintenance.tips':
+    'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.'
 };

src/locales/ja-JP/resources.ts

@@ -97,7 +97,15 @@ export default {
   'resources.worker.download.privatekey': 'Download Private Key',
   'resources.worker': 'Worker',
   'resources.modelfiles.form.exsting': 'Downloaded',
-  'resources.modelfiles.form.added': 'Added'
+  'resources.modelfiles.form.added': 'Added',
+  'resources.worker.maintenance.title': 'System Maintenance',
+  'resources.worker.maintenance.enable': 'Enter Maintenance Mode',
+  'resources.worker.maintenance.disable': 'Exit Maintenance Mode',
+  'resources.worker.maintenance.remark': 'Remark',
+  'resources.worker.maintenance.remark.rules':
+    'Please enter maintenance remarks',
+  'resources.worker.maintenance.tips':
+    'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.'
 };
 
 // ========== To-Do: Translate Keys (Remove After Translation) ==========
@@ -117,5 +125,11 @@ export default {
 // 14. 'resources.worker.download.privatekey': 'Download Private Key',
 // 15. 'resources.worker': 'Worker',
 // 16. 'resources.modelfiles.form.exsting': 'Downloaded',
-// 17. 'resources.modelfiles.form.added': 'Added'
+// 17. 'resources.modelfiles.form.added': 'Added',
+// 18. 'resources.worker.maintenance.title': 'System Maintenance',
+// 19. 'resources.worker.maintenance.enable': 'Enter Maintenance Mode',
+// 20. 'resources.worker.maintenance.disable': 'Exit Maintenance Mode',
+// 21. 'resources.worker.maintenance.remark': 'Remark',
+// 22. 'resources.worker.maintenance.remark.rules': 'Please enter maintenance remarks',
+// 23. 'resources.worker.maintenance.tips': 'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.'
 // ========== End of To-Do List ==========

src/locales/ru-RU/resources.ts

@@ -95,9 +95,22 @@ export default {
   'resources.worker.download.privatekey': 'Скачать приватный ключ',
   'resources.worker': 'Рабочий узел',
   'resources.modelfiles.form.exsting': 'Загружено',
-  'resources.modelfiles.form.added': 'Добавлено'
+  'resources.modelfiles.form.added': 'Добавлено',
+  'resources.worker.maintenance.title': 'System Maintenance',
+  'resources.worker.maintenance.enable': 'Enter Maintenance Mode',
+  'resources.worker.maintenance.disable': 'Exit Maintenance Mode',
+  'resources.worker.maintenance.remark': 'Remark',
+  'resources.worker.maintenance.remark.rules':
+    'Please enter maintenance remarks',
+  'resources.worker.maintenance.tips':
+    'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.'
 };
 
 // ========== To-Do: Translate Keys (Remove After Translation) ==========
-
+// 2. 'resources.worker.maintenance.title': 'System Maintenance',
+// 3. 'resources.worker.maintenance.enable': 'Enter Maintenance Mode',
+// 4. 'resources.worker.maintenance.disable': 'Exit Maintenance Mode',
+// 5. 'resources.worker.maintenance.remark': 'Remark',
+// 6. 'resources.worker.maintenance.remark.rules': 'Please enter maintenance remarks',
+// 7. 'resources.worker.maintenance.tips': 'When maintenance mode is enabled, the node will stop scheduling new model deployment tasks. Running instances will not be affected.'
 // ========== End of To-Do List ==========

src/locales/zh-CN/resources.ts

@@ -94,5 +94,12 @@ export default {
   'resources.model.instance': '模型实例',
   'resources.worker.download.privatekey': '下载私钥',
   'resources.modelfiles.form.exsting': '已下载',
-  'resources.modelfiles.form.added': '已添加'
+  'resources.modelfiles.form.added': '已添加',
+  'resources.worker.maintenance.title': '系统维护',
+  'resources.worker.maintenance.enable': '开启维护',
+  'resources.worker.maintenance.disable': '停止维护',
+  'resources.worker.maintenance.remark': '备注',
+  'resources.worker.maintenance.remark.rules': '请输入维护备注',
+  'resources.worker.maintenance.tips':
+    '启用维护后，节点将停止调度新的模型实例部署任务，正在运行的实例不会受到影响。'
 };

src/pages/cluster-management/components/add-worker-command.tsx

@@ -4,6 +4,7 @@ import React from 'react';
 
 type ViewModalProps = {
   currentGPU?: string;
+  workerIP?: string;
   registrationInfo: {
     token: string;
     image: string;
@@ -13,6 +14,7 @@ type ViewModalProps = {
 
 const AddWorkerCommand: React.FC<ViewModalProps> = ({
   registrationInfo,
+  workerIP,
   currentGPU
 }) => {
   const code = React.useMemo(() => {
@@ -21,10 +23,11 @@ const AddWorkerCommand: React.FC<ViewModalProps> = ({
       gpu: currentGPU || '',
       server: registrationInfo.server_url || origin,
       tag: '',
+      workerIP: '${WORKER_IP}',
       image: registrationInfo.image,
       token: registrationInfo.token || '${token}'
     });
-  }, [registrationInfo, currentGPU]);
+  }, [registrationInfo, currentGPU, workerIP]);
 
   return (
     <HighlightCode

src/pages/cluster-management/components/add-worker-step.tsx

@@ -89,7 +89,6 @@ const AddWorkerStep: React.FC<AddModalProps> = ({
   const [currentGPU, setCurrentGPU] = React.useState<string>(
     GPUDriverMap.NVIDIA
   );
-  React.useState<string>(GPUDriverMap.NVIDIA);
   const [workerCommand, setWorkerCommand] = React.useState<Record<string, any>>(
     {
       label: 'NVIDIA',

src/pages/resources/components/workers.tsx

@@ -20,6 +20,7 @@ import {
 } from '../apis';
 import { ListItem } from '../config/types';
 import useWorkerColumns from '../hooks/use-worker-columns';
+import useWorkerMaintenance from '../hooks/use-worker-maintenance';
 import UpdateLabels from './update-labels';
 import WorkerDetailModal from './worker-detail-modal';
 
@@ -46,6 +47,8 @@ const Workers: React.FC = () => {
     API: WORKERS_API
   });
   const [, setClusterSession] = useAtom(clusterSessionAtom);
+  const { MaintenanceModal, handleStopMaintenance, setOpenStatus } =
+    useWorkerMaintenance({ fetchData });
 
   const navigate = useNavigate();
   const intl = useIntl();
@@ -154,6 +157,17 @@ const Workers: React.FC = () => {
         name: record.name
       });
     }
+
+    if (val === 'star_maintenance') {
+      setOpenStatus({
+        open: true,
+        currentData: record
+      });
+    }
+
+    if (val === 'stop_maintenance') {
+      handleStopMaintenance(record);
+    }
   });
 
   const handleAddWorker = () => {
@@ -253,6 +267,7 @@ const Workers: React.FC = () => {
             setWorkerDetailStatus({ currentData: null, open: false })
           }
         />
+        {MaintenanceModal}
       </PageBox>
     </>
   );

src/pages/resources/config/gpu-driver.ts

@@ -134,25 +134,19 @@ const registerWorker = (params: {
   token: string;
   image: string;
   gpu: string;
+  workerIP?: string;
 }) => {
   const config = GPUsConfigs[params.gpu];
-  return `CONTAINER_NAME="gpustack-worker" \\
+  return `sudo docker run -d --name gpustack-worker \\
-WORKER_PORT="80" \\
-WORKER_IP="" \\
-sudo docker run -d --name \${CONTAINER_NAME} \\
       --restart=unless-stopped \\
       --privileged \\
-      --publish \${WORKER_PORT}:\${WORKER_PORT} \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\
       --volume /var/run/docker.sock:/var/run/docker.sock \\
       --volume gpustack-data:/var/lib/gpustack \\
       --runtime ${config.runtime} \\
       ${params.image} \\
       --server-url ${params.server} \\
       --token ${params.token} \\
-      --worker-port \${WORKER_PORT} \\
+      ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim();
-      --advertise-address \${WORKER_IP}`;
 };
 
 // avaliable for Ascend
@@ -162,17 +156,13 @@ const registerAscendWorker = (params: {
   token: string;
   image: string;
   gpu: string;
+  workerIP?: string;
 }) => {
   const config = GPUsConfigs[params.gpu];
-  return `CONTAINER_NAME="gpustack-worker" \\
+  return `sudo docker run -d --name gpustack-worker \\
-WORKER_PORT="80" \\
-WORKER_IP="" \\
-sudo docker run -d --name \${CONTAINER_NAME} \\
       --restart=unless-stopped \\
       --privileged \\
-      --publish \${WORKER_PORT}:\${WORKER_PORT} \\
+      --network=host \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\
       --env "ASCEND_VISIBLE_DEVICES=$(npu-smi info -m | tail -n 1 | awk '{print $1}') \\
       --volume /var/run/docker.sock:/var/run/docker.sock \\
       --volume gpustack-data:/var/lib/gpustack \\
@@ -180,8 +170,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\
       ${params.image} \\
       --server-url ${params.server} \\
       --token ${params.token} \\
-      --worker-port \${WORKER_PORT} \\
+      ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim();
-      --advertise-address \${WORKER_IP}`;
 };
 
 const registerHygonWorker = (params: {
@@ -190,17 +179,13 @@ const registerHygonWorker = (params: {
   token: string;
   image: string;
   gpu: string;
+  workerIP?: string;
 }) => {
   const config = GPUsConfigs[params.gpu];
-  return `CONTAINER_NAME="gpustack-worker" \\
+  return `sudo docker run -d --name gpustack-worker \\
-WORKER_PORT="80" \\
-WORKER_IP="" \\
-sudo docker run -d --name \${CONTAINER_NAME} \\
       --restart=unless-stopped \\
       --privileged \\
-      --publish \${WORKER_PORT}:\${WORKER_PORT} \\
+      --network=host \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\
       --env ROCM_PATH=/opt/dtk \\
       --env ROCM_SMI_LIB_PATH=/opt/hyhal/lib \\
       --volume /var/run/docker.sock:/var/run/docker.sock \\
@@ -210,8 +195,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\
       ${params.image} \\
       --server-url ${params.server} \\
       --token ${params.token} \\
-      --worker-port \${WORKER_PORT} \\
+      ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim();
-      --advertise-address \${WORKER_IP}`;
 };
 
 const registerIluvatarWorker = (params: {
@@ -220,17 +204,13 @@ const registerIluvatarWorker = (params: {
   token: string;
   image: string;
   gpu: string;
+  workerIP?: string;
 }) => {
   const config = GPUsConfigs[params.gpu];
-  return `CONTAINER_NAME="gpustack-worker" \\
+  return `sudo docker run -d --name gpustack-worker \\
-WORKER_PORT="80" \\
-WORKER_IP="" \\
-sudo docker run -d --name \${CONTAINER_NAME} \\
       --restart=unless-stopped \\
       --privileged \\
-      --publish \${WORKER_PORT}:\${WORKER_PORT} \\
+      --network=host \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\
       --volume /var/run/docker.sock:/var/run/docker.sock \\
       --volume gpustack-data:/var/lib/gpustack \\
       --volume /lib/modules:/lib/modules:ro \\
@@ -240,8 +220,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\
       ${params.image} \\
       --server-url ${params.server} \\
       --token ${params.token} \\
-      --worker-port \${WORKER_PORT} \\
+      ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim();
-      --advertise-address \${WORKER_IP}`;
 };
 
 const registerMetaXWorker = (params: {
@@ -250,17 +229,13 @@ const registerMetaXWorker = (params: {
   token: string;
   image: string;
   gpu: string;
+  workerIP?: string;
 }) => {
   const config = GPUsConfigs[params.gpu];
-  return `CONTAINER_NAME="gpustack-worker" \\
+  return `sudo docker run -d --name gpustack-worker \\
-WORKER_PORT="80" \\
-WORKER_IP="" \\
-sudo docker run -d --name \${CONTAINER_NAME} \\
       --restart=unless-stopped \\
       --privileged \\
-      --publish \${WORKER_PORT}:\${WORKER_PORT} \\
+      --network=host \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\
       --volume /var/run/docker.sock:/var/run/docker.sock \\
       --volume gpustack-data:/var/lib/gpustack \\
       --volume /opt/mxdriver:/opt/mxdriver:ro \\
@@ -268,8 +243,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\
       ${params.image} \\
       --server-url ${params.server} \\
       --token ${params.token} \\
-      --worker-port \${WORKER_PORT} \\
+      ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim();
-      --advertise-address \${WORKER_IP}`;
 };
 
 const registerCambriconWorker = (params: {
@@ -278,17 +252,13 @@ const registerCambriconWorker = (params: {
   token: string;
   image: string;
   gpu: string;
+  workerIP?: string;
 }) => {
   const config = GPUsConfigs[params.gpu];
-  return `CONTAINER_NAME="gpustack-worker" \\
+  return `sudo docker run -d --name gpustack-worker \\
-WORKER_PORT="80" \\
-WORKER_IP="" \\
-sudo docker run -d --name \${CONTAINER_NAME} \\
       --restart=unless-stopped \\
       --privileged \\
-      --publish \${WORKER_PORT}:\${WORKER_PORT} \\
+      --network=host \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT=true" \\
-      --env "GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME=\${CONTAINER_NAME}" \\
       --volume /var/run/docker.sock:/var/run/docker.sock \\
       --volume gpustack-data:/var/lib/gpustack \\
       --volume /usr/local/neuware:/usr/local/neuware:ro \\
@@ -296,8 +266,7 @@ sudo docker run -d --name \${CONTAINER_NAME} \\
       ${params.image} \\
       --server-url ${params.server} \\
       --token ${params.token} \\
-      --worker-port \${WORKER_PORT} \\
+      ${params.workerIP ? `--advertise-address ${params.workerIP}` : ''}`.trim();
-      --advertise-address \${WORKER_IP}`;
 };
 
 export const registerAddWokerCommandMap = {

src/pages/resources/config/index.ts

@@ -14,7 +14,8 @@ export const WorkerStatusMap = {
   deleting: 'deleting',
   error: 'error',
   pending: 'pending',
-  initializing: 'initializing'
+  initializing: 'initializing',
+  maintenance: 'maintenance'
 };
 
 export const WorkerStatusMapValue = {
@@ -25,7 +26,8 @@ export const WorkerStatusMapValue = {
   [WorkerStatusMap.deleting]: 'Deleting',
   [WorkerStatusMap.error]: 'Error',
   [WorkerStatusMap.initializing]: 'Initializing',
-  [WorkerStatusMap.pending]: 'Pending'
+  [WorkerStatusMap.pending]: 'Pending',
+  [WorkerStatusMap.maintenance]: 'Maintenance'
 };
 
 export const status: any = {
@@ -36,7 +38,8 @@ export const status: any = {
   [WorkerStatusMap.deleting]: StatusMaps.transitioning,
   [WorkerStatusMap.error]: StatusMaps.error,
   [WorkerStatusMap.pending]: StatusMaps.transitioning,
-  [WorkerStatusMap.initializing]: StatusMaps.transitioning
+  [WorkerStatusMap.initializing]: StatusMaps.transitioning,
+  [WorkerStatusMap.maintenance]: StatusMaps.warning
 };
 export const checkEnvCommand = (gpu: string) => {
   return {

src/pages/resources/config/types.ts

@@ -68,6 +68,10 @@ export interface ListItem {
   state_message: string;
   ssh_key_id: string;
   provision_progress: string;
+  maintenance: {
+    enabled: boolean;
+    message: string;
+  };
   status: {
     cpu: {
       total: number;

src/pages/resources/hooks/use-worker-columns.tsx

@@ -9,7 +9,9 @@ import {
   DeleteOutlined,
   DownloadOutlined,
   EditOutlined,
-  InfoCircleOutlined
+  InfoCircleOutlined,
+  SafetyOutlined,
+  ToolOutlined
 } from '@ant-design/icons';
 import { useIntl } from '@umijs/max';
 import { Tooltip } from 'antd';
@@ -31,6 +33,16 @@ const ActionList = [
     key: 'download_ssh_key',
     icon: <DownloadOutlined />
   },
+  {
+    label: 'resources.worker.maintenance.enable',
+    key: 'star_maintenance',
+    icon: <ToolOutlined />
+  },
+  {
+    label: 'resources.worker.maintenance.disable',
+    key: 'stop_maintenance',
+    icon: <SafetyOutlined />
+  },
   // {
   //   label: 'common.button.logs',
   //   locale: false,
@@ -51,6 +63,12 @@ const setActions = (row: ListItem) => {
     if (action.key === 'download_ssh_key') {
       return !!row.ssh_key_id;
     }
+    if (action.key === 'star_maintenance') {
+      return !row.maintenance?.enabled;
+    }
+    if (action.key === 'stop_maintenance') {
+      return row.maintenance?.enabled;
+    }
     return true;
   });
 };

src/pages/resources/hooks/use-worker-maintenance.tsx

@@ -0,0 +1,132 @@
+import ModalFooter from '@/components/modal-footer';
+import ScrollerModal from '@/components/scroller-modal';
+import SealInput from '@/components/seal-form/seal-input';
+import SealTextArea from '@/components/seal-form/seal-textarea';
+import { useIntl } from '@umijs/max';
+import { Form, message } from 'antd';
+import React from 'react';
+import { updateWorker } from '../apis';
+import { ListItem } from '../config/types';
+
+const useWorkerMaintenance = ({ fetchData }: { fetchData: () => void }) => {
+  const intl = useIntl();
+  const [form] = Form.useForm();
+  const [openStatus, setOpenStatus] = React.useState<{
+    open: boolean;
+    currentData: ListItem;
+  }>({
+    open: false,
+    currentData: {} as ListItem
+  });
+
+  const handleSumit = () => {
+    form.submit();
+  };
+
+  const onCancel = () => {
+    setOpenStatus({
+      open: false,
+      currentData: {} as ListItem
+    });
+  };
+
+  const onOk = async (values: any) => {
+    try {
+      await updateWorker(openStatus.currentData.id, {
+        name: openStatus.currentData.name,
+        labels: openStatus.currentData.labels,
+        maintenance: {
+          enabled: true,
+          message: values.remark
+        }
+      });
+      message.success(intl.formatMessage({ id: 'common.message.success' }));
+      fetchData();
+      onCancel();
+    } catch (error) {}
+  };
+
+  const handleStopMaintenance = async (row: ListItem) => {
+    try {
+      await updateWorker(row.id, {
+        name: row.name,
+        labels: row.labels,
+        maintenance: {
+          enabled: false,
+          message: ''
+        }
+      });
+      message.success(intl.formatMessage({ id: 'common.message.success' }));
+      fetchData();
+      onCancel();
+    } catch (error) {}
+  };
+
+  const MaintenanceModal = (
+    <ScrollerModal
+      title={intl.formatMessage({ id: 'resources.worker.maintenance.title' })}
+      open={openStatus.open}
+      centered={true}
+      onCancel={onCancel}
+      destroyOnHidden={true}
+      closeIcon={true}
+      maskClosable={false}
+      keyboard={false}
+      width={600}
+      maxContentHeight={'max(calc(100vh - 300px), 500px)'}
+      footer={
+        <ModalFooter onOk={handleSumit} onCancel={onCancel}></ModalFooter>
+      }
+    >
+      <Form
+        form={form}
+        onFinish={onOk}
+        preserve={false}
+        clearOnDestroy={true}
+        initialValues={{
+          name: openStatus.currentData.name
+        }}
+      >
+        <Form.Item name="name">
+          <SealInput.Input
+            label={intl.formatMessage({
+              id: 'common.table.name'
+            })}
+            disabled
+          />
+        </Form.Item>
+        <Form.Item
+          name="remark"
+          rules={[
+            {
+              required: true,
+              message: intl.formatMessage({
+                id: 'resources.worker.maintenance.remark.rules'
+              })
+            }
+          ]}
+        >
+          <SealTextArea
+            required
+            alwaysFocus
+            label={intl.formatMessage({
+              id: 'resources.worker.maintenance.remark'
+            })}
+            description={intl.formatMessage({
+              id: 'resources.worker.maintenance.tips'
+            })}
+            autoSize={{ minRows: 4, maxRows: 6 }}
+          ></SealTextArea>
+        </Form.Item>
+      </Form>
+    </ScrollerModal>
+  );
+
+  return {
+    MaintenanceModal,
+    handleStopMaintenance,
+    setOpenStatus
+  };
+};
+
+export default useWorkerMaintenance;