chore: model instance show gpu info

src/components/auto-tooltip/index.tsx

@@ -14,12 +14,14 @@ interface AutoTooltipProps extends React.ComponentProps<typeof Tag> {
   color?: string;
   style?: React.CSSProperties;
   ghost?: boolean;
+  showTitle?: boolean;
 }
 
 const AutoTooltip: React.FC<AutoTooltipProps> = ({
   children,
   maxWidth = '100%',
   ghost = false,
+  showTitle = false,
   ...tagProps
 }) => {
   const contentRef = useRef<HTMLDivElement>(null);
@@ -62,7 +64,7 @@ const AutoTooltip: React.FC<AutoTooltipProps> = ({
   );
 
   return (
-    <Tooltip title={isOverflowing ? children : ''}>
+    <Tooltip title={isOverflowing || showTitle ? children : ''}>
       {ghost ? (
         <div ref={contentRef} style={tagStyle}>
           {children}

src/components/seal-form/components/wrapper.less

@@ -159,6 +159,10 @@
     background-color: @bgColor !important;
   }
 
+  :global(.ant-select-selection-search) {
+    inset-inline-start: 8px !important;
+  }
+
   :global(.ant-select-selection-item) {
     height: @inputheight !important;
     padding-block: 5px !important;

src/components/seal-table/styles/row-children.less

@@ -3,7 +3,7 @@
   display: flex;
   align-items: center;
   height: 54px;
-  border-radius: var(--ant-table-header-border-radius);
+  border-radius: var(--border-radius-mdium);
   transition: all 0.2s ease;
 
   &:hover {

src/components/simple-table/header.tsx

@@ -0,0 +1,22 @@
+import React from 'react';
+
+interface TableHeaderProps {
+  columns: any[];
+}
+interface TableHeaderProps {
+  columns: any[];
+}
+const TableHeader = ({ columns }: TableHeaderProps) => {
+  return (
+    <tr>
+      {columns.map((column: any, index: number) => {
+        return (
+          <th key={index}>
+            <span className="cell-span">{column.title}</span>
+          </th>
+        );
+      })}
+    </tr>
+  );
+};
+export default React.memo(TableHeader);

src/components/simple-table/index.less

@@ -0,0 +1,33 @@
+.simple-table {
+  table-layout: auto;
+  width: 100%;
+  text-align: left;
+
+  &.simple-table-bordered {
+    border-collapse: collapse;
+    border-spacing: 0;
+
+    td {
+      border-bottom: 1px solid rgba(255, 255, 255, 10%);
+    }
+
+    th {
+      border-bottom: 1px solid rgba(255, 255, 255, 10%);
+    }
+
+    tr:last-child td {
+      border-bottom: none;
+    }
+  }
+
+  th {
+    height: 36px;
+    font-weight: 600;
+  }
+
+  .cell-span {
+    display: flex;
+    padding: 4px 6px;
+    min-height: 32px;
+  }
+}

src/components/simple-table/index.tsx

@@ -0,0 +1,38 @@
+import classNames from 'classnames';
+import React from 'react';
+import SimpleBar from 'simplebar-react';
+import 'simplebar-react/dist/simplebar.min.css';
+import TableHeader from './header';
+import './index.less';
+import TableRow from './row';
+
+interface SimpleTableProps {
+  columns: any[];
+  dataSource: any[];
+  bordered?: boolean;
+}
+const SimpleTabel: React.FC<SimpleTableProps> = (props) => {
+  const { columns, dataSource, bordered = true } = props;
+  return (
+    <SimpleBar style={{ maxHeight: 200 }}>
+      <table
+        className={classNames('simple-table', {
+          'simple-table-bordered': bordered
+        })}
+      >
+        <thead>
+          <TableHeader columns={columns}></TableHeader>
+        </thead>
+        <tbody>
+          {dataSource.map((item: any, index: number) => {
+            return (
+              <TableRow row={item} columns={columns} key="index"></TableRow>
+            );
+          })}
+        </tbody>
+      </table>
+    </SimpleBar>
+  );
+};
+
+export default React.memo(SimpleTabel);

src/components/simple-table/row.tsx

@@ -0,0 +1,25 @@
+import React from 'react';
+
+interface TableRowProps {
+  row: any;
+  columns: any;
+}
+const TableRow = ({ row, columns }: TableRowProps) => {
+  return (
+    <tr>
+      {columns.map((column: any, index: number) => {
+        return (
+          <td key={index}>
+            <span className="cell-span">
+              {column.render
+                ? column.render({ dataIndex: column.key, row: row })
+                : row[column.key]}
+            </span>
+          </td>
+        );
+      })}
+    </tr>
+  );
+};
+
+export default React.memo(TableRow);

src/global.less

@@ -23,6 +23,7 @@ html {
   --border-radius-base: 8px;
   --border-radius-middle: 20px;
   --border-radius-small: 8px;
+  --border-radius-mdium: 6px;
   --border-radius-mini: 4px;
   --color-white-1: rgba(255, 255, 255, 100%);
   --color-fill-sider: #f4f5f4;
@@ -555,6 +556,10 @@ body {
   background-size: 400% 100%;
 }
 
+.ant-modal {
+  max-width: 100vw;
+}
+
 .tooltip-wrapper {
   display: flex;
   flex-direction: column;

src/locales/en-US/models.ts

@@ -48,5 +48,10 @@ export default {
     'Automatically deploys model instances to appropriate GPUs/Workers based on current resource conditions.',
   'models.form.scheduletype.manual.tips':
     'Allows you to manually specify the GPUs/Workers to deploy the model instances to.',
-  'models.form.manual.schedule': 'Manual Schedule'
+  'models.form.manual.schedule': 'Manual Schedule',
+  'models.table.gpuindex': 'GPU Index',
+  'models.table.backend': 'Backends',
+  'models.table.acrossworker': 'Distribution Across Workers',
+  'models.table.cpuoffload': 'CPU Offload',
+  'models.table.layers': 'Layers'
 };

src/locales/zh-CN/models.ts

@@ -47,5 +47,10 @@ export default {
     '自动根据当前资源情况部署模型实例到合适的 GPU/Worker。',
   'models.form.scheduletype.manual.tips':
     '手动调度可指定模型实例部署的 GPU/Worker。',
-  'models.form.manual.schedule': '手动调度'
+  'models.form.manual.schedule': '手动调度',
+  'models.table.gpuindex': 'GPU 序号',
+  'models.table.backend': '后端',
+  'models.table.acrossworker': '跨 worker 推理',
+  'models.table.cpuoffload': 'CPU 卸载',
+  'models.table.layers': '层'
 };

src/locales/zh-CN/resources.ts

@@ -13,7 +13,7 @@ export default {
   'resources.form.placementStrategy': '放置策略',
   'resources.form.workerSelector': 'Worker 选择器',
   'resources.form.enableDistributedInferenceAcrossWorkers':
-    '允许跨节点分布式推理',
+    '允许跨 worker 分布式推理',
   'resources.form.spread.tips':
     '使得集群整体的资源在所有 Worker 之间分配得相对均匀。可能会在单个 Worker 上产生较多资源碎片。',
   'resources.form.binpack.tips':

src/pages/llmodels/components/instance-item.tsx

@@ -1,13 +1,18 @@
 import DropdownButtons from '@/components/drop-down-buttons';
 import RowChildren from '@/components/seal-table/components/row-children';
+import SimpleTabel from '@/components/simple-table';
 import StatusTag from '@/components/status-tag';
+import {
+  GPUDeviceItem,
+  ListItem as WorkerListItem
+} from '@/pages/resources/config/types';
 import {
   DeleteOutlined,
   FieldTimeOutlined,
   InfoCircleOutlined
 } from '@ant-design/icons';
 import { useIntl } from '@umijs/max';
-import { Col, Row, Space, Tooltip } from 'antd';
+import { Col, Divider, Row, Space, Tag, Tooltip } from 'antd';
 import dayjs from 'dayjs';
 import _ from 'lodash';
 import React, { useCallback } from 'react';
@@ -16,6 +21,8 @@ import { ModelInstanceListItem } from '../config/types';
 
 interface InstanceItemProps {
   list: ModelInstanceListItem[];
+  gpuDeviceList: GPUDeviceItem[];
+  workerList: WorkerListItem[];
   handleChildSelect: (
     val: string,
     item: ModelInstanceListItem,
@@ -25,12 +32,29 @@ interface InstanceItemProps {
 
 const InstanceItem: React.FC<InstanceItemProps> = ({
   list,
+  workerList,
   handleChildSelect
 }) => {
-  console.log('instance item====');
-
   const intl = useIntl();
 
+  const distributeCols = [
+    {
+      title: 'Worker',
+      key: 'worker_name'
+    },
+    {
+      title: 'IP',
+      key: 'worker_ip',
+      render: ({ row }: { row: ModelInstanceListItem }) => {
+        return row.port ? `${row.worker_ip}:${row.port}` : row.worker_ip;
+      }
+    },
+    {
+      title: intl.formatMessage({ id: 'models.table.gpuindex' }),
+      key: 'gpu_index'
+    }
+  ];
+
   const childActionList = [
     {
       label: 'common.button.viewlog',
@@ -70,6 +94,44 @@ const InstanceItem: React.FC<InstanceItemProps> = ({
       <div>
         <div>{item.worker_name}</div>
         <div>{workerIp}</div>
+        <div>
+          {intl.formatMessage({ id: 'models.table.gpuindex' })}: [
+          {_.join(item.gpu_indexes, ',')}]
+        </div>
+      </div>
+    );
+  };
+
+  const renderDistributionInfo = (item: ModelInstanceListItem) => {
+    const rpcServerList = item.distributed_servers?.rpc_servers || [];
+    const list = _.map(rpcServerList, (item: any) => {
+      const data = _.find(workerList, { id: item.worker_id });
+      return {
+        woker_name: data?.name,
+        work_ip: data.ip,
+        port: item.port,
+        gpu_index: item.gpu_index
+      };
+    });
+
+    const mainWorker = [
+      {
+        worker_name: `${item.worker_name}`,
+        worker_ip: `${item.worker_ip}`,
+        port: item.port,
+        gpu_index: `${item.gpu_indexes} (main)`
+      }
+    ];
+
+    return (
+      <div>
+        <h3 style={{ margin: 0 }}>
+          {intl.formatMessage({ id: 'models.table.backend' })}
+        </h3>
+        <SimpleTabel
+          columns={distributeCols}
+          dataSource={[...mainWorker, ...list]}
+        ></SimpleTabel>
       </div>
     );
   };
@@ -95,10 +157,81 @@ const InstanceItem: React.FC<InstanceItemProps> = ({
                     <InfoCircleOutlined />
                   </Tooltip>
                 </Col>
-                <Col span={6}></Col>
+                <Col span={6}>
+                  <span
+                    style={{ paddingLeft: '58px' }}
+                    className="flex align-center"
+                  >
+                    {item.computed_resource_claim?.total_layers !==
+                      item.computed_resource_claim?.offload_layers && (
+                      <Tooltip
+                        title={
+                          <span className="flex flex-center">
+                            <span>
+                              CPU:{' '}
+                              {_.subtract(
+                                item.computed_resource_claim?.total_layers,
+                                item.computed_resource_claim?.offload_layers
+                              ) || 0}{' '}
+                              {intl.formatMessage({
+                                id: 'models.table.layers'
+                              })}
+                            </span>
+                            <Divider
+                              type="vertical"
+                              style={{
+                                borderColor: '#fff',
+                                opacity: 0.5
+                              }}
+                            ></Divider>
+                            <span>
+                              GPU:{' '}
+                              {item.computed_resource_claim?.offload_layers}{' '}
+                              {intl.formatMessage({
+                                id: 'models.table.layers'
+                              })}
+                            </span>
+                          </span>
+                        }
+                      >
+                        <Tag
+                          color="cyan"
+                          style={{
+                            opacity: 0.75
+                          }}
+                        >
+                          <InfoCircleOutlined className="m-r-5" />
+                          {intl.formatMessage({
+                            id: 'models.table.cpuoffload'
+                          })}
+                        </Tag>
+                      </Tooltip>
+                    )}
+                    {item?.distributed_servers?.rpc_servers?.length && (
+                      <Tooltip
+                        overlayInnerStyle={{
+                          width: '400px'
+                        }}
+                        title={renderDistributionInfo(item)}
+                      >
+                        <Tag
+                          color="processing"
+                          style={{
+                            opacity: 0.75
+                          }}
+                        >
+                          <InfoCircleOutlined className="m-r-5" />
+                          {intl.formatMessage({
+                            id: 'models.table.acrossworker'
+                          })}
+                        </Tag>
+                      </Tooltip>
+                    )}
+                  </span>
+                </Col>
                 <Col span={4}>
                   <span
-                    style={{ paddingLeft: '68px' }}
+                    style={{ paddingLeft: '62px' }}
                     className="flex justify-center"
                   >
                     {item.state && (

src/pages/llmodels/components/table-list.tsx

@@ -10,7 +10,11 @@ import useExpandedRowKeys from '@/hooks/use-expanded-row-keys';
 import useTableRowSelection from '@/hooks/use-table-row-selection';
 import useTableSort from '@/hooks/use-table-sort';
 import ViewCodeModal from '@/pages/playground/components/view-code-modal';
-import { handleBatchRequest, platformCall } from '@/utils';
+import {
+  GPUDeviceItem,
+  ListItem as WorkerListItem
+} from '@/pages/resources/config/types';
+import { handleBatchRequest } from '@/utils';
 import {
   DeleteOutlined,
   DownOutlined,
@@ -51,22 +55,24 @@ interface ModelsProps {
     perPage: number;
     query?: string;
   };
+  gpuDeviceList: GPUDeviceItem[];
+  workerList: WorkerListItem[];
   dataSource: ListItem[];
   loading: boolean;
   total: number;
 }
 
 const Models: React.FC<ModelsProps> = ({
-  dataSource,
   handleNameChange,
   handleSearch,
   handlePageChange,
+  dataSource,
+  gpuDeviceList,
+  workerList,
   queryParams,
   loading,
   total
 }) => {
-  console.log('model list====2');
-  const platform = platformCall();
   const access = useAccess();
   const intl = useIntl();
   const navigate = useNavigate();
@@ -80,6 +86,7 @@ const Models: React.FC<ModelsProps> = ({
     params: {},
     show: false
   });
+
   const [openLogModal, setOpenLogModal] = useState(false);
   const [openAddModal, setOpenAddModal] = useState(false);
   const [openDeployModal, setOpenDeployModal] = useState<any>({
@@ -194,7 +201,6 @@ const Models: React.FC<ModelsProps> = ({
   }, []);
 
   const handleOnSort = (dataIndex: string, order: any) => {
-    console.log('handleTableChange=======', dataIndex, order);
     setSortOrder(order);
   };
 
@@ -213,15 +219,9 @@ const Models: React.FC<ModelsProps> = ({
     message.success(intl.formatMessage({ id: 'common.message.success' }));
   };
 
-  const handleAddModal = () => {
-    setOpenAddModal(true);
-    setTitle(intl.formatMessage({ id: 'models.button.deploy' }));
-  };
-
   const handleModalOk = useCallback(
     async (data: FormData) => {
       try {
-        console.log('data:', data);
         await updateModel({ data, id: currentData?.id as number });
         setOpenAddModal(false);
         message.success(intl.formatMessage({ id: 'common.message.success' }));
@@ -242,18 +242,21 @@ const Models: React.FC<ModelsProps> = ({
     });
   };
 
-  const handleCreateModel = useCallback(async (data: FormData) => {
+  const handleCreateModel = useCallback(
-    try {
+    async (data: FormData) => {
-      console.log('data:', data, openDeployModal);
+      try {
+        console.log('data:', data, openDeployModal);
 
-      await createModel({ data });
+        await createModel({ data });
-      setOpenDeployModal({
+        setOpenDeployModal({
-        ...openDeployModal,
+          ...openDeployModal,
-        show: false
+          show: false
-      });
+        });
-      message.success(intl.formatMessage({ id: 'common.message.success' }));
+        message.success(intl.formatMessage({ id: 'common.message.success' }));
-    } catch (error) {}
+      } catch (error) {}
-  }, []);
+    },
+    [openDeployModal]
+  );
 
   const handleLogModalCancel = useCallback(() => {
     setOpenLogModal(false);
@@ -327,7 +330,6 @@ const Models: React.FC<ModelsProps> = ({
 
   const handleSelect = useCallback((val: any, row: ListItem) => {
     if (val === 'edit') {
-      console.log('row=======', row);
       handleEdit(row);
     }
     if (val === 'chat') {
@@ -363,6 +365,8 @@ const Models: React.FC<ModelsProps> = ({
     return (
       <InstanceItem
         list={list}
+        gpuDeviceList={gpuDeviceList}
+        workerList={workerList}
         handleChildSelect={handleChildSelect}
       ></InstanceItem>
     );
@@ -502,7 +506,7 @@ const Models: React.FC<ModelsProps> = ({
             }}
             render={(text, record: ListItem) => {
               return (
-                <span>
+                <span style={{ paddingLeft: 7 }}>
                   {record.ready_replicas} / {record.replicas}
                 </span>
               );

src/pages/llmodels/components/view-logs-modal.tsx

@@ -2,7 +2,7 @@ import LogsViewer from '@/components/logs-viewer';
 import { FullscreenExitOutlined, FullscreenOutlined } from '@ant-design/icons';
 import { useIntl } from '@umijs/max';
 import { Button, Modal } from 'antd';
-import React, { useState } from 'react';
+import React, { useEffect, useState } from 'react';
 
 type ViewModalProps = {
   open: boolean;
@@ -11,7 +11,6 @@ type ViewModalProps = {
 };
 
 const ViewCodeModal: React.FC<ViewModalProps> = (props) => {
-  console.log('viewlogs======');
   const { open, url, onCancel } = props || {};
   const [modalSize, setModalSize] = useState<any>({
     width: 600,
@@ -29,6 +28,16 @@ const ViewCodeModal: React.FC<ViewModalProps> = (props) => {
     });
   };
 
+  useEffect(() => {
+    if (open) {
+      isFullScreenRef.current = false;
+      setModalSize({
+        width: '100%',
+        height: 'calc(100vh - 100px)'
+      });
+    }
+  }, [open]);
+
   return (
     <Modal
       title={
@@ -49,7 +58,7 @@ const ViewCodeModal: React.FC<ViewModalProps> = (props) => {
       destroyOnClose={true}
       closeIcon={true}
       maskClosable={false}
-      keyboard={false}
+      keyboard={true}
       width={modalSize.width}
       footer={null}
     >

src/pages/llmodels/config/file-type.ts

@@ -29,7 +29,12 @@ enum FileType {
   IQ2_M = 29,
   IQ4_XS = 30,
   IQ1_M = 31,
-  BF16 = 32
+  BF16 = 32,
+  Q4_0_4_4 = 33,
+  Q4_0_4_8 = 34,
+  Q4_0_8_8 = 35,
+  TQ1_0 = 36,
+  TQ2_0 = 37
 }
 
 export const fileTypeList = [
@@ -63,7 +68,12 @@ export const fileTypeList = [
   'IQ2_M',
   'IQ4_XS',
   'IQ1_M',
-  'BF16'
+  'BF16',
+  'Q4_0_4_4',
+  'Q4_0_4_8',
+  'Q4_0_8_8',
+  'TQ1_0',
+  'TQ2_0'
 ];
 
 export const getFileType = (fileName: string): string | null => {

src/pages/llmodels/config/types.ts

@@ -47,8 +47,16 @@ export interface ModelInstanceListItem {
   huggingface_repo_id: string;
   huggingface_filename: string;
   ollama_library_model_name: string;
+  distributed_servers?: {
+    rpc_servers: any[];
+  };
+  computed_resource_claim?: {
+    offload_layers: number;
+    total_layers: number;
+  };
   s3_address: string;
   worker_id: number;
+  gpu_indexes?: number[];
   worker_ip: string;
   gpu_index: number;
   pid: number;

src/pages/llmodels/index.tsx

@@ -3,6 +3,11 @@ import useSetChunkRequest, {
   createAxiosToken
 } from '@/hooks/use-chunk-request';
 import useUpdateChunkedList from '@/hooks/use-update-chunk-list';
+import { queryGpuDevicesList, queryWorkersList } from '@/pages/resources/apis';
+import {
+  GPUDeviceItem,
+  ListItem as WokerListItem
+} from '@/pages/resources/config/types';
 import _ from 'lodash';
 import { memo, useCallback, useEffect, useRef, useState } from 'react';
 import { MODELS_API, MODEL_INSTANCE_API, queryModelsList } from './apis';
@@ -25,6 +30,8 @@ const Models: React.FC = () => {
     loading: false,
     total: 0
   });
+  const [gpuDeviceList, setGpuDeviceList] = useState<GPUDeviceItem[]>([]);
+  const [workerList, setWorkerList] = useState<WokerListItem[]>([]);
   const [firstLoad, setFirstLoad] = useState(true);
   const chunkRequedtRef = useRef<any>();
   const chunkInstanceRequedtRef = useRef<any>();
@@ -46,6 +53,26 @@ const Models: React.FC = () => {
     }
   });
 
+  const getDeviceList = async () => {
+    try {
+      const data = await queryGpuDevicesList({ page: 1, perPage: 100 });
+      const gpuDeviceMap = _.groupBy(data.items, 'worker_name');
+      setGpuDeviceList(data.items || []);
+    } catch (error) {
+      // ingore
+    }
+  };
+
+  const getWorkerList = async () => {
+    try {
+      const data = await queryWorkersList({ page: 1, perPage: 100 });
+      setWorkerList(data.items || []);
+    } catch (error) {
+      // ingore
+      setWorkerList([]);
+    }
+  };
+
   const fetchData = useCallback(async () => {
     axiosToken?.cancel?.();
     axiosToken = createAxiosToken();
@@ -169,6 +196,10 @@ const Models: React.FC = () => {
     };
   }, [queryParams]);
 
+  useEffect(() => {
+    getWorkerList();
+  }, []);
+
   return (
     <TableContext.Provider
       value={{
@@ -183,6 +214,8 @@ const Models: React.FC = () => {
         queryParams={queryParams}
         loading={dataSource.loading}
         total={dataSource.total}
+        gpuDeviceList={gpuDeviceList}
+        workerList={workerList}
       ></TableList>
     </TableContext.Provider>
   );

src/pages/resources/components/workers.tsx

@@ -25,6 +25,7 @@ import { WorkerStatusMapValue, status } from '../config';
 import { Filesystem, GPUDeviceItem, ListItem } from '../config/types';
 import AddWorker from './add-worker';
 import UpdateLabels from './update-labels';
+
 const { Column } = Table;
 
 const ActionList = [