diff --git a/config/routes.ts b/config/routes.ts index 36f4e98b..4407b61c 100644 --- a/config/routes.ts +++ b/config/routes.ts @@ -114,6 +114,15 @@ export default [ defaultIcon: 'icon-models', access: 'canSeeUser', component: './llmodels/user-models' + }, + { + name: 'benchmark', + path: '/models/benchmark', + key: 'benchmark', + icon: 'icon-speed', + selectedIcon: 'icon-speed-filled', + defaultIcon: 'icon-speed', + component: './benchmark/index' } ] }, diff --git a/src/components/icon-font/index.tsx b/src/components/icon-font/index.tsx index 1b3f42f8..8111748d 100644 --- a/src/components/icon-font/index.tsx +++ b/src/components/icon-font/index.tsx @@ -2,7 +2,7 @@ import { createFromIconfontCN } from '@ant-design/icons'; // import './iconfont/iconfont.js'; const IconFont = createFromIconfontCN({ - scriptUrl: '//at.alicdn.com/t/c/font_4613488_7f4klfqdzm.js' + scriptUrl: '//at.alicdn.com/t/c/font_4613488_9wb9516md1k.js' }); export default IconFont; diff --git a/src/components/page-tools/index.tsx b/src/components/page-tools/index.tsx index d886ac1a..c7477838 100644 --- a/src/components/page-tools/index.tsx +++ b/src/components/page-tools/index.tsx @@ -112,7 +112,7 @@ export const FilterBar: React.FC = (props) => { return null; } return ( - + {handleClickPrimary ? ( actionType === 'dropdown' ? ( = forwardRef( return (
- - - + {segmentOptions.length > 0 && ( + + + + )} {children}
diff --git a/src/pages/benchmark/apis/index.ts b/src/pages/benchmark/apis/index.ts new file mode 100644 index 00000000..9d536458 --- /dev/null +++ b/src/pages/benchmark/apis/index.ts @@ -0,0 +1,71 @@ +import { request } from '@umijs/max'; +import { CancelToken } from 'axios'; +import { BenchmarkListItem, FormData } from '../config/types'; + +export const BENCHMARKS_API = '/benchmark'; +export const DATASETS_API = '/datasets'; + +export async function queryBenchmarkList( + params: Global.SearchParams, + options?: { + token?: CancelToken; + } +) { + return request>(`${BENCHMARKS_API}`, { + method: 'GET', + params, + cancelToken: options?.token + }); +} + +export async function createBenchmark(params: { data: FormData }) { + return request(`${BENCHMARKS_API}`, { + method: 'POST', + data: params.data + }); +} + +export async function updateBenchmark(params: { id: number; data: FormData }) { + return request(`${BENCHMARKS_API}/${params.id}`, { + method: 'PUT', + data: params.data + }); +} + +export async function deleteBenchmark(id: number) { + return request(`${BENCHMARKS_API}/${id}`, { + method: 'DELETE' + }); +} + +export async function queryBenchmarkLogs( + id: number, + options?: { + token?: CancelToken; + } +) { + return request(`${BENCHMARKS_API}/${id}/logs`, { + method: 'GET', + cancelToken: options?.token + }); +} + +export async function createBenchmarkResult(params: { id: number; data: any }) { + return request(`${BENCHMARKS_API}/${params.id}/result`, { + method: 'POST', + data: params.data + }); +} + +export async function queryDatasetList( + params: Global.SearchParams, + options?: { + token?: CancelToken; + } +) { + return request>(`${DATASETS_API}`, { + method: 'GET', + params, + cancelToken: options?.token + }); +} diff --git a/src/pages/benchmark/components/add-benchmark-modal.tsx b/src/pages/benchmark/components/add-benchmark-modal.tsx new file mode 100644 index 00000000..c1361e33 --- /dev/null +++ b/src/pages/benchmark/components/add-benchmark-modal.tsx @@ -0,0 +1,59 @@ +import { PageActionType } from '@/config/types'; +import FormDrawer from '@/pages/_components/form-drawer'; +import React, { useRef } from 'react'; +import { FormData, BenchmarkListItem as ListItem } from '../config/types'; + +import BenchmarkForm from '../forms'; + +type AddModalProps = { + title: string; + action: PageActionType; + open: boolean; + currentData?: ListItem; // Used when action is EDIT + onOk: (values: FormData) => void; + onCancel: () => void; +}; +const AddBenchmark: React.FC = ({ + title, + action, + open, + currentData, + onOk, + onCancel +}) => { + const form = useRef(null); + + const handleSubmit = () => { + form.current?.submit(); + }; + + const handleOk = async (data: FormData) => { + onOk({ + ...data + }); + }; + + const handleCancel = () => { + form.current?.resetFields(); + onCancel(); + }; + + return ( + + + + ); +}; + +export default AddBenchmark; diff --git a/src/pages/benchmark/components/right-actions.tsx b/src/pages/benchmark/components/right-actions.tsx new file mode 100644 index 00000000..2c91dbd8 --- /dev/null +++ b/src/pages/benchmark/components/right-actions.tsx @@ -0,0 +1,61 @@ +import { + DeleteOutlined, + PlusOutlined, + SettingOutlined +} from '@ant-design/icons'; +import { useIntl } from '@umijs/max'; +import { Button, Space } from 'antd'; +import React from 'react'; + +export interface RightActionsProps { + handleDeleteByBatch: () => void; + handleClickPrimary?: () => void; + handleSettingFields?: () => void; + handleCompare?: () => void; + buttonText?: string; + rowSelection: { + selectedRowKeys: React.Key[]; + }; +} + +const RightActions: React.FC = ({ + handleDeleteByBatch, + handleClickPrimary, + handleCompare, + handleSettingFields, + buttonText, + rowSelection +}) => { + const intl = useIntl(); + + return ( + + + + + + + ); +}; + +export default RightActions; diff --git a/src/pages/benchmark/config/form-context.ts b/src/pages/benchmark/config/form-context.ts new file mode 100644 index 00000000..0458c77b --- /dev/null +++ b/src/pages/benchmark/config/form-context.ts @@ -0,0 +1,20 @@ +import { PageActionType } from '@/config/types'; +import { createContext, useContext } from 'react'; + +interface FormContextProps { + action: PageActionType; +} + +const FormContext = createContext({} as FormContextProps); + +export const useFormContext = () => { + const context = useContext(FormContext); + if (!context) { + throw new Error( + 'useFormContext must be used within a FormContext.Provider' + ); + } + return context; +}; + +export default FormContext; diff --git a/src/pages/benchmark/config/index.ts b/src/pages/benchmark/config/index.ts new file mode 100644 index 00000000..61c56468 --- /dev/null +++ b/src/pages/benchmark/config/index.ts @@ -0,0 +1,29 @@ +import { StatusMaps } from '@/config'; +import { StatusType } from '@/config/types'; + +export const BenchmarkStatusValueMap = { + Pending: 'pending', + Claimed: 'claimed', + Running: 'running', + Completed: 'completed', + Error: 'error', + Unreachable: 'unreachable' +}; + +export const BenchmarkStatusLabelMap = { + [BenchmarkStatusValueMap.Pending]: 'Pending', + [BenchmarkStatusValueMap.Claimed]: 'Claimed', + [BenchmarkStatusValueMap.Running]: 'Running', + [BenchmarkStatusValueMap.Completed]: 'Completed', + [BenchmarkStatusValueMap.Error]: 'Error', + [BenchmarkStatusValueMap.Unreachable]: 'Unreachable' +}; + +export const BenchmarkStatus: Record = { + [BenchmarkStatusValueMap.Pending]: StatusMaps.transitioning, + [BenchmarkStatusValueMap.Claimed]: StatusMaps.warning, + [BenchmarkStatusValueMap.Running]: StatusMaps.success, + [BenchmarkStatusValueMap.Completed]: StatusMaps.success, + [BenchmarkStatusValueMap.Error]: StatusMaps.error, + [BenchmarkStatusValueMap.Unreachable]: StatusMaps.error +}; diff --git a/src/pages/benchmark/config/types.ts b/src/pages/benchmark/config/types.ts new file mode 100644 index 00000000..92481e3b --- /dev/null +++ b/src/pages/benchmark/config/types.ts @@ -0,0 +1,103 @@ +export interface ComputedResourceClaim { + is_unified_memory: boolean; + offload_layers: number; + total_layers: number; + ram: number; + vram: Record; + tensor_split: number[]; + vram_utilization: number; +} + +export interface ComputedResourceClaim1 { + is_unified_memory: boolean; + offload_layers: number; + total_layers: number; + ram: number; + vram: Record; + tensor_split: number[]; + vram_utilization: number; +} + +export interface SubordinateWorkersItem { + computed_resource_claim: ComputedResourceClaim1; + ports: number[]; + worker_id: number; + worker_name: string; + gpu_type: string; + gpu_indexes: number[]; + gpu_ids: string[]; +} + +export interface InstanceSnapshot { + computed_resource_claim: ComputedResourceClaim; + ports: number[]; + worker_id: number; + worker_name: string; + gpu_type: string; + gpu_indexes: number[]; + gpu_ids: string[]; + id: number; + name: string; + state: string; + state_message: string; + backend: string; + backend_version: string; + api_detected_backend_version: string; + subordinate_workers: SubordinateWorkersItem[]; +} + +export interface GPUSnapshot { + vendor: string; + type: string; + index: number; + device_index: number; + device_chip_index: number; + arch_family: string; + name: string; + uuid: string; + driver_version: string; + runtime_version: string; + compute_capability: string; + id: string; + worker_id: number; + worker_name: string; + memory_total: number; + core_total: number; +} + +export interface FormData { + name: string; + description: string; + labels: Record; + cluster_id: number; + model_id: number; + model_name: string; + model_instance_name: string; + dataset_id: number; + dataset_name: string; + dataset_source: string; + dataset_prompt_tokens: number; + dataset_output_tokens: number; + total_requests: number; + request_rate: number; + instance_snapshots: Record; + gpu_snapshots: Record; + state: string; + state_message: string; + worker_id: number; + gpu_summary: string; + gpu_vendor_summary: string; +} + +export interface BenchmarkListItem extends FormData { + id: number; + created_at: string; + updated_at: string; +} + +export interface DatasetListItem { + name: string; + source: string; + prompt_tokens: number; + output_tokens: number; +} diff --git a/src/pages/benchmark/forms/basic.tsx b/src/pages/benchmark/forms/basic.tsx new file mode 100644 index 00000000..2eee5175 --- /dev/null +++ b/src/pages/benchmark/forms/basic.tsx @@ -0,0 +1,34 @@ +import SealInput from '@/components/seal-form/seal-input'; +import { useIntl } from '@umijs/max'; +import { Form } from 'antd'; +import React from 'react'; +import { FormData } from '../config/types'; + +const BasicForm: React.FC = () => { + const intl = useIntl(); + const form = Form.useFormInstance(); + + return ( + + name="name" + rules={[ + { + required: true, + message: intl.formatMessage( + { id: 'common.form.rule.input' }, + { + name: intl.formatMessage({ id: 'common.table.name' }) + } + ) + } + ]} + > + + + ); +}; + +export default BasicForm; diff --git a/src/pages/benchmark/forms/index.tsx b/src/pages/benchmark/forms/index.tsx new file mode 100644 index 00000000..9b57e21f --- /dev/null +++ b/src/pages/benchmark/forms/index.tsx @@ -0,0 +1,109 @@ +import IconFont from '@/components/icon-font'; +import { PageAction } from '@/config'; +import { PageActionType } from '@/config/types'; +import { useWrapperContext } from '@/pages/_components/column-wrapper/use-wrapper-context'; +import ScrollSpyTabs from '@/pages/_components/scroll-spy-tabs'; +import { useIntl } from '@umijs/max'; +import { Form } from 'antd'; +import { + forwardRef, + useEffect, + useImperativeHandle, + useRef, + useState +} from 'react'; +import FormContext from '../config/form-context'; +import { FormData, BenchmarkListItem as ListItem } from '../config/types'; +import Basic from './basic'; + +interface ProviderFormProps { + ref?: any; + action: PageActionType; + currentData?: ListItem; // Used when action is EDIT + onFinish: (values: FormData) => Promise; +} + +const TABKeysMap = { + BASIC: 'basic', + SUPPORTEDMODELS: 'supportedModels', + CUSTOMCONFIG: 'customConfig', + ADVANCED: 'advanced' +}; + +const ProviderForm: React.FC = forwardRef((props, ref) => { + const { action, currentData, onFinish } = props; + const intl = useIntl(); + const [form] = Form.useForm(); + const { getScrollElementScrollableHeight } = useWrapperContext(); + const [activeKey, setActiveKey] = useState([TABKeysMap.BASIC]); + const scrollTabsRef = useRef(null); + + const segmentOptions = [ + { + value: TABKeysMap.BASIC, + label: 'Basic', + icon: , + field: 'name' + }, + { + value: TABKeysMap.SUPPORTEDMODELS, + label: 'Supported Models', + icon: , + field: 'supportedModels' + }, + { + value: TABKeysMap.ADVANCED, + label: intl.formatMessage({ id: 'resources.form.advanced' }), + icon: , + field: 'advanceConfig' + } + ]; + + const handleActiveChange = (key: string[]) => { + setActiveKey(key); + }; + + const handleOnCollapseChange = (keys: string | string[]) => { + setActiveKey(Array.isArray(keys) ? keys : [keys]); + }; + + useImperativeHandle(ref, () => ({ + submit: () => { + form.submit(); + }, + resetFields: () => { + form.resetFields(); + } + })); + + useEffect(() => { + if (action === PageAction.EDIT && currentData) { + form.setFieldsValue({ + ...currentData + }); + } + }, [form, currentData, action]); + + return ( + + +
+ + +
+ + ); +}); + +export default ProviderForm; diff --git a/src/pages/benchmark/hooks/use-benchmark-columns.tsx b/src/pages/benchmark/hooks/use-benchmark-columns.tsx new file mode 100644 index 00000000..e2d2b70d --- /dev/null +++ b/src/pages/benchmark/hooks/use-benchmark-columns.tsx @@ -0,0 +1,176 @@ +// columns.ts +import AutoTooltip from '@/components/auto-tooltip'; +import DropdownButtons from '@/components/drop-down-buttons'; +import icons from '@/components/icon-font/icons'; +import StatusTag from '@/components/status-tag'; +import { tableSorter } from '@/config/settings'; +import { useIntl } from '@umijs/max'; +import { ColumnsType } from 'antd/es/table'; +import dayjs from 'dayjs'; +import { useMemo } from 'react'; +import { BenchmarkStatus, BenchmarkStatusLabelMap } from '../config'; +import { BenchmarkListItem as ListItem } from '../config/types'; + +const actionList = [ + { + key: 'edit', + label: 'common.button.edit', + icon: icons.EditOutlined + }, + { + key: 'delete', + label: 'common.button.delete', + icon: icons.DeleteOutlined, + props: { + danger: true + } + } +]; + +const useBenchmarkColumns = ( + sortOrder: string[], + handleSelect: (val: string, record: ListItem) => void +): ColumnsType => { + const intl = useIntl(); + + return useMemo(() => { + return [ + { + title: intl.formatMessage({ id: 'common.table.name' }), + dataIndex: 'name', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.model' }), + dataIndex: 'model_name', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.dataset' }), + dataIndex: 'dataset_name', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'common.table.status' }), + dataIndex: 'state', + render: (value: number, record: ListItem) => ( + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.requestRate' }), + dataIndex: 'request_rate', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.gpu' }), + dataIndex: 'gpu_summary', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.itl' }), + dataIndex: 'itl', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.tpot' }), + dataIndex: 'tpot', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.ttft' }), + dataIndex: 'ttft', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.rps' }), + dataIndex: 'requests_per_second', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'benchmark.table.tps' }), + dataIndex: 'tokens_per_second', + sorter: tableSorter(1), + render: (text: string) => ( + + {text} + + ) + }, + { + title: intl.formatMessage({ id: 'common.table.createTime' }), + dataIndex: 'created_at', + sorter: tableSorter(3), + render: (value: string) => ( + {dayjs(value).format('YYYY-MM-DD HH:mm:ss')} + ) + }, + { + title: intl.formatMessage({ id: 'common.table.operation' }), + dataIndex: 'operations', + ellipsis: { + showTitle: false + }, + render: (value: string, record: ListItem) => ( + handleSelect(val, record)} + > + ) + } + ]; + }, [intl, handleSelect]); +}; + +export default useBenchmarkColumns; diff --git a/src/pages/benchmark/hooks/use-create-benchmark.ts b/src/pages/benchmark/hooks/use-create-benchmark.ts new file mode 100644 index 00000000..46bae5f2 --- /dev/null +++ b/src/pages/benchmark/hooks/use-create-benchmark.ts @@ -0,0 +1,46 @@ +import { PageAction } from '@/config'; +import { PageActionType } from '@/config/types'; +import { useState } from 'react'; +import { BenchmarkListItem as ListItem } from '../config/types'; + +const useCreateBenchmark = () => { + const [openModalStatus, setOpenModalStatus] = useState<{ + open: boolean; + action: PageActionType; + currentData?: ListItem; + title: string; + }>({ + open: false, + action: PageAction.CREATE, + currentData: undefined, + title: '' + }); + + const openModal = (action: PageActionType, title: string, row?: ListItem) => { + setOpenModalStatus({ + ...openModalStatus, + open: true, + title: title, + action, + currentData: row + }); + }; + + const closeModal = () => { + setOpenModalStatus({ + open: false, + action: PageAction.CREATE, + currentData: undefined, + title: '' + }); + }; + + return { + openBenchmarkModalStatus: openModalStatus, + setOpenBenchmarkModalStatus: setOpenModalStatus, + openBenchmarkModal: openModal, + closeBenchmarkModal: closeModal + }; +}; + +export default useCreateBenchmark; diff --git a/src/pages/benchmark/hooks/use-create-compare.ts b/src/pages/benchmark/hooks/use-create-compare.ts new file mode 100644 index 00000000..48cd213d --- /dev/null +++ b/src/pages/benchmark/hooks/use-create-compare.ts @@ -0,0 +1,40 @@ +import { PageActionType } from '@/config/types'; +import { useState } from 'react'; +import { BenchmarkListItem as ListItem } from '../config/types'; + +const useCreateCompare = () => { + const [openModalStatus, setOpenModalStatus] = useState<{ + open: boolean; + currentData?: ListItem[]; + }>({ + open: false, + currentData: undefined + }); + + const openModal = ( + action: PageActionType, + title: string, + rows?: ListItem[] + ) => { + setOpenModalStatus({ + open: true, + currentData: rows + }); + }; + + const closeModal = () => { + setOpenModalStatus({ + open: false, + currentData: undefined + }); + }; + + return { + openCompareModalStatus: openModalStatus, + setOpenCompareModalStatus: setOpenModalStatus, + openCompareModal: openModal, + closeCompareModal: closeModal + }; +}; + +export default useCreateCompare; diff --git a/src/pages/benchmark/index.tsx b/src/pages/benchmark/index.tsx new file mode 100644 index 00000000..38e7e6c9 --- /dev/null +++ b/src/pages/benchmark/index.tsx @@ -0,0 +1,172 @@ +import DeleteModal from '@/components/delete-modal'; +import IconFont from '@/components/icon-font'; +import { FilterBar } from '@/components/page-tools'; +import { PageAction } from '@/config'; +import { TABLE_SORT_DIRECTIONS } from '@/config/settings'; +import useTableFetch from '@/hooks/use-table-fetch'; +import { useIntl } from '@umijs/max'; +import { useMemoizedFn } from 'ahooks'; +import { ConfigProvider, Table, message } from 'antd'; +import _ from 'lodash'; +import NoResult from '../_components/no-result'; +import PageBox from '../_components/page-box'; +import { + createBenchmark, + deleteBenchmark, + queryBenchmarkList, + updateBenchmark +} from './apis'; +import AddBenchmarkModal from './components/add-benchmark-modal'; +import RightActions from './components/right-actions'; +import { FormData, BenchmarkListItem as ListItem } from './config/types'; +import useBenchmarkColumns from './hooks/use-benchmark-columns'; +import useCreateBenchmark from './hooks/use-create-benchmark'; + +const Benchmark: React.FC = () => { + const { + dataSource, + rowSelection, + queryParams, + sortOrder, + modalRef, + handleDelete, + handleDeleteBatch, + fetchData, + handlePageChange, + handleTableChange, + handleSearch, + handleNameChange + } = useTableFetch({ + fetchAPI: queryBenchmarkList, + deleteAPI: deleteBenchmark, + contentForDelete: 'menu.models.benchmark' + }); + const intl = useIntl(); + const { openBenchmarkModal, closeBenchmarkModal, openBenchmarkModalStatus } = + useCreateBenchmark(); + + const handleAddBenchmark = () => { + openBenchmarkModal(PageAction.CREATE, 'Add Benchmark'); + }; + + const handleModalOk = async (data: FormData) => { + const params = { + ...data + }; + try { + if (openBenchmarkModalStatus.action === PageAction.EDIT) { + await updateBenchmark({ + data: { + ...params + }, + id: openBenchmarkModalStatus.currentData!.id + }); + } else { + await createBenchmark({ data: params }); + } + fetchData(); + closeBenchmarkModal(); + message.success(intl.formatMessage({ id: 'common.message.success' })); + } catch (error) { + closeBenchmarkModal(); + } + }; + + const handleModalCancel = () => { + closeBenchmarkModal(); + }; + + const handleEditUser = (row: ListItem) => { + openBenchmarkModal(PageAction.EDIT, 'Edit Benchmark', row); + }; + + const handleSelect = useMemoizedFn((val: any, row: ListItem) => { + if (val === 'edit') { + handleEditUser(row); + } else if (val === 'delete') { + handleDelete({ ...row, name: row.name }); + } + }); + + const renderEmpty = (type?: string) => { + if (type !== 'Table') return; + return ( + } + filters={_.omit(queryParams, ['sort_by'])} + noFoundText={intl.formatMessage({ + id: 'noresult.benchmark.nofound' + })} + title={intl.formatMessage({ id: 'noresult.benchmark.title' })} + subTitle={intl.formatMessage({ + id: 'noresult.benchmark.subTitle' + })} + onClick={handleAddBenchmark} + buttonText={intl.formatMessage({ id: 'noresult.button.add' })} + > + ); + }; + + const columns = useBenchmarkColumns(sortOrder, handleSelect); + + return ( + <> + + + } + > + +
+ + + + + + ); +}; + +export default Benchmark; diff --git a/src/pages/llmodels/components/table-list.tsx b/src/pages/llmodels/components/table-list.tsx index 04f23999..cc687326 100644 --- a/src/pages/llmodels/components/table-list.tsx +++ b/src/pages/llmodels/components/table-list.tsx @@ -693,7 +693,7 @@ const Models: React.FC = ({ } right={ - + {page !== 'clusters' && (