diff --git a/src/locales/en-US/benchmark.ts b/src/locales/en-US/benchmark.ts index efe225c1..e5677b53 100644 --- a/src/locales/en-US/benchmark.ts +++ b/src/locales/en-US/benchmark.ts @@ -1,5 +1,5 @@ export default { - 'benchmark.title': 'Benchmark', + 'benchmark.title': 'Benchmarks', 'benchmark.button.add': 'Add Benchmark', 'benchmark.button.compare': 'Compare', 'benchmark.table.model': 'Model', diff --git a/src/locales/en-US/menu.ts b/src/locales/en-US/menu.ts index a668b614..470391a1 100644 --- a/src/locales/en-US/menu.ts +++ b/src/locales/en-US/menu.ts @@ -14,7 +14,7 @@ export default { 'menu.models.catalog': 'Model Catalog', 'menu.models.deployment': 'Deployments', 'menu.models.userModels': 'My Models', - 'menu.models.benchmark': 'Benchmark', + 'menu.models.benchmark': 'Benchmarks', 'menu.modelCatalog': 'Catalog', 'menu.resources': 'Resources', 'menu.apikeys': 'API Keys', diff --git a/src/locales/ja-JP/benchmark.ts b/src/locales/ja-JP/benchmark.ts index efe225c1..e5677b53 100644 --- a/src/locales/ja-JP/benchmark.ts +++ b/src/locales/ja-JP/benchmark.ts @@ -1,5 +1,5 @@ export default { - 'benchmark.title': 'Benchmark', + 'benchmark.title': 'Benchmarks', 'benchmark.button.add': 'Add Benchmark', 'benchmark.button.compare': 'Compare', 'benchmark.table.model': 'Model', diff --git a/src/locales/ja-JP/menu.ts b/src/locales/ja-JP/menu.ts index ec3dbba6..bb03622f 100644 --- a/src/locales/ja-JP/menu.ts +++ b/src/locales/ja-JP/menu.ts @@ -13,7 +13,7 @@ export default { 'menu.models.modelCatalog': 'カタログ', 'menu.models.catalog': 'モデルカタログ', 'menu.models.deployment': 'Deployment', - 'menu.models.benchmark': 'Benchmark', + 'menu.models.benchmark': 'Benchmarks', 'menu.modelCatalog': 'カタログ', 'menu.resources': 'リソース', 'menu.apikeys': 'APIキー', @@ -52,5 +52,5 @@ export default { // 12. 'menu.clusterManagement.clusterDetail': 'Cluster Detail', // 13. 'menu.clusterManagement.clusterCreate': 'Create Cluster', // 14. 'menu.resources.backendsList': 'Inference Backends', -// 15. 'menu.models.benchmark': 'Benchmark', +// 15. 'menu.models.benchmark': 'Benchmarks', // ========== End of To-Do List ========== diff --git a/src/locales/ru-RU/benchmark.ts b/src/locales/ru-RU/benchmark.ts index efe225c1..e5677b53 100644 --- a/src/locales/ru-RU/benchmark.ts +++ b/src/locales/ru-RU/benchmark.ts @@ -1,5 +1,5 @@ export default { - 'benchmark.title': 'Benchmark', + 'benchmark.title': 'Benchmarks', 'benchmark.button.add': 'Add Benchmark', 'benchmark.button.compare': 'Compare', 'benchmark.table.model': 'Model', diff --git a/src/locales/ru-RU/menu.ts b/src/locales/ru-RU/menu.ts index ae884f25..e44bcd40 100644 --- a/src/locales/ru-RU/menu.ts +++ b/src/locales/ru-RU/menu.ts @@ -13,7 +13,7 @@ export default { 'menu.models.modelCatalog': 'Каталог', 'menu.models.catalog': 'Каталог моделей', 'menu.models.deployment': 'Запуск', - 'menu.models.benchmark': 'Benchmark', + 'menu.models.benchmark': 'Benchmarks', 'menu.modelCatalog': 'Каталог', 'menu.resources': 'Ресурсы', 'menu.apikeys': 'API-ключи', @@ -38,5 +38,5 @@ export default { }; // ========== To-Do: Translate Keys (Remove After Translation) ========== -// 1. 'menu.models.benchmark': 'Benchmark', +// 1. 'menu.models.benchmark': 'Benchmarks', // ========== End of To-Do List ========== diff --git a/src/pages/benchmark/components/configure/benchmark.tsx b/src/pages/benchmark/components/configure/benchmark.tsx index bf010b29..9d12e623 100644 --- a/src/pages/benchmark/components/configure/benchmark.tsx +++ b/src/pages/benchmark/components/configure/benchmark.tsx @@ -22,15 +22,15 @@ const Benchmark: React.FC = () => { label: 'Token Length (In/Out)', children: ( - {detailData?.dataset_input_tokens || '-'} /{' '} + {detailData?.dataset_prompt_tokens || '-'} /{' '} {detailData?.dataset_output_tokens || '-'} ) }, { - key: '5', - label: 'Seed', - children: detailData?.seed || '-' + key: '7', + label: 'Total Requests', + children: detailData?.total_requests || '-' }, { key: '6', @@ -38,9 +38,9 @@ const Benchmark: React.FC = () => { children: detailData?.request_rate || '-' }, { - key: '7', - label: 'Total Requests', - children: detailData?.total_requests || '-' + key: '5', + label: 'Seed', + children: detailData?.seed || '-' } ]; diff --git a/src/pages/benchmark/components/configure/instance.tsx b/src/pages/benchmark/components/configure/instance.tsx index 363684df..3c2cd11e 100644 --- a/src/pages/benchmark/components/configure/instance.tsx +++ b/src/pages/benchmark/components/configure/instance.tsx @@ -1,7 +1,7 @@ import StatusTag from '@/components/status-tag'; import { InstanceStatusMapValue, status } from '@/pages/llmodels/config'; import { convertFileSize } from '@/utils'; -import { Descriptions } from 'antd'; +import { Descriptions, Flex, Tag } from 'antd'; import _ from 'lodash'; import React, { useMemo } from 'react'; import { useDetailContext } from '../../config/detail-context'; @@ -81,6 +81,100 @@ const Instance: React.FC = () => { ]; }, [detailData]); + const paramsItems = useMemo(() => { + const { snapshot } = detailData; + const [instanceName, instanceData] = + Object.entries(snapshot.instances || {})[0] || []; + return [ + { + key: '1', + label: 'Backend Parameters', + children: ( + + {instanceData?.backend_parameters.map( + (param: string, index: number) => ( + + {param} + + ) + )} + + ) + }, + { + key: '3', + label: 'Extended KV Cache', + children: ( + + {instanceData?.extended_kv_cache?.enabled ? ( + <> + + RAM-to-VRAM Ratio: + {instanceData?.extended_kv_cache?.ram_ratio} + + + Maximum RAM Size (GiB): + {instanceData?.extended_kv_cache?.ram_size} + + + Size of Cache Chunks: + {instanceData?.extended_kv_cache?.chunk_size} + + + ) : ( + '-' + )} + + ) + }, + { + key: '4', + label: 'Speculative Decoding', + children: ( + + {instanceData?.speculative_config?.enabled ? ( + <> + + Algorithm: + {instanceData?.speculative_config?.algorithm} + + + Draft Model: + {instanceData?.speculative_config?.draft_model} + + + Number of Draft Tokens: + + {instanceData?.speculative_config?.num_draft_tokens} + + + + ) : ( + '-' + )} + + ) + }, + { + key: '2', + label: 'Environment Variables', + children: ( + + {instanceData?.env + ? Object.entries(instanceData?.env || {}).map( + ([key, value], index: number) => ( + + {`${key}=${value}`} + + ) + ) + : '-'} + + ) + } + ]; + }, [detailData]); + return (
{ } }} > +
); }; diff --git a/src/pages/benchmark/components/summary/basic.tsx b/src/pages/benchmark/components/summary/basic.tsx index 99749d99..999e6780 100644 --- a/src/pages/benchmark/components/summary/basic.tsx +++ b/src/pages/benchmark/components/summary/basic.tsx @@ -2,7 +2,7 @@ import AutoTooltip from '@/components/auto-tooltip'; import { useIntl } from '@umijs/max'; import { Col, Descriptions, DescriptionsProps, Row, Statistic } from 'antd'; import dayjs from 'dayjs'; -import { round } from 'lodash'; +import _, { round } from 'lodash'; import React from 'react'; import styled from 'styled-components'; import { useDetailContext } from '../../config/detail-context'; @@ -32,6 +32,7 @@ const Box = styled.div` const Summary: React.FC = () => { const { detailData, clusterList } = useDetailContext(); const intl = useIntl(); + const rawMetrics = _.get(detailData, ['raw_metrics', 'benchmarks', '0'], {}); const items: DescriptionsProps['items'] = [ { @@ -54,15 +55,10 @@ const Summary: React.FC = () => { label: 'Dataset', children: detailData.dataset_name }, - { - key: '5', - label: 'Concurrency', - children: detailData.concurrency - }, { key: '5', label: 'Duration (s)', - children: detailData.duration + children: round(rawMetrics.duration || 0, 2) }, { key: '3', @@ -75,43 +71,68 @@ const Summary: React.FC = () => { label: 'Total Requests', key: 'total_requests', value: detailData.total_requests, + precision: 0, unit: '' }, { label: 'Success Rate', key: 'success_rate', - value: detailData.success_rate, + value: round( + (_.get(rawMetrics, 'metrics.request_totals.successful') / + _.get(rawMetrics, 'metrics.request_totals.total')) * + 100 || 0, + 1 + ), color: 'var(--ant-color-success)', + precision: 0, unit: '%' }, { - label: 'TPS', - key: 'tps', - value: detailData.tokens_per_second_mean, - unit: 't/s' + label: 'Incomplete Count', + key: 'incomplete_count', + value: _.get(rawMetrics, 'metrics.request_totals.incomplete'), + color: 'var(--ant-color-warning)', + unit: '' + }, + // { + // label: 'Error Count', + // key: 'error_count', + // value: _.get(rawMetrics, 'metrics.request_totals.error'), + // color: 'var(--ant-color-error)', + // unit: '' + // }, + { + label: 'Concurrency', + key: 'concurrency', + value: _.get(rawMetrics, 'metrics.request_concurrency.successful.mean'), + precision: 2, + unit: '' }, { label: 'RPS', key: 'rps', value: detailData.requests_per_second_mean, + precision: 2, unit: 'req/s' }, - // { - // label: 'ITL', - // key: 'itl', - // value: detailData.inter_token_latency_mean, - // unit: 'ms' - // }, + { + label: 'ITL', + key: 'itl', + value: detailData.inter_token_latency_mean, + unit: 'ms' + }, { label: 'TPOT', key: 'time_per_output_token_mean', value: detailData.time_per_output_token_mean, + precision: 2, unit: 'ms' }, { label: 'TTFT', key: 'time_to_first_token_mean', value: detailData.time_to_first_token_mean, + precision: 2, unit: 'ms' } ]; @@ -119,36 +140,36 @@ const Summary: React.FC = () => { const throughputFields = [ { key: '3', - label: 'Total throughput', - children: `${round(detailData.total_through || 0, 2)} t/s` + label: 'Total token throughput', + children: `${round(detailData.tokens_per_second_mean || 0, 2)} t/s` }, { key: '1', - label: 'Request throughput', - children: `${round(detailData.request_through || 0, 2)} t/s` + label: 'Request token throughput', + children: `${round(detailData.prompt_tokens_per_second_mean || 0, 2)} t/s` }, { key: '2', - label: 'Output throughput', - children: `${round(detailData.output_through || 0, 2)} t/s` + label: 'Output token throughput', + children: `${round(detailData.output_tokens_per_second_mean || 0, 2)} t/s` } ]; const latencyFields = [ { key: '1', - label: 'Request Latency', - children: `${round(detailData.request_latency_mean, 2)} t/s` + label: 'Avg Request Latency', + children: `${round(detailData.request_latency_mean, 2)} ms` }, { key: '2', label: 'Avg Time to First Token', - children: `${round(detailData.time_to_first_token_mean, 2)} t/s` + children: `${round(detailData.time_to_first_token_mean, 2)} ms` }, { key: '3', label: 'Avg Time Per Output Token', - children: `${round(detailData.time_per_output_token_mean, 2)} t/s` + children: `${round(detailData.time_per_output_token_mean, 2)} ms` } ]; @@ -156,12 +177,12 @@ const Summary: React.FC = () => { {cardFields.map((field) => ( - + { title: name } ]; - useEffect(() => { document.title = `${intl.formatMessage({ id: 'benchmark.title' })} - ${name}`; }, [name, intl]); diff --git a/src/pages/benchmark/forms/basic.tsx b/src/pages/benchmark/forms/basic.tsx index 5fa8e8b2..dd37e20d 100644 --- a/src/pages/benchmark/forms/basic.tsx +++ b/src/pages/benchmark/forms/basic.tsx @@ -1,4 +1,3 @@ -import LabelSelector from '@/components/label-selector'; import SealInput from '@/components/seal-form/seal-input'; import SealSelect from '@/components/seal-form/seal-select'; import { PageAction } from '@/config'; @@ -13,7 +12,6 @@ import { useQueryModelInstancesList } from '@/pages/llmodels/services/use-query- import { useQueryModelList } from '@/pages/llmodels/services/use-query-model-list'; import { useIntl } from '@umijs/max'; import { Form } from 'antd'; -import _ from 'lodash'; import React, { useEffect } from 'react'; import { useFormContext } from '../config/form-context'; import { FormData } from '../config/types'; @@ -21,7 +19,6 @@ import { FormData } from '../config/types'; const BasicForm: React.FC = () => { const intl = useIntl(); const form = Form.useFormInstance(); - const labels = Form.useWatch('labels', form); const { getRuleMessage } = useAppUtils(); const { action, open } = useFormContext(); const { @@ -43,10 +40,6 @@ const BasicForm: React.FC = () => { instanceList } = useQueryModelInstancesList(); - const handleLabelsChange = (labels: object) => { - form.setFieldValue('labels', labels); - }; - const onModelListOpenChange = async (open: boolean) => { if (open && modelList.length === 0) { await form.validateFields(['cluster_id']); @@ -186,41 +179,7 @@ const BasicForm: React.FC = () => { required > - - name="labels" - rules={[ - () => ({ - validator(rule, value) { - if (_.keys(value).length > 0) { - if (_.some(_.keys(value), (k: string) => !value[k])) { - return Promise.reject( - intl.formatMessage( - { - id: 'common.validate.value' - }, - { - name: intl.formatMessage({ - id: 'resources.form.label' - }) - } - ) - ); - } - } - return Promise.resolve(); - } - }) - ]} - > - - + name="description"> { + const form = Form.useFormInstance(); + const labels = Form.useWatch('labels', form); + const intl = useIntl(); + + const handleLabelsChange = (labels: object) => { + form.setFieldValue('labels', labels); + }; + + return ( + + name="labels" + rules={[ + () => ({ + validator(rule, value) { + if (_.keys(value).length > 0) { + if (_.some(_.keys(value), (k: string) => !value[k])) { + return Promise.reject( + intl.formatMessage( + { + id: 'common.validate.value' + }, + { + name: intl.formatMessage({ + id: 'resources.form.label' + }) + } + ) + ); + } + } + return Promise.resolve(); + } + }) + ]} + > + + + ); +}; + +export default LabelsForm;