diff --git a/src/atoms/models.ts b/src/atoms/models.ts index 5437f91b..5a5e53cc 100644 --- a/src/atoms/models.ts +++ b/src/atoms/models.ts @@ -23,3 +23,13 @@ export const clusterListAtom = atom< value: number; }[] >([]); + +export const backendOptionsAtom = atom< + { + value: string; + label: string; + default_backend_param: string[]; + default_version: string; + versions: { label: string; value: string }[]; + }[] +>([]); diff --git a/src/components/seal-form/seal-textarea.tsx b/src/components/seal-form/seal-textarea.tsx index 5f108d1f..74d8ee66 100644 --- a/src/components/seal-form/seal-textarea.tsx +++ b/src/components/seal-form/seal-textarea.tsx @@ -12,7 +12,12 @@ import { SealFormItemProps } from './types'; import Wrapper from './wrapper'; import InputWrapper from './wrapper/input'; -const LabelWrapper = styled.div` +const LabelWrapper = styled.div.attrs({ + className: 'seal-textarea-label' +})` + display: flex; + align-items: center; + justify-content: flex-start; background-color: var(--ant-color-bg-container); `; @@ -110,7 +115,7 @@ const SealTextArea: React.FC = ( {label}} + label={label} isFocus={isFocus} required={required} description={description} diff --git a/src/components/seal-form/wrapper/index.tsx b/src/components/seal-form/wrapper/index.tsx index 8bfdef33..47b478b4 100644 --- a/src/components/seal-form/wrapper/index.tsx +++ b/src/components/seal-form/wrapper/index.tsx @@ -154,7 +154,9 @@ export const Label = styled.div.attrs<{ `; // inner -const Inner = styled.div` +const Inner = styled.div.attrs({ + className: '__inner__' +})` width: 100%; display: flex; `; diff --git a/src/components/seal-form/wrapper/input.ts b/src/components/seal-form/wrapper/input.ts index 887ad012..174ae18e 100644 --- a/src/components/seal-form/wrapper/input.ts +++ b/src/components/seal-form/wrapper/input.ts @@ -163,6 +163,9 @@ const InputWrapper = styled.div` height: auto; padding-right: 10px; } + .ant-input-textarea-allow-clear.ant-input-affix-wrapper { + padding: 0; + } `; export default InputWrapper; diff --git a/src/components/seal-form/wrapper/select.ts b/src/components/seal-form/wrapper/select.ts index e25b8e35..d77f673b 100644 --- a/src/components/seal-form/wrapper/select.ts +++ b/src/components/seal-form/wrapper/select.ts @@ -97,10 +97,10 @@ const SelectWrapper = styled.div` align-items: center; height: 54px; - .ant-select-selection-search { - // top: 20px !important; - // inset-inline-start: ${INPUT_INNER_PADDING}px; + .ant-select-selection-wrap { + height: 100%; } + &.ant-select-auto-complete { .ant-select-selection-search { padding-inline-start: ${INPUT_INNER_PADDING}px; diff --git a/src/pages/llmodels/apis/evaluateWorker.ts b/src/pages/llmodels/apis/evaluateWorker.ts deleted file mode 100644 index c6dcc472..00000000 --- a/src/pages/llmodels/apis/evaluateWorker.ts +++ /dev/null @@ -1,52 +0,0 @@ -const MODEL_EVALUATIONS = '/model-evaluations'; - -let controller = new AbortController(); -let signal = controller.signal; - -self.onmessage = async (event) => { - const { list, modelSource, modelSourceMap } = event.data; - - const repoList = list.map((item: any) => ({ - source: modelSource, - ...(modelSource === modelSourceMap.huggingface_value - ? { huggingface_repo_id: item.name } - : { model_scope_model_id: item.name }) - })); - - try { - controller?.abort(); - controller = new AbortController(); - signal = controller.signal; - const response = await fetch(`v1/${MODEL_EVALUATIONS}`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - signal, - body: JSON.stringify({ model_specs: repoList }) - }); - - if (!response.ok) { - throw new Error(`HTTP error! Status: ${response.status}`); - } - - const evaluations = await response.json(); - const { results } = evaluations; - const resultList = list.map((item: any, index: number) => { - return { - ...item, - evaluateResult: results[index] || null - }; - }); - - self.postMessage({ success: true, resultList }); - } catch (error) { - self.postMessage({ success: false, resultList: list }); - } -}; - -self.onmessage = (event) => { - if (event.data === 'abort') { - controller.abort(); - } -}; diff --git a/src/pages/llmodels/apis/index.ts b/src/pages/llmodels/apis/index.ts index ca23f84a..2dc86d69 100644 --- a/src/pages/llmodels/apis/index.ts +++ b/src/pages/llmodels/apis/index.ts @@ -20,6 +20,8 @@ export const MODEL_INSTANCE_API = '/model-instances'; export const MODEL_EVALUATIONS = '/model-evaluations'; +export const BACKEND_LIST_API = '/inference-backend/list'; + const setProxyUrl = (url: string) => { return `/proxy?url=${encodeURIComponent(url)}`; }; @@ -396,3 +398,54 @@ export async function evaluationsModelSpec( }) }; } + +export async function queryBackendList() { + // return request<{ + // items: { + // backend_name: string; + // backend_show_name: string; + // from_config: boolean; + // default_version: string; + // default_backend_param: string[]; + // versions: string[]; + // }[]; + // }>(BACKEND_LIST_API, { + // method: 'GET' + // }); + return { + items: [ + { + backend_name: 'vllm', + backend_show_name: 'vLLM', + from_config: false, + default_version: '0.10.1.1', + default_backend_param: null, + versions: ['0.10.1.1', '0.10.0', '0.9.2', '0.8.5', '0.8.3'] + }, + { + backend_name: 'ascend-mindie', + backend_show_name: 'Ascend MindIE', + from_config: false, + default_version: null, + default_backend_param: null, + versions: null + }, + { + backend_name: 'custom', + backend_show_name: 'Custom', + from_config: false, + default_version: null, + default_backend_param: null, + versions: null + }, + { + backend_name: 'test', + backend_show_name: null, + from_config: true, + default_version: 'v1', + default_backend_param: ['--host=0.0.0.0'], + versions: ['v1'] + } + ] + }; +} diff --git a/src/pages/llmodels/components/advance-config.tsx b/src/pages/llmodels/components/advance-config.tsx index 5dc6015b..0978fbe0 100644 --- a/src/pages/llmodels/components/advance-config.tsx +++ b/src/pages/llmodels/components/advance-config.tsx @@ -2,22 +2,17 @@ import IconFont from '@/components/icon-font'; import LabelSelector from '@/components/label-selector'; import ListInput from '@/components/list-input'; import CheckboxField from '@/components/seal-form/checkbox-field'; -import SealInput from '@/components/seal-form/seal-input'; import SealSelect from '@/components/seal-form/seal-select'; import TooltipList from '@/components/tooltip-list'; -import { PageAction } from '@/config'; import { PageActionType } from '@/config/types'; import { useIntl } from '@umijs/max'; import { Collapse, Form, FormInstance, Typography } from 'antd'; import _ from 'lodash'; import React, { useCallback, useMemo } from 'react'; import { - backendLabelMap, backendParamsHolderTips, - backendTipsList, getBackendParamsTips, modelCategories, - modelSourceMap, placementStrategyOptions, ScheduleValueMap } from '../config'; @@ -26,6 +21,7 @@ import BackendParameters, { } from '../config/backend-parameters'; import { useFormContext } from '../config/form-context'; import { FormData } from '../config/types'; +import BackendFields from '../forms/backend-fields'; import dataformStyles from '../style/data-form.less'; import Performance from './performance'; @@ -35,8 +31,6 @@ interface AdvanceConfigProps { gpuOptions: Array; action: PageActionType; source: string; - backendOptions?: Global.BaseOption[]; - handleBackendChange?: (value: string) => void; } const placementStrategyTips = [ @@ -51,7 +45,7 @@ const placementStrategyTips = [ ]; const AdvanceConfig: React.FC = (props) => { - const { form, isGGUF, gpuOptions, source, backendOptions, action } = props; + const { form, isGGUF, gpuOptions, source, action } = props; const intl = useIntl(); const wokerSelector = Form.useWatch('worker_selector', form); const EnviromentVars = Form.useWatch('env', form); @@ -63,7 +57,7 @@ const AdvanceConfig: React.FC = (props) => { const placement_strategy = Form.useWatch('placement_strategy', form); const gpuSelectorIds = Form.useWatch(['gpu_selector', 'gpu_ids'], form); const worker_selector = Form.useWatch('worker_selector', form); - const { onValuesChange } = useFormContext(); + const { onValuesChange, onBackendChange, backendOptions } = useFormContext(); const paramsConfig = useMemo(() => { return _.get(BackendParameters, backend, []); @@ -154,47 +148,7 @@ const AdvanceConfig: React.FC = (props) => { options={modelCategories} > - - } - options={ - backendOptions ?? [ - { - label: backendLabelMap[backendOptionsMap.vllm], - value: backendOptionsMap.vllm, - disabled: - props.source === modelSourceMap.local_path_value - ? false - : isGGUF - }, - { - label: backendLabelMap[backendOptionsMap.ascendMindie], - value: backendOptionsMap.ascendMindie, - disabled: - props.source === modelSourceMap.local_path_value - ? false - : isGGUF - }, - { - label: backendLabelMap[backendOptionsMap.voxBox], - value: backendOptionsMap.voxBox, - disabled: - props.source === modelSourceMap.local_path_value - ? false - : props.source === modelSourceMap.ollama_library_value || - isGGUF - } - ] - } - disabled={ - action === PageAction.EDIT && - props.source !== modelSourceMap.local_path_value - } - > - + {scheduleType === ScheduleValueMap.Auto && ( <> name="placement_strategy"> @@ -257,51 +211,6 @@ const AdvanceConfig: React.FC = (props) => { )} - - - - - {intl.formatMessage({ id: 'models.form.releases' })} - - - - - ) - } - )} - > - - name="backend_parameters"> []; + backendOptions: BackendOption[]; sourceList?: Global.BaseOption[]; clusterList: Global.BaseOption[]; fields?: string[]; @@ -41,7 +46,7 @@ const DataForm: React.FC = forwardRef((props, ref) => { formKey, initialValues, sourceDisable = true, - backendOptions, + backendOptions = [], sourceList, clusterList = [], fields = ['source'], @@ -174,8 +179,10 @@ const DataForm: React.FC = forwardRef((props, ref) => { value={{ isGGUF: isGGUF, formKey: formKey, + source: props.source, pageAction: action, gpuOptions: gpuOptions, + backendOptions: backendOptions, onValuesChange: onValuesChange, onBackendChange: handleBackendChange }} @@ -276,8 +283,6 @@ const DataForm: React.FC = forwardRef((props, ref) => { isGGUF={isGGUF} action={action} source={props.source} - backendOptions={backendOptions} - handleBackendChange={handleBackendChange} > diff --git a/src/pages/llmodels/components/deploy-modal.tsx b/src/pages/llmodels/components/deploy-modal.tsx index 3c406573..f2e4e18b 100644 --- a/src/pages/llmodels/components/deploy-modal.tsx +++ b/src/pages/llmodels/components/deploy-modal.tsx @@ -9,7 +9,7 @@ import { FC, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import styled from 'styled-components'; import { defaultFormValues, deployFormKeyMap, modelSourceMap } from '../config'; import { backendOptionsMap } from '../config/backend-parameters'; -import { FormData, SourceType } from '../config/types'; +import { BackendOption, FormData, SourceType } from '../config/types'; import { MessageStatus, WarningStausOptions, @@ -58,6 +58,7 @@ type AddModalProps = { width?: string | number; initialValues?: any; deploymentType?: 'modelList' | 'modelFiles'; + backendOptions: BackendOption[]; clusterList: Global.BaseOption< number, { provider: string; state: string | number } @@ -541,6 +542,7 @@ const AddModal: FC = (props) => { onOk={handleOnOk} ref={form} isGGUF={isGGUF} + backendOptions={props.backendOptions} onBackendChange={handleBackendChange} onValuesChange={onValuesChange} > diff --git a/src/pages/llmodels/components/table-list.tsx b/src/pages/llmodels/components/table-list.tsx index 82897567..4c964db3 100644 --- a/src/pages/llmodels/components/table-list.tsx +++ b/src/pages/llmodels/components/table-list.tsx @@ -57,6 +57,7 @@ import { } from '../config/types'; import useFormInitialValues from '../hooks/use-form-initial-values'; import useModelsColumns from '../hooks/use-models-columns'; +import useQueryBackends from '../hooks/use-query-backends'; import APIAccessInfoModal from './api-access-info'; import DeployModal from './deploy-modal'; import Instances from './instances'; @@ -123,6 +124,7 @@ const Models: React.FC = ({ loadend, total }) => { + const { backendOptions } = useQueryBackends(); const { generateFormValues, clusterList, getClusterList } = useFormInitialValues(); const { saveScrollHeight, restoreScrollHeight } = useBodyScroll(); @@ -686,6 +688,7 @@ const Models: React.FC = ({ clusterList={clusterList} onCancel={handleModalCancel} onOk={handleModalOk} + backendOptions={backendOptions} > = ({ isGGUF={openDeployModal.isGGUF} hasLinuxWorker={openDeployModal.hasLinuxWorker} clusterList={clusterList} + backendOptions={backendOptions} onCancel={handleDeployModalCancel} onOk={handleCreateModel} > diff --git a/src/pages/llmodels/components/update-modal.tsx b/src/pages/llmodels/components/update-modal.tsx index aae326fc..bbd926ba 100644 --- a/src/pages/llmodels/components/update-modal.tsx +++ b/src/pages/llmodels/components/update-modal.tsx @@ -11,7 +11,7 @@ import { updateIgnoreFields } from '../config'; import { backendOptionsMap } from '../config/backend-parameters'; -import { FormData } from '../config/types'; +import { BackendOption, FormData } from '../config/types'; import { generateGPUSelector } from '../config/utils'; import { useCheckCompatibility } from '../hooks'; import ColumnWrapper from './column-wrapper'; @@ -26,6 +26,7 @@ type AddModalProps = { data: FormData; isGGUF: boolean; }; + backendOptions: BackendOption[]; clusterList: Global.BaseOption< number, { provider: string; state: string | number } @@ -325,6 +326,7 @@ const UpdateModal: React.FC = (props) => { onOk={handleOk} ref={formRef} isGGUF={isGGUF} + backendOptions={props.backendOptions} onBackendChange={handleAsyncBackendChange} onValuesChange={handleManulOnValuesChange} > diff --git a/src/pages/llmodels/config/form-context.ts b/src/pages/llmodels/config/form-context.ts index a8e7937c..fd794582 100644 --- a/src/pages/llmodels/config/form-context.ts +++ b/src/pages/llmodels/config/form-context.ts @@ -1,15 +1,16 @@ import { PageActionType } from '@/config/types'; import React from 'react'; -import { DeployFormKey } from './types'; +import { BackendOption, DeployFormKey } from './types'; interface FormContextProps { isGGUF?: boolean; formKey: DeployFormKey; - source?: string; + source: string; pageAction: PageActionType; gpuOptions?: any[]; + backendOptions: BackendOption[]; onValuesChange?: (changedValues: any, allValues: any) => void; - onBackendChange?: (backend: string) => void; + onBackendChange: (backend: string) => void; } interface CatalogFormContextProps { @@ -19,6 +20,10 @@ interface CatalogFormContextProps { onQuantizationChange: (val: string) => void; } +interface FormOuterContextProps { + sourceList?: Global.BaseOption[]; +} + export const FormContext = React.createContext( {} as FormContextProps ); @@ -27,6 +32,10 @@ export const CatalogFormContext = React.createContext( {} as CatalogFormContextProps ); +export const FormOuterContext = React.createContext( + {} as FormOuterContextProps +); + export const useFormContext = () => { const context = React.useContext(FormContext); if (!context) { @@ -44,3 +53,13 @@ export const useCatalogFormContext = () => { } return context; }; + +export const useFormOuterContext = () => { + const context = React.useContext(FormOuterContext); + if (!context) { + throw new Error( + 'useFormOuterContext must be used within a FormOuterProvider' + ); + } + return context; +}; diff --git a/src/pages/llmodels/config/index.ts b/src/pages/llmodels/config/index.ts index 3385c127..da71e583 100644 --- a/src/pages/llmodels/config/index.ts +++ b/src/pages/llmodels/config/index.ts @@ -324,7 +324,9 @@ export const excludeFields = [ 'scheduleType', 'placement_strategy', 'backend', - 'gpu_selector' + 'gpu_selector', + 'run_command', + 'image_name' ]; // ingore fields when compare old and new data diff --git a/src/pages/llmodels/config/types.ts b/src/pages/llmodels/config/types.ts index d8fcec7c..7c961d16 100644 --- a/src/pages/llmodels/config/types.ts +++ b/src/pages/llmodels/config/types.ts @@ -39,6 +39,8 @@ export type SourceType = | 'ollama_library'; export interface FormData { + image_name?: string; + run_command?: string; backend: string; restart_on_error?: boolean; env?: Record; @@ -242,3 +244,11 @@ export interface EvaluateResult { }; }; } + +export interface BackendOption { + value: string; + label: string; + default_backend_param: string[]; + default_version: string; + versions: { label: string; value: string }[]; +} diff --git a/src/pages/llmodels/forms/backend-fields.tsx b/src/pages/llmodels/forms/backend-fields.tsx new file mode 100644 index 00000000..a753b45a --- /dev/null +++ b/src/pages/llmodels/forms/backend-fields.tsx @@ -0,0 +1,134 @@ +import IconFont from '@/components/icon-font'; +import AutoComplete from '@/components/seal-form/auto-complete'; +import SealInput from '@/components/seal-form/seal-input'; +import SealSelect from '@/components/seal-form/seal-select'; +import TooltipList from '@/components/tooltip-list'; +import { PageAction } from '@/config'; +import { useIntl } from '@umijs/max'; +import { Form, Typography } from 'antd'; +import React, { useMemo } from 'react'; +import { + backendLabelMap, + backendTipsList, + getBackendParamsTips, + modelSourceMap +} from '../config'; +import { backendOptionsMap } from '../config/backend-parameters'; +import { useFormContext } from '../config/form-context'; +import { FormData } from '../config/types'; + +const BackendFields: React.FC = () => { + const intl = useIntl(); + const form = Form.useFormInstance(); + const { + isGGUF, + formKey, + pageAction: action, + source, + gpuOptions, + onValuesChange, + backendOptions, + onBackendChange + } = useFormContext(); + const backend = Form.useWatch('backend', form); + + const handleBackendVersionOnBlur = () => { + onValuesChange?.({}, form.getFieldsValue()); + }; + + const backendParamsTips = useMemo(() => { + return getBackendParamsTips(backend); + }, [backend]); + + const backendVersions = useMemo(() => { + if (!backend || backend === backendOptionsMap.custom) { + return []; + } + return ( + backendOptions.find((item) => item.value === backend)?.versions || [] + ); + }, [backend]); + + return ( + <> + + } + options={backendOptions} + disabled={ + action === PageAction.EDIT && + source !== modelSourceMap.local_path_value + } + > + + {backendOptionsMap.custom !== backend && ( + + + + + {intl.formatMessage({ id: 'models.form.releases' })} + + + + + ) + } + )} + > + + )} + {backend === backendOptionsMap.custom && ( + <> + name="image_name" rules={[{ required: true }]}> + + + name="run_command" rules={[{ required: true }]}> + + + + )} + + ); +}; + +export default BackendFields; diff --git a/src/pages/llmodels/hooks/index.ts b/src/pages/llmodels/hooks/index.ts index 9302ded4..ecacb0fa 100644 --- a/src/pages/llmodels/hooks/index.ts +++ b/src/pages/llmodels/hooks/index.ts @@ -128,10 +128,6 @@ export const checkCurrentbackend = (data: { return backendOptionsMap.voxBox; } - if (isGGUF) { - return backendOptionsMap.llamaBox; - } - if (checkOnlyAscendNPU(gpuOptions)) { return backendOptionsMap.ascendMindie; } @@ -229,11 +225,6 @@ export const useCheckCompatibility = () => { return clusterNames.join(', '); }; - const getCurrentCluster = (id: number) => { - const cluster = clusterList.find?.((item) => item.value === id); - return cluster?.label || ''; - }; - const handleCheckCompatibility = ( evaluateResult: EvaluateResult | null ): MessageStatus => { diff --git a/src/pages/llmodels/hooks/use-query-backends.ts b/src/pages/llmodels/hooks/use-query-backends.ts new file mode 100644 index 00000000..1cc3adda --- /dev/null +++ b/src/pages/llmodels/hooks/use-query-backends.ts @@ -0,0 +1,41 @@ +import { backendOptionsAtom } from '@/atoms/models'; +import { useAtom } from 'jotai'; +import { useEffect } from 'react'; +import { queryBackendList } from '../apis'; + +export default function useQueryBackends() { + const [backendOptions, setBackendOptions] = useAtom(backendOptionsAtom); + + const getBackendOptions = async () => { + try { + const res = await queryBackendList(); + const list = res?.items?.map((item) => { + return { + value: item.backend_name, + label: item.backend_show_name, + default_backend_param: item.default_backend_param || [], + default_version: item.default_version, + versions: (item.versions || []).map((version) => ({ + label: version, + value: version + })) + }; + }); + if (res?.items) { + setBackendOptions(list || []); + } + } catch (error) { + // ignore + setBackendOptions([]); + } + }; + + useEffect(() => { + getBackendOptions(); + }, []); + + return { + backendOptions, + getBackendOptions + }; +} diff --git a/src/pages/resources/components/model-files.tsx b/src/pages/resources/components/model-files.tsx index 1cc9cc5c..7cf0c04e 100644 --- a/src/pages/resources/components/model-files.tsx +++ b/src/pages/resources/components/model-files.tsx @@ -17,6 +17,7 @@ import { import { SourceType } from '@/pages/llmodels/config/types'; import DownloadModal from '@/pages/llmodels/download'; import { useGenerateWorkerOptions } from '@/pages/llmodels/hooks/use-form-initial-values'; +import useQueryBackends from '@/pages/llmodels/hooks/use-query-backends'; import { PageContainer } from '@ant-design/pro-components'; import { useIntl, useNavigate } from '@umijs/max'; import { useMemoizedFn } from 'ahooks'; @@ -64,7 +65,7 @@ const ModelFiles = () => { watch: true, contentForDelete: 'resources.modelfiles.modelfile' }); - + const { backendOptions } = useQueryBackends(); const intl = useIntl(); const { showSuccess } = useAppUtils(); const [downloadModalStatus, setDownlaodMoalStatus] = useState<{ @@ -106,9 +107,15 @@ const ModelFiles = () => { worker_id: value }); }; - const generateInitialValues = (record: ListItem, gpuOptions: any[]) => { + + const checkIsGGUF = (record: ListItem) => { const isGGUF = _.includes(record.resolved_paths?.[0], 'gguf'); const isOllama = !!record.ollama_library_model_name; + return isGGUF || isOllama; + }; + + const generateInitialValues = (record: ListItem, gpuOptions: any[]) => { + const isGGUF = checkIsGGUF(record); const audioModelTag = identifyModelTask( record.source, record.resolved_paths?.[0] @@ -140,12 +147,12 @@ const ModelFiles = () => { : {}, name: extractFileName(name), backend: checkCurrentbackend({ - isGGUF: !audioModelTag && (isGGUF || isOllama), + isGGUF: !audioModelTag && isGGUF, isAudio: !!audioModelTag, gpuOptions: gpuOptions, defaultBackend: backendOptionsMap.vllm }), - isGGUF: !audioModelTag && (isGGUF || isOllama) + isGGUF: !audioModelTag && isGGUF }; }; @@ -338,6 +345,7 @@ const ModelFiles = () => { workerOptions={readyWorkers} >