diff --git a/src/locales/en-US/models.ts b/src/locales/en-US/models.ts index 8d8c2a8b..4669ae6a 100644 --- a/src/locales/en-US/models.ts +++ b/src/locales/en-US/models.ts @@ -242,7 +242,7 @@ export default { 'models.form.backend.custom': 'User-defined', 'models.form.rules.name': 'Up to 63 characters; letters, numbers, dots (.), underscores (_), and hyphens (-) only; must start and end with an alphanumeric character.', - 'models.catalog.button.explore': 'Explore Other Models', + 'models.catalog.button.explore': 'Explore More Models', 'models.catalog.precision': 'Precision', 'models.form.gpuPerReplica.tips': 'Enter a custom number', 'models.form.generic_proxy': 'Enable Generic Proxy', diff --git a/src/locales/ja-JP/models.ts b/src/locales/ja-JP/models.ts index 295683e9..66241973 100644 --- a/src/locales/ja-JP/models.ts +++ b/src/locales/ja-JP/models.ts @@ -242,7 +242,7 @@ export default { 'models.form.backend.custom': 'User Defined', 'models.form.rules.name': 'Up to 63 characters; letters, numbers, dots (.), underscores (_), and hyphens (-) only; must start and end with an alphanumeric character.', - 'models.catalog.button.explore': 'Explore Other Models', + 'models.catalog.button.explore': 'Explore More Models', 'models.catalog.precision': 'Precision', 'models.form.gpuPerReplica.tips': 'Enter a custom number', 'models.form.generic_proxy': 'Enable Generic Proxy', @@ -321,7 +321,7 @@ export default { // 47. 'models.form.quantization': 'Quantization', // 48. 'models.form.backend.custom': 'User Defined', // 49. 'models.form.rules.name': 'Up to 63 characters; letters, numbers, dots (.), underscores (_), and hyphens (-) only; must start and end with an alphanumeric character.', -// 50. 'models.catalog.button.explore': 'Explore Other Models', +// 50. 'models.catalog.button.explore': 'Explore More Models', // 51. 'models.catalog.precision': 'Precision', // 52. 'models.form.gpuPerReplica.tips': 'Enter a custom number', // 53. 'models.form.generic_proxy': 'Enable Generic Proxy', diff --git a/src/locales/ru-RU/models.ts b/src/locales/ru-RU/models.ts index 54dcdc66..e7f5e74d 100644 --- a/src/locales/ru-RU/models.ts +++ b/src/locales/ru-RU/models.ts @@ -243,7 +243,7 @@ export default { 'models.form.backend.custom': 'Пользовательский', 'models.form.rules.name': 'До 63 символов; только буквы, цифры, точки (.), подчёркивания (_) и дефисы (-); должно начинаться и заканчиваться буквенно-цифровым символом.', - 'models.catalog.button.explore': 'Изучить другие модели', + 'models.catalog.button.explore': 'Explore More Models', 'models.catalog.precision': 'Точность', 'models.form.gpuPerReplica.tips': 'Введите произвольное число', 'models.form.generic_proxy': 'Включить универсальный прокси', @@ -255,5 +255,6 @@ export default { }; // ========== To-Do: Translate Keys (Remove After Translation) ========== -// 7. 'models.table.genericProxy': 'Refer to the curl example below. The proxy forwards requests with the /model/proxy prefix to the corresponding model. You need to specify the model name either in the X-GPUStack-Model request header or in the "model" property of the JSON body.' +// 1. 'models.catalog.button.explore': 'Explore More Models', +// 2. 'models.table.genericProxy': 'Refer to the curl example below. The proxy forwards requests with the /model/proxy prefix to the corresponding model. You need to specify the model name either in the X-GPUStack-Model request header or in the "model" property of the JSON body.' // ========== End of To-Do List ========== diff --git a/src/locales/zh-CN/models.ts b/src/locales/zh-CN/models.ts index 7d182e65..8a3af60a 100644 --- a/src/locales/zh-CN/models.ts +++ b/src/locales/zh-CN/models.ts @@ -229,7 +229,7 @@ export default { 'models.form.backend.custom': '用户定义', 'models.form.rules.name': '长度不超过 63 个字符,只能包含字母、数字、点(.)、下划线(_)和连字符(-),且必须以字母或数字开头和结尾。', - 'models.catalog.button.explore': '浏览其他模型', + 'models.catalog.button.explore': '浏览更多模型', 'models.catalog.precision': '精度', 'models.form.gpuPerReplica.tips': '输入自定义数值', 'models.form.generic_proxy': '启用通用代理', diff --git a/src/pages/llmodels/apis/index.ts b/src/pages/llmodels/apis/index.ts index d3e486af..db5b5bbc 100644 --- a/src/pages/llmodels/apis/index.ts +++ b/src/pages/llmodels/apis/index.ts @@ -384,6 +384,7 @@ export async function queryCatalogItemSpec( export async function evaluationsModelSpec( data: { + cluster_id: number; model_specs: EvaluateSpec[]; }, options: { token: any } diff --git a/src/pages/llmodels/components/deploy-builtin-modal.tsx b/src/pages/llmodels/components/deploy-builtin-modal.tsx index 002914fa..4555d1c6 100644 --- a/src/pages/llmodels/components/deploy-builtin-modal.tsx +++ b/src/pages/llmodels/components/deploy-builtin-modal.tsx @@ -191,25 +191,6 @@ const AddModal: React.FC = (props) => { size: number; quantization: string; }) => { - // const spec = _.find(specListRef.current, (item: CatalogSpec) => { - // if (data.size && data.quantization) { - // return ( - // item.size === data.size && - // item.backend === data.backend && - // item.quantization === data.quantization - // ); - // } - // if (data.size) { - // return item.size === data.size && item.backend === data.backend; - // } - // if (data.quantization) { - // return ( - // item.quantization === data.quantization && - // item.backend === data.backend - // ); - // } - // return item.backend === data.backend; - // }); const defaultSpec = _.find( specListRef.current, (item: CatalogSpec) => item.mode === data.mode diff --git a/src/pages/llmodels/components/search-model.tsx b/src/pages/llmodels/components/search-model.tsx index 688eafd4..96da2c8e 100644 --- a/src/pages/llmodels/components/search-model.tsx +++ b/src/pages/llmodels/components/search-model.tsx @@ -252,6 +252,7 @@ const SearchModel: React.FC = (props) => { checkTokenRef.current = createAxiosToken(); const evaluations = await evaluationsModelSpec( { + cluster_id: clusterId!, model_specs: repoList }, { diff --git a/src/pages/llmodels/config/button-actions.ts b/src/pages/llmodels/config/button-actions.ts index 7ecbfdfb..cdd39484 100644 --- a/src/pages/llmodels/config/button-actions.ts +++ b/src/pages/llmodels/config/button-actions.ts @@ -170,6 +170,10 @@ export const setModelActionList = (record: any) => { return record.replicas > 0; } + if (action.key === 'proxy') { + return record.generic_proxy; + } + return true; }); }; diff --git a/src/pages/llmodels/download/index.tsx b/src/pages/llmodels/download/index.tsx index a35738bb..20a9875b 100644 --- a/src/pages/llmodels/download/index.tsx +++ b/src/pages/llmodels/download/index.tsx @@ -86,6 +86,7 @@ const DownloadModel: React.FC = (props) => { const handleOnSelectModel = (item: any) => { setSelectedModel(item); + setFileName(''); }; const handleOk = async (values: any) => { diff --git a/src/pages/llmodels/forms/kv-cache.tsx b/src/pages/llmodels/forms/kv-cache.tsx index 53989902..4884d037 100644 --- a/src/pages/llmodels/forms/kv-cache.tsx +++ b/src/pages/llmodels/forms/kv-cache.tsx @@ -41,6 +41,15 @@ const KVCacheForm = () => { ); }; + const handleRamRatioChange = ( + value: number | null | string, + field: string + ) => { + if (!value) { + form.setFieldValue(['extended_kv_cache', field], null); + } + }; + const builtInBackend = useMemo(() => { const currentBackend = backendOptions.find( (item) => item.value === backend @@ -75,6 +84,7 @@ const KVCacheForm = () => { <> name={['extended_kv_cache', 'ram_ratio']}> handleRamRatioChange(value, 'ram_ratio')} label={intl.formatMessage({ id: 'models.form.ramRatio' })} description={intl.formatMessage({ id: 'models.form.ramRatio.tips' @@ -86,6 +96,7 @@ const KVCacheForm = () => { name={['extended_kv_cache', 'ram_size']}> handleRamRatioChange(value, 'ram_size')} label={intl.formatMessage({ id: 'models.form.ramSize' })} description={intl.formatMessage( { @@ -100,6 +111,7 @@ const KVCacheForm = () => { name={['extended_kv_cache', 'chunk_size']}> handleRamRatioChange(value, 'chunk_size')} label={intl.formatMessage({ id: 'models.form.chunkSize' })} description={intl.formatMessage({ id: 'models.form.chunkSize.tips' diff --git a/src/pages/llmodels/forms/speculative-decode.tsx b/src/pages/llmodels/forms/speculative-decode.tsx index 4b076edf..595dda55 100644 --- a/src/pages/llmodels/forms/speculative-decode.tsx +++ b/src/pages/llmodels/forms/speculative-decode.tsx @@ -182,6 +182,7 @@ const SpeculativeDecode = () => { label={intl.formatMessage({ id: 'models.form.numDraftTokens' })} min={1} step={1} + required precision={0} /> diff --git a/src/pages/llmodels/hooks/index.ts b/src/pages/llmodels/hooks/index.ts index 5bffaa93..5be8c31c 100644 --- a/src/pages/llmodels/hooks/index.ts +++ b/src/pages/llmodels/hooks/index.ts @@ -180,6 +180,7 @@ export const useCheckCompatibility = () => { }); const evalution = await evaluationsModelSpec( { + cluster_id: data.cluster_id, model_specs: [ { ..._.omit(data, ['scheduleType']), diff --git a/src/pages/llmodels/hooks/use-generic-proxy.tsx b/src/pages/llmodels/hooks/use-generic-proxy.tsx index b7e85e33..d14bec20 100644 --- a/src/pages/llmodels/hooks/use-generic-proxy.tsx +++ b/src/pages/llmodels/hooks/use-generic-proxy.tsx @@ -17,57 +17,72 @@ import { generateEmbeddingCurlCode } from '@/pages/playground/view-code/embeddin import { generateImageCurlCode } from '@/pages/playground/view-code/image'; import { generateLLmCurlCode } from '@/pages/playground/view-code/llm'; import { generateRerankCurlCode } from '@/pages/playground/view-code/rerank'; -import { formatCurlArgs } from '@/pages/playground/view-code/utils'; import { useIntl } from '@umijs/max'; import { useState } from 'react'; import { modelCategoriesMap } from '../config'; const API_MAP: Record< string, - { api: string; generateCurlCode: (args: any) => string } + { api: string; parameters: any; generateCurlCode: (args: any) => string } > = { [modelCategoriesMap.embedding]: { api: EMBEDDING_API, + parameters: { + query: 'What are the benefits of regular exercise?', + documents: [ + 'Regular physical activity helps improve cardiovascular health and mental well-being.', + 'Eating too much sugar can lead to health issues.', + 'Exercise is often done in gyms or outdoors.' + ] + }, generateCurlCode: generateEmbeddingCurlCode }, [modelCategoriesMap.llm]: { api: CHAT_API, + parameters: { + messages: [ + { + role: 'user', + content: 'Hello, introduce yourself' + } + ] + }, generateCurlCode: generateLLmCurlCode }, [modelCategoriesMap.image]: { api: CREAT_IMAGE_API, + parameters: {}, generateCurlCode: generateImageCurlCode }, [modelCategoriesMap.text_to_speech]: { api: AUDIO_TEXT_TO_SPEECH_API, + parameters: { + response_format: 'mp3', + input: '' + }, generateCurlCode: generateTextToSpeechCurlCode }, [modelCategoriesMap.speech_to_text]: { api: AUDIO_SPEECH_TO_TEXT_API, + parameters: {}, generateCurlCode: generateSpeechToTextCurlCode }, [modelCategoriesMap.reranker]: { api: RERANKER_API, + parameters: { + messages: [ + { + role: 'user', + content: 'Hello, introduce yourself' + } + ] + }, generateCurlCode: generateRerankCurlCode } }; const langOptions = [{ label: 'Curl', value: 'bash' }]; -const generateCode = ({ api: url, parameters }: Record) => { - const host = window.location.origin; - const api = '/model/proxy/v1/'; - - // ========================= Curl ========================= - const curlCode = ` -curl ${host}${api} \\ --H "Content-Type: application/json" \\ --H "Authorization: Bearer $\{YOUR_GPUSTACK_API_KEY}" \\ -${formatCurlArgs(parameters, false)}`.trim(); - - return curlCode; -}; - const useGenericProxy = () => { const intl = useIntl(); const [modalStatus, setModalStatus] = useState<{ @@ -88,6 +103,7 @@ const useGenericProxy = () => { return { category, api: `${MODEL_PROXY}${config.api}`, + parameters: config.parameters, generateCurlCode: config.generateCurlCode }; } @@ -95,12 +111,22 @@ const useGenericProxy = () => { return { category: modelCategoriesMap.llm, api: CHAT_API, + parameters: { + messages: [ + { + role: 'user', + content: 'Hello, introduce yourself' + } + ] + }, generateCurlCode: generateLLmCurlCode }; }; const openProxyModal = (data?: any) => { - const { api, generateCurlCode } = getModelCategory(data?.categories || []); + const { api, generateCurlCode, parameters } = getModelCategory( + data?.categories || [] + ); setModalStatus({ open: true, @@ -108,7 +134,8 @@ const useGenericProxy = () => { api, modelProxy: true, parameters: { - model: data?.name || '' + model: data?.name || '', + ...parameters } }) }); diff --git a/src/pages/playground/components/ground-left.tsx b/src/pages/playground/components/ground-left.tsx index 994b8b0f..adb2afa2 100644 --- a/src/pages/playground/components/ground-left.tsx +++ b/src/pages/playground/components/ground-left.tsx @@ -52,13 +52,13 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { handleAddNewMessage, handleClear, setMessageList, + formRef, tokenResult, messageList, loading } = useChatCompletion(scroller); const { handleOnValuesChange, - formRef, paramsRef, paramsConfig, initialValues, @@ -108,7 +108,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { return message; }; - const handleSendMessage = (message: Omit) => { + const handleSendMessage = async (message: Omit) => { const currentMessage = generateValidMessage(message); submitMessage({ system: systemMessage diff --git a/src/pages/playground/hooks/use-chat-completion.ts b/src/pages/playground/hooks/use-chat-completion.ts index cb479149..02ce0a6e 100644 --- a/src/pages/playground/hooks/use-chat-completion.ts +++ b/src/pages/playground/hooks/use-chat-completion.ts @@ -23,6 +23,7 @@ export default function useChatCompletion( const currentMessageRef = useRef(null); const messageListLengthCache = useRef(0); const reasonContentRef = useRef(''); + const formRef = useRef(null); const setMessageId = () => { messageId.current = messageId.current + 1; @@ -43,7 +44,6 @@ export default function useChatCompletion( }; const joinMessage = (chunk: any) => { - console.log('chunk:', chunk); setTokenResult({ ...(chunk?.usage ?? {}) }); @@ -217,6 +217,7 @@ export default function useChatCompletion( loading, tokenResult, messageList, + formRef, setMessageId, setMessageList, handleClear, diff --git a/src/pages/playground/hooks/use-init-meta.ts b/src/pages/playground/hooks/use-init-meta.ts index b68c1e89..3913d7a5 100644 --- a/src/pages/playground/hooks/use-init-meta.ts +++ b/src/pages/playground/hooks/use-init-meta.ts @@ -52,7 +52,6 @@ export const useInitLLmMeta = ( defaultValues = {}, defaultParamsConfig = [] } = options; - const formRef = useRef(null); const [searchParams] = useSearchParams(); const [modelMeta, setModelMeta] = useState({}); const [initialValues, setInitialValues] = useState({ @@ -76,8 +75,6 @@ export const useInitLLmMeta = ( return model; }, [model, modelList, isChat]); - const getLLamaBoxMeta = (meta: any) => {}; - const getMaxTokens = (meta: any) => { const { max_model_len, n_ctx, n_slot, max_total_tokens } = meta || {}; @@ -197,7 +194,6 @@ export const useInitLLmMeta = ( setInitialValues, setParams, setParamsConfig, - formRef, paramsConfig, initialValues, parameters,