diff --git a/src/locales/en-US/models.ts b/src/locales/en-US/models.ts
index 8d8c2a8b..4669ae6a 100644
--- a/src/locales/en-US/models.ts
+++ b/src/locales/en-US/models.ts
@@ -242,7 +242,7 @@ export default {
'models.form.backend.custom': 'User-defined',
'models.form.rules.name':
'Up to 63 characters; letters, numbers, dots (.), underscores (_), and hyphens (-) only; must start and end with an alphanumeric character.',
- 'models.catalog.button.explore': 'Explore Other Models',
+ 'models.catalog.button.explore': 'Explore More Models',
'models.catalog.precision': 'Precision',
'models.form.gpuPerReplica.tips': 'Enter a custom number',
'models.form.generic_proxy': 'Enable Generic Proxy',
diff --git a/src/locales/ja-JP/models.ts b/src/locales/ja-JP/models.ts
index 295683e9..66241973 100644
--- a/src/locales/ja-JP/models.ts
+++ b/src/locales/ja-JP/models.ts
@@ -242,7 +242,7 @@ export default {
'models.form.backend.custom': 'User Defined',
'models.form.rules.name':
'Up to 63 characters; letters, numbers, dots (.), underscores (_), and hyphens (-) only; must start and end with an alphanumeric character.',
- 'models.catalog.button.explore': 'Explore Other Models',
+ 'models.catalog.button.explore': 'Explore More Models',
'models.catalog.precision': 'Precision',
'models.form.gpuPerReplica.tips': 'Enter a custom number',
'models.form.generic_proxy': 'Enable Generic Proxy',
@@ -321,7 +321,7 @@ export default {
// 47. 'models.form.quantization': 'Quantization',
// 48. 'models.form.backend.custom': 'User Defined',
// 49. 'models.form.rules.name': 'Up to 63 characters; letters, numbers, dots (.), underscores (_), and hyphens (-) only; must start and end with an alphanumeric character.',
-// 50. 'models.catalog.button.explore': 'Explore Other Models',
+// 50. 'models.catalog.button.explore': 'Explore More Models',
// 51. 'models.catalog.precision': 'Precision',
// 52. 'models.form.gpuPerReplica.tips': 'Enter a custom number',
// 53. 'models.form.generic_proxy': 'Enable Generic Proxy',
diff --git a/src/locales/ru-RU/models.ts b/src/locales/ru-RU/models.ts
index 54dcdc66..e7f5e74d 100644
--- a/src/locales/ru-RU/models.ts
+++ b/src/locales/ru-RU/models.ts
@@ -243,7 +243,7 @@ export default {
'models.form.backend.custom': 'Пользовательский',
'models.form.rules.name':
'До 63 символов; только буквы, цифры, точки (.), подчёркивания (_) и дефисы (-); должно начинаться и заканчиваться буквенно-цифровым символом.',
- 'models.catalog.button.explore': 'Изучить другие модели',
+ 'models.catalog.button.explore': 'Explore More Models',
'models.catalog.precision': 'Точность',
'models.form.gpuPerReplica.tips': 'Введите произвольное число',
'models.form.generic_proxy': 'Включить универсальный прокси',
@@ -255,5 +255,6 @@ export default {
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
-// 7. 'models.table.genericProxy': 'Refer to the curl example below. The proxy forwards requests with the /model/proxy prefix to the corresponding model. You need to specify the model name either in the X-GPUStack-Model request header or in the "model" property of the JSON body.'
+// 1. 'models.catalog.button.explore': 'Explore More Models',
+// 2. 'models.table.genericProxy': 'Refer to the curl example below. The proxy forwards requests with the /model/proxy prefix to the corresponding model. You need to specify the model name either in the X-GPUStack-Model request header or in the "model" property of the JSON body.'
// ========== End of To-Do List ==========
diff --git a/src/locales/zh-CN/models.ts b/src/locales/zh-CN/models.ts
index 7d182e65..8a3af60a 100644
--- a/src/locales/zh-CN/models.ts
+++ b/src/locales/zh-CN/models.ts
@@ -229,7 +229,7 @@ export default {
'models.form.backend.custom': '用户定义',
'models.form.rules.name':
'长度不超过 63 个字符,只能包含字母、数字、点(.)、下划线(_)和连字符(-),且必须以字母或数字开头和结尾。',
- 'models.catalog.button.explore': '浏览其他模型',
+ 'models.catalog.button.explore': '浏览更多模型',
'models.catalog.precision': '精度',
'models.form.gpuPerReplica.tips': '输入自定义数值',
'models.form.generic_proxy': '启用通用代理',
diff --git a/src/pages/llmodels/apis/index.ts b/src/pages/llmodels/apis/index.ts
index d3e486af..db5b5bbc 100644
--- a/src/pages/llmodels/apis/index.ts
+++ b/src/pages/llmodels/apis/index.ts
@@ -384,6 +384,7 @@ export async function queryCatalogItemSpec(
export async function evaluationsModelSpec(
data: {
+ cluster_id: number;
model_specs: EvaluateSpec[];
},
options: { token: any }
diff --git a/src/pages/llmodels/components/deploy-builtin-modal.tsx b/src/pages/llmodels/components/deploy-builtin-modal.tsx
index 002914fa..4555d1c6 100644
--- a/src/pages/llmodels/components/deploy-builtin-modal.tsx
+++ b/src/pages/llmodels/components/deploy-builtin-modal.tsx
@@ -191,25 +191,6 @@ const AddModal: React.FC = (props) => {
size: number;
quantization: string;
}) => {
- // const spec = _.find(specListRef.current, (item: CatalogSpec) => {
- // if (data.size && data.quantization) {
- // return (
- // item.size === data.size &&
- // item.backend === data.backend &&
- // item.quantization === data.quantization
- // );
- // }
- // if (data.size) {
- // return item.size === data.size && item.backend === data.backend;
- // }
- // if (data.quantization) {
- // return (
- // item.quantization === data.quantization &&
- // item.backend === data.backend
- // );
- // }
- // return item.backend === data.backend;
- // });
const defaultSpec = _.find(
specListRef.current,
(item: CatalogSpec) => item.mode === data.mode
diff --git a/src/pages/llmodels/components/search-model.tsx b/src/pages/llmodels/components/search-model.tsx
index 688eafd4..96da2c8e 100644
--- a/src/pages/llmodels/components/search-model.tsx
+++ b/src/pages/llmodels/components/search-model.tsx
@@ -252,6 +252,7 @@ const SearchModel: React.FC = (props) => {
checkTokenRef.current = createAxiosToken();
const evaluations = await evaluationsModelSpec(
{
+ cluster_id: clusterId!,
model_specs: repoList
},
{
diff --git a/src/pages/llmodels/config/button-actions.ts b/src/pages/llmodels/config/button-actions.ts
index 7ecbfdfb..cdd39484 100644
--- a/src/pages/llmodels/config/button-actions.ts
+++ b/src/pages/llmodels/config/button-actions.ts
@@ -170,6 +170,10 @@ export const setModelActionList = (record: any) => {
return record.replicas > 0;
}
+ if (action.key === 'proxy') {
+ return record.generic_proxy;
+ }
+
return true;
});
};
diff --git a/src/pages/llmodels/download/index.tsx b/src/pages/llmodels/download/index.tsx
index a35738bb..20a9875b 100644
--- a/src/pages/llmodels/download/index.tsx
+++ b/src/pages/llmodels/download/index.tsx
@@ -86,6 +86,7 @@ const DownloadModel: React.FC = (props) => {
const handleOnSelectModel = (item: any) => {
setSelectedModel(item);
+ setFileName('');
};
const handleOk = async (values: any) => {
diff --git a/src/pages/llmodels/forms/kv-cache.tsx b/src/pages/llmodels/forms/kv-cache.tsx
index 53989902..4884d037 100644
--- a/src/pages/llmodels/forms/kv-cache.tsx
+++ b/src/pages/llmodels/forms/kv-cache.tsx
@@ -41,6 +41,15 @@ const KVCacheForm = () => {
);
};
+ const handleRamRatioChange = (
+ value: number | null | string,
+ field: string
+ ) => {
+ if (!value) {
+ form.setFieldValue(['extended_kv_cache', field], null);
+ }
+ };
+
const builtInBackend = useMemo(() => {
const currentBackend = backendOptions.find(
(item) => item.value === backend
@@ -75,6 +84,7 @@ const KVCacheForm = () => {
<>
name={['extended_kv_cache', 'ram_ratio']}>
handleRamRatioChange(value, 'ram_ratio')}
label={intl.formatMessage({ id: 'models.form.ramRatio' })}
description={intl.formatMessage({
id: 'models.form.ramRatio.tips'
@@ -86,6 +96,7 @@ const KVCacheForm = () => {
name={['extended_kv_cache', 'ram_size']}>
handleRamRatioChange(value, 'ram_size')}
label={intl.formatMessage({ id: 'models.form.ramSize' })}
description={intl.formatMessage(
{
@@ -100,6 +111,7 @@ const KVCacheForm = () => {
name={['extended_kv_cache', 'chunk_size']}>
handleRamRatioChange(value, 'chunk_size')}
label={intl.formatMessage({ id: 'models.form.chunkSize' })}
description={intl.formatMessage({
id: 'models.form.chunkSize.tips'
diff --git a/src/pages/llmodels/forms/speculative-decode.tsx b/src/pages/llmodels/forms/speculative-decode.tsx
index 4b076edf..595dda55 100644
--- a/src/pages/llmodels/forms/speculative-decode.tsx
+++ b/src/pages/llmodels/forms/speculative-decode.tsx
@@ -182,6 +182,7 @@ const SpeculativeDecode = () => {
label={intl.formatMessage({ id: 'models.form.numDraftTokens' })}
min={1}
step={1}
+ required
precision={0}
/>
diff --git a/src/pages/llmodels/hooks/index.ts b/src/pages/llmodels/hooks/index.ts
index 5bffaa93..5be8c31c 100644
--- a/src/pages/llmodels/hooks/index.ts
+++ b/src/pages/llmodels/hooks/index.ts
@@ -180,6 +180,7 @@ export const useCheckCompatibility = () => {
});
const evalution = await evaluationsModelSpec(
{
+ cluster_id: data.cluster_id,
model_specs: [
{
..._.omit(data, ['scheduleType']),
diff --git a/src/pages/llmodels/hooks/use-generic-proxy.tsx b/src/pages/llmodels/hooks/use-generic-proxy.tsx
index b7e85e33..d14bec20 100644
--- a/src/pages/llmodels/hooks/use-generic-proxy.tsx
+++ b/src/pages/llmodels/hooks/use-generic-proxy.tsx
@@ -17,57 +17,72 @@ import { generateEmbeddingCurlCode } from '@/pages/playground/view-code/embeddin
import { generateImageCurlCode } from '@/pages/playground/view-code/image';
import { generateLLmCurlCode } from '@/pages/playground/view-code/llm';
import { generateRerankCurlCode } from '@/pages/playground/view-code/rerank';
-import { formatCurlArgs } from '@/pages/playground/view-code/utils';
import { useIntl } from '@umijs/max';
import { useState } from 'react';
import { modelCategoriesMap } from '../config';
const API_MAP: Record<
string,
- { api: string; generateCurlCode: (args: any) => string }
+ { api: string; parameters: any; generateCurlCode: (args: any) => string }
> = {
[modelCategoriesMap.embedding]: {
api: EMBEDDING_API,
+ parameters: {
+ query: 'What are the benefits of regular exercise?',
+ documents: [
+ 'Regular physical activity helps improve cardiovascular health and mental well-being.',
+ 'Eating too much sugar can lead to health issues.',
+ 'Exercise is often done in gyms or outdoors.'
+ ]
+ },
generateCurlCode: generateEmbeddingCurlCode
},
[modelCategoriesMap.llm]: {
api: CHAT_API,
+ parameters: {
+ messages: [
+ {
+ role: 'user',
+ content: 'Hello, introduce yourself'
+ }
+ ]
+ },
generateCurlCode: generateLLmCurlCode
},
[modelCategoriesMap.image]: {
api: CREAT_IMAGE_API,
+ parameters: {},
generateCurlCode: generateImageCurlCode
},
[modelCategoriesMap.text_to_speech]: {
api: AUDIO_TEXT_TO_SPEECH_API,
+ parameters: {
+ response_format: 'mp3',
+ input: ''
+ },
generateCurlCode: generateTextToSpeechCurlCode
},
[modelCategoriesMap.speech_to_text]: {
api: AUDIO_SPEECH_TO_TEXT_API,
+ parameters: {},
generateCurlCode: generateSpeechToTextCurlCode
},
[modelCategoriesMap.reranker]: {
api: RERANKER_API,
+ parameters: {
+ messages: [
+ {
+ role: 'user',
+ content: 'Hello, introduce yourself'
+ }
+ ]
+ },
generateCurlCode: generateRerankCurlCode
}
};
const langOptions = [{ label: 'Curl', value: 'bash' }];
-const generateCode = ({ api: url, parameters }: Record) => {
- const host = window.location.origin;
- const api = '/model/proxy/v1/';
-
- // ========================= Curl =========================
- const curlCode = `
-curl ${host}${api} \\
--H "Content-Type: application/json" \\
--H "Authorization: Bearer $\{YOUR_GPUSTACK_API_KEY}" \\
-${formatCurlArgs(parameters, false)}`.trim();
-
- return curlCode;
-};
-
const useGenericProxy = () => {
const intl = useIntl();
const [modalStatus, setModalStatus] = useState<{
@@ -88,6 +103,7 @@ const useGenericProxy = () => {
return {
category,
api: `${MODEL_PROXY}${config.api}`,
+ parameters: config.parameters,
generateCurlCode: config.generateCurlCode
};
}
@@ -95,12 +111,22 @@ const useGenericProxy = () => {
return {
category: modelCategoriesMap.llm,
api: CHAT_API,
+ parameters: {
+ messages: [
+ {
+ role: 'user',
+ content: 'Hello, introduce yourself'
+ }
+ ]
+ },
generateCurlCode: generateLLmCurlCode
};
};
const openProxyModal = (data?: any) => {
- const { api, generateCurlCode } = getModelCategory(data?.categories || []);
+ const { api, generateCurlCode, parameters } = getModelCategory(
+ data?.categories || []
+ );
setModalStatus({
open: true,
@@ -108,7 +134,8 @@ const useGenericProxy = () => {
api,
modelProxy: true,
parameters: {
- model: data?.name || ''
+ model: data?.name || '',
+ ...parameters
}
})
});
diff --git a/src/pages/playground/components/ground-left.tsx b/src/pages/playground/components/ground-left.tsx
index 994b8b0f..adb2afa2 100644
--- a/src/pages/playground/components/ground-left.tsx
+++ b/src/pages/playground/components/ground-left.tsx
@@ -52,13 +52,13 @@ const GroundLeft: React.FC = forwardRef((props, ref) => {
handleAddNewMessage,
handleClear,
setMessageList,
+ formRef,
tokenResult,
messageList,
loading
} = useChatCompletion(scroller);
const {
handleOnValuesChange,
- formRef,
paramsRef,
paramsConfig,
initialValues,
@@ -108,7 +108,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => {
return message;
};
- const handleSendMessage = (message: Omit) => {
+ const handleSendMessage = async (message: Omit) => {
const currentMessage = generateValidMessage(message);
submitMessage({
system: systemMessage
diff --git a/src/pages/playground/hooks/use-chat-completion.ts b/src/pages/playground/hooks/use-chat-completion.ts
index cb479149..02ce0a6e 100644
--- a/src/pages/playground/hooks/use-chat-completion.ts
+++ b/src/pages/playground/hooks/use-chat-completion.ts
@@ -23,6 +23,7 @@ export default function useChatCompletion(
const currentMessageRef = useRef(null);
const messageListLengthCache = useRef(0);
const reasonContentRef = useRef('');
+ const formRef = useRef(null);
const setMessageId = () => {
messageId.current = messageId.current + 1;
@@ -43,7 +44,6 @@ export default function useChatCompletion(
};
const joinMessage = (chunk: any) => {
- console.log('chunk:', chunk);
setTokenResult({
...(chunk?.usage ?? {})
});
@@ -217,6 +217,7 @@ export default function useChatCompletion(
loading,
tokenResult,
messageList,
+ formRef,
setMessageId,
setMessageList,
handleClear,
diff --git a/src/pages/playground/hooks/use-init-meta.ts b/src/pages/playground/hooks/use-init-meta.ts
index b68c1e89..3913d7a5 100644
--- a/src/pages/playground/hooks/use-init-meta.ts
+++ b/src/pages/playground/hooks/use-init-meta.ts
@@ -52,7 +52,6 @@ export const useInitLLmMeta = (
defaultValues = {},
defaultParamsConfig = []
} = options;
- const formRef = useRef(null);
const [searchParams] = useSearchParams();
const [modelMeta, setModelMeta] = useState({});
const [initialValues, setInitialValues] = useState({
@@ -76,8 +75,6 @@ export const useInitLLmMeta = (
return model;
}, [model, modelList, isChat]);
- const getLLamaBoxMeta = (meta: any) => {};
-
const getMaxTokens = (meta: any) => {
const { max_model_len, n_ctx, n_slot, max_total_tokens } = meta || {};
@@ -197,7 +194,6 @@ export const useInitLLmMeta = (
setInitialValues,
setParams,
setParamsConfig,
- formRef,
paramsConfig,
initialValues,
parameters,