root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update vllm parameters, do not change backend when file type unknown

Browse Source
This commit is contained in:
jialin
2025-04-16 12:34:18 +08:00
parent ced45bec65
commit c831c19848
14 changed files with 298 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pages/llmodels/components/api-access-info.tsx
+1 -1
View File
@@ -118,7 +118,7 @@ const ApiAccessInfo = ({ open, data, onClose }: ApiAccessInfoProps) => {
</Tips>
<ApiAccessInfoWrapper>
<span className="label">
{intl.formatMessage({ id: 'models.table.apiAccessInfo.enpoint' })}
{intl.formatMessage({ id: 'models.table.apiAccessInfo.endpoint' })}
</span>
<span className="value">
<AutoTooltip ghost maxWidth={200}>
src/pages/llmodels/components/model-card.tsx
+14 -11
View File
@@ -277,18 +277,21 @@ const ModelCard: React.FC<{
return null;
};
const generateModeScopeImgLink = (imgSrc: string) => {
if (!imgSrc) {
const generateModeScopeImgLink = useCallback(
(imgSrc: string) => {
if (!imgSrc) {
return '';
}
if (modelSource === modelSourceMap.modelscope_value) {
return `https://modelscope.cn/api/v1/models/${modelData?.name}/repo?Revision=${modelData?.Revision}&View=true&FilePath=${imgSrc}`;
}
if (modelSource === modelSourceMap.huggingface_value) {
return `https://huggingface.co/${modelData?.id}/resolve/main/${imgSrc}`;
}
return '';
}
if (modelSource === modelSourceMap.modelscope_value) {
return `https://modelscope.cn/api/v1/models/${modelData?.name}/repo?Revision=${modelData?.Revision}&View=true&FilePath=${imgSrc}`;
}
if (modelSource === modelSourceMap.huggingface_value) {
return `https://huggingface.co/${modelData?.id}/resolve/main/${imgSrc}`;
}
return '';
};
},
[modelSource, modelData?.name, modelData?.id, modelData?.Revision]
);
useEffect(() => {
getModelCardData();
src/pages/llmodels/components/update-modal.tsx
+11 -9
View File
@@ -81,12 +81,13 @@ const UpdateModal: React.FC<AddModalProps> = (props) => {
}
};
const handleOnValuesChange = _.debounce((data: any) => {
const handleOnValuesChange = (data: any) => {
const formdata = form.getFieldsValue?.();
let alldata = {};
if (formdata.scheduleType === 'manual') {
alldata = {
..._.omit(formdata, ['worker_selector']),
env: formdata.env || {},
gpu_selector:
formdata.gpu_selector?.gpu_ids?.length > 0
? originFormData.current?.gpu_selector
@@ -95,6 +96,7 @@ const UpdateModal: React.FC<AddModalProps> = (props) => {
} else {
alldata = {
..._.omit(formdata, ['gpu_selector']),
env: formdata.env || {},
worker_selector: originFormData.current?.worker_selector || null
};
}
@@ -113,7 +115,7 @@ const UpdateModal: React.FC<AddModalProps> = (props) => {
})
});
}
}, 300);
};
// voxbox is not support multi gpu
const handleSetGPUIds = (backend: string) => {
@@ -168,14 +170,14 @@ const UpdateModal: React.FC<AddModalProps> = (props) => {
}
const isEndwithGGUF = _.endsWith(value, '.gguf');
const isBlobFile = value.split('/').pop().includes('sha256');
let backend = backendOptionsMap.llamaBox;
const isVllmOrAscend = [
backendOptionsMap.vllm,
backendOptionsMap.ascendMindie
].includes(form.getFieldValue('backend'));
let backend = form.getFieldValue('backend');
if (!isEndwithGGUF || !isBlobFile) {
backend = isVllmOrAscend ? formData!.backend : backendOptionsMap.vllm;
if (
!isEndwithGGUF &&
!isBlobFile &&
backend === backendOptionsMap.llamaBox
) {
backend = backendOptionsMap.vllm;
}
form.setFieldValue('backend', backend);
handleBackendChange?.(backend);
src/pages/llmodels/config/vllm-config.ts
+121 -96
View File
@@ -4,6 +4,11 @@ const options = [
value: '--uvicorn-log-level',
options: ['debug', 'info', 'warning', 'error', 'critical', 'trace']
},
{
label: '--disable-uvicorn-access-log',
value: '--disable-uvicorn-access-log',
options: []
},
{
label: '--allow-credentials',
value: '--allow-credentials',
@@ -59,6 +64,11 @@ const options = [
value: '--ssl-certfile',
options: []
},
{
label: '--enable-ssl-refresh',
value: '--enable-ssl-refresh',
options: []
},
{
label: '--ssl-ca-certs',
value: '--ssl-ca-certs',
@@ -89,6 +99,11 @@ const options = [
value: '--disable-frontend-multiprocessing',
options: []
},
{
label: '--enable-request-id-headers',
value: '--enable-request-id-headers',
options: []
},
{
label: '--enable-auto-tool-choice',
value: '--enable-auto-tool-choice',
@@ -104,6 +119,7 @@ const options = [
'jamba',
'llama3_json',
'granite-20b-fc',
'phi4_mini_json',
'granite',
'pythonic'
]
@@ -118,7 +134,8 @@ const options = [
'embed',
'classify',
'score',
'reward'
'reward',
'transcription'
]
},
{
@@ -141,6 +158,11 @@ const options = [
value: '--tokenizer',
options: []
},
{
label: '--hf-config-path',
value: '--hf-config-path',
options: []
},
{
label: '--skip-tokenizer-init',
value: '--skip-tokenizer-init',
@@ -164,7 +186,7 @@ const options = [
{
label: '--tokenizer-mode',
value: '--tokenizer-mode',
options: ['auto', 'slow', 'mistral']
options: ['auto', 'slow', 'mistral', 'custom']
},
{
label: '--trust-remote-code',
@@ -190,7 +212,8 @@ const options = [
'gguf',
'bitsandbytes',
'mistral',
'runai_streamer'
'runai_streamer',
'fastsafetensors'
]
},
{
@@ -213,6 +236,11 @@ const options = [
value: '--config-format',
options: ['auto', 'hf', 'mistral']
},
{
label: '--disable-cascade-attn',
value: '--disable-cascade-attn',
options: []
},
{
label: '--dtype',
value: '--dtype',
@@ -223,6 +251,16 @@ const options = [
value: '--kv-cache-dtype',
options: ['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3']
},
{
label: '--disable-chunked-mm-input',
value: '--disable-chunked-mm-input',
options: []
},
{
label: 'DISABLE_CHUNKED_MM_INPUT',
value: 'DISABLE_CHUNKED_MM_INPUT',
options: []
},
{
label: '--quantization-param-path',
value: '--quantization-param-path',
@@ -259,6 +297,16 @@ const options = [
value: '--worker-use-ray',
options: []
},
{
label: '--data-parallel-size',
value: '--data-parallel-size',
options: []
},
{
label: '--enable-expert-parallel',
value: '--enable-expert-parallel',
options: []
},
{
label: '--pipeline-parallel-size',
value: '--pipeline-parallel-size',
@@ -289,6 +337,11 @@ const options = [
value: '--enable-prefix-caching',
options: []
},
{
label: '--prefix-caching-hash-algo',
value: '--prefix-caching-hash-algo',
options: ['builtin', 'sha256']
},
{
label: '--disable-sliding-window',
value: '--disable-sliding-window',
@@ -334,6 +387,23 @@ const options = [
value: '--max-num-batched-tokens',
options: []
},
{
label: '--max-num-partial-prefills',
value: '--max-num-partial-prefills',
options: []
},
{
label: '--max-long-partial-prefills',
value: '--max-long-partial-prefills',
options: []
},
{
label: '--long-prefill-token-threshold',
value: '--long-prefill-token-threshold',
options: []
},
{
label: '--max-num-seqs',
value: '--max-num-seqs',
@@ -354,6 +424,17 @@ const options = [
value: '--hf-overrides',
options: []
},
{
label: '--hf-token',
value: '--hf-token',
options: []
},
{
label: 'HF_TOKEN',
value: 'HF_TOKEN',
options: []
},
{
label: '--disable-mm-preprocessor-cache',
value: '--disable-mm-preprocessor-cache',
@@ -368,23 +449,26 @@ const options = [
'deepspeedfp',
'tpu_int8',
'fp8',
'ptpc_fp8',
'fbgemm_fp8',
'modelopt',
'nvfp4',
'marlin',
'gguf',
'hqq',
'gptq_marlin_24',
'gptq_marlin',
'awq_marlin',
'gptq',
'quark',
'moe_wna16',
'compressed-tensors',
'bitsandbytes',
'qqq',
'hqq',
'experts_int8',
'neuron_quant',
'ipex',
'quark',
'moe_wna16',
'torchao',
'None'
]
},
@@ -506,7 +590,12 @@ const options = [
{
label: '--device',
value: '--device',
options: ['auto', 'cuda', 'neuron', 'cpu', 'openvino', 'tpu', 'xpu', 'hpu']
options: ['auto', 'cuda', 'neuron', 'cpu', 'tpu', 'xpu', 'hpu']
},
{
label: '--speculative-config',
value: '--speculative-config',
options: []
},
{
label: '--num-scheduler-steps',
@@ -528,105 +617,31 @@ const options = [
value: '--enable-chunked-prefill',
options: []
},
{
label: '--speculative-model',
value: '--speculative-model',
options: []
},
{
label: '--speculative-model-quantization',
value: '--speculative-model-quantization',
options: [
'aqlm',
'awq',
'deepspeedfp',
'tpu_int8',
'fp8',
'fbgemm_fp8',
'modelopt',
'marlin',
'gguf',
'gptq_marlin_24',
'gptq_marlin',
'awq_marlin',
'gptq',
'compressed-tensors',
'bitsandbytes',
'qqq',
'hqq',
'experts_int8',
'neuron_quant',
'ipex',
'quark',
'moe_wna16',
'None'
]
},
{
label: '--num-speculative-tokens',
value: '--num-speculative-tokens',
options: []
},
{
label: '--speculative-disable-mqa-scorer',
value: '--speculative-disable-mqa-scorer',
options: []
},
{
label: '--speculative-draft-tensor-parallel-size',
value: '--speculative-draft-tensor-parallel-size',
options: []
},
{
label: '--speculative-max-model-len',
value: '--speculative-max-model-len',
options: []
},
{
label: '--speculative-disable-by-batch-size',
value: '--speculative-disable-by-batch-size',
options: []
},
{
label: '--ngram-prompt-lookup-max',
value: '--ngram-prompt-lookup-max',
options: []
},
{
label: '--ngram-prompt-lookup-min',
value: '--ngram-prompt-lookup-min',
options: []
},
{
label: '--spec-decoding-acceptance-method',
value: '--spec-decoding-acceptance-method',
options: ['rejection_sampler', 'typical_acceptance_sampler']
},
{
label: '--typical-acceptance-sampler-posterior-threshold',
value: '--typical-acceptance-sampler-posterior-threshold',
options: []
},
{
label: '--typical-acceptance-sampler-posterior-alpha',
value: '--typical-acceptance-sampler-posterior-alpha',
options: []
},
{
label: '--disable-logprobs-during-spec-decoding',
value: '--disable-logprobs-during-spec-decoding',
options: []
},
{
label: '--model-loader-extra-config',
value: '--model-loader-extra-config',
options: []
},
{
label: '--scheduler-cls',
value: '--scheduler-cls',
options: []
},
{
label: '--use-tqdm-on-load',
value: '--use-tqdm-on-load',
options: []
},
{
label: '--ignore-patterns',
value: '--ignore-patterns',
options: []
},
{
label: '--show-hidden-metrics-for-version',
value: '--show-hidden-metrics-for-version',
options: []
},
{
label: '--preemption-mode',
value: '--preemption-mode',
@@ -713,6 +728,11 @@ const options = [
value: '--disable-log-requests',
options: []
},
{
label: '--additional-config',
value: '--additional-config',
options: []
},
{
label: '--max-log-len',
value: '--max-log-len',
@@ -727,6 +747,11 @@ const options = [
label: '--enable-prompt-tokens-details',
value: '--enable-prompt-tokens-details',
options: []
},
{
label: '--enable-server-load-tracking',
value: '--enable-server-load-tracking',
options: []
}
];
src/pages/llmodels/forms/local-path.tsx
+8 -3
View File
@@ -19,7 +19,7 @@ const LocalPathForm: React.FC = () => {
const formInnerCtx = useFormInnerContext();
const source = Form.useWatch('source', form);
const { onBackendChange } = formInnerCtx;
const { modelFileOptions, byBuiltIn } = formCtx;
const { byBuiltIn } = formCtx;
const { getRuleMessage } = useAppUtils();
const intl = useIntl();
const localPathCache = useRef<string>(form.getFieldValue('local_path') || '');
@@ -35,8 +35,13 @@ const LocalPathForm: React.FC = () => {
}
const isEndwithGGUF = _.endsWith(value, '.gguf');
const isBlobFile = value.split('/').pop().includes('sha256');
let backend = backendOptionsMap.llamaBox;
if (!isEndwithGGUF && !isBlobFile) {
let backend = form.getFieldValue('backend');
if (
!isEndwithGGUF &&
!isBlobFile &&
backend === backendOptionsMap.llamaBox
) {
backend = backendOptionsMap.vllm;
}
form.setFieldValue('backend', backend);
src/pages/resources/components/model-files.tsx
+106 -64
View File
@@ -145,6 +145,35 @@ const getWorkerName = (
return worker?.label || '';
};
const getModelInfo = (record: ListItem) => {
if (record.source === modelSourceMap.huggingface_value) {
return {
title: `${record.huggingface_repo_id}/${record.huggingface_filename}`,
filename: record.huggingface_filename
};
}
if (record.source === modelSourceMap.modelscope_value) {
return {
title: `${record.model_scope_model_id}/${record.model_scope_file_path}`,
filename: record.model_scope_file_path
};
}
if (record.source === modelSourceMap.ollama_library_value) {
return {
title: record.ollama_library_model_name,
filename: record.ollama_library_model_name
};
}
return {
title: record.local_path,
filename: _.split(record.local_path, /[\\/]/).pop()
};
};
const getResolvedPath = (pathList: string[]) => {
return _.split(pathList?.[0], /[\\/]/).pop();
};
const InstanceStatusTag = (props: { data: ListItem }) => {
const { data } = props;
if (!data.state) {
@@ -174,6 +203,80 @@ const InstanceStatusTag = (props: { data: ListItem }) => {
);
};
const RenderParts = (props: { record: ListItem }) => {
const { record } = props;
const intl = useIntl();
const parts = record.resolved_paths || [];
if (parts.length <= 1) {
return null;
}
const renderItem = () => {
return (
<ItemWrapper>
{parts.map((item: string, index: number) => {
return <li key={index}>{_.split(item, /[\\/]/).pop()}</li>;
})}
</ItemWrapper>
);
};
return (
<TooltipOverlayScroller title={renderItem()}>
<FilesTag color="purple" icon={<InfoCircleOutlined />}>
<span style={{ opacity: 1 }}>
{record.resolved_paths?.length}{' '}
{intl.formatMessage({ id: 'models.form.files' })}
</span>
</FilesTag>
</TooltipOverlayScroller>
);
};
const ResolvedPathColumn = (props: { record: ListItem }) => {
const { record } = props;
const intl = useIntl();
if (
!record.resolved_paths.length &&
record.state === ModelfileStateMap.Downloading
) {
const modelInfo = getModelInfo(record);
const { title, filename } = modelInfo;
return (
<AutoTooltip ghost showTitle title={title}>
<span
dangerouslySetInnerHTML={{
__html: intl.formatMessage(
{
id: 'resources.modelfiles.storagePath.holder'
},
{
name: filename
}
)
}}
></span>
</AutoTooltip>
);
}
return (
record.resolved_paths?.length > 0 && (
<PathWrapper>
<AutoTooltip
ghost
showTitle
title={
<TooltipTitle path={record.resolved_paths?.[0]}></TooltipTitle>
}
>
<span>{getResolvedPath(record.resolved_paths)}</span>
</AutoTooltip>
<RenderParts record={record}></RenderParts>
</PathWrapper>
)
);
};
const ModelFiles = () => {
const { getGPUList } = useGenerateFormEditInitialValues();
const { saveScrollHeight, restoreScrollHeight } = useBodyScroll();
@@ -308,34 +411,6 @@ const ModelFiles = () => {
};
};
const renderParts = (record: ListItem) => {
const parts = record.resolved_paths || [];
if (parts.length <= 1) {
return null;
}
const renderItem = () => {
return (
<ItemWrapper>
{parts.map((item: string, index: number) => {
return <li key={index}>{_.split(item, /[\\/]/).pop()}</li>;
})}
</ItemWrapper>
);
};
return (
<TooltipOverlayScroller title={renderItem()}>
<FilesTag color="purple" icon={<InfoCircleOutlined />}>
<span style={{ opacity: 1 }}>
{record.resolved_paths?.length}{' '}
{intl.formatMessage({ id: 'models.form.files' })}
</span>
</FilesTag>
</TooltipOverlayScroller>
);
};
const handleSelect = async (val: any, record: ListItem) => {
try {
if (val === 'delete') {
@@ -464,10 +539,6 @@ const ModelFiles = () => {
}
};
const getResolvedPath = (pathList: string[]) => {
return _.split(pathList?.[0], /[\\/]/).pop();
};
const columns = [
{
title: intl.formatMessage({ id: 'models.form.source' }),
@@ -509,38 +580,9 @@ const ModelFiles = () => {
title: intl.formatMessage({ id: 'resources.modelfiles.form.path' }),
dataIndex: 'resolved_paths',
width: '35%',
render: (text: string, record: ListItem) => {
if (
!record.resolved_paths.length &&
record.state === ModelfileStateMap.Downloading
) {
return (
<span>
{intl.formatMessage({
id: 'resources.modelfiles.storagePath.holder'
})}
</span>
);
}
return (
record.resolved_paths?.length > 0 && (
<PathWrapper>
<AutoTooltip
ghost
showTitle
title={
<TooltipTitle
path={record.resolved_paths?.[0]}
></TooltipTitle>
}
>
<span>{getResolvedPath(record.resolved_paths)}</span>
</AutoTooltip>
{renderParts(record)}
</PathWrapper>
)
);
}
render: (text: string, record: ListItem) => (
<ResolvedPathColumn record={record} />
)
},
{
title: intl.formatMessage({ id: 'resources.modelfiles.size' }),