diff --git a/config/routes.ts b/config/routes.ts index 36f4e98b..eb3b15b1 100644 --- a/config/routes.ts +++ b/config/routes.ts @@ -73,6 +73,16 @@ export default [ selectedIcon: 'icon-audio-filled', defaultIcon: 'icon-audio1', component: './playground/speech' + }, + { + name: 'video', + title: 'Video', + path: '/playground/video', + key: 'video', + icon: 'icon-video', + selectedIcon: 'icon-video-filled', + defaultIcon: 'icon-video', + component: './playground/video' } ] }, diff --git a/src/components/icon-font/index.tsx b/src/components/icon-font/index.tsx index 98db3a77..90249bc8 100644 --- a/src/components/icon-font/index.tsx +++ b/src/components/icon-font/index.tsx @@ -2,7 +2,7 @@ import { createFromIconfontCN } from '@ant-design/icons'; // import './iconfont/iconfont.js'; const IconFont = createFromIconfontCN({ - scriptUrl: '//at.alicdn.com/t/c/font_4613488_rajpu0blzkp.js' + scriptUrl: '//at.alicdn.com/t/c/font_4613488_h1excejg8p8.js' }); export default IconFont; diff --git a/src/layouts/index.tsx b/src/layouts/index.tsx index 7a82caea..aa12adb8 100644 --- a/src/layouts/index.tsx +++ b/src/layouts/index.tsx @@ -45,7 +45,8 @@ const NO_CONTAINER_PAGES = [ 'image', 'text2images', 'clusterDetail', - 'clusterCreate' + 'clusterCreate', + 'video' ]; const CHECK_RESOURCE_PATH = [ diff --git a/src/locales/en-US/menu.ts b/src/locales/en-US/menu.ts index f1a2af52..3d6ec220 100644 --- a/src/locales/en-US/menu.ts +++ b/src/locales/en-US/menu.ts @@ -6,6 +6,7 @@ export default { 'menu.playground.chat': 'Chat', 'menu.playground.speech': 'Audio', 'menu.playground.text2images': 'Image', + 'menu.playground.video': 'Video', 'menu.compare': 'Compare', 'menu.models': 'Models', 'menu.models.modelList': 'Deploy & Manage', diff --git a/src/locales/en-US/playground.ts b/src/locales/en-US/playground.ts index 7e062134..e99fe002 100644 --- a/src/locales/en-US/playground.ts +++ b/src/locales/en-US/playground.ts @@ -62,7 +62,8 @@ export default { 'playground.params.width': 'Width', 'playground.params.height': 'Height', 'playground.params.custom': 'Custom', - 'playground.params.empty.tips': 'Generated images will appear here', + 'playground.params.empty.tips': 'Generated images will appear here.', + 'playground.video.empty.tips': 'Generated video will appear here.', 'playground.params.standard': 'Standard', 'playground.params.hd': 'HD', 'playground.params.style.vivid': 'Vivid', diff --git a/src/locales/ja-JP/menu.ts b/src/locales/ja-JP/menu.ts index 3aa0e3f7..ecb7a995 100644 --- a/src/locales/ja-JP/menu.ts +++ b/src/locales/ja-JP/menu.ts @@ -6,6 +6,7 @@ export default { 'menu.playground.chat': 'チャット', 'menu.playground.speech': '音声', 'menu.playground.text2images': '画像生成', + 'menu.playground.video': '動画', 'menu.compare': '比較', 'menu.models': 'モデル', 'menu.models.modelList': 'デプロイと管理', diff --git a/src/locales/ja-JP/playground.ts b/src/locales/ja-JP/playground.ts index 13783e9d..4e0e8d7c 100644 --- a/src/locales/ja-JP/playground.ts +++ b/src/locales/ja-JP/playground.ts @@ -62,7 +62,8 @@ export default { 'playground.params.width': '幅', 'playground.params.height': '高さ', 'playground.params.custom': 'カスタム', - 'playground.params.empty.tips': '生成された画像はここに表示されます', + 'playground.params.empty.tips': '生成された画像はここに表示されます。', + 'playground.video.empty.tips': '生成された動画はここに表示されます。', 'playground.params.standard': '標準', 'playground.params.hd': 'HD', 'playground.params.style.vivid': '鮮やか', diff --git a/src/locales/ru-RU/menu.ts b/src/locales/ru-RU/menu.ts index 9a6ae346..5301b4d5 100644 --- a/src/locales/ru-RU/menu.ts +++ b/src/locales/ru-RU/menu.ts @@ -6,6 +6,7 @@ export default { 'menu.playground.chat': 'Чат', 'menu.playground.speech': 'Аудио', 'menu.playground.text2images': 'Генерация изображений', + 'menu.playground.video': 'Видео', 'menu.compare': 'Сравнение', 'menu.models': 'Модели', 'menu.models.modelList': 'Развертывание и управление', diff --git a/src/locales/ru-RU/playground.ts b/src/locales/ru-RU/playground.ts index fb6cb8c3..c7dfecdd 100644 --- a/src/locales/ru-RU/playground.ts +++ b/src/locales/ru-RU/playground.ts @@ -63,6 +63,7 @@ export default { 'playground.params.height': 'Высота', 'playground.params.custom': 'Дополнительно', 'playground.params.empty.tips': 'Здесь появятся сгенерированные изображения', + 'playground.video.empty.tips': 'Здесь появятся сгенерированные видео.', 'playground.params.standard': 'Стандарт', 'playground.params.hd': 'HD', 'playground.params.style.vivid': 'Яркий', diff --git a/src/locales/zh-CN/menu.ts b/src/locales/zh-CN/menu.ts index cb1bf0aa..ff9518c1 100644 --- a/src/locales/zh-CN/menu.ts +++ b/src/locales/zh-CN/menu.ts @@ -6,6 +6,7 @@ export default { 'menu.playground.chat': '对话', 'menu.playground.speech': '语音', 'menu.playground.text2images': '图像', + 'menu.playground.video': '视频', 'menu.compare': '多模型对比', 'menu.models': '模型', 'menu.models.modelList': '部署与管理', diff --git a/src/locales/zh-CN/playground.ts b/src/locales/zh-CN/playground.ts index b4a44481..d8f32e5a 100644 --- a/src/locales/zh-CN/playground.ts +++ b/src/locales/zh-CN/playground.ts @@ -67,6 +67,7 @@ export default { 'playground.params.style.vivid': '生动', 'playground.params.style.natural': '自然', 'playground.params.empty.tips': '生成的图片将出现在这里', + 'playground.video.empty.tips': '生成的视频将出现在这里', 'playground.embedding.documents': '文档', 'playground.embedding.addtext': '添加文本', 'playground.embedding.inputyourtext': '输入你的文本', diff --git a/src/pages/llmodels/config/backend-parameters/vllm.ts b/src/pages/llmodels/config/backend-parameters/vllm.ts index fb8df9e3..bd2e7279 100644 --- a/src/pages/llmodels/config/backend-parameters/vllm.ts +++ b/src/pages/llmodels/config/backend-parameters/vllm.ts @@ -499,11 +499,6 @@ const options: BackendParameter[] = [ value: '--enable-lora-bias', options: [] }, - { - label: '--rope-scaling', - value: '--rope-scaling', - options: [] - }, { label: '--rope-theta', value: '--rope-theta', diff --git a/src/pages/playground/apis/index.ts b/src/pages/playground/apis/index.ts index b47916ce..1f713cbc 100644 --- a/src/pages/playground/apis/index.ts +++ b/src/pages/playground/apis/index.ts @@ -5,6 +5,7 @@ import { request } from '@umijs/max'; export { GPUSTACK_API_BASE_URL, OPENAI_COMPATIBLE }; export const CHAT_API = `/${OPENAI_COMPATIBLE}/chat/completions`; +export const CREATE_VIDEO_API = `/${OPENAI_COMPATIBLE}/videos`; export const CREAT_IMAGE_API = `/${OPENAI_COMPATIBLE}/images/generations`; export const EDIT_IMAGE_API = `/${OPENAI_COMPATIBLE}/images/edits`; @@ -128,6 +129,25 @@ export const createImage = async (params: { return response.json(); }; +export const createVideo = async (params: { + data?: any; + signal?: AbortSignal; +}) => { + const response = await fetch(CREATE_VIDEO_API, { + method: 'POST', + body: JSON.stringify(params.data), + signal: params.signal, + headers: { + 'Content-Type': 'application/json' + } + }); + + if (!response.ok) { + return await errorHandler(response); + } + return response.json(); +}; + // ============ audio ============ export const textToSpeech = async (params: any, options?: any) => { const res = await fetch(AUDIO_TEXT_TO_SPEECH_API, { diff --git a/src/pages/playground/components/ground-video.tsx b/src/pages/playground/components/ground-video.tsx new file mode 100644 index 00000000..bf301f4b --- /dev/null +++ b/src/pages/playground/components/ground-video.tsx @@ -0,0 +1,275 @@ +import { setRouteCache } from '@/atoms/route-cache'; +import AlertInfo from '@/components/alert-info'; +import IconFont from '@/components/icon-font'; +import routeCachekey from '@/config/route-cachekey'; +import ThumbImg from '@/pages/playground/components/thumb-img'; +import { VideoCameraOutlined } from '@ant-design/icons'; +import { useIntl } from '@umijs/max'; +import { Button, Tooltip } from 'antd'; +import classNames from 'classnames'; +import _ from 'lodash'; +import 'overlayscrollbars/overlayscrollbars.css'; +import React, { + forwardRef, + useCallback, + useImperativeHandle, + useMemo, + useRef, + useState +} from 'react'; +import { CREATE_VIDEO_API } from '../apis'; +import { useInitVideoMeta } from '../hooks/use-init-video-meta'; +import useTextVideo from '../hooks/use-text-video'; +import '../style/ground-llm.less'; +import '../style/system-message-wrap.less'; +import { generateCode } from '../view-code/video'; +import DynamicParams from './dynamic-params'; +import MessageInput from './message-input'; +import ViewCommonCode from './view-common-code'; + +interface MessageProps { + modelList: Global.BaseOption[]; + loaded?: boolean; + ref?: any; +} + +const GroundVideo: React.FC = forwardRef((props, ref) => { + const { modelList } = props; + + const intl = useIntl(); + const [show, setShow] = useState(false); + const [collapse, setCollapse] = useState(false); + const scroller = useRef(null); + const paramsRef = useRef(null); + const inputRef = useRef(null); + + const { + handleOnValuesChange, + handleToggleParamsStyle, + setParams, + form, + formFields, + paramsConfig, + initialValues, + parameters, + isOpenaiCompatible + } = useInitVideoMeta(props, { + type: 'create' + }); + const { + loading, + tokenResult, + videoList, + promptList, + currentPrompt, + setCurrentPrompt, + handleClear, + handleStopConversation, + submitMessage + } = useTextVideo({ + scroller, + paramsRef, + API: CREATE_VIDEO_API + }); + + useImperativeHandle(ref, () => { + return { + viewCode() { + setShow(true); + }, + setCollapse() { + setCollapse(!collapse); + }, + collapse: collapse + }; + }); + + const generateNumber = (min: number, max: number) => { + return Math.floor(Math.random() * (max - min + 1) + min); + }; + + const handleRandomPrompt = useCallback(() => { + const randomIndex = generateNumber(0, promptList.length - 1); + const randomPrompt = promptList[randomIndex]; + inputRef.current?.handleInputChange({ + target: { + value: randomPrompt + } + }); + }, []); + + const finalParameters = useMemo(() => { + if (parameters.size === 'custom') { + return { + ..._.omit(parameters, ['width', 'height', 'random_seed', 'seed']), + size: + parameters.width && parameters.height + ? `${parameters.width}x${parameters.height}` + : '' + }; + } + return { + ..._.omit(parameters, ['width', 'height', 'random_seed', 'seed']) + }; + }, [parameters]); + + const viewCodeContent = useMemo(() => { + return generateCode({ + api: CREATE_VIDEO_API, + parameters: { + ...finalParameters, + prompt: currentPrompt + } + }); + }, [finalParameters, isOpenaiCompatible, currentPrompt]); + + const handleInputChange = (e: any) => { + setCurrentPrompt(e.target.value); + }; + + const generateParams = () => { + const params = { + ..._.omitBy(finalParameters, (value: string) => !value), + prompt: currentPrompt + }; + return params; + }; + + const handleSendMessage = async () => { + try { + await form.current?.form?.validateFields(); + if (!parameters.model) return; + const params = generateParams(); + console.log('generateParams:', params); + setParams({ + ...parameters + }); + + console.log('params:', params, parameters); + setRouteCache(routeCachekey['/playground/video'], true); + await submitMessage(params); + } catch (error) { + // console.log('error:', error); + } finally { + console.log('finally---------'); + setRouteCache(routeCachekey['/playground/video'], false); + } + }; + + const handleCloseViewCode = useCallback(() => { + setShow(false); + }, []); + + return ( +
+
+
+ <> +
+ + {!videoList.length && ( +
+ + + {intl.formatMessage({ + id: 'playground.video.empty.tips' + })} + +
+ )} +
+ +
+ {tokenResult && ( +
+ +
+ )} +
+ + + + + + } + /> +
+
+
+
+ +
+
+ +
+ ); +}); + +export default GroundVideo; diff --git a/src/pages/playground/config/video-parameters.ts b/src/pages/playground/config/video-parameters.ts new file mode 100644 index 00000000..dabe2a8e --- /dev/null +++ b/src/pages/playground/config/video-parameters.ts @@ -0,0 +1,54 @@ +import { ImageSizeItem } from './params-config'; +import { ParamsSchema } from './types'; + +export const videoSizeOptions: ImageSizeItem[] = [ + { label: '720x1280', value: '720x1280', width: 720, height: 1280 }, + { label: '1280x720', value: '1280x720', width: 1280, height: 720 }, + { label: '1024x1792', value: '1024x1792', width: 1024, height: 1792 }, + { + label: '1792x1024', + value: '1792x1024', + width: 1792, + height: 1024 + } +]; + +export const videoParamsConfig: ParamsSchema[] = [ + { + type: 'Select', + name: 'seconds', + options: [ + { label: '4', value: '4' }, + { label: '8', value: '8' }, + { label: '12', value: '12' } + ], + label: { + text: 'Duration (seconds)', + isLocalized: false + }, + rules: [ + { + required: false + } + ] + }, + { + type: 'Select', + name: 'size', + options: videoSizeOptions, + description: { + text: 'playground.params.size.description', + html: true, + isLocalized: true + }, + label: { + text: 'playground.params.size', + isLocalized: true + }, + rules: [ + { + required: false + } + ] + } +]; diff --git a/src/pages/playground/hooks/config.ts b/src/pages/playground/hooks/config.ts index 5cfa596b..8829adb6 100644 --- a/src/pages/playground/hooks/config.ts +++ b/src/pages/playground/hooks/config.ts @@ -40,6 +40,8 @@ export const IMG_METAKEYS = [ 'strength' ]; +export const VIDEO_METAKEYS = []; + export const llmInitialValues = { seed: null, stop: null, @@ -62,6 +64,8 @@ export const advancedFieldsDefaultValus = { // preview: 'preview_faster' }; +export const videoAdvancedDefaultValues = {}; + export const openaiCompatibleFieldsDefaultValus = { // quality: 'standard', style: null diff --git a/src/pages/playground/hooks/use-init-meta.ts b/src/pages/playground/hooks/use-init-meta.ts index f96f0a73..d4e714af 100644 --- a/src/pages/playground/hooks/use-init-meta.ts +++ b/src/pages/playground/hooks/use-init-meta.ts @@ -279,10 +279,13 @@ export const useInitImageMeta = ( ) => { if (sizeOptions.length) { const sizeConfig = ImageSizeConfig.map((item) => { - return { - ...item, - options: sizeOptions - }; + if (item.name === 'size') { + return { + ...item, + options: sizeOptions + }; + } + return item; }); return [...ImageCountConfig, ...sizeConfig]; } diff --git a/src/pages/playground/hooks/use-init-video-meta.ts b/src/pages/playground/hooks/use-init-video-meta.ts new file mode 100644 index 00000000..47b0f383 --- /dev/null +++ b/src/pages/playground/hooks/use-init-video-meta.ts @@ -0,0 +1,348 @@ +import { + ImageCustomSizeConfig, + ImageSizeItem +} from '@/pages/playground/config/params-config'; +import { generateRandomNumber } from '@/utils'; +import { useSearchParams } from '@umijs/max'; +import { useMemoizedFn } from 'ahooks'; +import _ from 'lodash'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; +import { ParamsSchema } from '../config/types'; +import { + videoParamsConfig, + videoSizeOptions as videoSizeList +} from '../config/video-parameters'; +import { + openaiCompatibleFieldsDefaultValus, + VIDEO_METAKEYS, + videoAdvancedDefaultValues +} from './config'; + +const videoInitialValues = { + size: '720x1280', + seconds: '4' +}; + +interface MessageProps { + modelList: Global.BaseOption[]; + model?: string; + loaded?: boolean; + isChat?: boolean; + ref?: any; +} + +export const useInitVideoMeta = ( + props: MessageProps, + options: { type: string } +) => { + const { modelList } = props; + const videoAdvancedParamsConfig: ParamsSchema[] = []; + const videoExtraConfig: ParamsSchema[] = []; + + const form = useRef(null); + const [searchParams] = useSearchParams(); + const [modelMeta, setModelMeta] = useState({}); + const [isOpenaiCompatible, setIsOpenaiCompatible] = useState(false); + const [videoSizeOptions, setVideoSizeOptions] = React.useState< + ImageSizeItem[] + >([]); + const [basicParamsConfig, setBasicParamsConfig] = React.useState< + ParamsSchema[] + >([...videoParamsConfig]); + const initialSeed = generateRandomNumber(); + + const [initialValues, setInitialValues] = useState({ + ...videoInitialValues, + ...videoAdvancedDefaultValues, + seed: initialSeed, + model: '' + }); + const [paramsConfig, setParamsConfig] = useState([ + ...videoParamsConfig, + ...videoAdvancedParamsConfig + ]); + + const [parameters, setParams] = useState({ + ...videoInitialValues, + ...videoAdvancedDefaultValues, + seed: initialSeed, + model: '' + }); + + const defaultModel = useMemo(() => { + return searchParams.get('model') || modelList?.[0]?.value || ''; + }, [modelList]); + + const cacheFormData = React.useRef>({ + ...videoInitialValues, + ...openaiCompatibleFieldsDefaultValus, + ...videoAdvancedDefaultValues + }); + + const getNewSizeOptions = (metaData: any) => { + const { max_height, max_width } = metaData || {}; + if (!max_height || !max_width) { + return videoSizeList; + } + const newSizeOptions = videoSizeList.filter((item) => { + return ( + (item.width <= max_width && item.height <= max_height) || + item.value === 'custom' + ); + }); + if ( + !newSizeOptions.find( + (item) => item.width === max_width && item.height === max_height + ) + ) { + newSizeOptions.push({ + width: max_width, + height: max_height, + label: `${max_width}x${max_height}`, + value: `${max_width}x${max_height}` + }); + } + return newSizeOptions; + }; + + const generateParamsConfig = ( + currentModel: any, + sizeOptions: ImageSizeItem[] + ) => { + if (sizeOptions.length) { + const sizeConfig = videoParamsConfig.map((item) => { + return { + ...item, + options: sizeOptions + }; + }); + return [...sizeConfig]; + } + const { max_height, max_width } = currentModel.meta || {}; + // generate custom size config + const customSizeConfig = _.cloneDeep(ImageCustomSizeConfig).map( + (item: any) => { + const max = item.name === 'height' ? max_height : max_width; + return { + ...item, + attrs: { + ...item.attrs, + max: max || item.attrs.max + } + }; + } + ); + return [...customSizeConfig]; + }; + + const extractIMGMeta = (meta: any) => { + const sizeOptions = getNewSizeOptions(meta); + return { + form: _.merge( + {}, + { ...videoInitialValues, ...videoAdvancedDefaultValues }, + { + ..._.pick(meta, VIDEO_METAKEYS), + size: sizeOptions.length + ? `${meta?.default_width || 720}x${meta?.default_height || 1280}` + : 'custom', + width: meta?.default_width || 720, + height: meta?.default_height || 1280 + } + ), + meta: meta, + sizeOptions: sizeOptions + }; + }; + + const updateParamsConfig = (values: { + size: string; + isOpenaiCompatible: boolean; + }) => { + // update config + if (values.size === 'custom') { + return [ + ...basicParamsConfig, + ...ImageCustomSizeConfig, + ...(values.isOpenaiCompatible + ? videoExtraConfig + : videoAdvancedParamsConfig) + ]; + } + return [ + ...basicParamsConfig, + ...(values.isOpenaiCompatible + ? videoExtraConfig + : videoAdvancedParamsConfig) + ]; + }; + + const updateCacheFormData = (values: Record) => { + _.merge(cacheFormData.current, values); + }; + const handleToggleParamsStyle = () => { + // update values + let values: any = {}; + if (!isOpenaiCompatible) { + values = _.pick( + cacheFormData.current, + _.keys({ + ...videoInitialValues, + ...openaiCompatibleFieldsDefaultValus + }) + ); + } else { + values = _.pick( + cacheFormData.current, + _.keys({ + ...videoInitialValues, + ...videoAdvancedDefaultValues + }) + ); + } + + // update config + const newParamsConfig = updateParamsConfig({ + size: values.size, + isOpenaiCompatible: !isOpenaiCompatible + }); + form.current?.form?.setFieldsValue({ + ...values, + model: parameters.model + }); + setParamsConfig(newParamsConfig); + setParams({ + ...values, + model: parameters.model + }); + setIsOpenaiCompatible(!isOpenaiCompatible); + updateCacheFormData({ + ...values, + model: parameters + }); + }; + + const formFields = useMemo(() => { + const fields = paramsConfig?.map((item) => item.name); + return fields?.join(','); + }, [paramsConfig]); + + const handleOnModelChange = useMemoizedFn((val: string) => { + if (!val) return; + const model = modelList.find((item) => item.value === val); + const { form: initialData, sizeOptions } = extractIMGMeta(model?.meta); + const newParamsConfig = generateParamsConfig(model, sizeOptions); + + if (!isOpenaiCompatible) { + setParamsConfig([...newParamsConfig, ...videoAdvancedParamsConfig]); + } else { + setParamsConfig(newParamsConfig); + } + setBasicParamsConfig(newParamsConfig); + setVideoSizeOptions(sizeOptions); + setModelMeta(model?.meta || {}); + setInitialValues({ + ...initialData, + seed: parameters.seed, + model: val + }); + setParams({ + ...initialData, + seed: parameters.seed, + model: val + }); + updateCacheFormData(initialData); + }); + + const handleOnValuesChange = useMemoizedFn( + (changeValues: Record, allValues: Record) => { + console.log('changeValues', changeValues); + // model change will reset all values + if (changeValues.model) { + handleOnModelChange(changeValues.model); + return; + } + + if (changeValues.size && changeValues.size === 'custom') { + setParamsConfig([ + ...basicParamsConfig, + ...ImageCustomSizeConfig, + ...(!isOpenaiCompatible + ? videoAdvancedParamsConfig + : videoExtraConfig) + ]); + setParams({ + ...allValues, + width: modelMeta.default_width || 720, + height: modelMeta.default_height || 1280 + }); + form.current?.form?.setFieldsValue({ + width: modelMeta.default_width || 720, + height: modelMeta.default_height || 1280 + }); + updateCacheFormData(changeValues); + } else if (changeValues.size && parameters.size === 'custom') { + setParamsConfig([ + ...basicParamsConfig, + ...(!isOpenaiCompatible + ? videoAdvancedParamsConfig + : videoExtraConfig) + ]); + setParams(allValues); + updateCacheFormData(changeValues); + } else if (_.isBoolean(changeValues.random_seed)) { + const seed = changeValues.random_seed + ? generateRandomNumber() + : parameters.seed; + + setParams({ + ...allValues, + seed: seed + }); + form.current?.form?.setFieldsValue({ + seed: seed + }); + updateCacheFormData(changeValues); + } else { + setParams(allValues); + updateCacheFormData(changeValues); + } + } + ); + + useEffect(() => { + if (defaultModel && modelList.length) { + handleOnModelChange(defaultModel); + } + }, [defaultModel, modelList.length]); + + return { + extractIMGMeta, + generateParamsConfig, + setVideoSizeOptions, + setBasicParamsConfig, + updateCacheFormData, + handleOnValuesChange, + handleToggleParamsStyle, + setParams, + setInitialValues, + updateParamsConfig, + setParamsConfig, + form, + modelMeta, + formFields, + paramsConfig, + initialValues, + parameters, + isOpenaiCompatible, + cacheFormData, + basicParamsConfig, + videoSizeOptions, + openaiCompatibleFieldsDefaultValus, + videoAdvancedDefaultValues, + videoAdvancedParamsConfig, + videoInitialValues, + ImageCustomSizeConfig, + videoExtraConfig + }; +}; diff --git a/src/pages/playground/hooks/use-text-video.ts b/src/pages/playground/hooks/use-text-video.ts new file mode 100644 index 00000000..6f45f65a --- /dev/null +++ b/src/pages/playground/hooks/use-text-video.ts @@ -0,0 +1,210 @@ +import useOverlayScroller from '@/hooks/use-overlay-scroller'; +import { extractErrorMessage, promptList } from '@/pages/playground/config'; +import { useIntl } from '@umijs/max'; +import _ from 'lodash'; +import { useEffect, useRef, useState } from 'react'; +import { createVideo } from '../apis'; + +export default function useTextVideo(props: any) { + const intl = useIntl(); + const { scroller, paramsRef } = props; + const [loading, setLoading] = useState(false); + const [tokenResult, setTokenResult] = useState(null); + const [videoList, setVideoList] = useState< + { + dataUrl: string; + height: number | string; + width: string | number; + maxHeight: string | number; + maxWidth: string | number; + uid: number; + span?: number; + loading?: boolean; + progress?: number; + preview?: boolean; + }[] + >([]); + const [currentPrompt, setCurrentPrompt] = useState(''); + const messageId = useRef(0); + const requestToken = useRef(null); + const { initialize } = useOverlayScroller(); + const { initialize: innitializeParams } = useOverlayScroller(); + const requestIdRef = useRef(0); + + useEffect(() => { + if (scroller.current) { + initialize(scroller.current); + } + }, [initialize]); + useEffect(() => { + if (paramsRef.current) { + innitializeParams(paramsRef.current); + } + }, [innitializeParams]); + + const updateRequestId = () => { + requestIdRef.current = requestIdRef.current + 1; + return requestIdRef.current; + }; + + const removeBase64Suffix = (str: string, suffix: string) => { + return str.endsWith(suffix) ? str.slice(0, -suffix.length) : str; + }; + + const setImageSize = (parameters: any) => { + let size: Record = { + span: 12 + }; + if (parameters.n === 1) { + size.span = 24; + } + if (parameters.n === 2) { + size.span = 12; + } + if (parameters.n === 3) { + size.span = 12; + } + if (parameters.n === 4) { + size.span = 12; + } + return size; + }; + + const setMessageId = () => { + messageId.current = messageId.current + 1; + return messageId.current; + }; + + const generateNumber = (min: number, max: number) => { + return Math.floor(Math.random() * (max - min + 1) + min); + }; + + const stopDebounce = _.debounce(() => { + setVideoList([]); + setLoading(false); + }, 200); + + const submitMessage = async (parameters: any) => { + try { + if (!parameters.model) return; + + requestToken.current?.abort?.('cancel'); + requestToken.current = new AbortController(); + const currentRequestId = updateRequestId(); + const size: any = setImageSize(parameters); + setLoading(true); + setMessageId(); + setTokenResult(null); + + const imgSize = _.split(parameters.size, 'x').map((item: string) => + _.toNumber(item) + ); + + let newImageList = Array(parameters.n) + .fill({}) + .map((item, index: number) => { + return { + dataUrl: 'data:image/png;base64,', + ...size, + progress: 0, + height: imgSize[1], + width: imgSize[0], + loading: true, + progressType: 'dashboard', + preview: false, + uid: setMessageId() + }; + }); + setVideoList(newImageList); + + const result = await createVideo({ + data: parameters, + signal: requestToken.current.signal + }); + + console.log('result:', result); + + if (result.error) { + setTokenResult({ + error: true, + errorMessage: extractErrorMessage(result) + }); + setVideoList([]); + return; + } + + // If the request ID has changed, ignore this chunk + if (requestIdRef.current !== currentRequestId) { + return; + } + if (result?.error) { + setTokenResult({ + error: true, + errorMessage: extractErrorMessage(result) + }); + return; + } + + result?.data?.forEach((item: any, index: number) => { + const imgItem = newImageList[index]; + + if (item.b64_json) { + imgItem.dataUrl = `data:image/png;base64,${item.b64_json}`; + } + + newImageList[index] = { + dataUrl: imgItem.dataUrl, + height: imgSize[1], + width: imgSize[0], + maxHeight: `${imgSize[1]}px`, + maxWidth: `${imgSize[0]}px`, + uid: imgItem.uid, + span: imgItem.span, + loading: false, + preview: true, + progress: 100 + }; + }); + console.log('newImageList:', newImageList); + setVideoList([...newImageList]); + } catch (error) { + console.log('error:', error); + updateRequestId(); + stopDebounce(); + requestToken.current?.abort?.('cancel'); + } finally { + setLoading(false); + } + }; + + const handleClear = () => { + setVideoList([]); + setTokenResult(null); + setCurrentPrompt(''); + }; + + const handleStopConversation = () => { + requestToken.current?.abort?.('stop'); + }; + + useEffect(() => { + return () => { + requestToken.current?.abort?.('cancel'); + }; + }, []); + + return { + loading, + tokenResult, + videoList, + promptList, + currentPrompt, + setVideoList, + setTokenResult, + setCurrentPrompt, + handleStopConversation, + generateNumber, + handleClear, + submitMessage + }; +} diff --git a/src/pages/playground/video.tsx b/src/pages/playground/video.tsx new file mode 100644 index 00000000..b3b907b5 --- /dev/null +++ b/src/pages/playground/video.tsx @@ -0,0 +1,116 @@ +import HotKeys from '@/config/hotkeys'; +import { ExtraContent } from '@/layouts/extraRender'; +import { modelCategoriesMap } from '@/pages/llmodels/config'; +import { useIntl } from '@umijs/max'; +import useMemoizedFn from 'ahooks/lib/useMemoizedFn'; +import { Divider } from 'antd'; +import classNames from 'classnames'; +import _ from 'lodash'; +import { useEffect, useRef, useState } from 'react'; +import { useHotkeys } from 'react-hotkeys-hook'; +import { PageContainerInner } from '../_components/page-box'; +import { queryModelsList } from './apis'; +import GroundVideo from './components/ground-video'; +import ViewCodeButtons from './components/view-code-buttons'; +import useCollapseLayout from './hooks/use-collapse-layout'; +import './style/play-ground.less'; + +const PlaygroundRerank: React.FC = () => { + const intl = useIntl(); + const groundRerankerRef = useRef(null); + const [rerankerModelList, setRerankerModelList] = useState< + Global.BaseOption[] + >([]); + const [loaded, setLoaded] = useState(false); + + useCollapseLayout({ + handler: () => { + groundRerankerRef.current?.setCollapse?.(); + }, + triggeredRef: groundRerankerRef.current + }); + + const handleViewCode = useMemoizedFn(() => { + groundRerankerRef.current?.viewCode?.(); + }); + + const handleToggleCollapse = useMemoizedFn(() => { + groundRerankerRef.current?.setCollapse?.(); + }); + + useEffect(() => { + const getModelListByReranker = async () => { + try { + const params = { + categories: modelCategoriesMap.reranker, + with_meta: true + }; + const res = await queryModelsList(params); + const list = _.map(res.data || [], (item: any) => { + return { + value: item.id, + label: item.id, + meta: item.meta + }; + }) as Global.BaseOption[]; + return list; + } catch (error) { + console.error(error); + return []; + } + }; + const fetchData = async () => { + try { + const [rerankerModelList] = await Promise.all([ + getModelListByReranker() + ]); + setRerankerModelList(rerankerModelList); + } catch (error) { + setLoaded(true); + } + }; + fetchData(); + }, []); + + useHotkeys( + HotKeys.RIGHT.join(','), + () => { + groundRerankerRef.current?.setCollapse?.(); + }, + { + preventDefault: true + } + ); + + return ( + , + , + + ]} + > +
+
+ +
+
+ + ); +}; + +export default PlaygroundRerank; diff --git a/src/pages/playground/view-code/video.ts b/src/pages/playground/view-code/video.ts new file mode 100644 index 00000000..4aa6ff83 --- /dev/null +++ b/src/pages/playground/view-code/video.ts @@ -0,0 +1,86 @@ +import _ from 'lodash'; +import { MODEL_PROXY } from '../apis'; +import { fomatNodeJsParams, formatCurlArgs } from './utils'; + +export const generateCurlCode = ({ + api: url, + parameters, + modelProxy, + isFormdata = false, + edit = false +}: Record) => { + const host = window.location.origin; + const api = modelProxy ? `${MODEL_PROXY}/\${YOUR_API_PATH}` : url; + + // ========================= Curl ========================= + let curlCode = ` +curl ${host}${api} \\ +-H "Content-Type: application/json" \\ +-H "Authorization: Bearer $\{YOUR_GPUSTACK_API_KEY}" \\${modelProxy ? `\n-H "X-GPUStack-Model: ${parameters.model}" \\` : ''} +${formatCurlArgs(parameters, isFormdata)}`.trim(); + + if (edit) { + curlCode = ` +curl ${host}${api} \\ +-H "Content-Type: multipart/form-data" \\ +-H "Authorization: Bearer $\{YOUR_GPUSTACK_API_KEY}" \\${modelProxy ? `\n-H "X-GPUStack-Model: ${parameters.model}" \\` : ''} +-F image="@image.png" \\ +-F mask="@mask.png" \\ +${formatCurlArgs(_.omit(parameters, ['mask', 'image']), isFormdata)}` + .trim() + .replace(/\\$/, ''); + } + + return curlCode; +}; + +export const generateCode = ({ + api: url, + parameters, + isFormdata = false, + edit = false +}: Record) => { + const host = window.location.origin; + const api = url; + + // ========================= Curl ========================= + let curlCode = generateCurlCode({ + api: url, + parameters, + isFormdata, + edit + }); + + // ========================= Python ========================= + const pythonCode = ` +import requests\n +url="${host}${api}" +headers = { + "Content-type": "application/json", + "Authorization": "Bearer $\{YOUR_GPUSTACK_API_KEY}" +} +data = ${JSON.stringify(parameters, null, 2).replace(/null/g, 'None')}\n +response = requests.post(url, headers=headers, json=data) +print(response.json()['data']['object'])`.trim(); + + // ========================= Node.js ========================= + const nodeJsCode = ` +const axios = require('axios'); + +const url = "${host}${api}"; +const headers = { + "Content-type": "application/json", + "Authorization": "Bearer $\{YOUR_GPUSTACK_API_KEY}" +}; +const data = ${fomatNodeJsParams(parameters)}; + +axios.post(url, data, { headers }).then((response) => { + console.log(response.data.object); +});`.trim(); + + return { + curlCode, + pythonCode, + nodeJsCode + }; +};