diff --git a/src/components/logs-viewer/virtual-log-list.tsx b/src/components/logs-viewer/virtual-log-list.tsx index 42f39746..8a7768ef 100644 --- a/src/components/logs-viewer/virtual-log-list.tsx +++ b/src/components/logs-viewer/virtual-log-list.tsx @@ -46,6 +46,11 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { const pageRef = useRef(page); const totalPageRef = useRef(totalPage); const isLoadingMoreRef = useRef(false); + const scrollPosRef = useRef({ + pos: 'bottom', + page: 1 + }); + const dataLenRef = useRef(0); useImperativeHandle(ref, () => ({ abort() { @@ -93,45 +98,54 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { const getLastPage = (data: string) => { const list = _.split(data.trim(), '\n'); let result = ''; - if (isLoadingMoreRef.current) { - setLoading(true); - } - if (!enableScorllLoad) { - result = list.join('\n'); - } else if (list.length <= pageSize) { - setTotalPage(1); - result = data; - } else { - const totalPage = Math.ceil(list.length / pageSize); - setTotalPage(totalPage); - setPage(() => totalPage); - pageRef.current = totalPage; - totalPageRef.current = totalPage; - const lastPage = list.slice(-pageSize).join('\n'); - result = lastPage; - } - debounceLoading(); + const totalPage = Math.ceil(list.length / pageSize); + console.log('loading== getLastPage ========', isLoadingMoreRef.current); + + pageRef.current = totalPage; + totalPageRef.current = totalPage; + const lastPageLogs = list.slice(-pageSize).join('\n'); + + result = lastPageLogs; + + setPage(totalPage); + setTotalPage(totalPage); + setScrollPos(['bottom', totalPage]); + scrollPosRef.current = { + pos: 'bottom', + page: totalPage + }; + return result; }; const getCurrentPage = () => { const list = _.split(cacheDataRef.current.trim(), '\n'); const totalPage = Math.ceil(list.length / pageSize); - + console.log('loading== getCurrentPage ========', isLoadingMoreRef.current); let newPage = pageRef.current; if (newPage < 1) { newPage = 1; } - + if (isLoadingMoreRef.current) { + setLoading(true); + } const start = (newPage - 1) * pageSize; const end = newPage * pageSize; const currentPage = list.slice(start, end).join('\n'); setPage(newPage); setTotalPage(totalPage); - if (pageRef.current === totalPageRef.current && scrollPos[0] === 'bottom') { + if ( + pageRef.current === totalPageRef.current && + scrollPosRef.current.pos === 'bottom' + ) { setScrollPos(['bottom', newPage]); + scrollPosRef.current = { + pos: 'bottom', + page: newPage + }; } + debounceLoading(); pageRef.current = newPage; logParseWorker.current.postMessage({ inputStr: currentPage @@ -150,6 +164,10 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { setPage(() => newPage); setScrollPos(['bottom', newPage]); + scrollPosRef.current = { + pos: 'bottom', + page: newPage + }; pageRef.current = newPage; logParseWorker.current.postMessage({ inputStr: prePage @@ -168,6 +186,10 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { setPage(() => newPage); setScrollPos(['top', newPage]); + scrollPosRef.current = { + pos: 'top', + page: newPage + }; pageRef.current = newPage; logParseWorker.current.postMessage({ inputStr: nextPage @@ -182,30 +204,28 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { const nextPage = list.slice(start, end).join('\n'); setPage(() => newPage); setScrollPos(['bottom', newPage]); + scrollPosRef.current = { + pos: 'bottom', + page: newPage + }; pageRef.current = newPage; logParseWorker.current.postMessage({ inputStr: nextPage }); }, [totalPage, page, pageSize]); - const debounceParseData = _.debounce(() => { - if (pageRef.current === totalPageRef.current) { - logParseWorker.current.postMessage({ - inputStr: getLastPage(cacheDataRef.current) - }); - } else { - getCurrentPage(); - } - }, 100); - const updateContent = (inputStr: string) => { const data = inputStr.replace(replaceLineRegex, '\n'); + dataLenRef.current = data.length; if (isClean(data)) { cacheDataRef.current = data; } else { cacheDataRef.current += data; } - if (pageRef.current === totalPageRef.current) { + if ( + pageRef.current === totalPageRef.current && + scrollPosRef.current.pos === 'bottom' + ) { logParseWorker.current.postMessage({ inputStr: getLastPage(cacheDataRef.current) }); @@ -234,21 +254,34 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { async (data: { isTop: boolean; isBottom: boolean }) => { const { isTop, isBottom } = data; setIsAtTop(isTop); - console.log('scroll========', { isTop, isBottom }); - // if (isBottom) { - // setScrollPos((pre) => { - // return ['bottom', pre[1]]; - // }); - // } else if (isTop) { - // setScrollPos((pre) => { - // return ['top', pre[1]]; - // }); - // } else { - // setScrollPos([]); - // } + console.log('scroll========', { + isTop, + isBottom, + loadMoreDone: loadMoreDone.current, + loading: loading, + loglen: dataLenRef.current + }); + if (isBottom) { + scrollPosRef.current = { + pos: 'bottom', + page: page + }; + } else if (isTop) { + scrollPosRef.current = { + pos: 'top', + page: page + }; + } else { + scrollPosRef.current = { + pos: 'middle', + page: page + }; + } if ( loading || - (logs.length > 0 && logs.length < pageSize && !loadMoreDone.current) || + (logs.length > 0 && + dataLenRef.current < pageSize && + !loadMoreDone.current) || !enableScorllLoad ) { return; @@ -272,16 +305,18 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { enableScorllLoad, page, totalPage, + setScrollPos, createChunkConnection ] ); const debouncedScroll = useCallback( _.debounce(() => { - if (scrollPos[0] === 'top') { + console.log('scrollPos+++++++++++=', scrollPos); + if (scrollPos[0] === 'top' && scrollPosRef.current.pos === 'top') { logListRef.current?.scrollToTop(); } - if (scrollPos[0] === 'bottom') { + if (scrollPos[0] === 'bottom' && scrollPosRef.current.pos === 'bottom') { logListRef.current?.scrollToBottom(); } }, 150), @@ -301,7 +336,6 @@ const LogsViewer: React.FC = forwardRef((props, ref) => { return (
-
{ decoder: TextDecoder, callback: (data: any) => void ) => { - const { done, value } = await reader.read(); - if (done) { - return; + const throttledCallback = throttle((data: any) => { + callback(data); + }, 200); + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + const chunk = decoder.decode(value, { stream: true }); + throttledCallback(chunk); } - - const chunk = decoder.decode(value, { stream: true }); - console.log('chunk===', chunk); - callback(chunk); - // console.log('chunkDataRef.current===2', chunkDataRef.current); - - await readTextEventStreamData(reader, decoder, callback); }; const readTextEventStreamDataByLine = async ( diff --git a/src/pages/playground/components/ground-left.tsx b/src/pages/playground/components/ground-left.tsx index b7b94663..e5d0ab60 100644 --- a/src/pages/playground/components/ground-left.tsx +++ b/src/pages/playground/components/ground-left.tsx @@ -228,6 +228,8 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { setMessageList(userMsg); }; + const throttleUpdatePosition = _.throttle(updateScrollerPosition, 100); + useEffect(() => { if (scroller.current) { initialize(scroller.current); @@ -242,6 +244,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { useEffect(() => { if (loading) { + console.log('loading:', loading); updateScrollerPosition(); } }, [messageList, loading]); diff --git a/src/pages/playground/components/params-settings.tsx b/src/pages/playground/components/params-settings.tsx index 9923e23e..22f1443e 100644 --- a/src/pages/playground/components/params-settings.tsx +++ b/src/pages/playground/components/params-settings.tsx @@ -34,7 +34,8 @@ const METAKEYS: Record = { stop: 'stop', temperature: 'temperature', top_p: 'top_p', - n_slot_ctx: 'max_tokens', + n_ctx: 'n_ctx', + n_slot: 'n_slot', max_model_len: 'max_tokens' }; @@ -108,9 +109,25 @@ const ParamsSettings: React.FC = ({ } return acc; }, {}); - form.setFieldsValue(obj); - setMetaData(obj); - return obj; + + let defaultMaxTokens = 1024; + + if (obj.n_ctx && obj.n_slot) { + defaultMaxTokens = _.divide(obj.n_ctx / 2, obj.n_slot); + } + + form.setFieldsValue({ + ..._.omit(obj, ['n_ctx', 'n_slot']), + max_tokens: defaultMaxTokens + }); + setMetaData({ + ...obj, + max_tokens: defaultMaxTokens + }); + return { + ..._.omit(obj, ['n_ctx', 'n_slot']), + max_tokens: defaultMaxTokens + }; }; useEffect(() => { @@ -120,18 +137,13 @@ const ParamsSettings: React.FC = ({ } const modelMetaData = handleModelChange(model); const mergeData = _.merge({}, initialValues, modelMetaData); - const defaultMaxTokens = modelMetaData?.max_tokens - ? _.divide(modelMetaData?.max_tokens, 2) - : 1024; form.setFieldsValue({ ...mergeData, - max_tokens: defaultMaxTokens, model: model }); setParams({ ...mergeData, - max_tokens: defaultMaxTokens, model: model }); setFirstLoad(false); @@ -259,7 +271,7 @@ const ParamsSettings: React.FC = ({ variant="borderless" > = (props) => { return commandCode?.registerWorker({ server: origin, tag: version, - token: '${mytoken}' + token: '${mytoken}', + workerip: '${myworkerip}' }); } return commandCode?.registerWorker({ server: origin, tag: `${version}-${activeKey}`, - token: '${mytoken}' + token: '${mytoken}', + workerip: '${myworkerip}' }); }, [versionInfo, activeKey, props.token, origin]); diff --git a/src/pages/resources/config/index.ts b/src/pages/resources/config/index.ts index 735340ad..92579d50 100644 --- a/src/pages/resources/config/index.ts +++ b/src/pages/resources/config/index.ts @@ -32,34 +32,59 @@ export const addWorkerGuide: Record = { cuda: { getToken: 'Get-Content -Path (Join-Path -Path $env:APPDATA -ChildPath "gpustack\\token") -Raw', - registerWorker(params: { server: string; tag: string; token: string }) { - return `docker run -d --gpus all --ipc=host --network=host gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token}`; + registerWorker(params: { + server: string; + tag: string; + token: string; + workerip: string; + }) { + return `docker run -d --name gpustack-worker --restart=unless-stopped --gpus all -p 10150:10150 -p 40000-41024:40000-41024 -p 50000-51024:50000-51024 --ipc=host -v gpustack-worker-data:/var/lib/gpustack gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token} --worker-ip ${params.workerip}`; } }, npu: { getToken: 'Get-Content -Path (Join-Path -Path $env:APPDATA -ChildPath "gpustack\\token") -Raw', - registerWorker(params: { server: string; tag: string; token: string }) { - return `docker run -d --ipc=host -e ASCEND_VISIBLE_DEVICES=0 --network=host gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token}`; + registerWorker(params: { + server: string; + tag: string; + token: string; + workerip: string; + }) { + return `docker run -d --name gpustack-worker --restart=unless-stopped -e ASCEND_VISIBLE_DEVICES=0 -p 10150:10150 -p 40000-41024:40000-41024 -p 50000-51024:50000-51024 --ipc=host -v gpustack-worker-data:/var/lib/gpustack gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token} --worker-ip ${params.workerip}`; } }, musa: { getToken: 'Get-Content -Path (Join-Path -Path $env:APPDATA -ChildPath "gpustack\\token") -Raw', - registerWorker(params: { server: string; tag: string; token: string }) { - return `docker run -d --ipc=host --network=host gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token}`; + registerWorker(params: { + server: string; + tag: string; + token: string; + workerip: string; + }) { + return `docker run -d --name gpustack-worker --restart=unless-stopped -p 10150:10150 -p 40000-41024:40000-41024 -p 50000-51024:50000-51024 --ipc=host -v gpustack-worker-data:/var/lib/gpustack gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token} --worker-ip ${params.workerip}`; } }, cpu: { getToken: 'Get-Content -Path (Join-Path -Path $env:APPDATA -ChildPath "gpustack\\token") -Raw', - registerWorker(params: { server: string; tag: string; token: string }) { - return `docker run -d --ipc=host --network=host gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token}`; + registerWorker(params: { + server: string; + tag: string; + token: string; + workerip: string; + }) { + return `docker run -d --name gpustack-worker --restart=unless-stopped -p 10150:10150 -p 40000-41024:40000-41024 -p 50000-51024:50000-51024 --ipc=host -v gpustack-worker-data:/var/lib/gpustack gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token} --worker-ip ${params.workerip}`; } }, rocm: { - registerWorker(params: { server: string; tag: string; token: string }) { - return `docker run -d --network=host --ipc=host --group-add=video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device /dev/kfd --device /dev/dri gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token}`; + registerWorker(params: { + server: string; + tag: string; + token: string; + workerip: string; + }) { + return `docker run -d --name gpustack-worker --restart=unless-stopped -p 10150:10150 -p 40000-41024:40000-41024 -p 50000-51024:50000-51024 --ipc=host --group-add=video --security-opt seccomp=unconfined --device /dev/kfd --device /dev/dri -v gpustack-worker-data:/var/lib/gpustack gpustack/gpustack:${params.tag} --server-url ${params.server} --token ${params.token} --worker-ip ${params.workerip}`; } }, container: { diff --git a/src/utils/fetch-chunk-data.ts b/src/utils/fetch-chunk-data.ts index db6ff41c..c4d3f4c8 100644 --- a/src/utils/fetch-chunk-data.ts +++ b/src/utils/fetch-chunk-data.ts @@ -1,4 +1,6 @@ +import { throttle } from 'lodash'; import qs from 'query-string'; + const extractStreamRegx = /data:\s*({.*?})(?=\n|$)/g; const extractJSON = (dataStr: string) => { @@ -119,28 +121,66 @@ export const fetchChunkedDataPostFormData = async (params: { }; export const readStreamData = async ( - reader: any, + reader: ReadableStreamDefaultReader, decoder: TextDecoder, - callback: (data: any) => void + callback: (data: any[]) => void, + throttleDelay = 200 ) => { - const { done, value } = await reader.read(); - if (done) { - return; + class BufferManager { + private buffer: any[] = []; + + public add(data: any) { + this.buffer.push(data); + } + + public flush() { + if (this.buffer.length > 0) { + const currentBuffer = [...this.buffer]; + this.buffer = []; + currentBuffer.forEach((item) => callback(item)); + } + } + + public getBuffer() { + return this.buffer; + } } - let chunk = decoder.decode(value, { stream: true }); + const bufferManager = new BufferManager(); - if (chunk.startsWith('error:')) { - const errorStr = chunk.slice(7).trim(); - const jsonData = JSON.parse(errorStr); - callback({ error: jsonData }); - } else { - extractJSON(chunk).forEach((data) => { - callback?.(data); - }); + const throttledCallback = throttle(() => { + bufferManager.flush(); + }, throttleDelay); + + let isReading = true; + + while (isReading) { + const { done, value } = await reader.read(); + + if (done) { + isReading = false; + bufferManager.flush(); + break; + } + + try { + const chunk = decoder.decode(value, { stream: true }); + + if (chunk.startsWith('error:')) { + const errorStr = chunk.slice(7).trim(); + const jsonData = JSON.parse(errorStr); + bufferManager.add({ error: jsonData }); + } else { + extractJSON(chunk).forEach((data) => { + bufferManager.add(data); + }); + } + + throttledCallback(); + } catch (error) { + bufferManager.add({ error }); + } } - - await readStreamData(reader, decoder, callback); }; // Process the remainder of the buffer