fix(llmodels): stale instances lingering after missed DELETE watch events

src/hooks/json-parser-worker.ts

@@ -8,24 +8,38 @@ const findValidJSONStrings = (inputStr: string) => {
     const openingBraceIndex = inputStr.indexOf('{', startIndex);
     if (openingBraceIndex === -1) break; // No more opening braces
 
-    let closingBraceIndex = openingBraceIndex;
+    // find the matching closing brace, ignoring braces inside string
+    // literals (e.g. a state_message containing `{`/`}`)
+    let closingBraceIndex = -1;
     let braceCount = 0;
+    let inString = false;
+    let escaped = false;
 
-    // find  couple of braces
-    while (closingBraceIndex < inputStr.length) {
-      if (inputStr[closingBraceIndex] === '{') {
+    for (let i = openingBraceIndex; i < inputStr.length; i++) {
+      const char = inputStr[i];
+      if (inString) {
+        if (escaped) {
+          escaped = false;
+        } else if (char === '\\') {
+          escaped = true;
+        } else if (char === '"') {
+          inString = false;
+        }
+      } else if (char === '"') {
+        inString = true;
+      } else if (char === '{') {
         braceCount++;
-      } else if (inputStr[closingBraceIndex] === '}') {
+      } else if (char === '}') {
         braceCount--;
+        if (braceCount === 0) {
+          closingBraceIndex = i;
+          break;
+        }
       }
-      if (braceCount === 0) {
-        break;
-      }
-      closingBraceIndex++;
     }
 
-    if (braceCount !== 0) {
-      //  no matching closing brace
+    if (closingBraceIndex === -1) {
+      //  no matching closing brace yet, wait for more data
       break;
     }
 
@@ -37,11 +51,11 @@ const findValidJSONStrings = (inputStr: string) => {
     try {
       const parsedData = JSON.parse(jsonString);
       validJSONStrings.push(parsedData);
-      startIndex = closingBraceIndex + 1;
     } catch (error) {
-      // mabye invalid JSON
-      break;
+      // skip the malformed segment instead of breaking, otherwise it jams
+      // the buffer and every later event on this stream is lost
     }
+    startIndex = closingBraceIndex + 1;
   }
 
   return {

src/hooks/use-chunk-request.ts

@@ -43,8 +43,11 @@ export const createAxiosToken = (): CancelTokenSource => {
 };
 
 export const sliceData = (data: string, loaded: number, loadedSize: any) => {
+  // `loaded` is a byte count while `data` is a UTF-16 string; with any
+  // non-ASCII payload the two drift apart, so track consumed characters by
+  // string length only
   const result = data.slice(loadedSize.current);
-  loadedSize.current = loaded;
+  loadedSize.current = data.length;
   return result;
 };
 

src/pages/llmodels/deployments.tsx

@@ -248,7 +248,13 @@ const Models = forwardRef((props, ref) => {
       chunkInstanceRequedtRef.current = setModelInstanceChunkRequest({
         url: `${MODEL_INSTANCE_API}`,
         params: {},
-        handler: updateInstanceHandler
+        handler: updateInstanceHandler,
+        beforeReconnect() {
+          // treat the reconnect snapshot as the new baseline, otherwise
+          // instances deleted while the stream was down linger in the cache
+          // (their DELETE events are never re-sent)
+          cacheInsDataListRef.current = [];
+        }
       });
     } catch (error) {
       // ignore
@@ -452,6 +458,20 @@ const Models = forwardRef((props, ref) => {
     };
   }, []);
 
+  // watch events can still be lost (stream hiccup, reconnect gap); a low
+  // frequency relist keeps the instance cache eventually consistent, so a
+  // missed DELETE event can't leave a stale instance behind for good
+  useEffect(() => {
+    const timer = setInterval(() => {
+      if (!isPageHidden.current) {
+        getAllModelInstances();
+      }
+    }, 60 * 1000);
+    return () => {
+      clearInterval(timer);
+    };
+  }, []);
+
   const setDisableExpand = useMemoizedFn((record: any) => {
     return !record?.replicas;
   });