root/goodbuddy
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

perf(activity): page, summarize and reconcile activity history in Main by index

Browse Source 
Schema 57 rebuilds activity_history_records with order_seq (newer = larger;
a move to the front assigns a new maximum) and stored generated columns
(record_id, occurrence, status, status_group, kind, request_id,
conversation_id, project_id, created_at) with indexes. The order is migrated
from record_order_json exactly; legacy duplicates keep their occurrence and
only the first is shown. record_order_json is no longer written.

ActivityHistoryRepository owns the SQL: page({before, limit <= 500, filter,
projectId, conversationId}) newest first by index, summary() with the counts
and conversation titles/status the Activity page derived from the full list,
reconcile() joined with tasks with a renderer-localized detail line, update()
applied directly in SQL without reading the history, replace() for the
legacy migration, clear(). New activity-history:page/summary/reconcile IPC
with zod validation; page and summary run on the readonly worker.
perf:main measures 24,000 records: startup load 106 ms on Main (14 MB) ->
first page + summary 13.5 ms on the worker (0.6 ms loop block, 114 KB).
This commit is contained in:
mesalogo
2026-10-03 21:42:52 +08:00
parent 6aab78b9fa
commit 352da66dbd
14 changed files with 963 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/assistant/assistant-database.ts
+56 -145
View File
@@ -6,14 +6,7 @@ import { appendConversationQuestionBlock } from '../../shared/conversation-quest
import { DatabaseSync } from 'node:sqlite'
import { ExecutionStatsReader } from './execution-stats-reader'
import { ReadonlyQueryReader, readWithFallback } from '../readonly-query-reader'
import {
isActivityHistoryPlanEmpty,
parseActivityHistorySnapshot,
planActivityHistoryReplace,
planActivityHistoryUpdate,
type ActivityHistoryPlan,
type ActivityHistoryState
} from './activity-history-plan'
import { ActivityHistoryRepository, migrateActivityHistoryOrder } from './activity-history-repository'
import { statSync } from 'node:fs'
import { MagicNoteStorage } from '../magic-notes/magic-note-storage'
import type { AssistantStorageProgress } from '../../shared/assistant-storage-contracts'
@@ -45,7 +38,9 @@ import type {
AssistantTask,
ExecutionStats,
ExecutionStatsInput,
ActivityHistoryPage,
ActivityHistorySnapshot,
ActivityHistorySummary,
ConversationQueueItem,
ConversationAnsweredQuestion,
ConversationBranchInput,
@@ -142,7 +137,7 @@ import {
SUBAGENT_PROGRESS_STORAGE_SCHEMA_VERSION
} from './subagent-progress-storage'
export const ASSISTANT_DATABASE_SCHEMA_VERSION = 56
export const ASSISTANT_DATABASE_SCHEMA_VERSION = 57
export type RemoteTaskEventInput = {
taskId: string
@@ -1974,7 +1969,7 @@ export class AssistantDatabase {
private static readonly executionStatsCacheTtlMs = 30_000
private static readonly executionStatsCacheLimit = 8
private database?: DatabaseSync
private activityHistoryCache?: { state: ActivityHistoryState; dataVersion: number }
private activityHistoryRepository?: { database: DatabaseSync; repository: ActivityHistoryRepository }
private executionStatsReader?: ExecutionStatsReader
private readonlyReader?: ReadonlyQueryReader
private readonlyWorkerPath?: string
@@ -2412,7 +2407,7 @@ export class AssistantDatabase {
this.readonlyReader = undefined
this.foldedSearchConnection = undefined
this.executionStatsCache.clear()
this.activityHistoryCache = undefined
this.activityHistoryRepository = undefined
this.database?.close()
this.database = undefined
this.subagentProgress = undefined
@@ -2472,7 +2467,7 @@ export class AssistantDatabase {
throw error
}
this.dirtyMagicNotes.clear()
this.activityHistoryCache = undefined
this.activityHistoryRepository?.repository.invalidate()
this.pendingMagicNoteCleanup.clear()
this.reconcileMagicNoteFiles()
this.options.onMagicNotesChanged?.()
@@ -5295,154 +5290,59 @@ export class AssistantDatabase {
return result
}
getActivityHistory(): ActivityHistorySnapshot {
/** Activity history storage and queries (PERF-15); the SQL lives in the repository. */
activityHistory(): ActivityHistoryRepository {
const database = this.requireDatabase()
const row = database
.prepare(
`SELECT record_order_json, legacy_history_may_be_incomplete
FROM activity_history
WHERE singleton = 1`
)
.get() as
| {
record_order_json: string
legacy_history_may_be_incomplete: number
}
| undefined
const records = database.prepare(
`SELECT records.record_json
FROM json_each(?) AS ordering
JOIN activity_history_records AS records ON records.record_key = ordering.value
ORDER BY CAST(ordering.key AS INTEGER)`
).all(row?.record_order_json ?? '[]') as Array<{ record_json: string }>
return activityHistorySnapshotSchema.parse({
records: records.map((record) => JSON.parse(record.record_json)),
legacyHistoryMayBeIncomplete:
row?.legacy_history_may_be_incomplete === 1
})
if (this.activityHistoryRepository?.database !== database) {
this.activityHistoryRepository = { database, repository: new ActivityHistoryRepository(database) }
}
return this.activityHistoryRepository.repository
}
/**
* Persists the whole newest-first list. Unchanged records are neither
* re-validated, re-serialized nor written; a fully unchanged snapshot does
* not open a write transaction (PERF-15).
*/
/** The whole stored list (tests, diagnostics); the renderer reads pages. */
getActivityHistory(): ActivityHistorySnapshot {
return this.activityHistory().snapshot()
}
/** Persists a whole newest-first list; used for the legacy localStorage migration. */
replaceActivityHistory(input: unknown): void {
const database = this.requireDatabase()
// Reuse is safe even from a stale cache: cached records are validated copies.
const { items, incomplete } = parseActivityHistorySnapshot(input, this.activityHistoryCache?.state)
const committed = this.committedActivityHistoryState(database)
const planned = committed && planActivityHistoryReplace(items, incomplete, committed)
if (planned && isActivityHistoryPlanEmpty(planned)) return
this.writeActivityHistoryPlan(database, (previous) => previous === committed && planned
? planned : planActivityHistoryReplace(items, incomplete, previous))
this.activityHistory().replace(input)
}
/** Applies incremental changes in one transaction (PERF-15). */
updateActivityHistory(input: unknown): void {
const database = this.requireDatabase()
this.writeActivityHistoryPlan(database, (previous) => planActivityHistoryUpdate(input, previous))
this.activityHistory().update(input)
}
/** Deletes every activity record in one transaction. */
clearActivityHistory(): void {
this.activityHistory().clear()
}
getActivityHistoryPage(input: unknown): ActivityHistoryPage {
return this.activityHistory().page(input)
}
getActivityHistorySummary(input: unknown): ActivityHistorySummary {
return this.readSnapshot(() => this.activityHistory().summary(input))
}
/** Pages and the summary on the readonly worker; same rules as listConversationSummariesAsync. */
getActivityHistoryPageAsync(input: unknown): Promise<ActivityHistoryPage> {
const database = this.requireDatabase()
database.exec('BEGIN IMMEDIATE')
try {
database.exec('DELETE FROM activity_history_records')
database.exec(
`UPDATE activity_history
SET record_order_json = '[]', legacy_history_may_be_incomplete = 0
WHERE singleton = 1`
)
database.exec('COMMIT')
} catch (error) {
database.exec('ROLLBACK')
throw error
}
this.activityHistoryCache = undefined
return readWithFallback(database.isTransaction ? undefined : this.readonlyQueryReader(),
'activityHistoryPage', [input], undefined, () => this.getActivityHistoryPage(input))
}
/** The cached committed state, when no other connection committed since. */
private committedActivityHistoryState(database: DatabaseSync): ActivityHistoryState | undefined {
const cache = this.activityHistoryCache
if (!cache || database.isTransaction) return undefined
const { data_version: dataVersion } = database.prepare('PRAGMA data_version').get() as { data_version: number }
return cache.dataVersion === dataVersion ? cache.state : undefined
getActivityHistorySummaryAsync(input: unknown): Promise<ActivityHistorySummary> {
const database = this.requireDatabase()
return readWithFallback(database.isTransaction ? undefined : this.readonlyQueryReader(),
'activityHistorySummary', [input], undefined, () => this.getActivityHistorySummary(input))
}
private writeActivityHistoryPlan(
database: DatabaseSync,
plan: (previous: ActivityHistoryState) => ActivityHistoryPlan
): void {
database.exec('BEGIN IMMEDIATE')
try {
const { data_version: dataVersion } = database.prepare('PRAGMA data_version').get() as { data_version: number }
const previous = this.activityHistoryCache?.dataVersion === dataVersion
? this.activityHistoryCache.state : this.readActivityHistoryState(database)
const result = plan(previous)
this.applyActivityHistoryPlan(database, result)
database.exec('COMMIT')
// Publish only committed records; a failed save must remain retryable.
this.activityHistoryCache = { state: result.next, dataVersion }
} catch (error) {
database.exec('ROLLBACK')
throw error
}
}
private readActivityHistoryState(database: DatabaseSync): ActivityHistoryState {
const rows = database.prepare(
'SELECT record_key, record_json FROM activity_history_records'
).all() as Array<{ record_key: string; record_json: string }>
const header = database.prepare(
'SELECT record_order_json, legacy_history_may_be_incomplete FROM activity_history WHERE singleton = 1'
).get() as { record_order_json: string; legacy_history_may_be_incomplete: number } | undefined
const order = header ? JSON.parse(header.record_order_json) as unknown : undefined
return {
entries: new Map(rows.map((row) => [row.record_key, { json: row.record_json }])),
...(Array.isArray(order) && order.every((key) => typeof key === 'string')
? { order, orderJson: header!.record_order_json } : {}),
...(header ? { incomplete: header.legacy_history_may_be_incomplete === 1 } : {})
}
}
private applyActivityHistoryPlan(database: DatabaseSync, plan: ActivityHistoryPlan): void {
if (plan.upserts.length > 0) {
const upsert = database.prepare(
`INSERT INTO activity_history_records (record_key, record_json) VALUES (?, ?)
ON CONFLICT(record_key) DO UPDATE SET record_json = excluded.record_json`
)
for (const [key, json] of plan.upserts) upsert.run(key, json)
}
if (plan.removes.length > 0) {
const remove = database.prepare('DELETE FROM activity_history_records WHERE record_key = ?')
for (const key of plan.removes) remove.run(key)
}
if (plan.headerChanged) {
database.prepare(
`UPDATE activity_history
SET record_order_json = ?, legacy_history_may_be_incomplete = ?
WHERE singleton = 1`
).run(plan.next.orderJson, Number(plan.next.incomplete))
}
}
/** Schema-44 migration: writes a parsed legacy snapshot into the new tables. */
private writeActivityHistory(
database: DatabaseSync,
snapshot: ActivityHistorySnapshot,
onProgress?: (processed: number) => void
): void {
const rows = database.prepare(
'SELECT record_key, record_json FROM activity_history_records'
).all() as Array<{ record_key: string; record_json: string }>
// The order column still holds the legacy records; always rewrite it.
const previous: ActivityHistoryState = {
entries: new Map(rows.map((row) => [row.record_key, { json: row.record_json }]))
}
this.applyActivityHistoryPlan(database, planActivityHistoryReplace(
snapshot.records.map((record) => ({ record })), snapshot.legacyHistoryMayBeIncomplete, previous, onProgress))
/** Ends records left running by requests that are no longer active (see the repository). */
reconcileActivityHistory(input: unknown): number {
return this.activityHistory().reconcile(input)
}
createTask(input: {
@@ -12088,11 +11988,13 @@ export class AssistantDatabase {
database.exec(`
CREATE TABLE activity_history_records (
record_key TEXT PRIMARY KEY NOT NULL,
record_json TEXT NOT NULL
record_json TEXT NOT NULL,
order_seq INTEGER NOT NULL DEFAULT 0
);
ALTER TABLE activity_history RENAME COLUMN records_json TO record_order_json;
`)
this.writeActivityHistory(database, snapshot, onProgress || isCancelled ? (processed) => {
// Writes order_seq directly; schema 57 then only adds the indexed columns.
new ActivityHistoryRepository(database).writeSnapshot(snapshot, onProgress || isCancelled ? (processed) => {
onProgress?.({ stage: 'converting', processed, total: snapshot.records.length, bytesBefore })
if (isCancelled?.()) throw new DOMException('Upgrade cancelled', 'AbortError')
} : undefined)
@@ -12345,6 +12247,15 @@ export class AssistantDatabase {
database.exec('PRAGMA user_version = 56; COMMIT;')
} catch (error) { database.exec('ROLLBACK'); throw error }
}
if (version.user_version < 57) {
database.exec('BEGIN IMMEDIATE')
try {
// Activity history order and indexed columns (PERF-15): pages read by index,
// and a move to the front no longer rewrites the order list.
migrateActivityHistoryOrder(database)
database.exec('PRAGMA user_version = 57; COMMIT;')
} catch (error) { database.exec('ROLLBACK'); throw error }
}
}
supervisionExperiences(): SupervisionExperienceStore {