feat: add computer control and managed browser
This commit is contained in:
lofyer
@@ -0,0 +1,13 @@
|
||||
import type {
|
||||
ComputerControlApprovalRequest,
|
||||
ComputerControlApprovalResult
|
||||
} from '../../shared/computer-control-contracts'
|
||||
|
||||
export const COMPUTER_CONTROL_APPROVAL_DEADLINE_MS = 120_000
|
||||
|
||||
export interface ComputerControlApprovalProvider {
|
||||
request(
|
||||
request: ComputerControlApprovalRequest,
|
||||
signal: AbortSignal
|
||||
): Promise<ComputerControlApprovalResult>
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import type {
|
||||
ComputerControlErrorCode,
|
||||
ComputerControlRisk
|
||||
} from '../../shared/computer-control-contracts'
|
||||
|
||||
export type ComputerControlAuditEvent = {
|
||||
timestamp: number
|
||||
taskId: string
|
||||
conversationId: string
|
||||
leaseId: string
|
||||
commandId: string
|
||||
action:
|
||||
| 'observe'
|
||||
| 'activate'
|
||||
| 'replace_text'
|
||||
| 'select_option'
|
||||
| 'scroll'
|
||||
risk: ComputerControlRisk
|
||||
outcome: 'completed' | 'denied' | 'failed' | 'outcome_unknown'
|
||||
errorCode?: ComputerControlErrorCode
|
||||
textLength?: number
|
||||
textDigest?: string
|
||||
}
|
||||
|
||||
export interface ComputerControlAuditSink {
|
||||
write(event: ComputerControlAuditEvent): void | Promise<void>
|
||||
}
|
||||
|
||||
export const digestComputerControlText = (text: string): string =>
|
||||
createHash('sha256').update(text, 'utf8').digest('hex')
|
||||
|
||||
export class InMemoryComputerControlAudit
|
||||
implements ComputerControlAuditSink
|
||||
{
|
||||
private readonly records: ComputerControlAuditEvent[] = []
|
||||
|
||||
constructor(private readonly maximumRecords = 1_000) {
|
||||
if (
|
||||
!Number.isInteger(maximumRecords) ||
|
||||
maximumRecords < 1 ||
|
||||
maximumRecords > 10_000
|
||||
) {
|
||||
throw new Error('Invalid computer control audit capacity')
|
||||
}
|
||||
}
|
||||
|
||||
write(event: ComputerControlAuditEvent): void {
|
||||
const bounded: ComputerControlAuditEvent = {
|
||||
...event,
|
||||
taskId: event.taskId.slice(0, 128),
|
||||
conversationId: event.conversationId.slice(0, 128),
|
||||
leaseId: event.leaseId.slice(0, 160),
|
||||
commandId: event.commandId.slice(0, 160)
|
||||
}
|
||||
this.records.push(Object.freeze(bounded))
|
||||
if (this.records.length > this.maximumRecords) {
|
||||
this.records.splice(0, this.records.length - this.maximumRecords)
|
||||
}
|
||||
}
|
||||
|
||||
entries(): readonly ComputerControlAuditEvent[] {
|
||||
return this.records.map((record) => ({ ...record }))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,896 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type {
|
||||
ComputerControlAction,
|
||||
ComputerControlApprovalRequest,
|
||||
ComputerControlApprovalResult,
|
||||
ComputerControlCommandResult,
|
||||
ComputerControlObservation
|
||||
} from '../../shared/computer-control-contracts'
|
||||
import { InMemoryComputerControlAudit } from './audit'
|
||||
import type { ComputerControlApprovalProvider } from './approval'
|
||||
import {
|
||||
ComputerControlBroker,
|
||||
type ComputerControlBrokerOptions
|
||||
} from './broker'
|
||||
import type {
|
||||
ComputerControlDriver,
|
||||
DriverElement,
|
||||
DriverObservation,
|
||||
DriverWindowIdentity,
|
||||
NativeElementIdentity
|
||||
} from './driver'
|
||||
import type { ComputerControlLeaseBinding } from './lease-store'
|
||||
|
||||
const binding: ComputerControlLeaseBinding = {
|
||||
taskId: 'task-1',
|
||||
conversationId: 'conversation-1',
|
||||
pid: 42,
|
||||
processStartTime: 100,
|
||||
windowIdentity: 'window-1'
|
||||
}
|
||||
|
||||
const commandId = (suffix: string): string =>
|
||||
`command_identifier_${suffix.padStart(6, '0')}`
|
||||
|
||||
class FakeDriver implements ComputerControlDriver {
|
||||
readonly available = true
|
||||
readonly injections: ComputerControlAction[] = []
|
||||
releaseCount = 0
|
||||
activeInjections = 0
|
||||
maximumActiveInjections = 0
|
||||
foreground: DriverWindowIdentity = {
|
||||
pid: binding.pid,
|
||||
processStartTime: binding.processStartTime,
|
||||
windowIdentity: binding.windowIdentity
|
||||
}
|
||||
injectGate?: Promise<void>
|
||||
foregroundGate?: Promise<void>
|
||||
|
||||
constructor(readonly elements: DriverElement[]) {}
|
||||
|
||||
async observe(): Promise<DriverObservation> {
|
||||
await this.foregroundGate
|
||||
return {
|
||||
window: { ...this.foreground },
|
||||
windowTitle: 'Test window',
|
||||
elements: this.elements.map((element) => ({ ...element }))
|
||||
}
|
||||
}
|
||||
|
||||
async getForegroundWindow(): Promise<DriverWindowIdentity> {
|
||||
await this.foregroundGate
|
||||
return { ...this.foreground }
|
||||
}
|
||||
|
||||
async resolveElement(
|
||||
nativeIdentity: NativeElementIdentity
|
||||
): Promise<DriverElement | undefined> {
|
||||
const element = this.elements.find(
|
||||
(candidate) => candidate.nativeIdentity === nativeIdentity
|
||||
)
|
||||
return element ? { ...element } : undefined
|
||||
}
|
||||
|
||||
async focusElement(
|
||||
nativeIdentity: NativeElementIdentity
|
||||
): Promise<boolean> {
|
||||
const element = this.elements.find(
|
||||
(candidate) => candidate.nativeIdentity === nativeIdentity
|
||||
)
|
||||
if (!element || !element.enabled) {
|
||||
return false
|
||||
}
|
||||
element.focused = true
|
||||
return true
|
||||
}
|
||||
|
||||
async inject(
|
||||
nativeIdentity: NativeElementIdentity,
|
||||
action: ComputerControlAction
|
||||
): Promise<void> {
|
||||
void nativeIdentity
|
||||
this.injections.push(action)
|
||||
this.activeInjections += 1
|
||||
this.maximumActiveInjections = Math.max(
|
||||
this.maximumActiveInjections,
|
||||
this.activeInjections
|
||||
)
|
||||
try {
|
||||
await this.injectGate
|
||||
} finally {
|
||||
this.activeInjections -= 1
|
||||
}
|
||||
}
|
||||
|
||||
async releaseInjectedInput(): Promise<void> {
|
||||
this.releaseCount += 1
|
||||
}
|
||||
}
|
||||
|
||||
class FakeApproval implements ComputerControlApprovalProvider {
|
||||
readonly requests: ComputerControlApprovalRequest[] = []
|
||||
handler?: (
|
||||
request: ComputerControlApprovalRequest,
|
||||
signal: AbortSignal
|
||||
) => Promise<ComputerControlApprovalResult>
|
||||
|
||||
async request(
|
||||
request: ComputerControlApprovalRequest,
|
||||
signal: AbortSignal
|
||||
): Promise<ComputerControlApprovalResult> {
|
||||
this.requests.push(request)
|
||||
if (this.handler) {
|
||||
return this.handler(request, signal)
|
||||
}
|
||||
return {
|
||||
approvalId: request.approvalId,
|
||||
decision: 'approve_once'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const makeElement = (
|
||||
role: DriverElement['role'],
|
||||
targetKind: DriverElement['targetKind'] = 'standard',
|
||||
name = 'Target'
|
||||
): DriverElement => ({
|
||||
nativeIdentity: {},
|
||||
role,
|
||||
name,
|
||||
enabled: true,
|
||||
focused: false,
|
||||
targetKind
|
||||
})
|
||||
|
||||
const makeHarness = (
|
||||
elements: DriverElement[],
|
||||
options: {
|
||||
now?: () => number
|
||||
driverDeadlineMs?: number
|
||||
approvalDeadlineMs?: number
|
||||
audit?: ComputerControlBrokerOptions['audit']
|
||||
fallbackAudit?: ComputerControlBrokerOptions['fallbackAudit']
|
||||
} = {}
|
||||
) => {
|
||||
let idSequence = 0
|
||||
const driver = new FakeDriver(elements)
|
||||
const approval = new FakeApproval()
|
||||
const audit = new InMemoryComputerControlAudit()
|
||||
const broker = new ComputerControlBroker({
|
||||
driver,
|
||||
approval,
|
||||
audit: options.audit ?? audit,
|
||||
fallbackAudit: options.fallbackAudit,
|
||||
now: options.now,
|
||||
driverDeadlineMs: options.driverDeadlineMs,
|
||||
approvalDeadlineMs: options.approvalDeadlineMs,
|
||||
createId: () =>
|
||||
`generated_identifier_${String(++idSequence).padStart(6, '0')}`
|
||||
})
|
||||
return { broker, driver, approval, audit }
|
||||
}
|
||||
|
||||
const observe = async (
|
||||
broker: ComputerControlBroker,
|
||||
leaseId: string,
|
||||
suffix: string,
|
||||
context = binding
|
||||
): Promise<ComputerControlObservation> => {
|
||||
const result = await broker.execute(
|
||||
{
|
||||
kind: 'observe',
|
||||
commandId: commandId(suffix),
|
||||
leaseId
|
||||
},
|
||||
context,
|
||||
new AbortController().signal
|
||||
)
|
||||
if (result.status !== 'observed') {
|
||||
throw new Error(`Observation failed: ${JSON.stringify(result)}`)
|
||||
}
|
||||
return result.observation
|
||||
}
|
||||
|
||||
const expectErrorCode = (
|
||||
result: ComputerControlCommandResult,
|
||||
code: string
|
||||
): void => {
|
||||
expect(result).toMatchObject({
|
||||
status: 'error',
|
||||
error: { code }
|
||||
})
|
||||
}
|
||||
|
||||
describe('ComputerControlBroker', () => {
|
||||
it('rejects stale and consumed refs while preserving command idempotency', async () => {
|
||||
let now = 1_000
|
||||
const { broker, driver } = makeHarness([makeElement('link')], {
|
||||
now: () => now
|
||||
})
|
||||
const lease = broker.createLease(binding)
|
||||
const staleObservation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'stale-observe'
|
||||
)
|
||||
now += 3_000
|
||||
const stale = await broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('stale-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: staleObservation.observationId,
|
||||
revision: staleObservation.revision,
|
||||
action: {
|
||||
kind: 'activate',
|
||||
elementRef: staleObservation.elements[0]?.ref
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(stale, 'observation_stale')
|
||||
|
||||
const fresh = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'fresh-observe'
|
||||
)
|
||||
const action = {
|
||||
kind: 'act' as const,
|
||||
commandId: commandId('fresh-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: fresh.observationId,
|
||||
revision: fresh.revision,
|
||||
action: {
|
||||
kind: 'activate' as const,
|
||||
elementRef: fresh.elements[0]?.ref
|
||||
}
|
||||
}
|
||||
const first = await broker.execute(
|
||||
action,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expect(first.status).toBe('completed')
|
||||
await expect(
|
||||
broker.execute(
|
||||
action,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
).resolves.toEqual(first)
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
|
||||
const consumed = await broker.execute(
|
||||
{ ...action, commandId: commandId('consumed') },
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(consumed, 'observation_consumed')
|
||||
})
|
||||
|
||||
it('rejects command ID reuse with different content', async () => {
|
||||
const { broker } = makeHarness([makeElement('link')])
|
||||
const lease = broker.createLease(binding)
|
||||
await observe(broker, lease.leaseId, 'same-id')
|
||||
const conflict = await broker.execute(
|
||||
{
|
||||
kind: 'observe',
|
||||
commandId: commandId('same-id'),
|
||||
leaseId: 'different_lease_identifier'
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(conflict, 'command_id_conflict')
|
||||
})
|
||||
|
||||
it.each([
|
||||
'protected',
|
||||
'password',
|
||||
'otp',
|
||||
'payment',
|
||||
'account_security',
|
||||
'privilege',
|
||||
'os_permission'
|
||||
] as const)('forbids %s targets before approval or injection', async (targetKind) => {
|
||||
const { broker, driver, approval } = makeHarness([
|
||||
makeElement('textbox', targetKind)
|
||||
])
|
||||
const lease = broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
`forbidden-${targetKind}`
|
||||
)
|
||||
expect(observation.elements[0]).toMatchObject({
|
||||
risk: 'forbidden',
|
||||
blocked: true
|
||||
})
|
||||
const result = await broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId(`act-${targetKind}`),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'replace_text',
|
||||
elementRef: observation.elements[0]?.ref,
|
||||
text: 'never injected'
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(result, 'forbidden')
|
||||
expect(approval.requests).toHaveLength(0)
|
||||
expect(driver.injections).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('requires a distinct once approval for every input and commit', async () => {
|
||||
const { broker, approval } = makeHarness([
|
||||
makeElement('textbox', 'standard', 'Editor'),
|
||||
makeElement('button', 'standard', 'Submit')
|
||||
])
|
||||
const lease = broker.createLease(binding)
|
||||
const inputObservation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'input-observe'
|
||||
)
|
||||
const input = await broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('input-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: inputObservation.observationId,
|
||||
revision: inputObservation.revision,
|
||||
action: {
|
||||
kind: 'replace_text',
|
||||
elementRef: inputObservation.elements[0]?.ref,
|
||||
text: 'hello'
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expect(input).toMatchObject({ status: 'completed', risk: 'input' })
|
||||
|
||||
const commitObservation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'commit-observe'
|
||||
)
|
||||
const commit = await broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('commit-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: commitObservation.observationId,
|
||||
revision: commitObservation.revision,
|
||||
action: {
|
||||
kind: 'activate',
|
||||
elementRef: commitObservation.elements[1]?.ref
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expect(commit).toMatchObject({
|
||||
status: 'completed',
|
||||
risk: 'commit'
|
||||
})
|
||||
expect(approval.requests.map((request) => request.risk)).toEqual([
|
||||
'input',
|
||||
'commit'
|
||||
])
|
||||
expect(
|
||||
approval.requests.every(
|
||||
(request) => !('text' in request)
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('handles approval denial, timeout, and cancellation fail-closed', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const deniedHarness = makeHarness([makeElement('textbox')])
|
||||
deniedHarness.approval.handler = async (request) => ({
|
||||
approvalId: request.approvalId,
|
||||
decision: 'deny'
|
||||
})
|
||||
const deniedLease = deniedHarness.broker.createLease(binding)
|
||||
const deniedObservation = await observe(
|
||||
deniedHarness.broker,
|
||||
deniedLease.leaseId,
|
||||
'denied-observe'
|
||||
)
|
||||
const denied = await deniedHarness.broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('denied-act'),
|
||||
leaseId: deniedLease.leaseId,
|
||||
observationId: deniedObservation.observationId,
|
||||
revision: deniedObservation.revision,
|
||||
action: {
|
||||
kind: 'replace_text',
|
||||
elementRef: deniedObservation.elements[0]?.ref,
|
||||
text: 'denied'
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(denied, 'approval_denied')
|
||||
|
||||
const timeoutHarness = makeHarness([makeElement('textbox')], {
|
||||
approvalDeadlineMs: 20
|
||||
})
|
||||
timeoutHarness.approval.handler = () =>
|
||||
new Promise(() => undefined)
|
||||
const timeoutLease = timeoutHarness.broker.createLease(binding)
|
||||
const timeoutObservation = await observe(
|
||||
timeoutHarness.broker,
|
||||
timeoutLease.leaseId,
|
||||
'timeout-observe'
|
||||
)
|
||||
const timeoutPromise = timeoutHarness.broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('timeout-act'),
|
||||
leaseId: timeoutLease.leaseId,
|
||||
observationId: timeoutObservation.observationId,
|
||||
revision: timeoutObservation.revision,
|
||||
action: {
|
||||
kind: 'replace_text',
|
||||
elementRef: timeoutObservation.elements[0]?.ref,
|
||||
text: 'timeout'
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(20)
|
||||
expectErrorCode(await timeoutPromise, 'approval_timeout')
|
||||
|
||||
const cancelledHarness = makeHarness([makeElement('textbox')])
|
||||
cancelledHarness.approval.handler = () =>
|
||||
new Promise(() => undefined)
|
||||
const cancelledLease =
|
||||
cancelledHarness.broker.createLease(binding)
|
||||
const cancelledObservation = await observe(
|
||||
cancelledHarness.broker,
|
||||
cancelledLease.leaseId,
|
||||
'cancel-observe'
|
||||
)
|
||||
const controller = new AbortController()
|
||||
const cancelledPromise = cancelledHarness.broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('cancel-act'),
|
||||
leaseId: cancelledLease.leaseId,
|
||||
observationId: cancelledObservation.observationId,
|
||||
revision: cancelledObservation.revision,
|
||||
action: {
|
||||
kind: 'replace_text',
|
||||
elementRef: cancelledObservation.elements[0]?.ref,
|
||||
text: 'cancelled'
|
||||
}
|
||||
},
|
||||
binding,
|
||||
controller.signal
|
||||
)
|
||||
await vi.waitFor(() => {
|
||||
expect(cancelledHarness.approval.requests).toHaveLength(1)
|
||||
})
|
||||
controller.abort()
|
||||
expectErrorCode(await cancelledPromise, 'cancelled')
|
||||
expect(
|
||||
cancelledHarness.broker.leases.peek(cancelledLease.leaseId)
|
||||
).toBeUndefined()
|
||||
expect(cancelledHarness.driver.releaseCount).toBeGreaterThan(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('reports outcome_unknown when cancellation races injection', async () => {
|
||||
let releaseInjection: (() => void) | undefined
|
||||
const { broker, driver } = makeHarness([makeElement('link')])
|
||||
driver.injectGate = new Promise<void>((resolve) => {
|
||||
releaseInjection = resolve
|
||||
})
|
||||
const lease = broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'race-observe'
|
||||
)
|
||||
const controller = new AbortController()
|
||||
const resultPromise = broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('race-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'activate',
|
||||
elementRef: observation.elements[0]?.ref
|
||||
}
|
||||
},
|
||||
binding,
|
||||
controller.signal
|
||||
)
|
||||
await vi.waitFor(() => {
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
})
|
||||
controller.abort()
|
||||
expectErrorCode(await resultPromise, 'outcome_unknown')
|
||||
releaseInjection?.()
|
||||
expect(driver.releaseCount).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('caches outcome_unknown when injection times out before late completion', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let releaseInjection: (() => void) | undefined
|
||||
const { broker, driver, audit } = makeHarness(
|
||||
[makeElement('link')],
|
||||
{ driverDeadlineMs: 20 }
|
||||
)
|
||||
const lease = broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'injection-timeout-observe'
|
||||
)
|
||||
driver.injectGate = new Promise<void>((resolve) => {
|
||||
releaseInjection = resolve
|
||||
})
|
||||
const command = {
|
||||
kind: 'act' as const,
|
||||
commandId: commandId('injection-timeout-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'activate' as const,
|
||||
elementRef: observation.elements[0]?.ref
|
||||
}
|
||||
}
|
||||
|
||||
const firstPromise = broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
await vi.waitFor(() => {
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(20)
|
||||
const first = await firstPromise
|
||||
expectErrorCode(first, 'outcome_unknown')
|
||||
|
||||
await expect(
|
||||
broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
).resolves.toEqual(first)
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
expect(audit.entries().at(-1)).toMatchObject({
|
||||
outcome: 'outcome_unknown',
|
||||
errorCode: 'outcome_unknown'
|
||||
})
|
||||
|
||||
releaseInjection?.()
|
||||
await Promise.resolve()
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('treats an uncertain injection driver failure as outcome_unknown', async () => {
|
||||
const { broker, driver, audit } = makeHarness([makeElement('link')])
|
||||
const lease = broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'injection-failure-observe'
|
||||
)
|
||||
driver.inject = vi.fn(async () => {
|
||||
throw new Error('driver disconnected during injection')
|
||||
})
|
||||
const command = {
|
||||
kind: 'act' as const,
|
||||
commandId: commandId('injection-failure-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'activate' as const,
|
||||
elementRef: observation.elements[0]?.ref
|
||||
}
|
||||
}
|
||||
|
||||
const first = await broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(first, 'outcome_unknown')
|
||||
await expect(
|
||||
broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
).resolves.toEqual(first)
|
||||
expect(driver.inject).toHaveBeenCalledOnce()
|
||||
expect(audit.entries().at(-1)).toMatchObject({
|
||||
outcome: 'outcome_unknown',
|
||||
errorCode: 'outcome_unknown'
|
||||
})
|
||||
})
|
||||
|
||||
it('isolates throwing primary and fallback audit sinks from cached results', async () => {
|
||||
const primaryWrite = vi.fn(async () => {
|
||||
throw new Error('primary audit unavailable')
|
||||
})
|
||||
const fallbackWrite = vi.fn(async () => {
|
||||
throw new Error('fallback audit unavailable')
|
||||
})
|
||||
const { broker, driver } = makeHarness([makeElement('link')], {
|
||||
audit: { write: primaryWrite },
|
||||
fallbackAudit: { write: fallbackWrite }
|
||||
})
|
||||
const lease = broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'throwing-audit-observe'
|
||||
)
|
||||
const command = {
|
||||
kind: 'act' as const,
|
||||
commandId: commandId('throwing-audit-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'activate' as const,
|
||||
elementRef: observation.elements[0]?.ref
|
||||
}
|
||||
}
|
||||
|
||||
const first = await broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expect(first).toMatchObject({ status: 'completed' })
|
||||
await expect(
|
||||
broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
).resolves.toEqual(first)
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
expect(primaryWrite).toHaveBeenCalledTimes(2)
|
||||
expect(fallbackWrite).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('isolates throwing audit sinks while caching a failed result', async () => {
|
||||
const primaryWrite = vi.fn(async () => {
|
||||
throw new Error('primary audit unavailable')
|
||||
})
|
||||
const fallbackWrite = vi.fn(async () => {
|
||||
throw new Error('fallback audit unavailable')
|
||||
})
|
||||
const { broker, driver } = makeHarness(
|
||||
[makeElement('textbox', 'protected')],
|
||||
{
|
||||
audit: { write: primaryWrite },
|
||||
fallbackAudit: { write: fallbackWrite }
|
||||
}
|
||||
)
|
||||
const lease = broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'throwing-failure-audit-observe'
|
||||
)
|
||||
const command = {
|
||||
kind: 'act' as const,
|
||||
commandId: commandId('throwing-failure-audit-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'replace_text' as const,
|
||||
elementRef: observation.elements[0]?.ref,
|
||||
text: 'not injected'
|
||||
}
|
||||
}
|
||||
|
||||
const first = await broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(first, 'forbidden')
|
||||
await expect(
|
||||
broker.execute(
|
||||
command,
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
).resolves.toEqual(first)
|
||||
expect(driver.injections).toHaveLength(0)
|
||||
expect(primaryWrite).toHaveBeenCalledTimes(2)
|
||||
expect(fallbackWrite).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('serializes actions for leases belonging to the same task', async () => {
|
||||
let releaseFirst: (() => void) | undefined
|
||||
const { broker, driver } = makeHarness([makeElement('link')])
|
||||
driver.injectGate = new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve
|
||||
})
|
||||
const firstLease = broker.createLease(binding)
|
||||
const secondLease = broker.createLease(binding)
|
||||
const firstObservation = await observe(
|
||||
broker,
|
||||
firstLease.leaseId,
|
||||
'serial-observe-1'
|
||||
)
|
||||
const secondObservation = await observe(
|
||||
broker,
|
||||
secondLease.leaseId,
|
||||
'serial-observe-2'
|
||||
)
|
||||
const actionFor = (
|
||||
leaseId: string,
|
||||
observation: ComputerControlObservation,
|
||||
suffix: string
|
||||
) =>
|
||||
broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId(suffix),
|
||||
leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'activate',
|
||||
elementRef: observation.elements[0]?.ref
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
const first = actionFor(
|
||||
firstLease.leaseId,
|
||||
firstObservation,
|
||||
'serial-act-1'
|
||||
)
|
||||
await vi.waitFor(() => {
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
})
|
||||
const second = actionFor(
|
||||
secondLease.leaseId,
|
||||
secondObservation,
|
||||
'serial-act-2'
|
||||
)
|
||||
await Promise.resolve()
|
||||
expect(driver.injections).toHaveLength(1)
|
||||
releaseFirst?.()
|
||||
driver.injectGate = undefined
|
||||
await expect(Promise.all([first, second])).resolves.toEqual([
|
||||
expect.objectContaining({ status: 'completed' }),
|
||||
expect.objectContaining({ status: 'completed' })
|
||||
])
|
||||
expect(driver.maximumActiveInjections).toBe(1)
|
||||
})
|
||||
|
||||
it('revokes on foreground identity mismatch and enforces driver deadlines', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const mismatchHarness = makeHarness([makeElement('link')])
|
||||
const mismatchLease =
|
||||
mismatchHarness.broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
mismatchHarness.broker,
|
||||
mismatchLease.leaseId,
|
||||
'mismatch-observe'
|
||||
)
|
||||
mismatchHarness.driver.foreground = {
|
||||
...mismatchHarness.driver.foreground,
|
||||
pid: 99
|
||||
}
|
||||
const mismatch = await mismatchHarness.broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('mismatch-act'),
|
||||
leaseId: mismatchLease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'activate',
|
||||
elementRef: observation.elements[0]?.ref
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expectErrorCode(mismatch, 'window_not_foreground')
|
||||
expect(
|
||||
mismatchHarness.broker.leases.peek(mismatchLease.leaseId)
|
||||
).toBeUndefined()
|
||||
|
||||
const timeoutHarness = makeHarness([makeElement('link')], {
|
||||
driverDeadlineMs: 20
|
||||
})
|
||||
timeoutHarness.driver.foregroundGate = new Promise(
|
||||
() => undefined
|
||||
)
|
||||
const timeoutLease = timeoutHarness.broker.createLease(binding)
|
||||
const timeoutPromise = timeoutHarness.broker.execute(
|
||||
{
|
||||
kind: 'observe',
|
||||
commandId: commandId('driver-timeout'),
|
||||
leaseId: timeoutLease.leaseId
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(20)
|
||||
expectErrorCode(await timeoutPromise, 'driver_timeout')
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('audits typed text only as length and digest', async () => {
|
||||
const { broker, approval, audit } = makeHarness([
|
||||
makeElement('textbox', 'standard', 'Editor')
|
||||
])
|
||||
const lease = broker.createLease(binding)
|
||||
const observation = await observe(
|
||||
broker,
|
||||
lease.leaseId,
|
||||
'audit-observe'
|
||||
)
|
||||
const secretText = 'private typed content'
|
||||
const result = await broker.execute(
|
||||
{
|
||||
kind: 'act',
|
||||
commandId: commandId('audit-act'),
|
||||
leaseId: lease.leaseId,
|
||||
observationId: observation.observationId,
|
||||
revision: observation.revision,
|
||||
action: {
|
||||
kind: 'replace_text',
|
||||
elementRef: observation.elements[0]?.ref,
|
||||
text: secretText
|
||||
}
|
||||
},
|
||||
binding,
|
||||
new AbortController().signal
|
||||
)
|
||||
expect(result.status).toBe('completed')
|
||||
const entry = audit.entries().at(-1)
|
||||
expect(entry).toMatchObject({
|
||||
action: 'replace_text',
|
||||
textLength: secretText.length
|
||||
})
|
||||
expect(entry?.textDigest).toMatch(/^[a-f0-9]{64}$/)
|
||||
expect(JSON.stringify(audit.entries())).not.toContain(secretText)
|
||||
expect(JSON.stringify(approval.requests)).not.toContain(secretText)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,729 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
computerControlApprovalResultSchema,
|
||||
computerControlRuntimeCommandSchema,
|
||||
type ComputerControlAction,
|
||||
type ComputerControlCommandResult,
|
||||
type ComputerControlRisk,
|
||||
type ComputerControlRuntimeCommand
|
||||
} from '../../shared/computer-control-contracts'
|
||||
import {
|
||||
digestComputerControlText,
|
||||
type ComputerControlAuditEvent,
|
||||
type ComputerControlAuditSink
|
||||
} from './audit'
|
||||
import {
|
||||
COMPUTER_CONTROL_APPROVAL_DEADLINE_MS,
|
||||
type ComputerControlApprovalProvider
|
||||
} from './approval'
|
||||
import { runWithDeadline } from './deadline'
|
||||
import {
|
||||
COMPUTER_CONTROL_DRIVER_DEADLINE_MS,
|
||||
type ComputerControlDriver,
|
||||
type DriverElement,
|
||||
type DriverWindowIdentity
|
||||
} from './driver'
|
||||
import {
|
||||
cancellationFailure,
|
||||
ComputerControlFailure
|
||||
} from './errors'
|
||||
import {
|
||||
ComputerControlLeaseStore,
|
||||
type ComputerControlLease,
|
||||
type ComputerControlLeaseBinding
|
||||
} from './lease-store'
|
||||
import { ComputerControlPerceptionStore } from './perception-store'
|
||||
import {
|
||||
classifyComputerControlAction,
|
||||
maximumRisk
|
||||
} from './risk-policy'
|
||||
|
||||
const INVALID_COMMAND_ID = 'invalid_command_000000'
|
||||
const MAX_CACHED_COMMANDS = 1_000
|
||||
|
||||
type BrokerExecutionContext = ComputerControlLeaseBinding
|
||||
|
||||
type InFlightCommand = {
|
||||
fingerprint: string
|
||||
result: Promise<ComputerControlCommandResult>
|
||||
}
|
||||
|
||||
export type ComputerControlBrokerOptions = {
|
||||
driver: ComputerControlDriver
|
||||
approval: ComputerControlApprovalProvider
|
||||
audit: ComputerControlAuditSink
|
||||
fallbackAudit?: ComputerControlAuditSink
|
||||
leases?: ComputerControlLeaseStore
|
||||
perceptions?: ComputerControlPerceptionStore
|
||||
driverDeadlineMs?: number
|
||||
approvalDeadlineMs?: number
|
||||
classify?: (
|
||||
action: ComputerControlAction,
|
||||
element: DriverElement
|
||||
) => ComputerControlRisk
|
||||
now?: () => number
|
||||
createId?: () => string
|
||||
}
|
||||
|
||||
export class ComputerControlBroker {
|
||||
readonly leases: ComputerControlLeaseStore
|
||||
readonly perceptions: ComputerControlPerceptionStore
|
||||
|
||||
private readonly driver: ComputerControlDriver
|
||||
private readonly approval: ComputerControlApprovalProvider
|
||||
private readonly audit: ComputerControlAuditSink
|
||||
private readonly fallbackAudit: ComputerControlAuditSink | undefined
|
||||
private readonly driverDeadlineMs: number
|
||||
private readonly approvalDeadlineMs: number
|
||||
private readonly classify?: ComputerControlBrokerOptions['classify']
|
||||
private readonly now: () => number
|
||||
private readonly createId: () => string
|
||||
private readonly queues = new Map<string, Promise<void>>()
|
||||
private readonly completed = new Map<
|
||||
string,
|
||||
{ fingerprint: string; result: ComputerControlCommandResult }
|
||||
>()
|
||||
private readonly inFlight = new Map<string, InFlightCommand>()
|
||||
|
||||
constructor(options: ComputerControlBrokerOptions) {
|
||||
this.driver = options.driver
|
||||
this.approval = options.approval
|
||||
this.audit = options.audit
|
||||
this.fallbackAudit = options.fallbackAudit
|
||||
this.now = options.now ?? Date.now
|
||||
this.createId = options.createId ?? randomUUID
|
||||
this.leases =
|
||||
options.leases ??
|
||||
new ComputerControlLeaseStore(this.now, this.createId)
|
||||
this.perceptions =
|
||||
options.perceptions ??
|
||||
new ComputerControlPerceptionStore(this.now, this.createId)
|
||||
this.driverDeadlineMs =
|
||||
options.driverDeadlineMs ?? COMPUTER_CONTROL_DRIVER_DEADLINE_MS
|
||||
this.approvalDeadlineMs =
|
||||
options.approvalDeadlineMs ??
|
||||
COMPUTER_CONTROL_APPROVAL_DEADLINE_MS
|
||||
this.classify = options.classify
|
||||
}
|
||||
|
||||
createLease(
|
||||
binding: ComputerControlLeaseBinding
|
||||
): ComputerControlLease {
|
||||
return this.leases.create(binding)
|
||||
}
|
||||
|
||||
revoke(leaseId: string): void {
|
||||
this.leases.revoke(leaseId)
|
||||
this.perceptions.revokeLease(leaseId)
|
||||
void this.releaseInjectedInput()
|
||||
}
|
||||
|
||||
revokeTask(taskId: string): void {
|
||||
const revokedLeaseIds = this.leases.revokeTask(taskId)
|
||||
for (const leaseId of revokedLeaseIds) {
|
||||
this.perceptions.revokeLease(leaseId)
|
||||
}
|
||||
void this.releaseInjectedInput()
|
||||
}
|
||||
|
||||
async execute(
|
||||
untrustedCommand: unknown,
|
||||
context: BrokerExecutionContext,
|
||||
signal: AbortSignal
|
||||
): Promise<ComputerControlCommandResult> {
|
||||
const parsed =
|
||||
computerControlRuntimeCommandSchema.safeParse(untrustedCommand)
|
||||
if (!parsed.success) {
|
||||
return this.errorResult(
|
||||
INVALID_COMMAND_ID,
|
||||
new ComputerControlFailure(
|
||||
'invalid_request',
|
||||
'Invalid computer control command'
|
||||
)
|
||||
)
|
||||
}
|
||||
const command = parsed.data
|
||||
const fingerprint = JSON.stringify({ command, context })
|
||||
|
||||
const cached = this.completed.get(command.commandId)
|
||||
if (cached) {
|
||||
return cached.fingerprint === fingerprint
|
||||
? cached.result
|
||||
: this.errorResult(
|
||||
command.commandId,
|
||||
new ComputerControlFailure(
|
||||
'command_id_conflict',
|
||||
'Computer control command ID was reused'
|
||||
)
|
||||
)
|
||||
}
|
||||
const active = this.inFlight.get(command.commandId)
|
||||
if (active) {
|
||||
return active.fingerprint === fingerprint
|
||||
? active.result
|
||||
: this.errorResult(
|
||||
command.commandId,
|
||||
new ComputerControlFailure(
|
||||
'command_id_conflict',
|
||||
'Computer control command ID was reused'
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
const result = this.serialize(context.taskId, () =>
|
||||
this.executeSerialized(command, context, signal)
|
||||
)
|
||||
this.inFlight.set(command.commandId, { fingerprint, result })
|
||||
result
|
||||
.then((settled) => {
|
||||
this.completed.set(command.commandId, {
|
||||
fingerprint,
|
||||
result: settled
|
||||
})
|
||||
while (this.completed.size > MAX_CACHED_COMMANDS) {
|
||||
const oldest = this.completed.keys().next().value
|
||||
if (oldest === undefined) {
|
||||
break
|
||||
}
|
||||
this.completed.delete(oldest)
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
this.inFlight.delete(command.commandId)
|
||||
})
|
||||
.catch(() => {
|
||||
// executeSerialized always returns a contract result.
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
private async executeSerialized(
|
||||
command: ComputerControlRuntimeCommand,
|
||||
context: BrokerExecutionContext,
|
||||
signal: AbortSignal
|
||||
): Promise<ComputerControlCommandResult> {
|
||||
let injectionStarted = false
|
||||
const cancel = (): void => {
|
||||
this.revoke(command.leaseId)
|
||||
}
|
||||
signal.addEventListener('abort', cancel, { once: true })
|
||||
|
||||
try {
|
||||
if (signal.aborted) {
|
||||
cancel()
|
||||
throw cancellationFailure()
|
||||
}
|
||||
const lease = this.leases.validate(command.leaseId, context)
|
||||
if (!this.driver.available) {
|
||||
throw new ComputerControlFailure(
|
||||
'driver_unavailable',
|
||||
'Computer control driver is unavailable'
|
||||
)
|
||||
}
|
||||
|
||||
if (command.kind === 'observe') {
|
||||
const result = await this.observe(command, lease, signal)
|
||||
await this.writeAudit({
|
||||
...this.auditBase(command, lease),
|
||||
action: 'observe',
|
||||
risk: 'observe',
|
||||
outcome: 'completed'
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
const result = await this.act(
|
||||
command,
|
||||
lease,
|
||||
signal,
|
||||
() => {
|
||||
injectionStarted = true
|
||||
}
|
||||
)
|
||||
return result
|
||||
} catch (error) {
|
||||
const failure =
|
||||
signal.aborted && injectionStarted
|
||||
? new ComputerControlFailure(
|
||||
'outcome_unknown',
|
||||
'Cancellation raced input injection'
|
||||
)
|
||||
: this.normalizeFailure(error)
|
||||
if (command.kind === 'act') {
|
||||
await this.writeFailureAudit(command, context, failure)
|
||||
}
|
||||
return this.errorResult(command.commandId, failure)
|
||||
} finally {
|
||||
signal.removeEventListener('abort', cancel)
|
||||
if (signal.aborted) {
|
||||
await this.releaseInjectedInput()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async observe(
|
||||
command: Extract<ComputerControlRuntimeCommand, { kind: 'observe' }>,
|
||||
lease: ComputerControlLease,
|
||||
signal: AbortSignal
|
||||
): Promise<ComputerControlCommandResult> {
|
||||
const observation = await this.driverCall(
|
||||
(driverSignal) => this.driver.observe(driverSignal),
|
||||
signal
|
||||
)
|
||||
this.assertWindowMatches(lease, observation.window)
|
||||
const contract = this.perceptions.create(
|
||||
lease.leaseId,
|
||||
observation,
|
||||
(element) =>
|
||||
this.classifyRisk(
|
||||
{
|
||||
kind: 'activate',
|
||||
elementRef: 'opaque_identifier_placeholder'
|
||||
},
|
||||
element
|
||||
)
|
||||
)
|
||||
return {
|
||||
status: 'observed',
|
||||
commandId: command.commandId,
|
||||
observation: contract
|
||||
}
|
||||
}
|
||||
|
||||
private async act(
|
||||
command: Extract<ComputerControlRuntimeCommand, { kind: 'act' }>,
|
||||
lease: ComputerControlLease,
|
||||
signal: AbortSignal,
|
||||
markInjectionStarted: () => void
|
||||
): Promise<ComputerControlCommandResult> {
|
||||
const stored = this.perceptions.resolve(
|
||||
lease.leaseId,
|
||||
command.observationId,
|
||||
command.revision,
|
||||
command.action.elementRef
|
||||
)
|
||||
let resolved = await this.validateTarget(
|
||||
lease,
|
||||
stored.driverElement,
|
||||
stored.nativeIdentity,
|
||||
signal
|
||||
)
|
||||
let risk = this.classifyRisk(command.action, resolved)
|
||||
if (risk === 'forbidden') {
|
||||
throw new ComputerControlFailure(
|
||||
'forbidden',
|
||||
'Computer control target is protected'
|
||||
)
|
||||
}
|
||||
|
||||
if (risk === 'input' || risk === 'commit') {
|
||||
await this.requestApproval(command, lease, resolved, risk, signal)
|
||||
}
|
||||
|
||||
resolved = await this.validateTarget(
|
||||
lease,
|
||||
stored.driverElement,
|
||||
stored.nativeIdentity,
|
||||
signal
|
||||
)
|
||||
const finalRisk = this.classifyRisk(command.action, resolved)
|
||||
if (finalRisk === 'forbidden') {
|
||||
throw new ComputerControlFailure(
|
||||
'forbidden',
|
||||
'Computer control target is protected'
|
||||
)
|
||||
}
|
||||
const elevatedRisk = maximumRisk(risk, finalRisk)
|
||||
if (elevatedRisk !== risk) {
|
||||
if (elevatedRisk === 'input' || elevatedRisk === 'commit') {
|
||||
await this.requestApproval(
|
||||
command,
|
||||
lease,
|
||||
resolved,
|
||||
elevatedRisk,
|
||||
signal
|
||||
)
|
||||
resolved = await this.validateTarget(
|
||||
lease,
|
||||
stored.driverElement,
|
||||
stored.nativeIdentity,
|
||||
signal
|
||||
)
|
||||
}
|
||||
risk = elevatedRisk
|
||||
}
|
||||
const injectionRisk = this.classifyRisk(command.action, resolved)
|
||||
if (injectionRisk === 'forbidden') {
|
||||
throw new ComputerControlFailure(
|
||||
'forbidden',
|
||||
'Computer control target is protected'
|
||||
)
|
||||
}
|
||||
const finalElevatedRisk = maximumRisk(risk, injectionRisk)
|
||||
if (finalElevatedRisk !== risk) {
|
||||
throw new ComputerControlFailure(
|
||||
'approval_denied',
|
||||
'Computer control risk changed after approval'
|
||||
)
|
||||
}
|
||||
risk = finalElevatedRisk
|
||||
|
||||
this.perceptions.consume(command.observationId)
|
||||
markInjectionStarted()
|
||||
try {
|
||||
await this.driverCall(
|
||||
(driverSignal) =>
|
||||
this.driver.inject(
|
||||
stored.nativeIdentity,
|
||||
command.action,
|
||||
driverSignal
|
||||
),
|
||||
signal
|
||||
)
|
||||
} catch {
|
||||
throw new ComputerControlFailure(
|
||||
'outcome_unknown',
|
||||
signal.aborted
|
||||
? 'Cancellation raced input injection'
|
||||
: 'Input injection outcome could not be verified'
|
||||
)
|
||||
}
|
||||
if (signal.aborted) {
|
||||
throw new ComputerControlFailure(
|
||||
'outcome_unknown',
|
||||
'Cancellation raced input injection'
|
||||
)
|
||||
}
|
||||
|
||||
await this.writeAudit({
|
||||
...this.auditBase(command, lease),
|
||||
action: command.action.kind,
|
||||
risk,
|
||||
outcome: 'completed',
|
||||
...this.redactedTextMetadata(command.action)
|
||||
})
|
||||
if (risk === 'forbidden') {
|
||||
throw new ComputerControlFailure(
|
||||
'forbidden',
|
||||
'Computer control target is protected'
|
||||
)
|
||||
}
|
||||
return {
|
||||
status: 'completed',
|
||||
commandId: command.commandId,
|
||||
risk
|
||||
}
|
||||
}
|
||||
|
||||
private async requestApproval(
|
||||
command: Extract<ComputerControlRuntimeCommand, { kind: 'act' }>,
|
||||
lease: ComputerControlLease,
|
||||
element: DriverElement,
|
||||
risk: 'input' | 'commit',
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
const approvalId = this.createId()
|
||||
const response = await runWithDeadline(
|
||||
(approvalSignal) =>
|
||||
this.approval.request(
|
||||
{
|
||||
approvalId,
|
||||
leaseId: lease.leaseId,
|
||||
commandId: command.commandId,
|
||||
risk,
|
||||
action:
|
||||
command.action.kind === 'scroll'
|
||||
? 'activate'
|
||||
: command.action.kind,
|
||||
targetName:
|
||||
this.boundedDisplayText(element.name, 256) ||
|
||||
element.role,
|
||||
...(command.action.kind === 'replace_text'
|
||||
? { textLength: command.action.text.length }
|
||||
: {})
|
||||
},
|
||||
approvalSignal
|
||||
),
|
||||
signal,
|
||||
this.approvalDeadlineMs,
|
||||
'approval_timeout'
|
||||
)
|
||||
const parsed = computerControlApprovalResultSchema.safeParse(response)
|
||||
if (!parsed.success || parsed.data.approvalId !== approvalId) {
|
||||
throw new ComputerControlFailure(
|
||||
'approval_denied',
|
||||
'Computer control approval response was invalid'
|
||||
)
|
||||
}
|
||||
if (parsed.data.decision !== 'approve_once') {
|
||||
throw new ComputerControlFailure(
|
||||
'approval_denied',
|
||||
'Computer control approval was denied'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private async validateTarget(
|
||||
lease: ComputerControlLease,
|
||||
original: DriverElement,
|
||||
nativeIdentity: DriverElement['nativeIdentity'],
|
||||
signal: AbortSignal
|
||||
): Promise<DriverElement> {
|
||||
const foreground = await this.driverCall(
|
||||
(driverSignal) =>
|
||||
this.driver.getForegroundWindow(driverSignal),
|
||||
signal
|
||||
)
|
||||
this.assertWindowMatches(lease, foreground)
|
||||
const resolved = await this.driverCall(
|
||||
(driverSignal) =>
|
||||
this.driver.resolveElement(nativeIdentity, driverSignal),
|
||||
signal
|
||||
)
|
||||
this.assertElementIdentity(original, resolved, nativeIdentity)
|
||||
if (!resolved.enabled) {
|
||||
throw new ComputerControlFailure(
|
||||
'focus_failed',
|
||||
'Computer control element is disabled'
|
||||
)
|
||||
}
|
||||
const focused = await this.driverCall(
|
||||
(driverSignal) =>
|
||||
this.driver.focusElement(nativeIdentity, driverSignal),
|
||||
signal
|
||||
)
|
||||
if (!focused) {
|
||||
throw new ComputerControlFailure(
|
||||
'focus_failed',
|
||||
'Computer control element could not be focused',
|
||||
true
|
||||
)
|
||||
}
|
||||
const verified = await this.driverCall(
|
||||
(driverSignal) =>
|
||||
this.driver.resolveElement(nativeIdentity, driverSignal),
|
||||
signal
|
||||
)
|
||||
this.assertElementIdentity(original, verified, nativeIdentity)
|
||||
if (!verified.focused) {
|
||||
throw new ComputerControlFailure(
|
||||
'focus_failed',
|
||||
'Computer control element focus was not verified',
|
||||
true
|
||||
)
|
||||
}
|
||||
const foregroundAfterFocus = await this.driverCall(
|
||||
(driverSignal) =>
|
||||
this.driver.getForegroundWindow(driverSignal),
|
||||
signal
|
||||
)
|
||||
this.assertWindowMatches(lease, foregroundAfterFocus)
|
||||
return verified
|
||||
}
|
||||
|
||||
private assertElementIdentity(
|
||||
original: DriverElement,
|
||||
resolved: DriverElement | undefined,
|
||||
nativeIdentity: DriverElement['nativeIdentity']
|
||||
): asserts resolved is DriverElement {
|
||||
if (
|
||||
!resolved ||
|
||||
resolved.nativeIdentity !== nativeIdentity ||
|
||||
resolved.role !== original.role ||
|
||||
resolved.name !== original.name
|
||||
) {
|
||||
throw new ComputerControlFailure(
|
||||
'element_identity_changed',
|
||||
'Computer control element identity changed',
|
||||
true
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private assertWindowMatches(
|
||||
lease: ComputerControlLease,
|
||||
window: DriverWindowIdentity
|
||||
): void {
|
||||
if (
|
||||
lease.pid !== window.pid ||
|
||||
lease.processStartTime !== window.processStartTime ||
|
||||
lease.windowIdentity !== window.windowIdentity
|
||||
) {
|
||||
this.revoke(lease.leaseId)
|
||||
throw new ComputerControlFailure(
|
||||
'window_not_foreground',
|
||||
'Leased window is not the foreground window',
|
||||
true
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private classifyRisk(
|
||||
action: ComputerControlAction,
|
||||
element: DriverElement
|
||||
): ComputerControlRisk {
|
||||
const baseline = classifyComputerControlAction(action, element)
|
||||
const additional = this.classify?.(action, element)
|
||||
return maximumRisk(baseline, additional)
|
||||
}
|
||||
|
||||
private driverCall<T>(
|
||||
operation: (signal: AbortSignal) => Promise<T>,
|
||||
signal: AbortSignal
|
||||
): Promise<T> {
|
||||
return runWithDeadline(
|
||||
operation,
|
||||
signal,
|
||||
this.driverDeadlineMs,
|
||||
'driver_timeout'
|
||||
)
|
||||
}
|
||||
|
||||
private serialize<T>(
|
||||
key: string,
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const previous = this.queues.get(key) ?? Promise.resolve()
|
||||
const result = previous.then(operation, operation)
|
||||
const tail = result.then(
|
||||
() => undefined,
|
||||
() => undefined
|
||||
)
|
||||
this.queues.set(key, tail)
|
||||
tail.finally(() => {
|
||||
if (this.queues.get(key) === tail) {
|
||||
this.queues.delete(key)
|
||||
}
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
private auditBase(
|
||||
command: ComputerControlRuntimeCommand,
|
||||
context: BrokerExecutionContext
|
||||
): Pick<
|
||||
ComputerControlAuditEvent,
|
||||
| 'timestamp'
|
||||
| 'taskId'
|
||||
| 'conversationId'
|
||||
| 'leaseId'
|
||||
| 'commandId'
|
||||
> {
|
||||
return {
|
||||
timestamp: this.now(),
|
||||
taskId: context.taskId,
|
||||
conversationId: context.conversationId,
|
||||
leaseId: command.leaseId,
|
||||
commandId: command.commandId
|
||||
}
|
||||
}
|
||||
|
||||
private redactedTextMetadata(
|
||||
action: ComputerControlAction
|
||||
): Pick<
|
||||
ComputerControlAuditEvent,
|
||||
'textLength' | 'textDigest'
|
||||
> {
|
||||
return action.kind === 'replace_text'
|
||||
? {
|
||||
textLength: action.text.length,
|
||||
textDigest: digestComputerControlText(action.text)
|
||||
}
|
||||
: {}
|
||||
}
|
||||
|
||||
private async writeFailureAudit(
|
||||
command: Extract<ComputerControlRuntimeCommand, { kind: 'act' }>,
|
||||
context: BrokerExecutionContext,
|
||||
failure: ComputerControlFailure
|
||||
): Promise<void> {
|
||||
const risk: ComputerControlRisk =
|
||||
failure.code === 'forbidden'
|
||||
? 'forbidden'
|
||||
: command.action.kind === 'replace_text' ||
|
||||
command.action.kind === 'select_option'
|
||||
? 'input'
|
||||
: command.action.kind === 'activate'
|
||||
? 'commit'
|
||||
: 'navigate'
|
||||
await this.writeAudit({
|
||||
...this.auditBase(command, context),
|
||||
action: command.action.kind,
|
||||
risk,
|
||||
outcome:
|
||||
failure.code === 'outcome_unknown'
|
||||
? 'outcome_unknown'
|
||||
: failure.code === 'approval_denied' ||
|
||||
failure.code === 'approval_timeout'
|
||||
? 'denied'
|
||||
: 'failed',
|
||||
errorCode: failure.code,
|
||||
...this.redactedTextMetadata(command.action)
|
||||
})
|
||||
}
|
||||
|
||||
private async writeAudit(event: ComputerControlAuditEvent): Promise<void> {
|
||||
try {
|
||||
await this.audit.write(event)
|
||||
return
|
||||
} catch {
|
||||
// Audit persistence must not alter the command's cached result.
|
||||
}
|
||||
if (!this.fallbackAudit || this.fallbackAudit === this.audit) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
await this.fallbackAudit.write(event)
|
||||
} catch {
|
||||
// The fallback is also best effort and isolated from execution.
|
||||
}
|
||||
}
|
||||
|
||||
private normalizeFailure(error: unknown): ComputerControlFailure {
|
||||
if (error instanceof ComputerControlFailure) {
|
||||
return error
|
||||
}
|
||||
return new ComputerControlFailure(
|
||||
'internal_error',
|
||||
'Computer control failed safely'
|
||||
)
|
||||
}
|
||||
|
||||
private boundedDisplayText(
|
||||
value: string,
|
||||
maximumLength: number
|
||||
): string {
|
||||
return [...value]
|
||||
.filter((character) => {
|
||||
const code = character.charCodeAt(0)
|
||||
return code > 31 && code !== 127
|
||||
})
|
||||
.join('')
|
||||
.trim()
|
||||
.slice(0, maximumLength)
|
||||
}
|
||||
|
||||
private errorResult(
|
||||
commandId: string,
|
||||
failure: ComputerControlFailure
|
||||
): ComputerControlCommandResult {
|
||||
return {
|
||||
status: 'error',
|
||||
commandId,
|
||||
error: failure.toContract()
|
||||
}
|
||||
}
|
||||
|
||||
private async releaseInjectedInput(): Promise<void> {
|
||||
try {
|
||||
await runWithDeadline(
|
||||
(signal) => this.driver.releaseInjectedInput(signal),
|
||||
new AbortController().signal,
|
||||
this.driverDeadlineMs,
|
||||
'driver_timeout'
|
||||
)
|
||||
} catch {
|
||||
// Cancellation cleanup is best effort and must not mask the result.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import type { ComputerControlAuditEvent } from './audit'
|
||||
import { DatabaseComputerControlAuditSink } from './database-audit'
|
||||
|
||||
const temporaryDirectories: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryDirectories.splice(0).map((directory) =>
|
||||
rm(directory, { recursive: true, force: true })
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
async function createDatabase(): Promise<{
|
||||
database: AssistantDatabase
|
||||
databasePath: string
|
||||
}> {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-control-audit-'))
|
||||
temporaryDirectories.push(directory)
|
||||
const databasePath = join(directory, 'assistant.sqlite')
|
||||
const database = new AssistantDatabase(databasePath)
|
||||
database.initialize('C:\\Workspace')
|
||||
return { database, databasePath }
|
||||
}
|
||||
|
||||
const createTask = (database: AssistantDatabase, taskId: string): void => {
|
||||
database.createTask({
|
||||
id: taskId,
|
||||
title: '计算机控制审计',
|
||||
instructions: '验证持久审计',
|
||||
workMode: 'execute'
|
||||
})
|
||||
}
|
||||
|
||||
describe('DatabaseComputerControlAuditSink', () => {
|
||||
it('persists redacted events idempotently across close and reopen', async () => {
|
||||
const { database, databasePath } = await createDatabase()
|
||||
const taskId = '00000000-0000-4000-8000-000000000401'
|
||||
const typedText = 'secret typed content 不得持久化'
|
||||
const textDigest = createHash('sha256')
|
||||
.update(typedText, 'utf8')
|
||||
.digest('hex')
|
||||
createTask(database, taskId)
|
||||
const sink = new DatabaseComputerControlAuditSink(database)
|
||||
const event: ComputerControlAuditEvent = {
|
||||
timestamp: 1_775_000_000_000,
|
||||
taskId,
|
||||
conversationId: 'conversation-control-0001',
|
||||
leaseId: 'lease_control_000000001',
|
||||
commandId: 'command_control_0000001',
|
||||
action: 'replace_text',
|
||||
risk: 'input',
|
||||
outcome: 'completed',
|
||||
textLength: typedText.length,
|
||||
textDigest
|
||||
}
|
||||
|
||||
sink.write(event)
|
||||
sink.write({ ...event, timestamp: event.timestamp + 1 })
|
||||
expect(database.listRecentComputerControlAudit()).toEqual([event])
|
||||
database.close()
|
||||
|
||||
const reopened = new AssistantDatabase(databasePath)
|
||||
reopened.initialize('C:\\Workspace')
|
||||
expect(reopened.listRecentComputerControlAudit(1)).toEqual([event])
|
||||
reopened.close()
|
||||
|
||||
const raw = new DatabaseSync(databasePath)
|
||||
const auditRows = raw
|
||||
.prepare('SELECT * FROM computer_control_actions')
|
||||
.all()
|
||||
const taskEvents = raw
|
||||
.prepare(
|
||||
`SELECT payload_json FROM task_events
|
||||
WHERE task_id = ? AND kind = 'computer_control'`
|
||||
)
|
||||
.all(taskId) as Array<{ payload_json: string }>
|
||||
expect(auditRows).toHaveLength(1)
|
||||
expect(taskEvents).toHaveLength(1)
|
||||
expect(JSON.stringify(auditRows)).not.toContain(typedText)
|
||||
expect(JSON.stringify(taskEvents)).not.toContain(typedText)
|
||||
expect(taskEvents[0]?.payload_json).toContain(textDigest)
|
||||
raw.close()
|
||||
expect((await readFile(databasePath)).toString('utf8')).not.toContain(
|
||||
typedText
|
||||
)
|
||||
})
|
||||
|
||||
it('enforces task foreign keys and cascades task audit deletion', async () => {
|
||||
const { database } = await createDatabase()
|
||||
const taskId = '00000000-0000-4000-8000-000000000402'
|
||||
const base: ComputerControlAuditEvent = {
|
||||
timestamp: 1_775_000_000_000,
|
||||
taskId,
|
||||
conversationId: 'conversation-control-0002',
|
||||
leaseId: 'lease_control_000000002',
|
||||
commandId: 'command_control_0000002',
|
||||
action: 'observe',
|
||||
risk: 'observe',
|
||||
outcome: 'completed'
|
||||
}
|
||||
|
||||
expect(() => database.persistComputerControlAudit(base)).toThrow()
|
||||
createTask(database, taskId)
|
||||
database.persistComputerControlAudit(base)
|
||||
database.clearAssistantData()
|
||||
expect(database.listRecentComputerControlAudit()).toEqual([])
|
||||
database.close()
|
||||
})
|
||||
|
||||
it('rejects invalid audit fields before persistence', async () => {
|
||||
const { database } = await createDatabase()
|
||||
const taskId = '00000000-0000-4000-8000-000000000403'
|
||||
createTask(database, taskId)
|
||||
const valid: ComputerControlAuditEvent = {
|
||||
timestamp: 1_775_000_000_000,
|
||||
taskId,
|
||||
conversationId: 'conversation-control-0003',
|
||||
leaseId: 'lease_control_000000003',
|
||||
commandId: 'command_control_0000003',
|
||||
action: 'observe',
|
||||
risk: 'observe',
|
||||
outcome: 'completed'
|
||||
}
|
||||
const invalidEvents: unknown[] = [
|
||||
{ ...valid, timestamp: Number.NaN },
|
||||
{ ...valid, taskId: '' },
|
||||
{ ...valid, conversationId: 'x'.repeat(129) },
|
||||
{ ...valid, leaseId: 'short' },
|
||||
{ ...valid, action: 'type_secret' },
|
||||
{ ...valid, risk: 'unsafe' },
|
||||
{ ...valid, outcome: 'failed' },
|
||||
{
|
||||
...valid,
|
||||
action: 'replace_text',
|
||||
risk: 'input',
|
||||
textLength: 10,
|
||||
textDigest: 'not-a-sha256'
|
||||
},
|
||||
{ ...valid, textLength: 1, textDigest: 'a'.repeat(64) }
|
||||
]
|
||||
|
||||
for (const invalid of invalidEvents) {
|
||||
expect(() =>
|
||||
database.persistComputerControlAudit(
|
||||
invalid as ComputerControlAuditEvent
|
||||
)
|
||||
).toThrow()
|
||||
}
|
||||
expect(() => database.listRecentComputerControlAudit(0)).toThrow()
|
||||
expect(database.listRecentComputerControlAudit()).toEqual([])
|
||||
database.close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,15 @@
|
||||
import type { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import type {
|
||||
ComputerControlAuditEvent,
|
||||
ComputerControlAuditSink
|
||||
} from './audit'
|
||||
|
||||
export class DatabaseComputerControlAuditSink
|
||||
implements ComputerControlAuditSink
|
||||
{
|
||||
constructor(private readonly database: AssistantDatabase) {}
|
||||
|
||||
write(event: ComputerControlAuditEvent): void {
|
||||
this.database.persistComputerControlAudit(event)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { cancellationFailure, ComputerControlFailure } from './errors'
|
||||
|
||||
export const runWithDeadline = async <T>(
|
||||
operation: (signal: AbortSignal) => Promise<T>,
|
||||
parentSignal: AbortSignal,
|
||||
deadlineMs: number,
|
||||
timeoutCode: 'driver_timeout' | 'approval_timeout'
|
||||
): Promise<T> => {
|
||||
if (parentSignal.aborted) {
|
||||
throw cancellationFailure()
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
let timedOut = false
|
||||
const cancel = (): void => {
|
||||
controller.abort(parentSignal.reason)
|
||||
}
|
||||
parentSignal.addEventListener('abort', cancel, { once: true })
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true
|
||||
controller.abort(new Error('Computer control deadline exceeded'))
|
||||
}, deadlineMs)
|
||||
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation(controller.signal),
|
||||
new Promise<never>((_resolve, reject) => {
|
||||
controller.signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
reject(
|
||||
timedOut
|
||||
? new ComputerControlFailure(
|
||||
timeoutCode,
|
||||
'Computer control request timed out',
|
||||
true
|
||||
)
|
||||
: cancellationFailure()
|
||||
)
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
})
|
||||
])
|
||||
} finally {
|
||||
clearTimeout(timeout)
|
||||
parentSignal.removeEventListener('abort', cancel)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import type {
|
||||
ComputerControlAction,
|
||||
ComputerControlElementRole
|
||||
} from '../../shared/computer-control-contracts'
|
||||
|
||||
export const COMPUTER_CONTROL_DRIVER_DEADLINE_MS = 2_000
|
||||
|
||||
export type NativeElementIdentity = object
|
||||
|
||||
export type ComputerControlTargetKind =
|
||||
| 'standard'
|
||||
| 'protected'
|
||||
| 'password'
|
||||
| 'otp'
|
||||
| 'payment'
|
||||
| 'account_security'
|
||||
| 'privilege'
|
||||
| 'os_permission'
|
||||
|
||||
export type DriverWindowIdentity = {
|
||||
pid: number
|
||||
processStartTime: number
|
||||
windowIdentity: string
|
||||
}
|
||||
|
||||
export type DriverElement = {
|
||||
nativeIdentity: NativeElementIdentity
|
||||
role: ComputerControlElementRole
|
||||
name: string
|
||||
enabled: boolean
|
||||
focused: boolean
|
||||
targetKind: ComputerControlTargetKind
|
||||
}
|
||||
|
||||
export type DriverObservation = {
|
||||
window: DriverWindowIdentity
|
||||
windowTitle: string
|
||||
elements: DriverElement[]
|
||||
}
|
||||
|
||||
export interface ComputerControlDriver {
|
||||
readonly available: boolean
|
||||
observe(signal: AbortSignal): Promise<DriverObservation>
|
||||
getForegroundWindow(signal: AbortSignal): Promise<DriverWindowIdentity>
|
||||
resolveElement(
|
||||
nativeIdentity: NativeElementIdentity,
|
||||
signal: AbortSignal
|
||||
): Promise<DriverElement | undefined>
|
||||
focusElement(
|
||||
nativeIdentity: NativeElementIdentity,
|
||||
signal: AbortSignal
|
||||
): Promise<boolean>
|
||||
inject(
|
||||
nativeIdentity: NativeElementIdentity,
|
||||
action: ComputerControlAction,
|
||||
signal: AbortSignal
|
||||
): Promise<void>
|
||||
releaseInjectedInput(signal: AbortSignal): Promise<void>
|
||||
}
|
||||
|
||||
export class UnavailableComputerControlDriver
|
||||
implements ComputerControlDriver
|
||||
{
|
||||
readonly available = false
|
||||
|
||||
async observe(): Promise<DriverObservation> {
|
||||
throw new Error('Computer control driver is unavailable')
|
||||
}
|
||||
|
||||
async getForegroundWindow(): Promise<DriverWindowIdentity> {
|
||||
throw new Error('Computer control driver is unavailable')
|
||||
}
|
||||
|
||||
async resolveElement(): Promise<DriverElement | undefined> {
|
||||
throw new Error('Computer control driver is unavailable')
|
||||
}
|
||||
|
||||
async focusElement(): Promise<boolean> {
|
||||
throw new Error('Computer control driver is unavailable')
|
||||
}
|
||||
|
||||
async inject(): Promise<void> {
|
||||
throw new Error('Computer control driver is unavailable')
|
||||
}
|
||||
|
||||
async releaseInjectedInput(): Promise<void> {
|
||||
// There cannot be injected input when no driver exists.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import type {
|
||||
ComputerControlError,
|
||||
ComputerControlErrorCode
|
||||
} from '../../shared/computer-control-contracts'
|
||||
|
||||
export class ComputerControlFailure extends Error {
|
||||
readonly code: ComputerControlErrorCode
|
||||
readonly retryable: boolean
|
||||
|
||||
constructor(
|
||||
code: ComputerControlErrorCode,
|
||||
message: string,
|
||||
retryable = false
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'ComputerControlFailure'
|
||||
this.code = code
|
||||
this.retryable = retryable
|
||||
}
|
||||
|
||||
toContract(): ComputerControlError {
|
||||
return {
|
||||
code: this.code,
|
||||
message: this.message.slice(0, 256) || 'Computer control failed',
|
||||
retryable: this.retryable
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const cancellationFailure = (): ComputerControlFailure =>
|
||||
new ComputerControlFailure('cancelled', 'Computer control was cancelled')
|
||||
@@ -0,0 +1,142 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
COMPUTER_CONTROL_LEASE_ABSOLUTE_MS,
|
||||
COMPUTER_CONTROL_LEASE_IDLE_MS,
|
||||
ComputerControlLeaseStore,
|
||||
type ComputerControlLeaseBinding
|
||||
} from './lease-store'
|
||||
|
||||
const binding: ComputerControlLeaseBinding = {
|
||||
taskId: 'task-1',
|
||||
conversationId: 'conversation-1',
|
||||
pid: 42,
|
||||
processStartTime: 100,
|
||||
windowIdentity: 'window-1'
|
||||
}
|
||||
|
||||
describe('ComputerControlLeaseStore', () => {
|
||||
it('enforces idle and absolute expiry', () => {
|
||||
let now = 1_000
|
||||
let sequence = 0
|
||||
const store = new ComputerControlLeaseStore(
|
||||
() => now,
|
||||
() => `lease_identifier_${++sequence}`
|
||||
)
|
||||
const idleLease = store.create(binding)
|
||||
now += COMPUTER_CONTROL_LEASE_IDLE_MS
|
||||
expect(() =>
|
||||
store.validate(idleLease.leaseId, binding)
|
||||
).toThrow('expired')
|
||||
|
||||
now = 1_000
|
||||
const absoluteLease = store.create(binding)
|
||||
for (let elapsed = 60_000; elapsed < COMPUTER_CONTROL_LEASE_ABSOLUTE_MS; elapsed += 60_000) {
|
||||
now = 1_000 + elapsed
|
||||
if (elapsed < COMPUTER_CONTROL_LEASE_ABSOLUTE_MS) {
|
||||
store.validate(absoluteLease.leaseId, binding)
|
||||
}
|
||||
}
|
||||
now = 1_000 + COMPUTER_CONTROL_LEASE_ABSOLUTE_MS
|
||||
expect(() =>
|
||||
store.validate(absoluteLease.leaseId, binding)
|
||||
).toThrow('expired')
|
||||
})
|
||||
|
||||
it('revokes and fails on task, conversation, PID, process, or window mismatch', () => {
|
||||
const variants: ComputerControlLeaseBinding[] = [
|
||||
{ ...binding, taskId: 'task-2' },
|
||||
{ ...binding, conversationId: 'conversation-2' },
|
||||
{ ...binding, pid: 43 },
|
||||
{ ...binding, processStartTime: 101 },
|
||||
{ ...binding, windowIdentity: 'window-2' }
|
||||
]
|
||||
let sequence = 0
|
||||
const store = new ComputerControlLeaseStore(
|
||||
() => 1_000,
|
||||
() => `lease_identifier_${++sequence}`
|
||||
)
|
||||
|
||||
for (const mismatch of variants) {
|
||||
const lease = store.create(binding)
|
||||
expect(() =>
|
||||
store.validate(lease.leaseId, mismatch)
|
||||
).toThrow('binding changed')
|
||||
expect(store.peek(lease.leaseId)).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('supports explicit lease and task revocation', () => {
|
||||
let sequence = 0
|
||||
const store = new ComputerControlLeaseStore(
|
||||
() => 1_000,
|
||||
() => `lease_identifier_${++sequence}`
|
||||
)
|
||||
const first = store.create(binding)
|
||||
const second = store.create({ ...binding, windowIdentity: 'window-2' })
|
||||
const other = store.create({ ...binding, taskId: 'task-2' })
|
||||
|
||||
store.revoke(first.leaseId)
|
||||
expect(store.peek(first.leaseId)).toBeUndefined()
|
||||
store.revokeTask(binding.taskId)
|
||||
expect(store.peek(second.leaseId)).toBeUndefined()
|
||||
expect(store.peek(other.leaseId)).toBeDefined()
|
||||
})
|
||||
|
||||
it('prunes all expired leases during create and validate', () => {
|
||||
let now = 1_000
|
||||
let sequence = 0
|
||||
const store = new ComputerControlLeaseStore(
|
||||
() => now,
|
||||
() => `lease_identifier_${++sequence}`
|
||||
)
|
||||
const expiredBeforeCreate = store.create(binding)
|
||||
now += COMPUTER_CONTROL_LEASE_IDLE_MS
|
||||
const current = store.create(binding)
|
||||
expect(store.peek(expiredBeforeCreate.leaseId)).toBeUndefined()
|
||||
|
||||
const expiresBeforeValidate = store.create({
|
||||
...binding,
|
||||
taskId: 'task-2'
|
||||
})
|
||||
now += COMPUTER_CONTROL_LEASE_IDLE_MS - 1
|
||||
store.validate(current.leaseId, binding)
|
||||
now += 1
|
||||
const newest = store.create({
|
||||
...binding,
|
||||
taskId: 'task-3'
|
||||
})
|
||||
store.validate(newest.leaseId, {
|
||||
...binding,
|
||||
taskId: 'task-3'
|
||||
})
|
||||
expect(store.peek(expiresBeforeValidate.leaseId)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('enforces a validated hard lease capacity by evicting oldest entries', () => {
|
||||
let sequence = 0
|
||||
const store = new ComputerControlLeaseStore(
|
||||
() => 1_000,
|
||||
() => `lease_identifier_${++sequence}`,
|
||||
{ maximumLeases: 2 }
|
||||
)
|
||||
const first = store.create(binding)
|
||||
const second = store.create({
|
||||
...binding,
|
||||
windowIdentity: 'window-2'
|
||||
})
|
||||
const third = store.create({
|
||||
...binding,
|
||||
windowIdentity: 'window-3'
|
||||
})
|
||||
|
||||
expect(store.peek(first.leaseId)).toBeUndefined()
|
||||
expect(store.peek(second.leaseId)).toBeDefined()
|
||||
expect(store.peek(third.leaseId)).toBeDefined()
|
||||
expect(
|
||||
() =>
|
||||
new ComputerControlLeaseStore(Date.now, undefined, {
|
||||
maximumLeases: 0
|
||||
})
|
||||
).toThrow('capacity')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,145 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { ComputerControlFailure } from './errors'
|
||||
import type { DriverWindowIdentity } from './driver'
|
||||
|
||||
export const COMPUTER_CONTROL_LEASE_IDLE_MS = 5 * 60 * 1_000
|
||||
export const COMPUTER_CONTROL_LEASE_ABSOLUTE_MS = 10 * 60 * 1_000
|
||||
export const COMPUTER_CONTROL_MAXIMUM_LEASES = 1_000
|
||||
|
||||
export type ComputerControlLeaseBinding = DriverWindowIdentity & {
|
||||
taskId: string
|
||||
conversationId: string
|
||||
}
|
||||
|
||||
export type ComputerControlLease = ComputerControlLeaseBinding & {
|
||||
leaseId: string
|
||||
createdAt: number
|
||||
lastUsedAt: number
|
||||
}
|
||||
|
||||
export type ComputerControlLeaseStoreOptions = {
|
||||
maximumLeases?: number
|
||||
}
|
||||
|
||||
export class ComputerControlLeaseStore {
|
||||
private readonly leases = new Map<string, ComputerControlLease>()
|
||||
private readonly maximumLeases: number
|
||||
|
||||
constructor(
|
||||
private readonly now: () => number = Date.now,
|
||||
private readonly createId: () => string = randomUUID,
|
||||
options: ComputerControlLeaseStoreOptions = {}
|
||||
) {
|
||||
this.maximumLeases =
|
||||
options.maximumLeases ?? COMPUTER_CONTROL_MAXIMUM_LEASES
|
||||
if (
|
||||
!Number.isSafeInteger(this.maximumLeases) ||
|
||||
this.maximumLeases < 1 ||
|
||||
this.maximumLeases > 10_000
|
||||
) {
|
||||
throw new Error('Invalid computer control lease capacity')
|
||||
}
|
||||
}
|
||||
|
||||
create(binding: ComputerControlLeaseBinding): ComputerControlLease {
|
||||
const timestamp = this.now()
|
||||
this.pruneExpired(timestamp)
|
||||
while (this.leases.size >= this.maximumLeases) {
|
||||
const oldest = this.leases.keys().next().value
|
||||
if (oldest === undefined) {
|
||||
break
|
||||
}
|
||||
this.leases.delete(oldest)
|
||||
}
|
||||
const lease: ComputerControlLease = {
|
||||
...binding,
|
||||
leaseId: this.createId(),
|
||||
createdAt: timestamp,
|
||||
lastUsedAt: timestamp
|
||||
}
|
||||
this.leases.set(lease.leaseId, lease)
|
||||
return { ...lease }
|
||||
}
|
||||
|
||||
peek(leaseId: string): ComputerControlLease | undefined {
|
||||
const lease = this.leases.get(leaseId)
|
||||
return lease ? { ...lease } : undefined
|
||||
}
|
||||
|
||||
validate(
|
||||
leaseId: string,
|
||||
binding: ComputerControlLeaseBinding
|
||||
): ComputerControlLease {
|
||||
const timestamp = this.now()
|
||||
const requestedLeaseExpired = this.isExpired(
|
||||
this.leases.get(leaseId),
|
||||
timestamp
|
||||
)
|
||||
this.pruneExpired(timestamp)
|
||||
const lease = this.leases.get(leaseId)
|
||||
if (!lease) {
|
||||
if (requestedLeaseExpired) {
|
||||
throw new ComputerControlFailure(
|
||||
'lease_expired',
|
||||
'Computer control lease expired'
|
||||
)
|
||||
}
|
||||
throw new ComputerControlFailure(
|
||||
'lease_not_found',
|
||||
'Computer control lease was not found'
|
||||
)
|
||||
}
|
||||
|
||||
if (
|
||||
lease.taskId !== binding.taskId ||
|
||||
lease.conversationId !== binding.conversationId ||
|
||||
lease.pid !== binding.pid ||
|
||||
lease.processStartTime !== binding.processStartTime ||
|
||||
lease.windowIdentity !== binding.windowIdentity
|
||||
) {
|
||||
this.leases.delete(leaseId)
|
||||
throw new ComputerControlFailure(
|
||||
'lease_mismatch',
|
||||
'Computer control lease binding changed'
|
||||
)
|
||||
}
|
||||
|
||||
lease.lastUsedAt = timestamp
|
||||
return { ...lease }
|
||||
}
|
||||
|
||||
revoke(leaseId: string): void {
|
||||
this.leases.delete(leaseId)
|
||||
}
|
||||
|
||||
revokeTask(taskId: string): string[] {
|
||||
const revoked: string[] = []
|
||||
for (const [leaseId, lease] of this.leases) {
|
||||
if (lease.taskId === taskId) {
|
||||
this.leases.delete(leaseId)
|
||||
revoked.push(leaseId)
|
||||
}
|
||||
}
|
||||
return revoked
|
||||
}
|
||||
|
||||
private pruneExpired(timestamp: number): void {
|
||||
for (const [leaseId, lease] of this.leases) {
|
||||
if (this.isExpired(lease, timestamp)) {
|
||||
this.leases.delete(leaseId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private isExpired(
|
||||
lease: ComputerControlLease | undefined,
|
||||
timestamp: number
|
||||
): boolean {
|
||||
return (
|
||||
lease !== undefined &&
|
||||
(timestamp - lease.lastUsedAt >= COMPUTER_CONTROL_LEASE_IDLE_MS ||
|
||||
timestamp - lease.createdAt >=
|
||||
COMPUTER_CONTROL_LEASE_ABSOLUTE_MS)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import {
|
||||
createHmac,
|
||||
randomBytes,
|
||||
randomUUID
|
||||
} from 'node:crypto'
|
||||
import type {
|
||||
ComputerControlObservation,
|
||||
ComputerControlRisk
|
||||
} from '../../shared/computer-control-contracts'
|
||||
import type {
|
||||
DriverElement,
|
||||
DriverObservation,
|
||||
NativeElementIdentity
|
||||
} from './driver'
|
||||
import { ComputerControlFailure } from './errors'
|
||||
|
||||
export const COMPUTER_CONTROL_OBSERVATION_FRESH_MS = 3_000
|
||||
export const COMPUTER_CONTROL_MAX_ELEMENTS = 200
|
||||
|
||||
type StoredElement = {
|
||||
nativeIdentity: NativeElementIdentity
|
||||
driverElement: DriverElement
|
||||
risk: ComputerControlRisk
|
||||
}
|
||||
|
||||
type StoredObservation = {
|
||||
contract: ComputerControlObservation
|
||||
elements: Map<string, StoredElement>
|
||||
consumed: boolean
|
||||
}
|
||||
|
||||
const boundedDisplayText = (
|
||||
value: string,
|
||||
maximumLength: number,
|
||||
fallback = ''
|
||||
): string => {
|
||||
const clean = [...value]
|
||||
.filter((character) => {
|
||||
const code = character.charCodeAt(0)
|
||||
return code > 31 && code !== 127
|
||||
})
|
||||
.join('')
|
||||
.trim()
|
||||
.slice(0, maximumLength)
|
||||
return clean || fallback
|
||||
}
|
||||
|
||||
export class ComputerControlPerceptionStore {
|
||||
private readonly observations = new Map<string, StoredObservation>()
|
||||
private readonly revisions = new Map<string, number>()
|
||||
|
||||
constructor(
|
||||
private readonly now: () => number = Date.now,
|
||||
private readonly createId: () => string = randomUUID,
|
||||
private readonly hmacKey: Buffer = randomBytes(32)
|
||||
) {}
|
||||
|
||||
create(
|
||||
leaseId: string,
|
||||
observation: DriverObservation,
|
||||
classify: (element: DriverElement) => ComputerControlRisk
|
||||
): ComputerControlObservation {
|
||||
if (observation.elements.length > COMPUTER_CONTROL_MAX_ELEMENTS) {
|
||||
throw new ComputerControlFailure(
|
||||
'internal_error',
|
||||
'Driver observation exceeded the element limit'
|
||||
)
|
||||
}
|
||||
|
||||
const revision = (this.revisions.get(leaseId) ?? 0) + 1
|
||||
this.revisions.set(leaseId, revision)
|
||||
this.deleteLeaseObservations(leaseId)
|
||||
|
||||
const observationId = this.createId()
|
||||
const storedElements = new Map<string, StoredElement>()
|
||||
const elements = observation.elements.map((element, index) => {
|
||||
const risk = classify(element)
|
||||
const ref = createHmac('sha256', this.hmacKey)
|
||||
.update(`${observationId}\u0000${revision}\u0000${index}`)
|
||||
.digest('base64url')
|
||||
storedElements.set(ref, {
|
||||
nativeIdentity: element.nativeIdentity,
|
||||
driverElement: element,
|
||||
risk
|
||||
})
|
||||
return {
|
||||
ref,
|
||||
role: element.role,
|
||||
name: boundedDisplayText(element.name, 256, element.role),
|
||||
enabled: element.enabled,
|
||||
focused: element.focused,
|
||||
risk,
|
||||
blocked: risk === 'forbidden'
|
||||
}
|
||||
})
|
||||
|
||||
const contract: ComputerControlObservation = {
|
||||
observationId,
|
||||
leaseId,
|
||||
revision,
|
||||
capturedAt: this.now(),
|
||||
windowTitle: boundedDisplayText(
|
||||
observation.windowTitle,
|
||||
256
|
||||
),
|
||||
elements
|
||||
}
|
||||
this.observations.set(observationId, {
|
||||
contract,
|
||||
elements: storedElements,
|
||||
consumed: false
|
||||
})
|
||||
return structuredClone(contract)
|
||||
}
|
||||
|
||||
resolve(
|
||||
leaseId: string,
|
||||
observationId: string,
|
||||
revision: number,
|
||||
elementRef: string
|
||||
): StoredElement {
|
||||
const observation = this.observations.get(observationId)
|
||||
if (!observation || observation.contract.leaseId !== leaseId) {
|
||||
throw new ComputerControlFailure(
|
||||
'observation_not_found',
|
||||
'Computer control observation was not found'
|
||||
)
|
||||
}
|
||||
if (observation.contract.revision !== revision) {
|
||||
throw new ComputerControlFailure(
|
||||
'observation_stale',
|
||||
'Computer control observation revision changed',
|
||||
true
|
||||
)
|
||||
}
|
||||
if (
|
||||
this.now() - observation.contract.capturedAt >=
|
||||
COMPUTER_CONTROL_OBSERVATION_FRESH_MS
|
||||
) {
|
||||
throw new ComputerControlFailure(
|
||||
'observation_stale',
|
||||
'Computer control observation is stale',
|
||||
true
|
||||
)
|
||||
}
|
||||
if (observation.consumed) {
|
||||
throw new ComputerControlFailure(
|
||||
'observation_consumed',
|
||||
'Computer control observation was already used'
|
||||
)
|
||||
}
|
||||
const element = observation.elements.get(elementRef)
|
||||
if (!element) {
|
||||
throw new ComputerControlFailure(
|
||||
'element_not_found',
|
||||
'Computer control element was not found'
|
||||
)
|
||||
}
|
||||
return element
|
||||
}
|
||||
|
||||
consume(observationId: string): void {
|
||||
const observation = this.observations.get(observationId)
|
||||
if (!observation) {
|
||||
throw new ComputerControlFailure(
|
||||
'observation_not_found',
|
||||
'Computer control observation was not found'
|
||||
)
|
||||
}
|
||||
if (observation.consumed) {
|
||||
throw new ComputerControlFailure(
|
||||
'observation_consumed',
|
||||
'Computer control observation was already used'
|
||||
)
|
||||
}
|
||||
observation.consumed = true
|
||||
}
|
||||
|
||||
revokeLease(leaseId: string): void {
|
||||
this.deleteLeaseObservations(leaseId)
|
||||
this.revisions.delete(leaseId)
|
||||
}
|
||||
|
||||
private deleteLeaseObservations(leaseId: string): void {
|
||||
for (const [observationId, observation] of this.observations) {
|
||||
if (observation.contract.leaseId === leaseId) {
|
||||
this.observations.delete(observationId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { DriverElement } from './driver'
|
||||
import {
|
||||
classifyComputerControlAction,
|
||||
maximumRisk
|
||||
} from './risk-policy'
|
||||
|
||||
const identity = {}
|
||||
const element: DriverElement = {
|
||||
nativeIdentity: identity,
|
||||
role: 'link',
|
||||
name: 'Next page',
|
||||
enabled: true,
|
||||
focused: true,
|
||||
targetKind: 'standard'
|
||||
}
|
||||
|
||||
describe('computer control risk policy', () => {
|
||||
it('classifies semantic actions and lets classification only raise risk', () => {
|
||||
expect(
|
||||
classifyComputerControlAction(
|
||||
{ kind: 'activate', elementRef: 'opaque_identifier_123456' },
|
||||
element
|
||||
)
|
||||
).toBe('navigate')
|
||||
expect(
|
||||
maximumRisk('commit', 'observe')
|
||||
).toBe('commit')
|
||||
expect(maximumRisk('input', 'commit')).toBe('commit')
|
||||
})
|
||||
|
||||
it.each([
|
||||
'protected',
|
||||
'password',
|
||||
'otp',
|
||||
'payment',
|
||||
'account_security',
|
||||
'privilege',
|
||||
'os_permission'
|
||||
] as const)('forbids %s targets', (targetKind) => {
|
||||
expect(
|
||||
classifyComputerControlAction(
|
||||
{ kind: 'activate', elementRef: 'opaque_identifier_123456' },
|
||||
{ ...element, targetKind }
|
||||
)
|
||||
).toBe('forbidden')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,79 @@
|
||||
import type {
|
||||
ComputerControlAction,
|
||||
ComputerControlElementRole,
|
||||
ComputerControlRisk
|
||||
} from '../../shared/computer-control-contracts'
|
||||
import type {
|
||||
ComputerControlTargetKind,
|
||||
DriverElement
|
||||
} from './driver'
|
||||
|
||||
const RISK_ORDER: Record<ComputerControlRisk, number> = {
|
||||
observe: 0,
|
||||
navigate: 1,
|
||||
input: 2,
|
||||
commit: 3,
|
||||
forbidden: 4
|
||||
}
|
||||
|
||||
const FORBIDDEN_TARGETS: ReadonlySet<ComputerControlTargetKind> =
|
||||
new Set([
|
||||
'protected',
|
||||
'password',
|
||||
'otp',
|
||||
'payment',
|
||||
'account_security',
|
||||
'privilege',
|
||||
'os_permission'
|
||||
])
|
||||
|
||||
const activationRisk = (
|
||||
role: ComputerControlElementRole
|
||||
): ComputerControlRisk => {
|
||||
switch (role) {
|
||||
case 'button':
|
||||
case 'checkbox':
|
||||
case 'radio':
|
||||
return 'commit'
|
||||
case 'textbox':
|
||||
case 'combobox':
|
||||
case 'scrollarea':
|
||||
return 'observe'
|
||||
default:
|
||||
return 'navigate'
|
||||
}
|
||||
}
|
||||
|
||||
export const maximumRisk = (
|
||||
baseline: ComputerControlRisk,
|
||||
classification?: ComputerControlRisk
|
||||
): ComputerControlRisk =>
|
||||
classification &&
|
||||
RISK_ORDER[classification] > RISK_ORDER[baseline]
|
||||
? classification
|
||||
: baseline
|
||||
|
||||
export const classifyComputerControlAction = (
|
||||
action: ComputerControlAction,
|
||||
element: DriverElement,
|
||||
driverClassification?: ComputerControlRisk
|
||||
): ComputerControlRisk => {
|
||||
if (FORBIDDEN_TARGETS.has(element.targetKind)) {
|
||||
return 'forbidden'
|
||||
}
|
||||
|
||||
let baseline: ComputerControlRisk
|
||||
switch (action.kind) {
|
||||
case 'replace_text':
|
||||
case 'select_option':
|
||||
baseline = 'input'
|
||||
break
|
||||
case 'scroll':
|
||||
baseline = 'navigate'
|
||||
break
|
||||
case 'activate':
|
||||
baseline = activationRisk(element.role)
|
||||
break
|
||||
}
|
||||
return maximumRisk(baseline, driverClassification)
|
||||
}
|
||||
Reference in New Issue
Block a user