clash-verge-rev/scripts/perf/run.mjs

import { spawn, execFileSync } from 'node:child_process'
import { createHash, randomUUID } from 'node:crypto'
import {
  mkdirSync,
  readFileSync,
  writeFileSync,
  openSync,
  closeSync,
  realpathSync,
} from 'node:fs'
import { createServer } from 'node:net'
import { resolve, dirname } from 'node:path'
import { setTimeout as sleep } from 'node:timers/promises'
import { parseArgs } from 'node:util'

import { createTauriCapabilities } from '@wdio/tauri-service'
import { remote } from 'webdriverio'
import { WebSocketServer } from 'ws'

import {
  delta,
  summarize,
  markdown,
  phases,
  roles,
  validStats,
  validWorkload,
  replayValues,
  SCHEMA,
  validObservation,
  validVisibility,
} from './report.mjs'

const { values } = parseArgs({
  args: process.argv.slice(2).filter((v) => v !== '--'),
  options: {
    output: { type: 'string', default: `target/perf/runs/${Date.now()}` },
    binary: { type: 'string', default: 'target/perf/build/clash-verge' },
    repeat: { type: 'string', default: '3' },
    seconds: { type: 'string', default: '20' },
    warmup: { type: 'string', default: '10' },
    'fail-phase': { type: 'string' },
  },
})
const seconds = Number(values.seconds),
  warmup = Number(values.warmup),
  repeat = Number(values.repeat)
const replayMs = 200
if (process.platform !== 'darwin') throw new Error('macOS only')
function readConsole() {
  const consoleState = JSON.parse(
    execFileSync('/usr/bin/plutil', ['-convert', 'json', '-o', '-', '-'], {
      input: execFileSync('/usr/sbin/ioreg', ['-n', 'Root', '-d1', '-a']),
      encoding: 'utf8',
    }),
  )
  const locked = consoleState.IOConsoleLocked
  if (typeof locked !== 'boolean')
    throw new Error('Environment unavailable: unknown console lock state')
  return locked
}
function checkConsole() {
  const locked = readConsole()
  if (locked)
    throw new Error(
      'Environment unavailable: an unlocked graphical session is required; window focus is not required',
    )
  return locked
}
const consoleLocked = checkConsole()
if (
  ![seconds, warmup, repeat].every(Number.isFinite) ||
  seconds < 5 ||
  warmup < 2 ||
  repeat < 1 ||
  !Number.isInteger(repeat)
)
  throw new Error('Require seconds >= 5, warmup >= 2, integer repeat >= 1')
const phaseTargets = phases.map(
  (_, i) =>
    Math.ceil(((warmup + 5) * 1000) / replayMs) +
    i * Math.ceil(((seconds + 2) * 1000) / replayMs),
)
if ((phaseTargets[2] * replayMs) / 1000 + seconds + 5 >= 600)
  throw new Error(
    'Total replay including transition allowance must be < 10 minutes',
  )
const binary = realpathSync(values.binary),
  output = resolve(values.output)
const webcontentBinary = realpathSync(
  '/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.WebContent.xpc/Contents/MacOS/com.apple.WebKit.WebContent',
)
const build = JSON.parse(readFileSync(`${binary}.json`))
const hash = (data) => createHash('sha256').update(data).digest('hex')
if (hash(readFileSync(binary)) !== build.binarySha256)
  throw new Error('Binary does not match build manifest')
try {
  execFileSync('git', ['check-ignore', '-q', '--no-index', output])
} catch {
  throw new Error('Output directory must be Git ignored')
}
mkdirSync(dirname(output), { recursive: true })
mkdirSync(output)
execFileSync('cc', ['-O2', 'scripts/perf/sample.c', '-o', `${output}/sample`])
const command = (name, args) =>
  execFileSync(name, args, { encoding: 'utf8' }).trim()
const report = {
  schema: SCHEMA,
  build,
  contract: {
    scenario: 'traffic',
    replayMs,
    consoleLocked,
    method: 'mach-rusage-v0-seconds/rss-bytes-v1',
    mode: build.mode,
    buildSettingsSha256: build.buildSettingsSha256,
    buildEnvironment: build.buildEnvironment,
    toolchain: build.toolchain,
    platform: command('sw_vers', []),
    machine: command('sysctl', ['-n', 'hw.model']),
    arch: process.arch,
    host: hash(command('scutil', ['--get', 'LocalHostName'])),
    hardware: command('sysctl', ['hw.ncpu', 'hw.memsize']),
    power: command('pmset', ['-g', 'custom']),
    powerSource: command('pmset', ['-g', 'batt']).split('\n')[0],
    webkit: command('defaults', [
      'read',
      '/System/Library/Frameworks/WebKit.framework/Resources/Info',
      'CFBundleVersion',
    ]),
    seconds,
    warmup,
    phaseTargets,
    transitionSeconds: 2,
    readinessAllowanceSeconds: 5,
    intervalMs: 1000,
    window: [1000, 700],
    windowPolicy: 'nonactivating-top-click-through-v1',
    rangeMinutes: 10,
    replay:
      'deterministic-v1, 5Hz, no prefill, append boundary, default bezier, blur pause false',
    automation:
      'wdio plugin 1.3.0; 20Hz macOS runloop pump plus one WebDriver observation/second',
  },
  runs: [],
}
report.units = {
  time: 'monotonic milliseconds',
  cpu: 'cumulative CPU seconds',
  percent: 'one CPU core = 100%',
  rss: 'resident size bytes',
  start: 'Mach process start ticks',
}
report.monotonicOriginEpochMs = performance.timeOrigin
report.harnessSha256 = hash(
  ['run.mjs', 'report.mjs', 'sample.c']
    .map((p) => readFileSync(`scripts/perf/${p}`))
    .join('\n'),
)
let interrupted = false
const abort = new AbortController()
const pause = (ms) => sleep(ms, undefined, { signal: abort.signal })
for (const signal of ['SIGINT', 'SIGTERM'])
  process.on(signal, () => {
    interrupted = true
    abort.abort()
  })
const assert = (value, reason) => {
  if (!value) throw new Error(reason)
}
const drawn = (s) => s.now >= s.draw && s.now - s.draw < 2500
const marker = (s) => ({
  ...s,
  pixelsSha256: hash(s.pixels),
  pixels: undefined,
  ua: undefined,
  token: undefined,
})
async function freePort() {
  const server = createServer()
  await new Promise((done) => server.listen(0, '127.0.0.1', done))
  const port = server.address().port
  await new Promise((done) => server.close(done))
  return port
}
for (let iteration = 0; iteration < repeat && !interrupted; iteration++) {
  const initialLocked = checkConsole()
  const token = randomUUID(),
    port = await freePort()
  const run = {
    token,
    ok: false,
    samples: [],
    observations: [],
    consoleStates: [
      { name: 'initial', locked: initialLocked, time: Date.now() },
    ],
    marks: [],
    stats: {},
    cleanup: {},
    replay: { sent: 0, maxLateMs: 0 },
  }
  report.runs.push(run)
  const replay = new WebSocketServer({ host: '127.0.0.1', port: 0 })
  await new Promise((done) => replay.on('listening', done))
  let sequence = 0,
    replayStarted,
    timer
  replay.on('connection', () => {
    if (replayStarted !== undefined) {
      run.replay.error = 'unexpected replay reconnection'
      clearTimeout(timer)
      return
    }
    replayStarted = performance.now()
    timer = setTimeout(send, replayMs)
  })
  const deadlineIndex = () =>
    Math.floor((performance.now() - replayStarted) / replayMs)
  function send() {
    const deadline = replayStarted + (sequence + 1) * replayMs
    const late = performance.now() - deadline
    if (late < 0) {
      timer = setTimeout(send, Math.ceil(-late))
      return
    }
    run.replay.maxLateMs = Math.max(run.replay.maxLateMs, late)
    if (late >= replayMs) {
      run.replay.error = `replay source missed a complete ${replayMs} ms period`
      return
    }
    sequence++
    run.replay.sent = sequence
    const timestamp = Date.now()
    const point = JSON.stringify({
      sequence,
      timestamp,
      ...replayValues(sequence),
    })
    for (const socket of replay.clients)
      if (socket.readyState === 1) socket.send(point)
    timer = setTimeout(
      send,
      Math.max(
        0,
        replayStarted + (sequence + 1) * replayMs - performance.now(),
      ),
    )
  }
  const log = openSync(`${output}/${iteration + 1}.log`, 'w')
  const child = spawn(binary, [], {
    env: {
      ...process.env,
      PERF_TOKEN: token,
      PERF_REPLAY_PORT: String(replay.address().port),
      TAURI_WEBDRIVER_PORT: String(port),
    },
    stdio: ['ignore', log, log],
  })
  closeSync(log)
  let browser, childError, previous, mainStart
  const identities = {}
  child.on('error', (error) => {
    childError = error
  })
  const exited = new Promise((done) => child.once('exit', done))
  const alive = () =>
    !childError && child.exitCode === null && child.signalCode === null
  const collect = (pids) =>
    command(`${output}/sample`, pids.map(String))
      .split('\n')
      .map((line) => {
        const [pid, start, cpu, rss, ...path] = line.split(' ')
        return {
          pid: Number(pid),
          start,
          cpu: Number(cpu),
          rss: Number(rss),
          path: path.join(' '),
          time: performance.now(),
        }
      })
  const identify = (role, row) => {
    assert(
      row.path === (role === 'main' ? binary : webcontentBinary),
      `unexpected ${role} process executable: ${row.path}`,
    )
    assert(
      BigInt(row.start) >= BigInt(mainStart),
      'process predates owned application',
    )
    const identity = `${row.pid}/${row.start}`
    assert(
      !identities[role] || identities[role] === identity,
      'process identity changed during scenario',
    )
    identities[role] = identity
  }
  const state = async (action = 'state', pixels = false) =>
    browser.execute(
      async (action, pixels) => {
        const native = await window.__TAURI_INTERNALS__.invoke('perf_state', {
          action,
        })
        const canvas = document.querySelector('canvas')
        return {
          ...native,
          now: Date.now(),
          hidden: document.hidden,
          focused: document.hasFocus(),
          received: Number(document.documentElement.dataset.perfReceived),
          source: Number(document.documentElement.dataset.perfSourceTime),
          gaps: Number(document.documentElement.dataset.perfGaps),
          worker: Number(document.documentElement.dataset.perfWorker),
          points: Number(document.documentElement.dataset.perfPoints),
          lastUp: Number(document.documentElement.dataset.perfLastUp),
          lastDown: Number(document.documentElement.dataset.perfLastDown),
          draw: Number(canvas?.dataset.perfDraw),
          pixels: pixels ? canvas?.toDataURL() : null,
          ua: navigator.userAgent,
        }
      },
      action,
      pixels,
    )
  const checked = async (action, pixels) => {
    assert(!interrupted, 'interrupted')
    assert(!run.replay.error, run.replay.error)
    assert(
      alive(),
      `owned app exited: ${childError ?? child.exitCode ?? child.signalCode}`,
    )
    const value = await state(action, pixels)
    assert(!run.replay.error, run.replay.error)
    run.lastObservation = { ...value, pixels: undefined, ua: undefined }
    assert(
      value.token === token && value.main === child.pid,
      'instance identity mismatch',
    )
    if (!identities.webview) {
      run.initialAttribution = collect([value.webview])[0]
      identify('webview', run.initialAttribution)
    }
    assert(
      value.nativeFocused === false && value.appActive === false,
      'measurement window unexpectedly activated',
    )
    assert(
      value.top === true && value.clickThrough === true,
      'measurement window must be natively topmost and click-through',
    )
    return value
  }
  try {
    mainStart = collect([child.pid])[0].start
    const deadline = Date.now() + 30000
    while (true) {
      assert(
        Date.now() < deadline && !interrupted && alive(),
        'embedded WebDriver readiness deadline or app exit',
      )
      try {
        if (
          (
            await fetch(`http://127.0.0.1:${port}/status`, {
              signal: AbortSignal.timeout(500),
            })
          ).ok
        )
          break
      } catch {}
      await pause(100)
    }
    const listener = command('lsof', [
      '-nP',
      '-a',
      '-p',
      String(child.pid),
      `-iTCP:${port}`,
      '-sTCP:LISTEN',
    ])
    assert(
      listener.includes(`127.0.0.1:${port}`) && !listener.includes(`*:${port}`),
      'driver must belong to this app and bind loopback',
    )
    const capabilities = createTauriCapabilities(binary)
    delete capabilities['wdio:tauriServiceOptions']
    browser = await remote({
      hostname: '127.0.0.1',
      port,
      capabilities,
      logLevel: 'error',
      connectionRetryCount: 0,
      connectionRetryTimeout: 5000,
    })
    const handles = await browser.getWindowHandles()
    assert(handles.length === 1, 'expected exactly one measurement WebView')
    await browser.switchToWindow(handles[0])
    // Replay's parent effect starts the socket after the graph installs its listeners.
    await browser.waitUntil(
      async () => {
        const s = await checked()
        return (
          validWorkload(s) &&
          s.received >= 3 &&
          s.now >= s.worker &&
          s.now - s.worker < 2500
        )
      },
      {
        timeout: 15000,
        interval: 100,
        timeoutMsg: 'traffic effects/Worker not ready before show',
      },
    )
    run.initialData = run.lastObservation
    await checked('show')
    await browser.waitUntil(
      async () => {
        const s = await checked()
        return validObservation({ phase: 'foreground', ...s })
      },
      {
        timeout: 15000,
        interval: 250,
        timeoutMsg: 'traffic data path/page not ready',
      },
    )
    run.userAgent = (await checked()).ua
    run.backgroundLoad = command('sysctl', ['-n', 'vm.loadavg'])
    const viewport = await browser.execute(() => [
      innerWidth,
      innerHeight,
      devicePixelRatio,
    ])
    run.viewport = { dom: viewport, driver: await browser.getWindowRect() }
    const driverScale = run.viewport.driver.width / 1000
    assert(
      viewport[0] === 1000 &&
        viewport[1] > 0 &&
        viewport[1] <= 700 &&
        (driverScale === 1 || driverScale === viewport[2]) &&
        run.viewport.driver.height / driverScale === 700,
      `unexpected window dimensions: ${JSON.stringify(run.viewport)}`,
    )
    assert(
      !report.contract.viewport ||
        JSON.stringify(report.contract.viewport) === JSON.stringify(viewport),
      'viewport changed between runs',
    )
    report.contract.viewport = viewport
    run.marks.push({ name: 'ready', time: Date.now() })
    const readyAt = performance.now()
    assert(
      phaseTargets[0] - (await checked()).received >=
        (warmup * 1000) / replayMs,
      'readiness exceeded fixed replay allowance',
    )
    for (const phase of phases) {
      run.consoleStates.push({
        name: phase,
        locked: readConsole(),
        time: Date.now(),
      })
      const operation = performance.now()
      let s = await checked(phase === 'hidden' ? 'hide' : 'show', true)
      assert(
        s.visible === (phase !== 'hidden'),
        `${phase}: native visibility mismatch`,
      )
      const beforeDraw = s.draw
      await browser.waitUntil(
        async () => {
          s = await checked('state', true)
          return (
            validVisibility({ phase, ...s }) &&
            (phase === 'hidden' || (s.draw > beforeDraw && drawn(s)))
          )
        },
        {
          timeout: 5000,
          interval: 100,
          timeoutMsg: `${phase}: window/Canvas readiness deadline`,
        },
      )
      const operationMs = performance.now() - operation
      const target = phaseTargets[phases.indexOf(phase)]
      await browser.waitUntil(
        async () => {
          s = await checked('state', true)
          assert(
            s.received <= target,
            `${phase}: missed replay anchor ${target}`,
          )
          return s.received === target
        },
        {
          timeout: Math.max(1000, (target - s.received) * replayMs + 1000),
          interval: 50,
          timeoutMsg: `${phase}: replay anchor deadline`,
        },
      )
      assert(
        validObservation({ phase, ...s }),
        `${phase}: invalid traffic observation`,
      )
      const first = s,
        start = performance.now(),
        firstIndex = deadlineIndex(),
        phaseEpoch = Date.now()
      if (phase === 'foreground') {
        run.actualWarmupMs = start - readyAt
        assert(
          run.actualWarmupMs >= warmup * 1000,
          'insufficient actual warmup',
        )
      }
      run.marks.push({
        name: phase,
        time: phaseEpoch,
        operationMs,
        transitionMs: start - operation,
        target,
        ...marker(s),
        deadlineIndex: firstIndex,
        senderSent: run.replay.sent,
      })
      previous = {}
      if (values['fail-phase'] === phase)
        throw new Error(`injected failure at ${phase}`)
      while (performance.now() - start < seconds * 1000) {
        const tick = performance.now()
        s = await checked()
        run.observations.push({ phase, ...run.lastObservation })
        assert(
          validObservation({ phase, ...s }),
          `${phase}: invalid visibility, workload or freshness`,
        )
        assert(
          performance.now() - replayStarted < 600000,
          'replay exceeded raw history window',
        )
        const samples = collect(roles.map((role) => s[role]))
        run.attribution = {
          main: s.main,
          webview: s.webview,
          observed: samples,
        }
        for (let i = 0; i < roles.length; i++) {
          const row = { ...samples[i], phase, role: roles[i] }
          identify(row.role, row)
          row.percent = delta(previous[row.role], row)
          row.cpuDelta =
            row.percent === null ? null : row.cpu - previous[row.role].cpu
          run.samples.push(row)
          previous[row.role] = row
        }
        await pause(Math.max(0, 1000 - (performance.now() - tick)))
      }
      s = await checked('state', true)
      const endIndex = deadlineIndex()
      const expected = endIndex - firstIndex
      assert(
        validObservation({ phase, ...s }) &&
          s.worker > first.worker &&
          Math.abs(s.received - first.received - expected) <= 2,
        `${phase}: unstable or lost workload`,
      )
      if (phase !== 'hidden')
        assert(
          s.draw > first.draw && s.pixels !== first.pixels,
          `${phase}: Canvas did not render fresh data`,
        )
      run.marks.push({
        name: `${phase}:end`,
        time: s.now,
        ...marker(s),
        deadlineIndex: endIndex,
        senderSent: run.replay.sent,
      })
      run.consoleStates.push({
        name: `${phase}:end`,
        locked: readConsole(),
        time: Date.now(),
      })
    }
    run.stats = summarize(run.samples, seconds)
    assert(
      validStats(run.stats),
      'insufficient coverage or process discontinuity',
    )
    run.ok = true
  } catch (error) {
    run.error = String(error)
    process.exitCode = 1
  } finally {
    await browser?.deleteSession().catch(() => {})
    try {
      if (alive() && collect([child.pid])[0].start === mainStart) {
        child.kill('SIGTERM')
        await Promise.race([exited, sleep(5000)])
        if (alive() && collect([child.pid])[0].start === mainStart) {
          child.kill('SIGKILL')
          await Promise.race([exited, sleep(2000)])
        }
      }
    } catch (error) {
      run.cleanup.error = String(error)
    }
    clearTimeout(timer)
    for (const socket of replay.clients) socket.terminate()
    await new Promise((done) => replay.close(done))
    run.cleanup.mainExited =
      child.exitCode !== null || child.signalCode !== null
    if (identities.webview) {
      const [pid, start] = identities.webview.split('/')
      const remains = () => {
        try {
          return collect([pid])[0].start === start
        } catch {
          return false
        }
      }
      const deadline = Date.now() + 5000
      while (remains() && Date.now() < deadline) await sleep(100)
      run.cleanup.webviewExited = !remains()
    }
    try {
      await fetch(`http://127.0.0.1:${port}/status`, {
        signal: AbortSignal.timeout(500),
      })
      run.cleanup.driverClosed = false
    } catch {
      run.cleanup.driverClosed = true
    }
    if (
      !run.cleanup.mainExited ||
      !run.cleanup.driverClosed ||
      run.cleanup.webviewExited === false
    ) {
      run.ok = false
      run.error = `${run.error ?? ''} cleanup failed`
      process.exitCode = 1
    }
    run.stats = summarize(run.samples, seconds)
    writeFileSync(
      `${output}/${iteration + 1}.json`,
      JSON.stringify(run, null, 2),
    )
    const columns = [
      'phase',
      'role',
      'time',
      'pid',
      'start',
      'cpu',
      'percent',
      'rss',
    ]
    writeFileSync(
      `${output}/${iteration + 1}.csv`,
      [
        columns.join(','),
        ...run.samples.map((r) =>
          columns.map((c) => r[c] ?? 'unavailable').join(','),
        ),
      ].join('\n'),
    )
    writeFileSync(`${output}/summary.json`, JSON.stringify(report, null, 2))
    writeFileSync(`${output}/report.md`, markdown(report))
  }
  console.log(`Run ${iteration + 1}: ${run.ok ? 'passed' : run.error}`)
}
console.log(`Reports: ${output}`)