TUI heap dump disk-fill risk and memory guards

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #41948 fix(tui): guard automatic heap dumps against disk fill (#21767)

  merged to main

  files: scripts/release.py, ui-tui/src/entry.tsx, ui-tui/src/lib/memory.test.ts, ui-tui/src/lib/memory.ts, ui-tui/src/lib/memoryMonitor.ts

  Show the diff

  • scripts/release.py

    @@ -65,6 +65,7 @@
         "[email protected]": "HeLLGURD",
         "[email protected]": "synapsesx",
         "[email protected]": "dirtyren",
    +    "[redacted:email]": "alarcritty",
         "[email protected]": "islam666",
         "[email protected]": "lsaether",
         "[email protected]": "JimStenstrom",

  • ui-tui/src/entry.tsx

    @@ -37,7 +37,7 @@ const gw = new GatewayClient()
     gw.start()
     
     const dumpNotice = (snap: MemorySnapshot, dump: HeapDumpResult | null) =>
    -  `hermes-tui: ${snap.level} memory (${formatBytes(snap.heapUsed)}) — auto heap dump → ${dump?.heapPath ?? '(failed)'}\n`
    +  `hermes-tui: ${snap.level} memory (${formatBytes(snap.heapUsed)}) — auto heap dump → ${dump?.heapPath ?? dump?.diagPath ?? '(failed)'}\n`
     
     setupGracefulExit({
       cleanups: [

  • ui-tui/src/lib/memory.test.ts

    @@ -0,0 +1,162 @@
    +import { mkdtempSync, readdirSync, rmSync, statSync, utimesSync, writeFileSync } from 'node:fs'
    +import { tmpdir } from 'node:os'
    +import { join } from 'node:path'
    +
    +import { afterEach, beforeEach, describe, expect, it } from 'vitest'
    +
    +import { performHeapDump } from './memory.js'
    +
    +const ENV_KEYS = ['HERMES_AUTO_HEAPDUMP', 'HERMES_HEAPDUMP_DIR', 'HERMES_HEAPDUMP_MAX_BYTES'] as const
    +
    +describe('performHeapDump auto opt-in gate (#21767)', () => {
    +  let saved: Record<string, string | undefined>
    +  let dir: string
    +
    +  beforeEach(() => {
    +    saved = {}
    +
    +    for (const k of ENV_KEYS) {
    +      saved[k] = process.env[k]
    +      delete process.env[k]
    +    }
    +
    +    dir = mkdtempSync(join(tmpdir(), 'hermes-heapdump-test-'))
    +    process.env.HERMES_HEAPDUMP_DIR = dir
    +  })
    +
    +  afterEach(() => {
    +    for (const k of ENV_KEYS) {
    +      if (saved[k] === undefined) {
    +        delete process.env[k]
    +      } else {
    +        process.env[k] = saved[k]
    +      }
    +    }
    +
    +    rmSync(dir, { force: true, recursive: true })
    +  })
    +
    +  it('writes diagnostics only for auto-high without HERMES_AUTO_HEAPDUMP', async () => {
    +    const result = await performHeapDump('auto-high')
    +
    +    expect(result.success).toBe(true)
    +    expect(result.suppressed).toBe(true)
    +    expect(result.diagPath).toBeDefined()
    +    expect(result.heapPath).toBeUndefined()
    +
    +    const files = readdirSync(dir)
    +    expect(files.some(f => f.endsWith('.diagnostics.json'))).toBe(true)
    +    expect(files.some(f => f.endsWith('.heapsnapshot'))).toBe(false)
    +  })
    +
    +  it('writes diagnostics only for auto-critical without HERMES_AUTO_HEAPDUMP', async () => {
    +    const result = await performHeapDump('auto-critical')
    +
    +    expect(result.success).toBe(true)
    +    expect(result.suppressed).toBe(true)
    +    expect(result.heapPath).toBeUndefined()
    +
    +    const files = readdirSync(dir)
    +    expect(files.some(f => f.endsWith('.heapsnapshot'))).toBe(false)
    +  })
    +
    +  it('writes both diagnostics and snapshot for auto-high when HERMES_AUTO_HEAPDUMP=1', async () => {
    +    process.env.HERMES_AUTO_HEAPDUMP = '1'
    +
    +    const result = await performHeapDump('auto-high')
    +
    +    expect(result.success).toBe(true)
    +    expect(result.suppressed).toBeUndefined()
    +    expect(result.diagPath).toBeDefined()
    +    expect(result.heapPath).toBeDefined()
    +
    +    const files = readdirSync(dir)
    +    expect(files.some(f => f.endsWith('.heapsnapshot'))).toBe(true)
    +  })
    +
    +  it('accepts truthy spellings (true|yes|on, case-insensitive) as opt-in', async () => {
    +    for (const value of ['true', 'YES', 'On']) {
    +      process.env.HERMES_AUTO_HEAPDUMP = value
    +      const result = await performHeapDump('auto-high')
    +
    +      expect(result.success).toBe(true)
    +      expect(result.heapPath).toBeDefined()
    +    }
    +  })
    +
    +  it('treats other values (0, off, garbage) as opt-out for auto triggers', async () => {
    +    for (const value of ['0', 'off', 'nope']) {
    +      process.env.HERMES_AUTO_HEAPDUMP = value
    +      const result = await performHeapDump('auto-high')
    +
    +      expect(result.success).toBe(true)
    +      expect(result.suppressed).toBe(true)
    +      expect(result.heapPath).toBeUndefined()
    +    }
    +  })
    +
    +  it('writes both for manual triggers regardless of HERMES_AUTO_HEAPDUMP', async () => {
    +    const result = await performHeapDump('manual')
    +
    +    expect(result.success).toBe(true)
    +    expect(result.suppressed).toBeUndefined()
    +    expect(result.heapPath).toBeDefined()
    +
    +    const files = readdirSync(dir)
    +    expect(files.some(f => f.endsWith('.heapsnapshot'))).toBe(true)
    +  })
    +})
    +
    +describe('heapdump retention guard (#21767)', () => {
    +  let savedDir: string | undefined
    +  let savedMax: string | undefined
    +  let dir: string
    +
    +  beforeEach(() => {
    +    savedDir = process.env.HERMES_HEAPDUMP_DIR
    +    savedMax = process.env.HERMES_HEAPDUMP_MAX_BYTES
    +    delete process.env.HERMES_AUTO_HEAPDUMP
    +    dir = mkdtempSync(join(tmpdir(), 'hermes-heapdump-prune-'))
    +    process.env.HERMES_HEAPDUMP_DIR = dir
    +  })
    +
    +  afterEach(() => {
    +    if (savedDir === undefined) {delete process.env.HERMES_HEAPDUMP_DIR}
    +    else {process.env.HERMES_HEAPDUMP_DIR = savedDir}
    +
    +    if (savedMax === undefined) {delete process.env.HERMES_HEAPDUMP_MAX_BYTES}
    +    else {process.env.HERMES_HEAPDUMP_MAX_BYTES = savedMax}
    +
    +    rmSync(dir, { force: true, recursive: true })
    +  })
    +
    +  it('evicts oldest files when total bytes exceed the cap, retaining the newest', async () => {
    +    // 4 pre-existing dumps, 1KB each, with ascending mtimes (oldest first).
    +    const blob = 'x'.repeat(1024)
    +    const now = Date.now()
    +
    +    for (let i = 0; i < 4; i++) {
    +      const p = join(dir, `old-${i}.heapsnapshot`)
    +      writeFileSync(p, blob)
    +      const t = (now - (4 - i) * 60_000) / 1000
    +      utimesSync(p, t, t)
    +    }
    +
    +    // Cap at 2KB → a fresh diagnostics write should trigger a prune down to ~cap.
    +    process.env.HERMES_HEAPDUMP_MAX_BYTES = String(2 * 1024)
    +
    +    const result = await performHeapDump('auto-high')
    +    expect(result.success).toBe(true)
    +
    +    const remaining = readdirSync(dir)
    +    const totalBytes = remaining.reduce((acc, f) => acc + statSync(join(dir, f)).size, 0)
    +    // Contract: prune evicts oldest-first until total <= cap, but always keeps
    +    // the single newest file even if it alone exceeds the cap. So either the
    +    // total is under cap, or exactly one (newest) file remains.
    +    expect(totalBytes <= 2 * 1024 || remaining.length === 1).toBe(true)
    +    // The old 1KB dumps must have been pruned down from the original four.
    +    expect(remaining.length).toBeLessThan(5)
    +    // The brand-new diagnostics sidecar must survive the prune.
    +    expect(remaining.some(f => f.endsWith('.diagnostics.json'))).toBe(true)
    +  })
    +})

  • ui-tui/src/lib/memory.ts

    @@ -1,5 +1,5 @@
     import { createWriteStream } from 'node:fs'
    -import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises'
    +import { mkdir, readdir, readFile, stat, unlink, writeFile } from 'node:fs/promises'
     import { homedir, tmpdir } from 'node:os'
     import { join } from 'node:path'
     import { pipeline } from 'node:stream/promises'
    @@ -51,6 +51,9 @@ export interface HeapDumpResult {
       diagPath?: string
       error?: string
       heapPath?: string
    +  // True when an auto trigger wrote diagnostics only and intentionally skipped
    +  // the heavy snapshot because HERMES_AUTO_HEAPDUMP was not enabled (#21767).
    +  suppressed?: boolean
       success: boolean
     }
     
    @@ -153,8 +156,26 @@ export async function performHeapDump(trigger: MemoryTrigger = 'manual'): Promis
         const heapPath = join(dir, `${base}.heapsnapshot`)
         const diagPath = join(dir, `${base}.diagnostics.json`)
     
    +    // The diagnostics JSON is KB-sized and the most useful artifact when a
    +    // full snapshot is suppressed by the auto-heapdump opt-in gate below.
         await writeFile(diagPath, JSON.stringify(diagnostics, null, 2), { mode: 0o600 })
    +
    +    // Auto triggers require explicit opt-in: multi-GiB snapshots written on
    +    // every threshold cross can fill the user's disk (issue #21767).
    +    const isAuto = trigger === 'auto-critical' || trigger === 'auto-high'
    +    const autoEnabled = /^(?:1|true|yes|on)$/i.test((process.env.HERMES_AUTO_HEAPDUMP ?? '').trim())
    +
    +    if (isAuto && !autoEnabled) {
    +      await pruneHeapdumps(dir).catch(() => undefined)
    +
    +      // Not an error: the dump did its job — it wrote the lightweight
    +      // diagnostics sidecar and intentionally skipped the heavy snapshot.
    +      // `heapPath` is omitted so callers/notices report diagnostics-only.
    +      return { diagPath, suppressed: true, success: true }
    +    }
    +
         await pipeline(getHeapSnapshot(), createWriteStream(heapPath, { mode: 0o600 }))
    +    await pruneHeapdumps(dir).catch(() => undefined)
     
         return { diagPath, heapPath, success: true }
       } catch (e) {
    @@ -162,6 +183,44 @@ export async function performHeapDump(trigger: MemoryTrigger = 'manual'): Promis
       }
     }
     
    +// Cap total bytes of files in `dir`, deleting oldest first. Covers both
    +// `.heapsnapshot` and `.diagnostics.json` artifacts so orphan sidecars from
    +// gated auto-triggers cannot accumulate without bound. The newest file is
    +// always retained even if it alone exceeds the cap.
    +async function pruneHeapdumps(dir: string): Promise<void> {
    +  const raw = process.env.HERMES_HEAPDUMP_MAX_BYTES?.trim()
    +  const parsed = raw ? Number(raw) : NaN
    +  const cap = Number.isFinite(parsed) && parsed > 0 ? parsed : 2 * 1024 ** 3
    +
    +  const names = await readdir(dir)
    +
    +  const stats = await Promise.all(
    +    names.map(async name => {
    +      const path = join(dir, name)
    +      const s = await stat(path).catch(() => null)
    +
    +      return s && s.isFile() ? { mtimeMs: s.mtimeMs, path, size: s.size } : null
    +    })
    +  )
    +
    +  const valid = stats.filter((s): s is { mtimeMs: number; path: string; size: number } => s !== null)
    +
    +  valid.sort((a, b) => b.mtimeMs - a.mtimeMs)
    +
    +  let total = valid.reduce((acc, s) => acc + s.size, 0)
    +
    +  while (total > cap && valid.length > 1) {
    +    const oldest = valid.pop()
    +
    +    if (!oldest) {
    +      break
    +    }
    +
    +    await unlink(oldest.path).catch(() => undefined)
    +    total -= oldest.size
    +  }
    +}
    +
     export function formatBytes(bytes: number): string {
       if (!Number.isFinite(bytes) || bytes <= 0) {
         return '0B'

  • ui-tui/src/lib/memoryMonitor.ts

    @@ -111,6 +111,14 @@ export function startMemoryMonitor({
       let warned = false
       const WARN_GROWTH_STEP = 150 * MB
     
    +  // Cooldown prevents repeated auto dumps when heap oscillates around the
    +  // threshold (issue #21767). `dumped` alone is not enough — it clears on
    +  // every transition back to `normal`.
    +  const cooldownRaw = process.env.HERMES_AUTO_HEAPDUMP_COOLDOWN_MS?.trim()
    +  const cooldownParsed = cooldownRaw ? Number(cooldownRaw) : NaN
    +  const cooldownMs = Number.isFinite(cooldownParsed) && cooldownParsed >= 0 ? cooldownParsed : 600_000
    +  let lastAutoDumpAt = 0
    +
       const tick = async () => {
         const { heapUsed, rss } = process.memoryUsage()
     
    @@ -137,7 +145,12 @@ export function startMemoryMonitor({
           return
         }
     
    +    if (Date.now() - lastAutoDumpAt < cooldownMs) {
    +      return
    +    }
    +
         inFlight.add(level)
    +    lastAutoDumpAt = Date.now()
     
         // Prune Ink content caches before dump/exit — half on 'high' (recoverable),
         // full on 'critical' (post-dump RSS reduction, keeps user running).