{"repo":"NousResearch/hermes-agent","public_id":"52383","display_label":"Cron job output logs filling disk storage","summary":"Cron job per-run output files accumulate without retention limits, causing disk exhaustion on long-running deployments. Fixes add a default retention policy to prevent unbounded log growth.","subsystem":"cron","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":52646},"resolution":{"state":"partially_resolved","by":"pr-52646"},"freshness":{"state":"verified","last_verified_at":"2026-06-25T23:00:13+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.3933333333333333},"fix_edges":[{"issue_number":52383,"pr_number":52646,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":52383,"title":"[Bug]: Hermes fills the disk on long-running deploys (\"no space left on device\") despite rotated app logs","url":null,"state":"closed"},{"kind":"issue","number":56894,"title":"182GB Log file in 24hours [Bug]: 🚨 CRITICAL: Disk is 100% FULL — This is why everything is failing!","url":null,"state":"closed"},{"kind":"pr","number":52407,"title":"fix: add default retention for cron output files to prevent disk fill (#52383)","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":52646,"title":"fix(cron): add default retention to per-run job output (#52383)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cron/jobs.py","hermes_cli/config.py","tests/cron/test_jobs.py"],"fix_hunks":[{"file":"cron/jobs.py","hunks":["@@ -1497,16 +1497,64 @@ def _get_due_jobs_locked() -> List[Dict[str, Any]]:\n     return due\n \n \n+# Per-run cron output (`cron/output/<job>/<timestamp>.md`) is written once per\n+# execution. Unlike the quick-snapshot store (`hermes_cli.backup`, capped at 20)\n+# it had no retention, so a frequently-scheduled job on a long-running deploy\n+# accumulated one file per run forever and could fill the disk (#52383). Keep the\n+# most recent N files per job; a non-positive value disables pruning (opt-out).\n+_CRON_OUTPUT_DEFAULT_KEEP = 50\n+\n+\n+def _cron_output_keep() -> int:\n+    \"\"\"Resolve the per-job output-file retention cap from config (``cron.output_retention``).\"\"\"\n+    try:\n+        from hermes_cli.config import load_config\n+        cfg = load_config() or {}\n+        cron_cfg = cfg.get(\"cron\", {}) if isinstance(cfg, dict) else {}\n+        return int(cron_cfg.get(\"output_retention\", _CRON_OUTPUT_DEFAULT_KEEP))\n+    except Exception:\n+        return _CRON_OUTPUT_DEFAULT_KEEP\n+\n+\n+def _prune_job_output(job_output_dir: Path, keep: int) -> int:\n+    \"\"\"Remove the oldest ``*.md`` run-output files beyond *keep*. Returns count deleted.\n+\n+    Mirrors the quick-snapshot retention in ``hermes_cli.backup._prune_quick_snapshots``:\n+    output filenames are timestamp-based (``%Y-%m-%d_%H-%M-%S.md``) so a reverse\n+    lexical sort orders newest-first, and everything past *keep* is the tail to\n+    drop. A non-positive *keep* disables pruning. Pruning failures are swallowed\n+    so they can never break output saving.\n+    \"\"\"\n+    if keep <= 0:\n+        return 0\n+    try:\n+        files = sorted(\n+            (f for f in job_output_dir.glob(\"*.md\") if f.is_file()),\n+            key=lambda f: f.name,\n+            reverse=True,\n+        )\n+    except OSError:\n+        return 0\n+    deleted = 0\n+    for stale in files[keep:]:\n+        try:\n+            stale.unlink()\n+            deleted += 1\n+        except OSError as exc:\n+            logger.debug(\"Failed to prune cron output %s: %s\", stale.name, exc)\n+    return deleted\n+\n+\n def save_job_output(job_id: str, output: str):\n     \"\"\"Save job output to file.\"\"\"\n     ensure_dirs()\n     job_output_dir = _job_output_dir(job_id)\n     job_output_dir.mkdir(parents=True, exist_ok=True)\n     _secure_dir(job_output_dir)\n-    \n+\n     timestamp = _hermes_now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n     output_file = job_output_dir / f\"{timestamp}.md\"\n-    \n+\n     fd, tmp_path = tempfile.mkstemp(dir=str(job_output_dir), suffix='.tmp', prefix='.output_')\n     try:\n         with os.fdopen(fd, 'w', encoding='utf-8') as f:","@@ -1521,7 +1569,10 @@ def save_job_output(job_id: str, output: str):\n         except OSError:\n             pass\n         raise\n-    \n+\n+    # Bound per-job output growth so long-running deploys don't fill the disk (#52383).\n+    _prune_job_output(job_output_dir, _cron_output_keep())\n+\n     return output_file\n \n "]},{"file":"hermes_cli/config.py","hunks":["@@ -2417,6 +2417,10 @@ def _ensure_hermes_home_managed(home: Path):\n         # 1 = serial (pre-v0.9 behaviour).\n         # Also overridable via HERMES_CRON_MAX_PARALLEL env var.\n         \"max_parallel_jobs\": None,\n+        # Per-job output-file retention: save_job_output keeps the N most\n+        # recent .md files and prunes older ones. 0 or negative disables\n+        # pruning (for operators who manage cleanup externally). Default 50.\n+        \"output_retention\": 50,\n     },\n \n     # Kanban multi-agent coordination — controls the dispatcher loop that"]},{"file":"tests/cron/test_jobs.py","hunks":["@@ -1242,3 +1242,75 @@ def test_rejects_absolute_job_id(self, tmp_cron_dir):\n         with pytest.raises(ValueError, match=\"output path\"):\n             save_job_output(str(tmp_cron_dir / \"outside\"), \"# Results\")\n         assert not (tmp_cron_dir / \"outside\").exists()\n+\n+\n+class TestCronOutputRetention:\n+    \"\"\"Per-run cron output must self-prune so long deploys don't fill the disk (#52383).\"\"\"\n+\n+    @staticmethod\n+    def _seed(d, count):\n+        d.mkdir(parents=True, exist_ok=True)\n+        names = [f\"2026-06-25_10-00-{i:02d}.md\" for i in range(count)]\n+        for n in names:\n+            (d / n).write_text(\"x\")\n+        return names\n+\n+    def test_prune_keeps_newest_n(self, tmp_path):\n+        from cron.jobs import _prune_job_output\n+        d = tmp_path / \"job\"\n+        names = self._seed(d, 10)\n+        assert _prune_job_output(d, keep=3) == 7\n+        assert sorted(p.name for p in d.glob(\"*.md\")) == names[-3:]\n+\n+    def test_prune_noop_when_under_cap(self, tmp_path):\n+        from cron.jobs import _prune_job_output\n+        d = tmp_path / \"job\"\n+        self._seed(d, 3)\n+        assert _prune_job_output(d, keep=5) == 0\n+        assert len(list(d.glob(\"*.md\"))) == 3\n+\n+    def test_prune_disabled_when_keep_non_positive(self, tmp_path):\n+        from cron.jobs import _prune_job_output\n+        d = tmp_path / \"job\"\n+        self._seed(d, 5)\n+        assert _prune_job_output(d, keep=0) == 0\n+        assert _prune_job_output(d, keep=-1) == 0\n+        assert len(list(d.glob(\"*.md\"))) == 5\n+\n+    def test_prune_ignores_non_md_and_temp_files(self, tmp_path):\n+        from cron.jobs import _prune_job_output\n+        d = tmp_path / \"job\"\n+        self._seed(d, 4)\n+        (d / \".output_abc.tmp\").write_text(\"partial\")\n+        (d / \"manifest.json\").write_text(\"{}\")\n+        _prune_job_output(d, keep=2)\n+        assert (d / \".output_abc.tmp\").exists()\n+        assert (d / \"manifest.json\").exists()\n+        assert len(list(d.glob(\"*.md\"))) == 2\n+\n+    def test_save_job_output_prunes_old_runs(self, tmp_cron_dir, monkeypatch):\n+        from cron.jobs import save_job_output, _job_output_dir\n+        monkeypatch.setattr(\"cron.jobs._cron_output_keep\", lambda: 3)\n+        seq = iter(\n+            datetime(2026, 6, 25, 10, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=i)\n+            for i in range(8)\n+        )\n+        monkeypatch.setattr(\"cron.jobs._hermes_now\", lambda: next(seq))\n+        for _ in range(8):\n+            save_job_output(\"job1\", \"report\")\n+        files = sorted(_job_output_dir(\"job1\").glob(\"*.md\"))\n+        assert len(files) == 3  # only the 3 most-recent runs survive\n+\n+    def test_cron_output_keep_reads_config(self, monkeypatch):\n+        import cron.jobs as jobs\n+        monkeypatch.setattr(\n+            \"hermes_cli.config.load_config\", lambda: {\"cron\": {\"output_retention\": 7}}\n+        )\n+        assert jobs._cron_output_keep() == 7\n+\n+    def test_cron_output_keep_defaults_on_bad_config(self, monkeypatch):\n+        import cron.jobs as jobs\n+        monkeypatch.setattr(\n+            \"hermes_cli.config.load_config\", lambda: {\"cron\": {\"output_retention\": \"oops\"}}\n+        )\n+        assert jobs._cron_output_keep() == jobs._CRON_OUTPUT_DEFAULT_KEEP\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}