{"repo":"NousResearch/hermes-agent","public_id":"7439","display_label":"Docker container cleanup on run failure","summary":"This cluster addresses a bug where Docker containers are left in a 'Created' (orphaned) state when 'docker run' or session initialization fails. Multiple fix attempts and follow-up PRs propose cleanup logic to handle these leaked containers.","subsystem":"environments","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":39412},"resolution":{"state":"resolved","by":"pr-39412"},"freshness":{"state":"verified","last_verified_at":"2026-06-05T00:19:08+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":7439,"pr_number":39412,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":7439,"title":"Docker containers leak in Created state when docker run fails","url":null,"state":"closed"},{"kind":"pr","number":7440,"title":"fix(docker): clean up orphaned container when docker run fails","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#7498","number":7498},{"kind":"pr","value":"#7439","number":7439}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":7498,"title":"fix(docker): clean up containers when docker run or init_session fails","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":11535,"title":"fix(docker): clean up orphaned containers when docker run fails","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":39412,"title":"fix(docker): clean up orphaned container when docker run fails (salvage #7440)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_docker_environment.py","tools/environments/docker.py"],"fix_hunks":[{"file":"tests/tools/test_docker_environment.py","hunks":["@@ -786,6 +786,84 @@ def test_reuse_falls_back_to_fresh_run_when_start_fails(monkeypatch):\n     assert run_invocations, \"fallback to fresh docker run must happen on start failure\"\n \n \n+def test_failed_docker_run_cleans_up_orphaned_container(monkeypatch):\n+    \"\"\"When ``docker run`` fails (e.g. exit 125), the partially-created\n+    container must be removed by name.\n+\n+    Docker can create the container object before failing to start it,\n+    leaving a stale ``Created`` container. The exited-only orphan reaper\n+    (``reap_orphan_containers``, ``status=exited``) never catches a\n+    ``Created`` orphan, so without this cleanup it leaks permanently.\n+    Regression for #7439. Salvage of #7440 (@Tranquil-Flow).\n+    \"\"\"\n+    monkeypatch.setattr(docker_env, \"find_docker\", lambda: \"/usr/bin/docker\")\n+    monkeypatch.setattr(docker_env, \"_get_active_profile_name\", lambda: \"default\")\n+\n+    cleanup_calls = []\n+\n+    def _run(cmd, **kwargs):\n+        if isinstance(cmd, list) and len(cmd) >= 2:\n+            sub = cmd[1]\n+            if sub == \"version\":\n+                return subprocess.CompletedProcess(cmd, 0, stdout=\"Docker version\", stderr=\"\")\n+            if sub == \"ps\":\n+                # No reusable container -> fall through to a fresh `docker run`.\n+                return subprocess.CompletedProcess(cmd, 0, stdout=\"\", stderr=\"\")\n+            if sub == \"run\":\n+                raise subprocess.CalledProcessError(\n+                    125, cmd, output=\"\", stderr=\"docker: Error response from daemon\"\n+                )\n+            if sub == \"rm\":\n+                cleanup_calls.append(list(cmd))\n+                return subprocess.CompletedProcess(cmd, 0, stdout=\"\", stderr=\"\")\n+        return subprocess.CompletedProcess(cmd, 0, stdout=\"\", stderr=\"\")\n+\n+    monkeypatch.setattr(docker_env.subprocess, \"run\", _run)\n+\n+    with pytest.raises(subprocess.CalledProcessError):\n+        _make_dummy_env()\n+\n+    assert len(cleanup_calls) == 1, \"docker rm should be called once for the orphaned container\"\n+    rm_cmd = cleanup_calls[0]\n+    assert rm_cmd[1] == \"rm\" and rm_cmd[2] == \"-f\"\n+    assert rm_cmd[3].startswith(\"hermes-\"), \"should remove the container by its generated name\"\n+\n+\n+def test_docker_run_timeout_cleans_up_orphaned_container(monkeypatch):\n+    \"\"\"When ``docker run`` times out (e.g. slow image pull), the\n+    partially-created container must be removed. Salvage of #7440\n+    (@Tranquil-Flow); regression for #7439.\n+    \"\"\"\n+    monkeypatch.setattr(docker_env, \"find_docker\", lambda: \"/usr/bin/docker\")\n+    monkeypatch.setattr(docker_env, \"_get_active_profile_name\", lambda: \"default\")\n+\n+    cleanup_calls = []\n+\n+    def _run(cmd, **kwargs):\n+        if isinstance(cmd, list) and len(cmd) >= 2:\n+            sub = cmd[1]\n+            if sub == \"version\":\n+                return subprocess.CompletedProcess(cmd, 0, stdout=\"Docker version\", stderr=\"\")\n+            if sub == \"ps\":\n+                return subprocess.CompletedProcess(cmd, 0, stdout=\"\", stderr=\"\")\n+            if sub == \"run\":\n+                raise subprocess.TimeoutExpired(cmd, 120)\n+            if sub == \"rm\":\n+                cleanup_calls.append(list(cmd))\n+                return subprocess.CompletedProcess(cmd, 0, stdout=\"\", stderr=\"\")\n+        return subprocess.CompletedProcess(cmd, 0, stdout=\"\", stderr=\"\")\n+\n+    monkeypatch.setattr(docker_env.subprocess, \"run\", _run)\n+\n+    with pytest.raises(subprocess.TimeoutExpired):\n+        _make_dummy_env()\n+\n+    assert len(cleanup_calls) == 1, \"docker rm should be called once for the orphaned container\"\n+    rm_cmd = cleanup_calls[0]\n+    assert rm_cmd[1] == \"rm\" and rm_cmd[2] == \"-f\"\n+    assert rm_cmd[3].startswith(\"hermes-\"), \"should remove the container by its generated name\"\n+\n+\n def test_no_reuse_when_persist_across_processes_disabled(monkeypatch):\n     \"\"\"Opt-out path: ``persist_across_processes=False`` skips the ps probe\n     entirely and always starts a fresh container, matching the pre-fix"]},{"file":"tools/environments/docker.py","hunks":["@@ -854,13 +854,31 @@ def __init__(\n                 \"sleep\", \"infinity\",  # no fixed lifetime — idle reaper handles cleanup\n             ]\n             logger.debug(f\"Starting container: {' '.join(run_cmd)}\")\n-            result = subprocess.run(\n-                run_cmd,\n-                capture_output=True,\n-                text=True,\n-                timeout=120,  # image pull may take a while\n-                check=True,\n-            )\n+            try:\n+                result = subprocess.run(\n+                    run_cmd,\n+                    capture_output=True,\n+                    text=True,\n+                    timeout=120,  # image pull may take a while\n+                    check=True,\n+                )\n+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:\n+                # Docker may create the container object before `docker run`\n+                # fails to start it (e.g. exit code 125 when the daemon isn't\n+                # ready, or a timeout mid-pull). That orphan is left in\n+                # \"Created\" state — which the exited-only orphan reaper\n+                # (reap_orphan_containers, status=exited) never catches, so it\n+                # leaks permanently. Remove it by its known name before\n+                # re-raising. See #7439.\n+                logger.warning(\n+                    \"docker run failed for %s, cleaning up orphaned container: %s\",\n+                    container_name, e,\n+                )\n+                subprocess.run(\n+                    [self._docker_exe, \"rm\", \"-f\", container_name],\n+                    capture_output=True, timeout=10,\n+                )\n+                raise\n             self._container_id = result.stdout.strip()\n             logger.info(f\"Started container {container_name} ({self._container_id[:12]})\")\n \n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}