{"repo":"NousResearch/hermes-agent","public_id":"38249","display_label":"Terminal env snapshot concurrency and atomicity fixes","summary":"Multiple bug reports and fix attempts address concurrent environment snapshot writes corrupting PATH, with specific attention to bash 3.2 on macOS where $BASHPID is unavailable and writers share a single temp file.","subsystem":"environments","counts":{"members":7,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":1,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":54098},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-28T09:08:57+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.2552380952380952},"fix_edges":[{"issue_number":38249,"pr_number":54098,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":38249,"title":"[Bug]: Concurrent terminal environment snapshots can corrupt PATH with declare -x entries","url":null,"state":"closed"},{"kind":"issue","number":57866,"title":"Env snapshot tears on macOS: $BASHPID is empty under /bin/bash 3.2, all concurrent writers share one temp file","url":null,"state":"closed"},{"kind":"pr","number":38267,"title":"fix(terminal): serialize environment snapshot execution","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#54098","number":54098}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":38279,"title":"fix(environments): use atomic file replacement for snapshot writes #38249","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"6a2958a52","number":0},{"kind":"pr","value":"#54098","number":54098}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":53159,"title":"fix(terminal): atomic env-snapshot replacement to prevent PATH corruption (#38249)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#54098","number":54098}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":54098,"title":"fix(terminal): atomic env-snapshot writes keyed on $BASHPID to prevent PATH corruption (#38249)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["infographic/atomic-env-snapshot-38249/infographic.png","tests/tools/test_base_environment.py","tools/environments/base.py","tools/environments/local.py"],"fix_hunks":[{"file":"tests/tools/test_base_environment.py","hunks":["@@ -90,6 +90,166 @@ def test_cd_failure_exit_126(self):\n         assert \"exit 126\" in wrapped\n \n \n+class TestAtomicSnapshotWrite:\n+    \"\"\"Regression for #38249: concurrent terminal calls in one session both\n+    source AND rewrite the shared env snapshot. A non-atomic ``export -p >\n+    snap`` truncates-then-writes in place, so a concurrent ``source snap`` can\n+    read a half-written file and embed ``declare -x``/``export`` fragments into\n+    PATH, breaking ``ls``/``git``/``tr`` with command-not-found. The write must\n+    assemble in a temp file and ``mv -f`` it into place (mv is atomic on POSIX\n+    same-fs), so a reader sees the old-or-new complete file, never a torn one.\n+    \"\"\"\n+\n+    def test_wrap_command_uses_atomic_temp_then_mv(self):\n+        env = _TestableEnv()\n+        env._snapshot_ready = True\n+        wrapped = env._wrap_command(\"echo hi\", \"/tmp\")\n+        # Env dump goes to a temp file, not directly over the live snapshot.\n+        assert \"export -p > \" in wrapped\n+        assert \".tmp.\" in wrapped\n+        # Then an atomic rename onto the real snapshot path.\n+        assert \"mv -f \" in wrapped\n+        # The env-dump must NOT write the live snapshot in place (the bug).\n+        snap = env._snapshot_path\n+        assert f\"export -p > {snap} \" not in wrapped\n+        assert f\"export -p > '{snap}'\" not in wrapped\n+\n+    def test_temp_path_uses_bashpid_not_dollardollar(self):\n+        \"\"\"The temp name MUST use ``$BASHPID`` (the real subshell PID), not\n+        ``$$``.  In ``&``-launched concurrent subshells ``$$`` stays the parent\n+        shell's PID, so two writers would pick the same temp name, clobber each\n+        other mid-write, and mv would publish a torn file — the corruption is\n+        only narrowed, not closed.  This is the bug shared by every prior PR in\n+        the #38249 cluster.\"\"\"\n+        env = _TestableEnv()\n+        env._snapshot_ready = True\n+        wrapped = env._wrap_command(\"echo hi\", \"/tmp\")\n+        assert \"$BASHPID\" in wrapped\n+        # The bare $$ temp form must be gone.\n+        assert \".tmp.$$\" not in wrapped\n+\n+    def test_temp_path_static_part_is_quoted_bashpid_outside(self):\n+        \"\"\"The static path portion must be shlex-quoted (Windows/Git-Bash\n+        ``C:/Users/...`` or spaces) while ``$BASHPID`` stays OUTSIDE the quotes\n+        so it still expands.\"\"\"\n+        env = _TestableEnv()\n+        env._snapshot_ready = True\n+        env._snapshot_path = \"/tmp/has space/hermes-snap-x.sh\"\n+        wrapped = env._wrap_command(\"echo hi\", \"/tmp\")\n+        # The static path (with its space) is shlex-quoted as a single word, with\n+        # $BASHPID appended OUTSIDE the quotes so it still expands at runtime.\n+        assert \"'/tmp/has space/hermes-snap-x.sh.tmp.'$BASHPID\" in wrapped\n+        # The space must never appear bare/unquoted in the temp token (that would\n+        # word-split into two args and break the redirect/mv).\n+        assert \" space/hermes-snap-x.sh.tmp.$BASHPID\" not in wrapped\n+\n+    def test_wrap_command_mv_chained_on_export_success(self):\n+        \"\"\"A failed/partial ``export -p`` must NOT mv a torn temp over a good\n+        snapshot.  The mv is chained with ``&&`` on the export, and the temp is\n+        removed on failure.\"\"\"\n+        env = _TestableEnv()\n+        env._snapshot_ready = True\n+        wrapped = env._wrap_command(\"echo hi\", \"/tmp\")\n+        assert \"export -p > \" in wrapped and \"&& mv -f \" in wrapped\n+        assert \"rm -f \" in wrapped  # temp cleanup on failure\n+\n+    def test_init_session_bootstrap_also_atomic_and_bashpid(self):\n+        \"\"\"The init_session bootstrap (first snapshot write) is the same shared\n+        file a concurrent command could source — it must be atomic and use\n+        ``$BASHPID`` too.\"\"\"\n+        env = _TestableEnv()\n+        captured = {}\n+\n+        def fake_run_bash(cmd_string, *, login=False, timeout=120, stdin_data=None):\n+            captured[\"cmd\"] = cmd_string\n+            raise RuntimeError(\"stop after capture\")  # we only need the script\n+\n+        env._run_bash = fake_run_bash  # type: ignore[assignment]\n+        try:\n+            env.init_session()\n+        except Exception:\n+            pass\n+        boot = captured.get(\"cmd\", \"\")\n+        assert \".tmp.\" in boot and \"mv -f \" in boot, boot\n+        assert \"$BASHPID\" in boot\n+        assert \".tmp.$$\" not in boot\n+\n+\n+class TestAtomicSnapshotConcurrencyBehavioral:\n+    \"\"\"Behavioral regression for #38249 — actually EXECUTES the generated\n+    snapshot write/read concurrently and asserts the file never tears.\n+\n+    The string-inspection tests prove the right script is emitted; this proves\n+    the emitted script's guarantee holds under real concurrency: N concurrent\n+    writers + readers, and the snapshot is ALWAYS a complete, parseable env\n+    dump — never truncated mid-line with a ``declare -x`` / ``export`` fragment\n+    that would corrupt PATH.  Crucially it uses ``$BASHPID`` (per-subshell\n+    unique), which is what closes the race; ``$$`` would still tear here.\n+    \"\"\"\n+\n+    def _run(self, script):\n+        import subprocess\n+        return subprocess.run([\"/bin/bash\", \"-c\", script], capture_output=True, text=True)\n+\n+    def test_concurrent_writes_never_tear_the_snapshot(self, tmp_path):\n+        import shutil\n+        if not shutil.which(\"bash\"):\n+            import pytest\n+            pytest.skip(\"bash required\")\n+        import shlex\n+        snap = str(tmp_path / \"hermes-snap-x.sh\")\n+        _q = shlex.quote\n+        _snap_tmp = _q(snap + \".tmp.\") + \"$BASHPID\"\n+        # One writer iteration = the exact atomic sequence _wrap_command emits.\n+        writer = (\n+            \"for i in $(seq 1 80); do \"\n+            \"export BIG_$i=$(head -c 600 /dev/zero | tr '\\\\0' x); \"\n+            f\"{{ export -p > {_snap_tmp} && mv -f {_snap_tmp} {_q(snap)}; }} \"\n+            f\"2>/dev/null || rm -f {_snap_tmp} 2>/dev/null || true; \"\n+            \"done\"\n+        )\n+        # Reader: repeatedly source the snapshot and check PATH never absorbs\n+        # an `export `/`declare -x` fragment (the corruption signature).\n+        reader = (\n+            \"export PATH=/usr/bin:/bin; \"\n+            \"for i in $(seq 1 160); do \"\n+            f\"( source {_q(snap)} >/dev/null 2>&1 || true; \"\n+            \"case \\\"$PATH\\\" in *'declare -x'*|*'export '*) echo CORRUPT;; esac ); \"\n+            \"done\"\n+        )\n+        self._run(f\"export -p > {_q(snap)}\")  # seed a valid snapshot\n+        # 4 concurrent writers + 4 readers, repeated.\n+        w = \" & \".join([writer] * 4)\n+        r = \" & \".join([reader] * 4)\n+        procs = [self._run(f\"{w} & {r} & wait\") for _ in range(3)]\n+        corrupt = any(\"CORRUPT\" in p.stdout for p in procs)\n+        assert not corrupt, \"snapshot tore — PATH absorbed a declare-x/export fragment\"\n+        final = self._run(f\"source {_q(snap)} >/dev/null 2>&1 && echo OK || echo BROKEN\")\n+        assert \"OK\" in final.stdout, f\"final snapshot not sourceable: {final.stdout} {final.stderr}\"\n+\n+    def test_failed_export_does_not_destroy_good_snapshot(self, tmp_path):\n+        \"\"\"If ``export -p`` fails, the ``&&``-chained mv must NOT clobber the\n+        existing good snapshot.\"\"\"\n+        import shutil\n+        if not shutil.which(\"bash\"):\n+            import pytest\n+            pytest.skip(\"bash required\")\n+        import shlex\n+        snap = str(tmp_path / \"snap.sh\")\n+        _q = shlex.quote\n+        self._run(f\"echo 'export GOOD=1' > {_q(snap)}\")  # seed good snapshot\n+        # Redirect export into an unwritable dir so the export side fails; mv\n+        # must then NOT run (&&) and not clobber snap.\n+        bad_tmp = _q(\"/nonexistent-dir/snap.tmp.\") + \"$BASHPID\"\n+        script = (\n+            f\"{{ export -p > {bad_tmp} && mv -f {bad_tmp} {_q(snap)}; }} \"\n+            f\"2>/dev/null || rm -f {bad_tmp} 2>/dev/null || true\"\n+        )\n+        self._run(script)\n+        out = self._run(f\"cat {_q(snap)}\")\n+        assert \"export GOOD=1\" in out.stdout, \"good snapshot was destroyed by a failed export\"\n+\n+\n class TestExtractCwdFromOutput:\n     def test_happy_path(self):\n         env = _TestableEnv()"]},{"file":"tools/environments/base.py","hunks":["@@ -371,13 +371,34 @@ def init_session(self):\n         # backends) into every terminal-tool response.\n         _quoted_snap = shlex.quote(self._snapshot_path)\n         _quoted_cwd_file = shlex.quote(self._cwd_file)\n+        # Use atomic file replacement: assemble the snapshot in a temp file,\n+        # then mv it over the final path.  This prevents concurrent source()\n+        # calls from reading a half-written snapshot when another terminal\n+        # command finishes and rewrites the env vars (issue #38249).  `mv` is\n+        # atomic on POSIX when src and dest are on the same filesystem, so\n+        # source() either sees the old complete snapshot or the new complete\n+        # one — never a partial/truncated file.\n+        #\n+        # The temp name MUST be unique per concurrent writer.  ``$$`` is the\n+        # bash PID, but in ``&``-launched subshells (how concurrent terminal\n+        # calls run) ``$$`` stays the *parent* shell's PID — so two concurrent\n+        # writers would pick the SAME temp name, clobber each other's temp\n+        # mid-write, and mv would then publish a torn file (the corruption is\n+        # only narrowed, not closed).  ``$BASHPID`` is the actual subshell PID\n+        # and is genuinely unique per writer, which closes the race.  The\n+        # static path is shlex-quoted (Windows/Git-Bash drive letters, spaces)\n+        # with ``$BASHPID`` left outside the quotes so it still expands.\n+        _snap_tmp = shlex.quote(self._snapshot_path + \".tmp.\") + \"$BASHPID\"\n         bootstrap = (\n-            f\"export -p > {_quoted_snap}\\n\"\n-            f\"declare -f | grep -vE '^_[^_]' >> {_quoted_snap}\\n\"\n-            f\"alias -p >> {_quoted_snap}\\n\"\n-            f\"echo 'shopt -s expand_aliases' >> {_quoted_snap}\\n\"\n-            f\"echo 'set +e' >> {_quoted_snap}\\n\"\n-            f\"echo 'set +u' >> {_quoted_snap}\\n\"\n+            f\"export -p > {_snap_tmp}\\n\"\n+            f\"declare -f | grep -vE '^_[^_]' >> {_snap_tmp}\\n\"\n+            f\"alias -p >> {_snap_tmp}\\n\"\n+            f\"echo 'shopt -s expand_aliases' >> {_snap_tmp}\\n\"\n+            f\"echo 'set +e' >> {_snap_tmp}\\n\"\n+            f\"echo 'set +u' >> {_snap_tmp}\\n\"\n+            # Publish atomically only if assembly succeeded; otherwise drop the\n+            # partial temp rather than leave it to be sourced or orphaned.\n+            f\"mv -f {_snap_tmp} {_quoted_snap} || rm -f {_snap_tmp}\\n\"\n             f\"builtin cd {_quoted_cwd} 2>/dev/null || true\\n\"\n             f\"pwd -P > {_quoted_cwd_file} 2>/dev/null || true\\n\"\n             f\"printf '\\\\n{self._cwd_marker}%s{self._cwd_marker}\\\\n' \\\"$(pwd -P)\\\"\\n\"","@@ -427,6 +448,14 @@ def _wrap_command(self, command: str, cwd: str) -> str:\n         # :meth:`init_session` for the same fix on the bootstrap block.\n         _quoted_snap = shlex.quote(self._snapshot_path)\n         _quoted_cwd_file = shlex.quote(self._cwd_file)\n+        # Use atomic file replacement for env snapshot updates (issue #38249).\n+        # Assemble into a per-writer-unique temp file, then mv to atomically\n+        # replace the snapshot so concurrent source() calls never read a\n+        # truncated/half-written file.  ``$BASHPID`` (not ``$$``) is the actual\n+        # subshell PID — unique per concurrent ``&``-launched writer — so two\n+        # writers never share a temp name and clobber each other before the mv.\n+        # Static path shlex-quoted (Windows/spaces); ``$BASHPID`` left to expand.\n+        _snap_tmp = shlex.quote(self._snapshot_path + \".tmp.\") + \"$BASHPID\"\n \n         parts = []\n ","@@ -451,9 +480,15 @@ def _wrap_command(self, command: str, cwd: str) -> str:\n         parts.append(f\"eval '{escaped}'\")\n         parts.append(\"__hermes_ec=$?\")\n \n-        # Re-dump env vars to snapshot (last-writer-wins for concurrent calls)\n+        # Re-dump env vars to snapshot (atomic replacement to avoid races).\n+        # Chain mv on the export succeeding so a failed/partial dump never\n+        # replaces a good snapshot; drop the temp on failure so it isn't\n+        # orphaned (cleaned up wholesale in LocalEnvironment.cleanup too).\n         if self._snapshot_ready:\n-            parts.append(f\"export -p > {_quoted_snap} 2>/dev/null || true\")\n+            parts.append(\n+                f\"{{ export -p > {_snap_tmp} && mv -f {_snap_tmp} {_quoted_snap}; }} \"\n+                f\"2>/dev/null || rm -f {_snap_tmp} 2>/dev/null || true\"\n+            )\n \n         # Write CWD to file (local reads this) and stdout marker (remote parses this)\n         parts.append(f\"pwd -P > {_quoted_cwd_file} 2>/dev/null || true\")"]},{"file":"tools/environments/local.py","hunks":["@@ -987,3 +987,14 @@ def cleanup(self):\n                 os.unlink(f)\n             except OSError:\n                 pass\n+        # Remove any orphaned atomic-write temp snapshots (snap.tmp.<bashpid>)\n+        # a failed/interrupted mv could have left behind (#38249).\n+        try:\n+            import glob\n+            for tmp in glob.glob(f\"{self._snapshot_path}.tmp.*\"):\n+                try:\n+                    os.unlink(tmp)\n+                except OSError:\n+                    pass\n+        except Exception:\n+            pass\n"]}]},{"kind":"pr","number":57867,"title":"fix(environments): portable per-writer PID for atomic snapshot writes on bash 3.2","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"911d380296","number":0},{"kind":"pr","value":"#54314","number":54314}],"category":"superseded_by_pr","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}