{"repo":"NousResearch/hermes-agent","public_id":"6035","display_label":"bulk file sync performance for remote backends","summary":"These issues and pull requests address slow session startup caused by excessive individual file uploads in Daytona and other remote backends (SSH, Modal). Work focuses on bulk/tar-based file transfer, removing unnecessary skill file syncs, and related configuration fixes.","subsystem":"environments","counts":{"members":10,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":3,"merged_prs":2,"attempt_prs":5},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":7538},"resolution":{"state":"partially_resolved","by":"pr-7538,pr-8014"},"freshness":{"state":"stale","last_verified_at":"2026-04-12T00:48:05+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.17155555555555554},"fix_edges":[{"issue_number":7362,"pr_number":7538,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":7465,"pr_number":8014,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":7467,"pr_number":8014,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":6035,"title":"[Bug]: New Daytona implementation requires hundreds of file uploads at every session start","url":null,"state":"closed"},{"kind":"issue","number":7362,"title":"Daytona backend: bulk upload, config bridge, silent disk cap","url":null,"state":"closed"},{"kind":"issue","number":7467,"title":"perf: bulk file sync for all remote backends (SSH, Modal, Daytona)","url":null,"state":"closed"},{"kind":"pr","number":6108,"title":"fix(environments): remove unnecessary skill file sync to sandboxes (#6035)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#6308","number":6308}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":6113,"title":"perf(daytona): remove unnecessary skills file sync (275s startup overhead)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#6308","number":6308}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":7447,"title":"fix(daytona): bulk upload, config bridge, silent disk cap (#7362)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#7538","number":7538}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":7538,"title":"fix(daytona): bulk upload, config bridge, silent disk cap (#7362)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/config.py","tests/tools/test_file_sync.py","tools/environments/daytona.py","tools/environments/file_sync.py"],"fix_hunks":[{"file":"hermes_cli/config.py","hunks":["@@ -2783,6 +2783,10 @@ def set_config_value(key: str, value: str):\n         \"terminal.timeout\": \"TERMINAL_TIMEOUT\",\n         \"terminal.sandbox_dir\": \"TERMINAL_SANDBOX_DIR\",\n         \"terminal.persistent_shell\": \"TERMINAL_PERSISTENT_SHELL\",\n+        \"terminal.container_cpu\": \"TERMINAL_CONTAINER_CPU\",\n+        \"terminal.container_memory\": \"TERMINAL_CONTAINER_MEMORY\",\n+        \"terminal.container_disk\": \"TERMINAL_CONTAINER_DISK\",\n+        \"terminal.container_persistent\": \"TERMINAL_CONTAINER_PERSISTENT\",\n     }\n     if key in _config_to_env_sync:\n         save_env_value(_config_to_env_sync[key], str(value))"]},{"file":"tests/tools/test_file_sync.py","hunks":["@@ -255,3 +255,57 @@ def test_file_disappears_between_list_and_upload(self, tmp_path):\n \n         mgr.sync(force=True)\n         upload.assert_not_called()  # _file_mtime_key returns None, skipped\n+\n+\n+class TestBulkUpload:\n+    \"\"\"Tests for the optional bulk_upload_fn callback.\"\"\"\n+\n+    def test_bulk_upload_used_when_provided(self, tmp_files):\n+        \"\"\"When bulk_upload_fn is set, it's called instead of per-file upload_fn.\"\"\"\n+        upload = MagicMock()\n+        bulk_upload = MagicMock()\n+        mgr = FileSyncManager(\n+            get_files_fn=_make_get_files(tmp_files),\n+            upload_fn=upload,\n+            delete_fn=MagicMock(),\n+            bulk_upload_fn=bulk_upload,\n+        )\n+\n+        mgr.sync(force=True)\n+        upload.assert_not_called()\n+        bulk_upload.assert_called_once()\n+        # All 3 files passed as a list of (host, remote) tuples\n+        files_arg = bulk_upload.call_args[0][0]\n+        assert len(files_arg) == 3\n+\n+    def test_fallback_to_upload_fn_when_no_bulk(self, tmp_files):\n+        \"\"\"Without bulk_upload_fn, per-file upload_fn is used (backwards compat).\"\"\"\n+        upload = MagicMock()\n+        mgr = FileSyncManager(\n+            get_files_fn=_make_get_files(tmp_files),\n+            upload_fn=upload,\n+            delete_fn=MagicMock(),\n+            bulk_upload_fn=None,\n+        )\n+\n+        mgr.sync(force=True)\n+        assert upload.call_count == 3\n+\n+    def test_bulk_upload_rollback_on_failure(self, tmp_files):\n+        \"\"\"Bulk upload failure rolls back synced state so next sync retries.\"\"\"\n+        bulk_upload = MagicMock(side_effect=RuntimeError(\"upload failed\"))\n+        mgr = FileSyncManager(\n+            get_files_fn=_make_get_files(tmp_files),\n+            upload_fn=MagicMock(),\n+            delete_fn=MagicMock(),\n+            bulk_upload_fn=bulk_upload,\n+        )\n+\n+        mgr.sync(force=True)  # fails, should rollback\n+\n+        # State rolled back: next sync should retry all files\n+        bulk_upload.side_effect = None\n+        bulk_upload.reset_mock()\n+        mgr.sync(force=True)\n+        bulk_upload.assert_called_once()\n+        assert len(bulk_upload.call_args[0][0]) == 3"]},{"file":"tools/environments/daytona.py","hunks":["@@ -9,7 +9,6 @@\n import math\n import shlex\n import threading\n-import warnings\n from pathlib import Path\n \n from tools.environments.base import (","@@ -63,10 +62,9 @@ def __init__(\n         memory_gib = max(1, math.ceil(memory / 1024))\n         disk_gib = max(1, math.ceil(disk / 1024))\n         if disk_gib > 10:\n-            warnings.warn(\n-                f\"Daytona: requested disk ({disk_gib}GB) exceeds platform limit (10GB). \"\n-                f\"Capping to 10GB.\",\n-                stacklevel=2,\n+            logger.warning(\n+                \"Daytona: requested disk (%dGB) exceeds platform limit (10GB). \"\n+                \"Capping to 10GB.\", disk_gib,\n             )\n             disk_gib = 10\n         resources = Resources(cpu=cpu, memory=memory_gib, disk=disk_gib)","@@ -129,6 +127,7 @@ def __init__(\n             get_files_fn=lambda: iter_sync_files(f\"{self._remote_home}/.hermes\"),\n             upload_fn=self._daytona_upload,\n             delete_fn=self._daytona_delete,\n+            bulk_upload_fn=self._daytona_bulk_upload,\n         )\n         self._sync_manager.sync(force=True)\n         self.init_session()","@@ -139,6 +138,30 @@ def _daytona_upload(self, host_path: str, remote_path: str) -> None:\n         self._sandbox.process.exec(f\"mkdir -p {parent}\")\n         self._sandbox.fs.upload_file(host_path, remote_path)\n \n+    def _daytona_bulk_upload(self, files: list[tuple[str, str]]) -> None:\n+        \"\"\"Upload many files in a single HTTP call via Daytona SDK.\n+\n+        Uses ``sandbox.fs.upload_files()`` which batches all files into one\n+        multipart POST, avoiding per-file TLS/HTTP overhead (~580 files\n+        goes from ~5 min to <2 s).\n+        \"\"\"\n+        from daytona.common.filesystem import FileUpload\n+\n+        if not files:\n+            return\n+\n+        # Pre-create all unique parent directories in one shell call\n+        parents = sorted({str(Path(remote).parent) for _, remote in files})\n+        if parents:\n+            mkdir_cmd = \"mkdir -p \" + \" \".join(shlex.quote(p) for p in parents)\n+            self._sandbox.process.exec(mkdir_cmd)\n+\n+        uploads = [\n+            FileUpload(source=host_path, destination=remote_path)\n+            for host_path, remote_path in files\n+        ]\n+        self._sandbox.fs.upload_files(uploads)\n+\n     def _daytona_delete(self, remote_paths: list[str]) -> None:\n         \"\"\"Batch-delete remote files via SDK exec.\"\"\"\n         self._sandbox.process.exec(quoted_rm_command(remote_paths))"]},{"file":"tools/environments/file_sync.py","hunks":["@@ -21,6 +21,7 @@\n \n # Transport callbacks provided by each backend\n UploadFn = Callable[[str, str], None]  # (host_path, remote_path) -> raises on failure\n+BulkUploadFn = Callable[[list[tuple[str, str]]], None]  # [(host_path, remote_path), ...] -> raises on failure\n DeleteFn = Callable[[list[str]], None]  # (remote_paths) -> raises on failure\n GetFilesFn = Callable[[], list[tuple[str, str]]]  # () -> [(host_path, remote_path), ...]\n ","@@ -76,9 +77,11 @@ def __init__(\n         upload_fn: UploadFn,\n         delete_fn: DeleteFn,\n         sync_interval: float = _SYNC_INTERVAL_SECONDS,\n+        bulk_upload_fn: BulkUploadFn | None = None,\n     ):\n         self._get_files_fn = get_files_fn\n         self._upload_fn = upload_fn\n+        self._bulk_upload_fn = bulk_upload_fn\n         self._delete_fn = delete_fn\n         self._synced_files: dict[str, tuple[float, int]] = {}  # remote_path -> (mtime, size)\n         self._last_sync_time: float = 0.0  # monotonic; 0 ensures first sync runs","@@ -129,9 +132,13 @@ def sync(self, *, force: bool = False) -> None:\n             logger.debug(\"file_sync: deleting %d stale remote file(s)\", len(to_delete))\n \n         try:\n-            for host_path, remote_path in to_upload:\n-                self._upload_fn(host_path, remote_path)\n-                logger.debug(\"file_sync: uploaded %s -> %s\", host_path, remote_path)\n+            if to_upload and self._bulk_upload_fn is not None:\n+                self._bulk_upload_fn(to_upload)\n+                logger.debug(\"file_sync: bulk-uploaded %d file(s)\", len(to_upload))\n+            else:\n+                for host_path, remote_path in to_upload:\n+                    self._upload_fn(host_path, remote_path)\n+                    logger.debug(\"file_sync: uploaded %s -> %s\", host_path, remote_path)\n \n             if to_delete:\n                 self._delete_fn(to_delete)\n"]}]},{"kind":"pr","number":7558,"title":"perf(ssh,modal): bulk file sync via tar pipe and tar/base64 archive","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#8014","number":8014}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":7560,"title":"perf(ssh,modal): bulk file sync via tar pipe and tar/base64 archive","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#7558","number":7558}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":8014,"title":"perf(ssh,modal): bulk file sync via tar pipe and tar/base64 archive","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_modal_bulk_upload.py","tests/tools/test_ssh_bulk_upload.py","tools/environments/daytona.py","tools/environments/file_sync.py","tools/environments/modal.py","tools/environments/ssh.py"],"fix_hunks":[{"file":"tests/tools/test_modal_bulk_upload.py","hunks":["@@ -0,0 +1,295 @@\n+\"\"\"Tests for Modal bulk upload via tar/base64 archive.\"\"\"\n+\n+import asyncio\n+import base64\n+import io\n+import tarfile\n+from pathlib import Path\n+from unittest.mock import AsyncMock, MagicMock, patch\n+\n+import pytest\n+\n+from tools.environments import modal as modal_env\n+\n+\n+def _make_mock_modal_env(monkeypatch, tmp_path):\n+    \"\"\"Create a minimal mock ModalEnvironment for testing upload methods.\n+\n+    Returns a ModalEnvironment-like object with _sandbox and _worker mocked.\n+    We don't call __init__ because it requires the Modal SDK.\n+    \"\"\"\n+    env = object.__new__(modal_env.ModalEnvironment)\n+    env._sandbox = MagicMock()\n+    env._worker = MagicMock()\n+    env._persistent = False\n+    env._task_id = \"test\"\n+    env._sync_manager = None\n+    return env\n+\n+\n+def _make_mock_stdin():\n+    \"\"\"Create a mock stdin that captures written data.\"\"\"\n+    stdin = MagicMock()\n+    written_chunks = []\n+\n+    def mock_write(data):\n+        written_chunks.append(data)\n+\n+    stdin.write = mock_write\n+    stdin.write_eof = MagicMock()\n+    stdin.drain = MagicMock()\n+    stdin.drain.aio = AsyncMock()\n+    stdin._written_chunks = written_chunks\n+    return stdin\n+\n+\n+def _wire_async_exec(env, exec_calls=None):\n+    \"\"\"Wire mock sandbox.exec.aio and a real run_coroutine on the env.\n+\n+    Optionally captures exec call args into *exec_calls* list.\n+    Returns (exec_calls, run_kwargs, stdin_mock).\n+    \"\"\"\n+    if exec_calls is None:\n+        exec_calls = []\n+    run_kwargs: dict = {}\n+    stdin_mock = _make_mock_stdin()\n+\n+    async def mock_exec_fn(*args, **kwargs):\n+        exec_calls.append(args)\n+        proc = MagicMock()\n+        proc.wait = MagicMock()\n+        proc.wait.aio = AsyncMock(return_value=0)\n+        proc.stdin = stdin_mock\n+        proc.stderr = MagicMock()\n+        proc.stderr.read = MagicMock()\n+        proc.stderr.read.aio = AsyncMock(return_value=\"\")\n+        return proc\n+\n+    env._sandbox.exec = MagicMock()\n+    env._sandbox.exec.aio = mock_exec_fn\n+\n+    def real_run_coroutine(coro, **kwargs):\n+        run_kwargs.update(kwargs)\n+        loop = asyncio.new_event_loop()\n+        try:\n+            return loop.run_until_complete(coro)\n+        finally:\n+            loop.close()\n+\n+    env._worker.run_coroutine = real_run_coroutine\n+    return exec_calls, run_kwargs, stdin_mock\n+\n+\n+class TestModalBulkUpload:\n+    \"\"\"Test _modal_bulk_upload method.\"\"\"\n+\n+    def test_empty_files_is_noop(self, monkeypatch, tmp_path):\n+        \"\"\"Empty file list should not call worker.run_coroutine.\"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+        env._modal_bulk_upload([])\n+        env._worker.run_coroutine.assert_not_called()\n+\n+    def test_tar_archive_contains_all_files(self, monkeypatch, tmp_path):\n+        \"\"\"The tar archive sent via stdin should contain all files.\"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+\n+        src_a = tmp_path / \"a.json\"\n+        src_b = tmp_path / \"b.py\"\n+        src_a.write_text(\"cred_content\")\n+        src_b.write_text(\"skill_content\")\n+\n+        files = [\n+            (str(src_a), \"/root/.hermes/credentials/a.json\"),\n+            (str(src_b), \"/root/.hermes/skills/b.py\"),\n+        ]\n+\n+        exec_calls, _, stdin_mock = _wire_async_exec(env)\n+        env._modal_bulk_upload(files)\n+\n+        # Verify the command reads from stdin (no echo with embedded payload)\n+        assert len(exec_calls) == 1\n+        args = exec_calls[0]\n+        assert args[0] == \"bash\"\n+        assert args[1] == \"-c\"\n+        cmd = args[2]\n+        assert \"mkdir -p\" in cmd\n+        assert \"base64 -d\" in cmd\n+        assert \"tar xzf\" in cmd\n+        assert \"-C /\" in cmd\n+\n+        # Reassemble the base64 payload from stdin chunks and verify tar contents\n+        payload = \"\".join(stdin_mock._written_chunks)\n+        tar_data = base64.b64decode(payload)\n+        buf = io.BytesIO(tar_data)\n+        with tarfile.open(fileobj=buf, mode=\"r:gz\") as tar:\n+            names = sorted(tar.getnames())\n+            assert \"root/.hermes/credentials/a.json\" in names\n+            assert \"root/.hermes/skills/b.py\" in names\n+\n+            # Verify content\n+            a_content = tar.extractfile(\"root/.hermes/credentials/a.json\").read()\n+            assert a_content == b\"cred_content\"\n+            b_content = tar.extractfile(\"root/.hermes/skills/b.py\").read()\n+            assert b_content == b\"skill_content\"\n+\n+        # Verify stdin was closed\n+        stdin_mock.write_eof.assert_called_once()\n+\n+    def test_mkdir_includes_all_parents(self, monkeypatch, tmp_path):\n+        \"\"\"Remote parent directories should be pre-created in the command.\"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+\n+        src = tmp_path / \"f.txt\"\n+        src.write_text(\"data\")\n+\n+        files = [\n+            (str(src), \"/root/.hermes/credentials/f.txt\"),\n+            (str(src), \"/root/.hermes/skills/deep/nested/f.txt\"),\n+        ]\n+\n+        exec_calls, _, _ = _wire_async_exec(env)\n+        env._modal_bulk_upload(files)\n+\n+        cmd = exec_calls[0][2]\n+        assert \"/root/.hermes/credentials\" in cmd\n+        assert \"/root/.hermes/skills/deep/nested\" in cmd\n+\n+    def test_single_exec_call(self, monkeypatch, tmp_path):\n+        \"\"\"Bulk upload should use exactly one exec call regardless of file count.\"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+\n+        files = []\n+        for i in range(20):\n+            src = tmp_path / f\"file_{i}.txt\"\n+            src.write_text(f\"content_{i}\")\n+            files.append((str(src), f\"/root/.hermes/cache/file_{i}.txt\"))\n+\n+        exec_calls, _, _ = _wire_async_exec(env)\n+        env._modal_bulk_upload(files)\n+\n+        # Should be exactly 1 exec call, not 20\n+        assert len(exec_calls) == 1\n+\n+    def test_bulk_upload_wired_in_filesyncmanager(self, monkeypatch):\n+        \"\"\"Verify ModalEnvironment passes bulk_upload_fn to FileSyncManager.\"\"\"\n+        captured_kwargs = {}\n+\n+        def capture_fsm(**kwargs):\n+            captured_kwargs.update(kwargs)\n+            return type(\"M\", (), {\"sync\": lambda self, **k: None})()\n+\n+        monkeypatch.setattr(modal_env, \"FileSyncManager\", capture_fsm)\n+\n+        # Create a minimal env without full __init__\n+        env = object.__new__(modal_env.ModalEnvironment)\n+        env._sandbox = MagicMock()\n+        env._worker = MagicMock()\n+        env._persistent = False\n+        env._task_id = \"test\"\n+\n+        # Manually call the part of __init__ that wires FileSyncManager\n+        from tools.environments.file_sync import iter_sync_files\n+        env._sync_manager = modal_env.FileSyncManager(\n+            get_files_fn=lambda: iter_sync_files(\"/root/.hermes\"),\n+            upload_fn=env._modal_upload,\n+            delete_fn=env._modal_delete,\n+            bulk_upload_fn=env._modal_bulk_upload,\n+        )\n+\n+        assert \"bulk_upload_fn\" in captured_kwargs\n+        assert captured_kwargs[\"bulk_upload_fn\"] is not None\n+        assert callable(captured_kwargs[\"bulk_upload_fn\"])\n+\n+    def test_timeout_set_to_120(self, monkeypatch, tmp_path):\n+        \"\"\"Bulk upload uses a 120s timeout (not the per-file 15s).\"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+\n+        src = tmp_path / \"f.txt\"\n+        src.write_text(\"data\")\n+        files = [(str(src), \"/root/.hermes/f.txt\")]\n+\n+        _, run_kwargs, _ = _wire_async_exec(env)\n+        env._modal_bulk_upload(files)\n+\n+        assert run_kwargs.get(\"timeout\") == 120\n+\n+    def test_nonzero_exit_raises(self, monkeypatch, tmp_path):\n+        \"\"\"Non-zero exit code from remote exec should raise RuntimeError.\"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+\n+        src = tmp_path / \"f.txt\"\n+        src.write_text(\"data\")\n+        files = [(str(src), \"/root/.hermes/f.txt\")]\n+\n+        stdin_mock = _make_mock_stdin()\n+\n+        async def mock_exec_fn(*args, **kwargs):\n+            proc = MagicMock()\n+            proc.wait = MagicMock()\n+            proc.wait.aio = AsyncMock(return_value=1)  # non-zero exit\n+            proc.stdin = stdin_mock\n+            proc.stderr = MagicMock()\n+            proc.stderr.read = MagicMock()\n+            proc.stderr.read.aio = AsyncMock(return_value=\"tar: error\")\n+            return proc\n+\n+        env._sandbox.exec = MagicMock()\n+        env._sandbox.exec.aio = mock_exec_fn\n+\n+        def real_run_coroutine(coro, **kwargs):\n+            loop = asyncio.new_event_loop()\n+            try:\n+                return loop.run_until_complete(coro)\n+            finally:\n+                loop.close()\n+\n+        env._worker.run_coroutine = real_run_coroutine\n+\n+        with pytest.raises(RuntimeError, match=\"Modal bulk upload failed\"):\n+            env._modal_bulk_upload(files)\n+\n+    def test_payload_not_in_command_string(self, monkeypatch, tmp_path):\n+        \"\"\"The base64 payload must NOT appear in the bash -c argument.\n+\n+        This is the core ARG_MAX fix: the payload goes through stdin,\n+        not embedded in the command string.\n+        \"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+\n+        src = tmp_path / \"f.txt\"\n+        src.write_text(\"some data to upload\")\n+        files = [(str(src), \"/root/.hermes/f.txt\")]\n+\n+        exec_calls, _, stdin_mock = _wire_async_exec(env)\n+        env._modal_bulk_upload(files)\n+\n+        # The command should NOT contain an echo with the payload\n+        cmd = exec_calls[0][2]\n+        assert \"echo\" not in cmd\n+        # The payload should go through stdin\n+        assert len(stdin_mock._written_chunks) > 0\n+\n+    def test_stdin_chunked_for_large_payloads(self, monkeypatch, tmp_path):\n+        \"\"\"Payloads larger than _STDIN_CHUNK_SIZE should be split into multiple writes.\"\"\"\n+        env = _make_mock_modal_env(monkeypatch, tmp_path)\n+\n+        # Use random bytes so gzip cannot compress them -- ensures the\n+        # base64 payload exceeds one 1 MB chunk.\n+        import os as _os\n+        src = tmp_path / \"large.bin\"\n+        src.write_bytes(_os.urandom(1024 * 1024 + 512 * 1024))\n+        files = [(str(src), \"/root/.hermes/large.bin\")]\n+\n+        exec_calls, _, stdin_mock = _wire_async_exec(env)\n+        env._modal_bulk_upload(files)\n+\n+        # Should have multiple stdin write chunks\n+        assert len(stdin_mock._written_chunks) >= 2\n+\n+        # Reassembled payload should still decode to valid tar\n+        payload = \"\".join(stdin_mock._written_chunks)\n+        tar_data = base64.b64decode(payload)\n+        buf = io.BytesIO(tar_data)\n+        with tarfile.open(fileobj=buf, mode=\"r:gz\") as tar:\n+            names = tar.getnames()\n+            assert \"root/.hermes/large.bin\" in names"]},{"file":"tests/tools/test_ssh_bulk_upload.py","hunks":["@@ -0,0 +1,517 @@\n+\"\"\"Tests for SSH bulk upload via tar pipe.\"\"\"\n+\n+import os\n+import subprocess\n+from pathlib import Path\n+from unittest.mock import MagicMock, patch\n+\n+import pytest\n+\n+from tools.environments import ssh as ssh_env\n+from tools.environments.file_sync import quoted_mkdir_command, unique_parent_dirs\n+from tools.environments.ssh import SSHEnvironment\n+\n+\n+def _mock_proc(*, returncode=0, poll_return=0, communicate_return=(b\"\", b\"\"),\n+               stderr_read=b\"\"):\n+    \"\"\"Create a MagicMock mimicking subprocess.Popen for tar/ssh pipes.\"\"\"\n+    m = MagicMock()\n+    m.stdout = MagicMock()\n+    m.returncode = returncode\n+    m.poll.return_value = poll_return\n+    m.communicate.return_value = communicate_return\n+    m.stderr = MagicMock()\n+    m.stderr.read.return_value = stderr_read\n+    return m\n+\n+\n[redacted:email]\n+def mock_env(monkeypatch):\n+    \"\"\"Create an SSHEnvironment with mocked connection/sync.\"\"\"\n+    monkeypatch.setattr(ssh_env.shutil, \"which\", lambda _name: \"/usr/bin/ssh\")\n+    monkeypatch.setattr(ssh_env.SSHEnvironment, \"_establish_connection\", lambda self: None)\n+    monkeypatch.setattr(ssh_env.SSHEnvironment, \"_detect_remote_home\", lambda self: \"/home/testuser\")\n+    monkeypatch.setattr(ssh_env.SSHEnvironment, \"_ensure_remote_dirs\", lambda self: None)\n+    monkeypatch.setattr(ssh_env.SSHEnvironment, \"init_session\", lambda self: None)\n+    monkeypatch.setattr(\n+        ssh_env, \"FileSyncManager\",\n+        lambda **kw: type(\"M\", (), {\"sync\": lambda self, **k: None})(),\n+    )\n+    return SSHEnvironment(host=\"example.com\", user=\"testuser\")\n+\n+\n+class TestSSHBulkUpload:\n+    \"\"\"Unit tests for _ssh_bulk_upload — tar pipe mechanics.\"\"\"\n+\n+    def test_empty_files_is_noop(self, mock_env):\n+        \"\"\"Empty file list should not spawn any subprocesses.\"\"\"\n+        with patch.object(subprocess, \"run\") as mock_run, \\\n+             patch.object(subprocess, \"Popen\") as mock_popen:\n+            mock_env._ssh_bulk_upload([])\n+            mock_run.assert_not_called()\n+            mock_popen.assert_not_called()\n+\n+    def test_mkdir_batched_into_single_call(self, mock_env, tmp_path):\n+        \"\"\"All parent directories should be created in one SSH call.\"\"\"\n+        # Create test files\n+        f1 = tmp_path / \"a.txt\"\n+        f1.write_text(\"aaa\")\n+        f2 = tmp_path / \"b.txt\"\n+        f2.write_text(\"bbb\")\n+\n+        files = [\n+            (str(f1), \"/home/testuser/.hermes/skills/a.txt\"),\n+            (str(f2), \"/home/testuser/.hermes/credentials/b.txt\"),\n+        ]\n+\n+        # Mock subprocess.run for mkdir and Popen for tar pipe\n+        mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0))\n+\n+        def make_proc(cmd, **kwargs):\n+            m = MagicMock()\n+            m.stdout = MagicMock()\n+            m.returncode = 0\n+            m.poll.return_value = 0\n+            m.communicate.return_value = (b\"\", b\"\")\n+            m.stderr = MagicMock()\n+            m.stderr.read.return_value = b\"\"\n+            return m\n+\n+        with patch.object(subprocess, \"run\", mock_run), \\\n+             patch.object(subprocess, \"Popen\", side_effect=make_proc):\n+            mock_env._ssh_bulk_upload(files)\n+\n+        # Exactly one subprocess.run call for mkdir\n+        assert mock_run.call_count == 1\n+        mkdir_cmd = mock_run.call_args[0][0]\n+        # Should contain mkdir -p with both parent dirs\n+        mkdir_str = \" \".join(mkdir_cmd)\n+        assert \"mkdir -p\" in mkdir_str\n+        assert \"/home/testuser/.hermes/skills\" in mkdir_str\n+        assert \"/home/testuser/.hermes/credentials\" in mkdir_str\n+\n+    def test_staging_symlinks_mirror_remote_layout(self, mock_env, tmp_path):\n+        \"\"\"Symlinks in staging dir should mirror the remote path structure.\"\"\"\n+        f1 = tmp_path / \"local_a.txt\"\n+        f1.write_text(\"content a\")\n+\n+        files = [\n+            (str(f1), \"/home/testuser/.hermes/skills/my_skill.md\"),\n+        ]\n+\n+        staging_paths = []\n+\n+        def capture_tar_cmd(cmd, **kwargs):\n+            if cmd[0] == \"tar\":\n+                # Capture the staging dir from -C argument\n+                c_idx = cmd.index(\"-C\")\n+                staging_dir = cmd[c_idx + 1]\n+                # Check the symlink exists\n+                expected = os.path.join(\n+                    staging_dir, \"home/testuser/.hermes/skills/my_skill.md\"\n+                )\n+                staging_paths.append(expected)\n+                assert os.path.islink(expected), f\"Expected symlink at {expected}\"\n+                assert os.readlink(expected) == os.path.abspath(str(f1))\n+\n+            mock = MagicMock()\n+            mock.stdout = MagicMock()\n+            mock.returncode = 0\n+            mock.poll.return_value = 0\n+            mock.communicate.return_value = (b\"\", b\"\")\n+            mock.stderr = MagicMock()\n+            mock.stderr.read.return_value = b\"\"\n+            return mock\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=capture_tar_cmd):\n+            mock_env._ssh_bulk_upload(files)\n+\n+        assert len(staging_paths) == 1, \"tar command should have been called\"\n+\n+    def test_tar_pipe_commands(self, mock_env, tmp_path):\n+        \"\"\"Verify tar and SSH commands are wired correctly.\"\"\"\n+        f1 = tmp_path / \"x.txt\"\n+        f1.write_text(\"x\")\n+\n+        files = [(str(f1), \"/home/testuser/.hermes/cache/x.txt\")]\n+\n+        popen_cmds = []\n+\n+        def capture_popen(cmd, **kwargs):\n+            popen_cmds.append(cmd)\n+            mock = MagicMock()\n+            mock.stdout = MagicMock()\n+            mock.returncode = 0\n+            mock.poll.return_value = 0\n+            mock.communicate.return_value = (b\"\", b\"\")\n+            mock.stderr = MagicMock()\n+            mock.stderr.read.return_value = b\"\"\n+            return mock\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=capture_popen):\n+            mock_env._ssh_bulk_upload(files)\n+\n+        assert len(popen_cmds) == 2, \"Should spawn tar + ssh processes\"\n+\n+        tar_cmd = popen_cmds[0]\n+        ssh_cmd = popen_cmds[1]\n+\n+        # tar: create, dereference symlinks, to stdout\n+        assert tar_cmd[0] == \"tar\"\n+        assert \"-chf\" in tar_cmd\n+        assert \"-\" in tar_cmd  # stdout\n+        assert \"-C\" in tar_cmd\n+\n+        # ssh: extract from stdin at /\n+        ssh_str = \" \".join(ssh_cmd)\n+        assert \"ssh\" in ssh_str\n+        assert \"tar xf - -C /\" in ssh_str\n+        assert \"[redacted:email]\" in ssh_str\n+\n+    def test_mkdir_failure_raises(self, mock_env, tmp_path):\n+        \"\"\"mkdir failure should raise RuntimeError before tar pipe.\"\"\"\n+        f1 = tmp_path / \"y.txt\"\n+        f1.write_text(\"y\")\n+        files = [(str(f1), \"/home/testuser/.hermes/skills/y.txt\")]\n+\n+        failed_run = subprocess.CompletedProcess([], 1, stderr=\"Permission denied\")\n+        with patch.object(subprocess, \"run\", return_value=failed_run):\n+            with pytest.raises(RuntimeError, match=\"remote mkdir failed\"):\n+                mock_env._ssh_bulk_upload(files)\n+\n+    def test_tar_create_failure_raises(self, mock_env, tmp_path):\n+        \"\"\"tar create failure should raise RuntimeError.\"\"\"\n+        f1 = tmp_path / \"z.txt\"\n+        f1.write_text(\"z\")\n+        files = [(str(f1), \"/home/testuser/.hermes/skills/z.txt\")]\n+\n+        mock_tar = MagicMock()\n+        mock_tar.stdout = MagicMock()\n+        mock_tar.returncode = 1\n+        mock_tar.poll.return_value = 1\n+        mock_tar.communicate.return_value = (b\"tar: error\", b\"\")\n+        mock_tar.stderr = MagicMock()\n+        mock_tar.stderr.read.return_value = b\"tar: error\"\n+\n+        mock_ssh = MagicMock()\n+        mock_ssh.communicate.return_value = (b\"\", b\"\")\n+        mock_ssh.returncode = 0\n+\n+        def popen_side_effect(cmd, **kwargs):\n+            if cmd[0] == \"tar\":\n+                return mock_tar\n+            return mock_ssh\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=popen_side_effect):\n+            with pytest.raises(RuntimeError, match=\"tar create failed\"):\n+                mock_env._ssh_bulk_upload(files)\n+\n+    def test_ssh_extract_failure_raises(self, mock_env, tmp_path):\n+        \"\"\"SSH tar extract failure should raise RuntimeError.\"\"\"\n+        f1 = tmp_path / \"w.txt\"\n+        f1.write_text(\"w\")\n+        files = [(str(f1), \"/home/testuser/.hermes/skills/w.txt\")]\n+\n+        mock_tar = MagicMock()\n+        mock_tar.stdout = MagicMock()\n+        mock_tar.returncode = 0\n+        mock_tar.poll.return_value = 0\n+        mock_tar.communicate.return_value = (b\"\", b\"\")\n+        mock_tar.stderr = MagicMock()\n+        mock_tar.stderr.read.return_value = b\"\"\n+\n+        mock_ssh = MagicMock()\n+        mock_ssh.communicate.return_value = (b\"\", b\"Permission denied\")\n+        mock_ssh.returncode = 1\n+\n+        def popen_side_effect(cmd, **kwargs):\n+            if cmd[0] == \"tar\":\n+                return mock_tar\n+            return mock_ssh\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=popen_side_effect):\n+            with pytest.raises(RuntimeError, match=\"tar extract over SSH failed\"):\n+                mock_env._ssh_bulk_upload(files)\n+\n+    def test_ssh_command_uses_control_socket(self, mock_env, tmp_path):\n+        \"\"\"SSH command for tar extract should reuse ControlMaster socket.\"\"\"\n+        f1 = tmp_path / \"c.txt\"\n+        f1.write_text(\"c\")\n+        files = [(str(f1), \"/home/testuser/.hermes/cache/c.txt\")]\n+\n+        popen_cmds = []\n+\n+        def capture_popen(cmd, **kwargs):\n+            popen_cmds.append(cmd)\n+            mock = MagicMock()\n+            mock.stdout = MagicMock()\n+            mock.returncode = 0\n+            mock.poll.return_value = 0\n+            mock.communicate.return_value = (b\"\", b\"\")\n+            mock.stderr = MagicMock()\n+            mock.stderr.read.return_value = b\"\"\n+            return mock\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=capture_popen):\n+            mock_env._ssh_bulk_upload(files)\n+\n+        # The SSH command (second Popen call) should include ControlPath\n+        ssh_cmd = popen_cmds[1]\n+        assert f\"ControlPath={mock_env.control_socket}\" in \" \".join(ssh_cmd)\n+\n+    def test_custom_port_and_key_in_ssh_command(self, monkeypatch, tmp_path):\n+        \"\"\"Bulk upload SSH command should include custom port and key.\"\"\"\n+        monkeypatch.setattr(ssh_env.shutil, \"which\", lambda _name: \"/usr/bin/ssh\")\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"_establish_connection\", lambda self: None)\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"_detect_remote_home\", lambda self: \"/home/u\")\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"_ensure_remote_dirs\", lambda self: None)\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"init_session\", lambda self: None)\n+        monkeypatch.setattr(\n+            ssh_env, \"FileSyncManager\",\n+            lambda **kw: type(\"M\", (), {\"sync\": lambda self, **k: None})(),\n+        )\n+        env = SSHEnvironment(host=\"h\", user=\"u\", port=2222, key_path=\"/my/key\")\n+\n+        f1 = tmp_path / \"d.txt\"\n+        f1.write_text(\"d\")\n+        files = [(str(f1), \"/home/u/.hermes/skills/d.txt\")]\n+\n+        run_cmds = []\n+        popen_cmds = []\n+\n+        def capture_run(cmd, **kwargs):\n+            run_cmds.append(cmd)\n+            return subprocess.CompletedProcess([], 0)\n+\n+        def capture_popen(cmd, **kwargs):\n+            popen_cmds.append(cmd)\n+            mock = MagicMock()\n+            mock.stdout = MagicMock()\n+            mock.returncode = 0\n+            mock.poll.return_value = 0\n+            mock.communicate.return_value = (b\"\", b\"\")\n+            mock.stderr = MagicMock()\n+            mock.stderr.read.return_value = b\"\"\n+            return mock\n+\n+        with patch.object(subprocess, \"run\", side_effect=capture_run), \\\n+             patch.object(subprocess, \"Popen\", side_effect=capture_popen):\n+            env._ssh_bulk_upload(files)\n+\n+        # Check mkdir SSH call includes port and key\n+        assert len(run_cmds) == 1\n+        mkdir_cmd = run_cmds[0]\n+        assert \"-p\" in mkdir_cmd and \"2222\" in mkdir_cmd\n+        assert \"-i\" in mkdir_cmd and \"/my/key\" in mkdir_cmd\n+\n+        # Check tar extract SSH call includes port and key\n+        ssh_cmd = popen_cmds[1]\n+        assert \"-p\" in ssh_cmd and \"2222\" in ssh_cmd\n+        assert \"-i\" in ssh_cmd and \"/my/key\" in ssh_cmd\n+\n+    def test_parent_dirs_deduplicated(self, mock_env, tmp_path):\n+        \"\"\"Multiple files in the same dir should produce one mkdir entry.\"\"\"\n+        f1 = tmp_path / \"a.txt\"\n+        f1.write_text(\"a\")\n+        f2 = tmp_path / \"b.txt\"\n+        f2.write_text(\"b\")\n+        f3 = tmp_path / \"c.txt\"\n+        f3.write_text(\"c\")\n+\n+        files = [\n+            (str(f1), \"/home/testuser/.hermes/skills/a.txt\"),\n+            (str(f2), \"/home/testuser/.hermes/skills/b.txt\"),\n+            (str(f3), \"/home/testuser/.hermes/credentials/c.txt\"),\n+        ]\n+\n+        run_cmds = []\n+\n+        def capture_run(cmd, **kwargs):\n+            run_cmds.append(cmd)\n+            return subprocess.CompletedProcess([], 0)\n+\n+        def make_mock_proc(cmd, **kwargs):\n+            mock = MagicMock()\n+            mock.stdout = MagicMock()\n+            mock.returncode = 0\n+            mock.poll.return_value = 0\n+            mock.communicate.return_value = (b\"\", b\"\")\n+            mock.stderr = MagicMock()\n+            mock.stderr.read.return_value = b\"\"\n+            return mock\n+\n+        with patch.object(subprocess, \"run\", side_effect=capture_run), \\\n+             patch.object(subprocess, \"Popen\", side_effect=make_mock_proc):\n+            mock_env._ssh_bulk_upload(files)\n+\n+        # Only one mkdir call\n+        assert len(run_cmds) == 1\n+        mkdir_str = \" \".join(run_cmds[0])\n+        # skills dir should appear exactly once despite two files\n+        assert mkdir_str.count(\"/home/testuser/.hermes/skills\") == 1\n+        assert \"/home/testuser/.hermes/credentials\" in mkdir_str\n+\n+    def test_tar_stdout_closed_for_sigpipe(self, mock_env, tmp_path):\n+        \"\"\"tar_proc.stdout must be closed so SIGPIPE propagates correctly.\"\"\"\n+        f1 = tmp_path / \"s.txt\"\n+        f1.write_text(\"s\")\n+        files = [(str(f1), \"/home/testuser/.hermes/skills/s.txt\")]\n+\n+        mock_tar_stdout = MagicMock()\n+\n+        def make_proc(cmd, **kwargs):\n+            mock = MagicMock()\n+            if cmd[0] == \"tar\":\n+                mock.stdout = mock_tar_stdout\n+            else:\n+                mock.stdout = MagicMock()\n+            mock.returncode = 0\n+            mock.poll.return_value = 0\n+            mock.communicate.return_value = (b\"\", b\"\")\n+            mock.stderr = MagicMock()\n+            mock.stderr.read.return_value = b\"\"\n+            return mock\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=make_proc):\n+            mock_env._ssh_bulk_upload(files)\n+\n+        mock_tar_stdout.close.assert_called_once()\n+\n+    def test_timeout_kills_both_processes(self, mock_env, tmp_path):\n+        \"\"\"TimeoutExpired during communicate should kill both processes.\"\"\"\n+        f1 = tmp_path / \"t.txt\"\n+        f1.write_text(\"t\")\n+        files = [(str(f1), \"/home/testuser/.hermes/skills/t.txt\")]\n+\n+        mock_tar = MagicMock()\n+        mock_tar.stdout = MagicMock()\n+        mock_tar.returncode = None\n+        mock_tar.poll.return_value = None\n+\n+        mock_ssh = MagicMock()\n+        mock_ssh.communicate.side_effect = subprocess.TimeoutExpired(\"ssh\", 120)\n+        mock_ssh.returncode = None\n+\n+        def make_proc(cmd, **kwargs):\n+            if cmd[0] == \"tar\":\n+                return mock_tar\n+            return mock_ssh\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=make_proc):\n+            with pytest.raises(RuntimeError, match=\"SSH bulk upload timed out\"):\n+                mock_env._ssh_bulk_upload(files)\n+\n+        mock_tar.kill.assert_called_once()\n+        mock_ssh.kill.assert_called_once()\n+\n+\n+class TestSSHBulkUploadWiring:\n+    \"\"\"Verify bulk_upload_fn is wired into FileSyncManager.\"\"\"\n+\n+    def test_filesyncmanager_receives_bulk_upload_fn(self, monkeypatch):\n+        \"\"\"SSHEnvironment should pass _ssh_bulk_upload to FileSyncManager.\"\"\"\n+        monkeypatch.setattr(ssh_env.shutil, \"which\", lambda _name: \"/usr/bin/ssh\")\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"_establish_connection\", lambda self: None)\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"_detect_remote_home\", lambda self: \"/root\")\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"_ensure_remote_dirs\", lambda self: None)\n+        monkeypatch.setattr(ssh_env.SSHEnvironment, \"init_session\", lambda self: None)\n+\n+        captured_kwargs = {}\n+\n+        class FakeSyncManager:\n+            def __init__(self, **kwargs):\n+                captured_kwargs.update(kwargs)\n+\n+            def sync(self, **kw):\n+                pass\n+\n+        monkeypatch.setattr(ssh_env, \"FileSyncManager\", FakeSyncManager)\n+\n+        env = SSHEnvironment(host=\"h\", user=\"u\")\n+\n+        assert \"bulk_upload_fn\" in captured_kwargs\n+        assert captured_kwargs[\"bulk_upload_fn\"] is not None\n+        # Should be the bound method\n+        assert callable(captured_kwargs[\"bulk_upload_fn\"])\n+\n+\n+class TestSharedHelpers:\n+    \"\"\"Direct unit tests for file_sync.py helpers.\"\"\"\n+\n+    def test_quoted_mkdir_command_basic(self):\n+        result = quoted_mkdir_command([\"/a\", \"/b/c\"])\n+        assert result == \"mkdir -p /a /b/c\"\n+\n+    def test_quoted_mkdir_command_quotes_special_chars(self):\n+        result = quoted_mkdir_command([\"/path/with spaces\", \"/path/'quotes'\"])\n+        assert \"mkdir -p\" in result\n+        # shlex.quote wraps in single quotes\n+        assert \"'/path/with spaces'\" in result\n+\n+    def test_quoted_mkdir_command_empty(self):\n+        result = quoted_mkdir_command([])\n+        assert result == \"mkdir -p \"\n+\n+    def test_unique_parent_dirs_deduplicates(self):\n+        files = [\n+            (\"/local/a.txt\", \"/remote/dir/a.txt\"),\n+            (\"/local/b.txt\", \"/remote/dir/b.txt\"),\n+            (\"/local/c.txt\", \"/remote/other/c.txt\"),\n+        ]\n+        result = unique_parent_dirs(files)\n+        assert result == [\"/remote/dir\", \"/remote/other\"]\n+\n+    def test_unique_parent_dirs_sorted(self):\n+        files = [\n+            (\"/local/z.txt\", \"/z/file.txt\"),\n+            (\"/local/a.txt\", \"/a/file.txt\"),\n+        ]\n+        result = unique_parent_dirs(files)\n+        assert result == [\"/a\", \"/z\"]\n+\n+    def test_unique_parent_dirs_empty(self):\n+        assert unique_parent_dirs([]) == []\n+\n+\n+class TestSSHBulkUploadEdgeCases:\n+    \"\"\"Edge cases for _ssh_bulk_upload.\"\"\"\n+\n+    def test_ssh_popen_failure_kills_tar(self, mock_env, tmp_path):\n+        \"\"\"If SSH Popen raises, tar process must be killed and cleaned up.\"\"\"\n+        f1 = tmp_path / \"e.txt\"\n+        f1.write_text(\"e\")\n+        files = [(str(f1), \"/home/testuser/.hermes/skills/e.txt\")]\n+\n+        mock_tar = _mock_proc()\n+\n+        call_count = 0\n+\n+        def failing_ssh_popen(cmd, **kwargs):\n+            nonlocal call_count\n+            call_count += 1\n+            if call_count == 1:\n+                return mock_tar  # tar Popen succeeds\n+            raise OSError(\"SSH binary not found\")\n+\n+        with patch.object(subprocess, \"run\",\n+                          return_value=subprocess.CompletedProcess([], 0)), \\\n+             patch.object(subprocess, \"Popen\", side_effect=failing_ssh_popen):\n+            with pytest.raises(OSError, match=\"SSH binary not found\"):\n+                mock_env._ssh_bulk_upload(files)\n+\n+        mock_tar.kill.assert_called_once()\n+        mock_tar.wait.assert_called_once()"]},{"file":"tools/environments/daytona.py","hunks":["@@ -15,7 +15,13 @@\n     BaseEnvironment,\n     _ThreadedProcessHandle,\n )\n-from tools.environments.file_sync import FileSyncManager, iter_sync_files, quoted_rm_command\n+from tools.environments.file_sync import (\n+    FileSyncManager,\n+    iter_sync_files,\n+    quoted_mkdir_command,\n+    quoted_rm_command,\n+    unique_parent_dirs,\n+)\n \n logger = logging.getLogger(__name__)\n ","@@ -150,11 +156,9 @@ def _daytona_bulk_upload(self, files: list[tuple[str, str]]) -> None:\n         if not files:\n             return\n \n-        # Pre-create all unique parent directories in one shell call\n-        parents = sorted({str(Path(remote).parent) for _, remote in files})\n+        parents = unique_parent_dirs(files)\n         if parents:\n-            mkdir_cmd = \"mkdir -p \" + \" \".join(shlex.quote(p) for p in parents)\n-            self._sandbox.process.exec(mkdir_cmd)\n+            self._sandbox.process.exec(quoted_mkdir_command(parents))\n \n         uploads = [\n             FileUpload(source=host_path, destination=remote_path)"]},{"file":"tools/environments/file_sync.py","hunks":["@@ -10,6 +10,7 @@\n import os\n import shlex\n import time\n+from pathlib import Path\n from typing import Callable\n \n from tools.environments.base import _file_mtime_key","@@ -60,6 +61,16 @@ def quoted_rm_command(remote_paths: list[str]) -> str:\n     return \"rm -f \" + \" \".join(shlex.quote(p) for p in remote_paths)\n \n \n+def quoted_mkdir_command(dirs: list[str]) -> str:\n+    \"\"\"Build a shell ``mkdir -p`` command for a batch of directories.\"\"\"\n+    return \"mkdir -p \" + \" \".join(shlex.quote(d) for d in dirs)\n+\n+\n+def unique_parent_dirs(files: list[tuple[str, str]]) -> list[str]:\n+    \"\"\"Extract sorted unique parent directories from (host, remote) pairs.\"\"\"\n+    return sorted({str(Path(remote).parent) for _, remote in files})\n+\n+\n class FileSyncManager:\n     \"\"\"Tracks local file changes and syncs to a remote environment.\n "]},{"file":"tools/environments/modal.py","hunks":["@@ -5,8 +5,11 @@\n \"\"\"\n \n import asyncio\n+import base64\n+import io\n import logging\n import shlex\n+import tarfile\n import threading\n from pathlib import Path\n from typing import Any, Optional","@@ -18,7 +21,13 @@\n     _load_json_store,\n     _save_json_store,\n )\n-from tools.environments.file_sync import FileSyncManager, iter_sync_files, quoted_rm_command\n+from tools.environments.file_sync import (\n+    FileSyncManager,\n+    iter_sync_files,\n+    quoted_mkdir_command,\n+    quoted_rm_command,\n+    unique_parent_dirs,\n+)\n \n logger = logging.getLogger(__name__)\n ","@@ -259,26 +268,84 @@ async def _create_sandbox(image_spec: Any):\n             get_files_fn=lambda: iter_sync_files(\"/root/.hermes\"),\n             upload_fn=self._modal_upload,\n             delete_fn=self._modal_delete,\n+            bulk_upload_fn=self._modal_bulk_upload,\n         )\n         self._sync_manager.sync(force=True)\n         self.init_session()\n \n     def _modal_upload(self, host_path: str, remote_path: str) -> None:\n-        \"\"\"Upload a single file via base64-over-exec.\"\"\"\n-        import base64\n+        \"\"\"Upload a single file via base64 piped through stdin.\"\"\"\n         content = Path(host_path).read_bytes()\n         b64 = base64.b64encode(content).decode(\"ascii\")\n         container_dir = str(Path(remote_path).parent)\n         cmd = (\n             f\"mkdir -p {shlex.quote(container_dir)} && \"\n-            f\"echo {shlex.quote(b64)} | base64 -d > {shlex.quote(remote_path)}\"\n+            f\"base64 -d > {shlex.quote(remote_path)}\"\n         )\n \n         async def _write():\n             proc = await self._sandbox.exec.aio(\"bash\", \"-c\", cmd)\n+            offset = 0\n+            chunk_size = self._STDIN_CHUNK_SIZE\n+            while offset < len(b64):\n+                proc.stdin.write(b64[offset:offset + chunk_size])\n+                await proc.stdin.drain.aio()\n+                offset += chunk_size\n+            proc.stdin.write_eof()\n+            await proc.stdin.drain.aio()\n             await proc.wait.aio()\n \n-        self._worker.run_coroutine(_write(), timeout=15)\n+        self._worker.run_coroutine(_write(), timeout=30)\n+\n+    # Modal SDK stdin buffer limit (legacy server path).  The command-router\n+    # path allows 16 MB, but we must stay under the smaller 2 MB cap for\n+    # compatibility.  Chunks are written below this threshold and flushed\n+    # individually via drain().\n+    _STDIN_CHUNK_SIZE = 1 * 1024 * 1024  # 1 MB — safe for both transport paths\n+\n+    def _modal_bulk_upload(self, files: list[tuple[str, str]]) -> None:\n+        \"\"\"Upload many files via tar archive piped through stdin.\n+\n+        Builds a gzipped tar archive in memory and streams it into a\n+        ``base64 -d | tar xzf -`` pipeline via the process's stdin,\n+        avoiding the Modal SDK's 64 KB ``ARG_MAX_BYTES`` exec-arg limit.\n+        \"\"\"\n+        if not files:\n+            return\n+\n+        buf = io.BytesIO()\n+        with tarfile.open(fileobj=buf, mode=\"w:gz\") as tar:\n+            for host_path, remote_path in files:\n+                tar.add(host_path, arcname=remote_path.lstrip(\"/\"))\n+        payload = base64.b64encode(buf.getvalue()).decode(\"ascii\")\n+\n+        parents = unique_parent_dirs(files)\n+        mkdir_part = quoted_mkdir_command(parents)\n+        cmd = f\"{mkdir_part} && base64 -d | tar xzf - -C /\"\n+\n+        async def _bulk():\n+            proc = await self._sandbox.exec.aio(\"bash\", \"-c\", cmd)\n+\n+            # Stream payload through stdin in chunks to stay under the\n+            # SDK's per-write buffer limit (2 MB legacy / 16 MB router).\n+            offset = 0\n+            chunk_size = self._STDIN_CHUNK_SIZE\n+            while offset < len(payload):\n+                proc.stdin.write(payload[offset:offset + chunk_size])\n+                await proc.stdin.drain.aio()\n+                offset += chunk_size\n+\n+            proc.stdin.write_eof()\n+            await proc.stdin.drain.aio()\n+\n+            exit_code = await proc.wait.aio()\n+            if exit_code != 0:\n+                stderr_text = await proc.stderr.read.aio()\n+                raise RuntimeError(\n+                    f\"Modal bulk upload failed (exit {exit_code}): {stderr_text}\"\n+                )\n+\n+        self._worker.run_coroutine(_bulk(), timeout=120)\n \n     def _modal_delete(self, remote_paths: list[str]) -> None:\n         \"\"\"Batch-delete remote files via exec.\"\"\""]},{"file":"tools/environments/ssh.py","hunks":["@@ -1,6 +1,7 @@\n \"\"\"SSH remote execution environment with ControlMaster connection persistence.\"\"\"\n \n import logging\n+import os\n import shlex\n import shutil\n import subprocess","@@ -8,7 +9,13 @@\n from pathlib import Path\n \n from tools.environments.base import BaseEnvironment, _popen_bash\n-from tools.environments.file_sync import FileSyncManager, iter_sync_files, quoted_rm_command\n+from tools.environments.file_sync import (\n+    FileSyncManager,\n+    iter_sync_files,\n+    quoted_mkdir_command,\n+    quoted_rm_command,\n+    unique_parent_dirs,\n+)\n \n logger = logging.getLogger(__name__)\n ","@@ -50,6 +57,7 @@ def __init__(self, host: str, user: str, cwd: str = \"~\",\n             get_files_fn=lambda: iter_sync_files(f\"{self._remote_home}/.hermes\"),\n             upload_fn=self._scp_upload,\n             delete_fn=self._ssh_delete,\n+            bulk_upload_fn=self._ssh_bulk_upload,\n         )\n         self._sync_manager.sync(force=True)\n ","@@ -107,9 +115,8 @@ def _ensure_remote_dirs(self) -> None:\n         \"\"\"Create base ~/.hermes directory tree on remote in one SSH call.\"\"\"\n         base = f\"{self._remote_home}/.hermes\"\n         dirs = [base, f\"{base}/skills\", f\"{base}/credentials\", f\"{base}/cache\"]\n-        mkdir_cmd = \"mkdir -p \" + \" \".join(shlex.quote(d) for d in dirs)\n         cmd = self._build_ssh_command()\n-        cmd.append(mkdir_cmd)\n+        cmd.append(quoted_mkdir_command(dirs))\n         subprocess.run(cmd, capture_output=True, text=True, timeout=10)\n \n     # _get_sync_files provided via iter_sync_files in FileSyncManager init","@@ -131,6 +138,84 @@ def _scp_upload(self, host_path: str, remote_path: str) -> None:\n         if result.returncode != 0:\n             raise RuntimeError(f\"scp failed: {result.stderr.strip()}\")\n \n+    def _ssh_bulk_upload(self, files: list[tuple[str, str]]) -> None:\n+        \"\"\"Upload many files in a single tar-over-SSH stream.\n+\n+        Pipes ``tar c`` on the local side through an SSH connection to\n+        ``tar x`` on the remote, transferring all files in one TCP stream\n+        instead of spawning a subprocess per file.  Directory creation is\n+        batched into a single ``mkdir -p`` call beforehand.\n+\n+        Typical improvement: ~580 files goes from O(N) scp round-trips\n+        to a single streaming transfer.\n+        \"\"\"\n+        if not files:\n+            return\n+\n+        parents = unique_parent_dirs(files)\n+        if parents:\n+            cmd = self._build_ssh_command()\n+            cmd.append(quoted_mkdir_command(parents))\n+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)\n+            if result.returncode != 0:\n+                raise RuntimeError(f\"remote mkdir failed: {result.stderr.strip()}\")\n+\n+        # Symlink staging avoids fragile GNU tar --transform rules.\n+        with tempfile.TemporaryDirectory(prefix=\"hermes-ssh-bulk-\") as staging:\n+            for host_path, remote_path in files:\n+                staged = os.path.join(staging, remote_path.lstrip(\"/\"))\n+                os.makedirs(os.path.dirname(staged), exist_ok=True)\n+                os.symlink(os.path.abspath(host_path), staged)\n+\n+            tar_cmd = [\"tar\", \"-chf\", \"-\", \"-C\", staging, \".\"]\n+            ssh_cmd = self._build_ssh_command()\n+            ssh_cmd.append(\"tar xf - -C /\")\n+\n+            tar_proc = subprocess.Popen(\n+                tar_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE\n+            )\n+            try:\n+                ssh_proc = subprocess.Popen(\n+                    ssh_cmd, stdin=tar_proc.stdout, stdout=subprocess.PIPE,\n+                    stderr=subprocess.PIPE,\n+                )\n+            except Exception:\n+                tar_proc.kill()\n+                tar_proc.wait()\n+                raise\n+\n+            # Allow tar_proc to receive SIGPIPE if ssh_proc exits early\n+            tar_proc.stdout.close()\n+\n+            try:\n+                _, ssh_stderr = ssh_proc.communicate(timeout=120)\n+                # Use communicate() instead of wait() to drain stderr and\n+                # avoid deadlock if tar produces more than PIPE_BUF of errors.\n+                tar_stderr_raw = b\"\"\n+                if tar_proc.poll() is None:\n+                    _, tar_stderr_raw = tar_proc.communicate(timeout=10)\n+                else:\n+                    tar_stderr_raw = tar_proc.stderr.read() if tar_proc.stderr else b\"\"\n+            except subprocess.TimeoutExpired:\n+                tar_proc.kill()\n+                ssh_proc.kill()\n+                tar_proc.wait()\n+                ssh_proc.wait()\n+                raise RuntimeError(\"SSH bulk upload timed out\")\n+\n+            if tar_proc.returncode != 0:\n+                raise RuntimeError(\n+                    f\"tar create failed (rc={tar_proc.returncode}): \"\n+                    f\"{tar_stderr_raw.decode(errors='replace').strip()}\"\n+                )\n+            if ssh_proc.returncode != 0:\n+                raise RuntimeError(\n+                    f\"tar extract over SSH failed (rc={ssh_proc.returncode}): \"\n+                    f\"{ssh_stderr.decode(errors='replace').strip()}\"\n+                )\n+\n+        logger.debug(\"SSH: bulk-uploaded %d file(s) via tar pipe\", len(files))\n+\n     def _ssh_delete(self, remote_paths: list[str]) -> None:\n         \"\"\"Batch-delete remote files in one SSH call.\"\"\"\n         cmd = self._build_ssh_command()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}