Profile clone-all bugs: recursion, bloat, symlinks

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #4417 fix(cli): support exporting the default root profile (#4366)

  merged to main

  files: hermes_cli/profiles.py, tests/hermes_cli/test_profiles.py

  Show the diff

  • hermes_cli/profiles.py

    @@ -58,6 +58,32 @@
         "processes.json",
     ]
     
    +# Directories/files to exclude when exporting the default (~/.hermes) profile.
    +# The default profile contains infrastructure (repo checkout, worktrees, DBs,
    +# caches, binaries) that named profiles don't have.  We exclude those so the
    +# export is a portable, reasonable-size archive of actual profile data.
    +_DEFAULT_EXPORT_EXCLUDE_ROOT = frozenset({
    +    # Infrastructure
    +    "hermes-agent",         # repo checkout (multi-GB)
    +    ".worktrees",           # git worktrees
    +    "profiles",             # other profiles — never recursive-export
    +    "bin",                  # installed binaries (tirith, etc.)
    +    "node_modules",         # npm packages
    +    # Databases & runtime state
    +    "state.db", "state.db-shm", "state.db-wal",
    +    "hermes_state.db",
    +    "response_store.db", "response_store.db-shm", "response_store.db-wal",
    +    "gateway.pid", "gateway_state.json", "processes.json",
    +    "auth.lock", "active_profile", ".update_check",
    +    "errors.log",
    +    ".hermes_history",
    +    # Caches (regenerated on use)
    +    "image_cache", "audio_cache", "document_cache",
    +    "browser_screenshots", "checkpoints",
    +    "sandboxes",
    +    "logs",                 # gateway logs
    +})
    +
     # Names that cannot be used as profile aliases
     _RESERVED_NAMES = frozenset({
         "hermes", "default", "test", "tmp", "root", "sudo",
    @@ -685,11 +711,37 @@ def get_active_profile_name() -> str:
     # Export / Import
     # ---------------------------------------------------------------------------
     
    +def _default_export_ignore(root_dir: Path):
    +    """Return an *ignore* callable for :func:`shutil.copytree`.
    +
    +    At the root level it excludes everything in ``_DEFAULT_EXPORT_EXCLUDE_ROOT``.
    +    At all levels it excludes ``__pycache__``, sockets, and temp files.
    +    """
    +
    +    def _ignore(directory: str, contents: list) -> set:
    +        ignored: set = set()
    +        for entry in contents:
    +            # Universal exclusions (any depth)
    +            if entry == "__pycache__" or entry.endswith((".sock", ".tmp")):
    +                ignored.add(entry)
    +            # npm lockfiles can appear at root
    +            elif entry in ("package.json", "package-lock.json"):
    +                ignored.add(entry)
    +        # Root-level exclusions
    +        if Path(directory) == root_dir:
    +            ignored.update(c for c in contents if c in _DEFAULT_EXPORT_EXCLUDE_ROOT)
    +        return ignored
    +
    +    return _ignore
    +
    +
     def export_profile(name: str, output_path: str) -> Path:
         """Export a profile to a tar.gz archive.
     
         Returns the output file path.
         """
    +    import tempfile
    +
         validate_profile_name(name)
         profile_dir = get_profile_dir(name)
         if not profile_dir.is_dir():
    @@ -698,6 +750,21 @@ def export_profile(name: str, output_path: str) -> Path:
         output = Path(output_path)
         # shutil.make_archive wants the base name without extension
         base = str(output).removesuffix(".tar.gz").removesuffix(".tgz")
    +
    +    if name == "default":
    +        # The default profile IS ~/.hermes itself — its parent is ~/ and its
    +        # directory name is ".hermes", not "default".  We stage a clean copy
    +        # under a temp dir so the archive contains ``default/...``.
    +        with tempfile.TemporaryDirectory() as tmpdir:
    +            staged = Path(tmpdir) / "default"
    +            shutil.copytree(
    +                profile_dir,
    +                staged,
    +                ignore=_default_export_ignore(profile_dir),
    +            )
    +            result = shutil.make_archive(base, "gztar", tmpdir, "default")
    +            return Path(result)
    +
         result = shutil.make_archive(base, "gztar", str(profile_dir.parent), name)
         return Path(result)
     
    @@ -788,6 +855,15 @@ def import_profile(archive_path: str, name: Optional[str] = None) -> Path:
                 "Specify it explicitly: hermes profile import <archive> --name <name>"
             )
     
    +    # Archives exported from the default profile have "default/" as top-level
    +    # dir.  Importing as "default" would target ~/.hermes itself — disallow
    +    # that and guide the user toward a named profile.
    +    if inferred_name == "default":
    +        raise ValueError(
    +            "Cannot import as 'default' — that is the built-in root profile (~/.hermes). "
    +            "Specify a different name: hermes profile import <archive> --name <name>"
    +        )
    +
         validate_profile_name(inferred_name)
         profile_dir = get_profile_dir(inferred_name)
         if profile_dir.exists():

  • tests/hermes_cli/test_profiles.py

    @@ -488,6 +488,149 @@ def test_export_nonexistent_raises(self, profile_env, tmp_path):
             with pytest.raises(FileNotFoundError):
                 export_profile("nonexistent", str(tmp_path / "out.tar.gz"))
     
    +    # ---------------------------------------------------------------
    +    # Default profile export / import
    +    # ---------------------------------------------------------------
    +
    +    def test_export_default_creates_valid_archive(self, profile_env, tmp_path):
    +        """Exporting the default profile produces a valid tar.gz."""
    +        default_dir = get_profile_dir("default")
    +        (default_dir / "config.yaml").write_text("model: test")
    +
    +        output = tmp_path / "export" / "default.tar.gz"
    +        output.parent.mkdir(parents=True, exist_ok=True)
    +        result = export_profile("default", str(output))
    +
    +        assert Path(result).exists()
    +        assert tarfile.is_tarfile(str(result))
    +
    +    def test_export_default_includes_profile_data(self, profile_env, tmp_path):
    +        """Profile data files end up in the archive."""
    +        default_dir = get_profile_dir("default")
    +        (default_dir / "config.yaml").write_text("model: test")
    +        (default_dir / ".env").write_text("KEY=val")
    +        (default_dir / "SOUL.md").write_text("Be nice.")
    +        mem_dir = default_dir / "memories"
    +        mem_dir.mkdir(exist_ok=True)
    +        (mem_dir / "MEMORY.md").write_text("remember this")
    +
    +        output = tmp_path / "export" / "default.tar.gz"
    +        output.parent.mkdir(parents=True, exist_ok=True)
    +        export_profile("default", str(output))
    +
    +        with tarfile.open(str(output), "r:gz") as tf:
    +            names = tf.getnames()
    +
    +        assert "default/config.yaml" in names
    +        assert "default/.env" in names
    +        assert "default/SOUL.md" in names
    +        assert "default/memories/MEMORY.md" in names
    +
    +    def test_export_default_excludes_infrastructure(self, profile_env, tmp_path):
    +        """Repo checkout, worktrees, profiles, databases are excluded."""
    +        default_dir = get_profile_dir("default")
    +        (default_dir / "config.yaml").write_text("ok")
    +
    +        # Create dirs/files that should be excluded
    +        for d in ("hermes-agent", ".worktrees", "profiles", "bin",
    +                  "image_cache", "logs", "sandboxes", "checkpoints"):
    +            sub = default_dir / d
    +            sub.mkdir(exist_ok=True)
    +            (sub / "marker.txt").write_text("excluded")
    +
    +        for f in ("state.db", "gateway.pid", "gateway_state.json",
    +                  "processes.json", "errors.log", ".hermes_history",
    +                  "active_profile", ".update_check", "auth.lock"):
    +            (default_dir / f).write_text("excluded")
    +
    +        output = tmp_path / "export" / "default.tar.gz"
    +        output.parent.mkdir(parents=True, exist_ok=True)
    +        export_profile("default", str(output))
    +
    +        with tarfile.open(str(output), "r:gz") as tf:
    +            names = tf.getnames()
    +
    +        # Config is present
    +        assert "default/config.yaml" in names
    +
    +        # Infrastructure excluded
    +        excluded_prefixes = [
    +            "default/hermes-agent", "default/.worktrees", "default/profiles",
    +            "default/bin", "default/image_cache", "default/logs",
    +            "default/sandboxes", "default/checkpoints",
    +        ]
    +        for prefix in excluded_prefixes:
    +            assert not any(n.startswith(prefix) for n in names), \
    +                f"Expected {prefix} to be excluded but found it in archive"
    +
    +        excluded_files = [
    +            "default/state.db", "default/gateway.pid",
    +            "default/gateway_state.json", "default/processes.json",
    +            "default/errors.log", "default/.hermes_history",
    +            "default/active_profile", "default/.update_check",
    +            "default/auth.lock",
    +        ]
    +        for f in excluded_files:
    +            assert f not in names, f"Expected {f} to be excluded"
    +
    +    def test_export_default_excludes_pycache_at_any_depth(self, profile_env, tmp_path):
    +        """__pycache__ dirs are excluded even inside nested directories."""
    +        default_dir = get_profile_dir("default")
    +        (default_dir / "config.yaml").write_text("ok")
    +        nested = default_dir / "skills" / "my-skill" / "__pycache__"
    +        nested.mkdir(parents=True)
    +        (nested / "cached.pyc").write_text("bytecode")
    +
    +        output = tmp_path / "export" / "default.tar.gz"
    +        output.parent.mkdir(parents=True, exist_ok=True)
    +        export_profile("default", str(output))
    +
    +        with tarfile.open(str(output), "r:gz") as tf:
    +            names = tf.getnames()
    +
    +        assert not any("__pycache__" in n for n in names)
    +
    +    def test_import_default_without_name_raises(self, profile_env, tmp_path):
    +        """Importing a default export without --name gives clear guidance."""
    +        default_dir = get_profile_dir("default")
    +        (default_dir / "config.yaml").write_text("ok")
    +
    +        archive = tmp_path / "export" / "default.tar.gz"
    +        archive.parent.mkdir(parents=True, exist_ok=True)
    +        export_profile("default", str(archive))
    +
    +        with pytest.raises(ValueError, match="Cannot import as 'default'"):
    +            import_profile(str(archive))
    +
    +    def test_import_default_with_explicit_default_name_raises(self, profile_env, tmp_path):
    +        """Explicitly importing as 'default' is also rejected."""
    +        default_dir = get_profile_dir("default")
    +        (default_dir / "config.yaml").write_text("ok")
    +
    +        archive = tmp_path / "export" / "default.tar.gz"
    +        archive.parent.mkdir(parents=True, exist_ok=True)
    +        export_profile("default", str(archive))
    +
    +        with pytest.raises(ValueError, match="Cannot import as 'default'"):
    +            import_profile(str(archive), name="default")
    +
    +    def test_import_default_export_with_new_name_roundtrip(self, profile_env, tmp_path):
    +        """Export default → import under a different name → data preserved."""
    +        default_dir = get_profile_dir("default")
    +        (default_dir / "config.yaml").write_text("model: opus")
    +        mem_dir = default_dir / "memories"
    +        mem_dir.mkdir(exist_ok=True)
    +        (mem_dir / "MEMORY.md").write_text("important fact")
    +
    +        archive = tmp_path / "export" / "default.tar.gz"
    +        archive.parent.mkdir(parents=True, exist_ok=True)
    +        export_profile("default", str(archive))
    +
    +        imported = import_profile(str(archive), name="backup")
    +        assert imported.is_dir()
    +        assert (imported / "config.yaml").read_text() == "model: opus"
    +        assert (imported / "memories" / "MEMORY.md").read_text() == "important fact"
    +
     
     # ===================================================================
     # TestProfileIsolation

• #22475 fix(profiles,banner): exclude infrastructure from --clone-all + fix stale update-check repo resolution (salvage of #5025, #5026, #21728)

  merged to main

  files: hermes_cli/banner.py, hermes_cli/profiles.py, tests/hermes_cli/test_profiles.py

  Show the diff

  • hermes_cli/banner.py

    @@ -206,9 +206,12 @@ def check_for_updates() -> Optional[int]:
         if embedded_rev:
             behind = _check_via_rev(embedded_rev)
         else:
    -        repo_dir = hermes_home / "hermes-agent"
    +        # Prefer the running code's location over the profile-scoped path.
    +        # $HERMES_HOME/hermes-agent/ may be a stale copy from --clone-all;
    +        # Path(__file__) always resolves to the actual installed checkout.
    +        repo_dir = Path(__file__).parent.parent.resolve()
             if not (repo_dir / ".git").exists():
    -            repo_dir = Path(__file__).parent.parent.resolve()
    +            repo_dir = hermes_home / "hermes-agent"
             if not (repo_dir / ".git").exists():
                 return None
             behind = _check_via_local_git(repo_dir)
    @@ -222,11 +225,16 @@ def check_for_updates() -> Optional[int]:
     
     
     def _resolve_repo_dir() -> Optional[Path]:
    -    """Return the active Hermes git checkout, or None if this isn't a git install."""
    -    hermes_home = get_hermes_home()
    -    repo_dir = hermes_home / "hermes-agent"
    +    """Return the active Hermes git checkout, or None if this isn't a git install.
    +
    +    Prefers the running code's location over the profile-scoped path
    +    because ``$HERMES_HOME/hermes-agent/`` may be a stale copy carried
    +    over by ``--clone-all``.
    +    """
    +    repo_dir = Path(__file__).parent.parent.resolve()
         if not (repo_dir / ".git").exists():
    -        repo_dir = Path(__file__).parent.parent.resolve()
    +        hermes_home = get_hermes_home()
    +        repo_dir = hermes_home / "hermes-agent"
         return repo_dir if (repo_dir / ".git").exists() else None
     
     

  • hermes_cli/profiles.py

    @@ -64,13 +64,39 @@
         "memories/USER.md",
     ]
     
    -# Runtime files stripped after --clone-all (shouldn't carry over)
    -_CLONE_ALL_STRIP = [
    +# Runtime files stripped after --clone-all (shouldn't carry over).
    +# Kept as a post-copy step rather than in the ignore filter because they
    +# are created dynamically during normal use and may be absent at copy time.
    +_CLONE_ALL_STRIP: list[str] = [
         "gateway.pid",
         "gateway_state.json",
         "processes.json",
     ]
     
    +# Infrastructure artifacts excluded from --clone-all when the source is the
    +# default profile (``~/.hermes``).  Named profiles never contain these
    +# directories at root, so the exclusion is gated to avoid silently dropping
    +# user data from a named-profile source.
    +#
    +# Rationale per item:
    +#   hermes-agent  — git repo checkout (~84 MB source + ~3 GB venv)
    +#   .worktrees    — git worktrees
    +#   profiles      — sibling named profiles (recursive copy never intended)
    +#   bin           — installed binaries (tirith etc., ~10 MB) shared per-host
    +#   node_modules  — npm packages (hundreds of MB)
    +#
    +# See ``_DEFAULT_EXPORT_EXCLUDE_ROOT`` below for the broader export-side
    +# exclusion list (export drops state.db / logs / caches too because the
    +# archive is a portable snapshot; clone-all keeps those because the cloned
    +# profile is meant to keep working immediately).
    +_CLONE_ALL_DEFAULT_EXCLUDE_ROOT: frozenset[str] = frozenset({
    +    "hermes-agent",
    +    ".worktrees",
    +    "profiles",
    +    "bin",
    +    "node_modules",
    +})
    +
     # Marker file written by `hermes profile create --no-skills`.  When present in
     # a profile's root, callers of seed_profile_skills() (fresh-create, `hermes
     # update`'s all-profile sync, the web dashboard) skip bundled-skill seeding
    @@ -89,23 +115,48 @@ def has_bundled_skills_opt_out(profile_dir: Path) -> bool:
     
     
     def _clone_all_copytree_ignore(source_dir: Path):
    -    """Ignore ``profiles/`` at the root of *source_dir* only.
    -
    -    ``~/.hermes`` contains ``profiles/<name>/`` for sibling named profiles.
    -    ``shutil.copytree`` would otherwise duplicate that entire tree inside the
    -    new profile (recursive ``.../profiles/.../profiles/...``). Export already
    -    excludes ``profiles`` via ``_DEFAULT_EXPORT_EXCLUDE_ROOT`` — match that
    -    behavior for ``--clone-all``.
    +    """Exclude infrastructure artifacts when cloning a profile via --clone-all.
    +
    +    Two categories:
    +      1. Root-level entries in ``_CLONE_ALL_DEFAULT_EXCLUDE_ROOT`` — known
    +         Hermes infrastructure directories that only the default profile
    +         (``~/.hermes``) ever contains.  Gated on ``source_dir`` actually
    +         being the default profile so a named-profile source never has its
    +         own data silently dropped.
    +      2. Universal exclusions at any depth — Python bytecode caches that
    +         are stale or regenerable (``__pycache__``, ``*.pyc``, ``*.pyo``)
    +         and runtime sockets / temp files (``*.sock``, ``*.tmp``).
    +
    +    The export-side ignore (``_default_export_ignore``) uses the same
    +    two-tier pattern with the broader ``_DEFAULT_EXPORT_EXCLUDE_ROOT`` set
    +    because the export archive is a portable snapshot rather than a live
    +    clone.
         """
         source_resolved = source_dir.resolve()
    +    is_default_source = source_resolved == _get_default_hermes_home().resolve()
     
         def _ignore(directory: str, names: List[str]) -> List[str]:
    -        try:
    -            if Path(directory).resolve() == source_resolved:
    -                return [n for n in names if n == "profiles"]
    -        except (OSError, ValueError):
    -            pass
    -        return []
    +        ignored: list[str] = []
    +        for entry in names:
    +            # Universal exclusions at any depth.
    +            if (
    +                entry == "__pycache__"
    +                or entry.endswith((".pyc", ".pyo", ".sock", ".tmp"))
    +            ):
    +                ignored.append(entry)
    +                continue
    +            # Root-level exclusions only apply when cloning the default profile.
    +            if is_default_source:
    +                try:
    +                    if Path(directory).resolve() == source_resolved:
    +                        if entry in _CLONE_ALL_DEFAULT_EXCLUDE_ROOT:
    +                            ignored.append(entry)
    +                except (OSError, ValueError):
    +                    # ``resolve()`` can fail on unusual FS layouts (broken
    +                    # symlinks, missing parents).  Fail open — better to
    +                    # over-copy than silently drop user data.
    +                    pass
    +        return ignored
     
         return _ignore
     

  • tests/hermes_cli/test_profiles.py

    @@ -244,6 +244,64 @@ def test_clone_all_excludes_sibling_profiles_tree(self, profile_env):
             assert (profile_dir / "memories" / "note.md").read_text() == "remember this"
             assert not (profile_dir / "profiles").exists()
     
    +    def test_clone_all_excludes_default_infrastructure(self, profile_env):
    +        """--clone-all from default profile excludes hermes-agent, .worktrees,
    +        bin, node_modules at root, plus __pycache__/*.pyc/*.pyo/*.sock/*.tmp
    +        at any depth.  Profile data (config, env, skills, sessions, logs,
    +        state.db) must be preserved — clone-all means "complete snapshot
    +        minus infrastructure."
    +        """
    +        tmp_path = profile_env
    +        default_home = tmp_path / ".hermes"
    +        # Simulate infrastructure dirs that only the default profile has
    +        (default_home / "hermes-agent" / ".git").mkdir(parents=True)
    +        (default_home / "hermes-agent" / "venv" / "bin").mkdir(parents=True)
    +        (default_home / "hermes-agent" / "README.md").write_text("repo")
    +        (default_home / ".worktrees" / "some-tree").mkdir(parents=True)
    +        (default_home / "profiles" / "other").mkdir(parents=True)
    +        (default_home / "profiles" / "other" / "config.yaml").write_text("x")
    +        (default_home / "bin").mkdir(exist_ok=True)
    +        (default_home / "bin" / "tool").write_text("binary")
    +        (default_home / "node_modules" / ".package-lock.json").mkdir(parents=True)
    +        # Bytecode + temp files at nested depth (universal exclusion)
    +        (default_home / "skills" / "my-skill" / "__pycache__").mkdir(parents=True)
    +        (default_home / "skills" / "my-skill" / "__pycache__" / "module.cpython-311.pyc").write_text("stale")
    +        (default_home / "skills" / "my-skill" / "module.pyc").write_text("stale")
    +        (default_home / "skills" / "my-skill" / "module.pyo").write_text("stale")
    +        (default_home / "data.sock").write_text("socket")
    +        (default_home / "data.tmp").write_text("tmp")
    +        # Profile data that SHOULD be copied
    +        (default_home / "skills" / "my-skill").mkdir(parents=True, exist_ok=True)
    +        (default_home / "skills" / "my-skill" / "SKILL.md").write_text("skill")
    +        (default_home / "config.yaml").write_text("model: gpt-4")
    +        (default_home / ".env").write_text("KEY=val")
    +        (default_home / "state.db").write_text("sessions-data")
    +        (default_home / "sessions").mkdir(exist_ok=True)
    +        (default_home / "logs").mkdir(exist_ok=True)
    +        (default_home / "logs" / "gateway.log").write_text("log")
    +
    +        profile_dir = create_profile("cloned", clone_all=True, no_alias=True)
    +
    +        # Infrastructure must be excluded
    +        assert not (profile_dir / "hermes-agent").exists()
    +        assert not (profile_dir / ".worktrees").exists()
    +        assert not (profile_dir / "profiles").exists()
    +        assert not (profile_dir / "bin").exists()
    +        assert not (profile_dir / "node_modules").exists()
    +        # Universal exclusions at any depth
    +        assert not (profile_dir / "data.sock").exists()
    +        assert not (profile_dir / "data.tmp").exists()
    +        assert not (profile_dir / "skills" / "my-skill" / "__pycache__").exists()
    +        assert not (profile_dir / "skills" / "my-skill" / "module.pyc").exists()
    +        assert not (profile_dir / "skills" / "my-skill" / "module.pyo").exists()
    +        # All profile data must be present
    +        assert (profile_dir / "skills" / "my-skill" / "SKILL.md").read_text() == "skill"
    +        assert (profile_dir / "config.yaml").read_text() == "model: gpt-4"
    +        assert (profile_dir / ".env").read_text() == "KEY=val"
    +        assert (profile_dir / "state.db").read_text() == "sessions-data"
    +        assert (profile_dir / "sessions").exists()
    +        assert (profile_dir / "logs" / "gateway.log").read_text() == "log"
    +
         def test_clone_config_missing_files_skipped(self, profile_env):
             """Clone config gracefully skips files that don't exist in source."""
             profile_dir = create_profile("coder", clone_config=True, no_alias=True)