Redaction fix for lowercase/dotted config secret keys

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #53590 fix(redact): mask passwords in lowercase/dotted config keys (#16413)

  merged to main

  files: agent/redact.py, tests/agent/test_redact.py

  Show the diff

  • agent/redact.py

    @@ -107,12 +107,60 @@
         r"ntn_[A-Za-z0-9]{10,}",            # Notion internal integration token
     ]
     
    -# ENV assignment patterns: KEY=value where KEY contains a secret-like name
    +# ENV assignment patterns: KEY=value where KEY contains a secret-like name.
    +# Uppercase keys tolerate spaces around "=" (e.g. ``FOO_SECRET = [redacted:secret-assignment] because
    +# an all-caps key is almost never prose/code.
     _SECRET_ENV_NAMES = r"(?:API_?KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|AUTH)"
     _ENV_ASSIGN_RE = re.compile(
         rf"([A-Z0-9_]{{0,50}}{_SECRET_ENV_NAMES}[A-Z0-9_]{{0,50}})\s*=\s*(['\"]?)(\S+)\2",
     )
     
    +# Lowercase / dotted / hyphenated config keys from config files
    +# (application.properties, .env, YAML-ish dumps): ``spring.datasource.password=[redacted:secret-assignment]
    +# ``app.api.key=xyz``, ``password=[redacted:secret-assignment] The uppercase _ENV_ASSIGN_RE above
    +# never matched these, so config-file passwords leaked verbatim (issue #16413).
    +#
    +# These run only in a config-file context, NOT in prose, code, or URLs — three
    +# carve-outs preserved from the original design (#4367 + the documented
    +# web-URL passthrough below):
    +#   1. The value is bounded by ``[^\s&]`` (stops at whitespace AND ``&``) so
    +#      form-urlencoded bodies are handled pair-by-pair (by _redact_form_body),
    +#      not greedily swallowed.
    +#   2. _CFG_DOTTED_RE only matches when the key is NAMESPACED (contains a dot),
    +#      which is unambiguously a config key — never a prose word.
    +#   3. _CFG_ANCHORED_RE matches a bare secret-word key only at line start
    +#      (optionally after ``export``), so conversational ``I have password=foo``
    +#      mid-sentence is left alone.
    +# The colon-form URL guard (skip when ``://`` present) lives at the call site.
    +_SECRET_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential|auth)"
    +_CFG_VALUE = r"(['\"]?)([^\s&]+?)\2(?=[\s&]|$)"
    +# Namespaced (dotted) key: the secret word may sit anywhere in a dotted path.
    +_CFG_DOTTED_RE = re.compile(
    +    rf"((?:[A-Za-z0-9_\-]+\.)+[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*"
    +    rf"|[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*\.[A-Za-z0-9_.\-]+)"
    +    rf"={_CFG_VALUE}",
    +    re.IGNORECASE,
    +)
    +# Line-anchored bare key: ``password=…`` / ``export api_key=…`` at start of line.
    +_CFG_ANCHORED_RE = re.compile(
    +    rf"(^[ \t]*(?:export[ \t]+)?[A-Za-z0-9_\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_\-]*)={_CFG_VALUE}",
    +    re.IGNORECASE | re.MULTILINE,
    +)
    +
    +# Unquoted YAML / colon config (e.g. ``password: [redacted:secret-assignment]
    +# ``spring.datasource.password: [redacted:secret-assignment] The secret keyword must be part of
    +# the KEY (anchored to the start of the line/indent), and the value is a single
    +# whitespace-free token — so prose like ``note: secret meeting`` (keyword in the
    +# value) and ``error: token expired`` are left alone. Bare ``auth`` is excluded
    +# from the key set so ``Authorization:`` / ``author:`` don't match (the former
    +# is masked by _AUTH_HEADER_RE); ``auth_token``/``auth-token`` still match via
    +# the ``token`` keyword. Quoted values defer to _JSON_FIELD_RE via the lookahead.
    +_YAML_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential)"
    +_YAML_ASSIGN_RE = re.compile(
    +    rf"(^[ \t]*[A-Za-z0-9_.\-]*{_YAML_CFG_NAMES}[A-Za-z0-9_.\-]*)(:[ \t]*)(?!['\"])([^\s&]+)",
    +    re.IGNORECASE | re.MULTILINE,
    +)
    +
     # JSON field patterns: "apiKey": "value", "token": "value", etc.
     _JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)"
     _JSON_FIELD_RE = re.compile(
    @@ -382,6 +430,13 @@ def _redact_env(m):
                     name, quote, value = m.group(1), m.group(2), m.group(3)
                     return f"{name}={quote}{_mask_token(value)}{quote}"
                 text = _ENV_ASSIGN_RE.sub(_redact_env, text)
    +            # Lowercase/dotted config keys (issue #16413). Skip URLs entirely —
    +            # web-URL query params are intentionally passed through (see note
    +            # near the bottom of this function); _DB_CONNSTR_RE still guards
    +            # connection-string passwords.
    +            if "://" not in text:
    +                text = _CFG_DOTTED_RE.sub(_redact_env, text)
    +                text = _CFG_ANCHORED_RE.sub(_redact_env, text)
     
             # JSON fields: "apiKey": "***"  (skip for code files — false positives)
             if ":" in text and '"' in text:
    @@ -390,6 +445,15 @@ def _redact_json(m):
                     return f'{key}: "{_mask_token(value)}"'
                 text = _JSON_FIELD_RE.sub(_redact_json, text)
     
    +        # Unquoted YAML / colon config: password: ***  (after JSON so quoted
    +        # values are handled there; the lookahead in _YAML_ASSIGN_RE skips
    +        # quotes). Skip URLs — web-URL query params pass through by design.
    +        if ":" in text and "://" not in text:
    +            def _redact_yaml(m):
    +                key, sep, value = m.group(1), m.group(2), m.group(3)
    +                return f"{key}{sep}{_mask_token(value)}"
    +            text = _YAML_ASSIGN_RE.sub(_redact_yaml, text)
    +
         # Authorization headers — _AUTH_HEADER_RE matches any scheme after
         # "[Proxy-]Authorization:" case-insensitively, so "uthorization" is the
         # cheapest substring gate that covers every casing without a casefold().

  • tests/agent/test_redact.py

    @@ -504,6 +504,81 @@ def test_multiline_text_not_form(self):
             assert "first=1" in redact_sensitive_text(text)
     
     
    +class TestLowercaseDottedConfigKeys:
    +    """Issue #16413 — config-file passwords in lowercase/dotted/colon keys
    +    must be redacted. The uppercase _ENV_ASSIGN_RE missed these, leaking
    +    `spring.datasource.password=...` and `password: ...` from `cat`'d config
    +    files. Carve-outs: prose, code (#4367), and web URLs are left untouched.
    +    """
    +
    +    def test_spring_dotted_password_assignment(self):
    +        text = "spring.datasource.password=[redacted:secret-assignment]"
    +        result = redact_sensitive_text(text)
    +        assert "Sup3rS3cret!" not in result
    +        assert "spring.datasource.password=" in result
    +
    +    def test_dotted_api_key_split_keyword(self):
    +        # 'api.key' splits the keyword across a dot — must still match.
    +        text = "app.api.key=ak_live_998877"
    +        result = redact_sensitive_text(text)
    +        assert "ak_live_998877" not in result
    +        assert "app.api.key=" in result
    +
    +    def test_bare_lowercase_password_at_line_start(self):
    +        text = "password=[redacted:secret-assignment]"
    +        result = redact_sensitive_text(text)
    +        assert "mysecretvalue123" not in result
    +
    +    def test_quoted_lowercase_value(self):
    +        text = "password='[redacted:secret-assignment]'"
    +        result = redact_sensitive_text(text)
    +        assert "mysecretvalue123" not in result
    +
    +    def test_yaml_unquoted_password(self):
    +        text = "password: [redacted:secret-assignment]"
    +        result = redact_sensitive_text(text)
    +        assert "Sup3rS3cret!" not in result
    +        assert "password:" in result
    +
    +    def test_yaml_indented_dotted(self):
    +        text = "spring:\n  datasource:\n    password: [redacted:secret-assignment]"
    +        result = redact_sensitive_text(text)
    +        assert "hunter2pass" not in result
    +
    +    def test_properties_file_dump(self):
    +        text = (
    +            "server.port=8080\n"
    +            "spring.datasource.username=admin\n"
    +            "spring.datasource.password=[redacted:secret-assignment]"
    +            "logging.level.root=INFO"
    +        )
    +        result = redact_sensitive_text(text)
    +        assert "Sup3rS3cret!" not in result
    +        assert "server.port=8080" in result  # non-secret keys preserved
    +        assert "username=admin" in result
    +
    +    # --- carve-outs: must NOT redact ---
    +
    +    def test_prose_mid_sentence_password_unchanged(self):
    +        # Not line-anchored, not dotted → conversational text, leave alone.
    +        text = "I have password=foo and other things"
    +        assert redact_sensitive_text(text) == text
    +
    +    def test_lowercase_code_assignment_unchanged(self):
    +        # #4367 regression — spaces around '=' in code.
    +        text = "const secret = await fetchSecret();"
    +        assert redact_sensitive_text(text) == text
    +
    +    def test_url_query_param_passes_through(self):
    +        # Web URLs are intentionally hands-off (documented design).
    +        text = "https://example.com/api?password=[redacted:secret-assignment]"
    +        assert redact_sensitive_text(text) == text
    +
    +    def test_prose_keyword_in_value_unchanged(self):
    +        text = "note: secret meeting at noon"
    +        assert redact_sensitive_text(text) == text
    +
    +
     class TestXaiToken:
         KEY = "xai-ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstu"