LLM token redaction incorrectly masking env variable lookups

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #58534 fix(redact): don't mask programmatic env lookups in KEY=value redaction (salvage #2854)

  merged to main

  files: agent/redact.py, tests/agent/test_redact.py

  Show the diff

  • agent/redact.py

    @@ -137,6 +137,14 @@
     # The colon-form URL guard (skip when ``://`` present) lives at the call site.
     _SECRET_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential|auth)"
     _CFG_VALUE = r"(['\"]?)([^\s&]+?)\2(?=[\s&]|$)"
    +
    +# Programmatic env lookups (``os.getenv(...)``, ``os.environ[...]``,
    +# ``os.environ.get(...)``, ``process.env.X``, ``$ENV{X}``) reference variable
    +# *names*, not secret values. When one appears as the VALUE of a KEY=... match
    +# it's a code snippet, not a leaked secret — skip redaction (issue #2852).
    +_ENV_LOOKUP_VALUE_RE = re.compile(
    +    r"^(?:os\.(?:getenv|environ)|process\.env|\$ENV\{)"
    +)
     # Namespaced (dotted) key: the secret word may sit anywhere in a dotted path.
     _CFG_DOTTED_RE = re.compile(
         rf"((?:[A-Za-z0-9_\-]+\.)+[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*"
    @@ -541,6 +549,11 @@ def redact_sensitive_text(
             if "=" in text:
                 def _redact_env(m):
                     name, quote, value = m.group(1), m.group(2), m.group(3)
    +                # Programmatic env lookups reference variable *names*, not
    +                # secret values — masking them corrupts code snippets in
    +                # prose/log contexts (issue #2852): ``KEY=os.getenv('X')``.
    +                if _ENV_LOOKUP_VALUE_RE.match(value):
    +                    return m.group(0)
                     return f"{name}={quote}{_mask_token(value)}{quote}"
                 text = _ENV_ASSIGN_RE.sub(_redact_env, text)
                 # Lowercase/dotted config keys (issue #16413). Skip URLs entirely —

  • tests/agent/test_redact.py

    @@ -124,6 +124,52 @@ def test_export_whitespace_preserved(self):
             assert "mypassword" not in result
     
     
    +class TestEnvLookupPreserved:
    +    """Programmatic env var lookups must not be corrupted (issue #2852)."""
    +
    +    def test_os_getenv_single_quote_uppercase_key(self):
    +        text = "MY_API_KEY=os.getenv('OPENAI_API_KEY')"
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_os_getenv_lowercase_config_key(self):
    +        text = "ha_token=os.getenv('HOMEASSISTANT_TOKEN')"
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_os_getenv_double_quote(self):
    +        text = 'API_TOKEN=os.getenv("MY_API_TOKEN")'
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_os_environ_get(self):
    +        text = "HA_TOKEN=os.environ.get('HOMEASSISTANT_TOKEN')"
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_os_environ_bracket(self):
    +        text = "MY_SECRET=[redacted:secret-assignment]'MY_SECRET']"
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_process_env(self):
    +        text = "api_key=[redacted:secret-assignment]"
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_real_env_value_still_redacted(self):
    +        text = "HOMEASSISTANT_TOKEN=[redacted:secret-assignment]"
    +        result = redact_sensitive_text(text, force=True)
    +        assert "eyJhbGciOiJIUzI1NiJ9" not in result
    +
    +    def test_real_lowercase_value_still_redacted(self):
    +        text = "password=[redacted:secret-assignment]"
    +        result = redact_sensitive_text(text, force=True)
    +        assert "hunter2hunter2" not in result
    +
    +    def test_multiline_prose_with_code_snippet(self):
    +        text = """Set it up like this:
    +    HA_TOKEN=os.getenv('HOMEASSISTANT_TOKEN')
    +    if not HA_TOKEN:
    +        raise ValueError('Missing credentials')"""
    +        result = redact_sensitive_text(text, force=True)
    +        assert "os.getenv('HOMEASSISTANT_TOKEN')" in result
    +
    +
     class TestJsonFields:
         def test_json_api_key(self):
             text = '{"apiKey": "[redacted:openai-key]"}'

• #58915 fix(redact): skip env-lookup exception for JSON/YAML config field redaction

  merged to main

  files: agent/redact.py, tests/agent/test_redact.py

  Show the diff

  • agent/redact.py

    @@ -570,6 +570,11 @@ def _redact_env(m):
             if ":" in text and '"' in text:
                 def _redact_json(m):
                     key, value = m.group(1), m.group(2)
    +                # Same programmatic-env-lookup exception as _redact_env above
    +                # (issue #2852): "apiKey": "os.getenv('X')" is a code snippet,
    +                # not a leaked secret value.
    +                if _ENV_LOOKUP_VALUE_RE.match(value):
    +                    return m.group(0)
                     return f'{key}: "{_mask_token(value)}"'
                 text = _JSON_FIELD_RE.sub(_redact_json, text)
     
    @@ -579,6 +584,11 @@ def _redact_json(m):
             if ":" in text and "://" not in text:
                 def _redact_yaml(m):
                     key, sep, value = m.group(1), m.group(2), m.group(3)
    +                # Same programmatic-env-lookup exception as _redact_env above
    +                # (issue #2852): api_key: os.getenv('X') is a code snippet,
    +                # not a leaked secret value.
    +                if _ENV_LOOKUP_VALUE_RE.match(value):
    +                    return m.group(0)
                     return f"{key}{sep}{_mask_token(value)}"
                 text = _YAML_ASSIGN_RE.sub(_redact_yaml, text)
     

  • tests/agent/test_redact.py

    @@ -185,6 +185,34 @@ def test_multiline_prose_with_code_snippet(self):
             result = redact_sensitive_text(text, force=True)
             assert "os.getenv('HOMEASSISTANT_TOKEN')" in result
     
    +    def test_json_field_os_getenv_preserved(self):
    +        # _redact_env has the env-lookup exception; _redact_json (a separate
    +        # closure, JSON key: "value" syntax) did not, and mangled this into
    +        # '"apiKey": "[redacted:secret-assignment]')"'.
    +        text = '{"apiKey": "[redacted:secret-assignment]'OPENAI_API_KEY\')"}'
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_json_field_os_environ_get_preserved(self):
    +        text = '{"token": "[redacted:secret-assignment]'MY_TOKEN\')"}'
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_json_field_real_value_still_redacted(self):
    +        text = '{"apiKey": "[redacted:openai-key]"}'
    +        result = redact_sensitive_text(text, force=True)
    +        assert "[redacted:openai-key]" not in result
    +
    +    def test_yaml_field_os_getenv_preserved(self):
    +        # Same exception missing from _redact_yaml (unquoted key: value
    +        # syntax) — mangled 'api_key: os.getenv("OPENAI_API_KEY")' into
    +        # 'api_key: [redacted:secret-assignment]")'.
    +        text = 'api_key: os.getenv("OPENAI_API_KEY")'
    +        assert redact_sensitive_text(text, force=True) == text
    +
    +    def test_yaml_field_real_value_still_redacted(self):
    +        text = "api_key: [redacted:openai-key]"
    +        result = redact_sensitive_text(text, force=True)
    +        assert "[redacted:openai-key]" not in result
    +
     
     class TestJsonFields:
         def test_json_api_key(self):