diff --git a/.gitignore b/.gitignore index f4e1db3..b2b0f57 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ htmlcov/ # ctxguard local config (the example ships instead) .ctxguard.toml +# local task notes +goal.md diff --git a/README.md b/README.md index b6d491f..5050196 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,14 @@ Run the synthetic demonstration with `python3 scripts/demo.py`. See - GitHub classic and fine-grained personal access tokens - Slack tokens - Stripe live and test secret keys +- GCP API keys (`AIza...`) and service-account JSON (content marker plus the filename rule) +- Azure storage, Service Bus, Event Hub, and IoT Hub connection string keys + (`AccountKey=...`, `SharedAccessKey=...`) +- Twilio API key SIDs and SendGrid API keys +- OpenAI-style keys (`sk-...`, `sk-proj-...`) and Anthropic keys (`sk-ant-...`) +- Database URLs with inline passwords: Postgres, MySQL/MariaDB (including + SQLAlchemy dialect+driver and Rails `mysql2://` schemes), MongoDB, Redis, + AMQP, and MSSQL - private key blocks and JWT-shaped strings - credential-like assignments that pass length, digit, and entropy thresholds - high-entropy values assigned to names containing `key`, `secret`, `token`, @@ -145,6 +153,19 @@ ctxguard --version preserves availability but means a broken hook does not provide protection. - Native Windows compatibility is not claimed; current testing covers the Python package and hook contract on Unix-like CI runners. +- Any non-placeholder password inline in a database URL is flagged, including + weak or "demo" ones (`hunter2`, `letmein123`). This is intentional: a weak + credential is still a working credential, and ctxguard can't judge which + ones are safe to expose. Allowlist known-safe fixtures in `.ctxguard.toml` + instead. +- GCP `AIza...`-style API keys are flagged even though some (notably Firebase + web config keys) are designed to be usable client-side. Restrict such keys + by API and referrer in the Google Cloud Console rather than relying on + ctxguard to tell intent apart; allowlist the specific file if it's noisy. +- A handful of well-known, publicly documented development constants (e.g. + the Azurite/Azure Storage Emulator default key) are excluded by name since + they're identical on every machine and not secrets. This list is small and + only grows for equally unambiguous cases. Use layered controls: least-privilege credentials, a secrets manager, `.gitignore`, repository secret scanning, short-lived tokens, and revocation. diff --git a/ctxguard/detectors.py b/ctxguard/detectors.py index d4ae71c..a1a78bd 100644 --- a/ctxguard/detectors.py +++ b/ctxguard/detectors.py @@ -142,6 +142,11 @@ def shannon_entropy(value: str) -> float: "unset", "disabled", "string", + "pass", + "pwd", + "1234", + "12345", + "123456", } _PLACEHOLDER_SUBSTRINGS = ( @@ -163,13 +168,27 @@ def shannon_entropy(value: str) -> float: "****", ) -_PLACEHOLDER_WRAPPED = re.compile(r"^(<.*>|\$\{.*\}|\{\{.*\}\}|%.*%|__.*__|\[.*\])$") +_PLACEHOLDER_WRAPPED = re.compile(r"^(<.*>|\$\{.*\}|\{.*\}|%.*%|__.*__|\[.*\]|\$\w+)$") + +# Well-known, publicly documented development keys that are constant across +# every install and are not secrets. Checked from is_placeholder() so every +# detector (not just the one whose regex happens to capture them) rejects +# them consistently, regardless of which pattern matches the surrounding text. +_KNOWN_BENIGN_VALUES = { + # Azurite / Azure Storage Emulator default account key (Microsoft docs) + ( + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==" + ).lower(), +} def is_placeholder(value: str) -> bool: v = value.strip().strip("\"'").lower() if not v or v in _PLACEHOLDER_EXACT: return True + if v in _KNOWN_BENIGN_VALUES: + return True if _PLACEHOLDER_WRAPPED.match(v): return True if any(s in v for s in _PLACEHOLDER_SUBSTRINGS): @@ -216,6 +235,31 @@ def _plausible_env_value(value: str, m: "re.Match") -> bool: return shannon_entropy(v) >= 3.0 +def _plausible_db_password(value: str, m: "re.Match") -> bool: + # any real inline password in a connection URL is worth flagging, even a + # weak one: the point is that no credential (strong or not) belongs + # inline in a URL that might enter an AI's context. only obvious docs + # placeholders (user:password@, user:pass@, ${VAR}) get a pass. + if is_placeholder(value): + return False + return shannon_entropy(value) >= 2.0 + + +def _plausible_azure_key(value: str, m: "re.Match") -> bool: + if is_placeholder(value): + return False + return _plausible_token(value, m) + + +def _plausible_gcp_service_account(value: str, m: "re.Match") -> bool: + # the bare service-account type marker alone also appears in docs and + # example snippets; require a private key nearby, which every real GCP + # service-account credential file has right next to it. + text = m.string + window = text[max(0, m.start() - 2000) : m.start() + 4000] + return "private_key" in window + + def _high_entropy_value(value: str, m: "re.Match") -> bool: v = value.strip() if len(v) < 16 or is_placeholder(v): @@ -283,6 +327,68 @@ class _Spec: 1, _plausible_token, ), + _Spec( + "gcp_api_key", + re.compile(r"\b(AIza[0-9A-Za-z_\-]{35})(?![0-9A-Za-z_\-])"), + "high", + 1, + _plausible_token, + ), + _Spec( + "gcp_service_account", + re.compile(r"[\"']type[\"']\s*:\s*[\"']service_account[\"']"), + "high", + 0, + _plausible_gcp_service_account, + ), + _Spec( + "azure_storage_account_key", + re.compile(r"(?i)\b(?:Account|SharedAccess)Key=([A-Za-z0-9+/]{40,}={0,2})"), + "high", + 1, + _plausible_azure_key, + ), + _Spec( + "twilio_api_key", + re.compile(r"\b(SK[0-9a-fA-F]{32})(?![0-9a-fA-F])"), + "high", + 1, + _plausible_token, + ), + _Spec( + "sendgrid_api_key", + re.compile(r"\b(SG\.[A-Za-z0-9_\-]{22}\.[A-Za-z0-9_\-]{43})(?![A-Za-z0-9_\-])"), + "high", + 1, + _plausible_token, + ), + _Spec( + "anthropic_api_key", + re.compile(r"\b(sk-ant-[A-Za-z0-9_\-]{32,})\b"), + "high", + 1, + _plausible_token, + ), + _Spec( + "openai_api_key", + re.compile( + r"\b(sk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{32,}" + r"|sk-[A-Za-z0-9]{48}(?![A-Za-z0-9]))" + ), + "high", + 1, + _plausible_token, + ), + _Spec( + "database_url_password", + re.compile( + r"(?i)\b(?:postgres(?:ql)?|mysql2?|mariadb|mongodb(?:\+srv)?|rediss?" + r"|amqps?|mssql)(?:\+[a-z0-9_]+)?://[^\s:@/]*:([^\s:@/]{4,})@" + ), + "high", + 1, + _plausible_db_password, + ), _Spec( "private_key_block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----"), diff --git a/tests/conftest.py b/tests/conftest.py index 050eed3..b56c967 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,20 @@ API_TOKEN_VAL = "8fk2mZpQ7wR4" + "xV1bN6cY3sD5" STRIPE_KEY_VAL = "whsec_F8kL2mP9" + "qR4sT7vX1yZ3bC6d" +GCP_API_VAL = "AIzaSyQ9w8E7r6T5y4U3i2" + "O1p0Q9w8E7r6T5y0k" +AZURE_ACCOUNT_VAL = ( + "aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0uVwX1yZ2aB3cD4eF5g" + + "H6iJ7kL8mN9oP0qR1sT2uV3wX4yZ5aB6cD7eF8g==" +) +TWILIO_VAL = "SK9f8e7d6c5b4a3928" + "1706f5e4d3c2b1a0" +SENDGRID_VAL = ( + "SG.aB1cD2eF3gH4iJ5kL6mN7o" + ".pQ8rS9tU0vW1xY2zA3bC4dE5fG6hI7jK8lM9nO0pQ1r" +) +OPENAI_VAL = "sk-aB1cD2eF3gH4iJ5kL6mN7oP8" + "qR9sT0uV1wX2yZ3aB4cD5eF6" +ANTHROPIC_VAL = "sk-ant-api03-qW3eR5tY7uI9oP1a" + "S2dF4gH6jK8lZ0xC5vB7nM9q" +PG_URL_PW = "v8Kq2Lw9" + "Xt4z" +MONGO_URL_PW = "Zp4tK9qX" + "2mR7w" + RAW_SECRETS = [ AWS_ID, AWS_SECRET, @@ -49,11 +63,21 @@ DB_PASSWORD_VAL, API_TOKEN_VAL, STRIPE_KEY_VAL, + GCP_API_VAL, + AZURE_ACCOUNT_VAL, + TWILIO_VAL, + SENDGRID_VAL, + OPENAI_VAL, + ANTHROPIC_VAL, + PG_URL_PW, + MONGO_URL_PW, ] # Expected totals when scanning tests/fixtures/secrets: # aws.txt 2, github.txt 2, slack.txt 1, stripe.txt 2, keys.txt 1, jwt.txt 1, # dotenv/.env 4 (1 filename + 3 env), settings.py 1, -# dummy.pem 1, credentials.json 1, id_rsa 1 (filename-only) -EXPECTED_SECRET_FINDINGS = 17 -EXPECTED_FLAGGED_FILES = 11 +# dummy.pem 1, credentials.json 1, id_rsa 1 (filename-only), +# gcp_sa.json 2 (type marker + private_key_id), cloud.txt 2 (gcp + azure), +# saas.txt 4 (twilio, sendgrid, openai, anthropic), dburls.txt 2 +EXPECTED_SECRET_FINDINGS = 27 +EXPECTED_FLAGGED_FILES = 15 diff --git a/tests/fixtures/clean/placeholders.env b/tests/fixtures/clean/placeholders.env index b713750..2fb7f17 100644 --- a/tests/fixtures/clean/placeholders.env +++ b/tests/fixtures/clean/placeholders.env @@ -9,3 +9,8 @@ AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY STRIPE_KEY=sk_live_XXXXXXXXXXXXXXXXXXXXXXXX GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +GOOGLE_API_KEY=AIzaXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +TWILIO_API_KEY=SK00000000000000000000000000000000 +SENDGRID_API_KEY=SG.XXXXXXXXXXXXXXXXXXXXXX.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +DATABASE_URL=postgres://user:password@localhost:5432/dev diff --git a/tests/fixtures/secrets/cloud.txt b/tests/fixtures/secrets/cloud.txt new file mode 100644 index 0000000..f63959b --- /dev/null +++ b/tests/fixtures/secrets/cloud.txt @@ -0,0 +1,3 @@ +cloud credentials (fixture, fake but format-valid) +google maps key for staging: AIzaSyQ9w8E7r6T5y4U3i2O1p0Q9w8E7r6T5y0k +AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=fixtureacct;AccountKey=aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0uVwX1yZ2aB3cD4eF5gH6iJ7kL8mN9oP0qR1sT2uV3wX4yZ5aB6cD7eF8g==;EndpointSuffix=core.windows.net diff --git a/tests/fixtures/secrets/dburls.txt b/tests/fixtures/secrets/dburls.txt new file mode 100644 index 0000000..38082ec --- /dev/null +++ b/tests/fixtures/secrets/dburls.txt @@ -0,0 +1,3 @@ +connection strings (fixture, fake credentials) +DATABASE_URL=postgres://svc_app:v8Kq2Lw9Xt4z@db.internal:5432/appdb +MONGO_URI=mongodb+srv://appuser:Zp4tK9qX2mR7w@cluster0.fixture.mongodb.net/prod diff --git a/tests/fixtures/secrets/gcp_sa.json b/tests/fixtures/secrets/gcp_sa.json new file mode 100644 index 0000000..7df7e6c --- /dev/null +++ b/tests/fixtures/secrets/gcp_sa.json @@ -0,0 +1,6 @@ +{ + "type": "service_account", + "project_id": "fixture-project", + "private_key_id": "0f1e2d3c4b5a69788796a5b4c3d2e1f001234567", + "client_email": "svc@fixture-project.iam.gserviceaccount.com" +} diff --git a/tests/fixtures/secrets/saas.txt b/tests/fixtures/secrets/saas.txt new file mode 100644 index 0000000..2fb1454 --- /dev/null +++ b/tests/fixtures/secrets/saas.txt @@ -0,0 +1,5 @@ +saas api keys (fixture, fake but format-valid) +twilio api key sid SK9f8e7d6c5b4a39281706f5e4d3c2b1a0 +sendgrid mailer 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 +openai for the bot sk-aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0uV1wX2yZ3aB4cD5eF6 +anthropic for eval sk-ant-api03-qW3eR5tY7uI9oP1aS2dF4gH6jK8lZ0xC5vB7nM9q diff --git a/tests/test_detectors.py b/tests/test_detectors.py index 9e60ef8..bbd162f 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -3,20 +3,28 @@ import pytest from conftest import ( + ANTHROPIC_VAL, AWS_ID, AWS_SECRET, + AZURE_ACCOUNT_VAL, CLEAN_DIR, DB_PASSWORD_VAL, ENTROPY_VAL, EXPECTED_SECRET_FINDINGS, + GCP_API_VAL, GH_PAT, GHP, JWT, + MONGO_URL_PW, + OPENAI_VAL, + PG_URL_PW, RAW_SECRETS, SECRETS_DIR, + SENDGRID_VAL, SLACK, STRIPE_LIVE, STRIPE_TEST, + TWILIO_VAL, ) from ctxguard import detectors as det @@ -116,6 +124,173 @@ def test_header_only_negative(self): assert det.scan_text("eyJhbGciOiJIUzI1NiJ9") == [] +class TestGcp: + def test_api_key_positive(self): + findings = det.scan_text(f"maps key {GCP_API_VAL}\n") + assert names(findings) == ["gcp_api_key"] + + def test_api_key_placeholder_negative(self): + assert det.scan_text("AIza" + "X" * 35) == [] + + def test_service_account_marker_positive(self): + # the bare marker alone is no longer sufficient (see + # test_marker_without_private_key_negative below); a real service + # account file always carries private_key right alongside it. + # Built via concatenation so this test file's own self-scan stays + # clean (same convention as the RAW_SECRETS constants in conftest.py). + marker = '"type": "service' + '_account", "private' + '_key_id": "abc123def456"' + assert names(det.scan_text(marker)) == ["gcp_service_account"] + + def test_service_account_other_type_negative(self): + assert det.scan_text('"type": "authorized_user_config"') == [] + + def test_marker_without_private_key_negative(self): + # bare type marker in docs/example snippets, no key material nearby + assert ( + det.scan_text('{"type": "service' + '_account", "project_id": "x"}') == [] + ) + + def test_marker_with_private_key_positive(self): + text = ( + '{"type": "service' + '_account", "private' + '_key_id": "abc", ' + '"private' + '_key": "not-a-real-pem-block"}' + ) + assert "gcp_service_account" in names(det.scan_text(text)) + + +class TestAzure: + def test_connection_string_positive(self): + text = f"DefaultEndpointsProtocol=https;AccountKey={AZURE_ACCOUNT_VAL};x=y" + findings = det.scan_text(text) + assert names(findings) == ["azure_storage_account_key"] + assert AZURE_ACCOUNT_VAL not in str(findings[0]) + + def test_low_entropy_negative(self): + assert det.scan_text("AccountKey=" + "A" * 70) == [] + + def test_shared_access_key_positive(self): + # Service Bus / Event Hubs / IoT Hub connection strings use + # SharedAccessKey=, not AccountKey=. Variable named to avoid tripping + # this file's own env_assignment detector on ` = "..."`. + sas_value = "nQx8VmPz4KtY2bWd7RjLq3hF6sN9vC1xB4mZ8kQz=" + text = ( + "Endpoint=sb://ns.servicebus.windows.net/;" + "SharedAccessKeyName=RootManageSharedAccessKey;" + f"SharedAccessKey={sas_value}" + ) + findings = det.scan_text(text) + assert names(findings) == ["azure_storage_account_key"] + assert sas_value not in str(findings[0]) + + def test_well_known_azurite_dev_key_negative(self): + # the public, constant Azurite/Storage-Emulator default key is not a + # secret; must be excluded everywhere, not just from this detector, + # since "AccountKey" also matches the generic env_assignment name filter + azurite_key = ( + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==" + ) + assert det.scan_text(f"AccountKey={azurite_key}") == [] + conn_string = ( + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + f"AccountKey={azurite_key};" + ) + assert det.scan_text(conn_string) == [] + + +class TestTwilio: + def test_positive(self): + assert names(det.scan_text(f"sid {TWILIO_VAL}\n")) == ["twilio_api_key"] + + def test_zeroed_negative(self): + assert det.scan_text("SK" + "0" * 32) == [] + + def test_too_short_negative(self): + assert det.scan_text("SKa1b2c3") == [] + + +class TestSendGrid: + def test_positive(self): + assert names(det.scan_text(SENDGRID_VAL)) == ["sendgrid_api_key"] + + def test_placeholder_negative(self): + fake = "SG." + "X" * 22 + "." + "X" * 43 + assert det.scan_text(fake) == [] + + def test_wrong_shape_negative(self): + assert det.scan_text("SG.short.token") == [] + + +class TestOpenAiStyle: + def test_classic_positive(self): + assert names(det.scan_text(OPENAI_VAL)) == ["openai_api_key"] + + def test_project_key_positive(self): + proj = "sk-proj-" + OPENAI_VAL[3:] + assert names(det.scan_text(proj)) == ["openai_api_key"] + + def test_anthropic_positive(self): + assert names(det.scan_text(ANTHROPIC_VAL)) == ["anthropic_api_key"] + + def test_placeholder_negative(self): + assert det.scan_text("sk-" + "x" * 48) == [] + + def test_too_short_negative(self): + assert det.scan_text("sk-abc123") == [] + + +class TestDatabaseUrl: + @pytest.mark.parametrize( + "url", + [ + "postgres://svc:{pw}@db.internal:5432/app", + "postgresql://svc:{pw}@db/app", + "mysql://app:{pw}@10.0.0.5/x", + "mongodb+srv://appuser:{pw}@cluster0.example.net/prod", + "redis://default:{pw}@cache:6379/0", + "amqp://worker:{pw}@mq:5672/vhost", + # SQLAlchemy dialect+driver URLs (default form in every + # Python/SQLAlchemy/Alembic config) + "postgresql+psycopg2://svc:{pw}@db.internal/app", + "postgresql+asyncpg://svc:{pw}@db/app", + "mysql+pymysql://svc:{pw}@db/app", + "mssql+pyodbc://svc:{pw}@db/app", + # Rails' DATABASE_URL scheme for MySQL + "mysql2://svc:{pw}@10.0.0.5/appdb", + # canonical empty-username Redis/Mongo auth URLs + "redis://:{pw}@cache.internal:6379/0", + "rediss://:{pw}@cache/0", + "mongodb://:{pw}@db/x", + ], + ) + def test_inline_password_positive(self, url): + findings = det.scan_text(url.format(pw=PG_URL_PW)) + assert names(findings) == ["database_url_password"] + assert PG_URL_PW not in str(findings[0]) + + def test_mongo_positive(self): + url = f"mongodb://svc:{MONGO_URL_PW}@db/x" + assert names(det.scan_text(url)) == ["database_url_password"] + + @pytest.mark.parametrize( + "url", + [ + "postgres://user:password@localhost:5432/dev", + "mysql://root:pass@127.0.0.1/x", + "postgresql://u:${DB_PASSWORD}@h/db", + "postgres://u:{password}@h/db", + "postgres://u:$PGPASSWORD@h/db", + "postgres://user@localhost/db", + "postgres://localhost:5432/db", + "https://example.com/path", + "postgres://:password@host/db", + "redis://:changeme@host/db", + ], + ) + def test_placeholder_or_no_password_negative(self, url): + assert det.scan_text(url) == [] + + class TestEnvAssignment: def test_positive(self): findings = det.scan_text(f"DB_PASSWORD={DB_PASSWORD_VAL}\n")