From f07e1a0e65202d93e82f65ae849d733b0f4e9a78 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:58:56 +0000 Subject: [PATCH 01/41] feat(logging): add shared backend logging package --- backend/shared/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 backend/shared/__init__.py diff --git a/backend/shared/__init__.py b/backend/shared/__init__.py new file mode 100644 index 00000000..71cd720a --- /dev/null +++ b/backend/shared/__init__.py @@ -0,0 +1 @@ +"""Shared backend utilities used by both Django and managed AI services.""" From f19b3f7d6bc4d8081bd7b32f763be580b66848b6 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:16 +0000 Subject: [PATCH 02/41] feat(logging): centralize sensitive log redaction --- backend/shared/logging_redaction.py | 248 ++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 backend/shared/logging_redaction.py diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py new file mode 100644 index 00000000..5322e1e5 --- /dev/null +++ b/backend/shared/logging_redaction.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import logging +import re +import traceback +from collections.abc import Mapping +from threading import Lock +from typing import Any + +REDACTED = "[REDACTED]" +REDACTED_BINARY = "[REDACTED_BINARY]" +MAX_REDACTION_DEPTH = 12 + +_SENSITIVE_KEYS = frozenset( + { + # Credentials and session material. + "access_key", + "access_token", + "api_key", + "authorization", + "client_secret", + "cookie", + "credentials", + "csrf_token", + "csrfmiddlewaretoken", + "id_token", + "password", + "private_key", + "refresh_token", + "secret", + "secret_access_key", + "session_token", + "set_cookie", + "token", + # Direct identifiers / PII. + "address", + "birth_date", + "date_of_birth", + "dob", + "document_number", + "email", + "first_name", + "full_name", + "ghana_card_number", + "last_name", + "middle_name", + "national_id", + "passport_number", + "phone", + "phone_number", + "postal_address", + "tax_identification_number", + "tin", + # Evidence locations and raw document / biometric material. + "biometric_payload", + "biometric_template", + "document_bytes", + "document_image", + "document_storage_key", + "face_embedding", + "face_image", + "image", + "image_base64", + "image_bytes", + "liveness_video", + "mrz", + "ocr_text", + "raw_document", + "raw_image", + "raw_ocr", + "selfie", + "selfie_image", + "selfie_storage_key", + "storage_key", + } +) + +_SENSITIVE_SUFFIXES = ( + "_access_token", + "_api_key", + "_authorization", + "_client_secret", + "_credential", + "_credentials", + "_password", + "_private_key", + "_refresh_token", + "_secret", + "_session_token", + "_storage_key", + "_token", +) + +_SENSITIVE_FRAGMENTS = ( + "biometric", + "face_embedding", + "image_base64", + "image_bytes", + "liveness_video", + "selfie_image", +) + +_BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+") +_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b") +_EMAIL_RE = re.compile( + r"(? str: + normalized = re.sub(r"[^a-z0-9]+", "_", str(key).strip().lower()) + return normalized.strip("_") + + +def is_sensitive_key(key: object) -> bool: + normalized = _normalize_key(key) + if not normalized: + return False + if normalized in _SENSITIVE_KEYS: + return True + if normalized.endswith(_SENSITIVE_SUFFIXES): + return True + return any(fragment in normalized for fragment in _SENSITIVE_FRAGMENTS) + + +def redact_text(value: str) -> str: + """Redact common secret and PII shapes from unstructured log text.""" + redacted = _BEARER_RE.sub("Bearer [REDACTED]", value) + redacted = _JWT_RE.sub(REDACTED, redacted) + redacted = _AWS_ACCESS_KEY_RE.sub(REDACTED, redacted) + redacted = _CREDENTIAL_ASSIGNMENT_RE.sub( + lambda match: f"{match.group(1)}={REDACTED}", redacted + ) + redacted = _EMAIL_RE.sub(REDACTED, redacted) + redacted = _IPV4_RE.sub(REDACTED, redacted) + redacted = _PHONE_RE.sub(REDACTED, redacted) + return redacted + + +def redact_value(value: Any, *, key: object | None = None, _depth: int = 0) -> Any: + """Return a logging-safe copy of a nested value. + + Redaction is key-aware for structured payloads and shape-aware for free text. + Bytes are never logged because they can contain document or biometric evidence. + """ + if key is not None and is_sensitive_key(key): + return REDACTED + if _depth >= MAX_REDACTION_DEPTH: + return "[REDACTED_DEPTH_LIMIT]" + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + return redact_text(value) + if isinstance(value, (bytes, bytearray, memoryview)): + return REDACTED_BINARY + if isinstance(value, Mapping): + return { + str(item_key): redact_value( + item_value, + key=item_key, + _depth=_depth + 1, + ) + for item_key, item_value in value.items() + } + if isinstance(value, tuple): + return tuple(redact_value(item, _depth=_depth + 1) for item in value) + if isinstance(value, list): + return [redact_value(item, _depth=_depth + 1) for item in value] + if isinstance(value, (set, frozenset)): + return [redact_value(item, _depth=_depth + 1) for item in value] + return value + + +def _redact_exception( + exc_info: tuple[type[BaseException], BaseException, Any], +) -> str: + rendered = "".join(traceback.format_exception(*exc_info)) + return redact_text(rendered) + + +def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: + """Sanitize message args, structured extras, stack text, and exception text in place.""" + record.msg = redact_value(record.msg) + if record.args: + if isinstance(record.args, Mapping): + record.args = redact_value(record.args) + else: + record.args = tuple(redact_value(item) for item in record.args) + + standard = set(logging.LogRecord(None, 0, "", 0, "", (), None).__dict__) + standard.update({"message", "asctime"}) + for field, value in list(record.__dict__.items()): + if field in standard or field.startswith("_"): + continue + record.__dict__[field] = redact_value(value, key=field) + + if record.stack_info: + record.stack_info = redact_text(record.stack_info) + if record.exc_info: + record.exc_text = _redact_exception(record.exc_info) + record.exc_info = None + elif record.exc_text: + record.exc_text = redact_text(record.exc_text) + return record + + +def _safe_make_record(self, *args, **kwargs): + record = _ORIGINAL_MAKE_RECORD(self, *args, **kwargs) + return sanitize_log_record(record) + + +def install_safe_logging() -> None: + """Install the process-wide logging redaction boundary exactly once.""" + global _INSTALLED + if _INSTALLED: + return + with _INSTALL_LOCK: + if _INSTALLED: + return + logging.Logger.makeRecord = _safe_make_record + _INSTALLED = True + + +__all__ = [ + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] From 720eb3a73c94aba07b3eca9e1b3fab6f0d9c1f66 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:25 +0000 Subject: [PATCH 03/41] feat(logging): expose safe logging to Django and workers --- backend/django/common/safe_logging.py | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 backend/django/common/safe_logging.py diff --git a/backend/django/common/safe_logging.py b/backend/django/common/safe_logging.py new file mode 100644 index 00000000..bf9971bd --- /dev/null +++ b/backend/django/common/safe_logging.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_BACKEND_ROOT = Path(__file__).resolve().parents[2] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from shared.logging_redaction import ( # noqa: E402 + REDACTED, + REDACTED_BINARY, + install_safe_logging, + is_sensitive_key, + redact_text, + redact_value, + sanitize_log_record, +) + +__all__ = [ + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] From 1f8e8b9471d7e43aa68b54238e43bf5fad79616a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:32 +0000 Subject: [PATCH 04/41] feat(logging): expose safe logging to managed AI --- backend/ai-service/app/core/safe_logging.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 backend/ai-service/app/core/safe_logging.py diff --git a/backend/ai-service/app/core/safe_logging.py b/backend/ai-service/app/core/safe_logging.py new file mode 100644 index 00000000..67d26ee5 --- /dev/null +++ b/backend/ai-service/app/core/safe_logging.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_BACKEND_ROOT = Path(__file__).resolve().parents[3] +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +from shared.logging_redaction import ( # noqa: E402 + REDACTED, + REDACTED_BINARY, + install_safe_logging, + is_sensitive_key, + redact_text, + redact_value, + sanitize_log_record, +) + +__all__ = [ + "REDACTED", + "REDACTED_BINARY", + "install_safe_logging", + "is_sensitive_key", + "redact_text", + "redact_value", + "sanitize_log_record", +] From 1b2fa8559898d91882e1f45af6f52b38c0033429 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:40 +0000 Subject: [PATCH 05/41] feat(logging): install redaction at Django startup --- backend/django/apps/core/apps.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/django/apps/core/apps.py b/backend/django/apps/core/apps.py index 68868e8b..665f028d 100644 --- a/backend/django/apps/core/apps.py +++ b/backend/django/apps/core/apps.py @@ -5,3 +5,10 @@ class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "apps.core" label = "core" + + def ready(self) -> None: + # Install the redaction boundary after Django configures logging but before + # request/worker code can emit application records. + from common.safe_logging import install_safe_logging + + install_safe_logging() From 4cb315f9ce64e2e25cedcfc7598fca9a125f451f Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:59:50 +0000 Subject: [PATCH 06/41] feat(logging): protect Celery startup logs --- backend/django/config/celery.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/django/config/celery.py b/backend/django/config/celery.py index 1d803399..2610c9d7 100644 --- a/backend/django/config/celery.py +++ b/backend/django/config/celery.py @@ -2,6 +2,12 @@ from celery import Celery +from common.safe_logging import install_safe_logging + + +# Celery can initialize its own logging before Django app-ready hooks run. Install +# the redaction boundary here as well so broker/startup and task logs are protected. +install_safe_logging() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.development") From 0a0359f4eaf7d0d7e0fd6ba73136ffd624e376e8 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:00 +0000 Subject: [PATCH 07/41] feat(logging): install redaction before AI imports --- backend/ai-service/app/main.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/backend/ai-service/app/main.py b/backend/ai-service/app/main.py index ad3b9286..c30ca18b 100644 --- a/backend/ai-service/app/main.py +++ b/backend/ai-service/app/main.py @@ -1,8 +1,14 @@ -from fastapi import FastAPI +from app.core.safe_logging import install_safe_logging -from app.core.auth import enforce_internal_token as _enforce_internal_token -from app.routers.health import healthcheck, readiness, router as health_router -from app.routers.processing import ( +# Install before importing processing modules so initialization failures and +# exception paths cannot emit raw evidence or credentials. +install_safe_logging() + +from fastapi import FastAPI # noqa: E402 + +from app.core.auth import enforce_internal_token as _enforce_internal_token # noqa: E402 +from app.routers.health import healthcheck, readiness, router as health_router # noqa: E402 +from app.routers.processing import ( # noqa: E402 document_classify, document_ocr, document_quality, @@ -10,8 +16,8 @@ liveness_check, router as processing_router, ) -from app.runtime import configure_runtime_environment -from app.schemas.processing import ( +from app.runtime import configure_runtime_environment # noqa: E402 +from app.schemas.processing import ( # noqa: E402 AIResultResponse, DocumentClassificationRequest, DocumentOCRRequest, @@ -21,7 +27,7 @@ LivenessCheckRequest, ReadinessResponse, ) -from app.settings import get_settings +from app.settings import get_settings # noqa: E402 settings = get_settings() From 3e3e62de4c56462e8b1409fe81a6313b83f16dc3 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:08 +0000 Subject: [PATCH 08/41] build(logging): include shared redaction package in Django image --- backend/django/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/django/Dockerfile b/backend/django/Dockerfile index 33111902..2ae3c89b 100644 --- a/backend/django/Dockerfile +++ b/backend/django/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends build-essential libpq-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/lists/* COPY backend/pyproject.toml backend/uv.lock /app/backend/ RUN uv export \ @@ -22,6 +22,7 @@ RUN uv export \ --output-file /tmp/requirements-django.txt \ && pip install --no-cache-dir -r /tmp/requirements-django.txt +COPY backend/shared /app/backend/shared COPY backend/django /app/backend/django COPY docs/openapi/identitycore-public-api.yaml /app/docs/openapi/identitycore-public-api.yaml From f2e80eb6039cea356856c3578f7e72ef09946190 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:15 +0000 Subject: [PATCH 09/41] fix(build): preserve apt cleanup path --- backend/django/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/django/Dockerfile b/backend/django/Dockerfile index 2ae3c89b..6a7b8056 100644 --- a/backend/django/Dockerfile +++ b/backend/django/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends build-essential libpq-dev \ - && rm -rf /var/lib/lists/* + && rm -rf /var/lib/apt/lists/* COPY backend/pyproject.toml backend/uv.lock /app/backend/ RUN uv export \ From fc7c45716579ad1b99c202e69281756f5e1bcc76 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:23 +0000 Subject: [PATCH 10/41] build(logging): include shared redaction package in AI image --- backend/ai-service/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/ai-service/Dockerfile b/backend/ai-service/Dockerfile index f327f2ee..55b81091 100644 --- a/backend/ai-service/Dockerfile +++ b/backend/ai-service/Dockerfile @@ -32,6 +32,7 @@ RUN uv export \ && pip install --no-cache-dir -r /tmp/requirements-ai.txt \ && pip install --no-cache-dir --force-reinstall opencv-contrib-python-headless==4.10.0.84 +COPY backend/shared /app/backend/shared COPY backend/ai-service /app/backend/ai-service WORKDIR /app/backend/ai-service From bae85b3aad0b36e897e575e9a2899515c0d7361a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:00:46 +0000 Subject: [PATCH 11/41] test(logging): cover Django Celery and exception redaction --- backend/django/common/test_safe_logging.py | 122 +++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 backend/django/common/test_safe_logging.py diff --git a/backend/django/common/test_safe_logging.py b/backend/django/common/test_safe_logging.py new file mode 100644 index 00000000..3c8fecd8 --- /dev/null +++ b/backend/django/common/test_safe_logging.py @@ -0,0 +1,122 @@ +import io +import logging + +from celery.utils.log import get_task_logger +from django.test import SimpleTestCase + +from common.safe_logging import REDACTED, REDACTED_BINARY, install_safe_logging, redact_value + + +class SafeLoggingTests(SimpleTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + install_safe_logging() + + def _capture(self, logger: logging.Logger, formatter: str = "%(levelname)s %(message)s"): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter(formatter)) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + self.addCleanup(logger.handlers.clear) + return stream + + def test_recursive_redaction_covers_secret_pii_and_biometric_fields(self): + payload = { + "authorization": "Bearer top-secret", + "profile": { + "email": "ada@example.test", + "phone_number": "+233241234567", + "document_number": "GHA-123456789", + "safe_status": "pending_review", + }, + "evidence": { + "selfie_image": "base64-sensitive-selfie", + "face_embedding": [0.1, 0.2, 0.3], + "document_storage_key": "tenant/evidence/front.jpg", + }, + "binary": b"document-bytes", + } + + redacted = redact_value(payload) + + self.assertEqual(redacted["authorization"], REDACTED) + self.assertEqual(redacted["profile"]["email"], REDACTED) + self.assertEqual(redacted["profile"]["phone_number"], REDACTED) + self.assertEqual(redacted["profile"]["document_number"], REDACTED) + self.assertEqual(redacted["profile"]["safe_status"], "pending_review") + self.assertEqual(redacted["evidence"]["selfie_image"], REDACTED) + self.assertEqual(redacted["evidence"]["face_embedding"], REDACTED) + self.assertEqual(redacted["evidence"]["document_storage_key"], REDACTED) + self.assertEqual(redacted["binary"], REDACTED_BINARY) + + def test_django_logger_redacts_message_arguments_and_structured_extra(self): + logger = logging.getLogger("django.identitycore.redaction-test") + stream = self._capture(logger, "%(message)s payload=%(payload)s") + + logger.info( + "request rejected email=%s Authorization: Bearer %s", + "ada@example.test", + "secret-access-token", + extra={ + "payload": { + "password": "never-log-this", + "selfie_image": "raw-biometric-data", + "safe_reason": "credentials_missing", + } + }, + ) + + output = stream.getvalue() + self.assertNotIn("ada@example.test", output) + self.assertNotIn("secret-access-token", output) + self.assertNotIn("never-log-this", output) + self.assertNotIn("raw-biometric-data", output) + self.assertIn("credentials_missing", output) + self.assertIn(REDACTED, output) + + def test_celery_task_logger_uses_the_same_redaction_boundary(self): + logger = get_task_logger("identitycore.redaction-test") + stream = self._capture(logger, "%(message)s context=%(context)s") + + logger.warning( + "worker retry token=%s", + "celery-secret-token", + extra={ + "context": { + "api_key": "provider-key", + "ocr_text": "raw document text", + "verification_id": "ver_safe_public_id", + } + }, + ) + + output = stream.getvalue() + self.assertNotIn("celery-secret-token", output) + self.assertNotIn("provider-key", output) + self.assertNotIn("raw document text", output) + self.assertIn("ver_safe_public_id", output) + + def test_exception_traceback_is_preserved_without_sensitive_values(self): + logger = logging.getLogger("identitycore.exception-redaction-test") + stream = self._capture(logger) + + try: + raise RuntimeError( + "token=runtime-secret email=ada@example.test " + "phone=+233241234567 document_number=GHA-123456789" + ) + except RuntimeError: + logger.exception("provider failed first_name=Ada") + + output = stream.getvalue() + self.assertIn("Traceback", output) + self.assertIn("RuntimeError", output) + self.assertNotIn("runtime-secret", output) + self.assertNotIn("ada@example.test", output) + self.assertNotIn("+233241234567", output) + self.assertNotIn("GHA-123456789", output) + self.assertNotIn("first_name=Ada", output) + self.assertIn("first_name=[REDACTED]", output) From f6d79a485b8e7b968837f54ac26b2e57829959ed Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:01:12 +0000 Subject: [PATCH 12/41] test(logging): cover managed AI redaction failures --- backend/ai-service/tests/test_safe_logging.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 backend/ai-service/tests/test_safe_logging.py diff --git a/backend/ai-service/tests/test_safe_logging.py b/backend/ai-service/tests/test_safe_logging.py new file mode 100644 index 00000000..702e0704 --- /dev/null +++ b/backend/ai-service/tests/test_safe_logging.py @@ -0,0 +1,73 @@ +import io +import logging + +from app.core.safe_logging import REDACTED, install_safe_logging, redact_value + + +def _capture(logger: logging.Logger, formatter: str = "%(message)s"): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter(formatter)) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + return stream + + +def test_managed_ai_uses_shared_nested_redaction_corpus(): + install_safe_logging() + payload = { + "authorization": "Bearer ai-secret", + "request": { + "email": "subject@example.test", + "document_number": "P1234567", + "safe_operation": "face_compare", + }, + "biometrics": { + "face_embedding": [0.12, 0.34], + "selfie_image": "base64-selfie", + }, + } + + redacted = redact_value(payload) + + assert redacted["authorization"] == REDACTED + assert redacted["request"]["email"] == REDACTED + assert redacted["request"]["document_number"] == REDACTED + assert redacted["request"]["safe_operation"] == "face_compare" + assert redacted["biometrics"]["face_embedding"] == REDACTED + assert redacted["biometrics"]["selfie_image"] == REDACTED + + +def test_managed_ai_logger_redacts_structured_context_and_exception_text(): + install_safe_logging() + logger = logging.getLogger("identitycore.ai.redaction-test") + stream = _capture(logger, "%(message)s context=%(context)s") + + try: + raise ValueError( + "api_key=provider-secret email=subject@example.test " + "image_base64=raw-biometric" + ) + except ValueError: + logger.exception( + "AI processing failed Authorization: Bearer %s", + "internal-shared-token", + extra={ + "context": { + "document_storage_key": "tenant/evidence/document.jpg", + "ocr_text": "raw OCR contents", + "operation": "document_ocr", + } + }, + ) + + output = stream.getvalue() + assert "provider-secret" not in output + assert "subject@example.test" not in output + assert "raw-biometric" not in output + assert "internal-shared-token" not in output + assert "tenant/evidence/document.jpg" not in output + assert "raw OCR contents" not in output + assert "document_ocr" in output + assert "Traceback" in output From c8f851cb3679e8aad6acf5e4215fc33e09059258 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:09 +0000 Subject: [PATCH 13/41] feat(logging): add frontend safe logging boundary --- .../packages/api-client/src/safe-logging.ts | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 frontend/packages/api-client/src/safe-logging.ts diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts new file mode 100644 index 00000000..40997145 --- /dev/null +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -0,0 +1,168 @@ +export const REDACTED = "[REDACTED]"; +export const REDACTED_BINARY = "[REDACTED_BINARY]"; + +const MAX_REDACTION_DEPTH = 12; + +const SENSITIVE_KEYS = new Set([ + "access_key", + "access_token", + "address", + "api_key", + "authorization", + "biometric_payload", + "biometric_template", + "birth_date", + "client_secret", + "cookie", + "credentials", + "csrf_token", + "date_of_birth", + "dob", + "document_bytes", + "document_image", + "document_number", + "document_storage_key", + "email", + "face_embedding", + "face_image", + "first_name", + "full_name", + "ghana_card_number", + "id_token", + "image", + "image_base64", + "image_bytes", + "last_name", + "liveness_video", + "middle_name", + "mrz", + "national_id", + "ocr_text", + "passport_number", + "password", + "phone", + "phone_number", + "postal_address", + "private_key", + "raw_document", + "raw_image", + "raw_ocr", + "refresh_token", + "secret", + "secret_access_key", + "selfie", + "selfie_image", + "selfie_storage_key", + "session_token", + "set_cookie", + "storage_key", + "tax_identification_number", + "tin", + "token", +]); + +const SENSITIVE_SUFFIXES = [ + "_access_token", + "_api_key", + "_authorization", + "_client_secret", + "_credential", + "_credentials", + "_password", + "_private_key", + "_refresh_token", + "_secret", + "_session_token", + "_storage_key", + "_token", +]; + +const SENSITIVE_FRAGMENTS = [ + "biometric", + "face_embedding", + "image_base64", + "image_bytes", + "liveness_video", + "selfie_image", +]; + +function normalizeKey(key: PropertyKey): string { + return String(key) + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); +} + +export function isSensitiveLogKey(key: PropertyKey): boolean { + const normalized = normalizeKey(key); + if (!normalized) return false; + return ( + SENSITIVE_KEYS.has(normalized) || + SENSITIVE_SUFFIXES.some((suffix) => normalized.endsWith(suffix)) || + SENSITIVE_FRAGMENTS.some((fragment) => normalized.includes(fragment)) + ); +} + +export function redactLogText(value: string): string { + return value + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]") + .replace( + /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, + REDACTED, + ) + .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED) + .replace( + /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^\s,;"'}]+)\2/gi, + (_match, label: string) => `${label}=${REDACTED}`, + ) + .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) + .replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, REDACTED) + .replace(/(?:\+?\d[\d ().-]{7,}\d)/g, REDACTED); +} + +export function redactLogValue( + value: unknown, + key?: PropertyKey, + depth = 0, +): unknown { + if (key !== undefined && isSensitiveLogKey(key)) return REDACTED; + if (depth >= MAX_REDACTION_DEPTH) return "[REDACTED_DEPTH_LIMIT]"; + if (value === null || value === undefined) return value; + if (typeof value === "string") return redactLogText(value); + if (typeof value === "number" || typeof value === "boolean") return value; + if (value instanceof Uint8Array || value instanceof ArrayBuffer) { + return REDACTED_BINARY; + } + if (Array.isArray(value)) { + return value.map((item) => redactLogValue(item, undefined, depth + 1)); + } + if (typeof value === "object") { + const output: Record = {}; + for (const [entryKey, entryValue] of Object.entries(value)) { + output[entryKey] = redactLogValue(entryValue, entryKey, depth + 1); + } + return output; + } + return redactLogText(String(value)); +} + +export type SafeLogLevel = "debug" | "info" | "warn" | "error"; + +const LOG_METHODS: Record void> = { + debug: console.debug.bind(console), + info: console.info.bind(console), + warn: console.warn.bind(console), + error: console.error.bind(console), +}; + +export function safeLog( + level: SafeLogLevel, + event: string, + context: Record = {}, +): void { + LOG_METHODS[level]({ + event: redactLogText(event), + context: redactLogValue(context), + }); +} From be3f15cb11f3d52f183569e8d4c1565b0e27f094 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:16 +0000 Subject: [PATCH 14/41] test(logging): add frontend redaction test build config --- .../packages/api-client/tsconfig.safe-logging.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 frontend/packages/api-client/tsconfig.safe-logging.json diff --git a/frontend/packages/api-client/tsconfig.safe-logging.json b/frontend/packages/api-client/tsconfig.safe-logging.json new file mode 100644 index 00000000..c33b4fd3 --- /dev/null +++ b/frontend/packages/api-client/tsconfig.safe-logging.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "Node", + "strict": true, + "noEmit": false, + "declaration": false, + "outDir": ".safe-logging-test", + "lib": ["ES2022", "DOM"] + }, + "include": ["src/safe-logging.ts"] +} From 15c2edcfc694b7521a401625afdac5eee62cb120 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:27 +0000 Subject: [PATCH 15/41] test(logging): add frontend adversarial redaction tests --- .../api-client/test/safe-logging.test.mjs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 frontend/packages/api-client/test/safe-logging.test.mjs diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs new file mode 100644 index 00000000..2d94251f --- /dev/null +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { rmSync } from "node:fs"; +import { createRequire } from "node:module"; +import { after, test } from "node:test"; + +const require = createRequire(import.meta.url); +const { + REDACTED, + REDACTED_BINARY, + redactLogText, + redactLogValue, +} = require("../.safe-logging-test/safe-logging.js"); + +after(() => { + rmSync(new URL("../.safe-logging-test", import.meta.url), { + force: true, + recursive: true, + }); +}); + +test("redacts nested credentials, PII, evidence, and binary values", () => { + const redacted = redactLogValue({ + authorization: "Bearer secret-token", + profile: { + email: "ada@example.test", + phone_number: "+233241234567", + document_number: "GHA-123456789", + safe_status: "pending_review", + }, + evidence: { + selfie_image: "base64-selfie", + face_embedding: [0.1, 0.2], + document_storage_key: "tenant/evidence/front.jpg", + }, + binary: new Uint8Array([1, 2, 3]), + }); + + assert.equal(redacted.authorization, REDACTED); + assert.equal(redacted.profile.email, REDACTED); + assert.equal(redacted.profile.phone_number, REDACTED); + assert.equal(redacted.profile.document_number, REDACTED); + assert.equal(redacted.profile.safe_status, "pending_review"); + assert.equal(redacted.evidence.selfie_image, REDACTED); + assert.equal(redacted.evidence.face_embedding, REDACTED); + assert.equal(redacted.evidence.document_storage_key, REDACTED); + assert.equal(redacted.binary, REDACTED_BINARY); +}); + +test("redacts secrets and identifiers embedded in free text", () => { + const output = redactLogText( + "token=top-secret email=ada@example.test phone=+233241234567 document_number=GHA-123 Authorization: Bearer bearer-secret", + ); + + assert.doesNotMatch( + output, + /top-secret|ada@example\.test|233241234567|GHA-123|bearer-secret/, + ); + assert.match(output, /\[REDACTED\]/); +}); From e6bbe3355eb5495c186063b2cc2daa2ff88126c9 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:38 +0000 Subject: [PATCH 16/41] test(logging): fail CI on unsafe frontend console logging --- frontend/scripts/check-safe-logging.mjs | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 frontend/scripts/check-safe-logging.mjs diff --git a/frontend/scripts/check-safe-logging.mjs b/frontend/scripts/check-safe-logging.mjs new file mode 100644 index 00000000..4e1e3013 --- /dev/null +++ b/frontend/scripts/check-safe-logging.mjs @@ -0,0 +1,59 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { dirname, extname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const frontendRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const allowedConsoleFile = resolve( + frontendRoot, + "packages/api-client/src/safe-logging.ts", +); +const sourceRoots = [ + "dashboard", + "developer-portal", + "identitycore", + "platform-admin", + "verification-portal", + "packages", +].map((path) => resolve(frontendRoot, path)); +const extensions = new Set([".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"]); +const directConsole = /\bconsole\.(?:debug|info|log|warn|error|trace)\s*\(/; +const skipDirectories = new Set([ + ".next", + ".safe-logging-test", + "coverage", + "dist", + "e2e", + "node_modules", + "test", + "tests", +]); + +function walk(path, findings) { + for (const entry of readdirSync(path, { withFileTypes: true })) { + if (entry.isDirectory() && skipDirectories.has(entry.name)) continue; + const absolute = join(path, entry.name); + if (entry.isDirectory()) { + walk(absolute, findings); + continue; + } + if (!extensions.has(extname(entry.name))) continue; + if (/\.(?:spec|test)\.[cm]?[jt]sx?$/.test(entry.name)) continue; + if (absolute === allowedConsoleFile) continue; + const source = readFileSync(absolute, "utf8"); + if (directConsole.test(source)) findings.push(relative(frontendRoot, absolute)); + } +} + +const findings = []; +for (const root of sourceRoots) walk(root, findings); + +if (findings.length) { + console.error( + [ + "Unsafe direct console logging is not allowed in frontend production source.", + "Use safeLog from @identitycore/api-client so sensitive context is redacted.", + ...findings.map((path) => ` - ${path}`), + ].join("\n"), + ); + process.exitCode = 1; +} From 273be6831f57a6f32749bf490ddb2027b1fa5adc Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:02:56 +0000 Subject: [PATCH 17/41] feat(logging): export frontend safe logger --- frontend/packages/api-client/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/packages/api-client/src/index.ts b/frontend/packages/api-client/src/index.ts index da2ff732..1d777429 100644 --- a/frontend/packages/api-client/src/index.ts +++ b/frontend/packages/api-client/src/index.ts @@ -1,3 +1,5 @@ +export * from "./safe-logging"; + export interface ApiSuccess { success: true; data: T; From 6b1dbbf35d9a850d49e4ffc6531c351fc6b55dca Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:03:02 +0000 Subject: [PATCH 18/41] test(logging): enforce safe frontend logging in lint --- frontend/packages/api-client/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/packages/api-client/package.json b/frontend/packages/api-client/package.json index db440b1f..9efdef4f 100644 --- a/frontend/packages/api-client/package.json +++ b/frontend/packages/api-client/package.json @@ -7,6 +7,9 @@ "exports": { ".": "./src/index.ts" }, + "scripts": { + "lint": "tsc -p tsconfig.json && tsc -p tsconfig.safe-logging.json && node --test test/safe-logging.test.mjs && node ../../scripts/check-safe-logging.mjs" + }, "devDependencies": { "typescript": "^5" } From 878335c30bb52d35bb2b8ab46c5f490d0621449a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:03:29 +0000 Subject: [PATCH 19/41] docs(logging): document safe telemetry and redaction rules --- docs/operations/logging-redaction.md | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/operations/logging-redaction.md diff --git a/docs/operations/logging-redaction.md b/docs/operations/logging-redaction.md new file mode 100644 index 00000000..c6210cbf --- /dev/null +++ b/docs/operations/logging-redaction.md @@ -0,0 +1,41 @@ +# Logging redaction and safe telemetry + +IdentityCore treats logs as an operational data stream, not as a place to store verification evidence or user data. Application logs must never contain credentials, session material, direct subject PII, raw document/OCR content, storage locations for evidence, or biometric payloads. + +## Backend boundary + +Django, Celery workers, storage/provider call paths, and the managed AI service use the shared redaction implementation in `backend/shared/logging_redaction.py`. + +The boundary is installed when Django starts, before Celery initializes its application logger, and before the AI service imports processing routes. It sanitizes every Python `LogRecord` after `extra` fields have been attached and before handlers format the record. This covers: + +- nested dictionaries/lists passed as structured logging context; +- positional and mapping logging arguments; +- bearer/JWT-like credentials and common credential assignments embedded in free text; +- common email, phone, and IP shapes in free text; +- exception messages and tracebacks; +- bytes and byte-like values, which are always replaced rather than rendered. + +Safe operational dimensions such as public request/verification IDs, operation names, status/reason codes, provider codes, durations, retry counts, and queue names may be logged when they do not themselves contain subject data. + +When a new secret, PII field, document field, or biometric representation is introduced, add its normalized key to the shared redaction corpus and add an adversarial test before using it in telemetry. + +## Frontend boundary + +Frontend code must use `safeLog` exported by `@identitycore/api-client`. The logger applies the same categories of key-aware and free-text redaction before calling the browser console. + +Direct `console.log`, `console.info`, `console.warn`, `console.error`, `console.debug`, or `console.trace` calls are rejected by the frontend lint gate for production source. This prevents a future component from bypassing the redaction helper with a raw API error, token, form state, capture payload, or server response. + +## What not to log + +Do not log: + +- authorization headers, API keys, passwords, cookies, refresh/session tokens, provider credentials, or private keys; +- names, email addresses, phone numbers, addresses, dates of birth, national/passport/document numbers, or tax identifiers; +- OCR/MRZ text, raw document images/bytes, evidence storage keys, selfies, face embeddings, liveness media, or biometric templates; +- complete request/response bodies from identity, provider, storage, or webhook operations. + +Prefer stable reason/error codes and public correlation identifiers over raw exception/request payloads. + +## Verification + +The CI suite includes adversarial tests for Django, Celery, managed AI, and frontend logging. Tests intentionally place sensitive values in nested structured fields, positional arguments, binary values, and exception text and assert that those values never reach formatted log output. Frontend lint also fails when production source introduces a direct console logging call. From 316a02175e5bf5102430743e5d9f452bb30a40d9 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:04:26 +0000 Subject: [PATCH 20/41] fix(logging): harden and optimize record sanitization --- backend/shared/logging_redaction.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 5322e1e5..d7822287 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -101,7 +101,9 @@ ) _BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+") -_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b") +_JWT_RE = re.compile( + r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b" +) _EMAIL_RE = re.compile( r"(? A return value if isinstance(value, str): return redact_text(value) + if isinstance(value, BaseException): + return f"{value.__class__.__name__}: {redact_text(str(value))}" if isinstance(value, (bytes, bytearray, memoryview)): return REDACTED_BINARY if isinstance(value, Mapping): - return { - str(item_key): redact_value( + redacted_mapping = {} + for item_key, item_value in value.items(): + safe_key = redact_text(str(item_key)) + redacted_mapping[safe_key] = redact_value( item_value, key=item_key, _depth=_depth + 1, ) - for item_key, item_value in value.items() - } + return redacted_mapping if isinstance(value, tuple): return tuple(redact_value(item, _depth=_depth + 1) for item in value) if isinstance(value, list): return [redact_value(item, _depth=_depth + 1) for item in value] if isinstance(value, (set, frozenset)): return [redact_value(item, _depth=_depth + 1) for item in value] - return value + return redact_text(str(value)) def _redact_exception( @@ -203,10 +215,8 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: else: record.args = tuple(redact_value(item) for item in record.args) - standard = set(logging.LogRecord(None, 0, "", 0, "", (), None).__dict__) - standard.update({"message", "asctime"}) for field, value in list(record.__dict__.items()): - if field in standard or field.startswith("_"): + if field in _STANDARD_LOG_RECORD_ATTRS or field.startswith("_"): continue record.__dict__[field] = redact_value(value, key=field) From 9f7fb3d2ea308658fd0080f477c94c6b0973c8dd Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:04:54 +0000 Subject: [PATCH 21/41] test(logging): prove provider storage and exception boundaries --- .../common/test_safe_logging_boundaries.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 backend/django/common/test_safe_logging_boundaries.py diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py new file mode 100644 index 00000000..50e507af --- /dev/null +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -0,0 +1,59 @@ +import io +import logging + +from django.test import SimpleTestCase + +from common.safe_logging import install_safe_logging + + +class SafeLoggingBoundaryTests(SimpleTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + install_safe_logging() + + def _capture(self, logger_name: str): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(logging.Formatter("%(message)s context=%(context)s")) + logger = logging.getLogger(logger_name) + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + self.addCleanup(logger.handlers.clear) + return logger, stream + + def test_storage_and_provider_loggers_share_the_global_boundary(self): + for logger_name in ("common.storage", "apps.providers.services"): + with self.subTest(logger=logger_name): + logger, stream = self._capture(logger_name) + logger.info( + "operation completed", + extra={ + "context": { + "storage_key": "tenant/evidence/private.jpg", + "client_secret": "provider-secret", + "provider_code": "safe-provider-code", + } + }, + ) + output = stream.getvalue() + self.assertNotIn("tenant/evidence/private.jpg", output) + self.assertNotIn("provider-secret", output) + self.assertIn("safe-provider-code", output) + + def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): + logger, stream = self._capture("identitycore.argument-redaction-test") + error = RuntimeError("token=exception-secret email=subject@example.test") + + logger.error( + "provider error: %s", + error, + extra={"context": {"subject@example.test": "lookup", "status": "failed"}}, + ) + + output = stream.getvalue() + self.assertNotIn("exception-secret", output) + self.assertNotIn("subject@example.test", output) + self.assertIn("RuntimeError", output) + self.assertIn("failed", output) From dc9bcb867af57256f9a02507c30a7467e06ae670 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:07:16 +0000 Subject: [PATCH 22/41] fix(logging): cover correlated identifiers and multiword PII --- backend/shared/logging_redaction.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index d7822287..3c5a84c5 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -32,16 +32,21 @@ "session_token", "set_cookie", "token", - # Direct identifiers / PII. + # Direct identifiers / PII / correlation values that may identify a subject. "address", "birth_date", + "client_ip", "date_of_birth", + "device_fingerprint", "dob", "document_number", "email", + "external_reference", "first_name", "full_name", "ghana_card_number", + "ip", + "ip_address", "last_name", "middle_name", "national_id", @@ -49,8 +54,12 @@ "phone", "phone_number", "postal_address", + "remote_addr", + "subject_id", "tax_identification_number", "tin", + "user_agent", + "verification_subject_id", # Evidence locations and raw document / biometric material. "biometric_payload", "biometric_template", @@ -82,13 +91,16 @@ "_client_secret", "_credential", "_credentials", + "_fingerprint", "_password", "_private_key", "_refresh_token", "_secret", "_session_token", "_storage_key", + "_subject_id", "_token", + "_user_agent", ) _SENSITIVE_FRAGMENTS = ( @@ -114,9 +126,11 @@ r"(?i)\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|" r"access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|" r"first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|" - r"national[_-]?id|date[_-]?of[_-]?birth|dob|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|" + r"national[_-]?id|date[_-]?of[_-]?birth|dob|external[_-]?reference|device[_-]?fingerprint|" + r"subject[_-]?id|verification[_-]?subject[_-]?id|client[_-]?ip|ip[_-]?address|remote[_-]?addr|" + r"user[_-]?agent|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|" r"biometric[_-]?(?:payload|template))\b" - r"\s*[:=]\s*([\"']?)([^\s,;\"'}]+)\2" + r"\s*[:=]\s*([\"']?)([^,;\n\r\"'}]+)\2" ) _AWS_ACCESS_KEY_RE = re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b") _STANDARD_LOG_RECORD_ATTRS = frozenset( From b1217a4874608c6be804f4b8bd2623dc94dad9e9 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:07:34 +0000 Subject: [PATCH 23/41] fix(logging): redact correlated frontend identifiers --- .../packages/api-client/src/safe-logging.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts index 40997145..3d482ed4 100644 --- a/frontend/packages/api-client/src/safe-logging.ts +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -12,17 +12,20 @@ const SENSITIVE_KEYS = new Set([ "biometric_payload", "biometric_template", "birth_date", + "client_ip", "client_secret", "cookie", "credentials", "csrf_token", "date_of_birth", + "device_fingerprint", "dob", "document_bytes", "document_image", "document_number", "document_storage_key", "email", + "external_reference", "face_embedding", "face_image", "first_name", @@ -32,6 +35,8 @@ const SENSITIVE_KEYS = new Set([ "image", "image_base64", "image_bytes", + "ip", + "ip_address", "last_name", "liveness_video", "middle_name", @@ -48,6 +53,7 @@ const SENSITIVE_KEYS = new Set([ "raw_image", "raw_ocr", "refresh_token", + "remote_addr", "secret", "secret_access_key", "selfie", @@ -56,9 +62,12 @@ const SENSITIVE_KEYS = new Set([ "session_token", "set_cookie", "storage_key", + "subject_id", "tax_identification_number", "tin", "token", + "user_agent", + "verification_subject_id", ]); const SENSITIVE_SUFFIXES = [ @@ -68,13 +77,16 @@ const SENSITIVE_SUFFIXES = [ "_client_secret", "_credential", "_credentials", + "_fingerprint", "_password", "_private_key", "_refresh_token", "_secret", "_session_token", "_storage_key", + "_subject_id", "_token", + "_user_agent", ]; const SENSITIVE_FRAGMENTS = [ @@ -113,7 +125,7 @@ export function redactLogText(value: string): string { ) .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED) .replace( - /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^\s,;"'}]+)\2/gi, + /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|external[_-]?reference|device[_-]?fingerprint|subject[_-]?id|verification[_-]?subject[_-]?id|client[_-]?ip|ip[_-]?address|remote[_-]?addr|user[_-]?agent|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^,;\n\r"'}]+)\2/gi, (_match, label: string) => `${label}=${REDACTED}`, ) .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) @@ -140,7 +152,11 @@ export function redactLogValue( if (typeof value === "object") { const output: Record = {}; for (const [entryKey, entryValue] of Object.entries(value)) { - output[entryKey] = redactLogValue(entryValue, entryKey, depth + 1); + output[redactLogText(entryKey)] = redactLogValue( + entryValue, + entryKey, + depth + 1, + ); } return output; } From 678cb3bda4ad34922886408617284a18a9e1eec6 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:07:56 +0000 Subject: [PATCH 24/41] test(logging): cover correlated and multiword PII --- .../packages/api-client/test/safe-logging.test.mjs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs index 2d94251f..2258c5ab 100644 --- a/frontend/packages/api-client/test/safe-logging.test.mjs +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -25,6 +25,10 @@ test("redacts nested credentials, PII, evidence, and binary values", () => { email: "ada@example.test", phone_number: "+233241234567", document_number: "GHA-123456789", + external_reference: "customer-4482", + device_fingerprint: "device-secret", + user_agent: "browser-fingerprint", + verification_subject_id: "vs_sensitive", safe_status: "pending_review", }, evidence: { @@ -39,6 +43,10 @@ test("redacts nested credentials, PII, evidence, and binary values", () => { assert.equal(redacted.profile.email, REDACTED); assert.equal(redacted.profile.phone_number, REDACTED); assert.equal(redacted.profile.document_number, REDACTED); + assert.equal(redacted.profile.external_reference, REDACTED); + assert.equal(redacted.profile.device_fingerprint, REDACTED); + assert.equal(redacted.profile.user_agent, REDACTED); + assert.equal(redacted.profile.verification_subject_id, REDACTED); assert.equal(redacted.profile.safe_status, "pending_review"); assert.equal(redacted.evidence.selfie_image, REDACTED); assert.equal(redacted.evidence.face_embedding, REDACTED); @@ -46,14 +54,14 @@ test("redacts nested credentials, PII, evidence, and binary values", () => { assert.equal(redacted.binary, REDACTED_BINARY); }); -test("redacts secrets and identifiers embedded in free text", () => { +test("redacts secrets and multiword identifiers embedded in free text", () => { const output = redactLogText( - "token=top-secret email=ada@example.test phone=+233241234567 document_number=GHA-123 Authorization: Bearer bearer-secret", + "full_name=Ada Lovelace; token=top-secret; email=ada@example.test; phone=+233241234567; external_reference=customer-4482; document_number=GHA-123; Authorization: Bearer bearer-secret", ); assert.doesNotMatch( output, - /top-secret|ada@example\.test|233241234567|GHA-123|bearer-secret/, + /Ada Lovelace|top-secret|ada@example\.test|233241234567|customer-4482|GHA-123|bearer-secret/, ); assert.match(output, /\[REDACTED\]/); }); From 2cfaece59b1ff83bd3d4b7d0507eb99cdf08fb41 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:08:25 +0000 Subject: [PATCH 25/41] test(logging): cover correlated backend identifiers --- .../common/test_safe_logging_boundaries.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 50e507af..7a47a221 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -33,6 +33,10 @@ def test_storage_and_provider_loggers_share_the_global_boundary(self): "context": { "storage_key": "tenant/evidence/private.jpg", "client_secret": "provider-secret", + "external_reference": "customer-4482", + "device_fingerprint": "device-secret", + "user_agent": "browser-fingerprint", + "verification_subject_id": "vs_sensitive", "provider_code": "safe-provider-code", } }, @@ -40,11 +44,15 @@ def test_storage_and_provider_loggers_share_the_global_boundary(self): output = stream.getvalue() self.assertNotIn("tenant/evidence/private.jpg", output) self.assertNotIn("provider-secret", output) + self.assertNotIn("customer-4482", output) + self.assertNotIn("device-secret", output) + self.assertNotIn("browser-fingerprint", output) + self.assertNotIn("vs_sensitive", output) self.assertIn("safe-provider-code", output) def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): logger, stream = self._capture("identitycore.argument-redaction-test") - error = RuntimeError("token=exception-secret email=subject@example.test") + error = RuntimeError("token=exception-secret; email=subject@example.test") logger.error( "provider error: %s", @@ -57,3 +65,15 @@ def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): self.assertNotIn("subject@example.test", output) self.assertIn("RuntimeError", output) self.assertIn("failed", output) + + def test_multiword_pii_in_free_text_is_fully_removed(self): + logger, stream = self._capture("identitycore.multiword-redaction-test") + logger.info( + "review full_name=Ada Lovelace; external_reference=customer-4482; status=pending", + extra={"context": {"status": "pending"}}, + ) + + output = stream.getvalue() + self.assertNotIn("Ada Lovelace", output) + self.assertNotIn("customer-4482", output) + self.assertIn("status=pending", output) From f3bb8888be28a80e6e369a07aa945b5d2d4c817a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:09:10 +0000 Subject: [PATCH 26/41] fix(logging): redact rendered messages without breaking format args --- backend/shared/logging_redaction.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 3c5a84c5..1dcc1704 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -221,13 +221,15 @@ def _redact_exception( def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: - """Sanitize message args, structured extras, stack text, and exception text in place.""" - record.msg = redact_value(record.msg) + """Sanitize rendered messages, structured extras, stack text, and exceptions.""" if record.args: - if isinstance(record.args, Mapping): - record.args = redact_value(record.args) - else: - record.args = tuple(redact_value(item) for item in record.args) + # Render once using Python logging's normal interpolation rules, then redact + # the complete result. Redacting the template before interpolation can remove + # placeholders and cause handlers to fail with formatting errors. + record.msg = redact_text(record.getMessage()) + record.args = () + else: + record.msg = redact_value(record.msg) for field, value in list(record.__dict__.items()): if field in _STANDARD_LOG_RECORD_ATTRS or field.startswith("_"): From 6cae92f0e0ac2394e464bd7ea45bc26d2cd50e88 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:10:26 +0000 Subject: [PATCH 27/41] fix(logging): contain malformed format strings safely --- backend/shared/logging_redaction.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 1dcc1704..b13527db 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -10,6 +10,7 @@ REDACTED = "[REDACTED]" REDACTED_BINARY = "[REDACTED_BINARY]" MAX_REDACTION_DEPTH = 12 +LOG_FORMAT_ERROR = "[LOG_FORMAT_ERROR]" _SENSITIVE_KEYS = frozenset( { @@ -224,9 +225,13 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: """Sanitize rendered messages, structured extras, stack text, and exceptions.""" if record.args: # Render once using Python logging's normal interpolation rules, then redact - # the complete result. Redacting the template before interpolation can remove - # placeholders and cause handlers to fail with formatting errors. - record.msg = redact_text(record.getMessage()) + # the complete result. If the caller supplied a malformed format string, + # discard the args instead of letting logging break request/worker execution. + try: + rendered_message = record.getMessage() + except (TypeError, ValueError): + rendered_message = f"{redact_value(record.msg)} {LOG_FORMAT_ERROR}" + record.msg = redact_text(str(rendered_message)) record.args = () else: record.msg = redact_value(record.msg) @@ -264,6 +269,7 @@ def install_safe_logging() -> None: __all__ = [ + "LOG_FORMAT_ERROR", "REDACTED", "REDACTED_BINARY", "install_safe_logging", From 23961ed373683903e0d81d9d32ca0d9e9a99611e Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:10:49 +0000 Subject: [PATCH 28/41] test(logging): contain malformed format strings --- .../django/common/test_safe_logging_boundaries.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 7a47a221..94faf4d3 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -3,7 +3,7 @@ from django.test import SimpleTestCase -from common.safe_logging import install_safe_logging +from common.safe_logging import LOG_FORMAT_ERROR, install_safe_logging class SafeLoggingBoundaryTests(SimpleTestCase): @@ -77,3 +77,16 @@ def test_multiword_pii_in_free_text_is_fully_removed(self): self.assertNotIn("Ada Lovelace", output) self.assertNotIn("customer-4482", output) self.assertIn("status=pending", output) + + def test_malformed_format_string_does_not_raise_or_render_arguments(self): + logger, stream = self._capture("identitycore.format-error-test") + logger.info( + "provider failed without placeholder", + "secret-that-must-not-render", + extra={"context": {"status": "failed"}}, + ) + + output = stream.getvalue() + self.assertNotIn("secret-that-must-not-render", output) + self.assertIn(LOG_FORMAT_ERROR, output) + self.assertIn("failed", output) From e2691729ae9315553462683dcb540539cd33921a Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:11:01 +0000 Subject: [PATCH 29/41] fix(logging): keep Django safe logging exports aligned --- backend/django/common/safe_logging.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/django/common/safe_logging.py b/backend/django/common/safe_logging.py index bf9971bd..5c0ef2dc 100644 --- a/backend/django/common/safe_logging.py +++ b/backend/django/common/safe_logging.py @@ -8,6 +8,7 @@ sys.path.insert(0, str(_BACKEND_ROOT)) from shared.logging_redaction import ( # noqa: E402 + LOG_FORMAT_ERROR, REDACTED, REDACTED_BINARY, install_safe_logging, @@ -18,6 +19,7 @@ ) __all__ = [ + "LOG_FORMAT_ERROR", "REDACTED", "REDACTED_BINARY", "install_safe_logging", From be97df11efcb014c9af1660956da074f112aebc1 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:11:11 +0000 Subject: [PATCH 30/41] fix(logging): keep AI safe logging exports aligned --- backend/ai-service/app/core/safe_logging.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/ai-service/app/core/safe_logging.py b/backend/ai-service/app/core/safe_logging.py index 67d26ee5..3fa281c7 100644 --- a/backend/ai-service/app/core/safe_logging.py +++ b/backend/ai-service/app/core/safe_logging.py @@ -8,6 +8,7 @@ sys.path.insert(0, str(_BACKEND_ROOT)) from shared.logging_redaction import ( # noqa: E402 + LOG_FORMAT_ERROR, REDACTED, REDACTED_BINARY, install_safe_logging, @@ -18,6 +19,7 @@ ) __all__ = [ + "LOG_FORMAT_ERROR", "REDACTED", "REDACTED_BINARY", "install_safe_logging", From c72a71d0effb23f096b9fa5731520201f50675db Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:30:11 +0000 Subject: [PATCH 31/41] Fix managed AI redaction acceptance test --- backend/ai-service/tests/test_safe_logging.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/ai-service/tests/test_safe_logging.py b/backend/ai-service/tests/test_safe_logging.py index 702e0704..0942bea3 100644 --- a/backend/ai-service/tests/test_safe_logging.py +++ b/backend/ai-service/tests/test_safe_logging.py @@ -35,8 +35,9 @@ def test_managed_ai_uses_shared_nested_redaction_corpus(): assert redacted["request"]["email"] == REDACTED assert redacted["request"]["document_number"] == REDACTED assert redacted["request"]["safe_operation"] == "face_compare" - assert redacted["biometrics"]["face_embedding"] == REDACTED - assert redacted["biometrics"]["selfie_image"] == REDACTED + # A container explicitly named "biometrics" is sensitive as a whole. The + # redactor intentionally fails closed instead of retaining its structure. + assert redacted["biometrics"] == REDACTED def test_managed_ai_logger_redacts_structured_context_and_exception_text(): From 6668efb9ded9b1136fa0475ec0f2d06077423a62 Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:30:22 +0000 Subject: [PATCH 32/41] Route verification portal errors through safe logging --- frontend/verification-portal/src/app/error.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/verification-portal/src/app/error.tsx b/frontend/verification-portal/src/app/error.tsx index e1b57b3a..008f7216 100644 --- a/frontend/verification-portal/src/app/error.tsx +++ b/frontend/verification-portal/src/app/error.tsx @@ -2,6 +2,7 @@ import { useEffect } from "react"; import { AlertTriangle } from "lucide-react"; +import { safeLog } from "@identitycore/api-client"; import { Button } from "@identitycore/ui"; import { VerificationShell } from "@/components/layout/verification-shell"; @@ -13,10 +14,11 @@ export default function GlobalError({ reset: () => void; }) { useEffect(() => { - // Errors here are boundary-level render failures, not the flow's own - // handled error state. Avoid logging identity evidence or session - // tokens; only the error/digest are safe to surface. - console.error(error); + safeLog("error", "verification_portal_render_error", { + error_name: error.name, + error_message: error.message, + digest: error.digest ?? "", + }); }, [error]); return ( From 24882cf5a7c0a6704a8a3ce34422c68a3c69f83c Mon Sep 17 00:00:00 2001 From: quarj0 <54241472+quarj0@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:30:34 +0000 Subject: [PATCH 33/41] Make safe logging lint AST-aware --- frontend/scripts/check-safe-logging.mjs | 50 ++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/frontend/scripts/check-safe-logging.mjs b/frontend/scripts/check-safe-logging.mjs index 4e1e3013..65fe4912 100644 --- a/frontend/scripts/check-safe-logging.mjs +++ b/frontend/scripts/check-safe-logging.mjs @@ -1,8 +1,16 @@ import { readFileSync, readdirSync } from "node:fs"; +import { createRequire } from "node:module"; import { dirname, extname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const frontendRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const require = createRequire(import.meta.url); +const ts = require( + resolve( + frontendRoot, + "packages/api-client/node_modules/typescript/lib/typescript.js", + ), +); const allowedConsoleFile = resolve( frontendRoot, "packages/api-client/src/safe-logging.ts", @@ -16,7 +24,7 @@ const sourceRoots = [ "packages", ].map((path) => resolve(frontendRoot, path)); const extensions = new Set([".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"]); -const directConsole = /\bconsole\.(?:debug|info|log|warn|error|trace)\s*\(/; +const consoleMethods = new Set(["debug", "info", "log", "warn", "error", "trace"]); const skipDirectories = new Set([ ".next", ".safe-logging-test", @@ -28,6 +36,42 @@ const skipDirectories = new Set([ "tests", ]); +function scriptKind(path) { + if (path.endsWith(".tsx")) return ts.ScriptKind.TSX; + if (path.endsWith(".jsx")) return ts.ScriptKind.JSX; + if (path.endsWith(".ts")) return ts.ScriptKind.TS; + return ts.ScriptKind.JS; +} + +function containsDirectConsoleCall(path, source) { + const sourceFile = ts.createSourceFile( + path, + source, + ts.ScriptTarget.Latest, + true, + scriptKind(path), + ); + let found = false; + + function visit(node) { + if (found) return; + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + node.expression.expression.text === "console" && + consoleMethods.has(node.expression.name.text) + ) { + found = true; + return; + } + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return found; +} + function walk(path, findings) { for (const entry of readdirSync(path, { withFileTypes: true })) { if (entry.isDirectory() && skipDirectories.has(entry.name)) continue; @@ -40,7 +84,9 @@ function walk(path, findings) { if (/\.(?:spec|test)\.[cm]?[jt]sx?$/.test(entry.name)) continue; if (absolute === allowedConsoleFile) continue; const source = readFileSync(absolute, "utf8"); - if (directConsole.test(source)) findings.push(relative(frontendRoot, absolute)); + if (containsDirectConsoleCall(absolute, source)) { + findings.push(relative(frontendRoot, absolute)); + } } } From e31f024100892b0f6cb2ec3652321614116f1ebd Mon Sep 17 00:00:00 2001 From: quarj0 Date: Fri, 11 Sep 2026 00:11:40 +0000 Subject: [PATCH 34/41] Fix structured log argument redaction before interpolation --- .../common/test_safe_logging_boundaries.py | 27 +++++++++++++++++++ backend/shared/logging_redaction.py | 12 ++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 94faf4d3..08f0d265 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -66,6 +66,33 @@ def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): self.assertIn("RuntimeError", output) self.assertIn("failed", output) + def test_interpolated_structures_and_bytes_are_redacted(self): + logger, stream = self._capture("identitycore.structured-arguments-test") + logger.info( + "payload=%s binary=%s attempts=%03d", + {"storage_key": "tenant/private.jpg", "status": "failed"}, + b"private-document-content", + 7, + extra={"context": {}}, + ) + output = stream.getvalue() + self.assertNotIn("tenant/private.jpg", output) + self.assertNotIn("private-document-content", output) + self.assertIn("failed", output) + self.assertIn("attempts=007", output) + + def test_mapping_interpolation_preserves_exception_type_and_numbers(self): + logger, stream = self._capture("identitycore.mapping-arguments-test") + logger.error( + "error=%(error)s; attempts=%(attempts)03d", + {"error": RuntimeError("token=private-token"), "attempts": 7}, + extra={"context": {}}, + ) + output = stream.getvalue() + self.assertNotIn("private-token", output) + self.assertIn("RuntimeError", output) + self.assertIn("attempts=007", output) + def test_multiword_pii_in_free_text_is_fully_removed(self): logger, stream = self._capture("identitycore.multiword-redaction-test") logger.info( diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index b13527db..1c28e323 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -114,9 +114,7 @@ ) _BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+") -_JWT_RE = re.compile( - r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b" -) +_JWT_RE = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b") _EMAIL_RE = re.compile( r"(? logging.LogRecord: """Sanitize rendered messages, structured extras, stack text, and exceptions.""" if record.args: + # Preserve structured redaction and exception types before interpolation + # turns arguments into plain text. Numeric arguments retain their types. + if isinstance(record.args, Mapping): + record.args = { + key: redact_value(value, key=key) for key, value in record.args.items() + } + else: + record.args = redact_value(record.args) # Render once using Python logging's normal interpolation rules, then redact # the complete result. If the caller supplied a malformed format string, # discard the args instead of letting logging break request/worker execution. From 25309bfe99206fc7cfac44515b0d9c9317a8d6de Mon Sep 17 00:00:00 2001 From: quarj0 Date: Fri, 11 Sep 2026 00:29:49 +0000 Subject: [PATCH 35/41] Address cross-service redaction review findings --- backend/ai-service/tests/test_safe_logging.py | 31 ++++++++++++ .../common/test_safe_logging_boundaries.py | 35 +++++++++++++ backend/shared/logging_redaction.py | 50 +++++++++++++------ .../packages/api-client/src/safe-logging.ts | 26 +++++++--- .../api-client/test/safe-logging.test.mjs | 46 +++++++++++++++++ 5 files changed, 167 insertions(+), 21 deletions(-) diff --git a/backend/ai-service/tests/test_safe_logging.py b/backend/ai-service/tests/test_safe_logging.py index 0942bea3..8e33a969 100644 --- a/backend/ai-service/tests/test_safe_logging.py +++ b/backend/ai-service/tests/test_safe_logging.py @@ -72,3 +72,34 @@ def test_managed_ai_logger_redacts_structured_context_and_exception_text(): assert "raw OCR contents" not in output assert "document_ocr" in output assert "Traceback" in output + + +def test_uvicorn_access_formatter_preserves_protocol_without_private_data(): + from uvicorn.logging import AccessFormatter + + install_safe_logging() + logger = logging.getLogger("uvicorn.access") + previous = (logger.handlers[:], logger.propagate, logger.level) + stream = _capture(logger) + logger.handlers[0].setFormatter( + AccessFormatter( + '%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False + ) + ) + try: + logger.info( + '%s - "%s %s HTTP/%s" %d', + "192.0.2.1:1234", + "GET", + "/check?access_token=private-token", + "1.1", + 200, + ) + output = stream.getvalue() + assert "GET" in output + assert "200 OK" in output + assert "private-token" not in output + assert "192.0.2.1" not in output + assert "HTTP/1.1" in output + finally: + logger.handlers, logger.propagate, logger.level = previous diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 08f0d265..89667114 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -117,3 +117,38 @@ def test_malformed_format_string_does_not_raise_or_render_arguments(self): self.assertNotIn("secret-that-must-not-render", output) self.assertIn(LOG_FORMAT_ERROR, output) self.assertIn("failed", output) + + def test_all_sensitive_keys_are_redacted_in_serialized_text(self): + from common.safe_logging import redact_text + from shared.logging_redaction import _SENSITIVE_KEYS + + for key in _SENSITIVE_KEYS: + for spelling in (key, key.replace("_", "-"), key.upper()): + for message in ( + f'{{"{spelling}":"private-value"}}', + f"?{spelling}=private-value&status=failed", + f"{spelling}='private-value'; status=failed", + ): + with self.subTest(message=message): + self.assertNotIn("private-value", redact_text(message)) + for message in ( + '{"context":{"access_token":"private-value"}}', + '{"face_embedding":["private-value", "second-private"]}', + '{"private_key":"private-value\\"still-private"}', + ): + self.assertNotIn("private-value", redact_text(message)) + self.assertNotIn("second-private", redact_text(message)) + self.assertNotIn("still-private", redact_text(message)) + + def test_underscore_prefixed_extras_are_sanitized(self): + logger, stream = self._capture("identitycore.private-extra-test") + logger.handlers[0].setFormatter(logging.Formatter("%(_password)s %(_context)s")) + logger.info( + "request failed", + extra={ + "_password": "private-password", + "_context": {"token": "private-token"}, + }, + ) + self.assertNotIn("private-password", stream.getvalue()) + self.assertNotIn("private-token", stream.getvalue()) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 1c28e323..7872b78b 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -26,6 +26,7 @@ "csrfmiddlewaretoken", "id_token", "password", + "passcode", "private_key", "refresh_token", "secret", @@ -121,15 +122,9 @@ ) _IPV4_RE = re.compile(r"(?[\w.-]+)[\"']?\s*[:=]\s*") +_ASSIGNMENT_VALUE_RE = re.compile( + r"\[REDACTED(?:_[A-Z_]+)?\][^,;\n\r&}]*|\[[^\n\r]*|\{[^\n\r]*|\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'|[^,;\n\r&}]+" ) _AWS_ACCESS_KEY_RE = re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b") _STANDARD_LOG_RECORD_ATTRS = frozenset( @@ -166,9 +161,20 @@ def redact_text(value: str) -> str: redacted = _BEARER_RE.sub("Bearer [REDACTED]", value) redacted = _JWT_RE.sub(REDACTED, redacted) redacted = _AWS_ACCESS_KEY_RE.sub(REDACTED, redacted) - redacted = _CREDENTIAL_ASSIGNMENT_RE.sub( - lambda match: f"{match.group(1)}={REDACTED}", redacted - ) + parts = [] + cursor = 0 + for match in _CREDENTIAL_ASSIGNMENT_RE.finditer(redacted): + if match.start() < cursor or not is_sensitive_key(match.group("label")): + continue + value_match = _ASSIGNMENT_VALUE_RE.match(redacted, match.end()) + if value_match is None: + continue + parts.extend( + (redacted[cursor : match.start()], f"{match.group('label')}={REDACTED}") + ) + cursor = value_match.end() + parts.append(redacted[cursor:]) + redacted = "".join(parts) redacted = _EMAIL_RE.sub(REDACTED, redacted) redacted = _IPV4_RE.sub(REDACTED, redacted) redacted = _PHONE_RE.sub(REDACTED, redacted) @@ -221,7 +227,23 @@ def _redact_exception( def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: """Sanitize rendered messages, structured extras, stack text, and exceptions.""" - if record.args: + if ( + record.name == "uvicorn.access" + and isinstance(record.args, tuple) + and len(record.args) == 5 + ): + # Uvicorn's AccessFormatter unpacks this tuple after getMessage(). Keep + # its protocol shape, but never expose client addresses or query strings. + client, method, path, version, status = record.args + record.args = ( + REDACTED, + redact_value(method), + REDACTED, + redact_value(version), + status, + ) + record.msg = '%s - "%s %s HTTP/%s" %d' + elif record.args: # Preserve structured redaction and exception types before interpolation # turns arguments into plain text. Numeric arguments retain their types. if isinstance(record.args, Mapping): @@ -243,7 +265,7 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: record.msg = redact_value(record.msg) for field, value in list(record.__dict__.items()): - if field in _STANDARD_LOG_RECORD_ATTRS or field.startswith("_"): + if field in _STANDARD_LOG_RECORD_ATTRS: continue record.__dict__[field] = redact_value(value, key=field) diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts index 3d482ed4..0f279377 100644 --- a/frontend/packages/api-client/src/safe-logging.ts +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -17,6 +17,7 @@ const SENSITIVE_KEYS = new Set([ "cookie", "credentials", "csrf_token", + "csrfmiddlewaretoken", "date_of_birth", "device_fingerprint", "dob", @@ -45,6 +46,7 @@ const SENSITIVE_KEYS = new Set([ "ocr_text", "passport_number", "password", + "passcode", "phone", "phone_number", "postal_address", @@ -117,17 +119,27 @@ export function isSensitiveLogKey(key: PropertyKey): boolean { } export function redactLogText(value: string): string { - return value + const redacted = value .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]") .replace( /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, REDACTED, ) - .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED) - .replace( - /\b(authorization|password|passcode|secret|token|api[_-]?key|client[_-]?secret|access[_-]?key|refresh[_-]?token|session[_-]?token|cookie|email|phone(?:_number)?|first[_-]?name|last[_-]?name|full[_-]?name|address|document[_-]?number|passport[_-]?number|national[_-]?id|date[_-]?of[_-]?birth|dob|external[_-]?reference|device[_-]?fingerprint|subject[_-]?id|verification[_-]?subject[_-]?id|client[_-]?ip|ip[_-]?address|remote[_-]?addr|user[_-]?agent|selfie(?:_image)?|image[_-]?base64|ocr[_-]?text|mrz|biometric[_-]?(?:payload|template))\b\s*[:=]\s*(["']?)([^,;\n\r"'}]+)\2/gi, - (_match, label: string) => `${label}=${REDACTED}`, - ) + .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED); + const assignment = /([\w.-]+)["']?\s*[:=]\s*/g; + const valuePattern = + /^(?:\[REDACTED(?:_[A-Z_]+)?\][^,;\n\r&}]*|\[[^\n\r]*|\{[^\n\r]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^,;\n\r&}]+)/; + let output = ""; + let cursor = 0; + for (const match of redacted.matchAll(assignment)) { + if (match.index < cursor || !isSensitiveLogKey(match[1])) continue; + const start = match.index + match[0].length; + const valueMatch = redacted.slice(start).match(valuePattern); + if (!valueMatch) continue; + output += redacted.slice(cursor, match.index) + `${match[1]}=${REDACTED}`; + cursor = start + valueMatch[0].length; + } + return (output + redacted.slice(cursor)) .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) .replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, REDACTED) .replace(/(?:\+?\d[\d ().-]{7,}\d)/g, REDACTED); @@ -143,7 +155,7 @@ export function redactLogValue( if (value === null || value === undefined) return value; if (typeof value === "string") return redactLogText(value); if (typeof value === "number" || typeof value === "boolean") return value; - if (value instanceof Uint8Array || value instanceof ArrayBuffer) { + if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) { return REDACTED_BINARY; } if (Array.isArray(value)) { diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs index 2258c5ab..47308e76 100644 --- a/frontend/packages/api-client/test/safe-logging.test.mjs +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -65,3 +65,49 @@ test("redacts secrets and multiword identifiers embedded in free text", () => { ); assert.match(output, /\[REDACTED\]/); }); + +test("redacts quoted keys, credential spellings, and nested serialized values", () => { + for (const key of [ + "access_token", + "id_token", + "private_key", + "secret_access_key", + "document_number", + "csrfmiddlewaretoken", + "storage_key", + "face_embedding", + ]) { + for (const spelling of [key, key.replaceAll("_", "-"), key.toUpperCase()]) { + for (const value of [ + `{"${spelling}":"private-value"}`, + `?${spelling}=private-value&status=failed`, + `${spelling}='private-value'; status=failed`, + ]) { + assert.doesNotMatch(redactLogText(value), /private-value/); + } + } + } + for (const value of [ + '{"context":{"access_token":"private-value"}}', + '{"face_embedding":["private-value", "second-private"]}', + '{"private_key":"private-value\\"still-private"}', + ]) { + assert.doesNotMatch( + redactLogText(value), + /private-value|second-private|still-private/, + ); + } +}); + +test("redacts all binary array views before object traversal", () => { + for (const view of [ + new Uint8ClampedArray([17]), + new Uint16Array([18]), + new Float32Array([0.25]), + new DataView(new ArrayBuffer(4)), + ]) { + assert.deepEqual(redactLogValue({ imageData: view }), { + imageData: REDACTED_BINARY, + }); + } +}); From 89c3f7ff1fe9462d4c45f724f7591740830a0c12 Mon Sep 17 00:00:00 2001 From: quarj0 Date: Fri, 11 Sep 2026 06:28:49 +0000 Subject: [PATCH 36/41] Close remaining credential and biometric logging gaps --- backend/ai-service/app/pipeline.py | 4 +- backend/ai-service/tests/test_safe_logging.py | 20 +++++++++ .../common/test_safe_logging_boundaries.py | 42 +++++++++++++++++++ backend/shared/logging_redaction.py | 27 +++++++++++- .../packages/api-client/src/safe-logging.ts | 34 +++++++++++++++ .../api-client/test/safe-logging.test.mjs | 38 +++++++++++++++++ 6 files changed, 160 insertions(+), 5 deletions(-) diff --git a/backend/ai-service/app/pipeline.py b/backend/ai-service/app/pipeline.py index 8792517f..9cca8a96 100644 --- a/backend/ai-service/app/pipeline.py +++ b/backend/ai-service/app/pipeline.py @@ -31,9 +31,7 @@ class ProcessingError(RuntimeError): class MediaAssetNotFoundError(RuntimeError): def __init__(self, storage_key: str, bucket_name: str): - message = ( - f"Media asset '{storage_key}' was not found in bucket '{bucket_name}'." - ) + message = "Media asset was not found in configured storage (media_asset_not_found)." super().__init__(message) self.storage_key = storage_key self.bucket_name = bucket_name diff --git a/backend/ai-service/tests/test_safe_logging.py b/backend/ai-service/tests/test_safe_logging.py index 8e33a969..fdfc744a 100644 --- a/backend/ai-service/tests/test_safe_logging.py +++ b/backend/ai-service/tests/test_safe_logging.py @@ -103,3 +103,23 @@ def test_uvicorn_access_formatter_preserves_protocol_without_private_data(): assert "HTTP/1.1" in output finally: logger.handlers, logger.propagate, logger.level = previous + + +def test_numpy_buffers_do_not_render_pixels_or_embeddings(): + import numpy as np + from app.core.safe_logging import REDACTED_BINARY + + for value in ( + np.array([17, 18], dtype=np.uint8), + np.array([0.125, 0.25], dtype=np.float32), + ): + assert redact_value({"frame": value}) == {"frame": REDACTED_BINARY} + + +def test_missing_media_error_does_not_expose_evidence_location(): + from app.pipeline import MediaAssetNotFoundError + + error = MediaAssetNotFoundError("tenant/evidence/private.jpg", "private-bucket") + assert "tenant/evidence/private.jpg" not in str(error) + assert "private-bucket" not in str(error) + assert "media_asset_not_found" in str(error) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 89667114..30fb1ae1 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -152,3 +152,45 @@ def test_underscore_prefixed_extras_are_sanitized(self): ) self.assertNotIn("private-password", stream.getvalue()) self.assertNotIn("private-token", stream.getvalue()) + + def test_deployed_credentials_signatures_and_ipv6_are_redacted(self): + from common.safe_logging import REDACTED, redact_text, redact_value + + for key in ( + "SECRET_KEY", + "DJANGO_SECRET_KEY", + "object_storage_access_key_id", + "object_storage_secret_access_key", + "aws_access_key_id", + "aws_secret_access_key", + "X-Amz-Signature", + "X-IdentityCore-Signature", + ): + self.assertEqual(redact_value({key: "private-value"})[key], REDACTED) + self.assertNotIn( + "private-value", redact_text(f"?{key}=private-value&status=ok") + ) + for address in ( + "2001:db8:1234:5678:9abc:def0:1234:5678", + "2001:db8::1", + "::1", + "::ffff:192.0.2.1", + "fe80::1%eth0", + ): + self.assertNotIn(address, redact_text(f"client connected from [{address}]")) + self.assertEqual( + redact_text("time 12:34:56; status=ok"), "time 12:34:56; status=ok" + ) + self.assertNotIn( + "private-value", redact_text("token=[REDACTED]private-value; status=ok") + ) + + def test_missing_mapping_key_and_overflow_do_not_escape_logging(self): + logger, stream = self._capture("identitycore.interpolation-failures-test") + for message, argument in ( + ("%(missing)s", {"present": "private-value"}), + ("%c", 0x110000), + ): + logger.info(message, argument, extra={"context": {}}) + self.assertEqual(stream.getvalue().count(LOG_FORMAT_ERROR), 2) + self.assertNotIn("private-value", stream.getvalue()) diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 7872b78b..65c5c39b 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ipaddress import logging import re import traceback @@ -16,6 +17,9 @@ { # Credentials and session material. "access_key", + "access_key_id", + "secret_key", + "signature", "access_token", "api_key", "authorization", @@ -87,6 +91,10 @@ ) _SENSITIVE_SUFFIXES = ( + "_secret_key", + "_access_key", + "_access_key_id", + "_signature", "_access_token", "_api_key", "_authorization", @@ -120,6 +128,7 @@ r"(?[\w.-]+)[\"']?\s*[:=]\s*") @@ -176,6 +185,15 @@ def redact_text(value: str) -> str: parts.append(redacted[cursor:]) redacted = "".join(parts) redacted = _EMAIL_RE.sub(REDACTED, redacted) + + def redact_ipv6(match): + try: + ipaddress.IPv6Address(match.group()) + except ValueError: + return match.group() + return REDACTED + + redacted = _IPV6_RE.sub(redact_ipv6, redacted) redacted = _IPV4_RE.sub(REDACTED, redacted) redacted = _PHONE_RE.sub(REDACTED, redacted) return redacted @@ -199,6 +217,11 @@ def redact_value(value: Any, *, key: object | None = None, _depth: int = 0) -> A return f"{value.__class__.__name__}: {redact_text(str(value))}" if isinstance(value, (bytes, bytearray, memoryview)): return REDACTED_BINARY + try: + with memoryview(value): + return REDACTED_BINARY + except TypeError: + pass if isinstance(value, Mapping): redacted_mapping = {} for item_key, item_value in value.items(): @@ -257,8 +280,8 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: # discard the args instead of letting logging break request/worker execution. try: rendered_message = record.getMessage() - except (TypeError, ValueError): - rendered_message = f"{redact_value(record.msg)} {LOG_FORMAT_ERROR}" + except Exception: + rendered_message = LOG_FORMAT_ERROR record.msg = redact_text(str(rendered_message)) record.args = () else: diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts index 0f279377..b5e45fed 100644 --- a/frontend/packages/api-client/src/safe-logging.ts +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -5,6 +5,9 @@ const MAX_REDACTION_DEPTH = 12; const SENSITIVE_KEYS = new Set([ "access_key", + "access_key_id", + "secret_key", + "signature", "access_token", "address", "api_key", @@ -73,6 +76,10 @@ const SENSITIVE_KEYS = new Set([ ]); const SENSITIVE_SUFFIXES = [ + "_secret_key", + "_access_key", + "_access_key_id", + "_signature", "_access_token", "_api_key", "_authorization", @@ -118,6 +125,30 @@ export function isSensitiveLogKey(key: PropertyKey): boolean { ); } +function isIpv6(value: string): boolean { + const address = value.split("%")[0]; + if (!address.includes(":")) return false; + let normalized = address; + if (address.includes(".")) { + const lastColon = address.lastIndexOf(":"); + const octets = address.slice(lastColon + 1).split("."); + if ( + octets.length !== 4 || + octets.some((part) => !/^\d{1,3}$/.test(part) || Number(part) > 255) + ) + return false; + normalized = address.slice(0, lastColon + 1) + "0:0"; + } + const parts = normalized.split(":"); + if (parts.some((part) => !/^[0-9a-f]{0,4}$/i.test(part))) return false; + if (!normalized.includes("::")) + return parts.length === 8 && parts.every(Boolean); + if (normalized.indexOf("::") !== normalized.lastIndexOf("::")) return false; + if (normalized.startsWith(":") && !normalized.startsWith("::")) return false; + if (normalized.endsWith(":") && !normalized.endsWith("::")) return false; + return parts.filter(Boolean).length < 8; +} + export function redactLogText(value: string): string { const redacted = value .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]") @@ -141,6 +172,9 @@ export function redactLogText(value: string): string { } return (output + redacted.slice(cursor)) .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) + .replace(/(? + isIpv6(address) ? REDACTED : address, + ) .replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, REDACTED) .replace(/(?:\+?\d[\d ().-]{7,}\d)/g, REDACTED); } diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs index 47308e76..7cd225ef 100644 --- a/frontend/packages/api-client/test/safe-logging.test.mjs +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -111,3 +111,41 @@ test("redacts all binary array views before object traversal", () => { }); } }); + +test("redacts deployed credential names, signatures, and IPv6 addresses", () => { + for (const key of [ + "SECRET_KEY", + "DJANGO_SECRET_KEY", + "object_storage_access_key_id", + "object_storage_secret_access_key", + "aws_access_key_id", + "aws_secret_access_key", + "X-Amz-Signature", + "X-IdentityCore-Signature", + ]) { + assert.equal(redactLogValue({ [key]: "private-value" })[key], REDACTED); + assert.doesNotMatch( + redactLogText(`?${key}=private-value&status=ok`), + /private-value/, + ); + } + for (const address of [ + "2001:db8:1234:5678:9abc:def0:1234:5678", + "2001:db8::1", + "::1", + "::ffff:192.0.2.1", + "fe80::1%eth0", + ]) { + assert.ok( + !redactLogText(`client connected from [${address}]`).includes(address), + ); + } + assert.equal( + redactLogText("time 12:34:56; status=ok"), + "time 12:34:56; status=ok", + ); + assert.doesNotMatch( + redactLogText("token=[REDACTED]private-value; status=ok"), + /private-value/, + ); +}); From b2dfd716152b34201121cf1b820e889a8d53fc1a Mon Sep 17 00:00:00 2001 From: quarj0 Date: Fri, 11 Sep 2026 06:39:42 +0000 Subject: [PATCH 37/41] Keep document promotion evidence keys out of warnings --- backend/django/apps/identity_documents/tasks.py | 6 +++--- backend/django/apps/identity_documents/tests.py | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/django/apps/identity_documents/tasks.py b/backend/django/apps/identity_documents/tasks.py index bce3b65e..6dade485 100644 --- a/backend/django/apps/identity_documents/tasks.py +++ b/backend/django/apps/identity_documents/tasks.py @@ -563,10 +563,10 @@ def process_identity_document_task(identity_document_id: str) -> str: promote_upload_to_media_by_storage_key(capture.storage_key) except Exception as exc: logger.warning( - "Failed to promote document upload %s for verification %s: %s", - capture.storage_key, + "Failed to promote document upload for verification %s (%s)", verification.public_id, - exc, + type(exc).__name__, + extra={"storage_key": capture.storage_key}, ) record_audit_event( tenant=verification.tenant, diff --git a/backend/django/apps/identity_documents/tests.py b/backend/django/apps/identity_documents/tests.py index 94799f75..052716f3 100644 --- a/backend/django/apps/identity_documents/tests.py +++ b/backend/django/apps/identity_documents/tests.py @@ -437,7 +437,10 @@ def test_process_identity_document_task_keeps_processing_when_promotion_fails( } mock_promote.side_effect = RuntimeError("storage unavailable") - result = process_identity_document_task(self.identity_document.public_id) + with self.assertLogs("apps.identity_documents.tasks", level="WARNING") as captured: + result = process_identity_document_task(self.identity_document.public_id) + self.assertNotIn(self.upload.storage_key, " ".join(captured.output)) + self.assertEqual(captured.records[0].storage_key, "[REDACTED]") self.assertEqual(result, IdentityDocumentStatus.PROCESSED) self.identity_document.refresh_from_db() From 1e372580d596e84dd51fcf0b2eee5b360d43c441 Mon Sep 17 00:00:00 2001 From: quarj0 Date: Fri, 11 Sep 2026 18:42:00 +0000 Subject: [PATCH 38/41] Cover equivalent console calls and camel case secrets --- .../common/test_safe_logging_boundaries.py | 18 ++++ backend/shared/logging_redaction.py | 4 +- frontend/packages/api-client/package.json | 2 +- .../packages/api-client/src/safe-logging.ts | 2 + .../api-client/test/safe-logging.test.mjs | 15 ++++ frontend/scripts/check-safe-logging.mjs | 82 ++++++++++++++----- frontend/scripts/check-safe-logging.test.mjs | 23 ++++++ 7 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 frontend/scripts/check-safe-logging.test.mjs diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 30fb1ae1..4aa07931 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -185,6 +185,24 @@ def test_deployed_credentials_signatures_and_ipv6_are_redacted(self): "private-value", redact_text("token=[REDACTED]private-value; status=ok") ) + def test_camel_case_sensitive_keys_are_redacted(self): + from common.safe_logging import REDACTED, redact_value + + redacted = redact_value( + { + "accessToken": "access-private", + "sessionToken": "session-private", + "clientSecret": "client-private", + "fullName": "Ada Private", + "safeStatus": "ready", + } + ) + self.assertEqual(redacted["accessToken"], REDACTED) + self.assertEqual(redacted["sessionToken"], REDACTED) + self.assertEqual(redacted["clientSecret"], REDACTED) + self.assertEqual(redacted["fullName"], REDACTED) + self.assertEqual(redacted["safeStatus"], "ready") + def test_missing_mapping_key_and_overflow_do_not_escape_logging(self): logger, stream = self._capture("identitycore.interpolation-failures-test") for message, argument in ( diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 65c5c39b..8e9c7062 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -150,7 +150,9 @@ def _normalize_key(key: object) -> str: - normalized = re.sub(r"[^a-z0-9]+", "_", str(key).strip().lower()) + text = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", str(key).strip()) + text = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", text) + normalized = re.sub(r"[^a-z0-9]+", "_", text.lower()) return normalized.strip("_") diff --git a/frontend/packages/api-client/package.json b/frontend/packages/api-client/package.json index 9efdef4f..18a252a8 100644 --- a/frontend/packages/api-client/package.json +++ b/frontend/packages/api-client/package.json @@ -8,7 +8,7 @@ ".": "./src/index.ts" }, "scripts": { - "lint": "tsc -p tsconfig.json && tsc -p tsconfig.safe-logging.json && node --test test/safe-logging.test.mjs && node ../../scripts/check-safe-logging.mjs" + "lint": "tsc -p tsconfig.json && tsc -p tsconfig.safe-logging.json && node --test test/safe-logging.test.mjs ../../scripts/check-safe-logging.test.mjs && node ../../scripts/check-safe-logging.mjs" }, "devDependencies": { "typescript": "^5" diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts index b5e45fed..fa77f8a4 100644 --- a/frontend/packages/api-client/src/safe-logging.ts +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -110,6 +110,8 @@ const SENSITIVE_FRAGMENTS = [ function normalizeKey(key: PropertyKey): string { return String(key) .trim() + .replace(/([a-z0-9])([A-Z])/g, "$1_$2") + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2") .toLowerCase() .replace(/[^a-z0-9]+/g, "_") .replace(/^_+|_+$/g, ""); diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs index 7cd225ef..197fca63 100644 --- a/frontend/packages/api-client/test/safe-logging.test.mjs +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -149,3 +149,18 @@ test("redacts deployed credential names, signatures, and IPv6 addresses", () => /private-value/, ); }); + +test("redacts camelCase credential and identity keys", () => { + const redacted = redactLogValue({ + accessToken: "access-private", + sessionToken: "session-private", + clientSecret: "client-private", + fullName: "Ada Private", + safeStatus: "ready", + }); + assert.equal(redacted.accessToken, REDACTED); + assert.equal(redacted.sessionToken, REDACTED); + assert.equal(redacted.clientSecret, REDACTED); + assert.equal(redacted.fullName, REDACTED); + assert.equal(redacted.safeStatus, "ready"); +}); diff --git a/frontend/scripts/check-safe-logging.mjs b/frontend/scripts/check-safe-logging.mjs index 65fe4912..00b1a230 100644 --- a/frontend/scripts/check-safe-logging.mjs +++ b/frontend/scripts/check-safe-logging.mjs @@ -1,7 +1,7 @@ import { readFileSync, readdirSync } from "node:fs"; import { createRequire } from "node:module"; import { dirname, extname, join, relative, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const frontendRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const require = createRequire(import.meta.url); @@ -43,7 +43,51 @@ function scriptKind(path) { return ts.ScriptKind.JS; } -function containsDirectConsoleCall(path, source) { +function propertyName(node) { + if (ts.isIdentifier(node) || ts.isStringLiteralLike(node)) return node.text; + return null; +} + +function isConsoleObject(node) { + if (ts.isIdentifier(node)) return node.text === "console"; + if (ts.isPropertyAccessExpression(node)) { + return ( + ts.isIdentifier(node.expression) && + ["window", "globalThis"].includes(node.expression.text) && + node.name.text === "console" + ); + } + if (ts.isElementAccessExpression(node)) { + return ( + ts.isIdentifier(node.expression) && + ["window", "globalThis"].includes(node.expression.text) && + propertyName(node.argumentExpression) === "console" + ); + } + return false; +} + +function isConsoleMethodReference(node) { + if (ts.isPropertyAccessExpression(node)) { + return isConsoleObject(node.expression) && consoleMethods.has(node.name.text); + } + if (ts.isElementAccessExpression(node)) { + const method = propertyName(node.argumentExpression); + return isConsoleObject(node.expression) && method !== null && consoleMethods.has(method); + } + return false; +} + +function destructuresConsoleMethod(node) { + if (!ts.isVariableDeclaration(node) || !ts.isObjectBindingPattern(node.name)) return false; + if (!node.initializer || !isConsoleObject(node.initializer)) return false; + return node.name.elements.some((element) => { + const name = propertyName(element.propertyName ?? element.name); + return name !== null && consoleMethods.has(name); + }); +} + +export function containsUnsafeConsoleUse(path, source) { const sourceFile = ts.createSourceFile( path, source, @@ -56,11 +100,9 @@ function containsDirectConsoleCall(path, source) { function visit(node) { if (found) return; if ( - ts.isCallExpression(node) && - ts.isPropertyAccessExpression(node.expression) && - ts.isIdentifier(node.expression.expression) && - node.expression.expression.text === "console" && - consoleMethods.has(node.expression.name.text) + ((ts.isCallExpression(node) && isConsoleMethodReference(node.expression)) || + isConsoleMethodReference(node) || + destructuresConsoleMethod(node)) ) { found = true; return; @@ -84,22 +126,24 @@ function walk(path, findings) { if (/\.(?:spec|test)\.[cm]?[jt]sx?$/.test(entry.name)) continue; if (absolute === allowedConsoleFile) continue; const source = readFileSync(absolute, "utf8"); - if (containsDirectConsoleCall(absolute, source)) { + if (containsUnsafeConsoleUse(absolute, source)) { findings.push(relative(frontendRoot, absolute)); } } } -const findings = []; -for (const root of sourceRoots) walk(root, findings); +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const findings = []; + for (const root of sourceRoots) walk(root, findings); -if (findings.length) { - console.error( - [ - "Unsafe direct console logging is not allowed in frontend production source.", - "Use safeLog from @identitycore/api-client so sensitive context is redacted.", - ...findings.map((path) => ` - ${path}`), - ].join("\n"), - ); - process.exitCode = 1; + if (findings.length) { + console.error( + [ + "Unsafe direct console logging is not allowed in frontend production source.", + "Use safeLog from @identitycore/api-client so sensitive context is redacted.", + ...findings.map((path) => ` - ${path}`), + ].join("\n"), + ); + process.exitCode = 1; + } } diff --git a/frontend/scripts/check-safe-logging.test.mjs b/frontend/scripts/check-safe-logging.test.mjs new file mode 100644 index 00000000..d9863b84 --- /dev/null +++ b/frontend/scripts/check-safe-logging.test.mjs @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { containsUnsafeConsoleUse } from "./check-safe-logging.mjs"; + +for (const source of [ + 'console["error"]("secret")', + 'window.console.error("secret")', + 'globalThis.console["warn"]("secret")', + 'window["console"]["info"]("secret")', + 'const { error } = console; error("secret")', + 'const { warn: report } = globalThis.console; report("secret")', + 'const report = console.error; report("secret")', +]) { + test(`rejects equivalent console use: ${source}`, () => { + assert.equal(containsUnsafeConsoleUse("fixture.ts", source), true); + }); +} + +test("allows unrelated methods and safe logging", () => { + assert.equal(containsUnsafeConsoleUse("fixture.ts", 'safeLog("error", "event")'), false); + assert.equal(containsUnsafeConsoleUse("fixture.ts", 'reporter.error("safe code")'), false); +}); From aacd38b78cf1ed08475f983b779542d56dca26fe Mon Sep 17 00:00:00 2001 From: quarj0 Date: Fri, 11 Sep 2026 19:24:36 +0000 Subject: [PATCH 39/41] Close remaining safe logging bypasses --- .../common/test_safe_logging_boundaries.py | 5 ++ backend/shared/logging_redaction.py | 48 ++++++++++++++++--- .../packages/api-client/src/safe-logging.ts | 43 ++++++++++++++--- .../api-client/test/safe-logging.test.mjs | 4 +- frontend/scripts/check-safe-logging.mjs | 9 ++-- frontend/scripts/check-safe-logging.test.mjs | 4 ++ 6 files changed, 94 insertions(+), 19 deletions(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 4aa07931..7e58ee56 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -135,10 +135,15 @@ def test_all_sensitive_keys_are_redacted_in_serialized_text(self): '{"context":{"access_token":"private-value"}}', '{"face_embedding":["private-value", "second-private"]}', '{"private_key":"private-value\\"still-private"}', + "full_name=Doe, Jane", + 'face_embedding=[\n 0.123,\n 0.456\n], "status":"failed"', ): self.assertNotIn("private-value", redact_text(message)) self.assertNotIn("second-private", redact_text(message)) self.assertNotIn("still-private", redact_text(message)) + self.assertNotIn("Jane", redact_text(message)) + self.assertNotIn("0.123", redact_text(message)) + self.assertNotIn("0.456", redact_text(message)) def test_underscore_prefixed_extras_are_sanitized(self): logger, stream = self._capture("identitycore.private-extra-test") diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 8e9c7062..696a26f5 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -132,8 +132,8 @@ _IPV4_RE = re.compile(r"(?[\w.-]+)[\"']?\s*[:=]\s*") -_ASSIGNMENT_VALUE_RE = re.compile( - r"\[REDACTED(?:_[A-Z_]+)?\][^,;\n\r&}]*|\[[^\n\r]*|\{[^\n\r]*|\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'|[^,;\n\r&}]+" +_NEXT_ASSIGNMENT_BOUNDARY_RE = re.compile( + r"[;&\n\r](?=\s*[\"']?[\w.-]+[\"']?\s*[:=])" ) _AWS_ACCESS_KEY_RE = re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b") _STANDARD_LOG_RECORD_ATTRS = frozenset( @@ -167,6 +167,44 @@ def is_sensitive_key(key: object) -> bool: return any(fragment in normalized for fragment in _SENSITIVE_FRAGMENTS) +def _assignment_value_end(value: str, start: int) -> int: + if start >= len(value): + return start + opener = value[start] + if opener in "\"'": + escaped = False + for index in range(start + 1, len(value)): + character = value[index] + if character == opener and not escaped: + return index + 1 + escaped = character == "\\" and not escaped + if character != "\\": + escaped = False + elif opener in "[{" and not value.startswith("[REDACTED", start): + pairs = {"[": "]", "{": "}"} + stack = [pairs[opener]] + quote = None + escaped = False + for index in range(start + 1, len(value)): + character = value[index] + if quote: + if character == quote and not escaped: + quote = None + escaped = character == "\\" and not escaped + if character != "\\": + escaped = False + elif character in "\"'": + quote = character + elif character in pairs: + stack.append(pairs[character]) + elif stack and character == stack[-1]: + stack.pop() + if not stack: + return index + 1 + boundary = _NEXT_ASSIGNMENT_BOUNDARY_RE.search(value, start) + return boundary.start() if boundary else len(value) + + def redact_text(value: str) -> str: """Redact common secret and PII shapes from unstructured log text.""" redacted = _BEARER_RE.sub("Bearer [REDACTED]", value) @@ -177,13 +215,11 @@ def redact_text(value: str) -> str: for match in _CREDENTIAL_ASSIGNMENT_RE.finditer(redacted): if match.start() < cursor or not is_sensitive_key(match.group("label")): continue - value_match = _ASSIGNMENT_VALUE_RE.match(redacted, match.end()) - if value_match is None: - continue + value_end = _assignment_value_end(redacted, match.end()) parts.extend( (redacted[cursor : match.start()], f"{match.group('label')}={REDACTED}") ) - cursor = value_match.end() + cursor = value_end parts.append(redacted[cursor:]) redacted = "".join(parts) redacted = _EMAIL_RE.sub(REDACTED, redacted) diff --git a/frontend/packages/api-client/src/safe-logging.ts b/frontend/packages/api-client/src/safe-logging.ts index fa77f8a4..6f514159 100644 --- a/frontend/packages/api-client/src/safe-logging.ts +++ b/frontend/packages/api-client/src/safe-logging.ts @@ -127,6 +127,42 @@ export function isSensitiveLogKey(key: PropertyKey): boolean { ); } +function assignmentValueEnd(value: string, start: number): number { + if (start >= value.length) return start; + const opener = value[start]; + if (opener === '"' || opener === "'") { + let escaped = false; + for (let index = start + 1; index < value.length; index += 1) { + const character = value[index]; + if (character === opener && !escaped) return index + 1; + escaped = character === "\\" && !escaped; + if (character !== "\\") escaped = false; + } + } else if ((opener === "[" || opener === "{") && !value.startsWith("[REDACTED", start)) { + const pairs: Record = { "[": "]", "{": "}" }; + const stack = [pairs[opener]]; + let quote: string | undefined; + let escaped = false; + for (let index = start + 1; index < value.length; index += 1) { + const character = value[index]; + if (quote) { + if (character === quote && !escaped) quote = undefined; + escaped = character === "\\" && !escaped; + if (character !== "\\") escaped = false; + } else if (character === '"' || character === "'") { + quote = character; + } else if (character in pairs) { + stack.push(pairs[character]); + } else if (character === stack.at(-1)) { + stack.pop(); + if (stack.length === 0) return index + 1; + } + } + } + const boundary = value.slice(start).search(/[;&\n\r](?=\s*["']?[\w.-]+["']?\s*[:=])/); + return boundary === -1 ? value.length : start + boundary; +} + function isIpv6(value: string): boolean { const address = value.split("%")[0]; if (!address.includes(":")) return false; @@ -160,17 +196,12 @@ export function redactLogText(value: string): string { ) .replace(/\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g, REDACTED); const assignment = /([\w.-]+)["']?\s*[:=]\s*/g; - const valuePattern = - /^(?:\[REDACTED(?:_[A-Z_]+)?\][^,;\n\r&}]*|\[[^\n\r]*|\{[^\n\r]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|[^,;\n\r&}]+)/; let output = ""; let cursor = 0; for (const match of redacted.matchAll(assignment)) { if (match.index < cursor || !isSensitiveLogKey(match[1])) continue; - const start = match.index + match[0].length; - const valueMatch = redacted.slice(start).match(valuePattern); - if (!valueMatch) continue; output += redacted.slice(cursor, match.index) + `${match[1]}=${REDACTED}`; - cursor = start + valueMatch[0].length; + cursor = assignmentValueEnd(redacted, match.index + match[0].length); } return (output + redacted.slice(cursor)) .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, REDACTED) diff --git a/frontend/packages/api-client/test/safe-logging.test.mjs b/frontend/packages/api-client/test/safe-logging.test.mjs index 197fca63..9eeb376c 100644 --- a/frontend/packages/api-client/test/safe-logging.test.mjs +++ b/frontend/packages/api-client/test/safe-logging.test.mjs @@ -91,10 +91,12 @@ test("redacts quoted keys, credential spellings, and nested serialized values", '{"context":{"access_token":"private-value"}}', '{"face_embedding":["private-value", "second-private"]}', '{"private_key":"private-value\\"still-private"}', + "full_name=Doe, Jane", + 'face_embedding=[\n 0.123,\n 0.456\n], "status":"failed"', ]) { assert.doesNotMatch( redactLogText(value), - /private-value|second-private|still-private/, + /private-value|second-private|still-private|Jane|0\.123|0\.456/, ); } }); diff --git a/frontend/scripts/check-safe-logging.mjs b/frontend/scripts/check-safe-logging.mjs index 00b1a230..681d0ab8 100644 --- a/frontend/scripts/check-safe-logging.mjs +++ b/frontend/scripts/check-safe-logging.mjs @@ -24,7 +24,6 @@ const sourceRoots = [ "packages", ].map((path) => resolve(frontendRoot, path)); const extensions = new Set([".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"]); -const consoleMethods = new Set(["debug", "info", "log", "warn", "error", "trace"]); const skipDirectories = new Set([ ".next", ".safe-logging-test", @@ -69,11 +68,10 @@ function isConsoleObject(node) { function isConsoleMethodReference(node) { if (ts.isPropertyAccessExpression(node)) { - return isConsoleObject(node.expression) && consoleMethods.has(node.name.text); + return isConsoleObject(node.expression); } if (ts.isElementAccessExpression(node)) { - const method = propertyName(node.argumentExpression); - return isConsoleObject(node.expression) && method !== null && consoleMethods.has(method); + return isConsoleObject(node.expression); } return false; } @@ -82,8 +80,7 @@ function destructuresConsoleMethod(node) { if (!ts.isVariableDeclaration(node) || !ts.isObjectBindingPattern(node.name)) return false; if (!node.initializer || !isConsoleObject(node.initializer)) return false; return node.name.elements.some((element) => { - const name = propertyName(element.propertyName ?? element.name); - return name !== null && consoleMethods.has(name); + return propertyName(element.propertyName ?? element.name) !== null; }); } diff --git a/frontend/scripts/check-safe-logging.test.mjs b/frontend/scripts/check-safe-logging.test.mjs index d9863b84..bb5ba884 100644 --- a/frontend/scripts/check-safe-logging.test.mjs +++ b/frontend/scripts/check-safe-logging.test.mjs @@ -11,6 +11,10 @@ for (const source of [ 'const { error } = console; error("secret")', 'const { warn: report } = globalThis.console; report("secret")', 'const report = console.error; report("secret")', + 'console.table({ accessToken: "secret" })', + 'console.dir({ credentials: "secret" })', + 'console.assert(false, "secret")', + 'console[method]("secret")', ]) { test(`rejects equivalent console use: ${source}`, () => { assert.equal(containsUnsafeConsoleUse("fixture.ts", source), true); From ce6abb53d553aecd07ac109dd6f87d842a3fcd18 Mon Sep 17 00:00:00 2001 From: quarj0 Date: Sat, 12 Sep 2026 07:08:25 +0000 Subject: [PATCH 40/41] Preserve sanitized exception reports for email handlers --- .../common/test_safe_logging_boundaries.py | 31 +++++++++++++++++++ backend/shared/logging_redaction.py | 10 +++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 7e58ee56..40035452 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -2,10 +2,22 @@ import logging from django.test import SimpleTestCase +from django.utils.log import AdminEmailHandler from common.safe_logging import LOG_FORMAT_ERROR, install_safe_logging +class CapturingAdminEmailHandler(AdminEmailHandler): + def __init__(self): + super().__init__() + self.sent_messages = [] + + def send_mail( + self, subject, message, *args, fail_silently=False, html_message=None, **kwargs + ): + self.sent_messages.append((subject, message, html_message)) + + class SafeLoggingBoundaryTests(SimpleTestCase): @classmethod def setUpClass(cls): @@ -66,6 +78,25 @@ def test_exception_objects_and_sensitive_mapping_keys_are_sanitized(self): self.assertIn("RuntimeError", output) self.assertIn("failed", output) + def test_admin_email_handler_receives_sanitized_exception_context(self): + handler = CapturingAdminEmailHandler() + logger = logging.getLogger("django.request.safe-email-test") + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.ERROR) + self.addCleanup(logger.handlers.clear) + + try: + raise RuntimeError("token=email-secret; status=failed") + except RuntimeError: + logger.exception("request failed") + + self.assertEqual(len(handler.sent_messages), 1) + message = handler.sent_messages[0][1] + self.assertIn("test_admin_email_handler_receives_sanitized_exception_context", message) + self.assertIn("RuntimeError", message) + self.assertNotIn("email-secret", message) + def test_interpolated_structures_and_bytes_are_redacted(self): logger, stream = self._capture("identitycore.structured-arguments-test") logger.info( diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index 696a26f5..bf0060c9 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -13,6 +13,10 @@ MAX_REDACTION_DEPTH = 12 LOG_FORMAT_ERROR = "[LOG_FORMAT_ERROR]" + +class _SanitizedLogException(Exception): + """Exception wrapper consumable by handlers without exposing live frames.""" + _SENSITIVE_KEYS = frozenset( { # Credentials and session material. @@ -334,7 +338,11 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: record.stack_info = redact_text(record.stack_info) if record.exc_info: record.exc_text = _redact_exception(record.exc_info) - record.exc_info = None + record.exc_info = ( + _SanitizedLogException, + _SanitizedLogException(record.exc_text), + None, + ) elif record.exc_text: record.exc_text = redact_text(record.exc_text) return record From dc845c25357e755e78d160400552f2c4c02da08c Mon Sep 17 00:00:00 2001 From: quarj0 Date: Sat, 12 Sep 2026 07:31:30 +0000 Subject: [PATCH 41/41] Preserve sanitized Django request reports --- .../common/test_safe_logging_boundaries.py | 38 ++++++++- backend/shared/logging_redaction.py | 85 ++++++++++++++++++- 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/backend/django/common/test_safe_logging_boundaries.py b/backend/django/common/test_safe_logging_boundaries.py index 40035452..8dd6d1c6 100644 --- a/backend/django/common/test_safe_logging_boundaries.py +++ b/backend/django/common/test_safe_logging_boundaries.py @@ -2,6 +2,7 @@ import logging from django.test import SimpleTestCase +from django.test import RequestFactory, override_settings from django.utils.log import AdminEmailHandler from common.safe_logging import LOG_FORMAT_ERROR, install_safe_logging @@ -93,10 +94,45 @@ def test_admin_email_handler_receives_sanitized_exception_context(self): self.assertEqual(len(handler.sent_messages), 1) message = handler.sent_messages[0][1] - self.assertIn("test_admin_email_handler_receives_sanitized_exception_context", message) + self.assertIn( + "test_admin_email_handler_receives_sanitized_exception_context", message + ) self.assertIn("RuntimeError", message) self.assertNotIn("email-secret", message) + @override_settings(INTERNAL_IPS=["10.0.0.1"]) + def test_admin_email_handler_receives_safe_request_context(self): + handler = CapturingAdminEmailHandler() + logger = logging.getLogger("django.request.safe-request-email-test") + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.ERROR) + self.addCleanup(logger.handlers.clear) + request = RequestFactory().post( + "/api/v1/verifications/", + {"email": "subject@example.test", "token": "request-secret"}, + HTTP_AUTHORIZATION="Bearer auth-secret", + REMOTE_ADDR="10.0.0.1", + ) + request.user = "subject@example.test" + + try: + raise RuntimeError("token=exception-secret") + except RuntimeError: + logger.exception("request failed", extra={"request": request}) + + self.assertEqual(len(handler.sent_messages), 1) + subject, message, html_message = handler.sent_messages[0] + self.assertIn("internal IP", subject) + self.assertIn("/api/v1/verifications/", message) + self.assertIn("POST", message) + self.assertIn("REMOTE_ADDR = '[REDACTED]'", message) + complete_report = message + (html_message or "") + self.assertNotIn("subject@example.test", complete_report) + self.assertNotIn("request-secret", complete_report) + self.assertNotIn("auth-secret", complete_report) + self.assertNotIn("exception-secret", complete_report) + def test_interpolated_structures_and_bytes_are_redacted(self): logger, stream = self._capture("identitycore.structured-arguments-test") logger.info( diff --git a/backend/shared/logging_redaction.py b/backend/shared/logging_redaction.py index bf0060c9..787c6493 100644 --- a/backend/shared/logging_redaction.py +++ b/backend/shared/logging_redaction.py @@ -4,6 +4,7 @@ import logging import re import traceback +from copy import copy from collections.abc import Mapping from threading import Lock from typing import Any @@ -17,6 +18,18 @@ class _SanitizedLogException(Exception): """Exception wrapper consumable by handlers without exposing live frames.""" + +class _SanitizedRequestMeta(dict): + def __init__(self, *args, classification_ip: str, **kwargs): + super().__init__(*args, **kwargs) + self._classification_ip = classification_ip + + def get(self, key, default=None): + if key == "REMOTE_ADDR": + return self._classification_ip + return super().get(key, default) + + _SENSITIVE_KEYS = frozenset( { # Credentials and session material. @@ -136,9 +149,7 @@ class _SanitizedLogException(Exception): _IPV4_RE = re.compile(r"(?[\w.-]+)[\"']?\s*[:=]\s*") -_NEXT_ASSIGNMENT_BOUNDARY_RE = re.compile( - r"[;&\n\r](?=\s*[\"']?[\w.-]+[\"']?\s*[:=])" -) +_NEXT_ASSIGNMENT_BOUNDARY_RE = re.compile(r"[;&\n\r](?=\s*[\"']?[\w.-]+[\"']?\s*[:=])") _AWS_ACCESS_KEY_RE = re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b") _STANDARD_LOG_RECORD_ATTRS = frozenset( { @@ -290,6 +301,65 @@ def _redact_exception( return redact_text(rendered) +def _sanitize_django_request(value: Any) -> Any: + """Return a shallow request copy that Django's error reporter can inspect safely.""" + if not ( + value.__class__.__module__.startswith("django.") + and hasattr(value, "META") + and hasattr(value, "method") + and hasattr(value, "path") + ): + return None + + sanitized = copy(value) + safe_meta_keys = { + "PATH_INFO", + "REQUEST_METHOD", + "SCRIPT_NAME", + "SERVER_NAME", + "SERVER_PORT", + "SERVER_PROTOCOL", + } + remote_addr = value.META.get("REMOTE_ADDR", "") + try: + from django.conf import settings + + internal_ips = set(settings.INTERNAL_IPS) + except Exception: + internal_ips = set() + classification_ip = ( + next(iter(internal_ips)) + if remote_addr in internal_ips and internal_ips + else REDACTED + ) + sanitized.META = _SanitizedRequestMeta( + { + key: redact_text(str(item)) + for key, item in value.META.items() + if key in safe_meta_keys + }, + classification_ip=classification_ip, + ) + sanitized.META["REMOTE_ADDR"] = REDACTED + + for attribute in ("GET", "POST", "FILES"): + original = getattr(value, attribute, None) + if original is None: + continue + safe_values = original.copy() + if hasattr(safe_values, "setlist"): + for key in safe_values: + safe_values.setlist(key, [REDACTED]) + else: + safe_values = {str(key): REDACTED for key in safe_values} + setattr(sanitized, f"_{attribute.lower()}", safe_values) + + sanitized.COOKIES = {str(key): REDACTED for key in value.COOKIES} + sanitized.user = REDACTED + sanitized._body = b"" + return sanitized + + def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: """Sanitize rendered messages, structured extras, stack text, and exceptions.""" if ( @@ -332,7 +402,14 @@ def sanitize_log_record(record: logging.LogRecord) -> logging.LogRecord: for field, value in list(record.__dict__.items()): if field in _STANDARD_LOG_RECORD_ATTRS: continue - record.__dict__[field] = redact_value(value, key=field) + sanitized_request = ( + _sanitize_django_request(value) if field == "request" else None + ) + record.__dict__[field] = ( + sanitized_request + if sanitized_request is not None + else redact_value(value, key=field) + ) if record.stack_info: record.stack_info = redact_text(record.stack_info)