From 20d0fa673d45b5d44c8aa4b0d6d94a67bda6f345 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:27:58 -0400 Subject: [PATCH 01/47] Add durable filing draft models --- efile_app/efile/models.py | 172 +++++++++++++++++++++++++++++++++++++- 1 file changed, 169 insertions(+), 3 deletions(-) diff --git a/efile_app/efile/models.py b/efile_app/efile/models.py index 11a6518..873f171 100644 --- a/efile_app/efile/models.py +++ b/efile_app/efile/models.py @@ -1,6 +1,8 @@ # models.py - Optional extension to store additional user information +from django.conf import settings from django.contrib.auth.models import AbstractUser from django.db import models +from django.utils import timezone class UserProfile(AbstractUser): @@ -28,6 +30,170 @@ class Meta: verbose_name_plural = "User Profiles" -# Don't forget to run migrations if you add this model: -# python manage.py makemigrations -# python manage.py migrate --run-syncdb +class FilingDraft(models.Model): + """Durable aggregate for a single in-progress or submitted court filing.""" + + class Status(models.TextChoices): + DRAFT = "draft", "Draft" + SUBMITTING = "submitting", "Submitting" + SUBMITTED = "submitted", "Submitted" + ERROR = "error", "Error" + ABANDONED = "abandoned", "Abandoned" + + class WorkflowStep(models.TextChoices): + OPTIONS = "options", "Options" + UPLOAD_FIRST = "upload_first", "Upload lead document" + CASE_INFORMATION = "case_information", "Case information" + DOCUMENTS = "documents", "Documents" + PAYMENT = "payment", "Payment" + REVIEW = "review", "Review" + CONFIRMATION = "confirmation", "Confirmation" + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="filing_drafts", + ) + session_key = models.CharField(max_length=80, blank=True, db_index=True) + session_id = models.CharField(max_length=100, blank=True, db_index=True) + jurisdiction = models.CharField(max_length=40, db_index=True) + status = models.CharField(max_length=20, choices=Status.choices, default=Status.DRAFT, db_index=True) + current_step = models.CharField(max_length=64, choices=WorkflowStep.choices, default=WorkflowStep.OPTIONS) + + existing_case = models.CharField(max_length=20, blank=True) + court_code = models.CharField(max_length=100, blank=True) + court_name = models.CharField(max_length=255, blank=True) + case_category_code = models.CharField(max_length=100, blank=True) + case_category_name = models.CharField(max_length=255, blank=True) + case_type_code = models.CharField(max_length=100, blank=True) + case_type_name = models.CharField(max_length=255, blank=True) + case_subtype_code = models.CharField(max_length=100, blank=True) + case_subtype_name = models.CharField(max_length=255, blank=True) + filing_type_code = models.CharField(max_length=100, blank=True) + filing_type_name = models.CharField(max_length=255, blank=True) + document_type_code = models.CharField(max_length=100, blank=True) + document_type_name = models.CharField(max_length=255, blank=True) + + previous_case_id = models.CharField(max_length=255, blank=True) + docket_number = models.CharField(max_length=255, blank=True) + + selected_payment_account_id = models.CharField(max_length=255, blank=True) + selected_payment_account_name = models.CharField(max_length=255, blank=True) + + optional_services = models.JSONField(default=list, blank=True) + extracted_guesses = models.JSONField(default=dict, blank=True) + extra_case_data = models.JSONField(default=dict, blank=True) + submission_response = models.JSONField(default=dict, blank=True) + + submitted_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["-updated_at"] + indexes = [ + models.Index(fields=["user", "status"]), + models.Index(fields=["jurisdiction", "status"]), + models.Index(fields=["status", "updated_at"]), + ] + + def __str__(self): + return f"{self.get_status_display()} filing draft #{self.pk} ({self.jurisdiction})" + + def mark_submitted(self, response_data): + self.status = self.Status.SUBMITTED + self.current_step = self.WorkflowStep.CONFIRMATION + self.submission_response = response_data or {} + self.submitted_at = timezone.now() + self.save(update_fields=["status", "current_step", "submission_response", "submitted_at", "updated_at"]) + + def mark_error(self, response_data): + self.status = self.Status.ERROR + self.submission_response = response_data or {} + self.save(update_fields=["status", "submission_response", "updated_at"]) + + +class FilingDocument(models.Model): + """Uploaded document that belongs to a filing draft.""" + + class Role(models.TextChoices): + LEAD = "lead", "Lead document" + SUPPORTING = "supporting", "Supporting document" + + draft = models.ForeignKey(FilingDraft, on_delete=models.CASCADE, related_name="documents") + role = models.CharField(max_length=20, choices=Role.choices) + sort_order = models.PositiveIntegerField(default=0) + + name = models.CharField(max_length=255, blank=True) + original_filename = models.CharField(max_length=255, blank=True) + size = models.PositiveBigIntegerField(blank=True, null=True) + content_type = models.CharField(max_length=255, blank=True) + s3_key = models.CharField(max_length=1024, blank=True) + public_url = models.URLField(max_length=2048, blank=True) + + filing_type_code = models.CharField(max_length=100, blank=True) + filing_type_name = models.CharField(max_length=255, blank=True) + document_type_code = models.CharField(max_length=100, blank=True) + document_type_name = models.CharField(max_length=255, blank=True) + filing_component_code = models.CharField(max_length=100, blank=True) + filing_component_name = models.CharField(max_length=255, blank=True) + + courtesy_copy_email = models.EmailField(blank=True) + metadata = models.JSONField(default=dict, blank=True) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["role", "sort_order", "created_at"] + constraints = [ + models.UniqueConstraint(fields=["draft", "role", "sort_order"], name="unique_document_order_per_draft_role"), + ] + + def __str__(self): + return self.name or f"{self.get_role_display()} for draft #{self.draft_id}" + + +class FilingParty(models.Model): + """Person or organization associated with a filing draft.""" + + draft = models.ForeignKey(FilingDraft, on_delete=models.CASCADE, related_name="parties") + role = models.CharField(max_length=50) + sort_order = models.PositiveIntegerField(default=0) + + party_type = models.CharField(max_length=100, blank=True) + external_party_id = models.CharField(max_length=255, blank=True) + + first_name = models.CharField(max_length=100, blank=True) + middle_name = models.CharField(max_length=100, blank=True) + last_name = models.CharField(max_length=100, blank=True) + suffix = models.CharField(max_length=50, blank=True) + organization_name = models.CharField(max_length=255, blank=True) + + email = models.EmailField(blank=True) + phone = models.CharField(max_length=50, blank=True) + + address_line_1 = models.CharField(max_length=255, blank=True) + address_line_2 = models.CharField(max_length=255, blank=True) + city = models.CharField(max_length=100, blank=True) + state = models.CharField(max_length=50, blank=True) + zip_code = models.CharField(max_length=20, blank=True) + country = models.CharField(max_length=2, default="US", blank=True) + + metadata = models.JSONField(default=dict, blank=True) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["role", "sort_order", "created_at"] + constraints = [ + models.UniqueConstraint(fields=["draft", "role", "sort_order"], name="unique_party_order_per_draft_role"), + ] + verbose_name_plural = "Filing parties" + + def __str__(self): + display_name = " ".join(part for part in [self.first_name, self.middle_name, self.last_name] if part) + return display_name or self.organization_name or f"{self.role} for draft #{self.draft_id}" From c28dc88f92a52d5f2a96315626ec2346078f01d7 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:28:14 -0400 Subject: [PATCH 02/47] Register filing draft models in admin --- efile_app/efile/admin.py | 87 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 efile_app/efile/admin.py diff --git a/efile_app/efile/admin.py b/efile_app/efile/admin.py new file mode 100644 index 0000000..9574799 --- /dev/null +++ b/efile_app/efile/admin.py @@ -0,0 +1,87 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin + +from .models import FilingDocument, FilingDraft, FilingParty, UserProfile + + +class FilingDocumentInline(admin.TabularInline): + model = FilingDocument + extra = 0 + fields = ( + "role", + "sort_order", + "name", + "filing_type_code", + "document_type_code", + "filing_component_code", + "public_url", + ) + readonly_fields = ("created_at", "updated_at") + + +class FilingPartyInline(admin.TabularInline): + model = FilingParty + extra = 0 + fields = ("role", "sort_order", "party_type", "first_name", "last_name", "email", "phone") + readonly_fields = ("created_at", "updated_at") + + +@admin.register(UserProfile) +class UserProfileAdmin(UserAdmin): + fieldsets = UserAdmin.fieldsets + ( + ( + "eFile profile", + { + "fields": ( + "tyler_jurisdiction", + "tyler_user_id", + "email_updates", + "text_updates", + ) + }, + ), + ) + list_display = ("username", "email", "tyler_jurisdiction", "tyler_user_id", "is_staff") + + +@admin.register(FilingDraft) +class FilingDraftAdmin(admin.ModelAdmin): + inlines = [FilingDocumentInline, FilingPartyInline] + list_display = ( + "id", + "user", + "jurisdiction", + "status", + "current_step", + "court_code", + "case_type_code", + "updated_at", + ) + list_filter = ("jurisdiction", "status", "current_step", "created_at", "updated_at") + search_fields = ( + "id", + "session_id", + "court_code", + "court_name", + "case_type_code", + "case_type_name", + "docket_number", + "previous_case_id", + "user__username", + "user__email", + ) + readonly_fields = ("created_at", "updated_at", "submitted_at") + + +@admin.register(FilingDocument) +class FilingDocumentAdmin(admin.ModelAdmin): + list_display = ("id", "draft", "role", "sort_order", "name", "document_type_code", "updated_at") + list_filter = ("role", "content_type", "created_at", "updated_at") + search_fields = ("name", "original_filename", "s3_key", "public_url", "draft__id") + + +@admin.register(FilingParty) +class FilingPartyAdmin(admin.ModelAdmin): + list_display = ("id", "draft", "role", "party_type", "first_name", "last_name", "email") + list_filter = ("role", "party_type", "created_at", "updated_at") + search_fields = ("first_name", "middle_name", "last_name", "organization_name", "email", "draft__id") From 1da660de1c4c4ef2ae987b02d794bdb71e7cf1f1 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:28:20 -0400 Subject: [PATCH 03/47] Create services package --- efile_app/efile/services/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 efile_app/efile/services/__init__.py diff --git a/efile_app/efile/services/__init__.py b/efile_app/efile/services/__init__.py new file mode 100644 index 0000000..e69de29 From 34ddd2df3ad56463c85307db8c93df9bf39492ef Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:29:16 -0400 Subject: [PATCH 04/47] Add draft service module placeholder --- efile_app/efile/services/drafts.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 efile_app/efile/services/drafts.py diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py new file mode 100644 index 0000000..e01f4f3 --- /dev/null +++ b/efile_app/efile/services/drafts.py @@ -0,0 +1 @@ +"""Durable draft helpers.""" From b924397c22ef1c95981e3cb6ccf376b6297ca892 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:30:17 -0400 Subject: [PATCH 05/47] Add durable draft bridge services --- efile_app/efile/services/drafts.py | 318 ++++++++++++++++++++++++++++- 1 file changed, 317 insertions(+), 1 deletion(-) diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py index e01f4f3..3fa53bc 100644 --- a/efile_app/efile/services/drafts.py +++ b/efile_app/efile/services/drafts.py @@ -1 +1,317 @@ -"""Durable draft helpers.""" +"""Bridge helpers for moving the current session-backed flow to durable drafts.""" + +from __future__ import annotations + +import uuid +from collections.abc import Mapping +from typing import Any + +from django.db import transaction + +from efile.models import FilingDocument, FilingDraft, FilingParty + + +CASE_FIELD_MAPPINGS: dict[str, tuple[str, ...]] = { + "existing_case": ("existing_case",), + "court_code": ("court_code", "court"), + "court_name": ("court_name",), + "case_category_code": ("case_category_code", "case_category"), + "case_category_name": ("case_category_name",), + "case_type_code": ("case_type_code", "case_type"), + "case_type_name": ("case_type_name",), + "case_subtype_code": ("case_subtype_code", "case_subtype"), + "case_subtype_name": ("case_subtype_name",), + "filing_type_code": ("filing_type_code", "filing_type", "filing_type_id"), + "filing_type_name": ("filing_type_name",), + "document_type_code": ("document_type_code", "document_type"), + "document_type_name": ("document_type_name",), + "previous_case_id": ("previous_case_id", "case_tracking_id"), + "docket_number": ("docket_number", "case_docket_id"), + "selected_payment_account_id": ("selected_payment_account_id", "selected_payment_account", "payment_account_id"), + "selected_payment_account_name": ("selected_payment_account_name", "payment_account_name"), +} + + +def first_value(data: Mapping[str, Any], *keys: str) -> Any: + for key in keys: + value = data.get(key) + if value not in (None, ""): + return value + return "" + + +def as_string(value: Any) -> str: + return "" if value in (None, "") else str(value) + + +def authenticated_user(request): + user = getattr(request, "user", None) + return user if getattr(user, "is_authenticated", False) else None + + +def ensure_session_key(request) -> str: + if not getattr(request.session, "session_key", None): + request.session.create() + return request.session.session_key or "" + + +def get_active_draft(request) -> FilingDraft | None: + draft_id = request.session.get("filing_draft_id") + if draft_id: + try: + return FilingDraft.objects.get(pk=draft_id) + except FilingDraft.DoesNotExist: + request.session.pop("filing_draft_id", None) + request.session.modified = True + + user = authenticated_user(request) + if user: + draft = ( + FilingDraft.objects.filter(user=user, status__in=[FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR]) + .order_by("-updated_at") + .first() + ) + if draft: + request.session["filing_draft_id"] = draft.pk + request.session.modified = True + return draft + + session_key = getattr(request.session, "session_key", None) + if session_key: + draft = ( + FilingDraft.objects.filter(session_key=session_key, status__in=[FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR]) + .order_by("-updated_at") + .first() + ) + if draft: + request.session["filing_draft_id"] = draft.pk + request.session.modified = True + return draft + + return None + + +@transaction.atomic +def create_draft(request, jurisdiction: str, *, current_step: str = FilingDraft.WorkflowStep.OPTIONS) -> FilingDraft: + session_key = ensure_session_key(request) + session_id = request.session.get("session_id") or str(uuid.uuid4()) + request.session["session_id"] = session_id + request.session["jurisdiction"] = jurisdiction + + draft = FilingDraft.objects.create( + user=authenticated_user(request), + session_key=session_key, + session_id=session_id, + jurisdiction=jurisdiction, + current_step=str(current_step), + ) + request.session["filing_draft_id"] = draft.pk + request.session.modified = True + return draft + + +@transaction.atomic +def ensure_draft(request, jurisdiction: str | None = None, *, current_step: str | None = None) -> FilingDraft: + draft = get_active_draft(request) + if draft is None: + return create_draft( + request, + jurisdiction or request.session.get("jurisdiction") or "", + current_step=current_step or FilingDraft.WorkflowStep.OPTIONS, + ) + + update_fields = [] + if jurisdiction and draft.jurisdiction != jurisdiction: + draft.jurisdiction = jurisdiction + update_fields.append("jurisdiction") + if current_step and draft.current_step != str(current_step): + draft.current_step = str(current_step) + update_fields.append("current_step") + if update_fields: + update_fields.append("updated_at") + draft.save(update_fields=update_fields) + + request.session["filing_draft_id"] = draft.pk + request.session.modified = True + return draft + + +@transaction.atomic +def update_draft_from_case_data( + draft: FilingDraft, + case_data: Mapping[str, Any] | None, + *, + current_step: str | None = None, +) -> FilingDraft: + data = dict(case_data or {}) + update_fields = [] + + for draft_field, source_keys in CASE_FIELD_MAPPINGS.items(): + value = as_string(first_value(data, *source_keys)) + if getattr(draft, draft_field) != value: + setattr(draft, draft_field, value) + update_fields.append(draft_field) + + optional_services = data.get("optional_services") or [] + if draft.optional_services != optional_services: + draft.optional_services = optional_services + update_fields.append("optional_services") + + if draft.extra_case_data != data: + draft.extra_case_data = data + update_fields.append("extra_case_data") + + if current_step and draft.current_step != str(current_step): + draft.current_step = str(current_step) + update_fields.append("current_step") + + if update_fields: + update_fields.append("updated_at") + draft.save(update_fields=sorted(set(update_fields))) + + sync_parties_from_case_data(draft, data) + return draft + + +@transaction.atomic +def sync_documents_from_upload_data( + draft: FilingDraft, + upload_data: Mapping[str, Any] | None, + *, + current_step: str | None = None, +) -> FilingDraft: + data = dict(upload_data or {}) + files = dict(data.get("files") or {}) + lead_document = files.get("lead") + if lead_document: + upsert_document(draft, FilingDocument.Role.LEAD, lead_document, data, sort_order=0) + + supporting_documents = files.get("supporting") or [] + supporting_configs = data.get("supporting_documents") or [] + kept_orders = [] + for index, document in enumerate(supporting_documents): + config = supporting_configs[index] if index < len(supporting_configs) else {} + upsert_document(draft, FilingDocument.Role.SUPPORTING, document, config, sort_order=index) + kept_orders.append(index) + + if kept_orders: + FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING).exclude(sort_order__in=kept_orders).delete() + else: + FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING).delete() + + guesses = data.get("guesses") or {} + update_fields = [] + if guesses and draft.extracted_guesses != guesses: + draft.extracted_guesses = guesses + update_fields.append("extracted_guesses") + if current_step and draft.current_step != str(current_step): + draft.current_step = str(current_step) + update_fields.append("current_step") + if update_fields: + update_fields.append("updated_at") + draft.save(update_fields=update_fields) + + return draft + + +def sync_parties_from_case_data(draft: FilingDraft, case_data: Mapping[str, Any]) -> None: + petitioner = { + "party_type": as_string(first_value(case_data, "petitioner_party_type", "party_type", "determined_party_type")), + "first_name": as_string(first_value(case_data, "petitioner_first_name", "first_name")), + "last_name": as_string(first_value(case_data, "petitioner_last_name", "last_name")), + "email": as_string(first_value(case_data, "petitioner_email", "email")), + "phone": as_string(first_value(case_data, "petitioner_phone", "phone")), + "address_line_1": as_string(first_value(case_data, "petitioner_address", "address", "address_line_1")), + "metadata": {key: value for key, value in case_data.items() if key.startswith("petitioner_")}, + } + if any(petitioner.get(key) for key in ("party_type", "first_name", "last_name", "email")): + FilingParty.objects.update_or_create(draft=draft, role="petitioner", sort_order=0, defaults=petitioner) + + name_sought = { + "party_type": as_string(first_value(case_data, "new_name_party_type")), + "first_name": as_string(first_value(case_data, "new_first_name")), + "middle_name": as_string(first_value(case_data, "new_middle_name")), + "last_name": as_string(first_value(case_data, "new_last_name")), + "suffix": as_string(first_value(case_data, "new_suffix")), + "metadata": {key: value for key, value in case_data.items() if key.startswith("new_") or key.startswith("reason_")}, + } + if any(name_sought.get(key) for key in ("party_type", "first_name", "last_name")): + FilingParty.objects.update_or_create(draft=draft, role="name_sought", sort_order=0, defaults=name_sought) + + +def upsert_document( + draft: FilingDraft, + role: str, + document: Mapping[str, Any], + config: Mapping[str, Any], + *, + sort_order: int, +) -> FilingDocument: + document_data = dict(document or {}) + config_data = dict(config or {}) + defaults = { + "name": as_string(first_value(document_data, "name", "filename", "original_filename")), + "original_filename": as_string(first_value(document_data, "original_filename", "filename", "name")), + "size": positive_int(first_value(document_data, "size", "file_size")), + "content_type": as_string(first_value(document_data, "content_type", "mime_type", "type")), + "s3_key": as_string(first_value(document_data, "s3_key", "key")), + "public_url": as_string(first_value(document_data, "url", "s3_url", "file_url", "download_url")), + "filing_type_code": as_string(first_value(config_data, "filing_type", "lead_filing_type", "filing_type_code")), + "filing_type_name": as_string(first_value(config_data, "filing_type_name", "lead_filing_type_name")), + "document_type_code": as_string(first_value(config_data, "document_type", "lead_document_type", "document_type_code")), + "document_type_name": as_string(first_value(config_data, "document_type_name", "lead_document_type_name")), + "filing_component_code": as_string(first_value(config_data, "filing_component", "lead_filing_component", "filing_component_code")), + "filing_component_name": as_string(first_value(config_data, "filing_component_name", "lead_filing_component_name")), + "courtesy_copy_email": as_string(first_value(config_data, "cc_email", "lead_cc_email")), + "metadata": {"file": document_data, "config": config_data}, + } + document, _created = FilingDocument.objects.update_or_create( + draft=draft, + role=role, + sort_order=sort_order, + defaults=defaults, + ) + return document + + +def draft_snapshot(draft: FilingDraft | None) -> dict[str, Any] | None: + if draft is None: + return None + return { + "id": draft.pk, + "jurisdiction": draft.jurisdiction, + "status": draft.status, + "current_step": draft.current_step, + "existing_case": draft.existing_case, + "court_code": draft.court_code, + "court_name": draft.court_name, + "case_category_code": draft.case_category_code, + "case_category_name": draft.case_category_name, + "case_type_code": draft.case_type_code, + "case_type_name": draft.case_type_name, + "filing_type_code": draft.filing_type_code, + "filing_type_name": draft.filing_type_name, + "document_type_code": draft.document_type_code, + "document_type_name": draft.document_type_name, + "previous_case_id": draft.previous_case_id, + "docket_number": draft.docket_number, + "selected_payment_account_id": draft.selected_payment_account_id, + "selected_payment_account_name": draft.selected_payment_account_name, + "optional_services": draft.optional_services, + "extracted_guesses": draft.extracted_guesses, + "document_count": draft.documents.count(), + "party_count": draft.parties.count(), + "created_at": draft.created_at.isoformat() if draft.created_at else None, + "updated_at": draft.updated_at.isoformat() if draft.updated_at else None, + "submitted_at": draft.submitted_at.isoformat() if draft.submitted_at else None, + } + + +def positive_int(value: Any) -> int | None: + if value in (None, ""): + return None + try: + value = int(value) + except (TypeError, ValueError): + return None + return value if value >= 0 else None From d00aa825d9dd570ff81a8091321152b583b8fdd4 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:31:02 -0400 Subject: [PATCH 06/47] Add draft view module placeholder --- efile_app/efile/views/draft_views.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 efile_app/efile/views/draft_views.py diff --git a/efile_app/efile/views/draft_views.py b/efile_app/efile/views/draft_views.py new file mode 100644 index 0000000..c10be84 --- /dev/null +++ b/efile_app/efile/views/draft_views.py @@ -0,0 +1 @@ +"""Draft views.""" From 812e3bd1540043e8ab37340e1cf7963407f86423 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:31:17 -0400 Subject: [PATCH 07/47] Add durable draft views --- efile_app/efile/views/draft_views.py | 45 +++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/efile_app/efile/views/draft_views.py b/efile_app/efile/views/draft_views.py index c10be84..c8ff550 100644 --- a/efile_app/efile/views/draft_views.py +++ b/efile_app/efile/views/draft_views.py @@ -1 +1,44 @@ -"""Draft views.""" +import json +import logging + +from django.http import JsonResponse +from django.views.decorators.http import require_http_methods + +from efile.services.drafts import create_draft, draft_snapshot, get_active_draft +from efile.utils.django_helpers import flush_cache_stay_logged_in +from efile.workflow import WorkflowStepKey, get_step_url + +logger = logging.getLogger(__name__) + + +@require_http_methods(["POST"]) +def create_draft_view(request, jurisdiction): + """Start a durable draft for the current user/session.""" + + if not request.user.is_authenticated: + return JsonResponse({"success": False, "error": "Authentication required"}, status=401) + + try: + json.loads(request.body or "{}") + except json.JSONDecodeError: + return JsonResponse({"success": False, "error": "Invalid JSON data"}, status=400) + + flush_cache_stay_logged_in(request.session) + draft = create_draft(request, jurisdiction, current_step=WorkflowStepKey.UPLOAD_FIRST) + logger.info("Created durable draft id=%s jurisdiction=%s", draft.pk, jurisdiction) + + return JsonResponse( + { + "success": True, + "data": {"filing_draft": draft_snapshot(draft)}, + "redirect_url": get_step_url(WorkflowStepKey.UPLOAD_FIRST, jurisdiction), + } + ) + + +@require_http_methods(["GET"]) +def get_current_draft_view(request): + """Return the durable draft attached to this session/user.""" + + draft = get_active_draft(request) + return JsonResponse({"success": True, "data": {"filing_draft": draft_snapshot(draft)}}) From 7a81c250d30d2909d887e6a96df84f37449c970f Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:31:23 -0400 Subject: [PATCH 08/47] Expose active durable draft on options page --- efile_app/efile/views/options.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/efile_app/efile/views/options.py b/efile_app/efile/views/options.py index f6d6182..5c7f9c0 100644 --- a/efile_app/efile/views/options.py +++ b/efile_app/efile/views/options.py @@ -1,18 +1,23 @@ from django.shortcuts import render +from django.views.decorators.csrf import ensure_csrf_cookie from efile.api.suffolk_api_views import get_tyler_token +from efile.services.drafts import draft_snapshot, get_active_draft from ..utils.case_data_utils import get_case_data from ..workflow import WorkflowStepKey, get_workflow_context +@ensure_csrf_cookie def efile_options(request, jurisdiction): """Options view that displays saved case data and provides next steps.""" # Get case data from session if request.user.is_authenticated: case_data = get_case_data(request) + active_draft = get_active_draft(request) else: case_data = {} + active_draft = None is_logged_in = request.user.is_authenticated if not get_tyler_token(request, jurisdiction): @@ -22,7 +27,8 @@ def efile_options(request, jurisdiction): context = { "is_logged_in": is_logged_in, "case_data": case_data, - "has_case_data": bool(case_data), + "filing_draft": draft_snapshot(active_draft), + "has_case_data": bool(case_data or active_draft), } context.update(get_workflow_context(WorkflowStepKey.OPTIONS, jurisdiction)) From 51f8c0063812991971d44fc81e754ce5877e60c3 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:31:35 -0400 Subject: [PATCH 09/47] Ensure lead upload is attached to a durable draft --- efile_app/efile/views/upload_first.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/efile_app/efile/views/upload_first.py b/efile_app/efile/views/upload_first.py index 1a63de6..fa538ad 100644 --- a/efile_app/efile/views/upload_first.py +++ b/efile_app/efile/views/upload_first.py @@ -4,6 +4,7 @@ from django.shortcuts import redirect, render from efile.api.suffolk_api_views import get_tyler_token +from efile.services.drafts import draft_snapshot, ensure_draft from ..utils.case_data_utils import ( get_case_classification, @@ -38,6 +39,8 @@ def efile_upload_first(request, jurisdiction): request.session["jurisdiction"] = jurisdiction request.session.modified = True + filing_draft = ensure_draft(request, jurisdiction, current_step=WorkflowStepKey.UPLOAD_FIRST) + # Could visit here from a back button press, so use upload data if any upload_data = get_upload_data(request) @@ -52,6 +55,7 @@ def efile_upload_first(request, jurisdiction): context = { "is_logged_in": is_logged_in, "upload_data": upload_data, + "filing_draft": draft_snapshot(filing_draft), "petitioner_info": petitioner_info, "name_sought_info": name_sought_info, "case_classification": case_classification, From 79308a2f04782fd5ad7ea6ff5f6dba2469e7cb47 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:31:59 -0400 Subject: [PATCH 10/47] Expose durable draft in case data API --- efile_app/efile/views/api_views.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/efile_app/efile/views/api_views.py b/efile_app/efile/views/api_views.py index 4e17ccc..800ef2e 100644 --- a/efile_app/efile/views/api_views.py +++ b/efile_app/efile/views/api_views.py @@ -8,6 +8,8 @@ from django.views import View from django.views.decorators.csrf import ensure_csrf_cookie +from efile.services.drafts import draft_snapshot, get_active_draft, update_draft_from_case_data + logger = logging.getLogger(__name__) @@ -23,9 +25,13 @@ def get(self, request): if value: data[param_to_check] = value + draft = get_active_draft(request) + if draft: + data["filing_draft"] = draft_snapshot(draft) + return JsonResponse({"success": True, "data": data}) except Exception: - logger.exception("Error retrieving case data: {e}") + logger.exception("Error retrieving case data") return JsonResponse({"success": False, "error": "Server error occurred"}, status=500) @@ -75,6 +81,10 @@ def post(self, request): request.session["case_data"] = case_data request.session.modified = True + draft = get_active_draft(request) + if draft: + update_draft_from_case_data(draft, case_data) + logger.debug(f"Saved case data to session: {case_data}") return JsonResponse({"success": True, "message": "Case data saved successfully"}) @@ -82,7 +92,7 @@ def post(self, request): except json.JSONDecodeError: return JsonResponse({"success": False, "error": "Invalid JSON data"}, status=400) except Exception: - logger.exception("Error saving case data: {e}") + logger.exception("Error saving case data") return JsonResponse({"success": False, "error": "Server error occurred"}, status=500) From 636f434652930faa55e20d321615d520809b52d9 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:32:34 -0400 Subject: [PATCH 11/47] Add durable draft routes --- efile_app/efile/urls.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/efile_app/efile/urls.py b/efile_app/efile/urls.py index 4748e21..225340c 100644 --- a/efile_app/efile/urls.py +++ b/efile_app/efile/urls.py @@ -5,6 +5,7 @@ from .views.api_views import get_case_data_api, get_filing_components from .views.choose_jurisdiction import choose_jurisdiction from .views.confirmation import filing_confirmation +from .views.draft_views import create_draft_view, get_current_draft_view from .views.expert_form import efile_expert_form from .views.filing_statuses import filing_statuses from .views.login import efile_login, efile_logout, efile_password_reset @@ -45,6 +46,7 @@ def jurisdiction_homepage(request, jurisdiction): path("jurisdiction//register/", efile_register, name="efile_register"), path("jurisdiction//password_reset/", efile_password_reset, name="efile_password_reset"), path("jurisdiction//options/", efile_options, name="efile_options"), + path("jurisdiction//drafts/", create_draft_view, name="create_draft"), path("jurisdiction//filing_statuses/", filing_statuses, name="filing_statuses"), path("jurisdiction//expert_form/", efile_expert_form, name="expert_form"), path("jurisdiction//upload_first/", efile_upload_first, name="upload_first"), @@ -55,6 +57,7 @@ def jurisdiction_homepage(request, jurisdiction): # Session API endpoints path("api/get-case-data/", get_case_data_api, name="get_case_data_api"), path("api/get-filing-components/", get_filing_components, name="get_filing_components"), + path("api/draft/", get_current_draft_view, name="get_current_draft"), path("api/save-case-data/", api_save_case_data, name="save_case_data_api"), path("api/save-upload-data/", save_upload_data_to_session, name="save_upload_data_to_session"), path("api/save-upload-data-first/", save_upload_first_data, name="save_upload_data_to_session"), From 30f7a1a267bf45e3fe5772233d7818c7b9d0a220 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:32:59 -0400 Subject: [PATCH 12/47] Start new drafts through durable draft endpoint --- efile_app/efile/templates/efile/options.html | 25 ++++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/efile_app/efile/templates/efile/options.html b/efile_app/efile/templates/efile/options.html index 48cc806..6eed8ea 100644 --- a/efile_app/efile/templates/efile/options.html +++ b/efile_app/efile/templates/efile/options.html @@ -97,10 +97,8 @@

{% translate "View past filings" %}

// Function to save existing_case data and redirect to expert form or case details function goToExpertForm(new_or_existing) { if (new_or_existing === "new") { - // Save the existing_case selection - makeNewCaseData().then(() => { - // For new cases, go directly to expert form with cache clearing and options page flag - window.location.href = `/jurisdiction/{{jurisdiction}}/upload_first/?clear_session=true&from_options=true`; + makeNewDraft().then((result) => { + window.location.href = result.redirect_url || `/jurisdiction/{{jurisdiction}}/upload_first/`; }); } else { // TODO(brycew): go straight to the page where the existing session was, reload everything @@ -121,9 +119,26 @@

{% translate "View past filings" %}

return false; } + async function makeNewDraft() { + try { + sessionStorage.removeItem('session_id'); + const result = await apiUtils.post(`/jurisdiction/{{jurisdiction}}/drafts/`, {}); + if (!result.success) { + throw new Error(result.error || 'Unable to create draft'); + } + return result; + } catch (error) { + console.warn('Error creating draft; falling back to session start:', error); + await makeNewCaseData(); + return { + redirect_url: `/jurisdiction/{{jurisdiction}}/upload_first/?clear_session=true&from_options=true` + }; + } + } + // Function to save existing_case data to the session async function makeNewCaseData() { - // Save to session storage as immediate fallback + // Save to session storage as immediate fallback for the legacy start path let new_uuid = self.crypto.randomUUID(); sessionStorage.setItem('session_id', new_uuid); try { From f6a238c916dda191798a1327e61318f4bc8bdb0b Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:33:07 -0400 Subject: [PATCH 13/47] Create migrations package --- efile_app/efile/migrations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 efile_app/efile/migrations/__init__.py diff --git a/efile_app/efile/migrations/__init__.py b/efile_app/efile/migrations/__init__.py new file mode 100644 index 0000000..e69de29 From 2bff193ef3c952431486170466bc6753525ac785 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:33:40 -0400 Subject: [PATCH 14/47] Add initial migration for durable draft models --- efile_app/efile/migrations/0001_initial.py | 271 +++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 efile_app/efile/migrations/0001_initial.py diff --git a/efile_app/efile/migrations/0001_initial.py b/efile_app/efile/migrations/0001_initial.py new file mode 100644 index 0000000..50f4d5b --- /dev/null +++ b/efile_app/efile/migrations/0001_initial.py @@ -0,0 +1,271 @@ +# Generated by hand for the durable filing draft model foundation. + +import django.contrib.auth.validators +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("auth", "0012_alter_user_first_name_max_length"), + ] + + operations = [ + migrations.CreateModel( + name="UserProfile", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("password", models.CharField(max_length=128, verbose_name="password")), + ("last_login", models.DateTimeField(blank=True, null=True, verbose_name="last login")), + ( + "is_superuser", + models.BooleanField( + default=False, + help_text="Designates that this user has all permissions without explicitly assigning them.", + verbose_name="superuser status", + ), + ), + ( + "username", + models.CharField( + error_messages={"unique": "A user with that username already exists."}, + help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.", + max_length=150, + unique=True, + validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], + verbose_name="username", + ), + ), + ("first_name", models.CharField(blank=True, max_length=150, verbose_name="first name")), + ("last_name", models.CharField(blank=True, max_length=150, verbose_name="last name")), + ("email", models.EmailField(blank=True, max_length=254, verbose_name="email address")), + ( + "is_staff", + models.BooleanField( + default=False, + help_text="Designates whether the user can log into this admin site.", + verbose_name="staff status", + ), + ), + ( + "is_active", + models.BooleanField( + default=True, + help_text=( + "Designates whether this user should be treated as active. " + "Unselect this instead of deleting accounts." + ), + verbose_name="active", + ), + ), + ("date_joined", models.DateTimeField(default=django.utils.timezone.now, verbose_name="date joined")), + ("tyler_jurisdiction", models.CharField(max_length=20)), + ("tyler_user_id", models.CharField(blank=True, max_length=100, null=True)), + ("email_updates", models.BooleanField(default=False)), + ("text_updates", models.BooleanField(default=False)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "groups", + models.ManyToManyField( + blank=True, + help_text=( + "The groups this user belongs to. A user will get all permissions granted to each of " + "their groups." + ), + related_name="user_set", + related_query_name="user", + to="auth.group", + verbose_name="groups", + ), + ), + ( + "user_permissions", + models.ManyToManyField( + blank=True, + help_text="Specific permissions for this user.", + related_name="user_set", + related_query_name="user", + to="auth.permission", + verbose_name="user permissions", + ), + ), + ], + options={ + "verbose_name": "User Profile", + "verbose_name_plural": "User Profiles", + }, + ), + migrations.CreateModel( + name="FilingDraft", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("session_key", models.CharField(blank=True, db_index=True, max_length=80)), + ("session_id", models.CharField(blank=True, db_index=True, max_length=100)), + ("jurisdiction", models.CharField(db_index=True, max_length=40)), + ( + "status", + models.CharField( + choices=[ + ("draft", "Draft"), + ("submitting", "Submitting"), + ("submitted", "Submitted"), + ("error", "Error"), + ("abandoned", "Abandoned"), + ], + db_index=True, + default="draft", + max_length=20, + ), + ), + ( + "current_step", + models.CharField( + choices=[ + ("options", "Options"), + ("upload_first", "Upload lead document"), + ("case_information", "Case information"), + ("documents", "Documents"), + ("payment", "Payment"), + ("review", "Review"), + ("confirmation", "Confirmation"), + ], + default="options", + max_length=64, + ), + ), + ("existing_case", models.CharField(blank=True, max_length=20)), + ("court_code", models.CharField(blank=True, max_length=100)), + ("court_name", models.CharField(blank=True, max_length=255)), + ("case_category_code", models.CharField(blank=True, max_length=100)), + ("case_category_name", models.CharField(blank=True, max_length=255)), + ("case_type_code", models.CharField(blank=True, max_length=100)), + ("case_type_name", models.CharField(blank=True, max_length=255)), + ("case_subtype_code", models.CharField(blank=True, max_length=100)), + ("case_subtype_name", models.CharField(blank=True, max_length=255)), + ("filing_type_code", models.CharField(blank=True, max_length=100)), + ("filing_type_name", models.CharField(blank=True, max_length=255)), + ("document_type_code", models.CharField(blank=True, max_length=100)), + ("document_type_name", models.CharField(blank=True, max_length=255)), + ("previous_case_id", models.CharField(blank=True, max_length=255)), + ("docket_number", models.CharField(blank=True, max_length=255)), + ("selected_payment_account_id", models.CharField(blank=True, max_length=255)), + ("selected_payment_account_name", models.CharField(blank=True, max_length=255)), + ("optional_services", models.JSONField(blank=True, default=list)), + ("extracted_guesses", models.JSONField(blank=True, default=dict)), + ("extra_case_data", models.JSONField(blank=True, default=dict)), + ("submission_response", models.JSONField(blank=True, default=dict)), + ("submitted_at", models.DateTimeField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="filing_drafts", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["-updated_at"], + "indexes": [ + models.Index(fields=["user", "status"], name="efile_filing_user_id_3f2f47_idx"), + models.Index(fields=["jurisdiction", "status"], name="efile_filing_jurisdi_7b6b99_idx"), + models.Index(fields=["status", "updated_at"], name="efile_filing_status_4f8dcf_idx"), + ], + }, + ), + migrations.CreateModel( + name="FilingDocument", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("role", models.CharField(choices=[("lead", "Lead document"), ("supporting", "Supporting document")], max_length=20)), + ("sort_order", models.PositiveIntegerField(default=0)), + ("name", models.CharField(blank=True, max_length=255)), + ("original_filename", models.CharField(blank=True, max_length=255)), + ("size", models.PositiveBigIntegerField(blank=True, null=True)), + ("content_type", models.CharField(blank=True, max_length=255)), + ("s3_key", models.CharField(blank=True, max_length=1024)), + ("public_url", models.URLField(blank=True, max_length=2048)), + ("filing_type_code", models.CharField(blank=True, max_length=100)), + ("filing_type_name", models.CharField(blank=True, max_length=255)), + ("document_type_code", models.CharField(blank=True, max_length=100)), + ("document_type_name", models.CharField(blank=True, max_length=255)), + ("filing_component_code", models.CharField(blank=True, max_length=100)), + ("filing_component_name", models.CharField(blank=True, max_length=255)), + ("courtesy_copy_email", models.EmailField(blank=True, max_length=254)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "draft", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="documents", + to="efile.filingdraft", + ), + ), + ], + options={ + "ordering": ["role", "sort_order", "created_at"], + }, + ), + migrations.CreateModel( + name="FilingParty", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("role", models.CharField(max_length=50)), + ("sort_order", models.PositiveIntegerField(default=0)), + ("party_type", models.CharField(blank=True, max_length=100)), + ("external_party_id", models.CharField(blank=True, max_length=255)), + ("first_name", models.CharField(blank=True, max_length=100)), + ("middle_name", models.CharField(blank=True, max_length=100)), + ("last_name", models.CharField(blank=True, max_length=100)), + ("suffix", models.CharField(blank=True, max_length=50)), + ("organization_name", models.CharField(blank=True, max_length=255)), + ("email", models.EmailField(blank=True, max_length=254)), + ("phone", models.CharField(blank=True, max_length=50)), + ("address_line_1", models.CharField(blank=True, max_length=255)), + ("address_line_2", models.CharField(blank=True, max_length=255)), + ("city", models.CharField(blank=True, max_length=100)), + ("state", models.CharField(blank=True, max_length=50)), + ("zip_code", models.CharField(blank=True, max_length=20)), + ("country", models.CharField(blank=True, default="US", max_length=2)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "draft", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="parties", + to="efile.filingdraft", + ), + ), + ], + options={ + "verbose_name_plural": "Filing parties", + "ordering": ["role", "sort_order", "created_at"], + }, + ), + migrations.AddConstraint( + model_name="filingdocument", + constraint=models.UniqueConstraint( + fields=("draft", "role", "sort_order"), + name="unique_document_order_per_draft_role", + ), + ), + migrations.AddConstraint( + model_name="filingparty", + constraint=models.UniqueConstraint( + fields=("draft", "role", "sort_order"), name="unique_party_order_per_draft_role" + ), + ), + ] From ca5fe5e20dc90920e9385a67a6ff7ba7d6ab4696 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:34:02 -0400 Subject: [PATCH 15/47] Add durable draft model and service tests --- efile_app/efile/tests/test_durable_drafts.py | 142 +++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 efile_app/efile/tests/test_durable_drafts.py diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py new file mode 100644 index 0000000..e97edb9 --- /dev/null +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -0,0 +1,142 @@ +import json + +import pytest +from django.urls import reverse + +from efile.models import FilingDocument, FilingDraft, FilingParty +from efile.services.drafts import draft_snapshot, sync_documents_from_upload_data, update_draft_from_case_data + + +@pytest.mark.django_db +def test_update_draft_from_case_data_normalizes_known_fields(): + draft = FilingDraft.objects.create(jurisdiction="illinois") + case_data = { + "court": "cook:cd", + "court_name": "Cook County Circuit Court", + "case_category": "MR", + "case_category_name": "Miscellaneous Remedy", + "case_type": "Name Change", + "case_type_name": "Change of Name", + "filing_type": "motion", + "filing_type_name": "Motion", + "document_type": "petition", + "document_type_name": "Petition", + "selected_payment_account": "pay-123", + "selected_payment_account_name": "Card ending in 4242", + "optional_services": ["certified_copy"], + "petitioner_first_name": "Ada", + "petitioner_last_name": "Lovelace", + "petitioner_email": "ada@example.com", + "new_first_name": "Augusta Ada", + "new_last_name": "Lovelace", + } + + update_draft_from_case_data(draft, case_data, current_step=FilingDraft.WorkflowStep.CASE_INFORMATION) + draft.refresh_from_db() + + assert draft.current_step == FilingDraft.WorkflowStep.CASE_INFORMATION + assert draft.court_code == "cook:cd" + assert draft.case_category_code == "MR" + assert draft.case_type_code == "Name Change" + assert draft.filing_type_code == "motion" + assert draft.document_type_code == "petition" + assert draft.selected_payment_account_id == "pay-123" + assert draft.optional_services == ["certified_copy"] + assert draft.extra_case_data["petitioner_first_name"] == "Ada" + + petitioner = FilingParty.objects.get(draft=draft, role="petitioner") + assert petitioner.first_name == "Ada" + assert petitioner.last_name == "Lovelace" + assert petitioner.email == "ada@example.com" + + name_sought = FilingParty.objects.get(draft=draft, role="name_sought") + assert name_sought.first_name == "Augusta Ada" + + +@pytest.mark.django_db +def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(): + draft = FilingDraft.objects.create(jurisdiction="illinois") + upload_data = { + "files": { + "lead": { + "name": "petition.pdf", + "url": "https://example.com/petition.pdf", + "s3_key": "drafts/petition.pdf", + "size": 1234, + "content_type": "application/pdf", + }, + "supporting": [ + { + "name": "order.pdf", + "url": "https://example.com/order.pdf", + "size": 4321, + "content_type": "application/pdf", + } + ], + }, + "guesses": {"court": "Cook County"}, + "lead_filing_type": "efile", + "lead_document_type": "petition", + "lead_filing_component": "lead", + "supporting_documents": [ + { + "filing_type": "attachment", + "document_type": "exhibit", + "filing_component": "supporting", + "cc_email": "copy@example.com", + } + ], + } + + sync_documents_from_upload_data(draft, upload_data, current_step=FilingDraft.WorkflowStep.DOCUMENTS) + draft.refresh_from_db() + + assert draft.current_step == FilingDraft.WorkflowStep.DOCUMENTS + assert draft.extracted_guesses == {"court": "Cook County"} + + lead = FilingDocument.objects.get(draft=draft, role=FilingDocument.Role.LEAD) + assert lead.name == "petition.pdf" + assert lead.s3_key == "drafts/petition.pdf" + assert lead.filing_type_code == "efile" + + supporting = FilingDocument.objects.get(draft=draft, role=FilingDocument.Role.SUPPORTING) + assert supporting.name == "order.pdf" + assert supporting.document_type_code == "exhibit" + assert supporting.courtesy_copy_email == "copy@example.com" + + +@pytest.mark.django_db +def test_draft_snapshot_is_json_serializable(): + draft = FilingDraft.objects.create(jurisdiction="illinois", court_code="cook:cd") + + snapshot = draft_snapshot(draft) + + assert snapshot["id"] == draft.pk + assert snapshot["court_code"] == "cook:cd" + json.dumps(snapshot) + + +@pytest.mark.django_db +def test_create_draft_view_creates_durable_draft(client, django_user_model): + user = django_user_model.objects.create_user( + username="testuser", + password="testpass123", + tyler_jurisdiction="illinois", + ) + client.force_login(user) + + response = client.post( + reverse("create_draft", kwargs={"jurisdiction": "illinois"}), + data={}, + content_type="application/json", + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["success"] is True + assert payload["redirect_url"] == reverse("upload_first", kwargs={"jurisdiction": "illinois"}) + + draft = FilingDraft.objects.get(user=user) + assert draft.jurisdiction == "illinois" + assert draft.current_step == FilingDraft.WorkflowStep.UPLOAD_FIRST + assert payload["data"]["filing_draft"]["id"] == draft.pk From a982d253556bf0b16a938caaf0a9cf261dd09ace Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 17:34:12 -0400 Subject: [PATCH 16/47] Document durable draft model migration path --- docs/filing-draft-data-model.md | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/filing-draft-data-model.md diff --git a/docs/filing-draft-data-model.md b/docs/filing-draft-data-model.md new file mode 100644 index 0000000..bc13d38 --- /dev/null +++ b/docs/filing-draft-data-model.md @@ -0,0 +1,34 @@ +# Filing draft data model foundation + +This PR introduces the durable filing aggregate that future filing-flow PRs can migrate toward. + +## New source-of-truth models + +- `FilingDraft`: one filing workflow instance. It owns jurisdiction, status, current workflow step, case classification, existing-case identifiers, selected payment account, extracted guesses, temporary extra case data, and submission response. +- `FilingDocument`: one uploaded lead or supporting document. It owns S3/public URL metadata, file metadata, filing/document/component codes and names, courtesy copy email, and document order. +- `FilingParty`: one party/person associated with the draft. It owns role, party type, contact, name, and address fields. + +The model intentionally keeps `extra_case_data` and document/party `metadata` JSON fields as temporary escape hatches so the UI can move off session blobs incrementally without blocking on a perfect schema. + +## Backwards-compatible bridge + +The existing session-backed flow still works. This PR adds service helpers that can shadow-write the current session shapes into the durable models: + +- `create_draft()` +- `ensure_draft()` +- `update_draft_from_case_data()` +- `sync_documents_from_upload_data()` +- `draft_snapshot()` + +The options/start flow now creates a `FilingDraft` through `POST /jurisdiction//drafts/` and redirects to the workflow's first document-upload step. The old session start path remains as a browser fallback. + +## Future migration sequence + +This is intended as the foundation for follow-up PRs: + +1. Update the remaining case-data save APIs to use `update_draft_from_case_data()`. +2. Update upload APIs to use `sync_documents_from_upload_data()`. +3. Move payment selection into explicit draft fields. +4. Render review from `FilingDraft`, `FilingDocument`, and `FilingParty`. +5. Move final submission to a draft-backed submission service. +6. Remove legacy arbitrary session-merge APIs and localStorage/sessionStorage persistence. From 8c16ad90227481af1128d0d14531bf891d9d5a1d Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 19:08:23 -0400 Subject: [PATCH 17/47] Refactor PR, add clear django db migration --- docs/filing-draft-data-model.md | 26 +- efile_app/efile/admin.py | 1 - efile_app/efile/migrations/0001_initial.py | 180 +--------- .../efile/migrations/0002_filing_drafts.py | 185 +++++++++++ efile_app/efile/models.py | 33 +- efile_app/efile/services/current_drafts.py | 105 ++++++ efile_app/efile/services/drafts.py | 310 +++--------------- .../efile/services/legacy_draft_bridge.py | 263 +++++++++++++++ efile_app/efile/tests/test_durable_drafts.py | 142 +++++++- efile_app/efile/views/api_views.py | 10 +- efile_app/efile/views/draft_views.py | 14 +- efile_app/efile/views/options.py | 5 +- efile_app/efile/views/session_api.py | 11 + efile_app/efile/views/upload_first.py | 5 +- efile_app/efile/workflow.py | 6 + fly.toml | 2 +- 16 files changed, 812 insertions(+), 486 deletions(-) create mode 100644 efile_app/efile/migrations/0002_filing_drafts.py create mode 100644 efile_app/efile/services/current_drafts.py create mode 100644 efile_app/efile/services/legacy_draft_bridge.py diff --git a/docs/filing-draft-data-model.md b/docs/filing-draft-data-model.md index bc13d38..e22681f 100644 --- a/docs/filing-draft-data-model.md +++ b/docs/filing-draft-data-model.md @@ -10,25 +10,29 @@ This PR introduces the durable filing aggregate that future filing-flow PRs can The model intentionally keeps `extra_case_data` and document/party `metadata` JSON fields as temporary escape hatches so the UI can move off session blobs incrementally without blocking on a perfect schema. -## Backwards-compatible bridge +## Service boundaries -The existing session-backed flow still works. This PR adds service helpers that can shadow-write the current session shapes into the durable models: +The durable draft code is split into three layers: -- `create_draft()` -- `ensure_draft()` -- `update_draft_from_case_data()` -- `sync_documents_from_upload_data()` -- `draft_snapshot()` +- `services/drafts.py` contains request-independent operations on durable models. +- `services/current_drafts.py` resolves the current browser's draft while enforcing ownership and jurisdiction. The session contains only the current draft ID; it is not a state store. +- `services/legacy_draft_bridge.py` is the temporary compatibility adapter that translates the old `case_data` and `upload_data` blobs. Only legacy session endpoints import it, so it can be removed without changing the durable service. -The options/start flow now creates a `FilingDraft` through `POST /jurisdiction//drafts/` and redirects to the workflow's first document-upload step. The old session start path remains as a browser fallback. +The options/start flow creates a `FilingDraft` through `POST /jurisdiction//drafts/` and redirects to the workflow's first document-upload step. The old session start path remains as a browser fallback. While that flow remains, its actual case and upload save endpoints shadow-write through the compatibility adapter. + +Drafts require an authenticated owner. Every current-draft lookup verifies that owner, active status, and, when known, jurisdiction before returning the object. + +## Migration rollout + +`efile` previously had no migrations even though existing environments may already have the `UserProfile` table. Migration `0001` therefore contains only that baseline model, and migration `0002` creates the new filing tables. Existing environments must run `migrate --fake-initial`; the Fly release command includes that option. Fresh databases apply both migrations normally. ## Future migration sequence This is intended as the foundation for follow-up PRs: -1. Update the remaining case-data save APIs to use `update_draft_from_case_data()`. -2. Update upload APIs to use `sync_documents_from_upload_data()`. +1. Replace legacy case-data endpoints with typed draft updates. +2. Replace legacy upload endpoints with typed document updates. 3. Move payment selection into explicit draft fields. 4. Render review from `FilingDraft`, `FilingDocument`, and `FilingParty`. 5. Move final submission to a draft-backed submission service. -6. Remove legacy arbitrary session-merge APIs and localStorage/sessionStorage persistence. +6. Remove `legacy_draft_bridge.py`, arbitrary session-merge APIs, and browser storage persistence. diff --git a/efile_app/efile/admin.py b/efile_app/efile/admin.py index 9574799..ee98bef 100644 --- a/efile_app/efile/admin.py +++ b/efile_app/efile/admin.py @@ -60,7 +60,6 @@ class FilingDraftAdmin(admin.ModelAdmin): list_filter = ("jurisdiction", "status", "current_step", "created_at", "updated_at") search_fields = ( "id", - "session_id", "court_code", "court_name", "case_type_code", diff --git a/efile_app/efile/migrations/0001_initial.py b/efile_app/efile/migrations/0001_initial.py index 50f4d5b..07c31ed 100644 --- a/efile_app/efile/migrations/0001_initial.py +++ b/efile_app/efile/migrations/0001_initial.py @@ -1,13 +1,18 @@ -# Generated by hand for the durable filing draft model foundation. +# Generated by Django 5.2.5 on 2026-06-29 +import django.contrib.auth.models import django.contrib.auth.validators -import django.db.models.deletion import django.utils.timezone -from django.conf import settings from django.db import migrations, models class Migration(migrations.Migration): + """Baseline the pre-existing custom user table. + + Existing environments should apply this with ``migrate --fake-initial``; + fresh databases create the table normally. + """ + initial = True dependencies = [ @@ -99,173 +104,8 @@ class Migration(migrations.Migration): "verbose_name": "User Profile", "verbose_name_plural": "User Profiles", }, - ), - migrations.CreateModel( - name="FilingDraft", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("session_key", models.CharField(blank=True, db_index=True, max_length=80)), - ("session_id", models.CharField(blank=True, db_index=True, max_length=100)), - ("jurisdiction", models.CharField(db_index=True, max_length=40)), - ( - "status", - models.CharField( - choices=[ - ("draft", "Draft"), - ("submitting", "Submitting"), - ("submitted", "Submitted"), - ("error", "Error"), - ("abandoned", "Abandoned"), - ], - db_index=True, - default="draft", - max_length=20, - ), - ), - ( - "current_step", - models.CharField( - choices=[ - ("options", "Options"), - ("upload_first", "Upload lead document"), - ("case_information", "Case information"), - ("documents", "Documents"), - ("payment", "Payment"), - ("review", "Review"), - ("confirmation", "Confirmation"), - ], - default="options", - max_length=64, - ), - ), - ("existing_case", models.CharField(blank=True, max_length=20)), - ("court_code", models.CharField(blank=True, max_length=100)), - ("court_name", models.CharField(blank=True, max_length=255)), - ("case_category_code", models.CharField(blank=True, max_length=100)), - ("case_category_name", models.CharField(blank=True, max_length=255)), - ("case_type_code", models.CharField(blank=True, max_length=100)), - ("case_type_name", models.CharField(blank=True, max_length=255)), - ("case_subtype_code", models.CharField(blank=True, max_length=100)), - ("case_subtype_name", models.CharField(blank=True, max_length=255)), - ("filing_type_code", models.CharField(blank=True, max_length=100)), - ("filing_type_name", models.CharField(blank=True, max_length=255)), - ("document_type_code", models.CharField(blank=True, max_length=100)), - ("document_type_name", models.CharField(blank=True, max_length=255)), - ("previous_case_id", models.CharField(blank=True, max_length=255)), - ("docket_number", models.CharField(blank=True, max_length=255)), - ("selected_payment_account_id", models.CharField(blank=True, max_length=255)), - ("selected_payment_account_name", models.CharField(blank=True, max_length=255)), - ("optional_services", models.JSONField(blank=True, default=list)), - ("extracted_guesses", models.JSONField(blank=True, default=dict)), - ("extra_case_data", models.JSONField(blank=True, default=dict)), - ("submission_response", models.JSONField(blank=True, default=dict)), - ("submitted_at", models.DateTimeField(blank=True, null=True)), - ("created_at", models.DateTimeField(auto_now_add=True)), - ("updated_at", models.DateTimeField(auto_now=True)), - ( - "user", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="filing_drafts", - to=settings.AUTH_USER_MODEL, - ), - ), - ], - options={ - "ordering": ["-updated_at"], - "indexes": [ - models.Index(fields=["user", "status"], name="efile_filing_user_id_3f2f47_idx"), - models.Index(fields=["jurisdiction", "status"], name="efile_filing_jurisdi_7b6b99_idx"), - models.Index(fields=["status", "updated_at"], name="efile_filing_status_4f8dcf_idx"), - ], - }, - ), - migrations.CreateModel( - name="FilingDocument", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("role", models.CharField(choices=[("lead", "Lead document"), ("supporting", "Supporting document")], max_length=20)), - ("sort_order", models.PositiveIntegerField(default=0)), - ("name", models.CharField(blank=True, max_length=255)), - ("original_filename", models.CharField(blank=True, max_length=255)), - ("size", models.PositiveBigIntegerField(blank=True, null=True)), - ("content_type", models.CharField(blank=True, max_length=255)), - ("s3_key", models.CharField(blank=True, max_length=1024)), - ("public_url", models.URLField(blank=True, max_length=2048)), - ("filing_type_code", models.CharField(blank=True, max_length=100)), - ("filing_type_name", models.CharField(blank=True, max_length=255)), - ("document_type_code", models.CharField(blank=True, max_length=100)), - ("document_type_name", models.CharField(blank=True, max_length=255)), - ("filing_component_code", models.CharField(blank=True, max_length=100)), - ("filing_component_name", models.CharField(blank=True, max_length=255)), - ("courtesy_copy_email", models.EmailField(blank=True, max_length=254)), - ("metadata", models.JSONField(blank=True, default=dict)), - ("created_at", models.DateTimeField(auto_now_add=True)), - ("updated_at", models.DateTimeField(auto_now=True)), - ( - "draft", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="documents", - to="efile.filingdraft", - ), - ), - ], - options={ - "ordering": ["role", "sort_order", "created_at"], - }, - ), - migrations.CreateModel( - name="FilingParty", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("role", models.CharField(max_length=50)), - ("sort_order", models.PositiveIntegerField(default=0)), - ("party_type", models.CharField(blank=True, max_length=100)), - ("external_party_id", models.CharField(blank=True, max_length=255)), - ("first_name", models.CharField(blank=True, max_length=100)), - ("middle_name", models.CharField(blank=True, max_length=100)), - ("last_name", models.CharField(blank=True, max_length=100)), - ("suffix", models.CharField(blank=True, max_length=50)), - ("organization_name", models.CharField(blank=True, max_length=255)), - ("email", models.EmailField(blank=True, max_length=254)), - ("phone", models.CharField(blank=True, max_length=50)), - ("address_line_1", models.CharField(blank=True, max_length=255)), - ("address_line_2", models.CharField(blank=True, max_length=255)), - ("city", models.CharField(blank=True, max_length=100)), - ("state", models.CharField(blank=True, max_length=50)), - ("zip_code", models.CharField(blank=True, max_length=20)), - ("country", models.CharField(blank=True, default="US", max_length=2)), - ("metadata", models.JSONField(blank=True, default=dict)), - ("created_at", models.DateTimeField(auto_now_add=True)), - ("updated_at", models.DateTimeField(auto_now=True)), - ( - "draft", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="parties", - to="efile.filingdraft", - ), - ), + managers=[ + ("objects", django.contrib.auth.models.UserManager()), ], - options={ - "verbose_name_plural": "Filing parties", - "ordering": ["role", "sort_order", "created_at"], - }, - ), - migrations.AddConstraint( - model_name="filingdocument", - constraint=models.UniqueConstraint( - fields=("draft", "role", "sort_order"), - name="unique_document_order_per_draft_role", - ), - ), - migrations.AddConstraint( - model_name="filingparty", - constraint=models.UniqueConstraint( - fields=("draft", "role", "sort_order"), name="unique_party_order_per_draft_role" - ), ), ] diff --git a/efile_app/efile/migrations/0002_filing_drafts.py b/efile_app/efile/migrations/0002_filing_drafts.py new file mode 100644 index 0000000..aec407d --- /dev/null +++ b/efile_app/efile/migrations/0002_filing_drafts.py @@ -0,0 +1,185 @@ +# Generated by Django 5.2.5 on 2026-06-29 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("efile", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="FilingDraft", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("jurisdiction", models.CharField(db_index=True, max_length=40)), + ( + "status", + models.CharField( + choices=[ + ("draft", "Draft"), + ("submitting", "Submitting"), + ("submitted", "Submitted"), + ("error", "Error"), + ("abandoned", "Abandoned"), + ], + db_index=True, + default="draft", + max_length=20, + ), + ), + ( + "current_step", + models.CharField( + choices=[ + ("options", "Options"), + ("upload_first", "Upload lead document"), + ("case_information", "Case information"), + ("documents", "Documents"), + ("payment", "Payment"), + ("review", "Review"), + ("confirmation", "Confirmation"), + ], + default="options", + max_length=64, + ), + ), + ("existing_case", models.CharField(blank=True, max_length=20)), + ("court_code", models.CharField(blank=True, max_length=100)), + ("court_name", models.CharField(blank=True, max_length=255)), + ("case_category_code", models.CharField(blank=True, max_length=100)), + ("case_category_name", models.CharField(blank=True, max_length=255)), + ("case_type_code", models.CharField(blank=True, max_length=100)), + ("case_type_name", models.CharField(blank=True, max_length=255)), + ("case_subtype_code", models.CharField(blank=True, max_length=100)), + ("case_subtype_name", models.CharField(blank=True, max_length=255)), + ("filing_type_code", models.CharField(blank=True, max_length=100)), + ("filing_type_name", models.CharField(blank=True, max_length=255)), + ("document_type_code", models.CharField(blank=True, max_length=100)), + ("document_type_name", models.CharField(blank=True, max_length=255)), + ("previous_case_id", models.CharField(blank=True, max_length=255)), + ("docket_number", models.CharField(blank=True, max_length=255)), + ("selected_payment_account_id", models.CharField(blank=True, max_length=255)), + ("selected_payment_account_name", models.CharField(blank=True, max_length=255)), + ("optional_services", models.JSONField(blank=True, default=list)), + ("extracted_guesses", models.JSONField(blank=True, default=dict)), + ("extra_case_data", models.JSONField(blank=True, default=dict)), + ("submission_response", models.JSONField(blank=True, default=dict)), + ("submitted_at", models.DateTimeField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="filing_drafts", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["-updated_at"], + "indexes": [ + models.Index(fields=["user", "status"], name="draft_user_status_idx"), + models.Index(fields=["jurisdiction", "status"], name="draft_jurisdiction_status_idx"), + models.Index(fields=["status", "updated_at"], name="draft_status_updated_idx"), + ], + }, + ), + migrations.CreateModel( + name="FilingDocument", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "role", + models.CharField( + choices=[("lead", "Lead document"), ("supporting", "Supporting document")], + max_length=20, + ), + ), + ("sort_order", models.PositiveIntegerField(default=0)), + ("name", models.CharField(blank=True, max_length=255)), + ("original_filename", models.CharField(blank=True, max_length=255)), + ("size", models.PositiveBigIntegerField(blank=True, null=True)), + ("content_type", models.CharField(blank=True, max_length=255)), + ("s3_key", models.CharField(blank=True, max_length=1024)), + ("public_url", models.URLField(blank=True, max_length=2048)), + ("filing_type_code", models.CharField(blank=True, max_length=100)), + ("filing_type_name", models.CharField(blank=True, max_length=255)), + ("document_type_code", models.CharField(blank=True, max_length=100)), + ("document_type_name", models.CharField(blank=True, max_length=255)), + ("filing_component_code", models.CharField(blank=True, max_length=100)), + ("filing_component_name", models.CharField(blank=True, max_length=255)), + ("courtesy_copy_email", models.EmailField(blank=True, max_length=254)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "draft", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="documents", + to="efile.filingdraft", + ), + ), + ], + options={ + "ordering": ["role", "sort_order", "created_at"], + }, + ), + migrations.CreateModel( + name="FilingParty", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("role", models.CharField(max_length=50)), + ("sort_order", models.PositiveIntegerField(default=0)), + ("party_type", models.CharField(blank=True, max_length=100)), + ("external_party_id", models.CharField(blank=True, max_length=255)), + ("first_name", models.CharField(blank=True, max_length=100)), + ("middle_name", models.CharField(blank=True, max_length=100)), + ("last_name", models.CharField(blank=True, max_length=100)), + ("suffix", models.CharField(blank=True, max_length=50)), + ("organization_name", models.CharField(blank=True, max_length=255)), + ("email", models.EmailField(blank=True, max_length=254)), + ("phone", models.CharField(blank=True, max_length=50)), + ("address_line_1", models.CharField(blank=True, max_length=255)), + ("address_line_2", models.CharField(blank=True, max_length=255)), + ("city", models.CharField(blank=True, max_length=100)), + ("state", models.CharField(blank=True, max_length=50)), + ("zip_code", models.CharField(blank=True, max_length=20)), + ("country", models.CharField(blank=True, default="US", max_length=2)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "draft", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="parties", + to="efile.filingdraft", + ), + ), + ], + options={ + "verbose_name_plural": "Filing parties", + "ordering": ["role", "sort_order", "created_at"], + }, + ), + migrations.AddConstraint( + model_name="filingdocument", + constraint=models.UniqueConstraint( + fields=("draft", "role", "sort_order"), + name="unique_document_order_per_draft_role", + ), + ), + migrations.AddConstraint( + model_name="filingparty", + constraint=models.UniqueConstraint( + fields=("draft", "role", "sort_order"), + name="unique_party_order_per_draft_role", + ), + ), + ] diff --git a/efile_app/efile/models.py b/efile_app/efile/models.py index 873f171..7e52f7c 100644 --- a/efile_app/efile/models.py +++ b/efile_app/efile/models.py @@ -4,6 +4,8 @@ from django.db import models from django.utils import timezone +from efile.workflow import WorkflowStepKey, get_workflow_step_choices + class UserProfile(AbstractUser): """ @@ -40,27 +42,18 @@ class Status(models.TextChoices): ERROR = "error", "Error" ABANDONED = "abandoned", "Abandoned" - class WorkflowStep(models.TextChoices): - OPTIONS = "options", "Options" - UPLOAD_FIRST = "upload_first", "Upload lead document" - CASE_INFORMATION = "case_information", "Case information" - DOCUMENTS = "documents", "Documents" - PAYMENT = "payment", "Payment" - REVIEW = "review", "Review" - CONFIRMATION = "confirmation", "Confirmation" - user = models.ForeignKey( settings.AUTH_USER_MODEL, - blank=True, - null=True, on_delete=models.CASCADE, related_name="filing_drafts", ) - session_key = models.CharField(max_length=80, blank=True, db_index=True) - session_id = models.CharField(max_length=100, blank=True, db_index=True) jurisdiction = models.CharField(max_length=40, db_index=True) status = models.CharField(max_length=20, choices=Status.choices, default=Status.DRAFT, db_index=True) - current_step = models.CharField(max_length=64, choices=WorkflowStep.choices, default=WorkflowStep.OPTIONS) + current_step = models.CharField( + max_length=64, + choices=get_workflow_step_choices(), + default=WorkflowStepKey.OPTIONS, + ) existing_case = models.CharField(max_length=20, blank=True) court_code = models.CharField(max_length=100, blank=True) @@ -94,9 +87,9 @@ class WorkflowStep(models.TextChoices): class Meta: ordering = ["-updated_at"] indexes = [ - models.Index(fields=["user", "status"]), - models.Index(fields=["jurisdiction", "status"]), - models.Index(fields=["status", "updated_at"]), + models.Index(fields=["user", "status"], name="draft_user_status_idx"), + models.Index(fields=["jurisdiction", "status"], name="draft_jurisdiction_status_idx"), + models.Index(fields=["status", "updated_at"], name="draft_status_updated_idx"), ] def __str__(self): @@ -104,7 +97,7 @@ def __str__(self): def mark_submitted(self, response_data): self.status = self.Status.SUBMITTED - self.current_step = self.WorkflowStep.CONFIRMATION + self.current_step = WorkflowStepKey.CONFIRMATION self.submission_response = response_data or {} self.submitted_at = timezone.now() self.save(update_fields=["status", "current_step", "submission_response", "submitted_at", "updated_at"]) @@ -149,7 +142,9 @@ class Role(models.TextChoices): class Meta: ordering = ["role", "sort_order", "created_at"] constraints = [ - models.UniqueConstraint(fields=["draft", "role", "sort_order"], name="unique_document_order_per_draft_role"), + models.UniqueConstraint( + fields=["draft", "role", "sort_order"], name="unique_document_order_per_draft_role" + ), ] def __str__(self): diff --git a/efile_app/efile/services/current_drafts.py b/efile_app/efile/services/current_drafts.py new file mode 100644 index 0000000..00c84e7 --- /dev/null +++ b/efile_app/efile/services/current_drafts.py @@ -0,0 +1,105 @@ +"""Select the durable filing draft associated with the current request.""" + +from __future__ import annotations + +from django.db import transaction + +from efile.models import FilingDraft +from efile.services.drafts import create_draft, get_active_draft, set_current_step +from efile.workflow import WorkflowStepKey + +CURRENT_DRAFT_SESSION_KEY = "filing_draft_id" + + +def _authenticated_user(request): + user = getattr(request, "user", None) + return user if getattr(user, "is_authenticated", False) else None + + +def attach_current_draft(request, draft: FilingDraft) -> None: + """Remember which durable draft this browser is editing.""" + + request.session[CURRENT_DRAFT_SESSION_KEY] = draft.pk + request.session["jurisdiction"] = draft.jurisdiction + request.session.modified = True + + +def clear_current_draft(request) -> None: + if CURRENT_DRAFT_SESSION_KEY in request.session: + del request.session[CURRENT_DRAFT_SESSION_KEY] + request.session.modified = True + + +def get_current_draft( + request, + *, + jurisdiction: str | None = None, + resume_latest: bool = True, +) -> FilingDraft | None: + """Resolve the current user's draft without trusting a bare session ID. + + The session only stores a pointer. Ownership, active status, and (when + supplied) jurisdiction are enforced on every lookup. + """ + + user = _authenticated_user(request) + if user is None: + clear_current_draft(request) + return None + + draft_id = request.session.get(CURRENT_DRAFT_SESSION_KEY) + if draft_id is not None: + try: + draft_id = int(draft_id) + except (TypeError, ValueError): + clear_current_draft(request) + draft_id = None + + if draft_id is not None: + draft = get_active_draft(user=user, draft_id=draft_id, jurisdiction=jurisdiction) + if draft is not None: + return draft + clear_current_draft(request) + + if not resume_latest: + return None + + draft = get_active_draft(user=user, jurisdiction=jurisdiction) + if draft is not None: + attach_current_draft(request, draft) + return draft + + +@transaction.atomic +def create_current_draft( + request, + jurisdiction: str, + *, + current_step: WorkflowStepKey | str = WorkflowStepKey.OPTIONS, +) -> FilingDraft: + draft = create_draft( + user=_authenticated_user(request), + jurisdiction=jurisdiction, + current_step=current_step, + ) + attach_current_draft(request, draft) + return draft + + +@transaction.atomic +def ensure_current_draft( + request, + jurisdiction: str, + *, + current_step: WorkflowStepKey | str | None = None, +) -> FilingDraft: + draft = get_current_draft(request, jurisdiction=jurisdiction) + if draft is None: + return create_current_draft( + request, + jurisdiction, + current_step=current_step or WorkflowStepKey.OPTIONS, + ) + if current_step is not None: + set_current_step(draft, current_step) + return draft diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py index 3fa53bc..9bb1344 100644 --- a/efile_app/efile/services/drafts.py +++ b/efile_app/efile/services/drafts.py @@ -1,280 +1,80 @@ -"""Bridge helpers for moving the current session-backed flow to durable drafts.""" +"""Operations on the durable filing draft aggregate. + +This module deliberately has no dependency on HTTP requests, sessions, or the +legacy session data shapes. Request/session selection lives in +``current_drafts``; legacy data translation lives in ``legacy_draft_bridge``. +""" from __future__ import annotations -import uuid -from collections.abc import Mapping from typing import Any from django.db import transaction +from django.db.models import QuerySet -from efile.models import FilingDocument, FilingDraft, FilingParty - - -CASE_FIELD_MAPPINGS: dict[str, tuple[str, ...]] = { - "existing_case": ("existing_case",), - "court_code": ("court_code", "court"), - "court_name": ("court_name",), - "case_category_code": ("case_category_code", "case_category"), - "case_category_name": ("case_category_name",), - "case_type_code": ("case_type_code", "case_type"), - "case_type_name": ("case_type_name",), - "case_subtype_code": ("case_subtype_code", "case_subtype"), - "case_subtype_name": ("case_subtype_name",), - "filing_type_code": ("filing_type_code", "filing_type", "filing_type_id"), - "filing_type_name": ("filing_type_name",), - "document_type_code": ("document_type_code", "document_type"), - "document_type_name": ("document_type_name",), - "previous_case_id": ("previous_case_id", "case_tracking_id"), - "docket_number": ("docket_number", "case_docket_id"), - "selected_payment_account_id": ("selected_payment_account_id", "selected_payment_account", "payment_account_id"), - "selected_payment_account_name": ("selected_payment_account_name", "payment_account_name"), -} - - -def first_value(data: Mapping[str, Any], *keys: str) -> Any: - for key in keys: - value = data.get(key) - if value not in (None, ""): - return value - return "" - - -def as_string(value: Any) -> str: - return "" if value in (None, "") else str(value) - - -def authenticated_user(request): - user = getattr(request, "user", None) - return user if getattr(user, "is_authenticated", False) else None - - -def ensure_session_key(request) -> str: - if not getattr(request.session, "session_key", None): - request.session.create() - return request.session.session_key or "" - - -def get_active_draft(request) -> FilingDraft | None: - draft_id = request.session.get("filing_draft_id") - if draft_id: - try: - return FilingDraft.objects.get(pk=draft_id) - except FilingDraft.DoesNotExist: - request.session.pop("filing_draft_id", None) - request.session.modified = True - - user = authenticated_user(request) - if user: - draft = ( - FilingDraft.objects.filter(user=user, status__in=[FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR]) - .order_by("-updated_at") - .first() - ) - if draft: - request.session["filing_draft_id"] = draft.pk - request.session.modified = True - return draft +from efile.models import FilingDraft +from efile.workflow import WorkflowStepKey - session_key = getattr(request.session, "session_key", None) - if session_key: - draft = ( - FilingDraft.objects.filter(session_key=session_key, status__in=[FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR]) - .order_by("-updated_at") - .first() - ) - if draft: - request.session["filing_draft_id"] = draft.pk - request.session.modified = True - return draft +ACTIVE_DRAFT_STATUSES = (FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR) - return None +def active_drafts_for(user, *, jurisdiction: str | None = None) -> QuerySet[FilingDraft]: + """Return active drafts owned by ``user``, newest first.""" -@transaction.atomic -def create_draft(request, jurisdiction: str, *, current_step: str = FilingDraft.WorkflowStep.OPTIONS) -> FilingDraft: - session_key = ensure_session_key(request) - session_id = request.session.get("session_id") or str(uuid.uuid4()) - request.session["session_id"] = session_id - request.session["jurisdiction"] = jurisdiction - - draft = FilingDraft.objects.create( - user=authenticated_user(request), - session_key=session_key, - session_id=session_id, - jurisdiction=jurisdiction, - current_step=str(current_step), - ) - request.session["filing_draft_id"] = draft.pk - request.session.modified = True - return draft - - -@transaction.atomic -def ensure_draft(request, jurisdiction: str | None = None, *, current_step: str | None = None) -> FilingDraft: - draft = get_active_draft(request) - if draft is None: - return create_draft( - request, - jurisdiction or request.session.get("jurisdiction") or "", - current_step=current_step or FilingDraft.WorkflowStep.OPTIONS, - ) - - update_fields = [] - if jurisdiction and draft.jurisdiction != jurisdiction: - draft.jurisdiction = jurisdiction - update_fields.append("jurisdiction") - if current_step and draft.current_step != str(current_step): - draft.current_step = str(current_step) - update_fields.append("current_step") - if update_fields: - update_fields.append("updated_at") - draft.save(update_fields=update_fields) - - request.session["filing_draft_id"] = draft.pk - request.session.modified = True - return draft + drafts = FilingDraft.objects.filter(user=user, status__in=ACTIVE_DRAFT_STATUSES) + if jurisdiction is not None: + drafts = drafts.filter(jurisdiction=jurisdiction) + return drafts.order_by("-updated_at") -@transaction.atomic -def update_draft_from_case_data( - draft: FilingDraft, - case_data: Mapping[str, Any] | None, +def get_active_draft( *, - current_step: str | None = None, -) -> FilingDraft: - data = dict(case_data or {}) - update_fields = [] - - for draft_field, source_keys in CASE_FIELD_MAPPINGS.items(): - value = as_string(first_value(data, *source_keys)) - if getattr(draft, draft_field) != value: - setattr(draft, draft_field, value) - update_fields.append(draft_field) - - optional_services = data.get("optional_services") or [] - if draft.optional_services != optional_services: - draft.optional_services = optional_services - update_fields.append("optional_services") - - if draft.extra_case_data != data: - draft.extra_case_data = data - update_fields.append("extra_case_data") - - if current_step and draft.current_step != str(current_step): - draft.current_step = str(current_step) - update_fields.append("current_step") - - if update_fields: - update_fields.append("updated_at") - draft.save(update_fields=sorted(set(update_fields))) + user, + draft_id: int | str | None = None, + jurisdiction: str | None = None, +) -> FilingDraft | None: + """Get an owned active draft by ID, or the user's most recent draft.""" - sync_parties_from_case_data(draft, data) - return draft + drafts = active_drafts_for(user, jurisdiction=jurisdiction) + if draft_id is not None: + return drafts.filter(pk=draft_id).first() + return drafts.first() @transaction.atomic -def sync_documents_from_upload_data( - draft: FilingDraft, - upload_data: Mapping[str, Any] | None, +def create_draft( *, - current_step: str | None = None, + user, + jurisdiction: str, + current_step: WorkflowStepKey | str = WorkflowStepKey.OPTIONS, ) -> FilingDraft: - data = dict(upload_data or {}) - files = dict(data.get("files") or {}) - lead_document = files.get("lead") - if lead_document: - upsert_document(draft, FilingDocument.Role.LEAD, lead_document, data, sort_order=0) - - supporting_documents = files.get("supporting") or [] - supporting_configs = data.get("supporting_documents") or [] - kept_orders = [] - for index, document in enumerate(supporting_documents): - config = supporting_configs[index] if index < len(supporting_configs) else {} - upsert_document(draft, FilingDocument.Role.SUPPORTING, document, config, sort_order=index) - kept_orders.append(index) + """Create a durable draft owned by an authenticated user.""" - if kept_orders: - FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING).exclude(sort_order__in=kept_orders).delete() - else: - FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING).delete() + if not getattr(user, "is_authenticated", False): + raise ValueError("A filing draft must have an authenticated owner") + if not jurisdiction: + raise ValueError("A filing draft must have a jurisdiction") - guesses = data.get("guesses") or {} - update_fields = [] - if guesses and draft.extracted_guesses != guesses: - draft.extracted_guesses = guesses - update_fields.append("extracted_guesses") - if current_step and draft.current_step != str(current_step): - draft.current_step = str(current_step) - update_fields.append("current_step") - if update_fields: - update_fields.append("updated_at") - draft.save(update_fields=update_fields) - - return draft - - -def sync_parties_from_case_data(draft: FilingDraft, case_data: Mapping[str, Any]) -> None: - petitioner = { - "party_type": as_string(first_value(case_data, "petitioner_party_type", "party_type", "determined_party_type")), - "first_name": as_string(first_value(case_data, "petitioner_first_name", "first_name")), - "last_name": as_string(first_value(case_data, "petitioner_last_name", "last_name")), - "email": as_string(first_value(case_data, "petitioner_email", "email")), - "phone": as_string(first_value(case_data, "petitioner_phone", "phone")), - "address_line_1": as_string(first_value(case_data, "petitioner_address", "address", "address_line_1")), - "metadata": {key: value for key, value in case_data.items() if key.startswith("petitioner_")}, - } - if any(petitioner.get(key) for key in ("party_type", "first_name", "last_name", "email")): - FilingParty.objects.update_or_create(draft=draft, role="petitioner", sort_order=0, defaults=petitioner) + return FilingDraft.objects.create( + user=user, + jurisdiction=jurisdiction, + current_step=str(current_step), + ) - name_sought = { - "party_type": as_string(first_value(case_data, "new_name_party_type")), - "first_name": as_string(first_value(case_data, "new_first_name")), - "middle_name": as_string(first_value(case_data, "new_middle_name")), - "last_name": as_string(first_value(case_data, "new_last_name")), - "suffix": as_string(first_value(case_data, "new_suffix")), - "metadata": {key: value for key, value in case_data.items() if key.startswith("new_") or key.startswith("reason_")}, - } - if any(name_sought.get(key) for key in ("party_type", "first_name", "last_name")): - FilingParty.objects.update_or_create(draft=draft, role="name_sought", sort_order=0, defaults=name_sought) +def set_current_step(draft: FilingDraft, current_step: WorkflowStepKey | str) -> FilingDraft: + """Advance or rewind a draft's current UI step when it changed.""" -def upsert_document( - draft: FilingDraft, - role: str, - document: Mapping[str, Any], - config: Mapping[str, Any], - *, - sort_order: int, -) -> FilingDocument: - document_data = dict(document or {}) - config_data = dict(config or {}) - defaults = { - "name": as_string(first_value(document_data, "name", "filename", "original_filename")), - "original_filename": as_string(first_value(document_data, "original_filename", "filename", "name")), - "size": positive_int(first_value(document_data, "size", "file_size")), - "content_type": as_string(first_value(document_data, "content_type", "mime_type", "type")), - "s3_key": as_string(first_value(document_data, "s3_key", "key")), - "public_url": as_string(first_value(document_data, "url", "s3_url", "file_url", "download_url")), - "filing_type_code": as_string(first_value(config_data, "filing_type", "lead_filing_type", "filing_type_code")), - "filing_type_name": as_string(first_value(config_data, "filing_type_name", "lead_filing_type_name")), - "document_type_code": as_string(first_value(config_data, "document_type", "lead_document_type", "document_type_code")), - "document_type_name": as_string(first_value(config_data, "document_type_name", "lead_document_type_name")), - "filing_component_code": as_string(first_value(config_data, "filing_component", "lead_filing_component", "filing_component_code")), - "filing_component_name": as_string(first_value(config_data, "filing_component_name", "lead_filing_component_name")), - "courtesy_copy_email": as_string(first_value(config_data, "cc_email", "lead_cc_email")), - "metadata": {"file": document_data, "config": config_data}, - } - document, _created = FilingDocument.objects.update_or_create( - draft=draft, - role=role, - sort_order=sort_order, - defaults=defaults, - ) - return document + step = str(current_step) + if draft.current_step != step: + draft.current_step = step + draft.save(update_fields=["current_step", "updated_at"]) + return draft def draft_snapshot(draft: FilingDraft | None) -> dict[str, Any] | None: + """Return the stable, JSON-safe representation exposed to the UI.""" + if draft is None: return None return { @@ -289,6 +89,8 @@ def draft_snapshot(draft: FilingDraft | None) -> dict[str, Any] | None: "case_category_name": draft.case_category_name, "case_type_code": draft.case_type_code, "case_type_name": draft.case_type_name, + "case_subtype_code": draft.case_subtype_code, + "case_subtype_name": draft.case_subtype_name, "filing_type_code": draft.filing_type_code, "filing_type_name": draft.filing_type_name, "document_type_code": draft.document_type_code, @@ -305,13 +107,3 @@ def draft_snapshot(draft: FilingDraft | None) -> dict[str, Any] | None: "updated_at": draft.updated_at.isoformat() if draft.updated_at else None, "submitted_at": draft.submitted_at.isoformat() if draft.submitted_at else None, } - - -def positive_int(value: Any) -> int | None: - if value in (None, ""): - return None - try: - value = int(value) - except (TypeError, ValueError): - return None - return value if value >= 0 else None diff --git a/efile_app/efile/services/legacy_draft_bridge.py b/efile_app/efile/services/legacy_draft_bridge.py new file mode 100644 index 0000000..1ef1783 --- /dev/null +++ b/efile_app/efile/services/legacy_draft_bridge.py @@ -0,0 +1,263 @@ +"""Translate legacy session blobs into the durable filing draft aggregate. + +Only the session-backed endpoints should import this module. It can be removed +once those endpoints write typed draft fields directly. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from django.db import transaction + +from efile.models import FilingDocument, FilingDraft, FilingParty +from efile.services.current_drafts import get_current_draft +from efile.workflow import WorkflowStepKey + +CASE_FIELD_MAPPINGS: dict[str, tuple[str, ...]] = { + "existing_case": ("existing_case",), + "court_code": ("court_code", "court"), + "court_name": ("court_name",), + "case_category_code": ("case_category_code", "case_category"), + "case_category_name": ("case_category_name",), + "case_type_code": ("case_type_code", "case_type"), + "case_type_name": ("case_type_name",), + "case_subtype_code": ("case_subtype_code", "case_subtype"), + "case_subtype_name": ("case_subtype_name",), + "filing_type_code": ("filing_type_code", "filing_type", "filing_type_id"), + "filing_type_name": ("filing_type_name",), + "document_type_code": ("document_type_code", "document_type"), + "document_type_name": ("document_type_name",), + "previous_case_id": ("previous_case_id", "case_tracking_id"), + "docket_number": ("docket_number", "case_docket_id"), + "selected_payment_account_id": ( + "selected_payment_account_id", + "selected_payment_account", + "payment_account_id", + ), + "selected_payment_account_name": ("selected_payment_account_name", "payment_account_name"), +} + +_MISSING = object() + + +def _first_present(data: Mapping[str, Any], *keys: str) -> Any: + for key in keys: + if key in data: + return data[key] + return _MISSING + + +def _first_value(data: Mapping[str, Any], *keys: str) -> Any: + for key in keys: + value = data.get(key) + if value not in (None, ""): + return value + return "" + + +def _as_string(value: Any) -> str: + return "" if value in (None, "") else str(value) + + +@transaction.atomic +def update_draft_from_case_data( + draft: FilingDraft, + case_data: Mapping[str, Any] | None, + *, + current_step: WorkflowStepKey | str | None = None, +) -> FilingDraft: + """Mirror one complete legacy ``case_data`` blob into ``draft``.""" + + data = dict(case_data or {}) + update_fields = [] + + for draft_field, source_keys in CASE_FIELD_MAPPINGS.items(): + source_value = _first_present(data, *source_keys) + if source_value is _MISSING: + continue + value = _as_string(source_value) + if getattr(draft, draft_field) != value: + setattr(draft, draft_field, value) + update_fields.append(draft_field) + + if "optional_services" in data: + optional_services = data.get("optional_services") or [] + if draft.optional_services != optional_services: + draft.optional_services = optional_services + update_fields.append("optional_services") + + if draft.extra_case_data != data: + draft.extra_case_data = data + update_fields.append("extra_case_data") + + if current_step is not None and draft.current_step != str(current_step): + draft.current_step = str(current_step) + update_fields.append("current_step") + + if update_fields: + draft.save(update_fields=sorted({*update_fields, "updated_at"})) + + _sync_parties_from_case_data(draft, data) + return draft + + +@transaction.atomic +def sync_documents_from_upload_data( + draft: FilingDraft, + upload_data: Mapping[str, Any] | None, + *, + current_step: WorkflowStepKey | str | None = None, +) -> FilingDraft: + """Mirror one complete legacy ``upload_data`` blob into ``draft``.""" + + data = dict(upload_data or {}) + files = dict(data.get("files") or {}) + lead_document = files.get("lead") + if lead_document: + _upsert_document(draft, FilingDocument.Role.LEAD, lead_document, data, sort_order=0) + else: + FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.LEAD).delete() + + supporting_documents = files.get("supporting") or [] + supporting_configs = data.get("supporting_documents") or [] + kept_orders = [] + for index, document in enumerate(supporting_documents): + config = supporting_configs[index] if index < len(supporting_configs) else {} + _upsert_document(draft, FilingDocument.Role.SUPPORTING, document, config, sort_order=index) + kept_orders.append(index) + + supporting = FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING) + if kept_orders: + supporting.exclude(sort_order__in=kept_orders).delete() + else: + supporting.delete() + + guesses = data.get("guesses") or {} + update_fields = [] + if draft.extracted_guesses != guesses: + draft.extracted_guesses = guesses + update_fields.append("extracted_guesses") + if current_step is not None and draft.current_step != str(current_step): + draft.current_step = str(current_step) + update_fields.append("current_step") + draft.save(update_fields=[*update_fields, "updated_at"]) + + return draft + + +def sync_current_draft_case_data(request, case_data: Mapping[str, Any]) -> FilingDraft | None: + """Compatibility hook for a session endpoint that just saved case data.""" + + jurisdiction = _as_string( + _first_value(case_data, "jurisdiction_id", "jurisdiction") or request.session.get("jurisdiction") + ) + draft = get_current_draft(request, jurisdiction=jurisdiction or None, resume_latest=False) + if draft is not None: + update_draft_from_case_data(draft, case_data) + return draft + + +def sync_current_draft_upload_data(request, upload_data: Mapping[str, Any]) -> FilingDraft | None: + """Compatibility hook for a session endpoint that just saved upload data.""" + + jurisdiction = _as_string(request.session.get("jurisdiction")) + draft = get_current_draft(request, jurisdiction=jurisdiction or None, resume_latest=False) + if draft is not None: + sync_documents_from_upload_data(draft, upload_data) + return draft + + +def _sync_parties_from_case_data(draft: FilingDraft, case_data: Mapping[str, Any]) -> None: + petitioner = { + "party_type": _as_string( + _first_value(case_data, "petitioner_party_type", "party_type", "determined_party_type") + ), + "first_name": _as_string(_first_value(case_data, "petitioner_first_name", "first_name")), + "middle_name": _as_string(_first_value(case_data, "petitioner_middle_name", "middle_name")), + "last_name": _as_string(_first_value(case_data, "petitioner_last_name", "last_name")), + "suffix": _as_string(_first_value(case_data, "petitioner_suffix", "suffix")), + "email": _as_string(_first_value(case_data, "petitioner_email", "email")), + "phone": _as_string(_first_value(case_data, "petitioner_phone", "phone")), + "address_line_1": _as_string(_first_value(case_data, "petitioner_address", "address", "address_line_1")), + "address_line_2": _as_string(_first_value(case_data, "petitioner_address_line_2", "address_line2")), + "city": _as_string(_first_value(case_data, "petitioner_city", "city")), + "state": _as_string(_first_value(case_data, "petitioner_state", "state")), + "zip_code": _as_string(_first_value(case_data, "petitioner_zip", "zip", "zip_code")), + "metadata": {key: value for key, value in case_data.items() if key.startswith("petitioner_")}, + } + _upsert_or_delete_party(draft, "petitioner", petitioner) + + name_sought = { + "party_type": _as_string(_first_value(case_data, "new_name_party_type")), + "first_name": _as_string(_first_value(case_data, "new_first_name")), + "middle_name": _as_string(_first_value(case_data, "new_middle_name")), + "last_name": _as_string(_first_value(case_data, "new_last_name")), + "suffix": _as_string(_first_value(case_data, "new_suffix")), + "metadata": { + key: value for key, value in case_data.items() if key.startswith("new_") or key.startswith("reason_") + }, + } + _upsert_or_delete_party(draft, "name_sought", name_sought) + + +def _upsert_or_delete_party(draft: FilingDraft, role: str, values: dict[str, Any]) -> None: + meaningful_fields = set(values) - {"metadata"} + if any(values[field] for field in meaningful_fields): + FilingParty.objects.update_or_create(draft=draft, role=role, sort_order=0, defaults=values) + else: + FilingParty.objects.filter(draft=draft, role=role, sort_order=0).delete() + + +def _upsert_document( + draft: FilingDraft, + role: str, + document: Mapping[str, Any], + config: Mapping[str, Any], + *, + sort_order: int, +) -> FilingDocument: + document_data = dict(document or {}) + config_data = dict(config or {}) + defaults = { + "name": _as_string(_first_value(document_data, "name", "filename", "original_filename")), + "original_filename": _as_string(_first_value(document_data, "original_filename", "filename", "name")), + "size": _positive_int(_first_value(document_data, "size", "file_size")), + "content_type": _as_string(_first_value(document_data, "content_type", "mime_type", "type")), + "s3_key": _as_string(_first_value(document_data, "s3_key", "key")), + "public_url": _as_string(_first_value(document_data, "url", "s3_url", "file_url", "download_url")), + "filing_type_code": _as_string( + _first_value(config_data, "filing_type", "lead_filing_type", "filing_type_code") + ), + "filing_type_name": _as_string(_first_value(config_data, "filing_type_name", "lead_filing_type_name")), + "document_type_code": _as_string( + _first_value(config_data, "document_type", "lead_document_type", "document_type_code") + ), + "document_type_name": _as_string(_first_value(config_data, "document_type_name", "lead_document_type_name")), + "filing_component_code": _as_string( + _first_value(config_data, "filing_component", "lead_filing_component", "filing_component_code") + ), + "filing_component_name": _as_string( + _first_value(config_data, "filing_component_name", "lead_filing_component_name") + ), + "courtesy_copy_email": _as_string(_first_value(config_data, "cc_email", "lead_cc_email")), + "metadata": {"file": document_data, "config": config_data}, + } + document, _created = FilingDocument.objects.update_or_create( + draft=draft, + role=role, + sort_order=sort_order, + defaults=defaults, + ) + return document + + +def _positive_int(value: Any) -> int | None: + if value in (None, ""): + return None + try: + value = int(value) + except (TypeError, ValueError): + return None + return value if value >= 0 else None diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index e97edb9..50ed8c6 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -4,12 +4,16 @@ from django.urls import reverse from efile.models import FilingDocument, FilingDraft, FilingParty -from efile.services.drafts import draft_snapshot, sync_documents_from_upload_data, update_draft_from_case_data +from efile.services.current_drafts import CURRENT_DRAFT_SESSION_KEY, get_current_draft +from efile.services.drafts import draft_snapshot +from efile.services.legacy_draft_bridge import sync_documents_from_upload_data, update_draft_from_case_data +from efile.workflow import WorkflowStepKey, get_workflow_step_choices @pytest.mark.django_db -def test_update_draft_from_case_data_normalizes_known_fields(): - draft = FilingDraft.objects.create(jurisdiction="illinois") +def test_update_draft_from_case_data_normalizes_known_fields(django_user_model): + user = django_user_model.objects.create_user(username="draft-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") case_data = { "court": "cook:cd", "court_name": "Cook County Circuit Court", @@ -31,10 +35,10 @@ def test_update_draft_from_case_data_normalizes_known_fields(): "new_last_name": "Lovelace", } - update_draft_from_case_data(draft, case_data, current_step=FilingDraft.WorkflowStep.CASE_INFORMATION) + update_draft_from_case_data(draft, case_data, current_step=WorkflowStepKey.CASE_INFORMATION) draft.refresh_from_db() - assert draft.current_step == FilingDraft.WorkflowStep.CASE_INFORMATION + assert draft.current_step == WorkflowStepKey.CASE_INFORMATION assert draft.court_code == "cook:cd" assert draft.case_category_code == "MR" assert draft.case_type_code == "Name Change" @@ -54,8 +58,9 @@ def test_update_draft_from_case_data_normalizes_known_fields(): @pytest.mark.django_db -def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(): - draft = FilingDraft.objects.create(jurisdiction="illinois") +def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(django_user_model): + user = django_user_model.objects.create_user(username="document-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") upload_data = { "files": { "lead": { @@ -88,10 +93,10 @@ def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents() ], } - sync_documents_from_upload_data(draft, upload_data, current_step=FilingDraft.WorkflowStep.DOCUMENTS) + sync_documents_from_upload_data(draft, upload_data, current_step=WorkflowStepKey.DOCUMENTS) draft.refresh_from_db() - assert draft.current_step == FilingDraft.WorkflowStep.DOCUMENTS + assert draft.current_step == WorkflowStepKey.DOCUMENTS assert draft.extracted_guesses == {"court": "Cook County"} lead = FilingDocument.objects.get(draft=draft, role=FilingDocument.Role.LEAD) @@ -106,11 +111,13 @@ def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents() @pytest.mark.django_db -def test_draft_snapshot_is_json_serializable(): - draft = FilingDraft.objects.create(jurisdiction="illinois", court_code="cook:cd") +def test_draft_snapshot_is_json_serializable(django_user_model): + user = django_user_model.objects.create_user(username="snapshot-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois", court_code="cook:cd") snapshot = draft_snapshot(draft) + assert snapshot is not None assert snapshot["id"] == draft.pk assert snapshot["court_code"] == "cook:cd" json.dumps(snapshot) @@ -138,5 +145,116 @@ def test_create_draft_view_creates_durable_draft(client, django_user_model): draft = FilingDraft.objects.get(user=user) assert draft.jurisdiction == "illinois" - assert draft.current_step == FilingDraft.WorkflowStep.UPLOAD_FIRST + assert draft.current_step == WorkflowStepKey.UPLOAD_FIRST assert payload["data"]["filing_draft"]["id"] == draft.pk + + +@pytest.mark.django_db +def test_current_draft_enforces_owner(client, django_user_model): + illinois_user = django_user_model.objects.create_user(username="illinois-user", tyler_jurisdiction="illinois") + other_user = django_user_model.objects.create_user(username="other-user", tyler_jurisdiction="massachusetts") + other_draft = FilingDraft.objects.create(user=other_user, jurisdiction="illinois") + expected_draft = FilingDraft.objects.create(user=illinois_user, jurisdiction="illinois") + + client.force_login(illinois_user) + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = other_draft.pk + session.save() + + response = client.get(reverse("get_current_draft")) + + assert response.status_code == 200 + assert response.json()["data"]["filing_draft"]["id"] == expected_draft.pk + + +@pytest.mark.django_db +def test_current_draft_does_not_cross_jurisdictions(client, django_user_model): + user = django_user_model.objects.create_user(username="multi-state-user", tyler_jurisdiction="illinois") + illinois_draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + massachusetts_draft = FilingDraft.objects.create(user=user, jurisdiction="massachusetts") + client.force_login(user) + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = massachusetts_draft.pk + session.save() + + request = type("Request", (), {"user": user, "session": client.session})() + current = get_current_draft(request, jurisdiction="illinois") + + assert current == illinois_draft + + +@pytest.mark.django_db +def test_legacy_case_endpoint_mirrors_into_current_draft(client, django_user_model): + user = django_user_model.objects.create_user(username="bridge-user", tyler_jurisdiction="illinois") + client.force_login(user) + client.post( + reverse("create_draft", kwargs={"jurisdiction": "illinois"}), + data={}, + content_type="application/json", + ) + + response = client.post( + reverse("save_case_data_api"), + data={ + "jurisdiction": "illinois", + "data": { + "existing_case": "no", + "court": "cook:cd", + "case_type": "Name Change", + "petitioner_first_name": "Ada", + "petitioner_last_name": "Lovelace", + }, + }, + content_type="application/json", + ) + + assert response.status_code == 200 + draft = FilingDraft.objects.get(user=user) + assert draft.court_code == "cook:cd" + assert draft.case_type_code == "Name Change" + assert draft.parties.get(role="petitioner").first_name == "Ada" + + +@pytest.mark.django_db +def test_model_step_choices_follow_workflow_registry(): + current_step = FilingDraft._meta.get_field("current_step") + + assert tuple(current_step.choices) == get_workflow_step_choices() + + +@pytest.mark.django_db +def test_legacy_partial_case_update_does_not_clear_omitted_fields(django_user_model): + user = django_user_model.objects.create_user(username="partial-update-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create( + user=user, + jurisdiction="illinois", + court_code="old-code", + court_name="Court name to preserve", + ) + + update_draft_from_case_data(draft, {"court": "new-code"}) + draft.refresh_from_db() + + assert draft.court_code == "new-code" + assert draft.court_name == "Court name to preserve" + + +@pytest.mark.django_db +def test_legacy_upload_sync_removes_state_missing_from_complete_blob(django_user_model): + user = django_user_model.objects.create_user(username="upload-replace-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create( + user=user, + jurisdiction="illinois", + extracted_guesses={"court": "Old guess"}, + ) + FilingDocument.objects.create( + draft=draft, + role=FilingDocument.Role.LEAD, + name="old.pdf", + ) + + sync_documents_from_upload_data(draft, {"files": {}, "guesses": {}}) + draft.refresh_from_db() + + assert draft.extracted_guesses == {} + assert not draft.documents.exists() diff --git a/efile_app/efile/views/api_views.py b/efile_app/efile/views/api_views.py index 800ef2e..905aabf 100644 --- a/efile_app/efile/views/api_views.py +++ b/efile_app/efile/views/api_views.py @@ -8,7 +8,9 @@ from django.views import View from django.views.decorators.csrf import ensure_csrf_cookie -from efile.services.drafts import draft_snapshot, get_active_draft, update_draft_from_case_data +from efile.services.current_drafts import get_current_draft +from efile.services.drafts import draft_snapshot +from efile.services.legacy_draft_bridge import sync_current_draft_case_data logger = logging.getLogger(__name__) @@ -25,7 +27,7 @@ def get(self, request): if value: data[param_to_check] = value - draft = get_active_draft(request) + draft = get_current_draft(request, resume_latest=False) if draft: data["filing_draft"] = draft_snapshot(draft) @@ -81,9 +83,7 @@ def post(self, request): request.session["case_data"] = case_data request.session.modified = True - draft = get_active_draft(request) - if draft: - update_draft_from_case_data(draft, case_data) + sync_current_draft_case_data(request, case_data) logger.debug(f"Saved case data to session: {case_data}") diff --git a/efile_app/efile/views/draft_views.py b/efile_app/efile/views/draft_views.py index c8ff550..391475a 100644 --- a/efile_app/efile/views/draft_views.py +++ b/efile_app/efile/views/draft_views.py @@ -4,7 +4,8 @@ from django.http import JsonResponse from django.views.decorators.http import require_http_methods -from efile.services.drafts import create_draft, draft_snapshot, get_active_draft +from efile.services.current_drafts import create_current_draft, get_current_draft +from efile.services.drafts import draft_snapshot from efile.utils.django_helpers import flush_cache_stay_logged_in from efile.workflow import WorkflowStepKey, get_step_url @@ -19,12 +20,14 @@ def create_draft_view(request, jurisdiction): return JsonResponse({"success": False, "error": "Authentication required"}, status=401) try: - json.loads(request.body or "{}") + payload = json.loads(request.body or "{}") except json.JSONDecodeError: return JsonResponse({"success": False, "error": "Invalid JSON data"}, status=400) + if not isinstance(payload, dict): + return JsonResponse({"success": False, "error": "JSON body must be an object"}, status=400) flush_cache_stay_logged_in(request.session) - draft = create_draft(request, jurisdiction, current_step=WorkflowStepKey.UPLOAD_FIRST) + draft = create_current_draft(request, jurisdiction, current_step=WorkflowStepKey.UPLOAD_FIRST) logger.info("Created durable draft id=%s jurisdiction=%s", draft.pk, jurisdiction) return JsonResponse( @@ -40,5 +43,8 @@ def create_draft_view(request, jurisdiction): def get_current_draft_view(request): """Return the durable draft attached to this session/user.""" - draft = get_active_draft(request) + if not request.user.is_authenticated: + return JsonResponse({"success": False, "error": "Authentication required"}, status=401) + + draft = get_current_draft(request) return JsonResponse({"success": True, "data": {"filing_draft": draft_snapshot(draft)}}) diff --git a/efile_app/efile/views/options.py b/efile_app/efile/views/options.py index 5c7f9c0..c85110c 100644 --- a/efile_app/efile/views/options.py +++ b/efile_app/efile/views/options.py @@ -2,7 +2,8 @@ from django.views.decorators.csrf import ensure_csrf_cookie from efile.api.suffolk_api_views import get_tyler_token -from efile.services.drafts import draft_snapshot, get_active_draft +from efile.services.current_drafts import get_current_draft +from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import get_case_data from ..workflow import WorkflowStepKey, get_workflow_context @@ -14,7 +15,7 @@ def efile_options(request, jurisdiction): # Get case data from session if request.user.is_authenticated: case_data = get_case_data(request) - active_draft = get_active_draft(request) + active_draft = get_current_draft(request, jurisdiction=jurisdiction) else: case_data = {} active_draft = None diff --git a/efile_app/efile/views/session_api.py b/efile_app/efile/views/session_api.py index 6bde809..b8bbeb7 100644 --- a/efile_app/efile/views/session_api.py +++ b/efile_app/efile/views/session_api.py @@ -8,6 +8,11 @@ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods +from efile.services.legacy_draft_bridge import ( + sync_current_draft_case_data, + sync_current_draft_upload_data, +) + from ..utils.case_data_utils import clear_case_data, clear_upload_data, get_upload_data from ..utils.llms import LlmError, extract_fields_from_file from ..utils.proxy_connection import get_party_type_code_from_api @@ -219,6 +224,7 @@ def save_form_data_to_session(request): # Persist to session request.session["case_data"] = case_data request.session.modified = True + sync_current_draft_case_data(request, case_data) return JsonResponse({"success": True, "message": "Case data saved to session"}) @@ -271,6 +277,7 @@ def save_upload_first_data(request): # Save to session request.session["upload_data"] = upload_data request.session.modified = True + sync_current_draft_upload_data(request, upload_data) logger.info("Successfully saved upload data to session") return JsonResponse({"success": True, "message": "Upload data saved to session"}) @@ -309,6 +316,7 @@ def save_upload_data_to_session(request): # Save to session request.session["upload_data"] = upload_data request.session.modified = True + sync_current_draft_upload_data(request, upload_data) logger.info("Successfully saved upload data to session") return JsonResponse({"success": True, "message": "Upload data saved to session"}) @@ -580,6 +588,7 @@ def api_save_case_data(request): request.session["case_data"] = case_data request.session.modified = True + sync_current_draft_case_data(request, case_data) return JsonResponse( {"success": True, "data": {"existing_case": existing_case, "saved_fields": list(form_data.keys())}} @@ -667,6 +676,7 @@ def fetch_and_save_party_type(request): case_data["petitioner_party_type"] = party_type_code request.session["case_data"] = case_data request.session.modified = True + sync_current_draft_case_data(request, case_data) logger.debug(f"Saved party type to session: {party_type_code}") @@ -700,6 +710,7 @@ def save_party_type_to_session(request): case_data["available_party_types"] = party_types_available request.session["case_data"] = case_data request.session.modified = True + sync_current_draft_case_data(request, case_data) logger.debug(f"Saved party type to session: {party_type}") diff --git a/efile_app/efile/views/upload_first.py b/efile_app/efile/views/upload_first.py index fa538ad..39008d9 100644 --- a/efile_app/efile/views/upload_first.py +++ b/efile_app/efile/views/upload_first.py @@ -4,7 +4,8 @@ from django.shortcuts import redirect, render from efile.api.suffolk_api_views import get_tyler_token -from efile.services.drafts import draft_snapshot, ensure_draft +from efile.services.current_drafts import ensure_current_draft +from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import ( get_case_classification, @@ -39,7 +40,7 @@ def efile_upload_first(request, jurisdiction): request.session["jurisdiction"] = jurisdiction request.session.modified = True - filing_draft = ensure_draft(request, jurisdiction, current_step=WorkflowStepKey.UPLOAD_FIRST) + filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.UPLOAD_FIRST) # Could visit here from a back button press, so use upload data if any upload_data = get_upload_data(request) diff --git a/efile_app/efile/workflow.py b/efile_app/efile/workflow.py index d2445d2..9213899 100644 --- a/efile_app/efile/workflow.py +++ b/efile_app/efile/workflow.py @@ -60,6 +60,12 @@ def get_workflow_steps() -> tuple[WorkflowStep, ...]: return FILING_WORKFLOW +def get_workflow_step_choices() -> tuple[tuple[str, str], ...]: + """Return Django model choices derived from the workflow registry.""" + + return tuple((step.key.value, step.label) for step in FILING_WORKFLOW) + + def get_step(step_key: WorkflowStepKey | str) -> WorkflowStep: try: return next(step for step in FILING_WORKFLOW if step.key == step_key) diff --git a/fly.toml b/fly.toml index 2ac699c..b81c826 100644 --- a/fly.toml +++ b/fly.toml @@ -13,7 +13,7 @@ primary_region = 'lax' [deploy] # Run database migrations before each release - release_command = "uv run python manage.py migrate --noinput" + release_command = "uv run python manage.py migrate --noinput --fake-initial" [http_service] internal_port = 8000 From c11213749ec4dae89be5e9950a8313eb4a7f22b2 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Mon, 29 Jun 2026 19:13:40 -0400 Subject: [PATCH 18/47] Type fixes --- efile_app/efile/admin.py | 4 +++- efile_app/efile/services/drafts.py | 6 +++--- efile_app/efile/tests/test_workflow.py | 2 ++ efile_app/efile/views/options.py | 5 +++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/efile_app/efile/admin.py b/efile_app/efile/admin.py index ee98bef..8434f16 100644 --- a/efile_app/efile/admin.py +++ b/efile_app/efile/admin.py @@ -1,3 +1,5 @@ +from typing import Any, cast + from django.contrib import admin from django.contrib.auth.admin import UserAdmin @@ -28,7 +30,7 @@ class FilingPartyInline(admin.TabularInline): @admin.register(UserProfile) class UserProfileAdmin(UserAdmin): - fieldsets = UserAdmin.fieldsets + ( + fieldsets = cast(tuple[Any, ...], UserAdmin.fieldsets) + ( ( "eFile profile", { diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py index 9bb1344..9632a6c 100644 --- a/efile_app/efile/services/drafts.py +++ b/efile_app/efile/services/drafts.py @@ -12,7 +12,7 @@ from django.db import transaction from django.db.models import QuerySet -from efile.models import FilingDraft +from efile.models import FilingDocument, FilingDraft, FilingParty from efile.workflow import WorkflowStepKey ACTIVE_DRAFT_STATUSES = (FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR) @@ -101,8 +101,8 @@ def draft_snapshot(draft: FilingDraft | None) -> dict[str, Any] | None: "selected_payment_account_name": draft.selected_payment_account_name, "optional_services": draft.optional_services, "extracted_guesses": draft.extracted_guesses, - "document_count": draft.documents.count(), - "party_count": draft.parties.count(), + "document_count": FilingDocument.objects.filter(draft=draft).count(), + "party_count": FilingParty.objects.filter(draft=draft).count(), "created_at": draft.created_at.isoformat() if draft.created_at else None, "updated_at": draft.updated_at.isoformat() if draft.updated_at else None, "submitted_at": draft.submitted_at.isoformat() if draft.submitted_at else None, diff --git a/efile_app/efile/tests/test_workflow.py b/efile_app/efile/tests/test_workflow.py index 98437bd..51b7cf6 100644 --- a/efile_app/efile/tests/test_workflow.py +++ b/efile_app/efile/tests/test_workflow.py @@ -59,12 +59,14 @@ def test_get_previous_step_returns_none_for_first_step(): def test_get_previous_step_returns_prior_step(): previous_step = get_previous_step(WorkflowStepKey.CASE_INFORMATION) + assert previous_step is not None assert previous_step.key == WorkflowStepKey.UPLOAD_FIRST def test_get_next_step_returns_following_step(): next_step = get_next_step(WorkflowStepKey.CASE_INFORMATION) + assert next_step is not None assert next_step.key == WorkflowStepKey.DOCUMENTS diff --git a/efile_app/efile/views/options.py b/efile_app/efile/views/options.py index c85110c..b376cef 100644 --- a/efile_app/efile/views/options.py +++ b/efile_app/efile/views/options.py @@ -1,5 +1,5 @@ +from django.middleware.csrf import get_token from django.shortcuts import render -from django.views.decorators.csrf import ensure_csrf_cookie from efile.api.suffolk_api_views import get_tyler_token from efile.services.current_drafts import get_current_draft @@ -9,9 +9,10 @@ from ..workflow import WorkflowStepKey, get_workflow_context -@ensure_csrf_cookie def efile_options(request, jurisdiction): """Options view that displays saved case data and provides next steps.""" + get_token(request) + # Get case data from session if request.user.is_authenticated: case_data = get_case_data(request) From 53781728ebac42223149872d127e91969cb9e141 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:09:42 -0400 Subject: [PATCH 19/47] Enforce jurisdiction token before creating durable drafts --- efile_app/efile/views/draft_views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/efile_app/efile/views/draft_views.py b/efile_app/efile/views/draft_views.py index 391475a..e2f0068 100644 --- a/efile_app/efile/views/draft_views.py +++ b/efile_app/efile/views/draft_views.py @@ -4,6 +4,7 @@ from django.http import JsonResponse from django.views.decorators.http import require_http_methods +from efile.api.suffolk_api_views import get_tyler_token from efile.services.current_drafts import create_current_draft, get_current_draft from efile.services.drafts import draft_snapshot from efile.utils.django_helpers import flush_cache_stay_logged_in @@ -18,6 +19,8 @@ def create_draft_view(request, jurisdiction): if not request.user.is_authenticated: return JsonResponse({"success": False, "error": "Authentication required"}, status=401) + if not get_tyler_token(request, jurisdiction): + return JsonResponse({"success": False, "error": "Jurisdiction authorization required"}, status=403) try: payload = json.loads(request.body or "{}") From e6a7da976bf09b46c71ba3fde975258a61c9b570 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:09:51 -0400 Subject: [PATCH 20/47] Track durable draft step on case information view --- efile_app/efile/views/expert_form.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/efile_app/efile/views/expert_form.py b/efile_app/efile/views/expert_form.py index 96dfd99..61457d5 100644 --- a/efile_app/efile/views/expert_form.py +++ b/efile_app/efile/views/expert_form.py @@ -3,6 +3,8 @@ from django.shortcuts import redirect, render from efile.api.suffolk_api_views import get_tyler_token +from efile.services.current_drafts import ensure_current_draft +from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import get_upload_data from ..workflow import WorkflowStepKey, get_workflow_context @@ -18,6 +20,8 @@ def efile_expert_form(request, jurisdiction): if not request.user.is_authenticated: return redirect("efile_login", jurisdiction=jurisdiction) + filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.CASE_INFORMATION) + # Get auth tokens from session if available auth_tokens = request.session.get("auth_tokens", None) # Log only presence/keys, not token values @@ -52,6 +56,7 @@ def efile_expert_form(request, jurisdiction): context = { "is_logged_in": is_logged_in, "case_data": case_data, + "filing_draft": draft_snapshot(filing_draft), "guessed_court": upload_data.get("guesses", {}).get("court"), "guessed_case_category": upload_data.get("guesses", {}).get("case type"), "guessed_case_type": upload_data.get("guesses", {}).get("case category"), From afe3515e88238119dba7c663b5aa03e121a3fa8c Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:10:00 -0400 Subject: [PATCH 21/47] Track durable draft step on document upload view --- efile_app/efile/views/upload.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/efile_app/efile/views/upload.py b/efile_app/efile/views/upload.py index 3393cb5..dadbdff 100644 --- a/efile_app/efile/views/upload.py +++ b/efile_app/efile/views/upload.py @@ -5,6 +5,8 @@ from django.utils.translation import gettext from efile.api.suffolk_api_views import get_tyler_token +from efile.services.current_drafts import ensure_current_draft +from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import ( get_case_classification, @@ -30,6 +32,8 @@ def efile_upload(request, jurisdiction): messages.error(request, gettext("Please complete the case details first.")) return redirect("efile_options", jurisdiction=jurisdiction) + filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.DOCUMENTS) + petitioner_info = get_petitioner_info(request) name_sought_info = get_name_sought_info(request) case_classification = get_case_classification(request) @@ -47,6 +51,7 @@ def efile_upload(request, jurisdiction): "is_logged_in": is_logged_in, "case_data": case_data, "upload_data": upload_data, + "filing_draft": draft_snapshot(filing_draft), "petitioner_info": petitioner_info, "name_sought_info": name_sought_info, "case_classification": case_classification, From e5dede1a61f54dee7748a5ca4ff6151da23d1c8e Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:10:07 -0400 Subject: [PATCH 22/47] Track durable draft step on payment view --- efile_app/efile/views/payment.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/efile_app/efile/views/payment.py b/efile_app/efile/views/payment.py index 8462887..39dba12 100644 --- a/efile_app/efile/views/payment.py +++ b/efile_app/efile/views/payment.py @@ -5,6 +5,8 @@ from django.shortcuts import redirect, render from efile.api.suffolk_api_views import get_tyler_token +from efile.services.current_drafts import ensure_current_draft +from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import get_case_data from ..workflow import WorkflowStepKey, get_workflow_context @@ -14,6 +16,9 @@ def efile_payment(request, jurisdiction): """Review view for case details before final submission.""" + if not request.user.is_authenticated: + return redirect("efile_login", jurisdiction=jurisdiction) + # Get case data from session case_data = get_case_data(request) logger.debug("Review view case_data %s", case_data) @@ -28,6 +33,8 @@ def efile_payment(request, jurisdiction): messages.error(request, "Please complete the case details first.") return redirect("expert_form", jurisdiction=jurisdiction) + filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.PAYMENT) + new_toga_url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction}/payments/new-toga-account" is_logged_in = request.user.is_authenticated @@ -37,6 +44,7 @@ def efile_payment(request, jurisdiction): "is_logged_in": is_logged_in, "new_toga_url": new_toga_url, "case_data": case_data, + "filing_draft": draft_snapshot(filing_draft), } context.update(get_workflow_context(WorkflowStepKey.PAYMENT, jurisdiction)) From 12aa8b56a72765e0096ef60cda7a5fe032b8c6e9 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:10:20 -0400 Subject: [PATCH 23/47] Track durable draft step on review view --- efile_app/efile/views/review.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/efile_app/efile/views/review.py b/efile_app/efile/views/review.py index d980aee..fa92398 100644 --- a/efile_app/efile/views/review.py +++ b/efile_app/efile/views/review.py @@ -5,6 +5,8 @@ from django.shortcuts import redirect, render from efile.api.suffolk_api_views import get_tyler_token +from efile.services.current_drafts import ensure_current_draft +from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import get_case_classification, get_case_data, get_name_sought_info, get_petitioner_info from ..workflow import WorkflowStepKey, get_workflow_context @@ -14,6 +16,8 @@ def case_review(request, jurisdiction): """Review view for case details before final submission.""" + if not request.user.is_authenticated: + return redirect("efile_login", jurisdiction=jurisdiction) # Get case data from session case_data = get_case_data(request) @@ -29,6 +33,8 @@ def case_review(request, jurisdiction): messages.error(request, "Please complete the case details first.") return redirect("expert_form", jurisdiction=jurisdiction) + filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.REVIEW) + # Get organized case information petitioner_info = get_petitioner_info(request) name_sought_info = get_name_sought_info(request) @@ -104,6 +110,7 @@ def case_review(request, jurisdiction): "is_logged_in": is_logged_in, "new_toga_url": new_toga_url, "case_data": case_data, + "filing_draft": draft_snapshot(filing_draft), "review_sections": review_sections, "friendly_names": { "case_type": friendly_case_type, From 6cdec3ff7667451ffd1e1ef3947138ef90df38ed Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:10:42 -0400 Subject: [PATCH 24/47] Require jurisdiction token before ensuring upload draft --- efile_app/efile/views/upload_first.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/efile_app/efile/views/upload_first.py b/efile_app/efile/views/upload_first.py index 39008d9..bf29a56 100644 --- a/efile_app/efile/views/upload_first.py +++ b/efile_app/efile/views/upload_first.py @@ -26,6 +26,9 @@ def efile_upload_first(request, jurisdiction): if not request.user.is_authenticated: return redirect("efile_login", jurisdiction=jurisdiction) + if not get_tyler_token(request, jurisdiction): + return redirect("efile_login", jurisdiction=jurisdiction) + # Check if we need to clear cache (only when explicitly coming from options page button) clear_session = request.GET.get("clear_session", "false").lower() == "true" from_options = request.GET.get("from_options", "false").lower() == "true" @@ -50,11 +53,8 @@ def efile_upload_first(request, jurisdiction): name_sought_info = get_name_sought_info(request) case_classification = get_case_classification(request) - is_logged_in = request.user.is_authenticated - if not get_tyler_token(request, jurisdiction): - is_logged_in = False context = { - "is_logged_in": is_logged_in, + "is_logged_in": True, "upload_data": upload_data, "filing_draft": draft_snapshot(filing_draft), "petitioner_info": petitioner_info, From cf345c3c529cf049402c8dfe55716591d7e7b939 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:10:53 -0400 Subject: [PATCH 25/47] Require jurisdiction token before case information draft updates --- efile_app/efile/views/expert_form.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/efile_app/efile/views/expert_form.py b/efile_app/efile/views/expert_form.py index 61457d5..bfff5ce 100644 --- a/efile_app/efile/views/expert_form.py +++ b/efile_app/efile/views/expert_form.py @@ -20,6 +20,9 @@ def efile_expert_form(request, jurisdiction): if not request.user.is_authenticated: return redirect("efile_login", jurisdiction=jurisdiction) + if not get_tyler_token(request, jurisdiction): + return redirect("efile_login", jurisdiction=jurisdiction) + filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.CASE_INFORMATION) # Get auth tokens from session if available @@ -49,12 +52,9 @@ def efile_expert_form(request, jurisdiction): party_fields = ["petitioner_first_name", "petitioner_last_name", "new_first_name", "new_last_name"] has_party_info = all(case_data.get(field) for field in party_fields) - is_logged_in = request.user.is_authenticated - if not get_tyler_token(request, jurisdiction): - is_logged_in = False # Display the form for data collection with existing data populated context = { - "is_logged_in": is_logged_in, + "is_logged_in": True, "case_data": case_data, "filing_draft": draft_snapshot(filing_draft), "guessed_court": upload_data.get("guesses", {}).get("court"), From cd4c54986fab4552ea3a915f5c68f4441a41f59a Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:11:01 -0400 Subject: [PATCH 26/47] Require jurisdiction token before document draft updates --- efile_app/efile/views/upload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/efile_app/efile/views/upload.py b/efile_app/efile/views/upload.py index dadbdff..b2f7d4a 100644 --- a/efile_app/efile/views/upload.py +++ b/efile_app/efile/views/upload.py @@ -26,6 +26,9 @@ def efile_upload(request, jurisdiction): if not request.user.is_authenticated: return redirect("efile_login", jurisdiction=jurisdiction) + if not get_tyler_token(request, jurisdiction): + return redirect("efile_login", jurisdiction=jurisdiction) + case_data = get_case_data(request) if not case_data: @@ -44,11 +47,8 @@ def efile_upload(request, jurisdiction): upload_data = get_upload_data(request) - is_logged_in = request.user.is_authenticated - if not get_tyler_token(request, jurisdiction): - is_logged_in = False context = { - "is_logged_in": is_logged_in, + "is_logged_in": True, "case_data": case_data, "upload_data": upload_data, "filing_draft": draft_snapshot(filing_draft), From 11282c986225b5e4dfb00d6137f20ec6b106105c Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:11:09 -0400 Subject: [PATCH 27/47] Require jurisdiction token before payment draft updates --- efile_app/efile/views/payment.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/efile_app/efile/views/payment.py b/efile_app/efile/views/payment.py index 39dba12..ed38bfe 100644 --- a/efile_app/efile/views/payment.py +++ b/efile_app/efile/views/payment.py @@ -19,6 +19,9 @@ def efile_payment(request, jurisdiction): if not request.user.is_authenticated: return redirect("efile_login", jurisdiction=jurisdiction) + if not get_tyler_token(request, jurisdiction): + return redirect("efile_login", jurisdiction=jurisdiction) + # Get case data from session case_data = get_case_data(request) logger.debug("Review view case_data %s", case_data) @@ -37,11 +40,8 @@ def efile_payment(request, jurisdiction): new_toga_url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction}/payments/new-toga-account" - is_logged_in = request.user.is_authenticated - if not get_tyler_token(request, jurisdiction): - is_logged_in = False context = { - "is_logged_in": is_logged_in, + "is_logged_in": True, "new_toga_url": new_toga_url, "case_data": case_data, "filing_draft": draft_snapshot(filing_draft), From 81d16fe939171df16f6a1b9a56903a945017519c Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:11:23 -0400 Subject: [PATCH 28/47] Require jurisdiction token before review draft updates --- efile_app/efile/views/review.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/efile_app/efile/views/review.py b/efile_app/efile/views/review.py index fa92398..982d2b9 100644 --- a/efile_app/efile/views/review.py +++ b/efile_app/efile/views/review.py @@ -19,6 +19,9 @@ def case_review(request, jurisdiction): if not request.user.is_authenticated: return redirect("efile_login", jurisdiction=jurisdiction) + if not get_tyler_token(request, jurisdiction): + return redirect("efile_login", jurisdiction=jurisdiction) + # Get case data from session case_data = get_case_data(request) logger.debug("Review view case_data %s", case_data) @@ -103,11 +106,8 @@ def case_review(request, jurisdiction): new_toga_url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction}/payments/new-toga-account" - is_logged_in = request.user.is_authenticated - if not get_tyler_token(request, jurisdiction): - is_logged_in = False context = { - "is_logged_in": is_logged_in, + "is_logged_in": True, "new_toga_url": new_toga_url, "case_data": case_data, "filing_draft": draft_snapshot(filing_draft), From d400859773050d9df98c076073b68878c9caaeb8 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:11:32 -0400 Subject: [PATCH 29/47] Mirror final submission results into durable drafts --- efile_app/efile/views/submission.py | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 efile_app/efile/views/submission.py diff --git a/efile_app/efile/views/submission.py b/efile_app/efile/views/submission.py new file mode 100644 index 0000000..459c7bb --- /dev/null +++ b/efile_app/efile/views/submission.py @@ -0,0 +1,58 @@ +import json +import logging + +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods + +from efile.services.current_drafts import clear_current_draft, get_current_draft + +from .session_api import submit_final_filing as legacy_submit_final_filing + +logger = logging.getLogger(__name__) + + +def _json_payload(response: JsonResponse) -> dict: + try: + return json.loads(response.content.decode(response.charset or "utf-8")) + except (AttributeError, UnicodeDecodeError, ValueError): + return {} + + +def _submission_attempt_failed(response: JsonResponse, payload: dict) -> bool: + if "api_status_code" in payload: + return True + + error = payload.get("error") + if not isinstance(error, str): + return False + + return error.startswith("Filing submission failed:") or error.startswith("Network error during filing submission:") + + +@csrf_exempt +@require_http_methods(["POST"]) +def submit_final_filing(request): + """Submit through the legacy session path and mirror the result to the durable draft.""" + + jurisdiction = request.session.get("jurisdiction") + draft = get_current_draft(request, jurisdiction=jurisdiction, resume_latest=False) + + response = legacy_submit_final_filing(request) + payload = _json_payload(response) + + if draft is None: + return response + + if response.status_code < 400 and payload.get("success") is True: + draft.mark_submitted(payload.get("api_response") or {}) + clear_current_draft(request) + elif _submission_attempt_failed(response, payload): + draft.mark_error( + { + "status_code": response.status_code, + "response": payload, + } + ) + + return response From 6a75cf56f202a92ddcd0d3d0c0b8c21c8b50c0d6 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:11:45 -0400 Subject: [PATCH 30/47] Route final submission through durable draft wrapper --- efile_app/efile/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/efile_app/efile/urls.py b/efile_app/efile/urls.py index 225340c..8fe82ed 100644 --- a/efile_app/efile/urls.py +++ b/efile_app/efile/urls.py @@ -22,8 +22,8 @@ save_party_type_to_session, save_upload_data_to_session, save_upload_first_data, - submit_final_filing, ) +from .views.submission import submit_final_filing from .views.upload import efile_upload from .views.upload_first import efile_upload_first From 0b90257698436cbe28d769d8f57e06f4756d7e30 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Fri, 10 Jul 2026 10:12:21 -0400 Subject: [PATCH 31/47] Add durable draft authorization and submission regression tests --- efile_app/efile/tests/test_durable_drafts.py | 94 ++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index 50ed8c6..87ca9a2 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -10,6 +10,33 @@ from efile.workflow import WorkflowStepKey, get_workflow_step_choices +class FakeApiResponse: + def __init__(self, status_code, payload): + self.status_code = status_code + self._payload = payload + self.text = json.dumps(payload) + self.headers = {} + + def json(self): + return self._payload + + +def _authorize_jurisdiction_session(client, jurisdiction="illinois"): + session = client.session + session["auth_tokens"] = {f"TYLER-TOKEN-{jurisdiction.upper()}": "token"} + session.save() + + +def _set_submission_session(client, draft, jurisdiction="illinois"): + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = draft.pk + session["jurisdiction"] = jurisdiction + session["auth_tokens"] = {f"TYLER-TOKEN-{jurisdiction.upper()}": "token"} + session["case_data"] = {"court": "cook:cd"} + session["upload_data"] = {"files": {"lead": {"url": "https://example.com/petition.pdf"}}} + session.save() + + @pytest.mark.django_db def test_update_draft_from_case_data_normalizes_known_fields(django_user_model): user = django_user_model.objects.create_user(username="draft-owner", tyler_jurisdiction="illinois") @@ -131,6 +158,7 @@ def test_create_draft_view_creates_durable_draft(client, django_user_model): tyler_jurisdiction="illinois", ) client.force_login(user) + _authorize_jurisdiction_session(client) response = client.post( reverse("create_draft", kwargs={"jurisdiction": "illinois"}), @@ -149,6 +177,22 @@ def test_create_draft_view_creates_durable_draft(client, django_user_model): assert payload["data"]["filing_draft"]["id"] == draft.pk +@pytest.mark.django_db +def test_create_draft_view_requires_jurisdiction_token(client, django_user_model): + user = django_user_model.objects.create_user(username="no-token", tyler_jurisdiction="illinois") + client.force_login(user) + + response = client.post( + reverse("create_draft", kwargs={"jurisdiction": "illinois"}), + data={}, + content_type="application/json", + ) + + assert response.status_code == 403 + assert response.json()["success"] is False + assert not FilingDraft.objects.exists() + + @pytest.mark.django_db def test_current_draft_enforces_owner(client, django_user_model): illinois_user = django_user_model.objects.create_user(username="illinois-user", tyler_jurisdiction="illinois") @@ -187,6 +231,7 @@ def test_current_draft_does_not_cross_jurisdictions(client, django_user_model): def test_legacy_case_endpoint_mirrors_into_current_draft(client, django_user_model): user = django_user_model.objects.create_user(username="bridge-user", tyler_jurisdiction="illinois") client.force_login(user) + _authorize_jurisdiction_session(client) client.post( reverse("create_draft", kwargs={"jurisdiction": "illinois"}), data={}, @@ -215,6 +260,55 @@ def test_legacy_case_endpoint_mirrors_into_current_draft(client, django_user_mod assert draft.parties.get(role="petitioner").first_name == "Ada" +@pytest.mark.django_db +def test_final_submission_marks_current_draft_submitted(client, django_user_model, monkeypatch): + def fake_post(*_args, **_kwargs): + return FakeApiResponse(201, {"filing_id": "abc-123"}) + + monkeypatch.setattr("requests.post", fake_post) + user = django_user_model.objects.create_user(username="submit-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois", current_step=WorkflowStepKey.REVIEW) + client.force_login(user) + _set_submission_session(client, draft) + + response = client.post( + reverse("submit_final_filing"), + data={"confirm_submission": True, "efile_data": {"al_court_bundle": {}}}, + content_type="application/json", + ) + + assert response.status_code == 200 + draft.refresh_from_db() + assert draft.status == FilingDraft.Status.SUBMITTED + assert draft.current_step == WorkflowStepKey.CONFIRMATION + assert draft.submission_response == {"filing_id": "abc-123"} + assert CURRENT_DRAFT_SESSION_KEY not in client.session + + +@pytest.mark.django_db +def test_final_submission_marks_current_draft_error_on_api_failure(client, django_user_model, monkeypatch): + def fake_post(*_args, **_kwargs): + return FakeApiResponse(400, {"error": "Rejected", "validation_errors": ["bad bundle"]}) + + monkeypatch.setattr("requests.post", fake_post) + user = django_user_model.objects.create_user(username="error-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois", current_step=WorkflowStepKey.REVIEW) + client.force_login(user) + _set_submission_session(client, draft) + + response = client.post( + reverse("submit_final_filing"), + data={"confirm_submission": True, "efile_data": {"al_court_bundle": {}}}, + content_type="application/json", + ) + + assert response.status_code == 400 + draft.refresh_from_db() + assert draft.status == FilingDraft.Status.ERROR + assert draft.submission_response["status_code"] == 400 + assert draft.submission_response["response"]["api_status_code"] == 400 + + @pytest.mark.django_db def test_model_step_choices_follow_workflow_registry(): current_step = FilingDraft._meta.get_field("current_step") From c365f45c6dba29fc7acc4b57c55a36f788f94556 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Tue, 21 Jul 2026 18:30:35 -0400 Subject: [PATCH 32/47] Make the durable draft model the source of truth Replace the session-blob store with the FilingDraft aggregate. The legacy_draft_bridge that mirrored session blobs into the model is gone, along with the two-formats problem it created: case_data_utils now (de)serializes the model on every read/write, keeping its request-based signatures so views, templates, the submission path, and the browser wire format are unchanged. The model is fully typed -- extra_case_data and the document/party metadata catch-alls are removed. services/drafts.py holds an explicit wire<->model map that is the entire contract; unknown keys are not persisted, so adding a field a screen needs means adding a typed column and a map entry. Parties are typed FilingParty rows (petitioner, new_name, respondent, other) matching the canonical client-side model. Also cut over every live writer to the model, require auth to persist a draft, finalize the draft on submit via mark_submitted, and drop the dead server-side submission path (transform_case_data_to_filing_payload) plus the unused admin.py and the now-outdated design doc. Migration 0002 is amended in place (unmerged; DB wiped on merge). Co-Authored-By: Claude Opus 4.8 --- docs/filing-draft-data-model.md | 38 -- efile_app/efile/admin.py | 88 ----- efile_app/efile/api/filing_views.py | 283 +------------- efile_app/efile/api/suffolk_api_views.py | 24 +- .../efile/migrations/0002_filing_drafts.py | 4 +- efile_app/efile/models.py | 6 +- efile_app/efile/services/drafts.py | 361 +++++++++++++++++- .../efile/services/legacy_draft_bridge.py | 263 ------------- efile_app/efile/tests/test_durable_drafts.py | 151 ++++++-- efile_app/efile/tests/tests.py | 3 + efile_app/efile/utils/case_data_utils.py | 96 +++-- efile_app/efile/views/api_views.py | 85 +---- efile_app/efile/views/session_api.py | 247 ++++-------- 13 files changed, 612 insertions(+), 1037 deletions(-) delete mode 100644 docs/filing-draft-data-model.md delete mode 100644 efile_app/efile/admin.py delete mode 100644 efile_app/efile/services/legacy_draft_bridge.py diff --git a/docs/filing-draft-data-model.md b/docs/filing-draft-data-model.md deleted file mode 100644 index e22681f..0000000 --- a/docs/filing-draft-data-model.md +++ /dev/null @@ -1,38 +0,0 @@ -# Filing draft data model foundation - -This PR introduces the durable filing aggregate that future filing-flow PRs can migrate toward. - -## New source-of-truth models - -- `FilingDraft`: one filing workflow instance. It owns jurisdiction, status, current workflow step, case classification, existing-case identifiers, selected payment account, extracted guesses, temporary extra case data, and submission response. -- `FilingDocument`: one uploaded lead or supporting document. It owns S3/public URL metadata, file metadata, filing/document/component codes and names, courtesy copy email, and document order. -- `FilingParty`: one party/person associated with the draft. It owns role, party type, contact, name, and address fields. - -The model intentionally keeps `extra_case_data` and document/party `metadata` JSON fields as temporary escape hatches so the UI can move off session blobs incrementally without blocking on a perfect schema. - -## Service boundaries - -The durable draft code is split into three layers: - -- `services/drafts.py` contains request-independent operations on durable models. -- `services/current_drafts.py` resolves the current browser's draft while enforcing ownership and jurisdiction. The session contains only the current draft ID; it is not a state store. -- `services/legacy_draft_bridge.py` is the temporary compatibility adapter that translates the old `case_data` and `upload_data` blobs. Only legacy session endpoints import it, so it can be removed without changing the durable service. - -The options/start flow creates a `FilingDraft` through `POST /jurisdiction//drafts/` and redirects to the workflow's first document-upload step. The old session start path remains as a browser fallback. While that flow remains, its actual case and upload save endpoints shadow-write through the compatibility adapter. - -Drafts require an authenticated owner. Every current-draft lookup verifies that owner, active status, and, when known, jurisdiction before returning the object. - -## Migration rollout - -`efile` previously had no migrations even though existing environments may already have the `UserProfile` table. Migration `0001` therefore contains only that baseline model, and migration `0002` creates the new filing tables. Existing environments must run `migrate --fake-initial`; the Fly release command includes that option. Fresh databases apply both migrations normally. - -## Future migration sequence - -This is intended as the foundation for follow-up PRs: - -1. Replace legacy case-data endpoints with typed draft updates. -2. Replace legacy upload endpoints with typed document updates. -3. Move payment selection into explicit draft fields. -4. Render review from `FilingDraft`, `FilingDocument`, and `FilingParty`. -5. Move final submission to a draft-backed submission service. -6. Remove `legacy_draft_bridge.py`, arbitrary session-merge APIs, and browser storage persistence. diff --git a/efile_app/efile/admin.py b/efile_app/efile/admin.py deleted file mode 100644 index 8434f16..0000000 --- a/efile_app/efile/admin.py +++ /dev/null @@ -1,88 +0,0 @@ -from typing import Any, cast - -from django.contrib import admin -from django.contrib.auth.admin import UserAdmin - -from .models import FilingDocument, FilingDraft, FilingParty, UserProfile - - -class FilingDocumentInline(admin.TabularInline): - model = FilingDocument - extra = 0 - fields = ( - "role", - "sort_order", - "name", - "filing_type_code", - "document_type_code", - "filing_component_code", - "public_url", - ) - readonly_fields = ("created_at", "updated_at") - - -class FilingPartyInline(admin.TabularInline): - model = FilingParty - extra = 0 - fields = ("role", "sort_order", "party_type", "first_name", "last_name", "email", "phone") - readonly_fields = ("created_at", "updated_at") - - -@admin.register(UserProfile) -class UserProfileAdmin(UserAdmin): - fieldsets = cast(tuple[Any, ...], UserAdmin.fieldsets) + ( - ( - "eFile profile", - { - "fields": ( - "tyler_jurisdiction", - "tyler_user_id", - "email_updates", - "text_updates", - ) - }, - ), - ) - list_display = ("username", "email", "tyler_jurisdiction", "tyler_user_id", "is_staff") - - -@admin.register(FilingDraft) -class FilingDraftAdmin(admin.ModelAdmin): - inlines = [FilingDocumentInline, FilingPartyInline] - list_display = ( - "id", - "user", - "jurisdiction", - "status", - "current_step", - "court_code", - "case_type_code", - "updated_at", - ) - list_filter = ("jurisdiction", "status", "current_step", "created_at", "updated_at") - search_fields = ( - "id", - "court_code", - "court_name", - "case_type_code", - "case_type_name", - "docket_number", - "previous_case_id", - "user__username", - "user__email", - ) - readonly_fields = ("created_at", "updated_at", "submitted_at") - - -@admin.register(FilingDocument) -class FilingDocumentAdmin(admin.ModelAdmin): - list_display = ("id", "draft", "role", "sort_order", "name", "document_type_code", "updated_at") - list_filter = ("role", "content_type", "created_at", "updated_at") - search_fields = ("name", "original_filename", "s3_key", "public_url", "draft__id") - - -@admin.register(FilingParty) -class FilingPartyAdmin(admin.ModelAdmin): - list_display = ("id", "draft", "role", "party_type", "first_name", "last_name", "email") - list_filter = ("role", "party_type", "created_at", "updated_at") - search_fields = ("first_name", "middle_name", "last_name", "organization_name", "email", "draft__id") diff --git a/efile_app/efile/api/filing_views.py b/efile_app/efile/api/filing_views.py index 2b3fc65..5e6cddd 100644 --- a/efile_app/efile/api/filing_views.py +++ b/efile_app/efile/api/filing_views.py @@ -14,7 +14,7 @@ from efile.utils.jurisdiction_stuff import get_jurisdiction_from_request from ..utils.case_data_utils import get_case_data -from ..utils.proxy_connection import get_headers, get_party_type_code_from_api +from ..utils.proxy_connection import get_headers from .base import APIResponseMixin logger = logging.getLogger(__name__) @@ -172,7 +172,7 @@ def payment_fees(request): jurisdiction_id = request.session.get("jurisdiction") auth_tokens = request.session.get("auth_tokens", {}) - case_data = request.session.get("case_data", {}) + case_data = get_case_data(request) court_id = case_data.get("court", "") url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filing/fees" @@ -321,285 +321,6 @@ def delete_filing(request, filing_id): return FilingAPIViews.error_response(f"Error: {str(e)}") -@csrf_exempt -@require_http_methods(["POST"]) -def _unused_create_filing(request): - """Create a filing with Suffolk LIT Lab API using collected case data.""" - - try: - # Get case data from session - case_data = get_case_data(request) - - if not case_data: - return JsonResponse( - {"success": False, "error": "No case data found. Please complete the expert form first."}, status=400 - ) - - # Get auth tokens - auth_tokens = request.session.get("auth_tokens") - if not auth_tokens or "token" not in auth_tokens: - return JsonResponse( - {"success": False, "error": "Authentication required. Please log in first."}, status=401 - ) - - # Transform case data to Suffolk API payload format - filing_payload = transform_case_data_to_filing_payload(case_data, request) - - # Make POST request to Suffolk LIT Lab filing API - path = "/filings/" - api_url = f"{settings.EFSP_URL}{path}" - - headers = {"Authorization": f"Bearer {auth_tokens['token']}", "Content-Type": "application/json"} - - # Safe pre-request logging - logger.debug( - "POST %s with headers keys=%s payload keys=%s", - api_url, - list(headers.keys()), - list(filing_payload.keys()), - ) - response = requests.post(api_url, headers=headers, json=filing_payload, timeout=30) - logger.debug( - "Create filing response: status=%s content_type=%s", - response.status_code, - response.headers.get("Content-Type"), - ) - - if response.status_code == 201: - # Filing created successfully - filing_data = response.json() - - # Save filing ID to session for future reference - request.session["current_filing_id"] = filing_data.get("id") - request.session.modified = True - - return JsonResponse( - { - "success": True, - "filing_id": filing_data.get("id"), - "message": "Filing created successfully", - "data": filing_data, - } - ) - else: - # API error - error_detail = response.text - try: - error_json = response.json() - error_detail = error_json.get("detail", error_json) - except ValueError: - pass - - return JsonResponse( - { - "success": False, - "error": f"Filing creation failed: {error_detail}", - "status_code": response.status_code, - }, - status=response.status_code, - ) - - except requests.RequestException as e: - return JsonResponse({"success": False, "error": f"Network error: {str(e)}"}, status=500) - except Exception as e: - return JsonResponse({"success": False, "error": f"Unexpected error: {str(e)}"}, status=500) - - -def transform_case_data_to_filing_payload(case_data, request=None): - """ - Transform collected case data into Suffolk LIT Lab API filing payload format. - """ - - # Base filing payload structure - payload = { - "jurisdiction": case_data.get("jurisdiction"), - "court": case_data.get("court"), - "category": case_data.get("case_category"), - "case_type": case_data.get("case_type"), - "filing_type": case_data.get("filing_type"), - "document_type": case_data.get("document_type"), - "parties": [], - "optional_services": case_data.get("optional_services", []), - } - - # Add petitioner party if this is a name change case - if "name change" in case_data.get("case_type", "").lower(): - # Add petitioner - if case_data.get("petitioner_first_name") or case_data.get("petitioner_last_name"): - # Use party type from session data, or determine it from API - party_type = case_data.get("petitioner_party_type") - if not party_type: - party_type = determine_party_type_for_new_case(case_data) - - petitioner = { - "party_type": party_type, - "name": { - "first": case_data.get("petitioner_first_name", ""), - "last": case_data.get("petitioner_last_name", ""), - "full": ( - f"{case_data.get('petitioner_first_name', '')} {case_data.get('petitioner_last_name', '')}" - ).strip(), - }, - "address": case_data.get("petitioner_address", ""), - "role": "Petitioner", # Keep role as readable text - } - payload["parties"].append(petitioner) - else: - # For non-name change cases, still add the party information if available - if case_data.get("first_name") or case_data.get("last_name"): - # Determine party type based on existing case status - existing_case = None - if request: - existing_case = request.session.get("existing_case") - - # Also check if existing_case is stored in case_data - if not existing_case: - existing_case = case_data.get("existing_case") - - if existing_case == "yes": - # When responding to existing case, use intelligent party type determination - party_type = case_data.get("party_type") or determine_party_type_for_existing_case(case_data) - else: - # For new cases, use intelligent party type determination - party_type = case_data.get("party_type") or determine_party_type_for_new_case(case_data) - - # Determine role name for display (keep as readable text) - if existing_case == "yes": - role_name = "Defendant" if "DEF" in party_type else "Respondent" if "RES" in party_type else "Party" - else: - role_name = "Petitioner" if "PET" in party_type else "Plaintiff" if "PLA" in party_type else "Party" - - party = { - "party_type": party_type, - "name": { - "first": case_data.get("first_name", ""), - "last": case_data.get("last_name", ""), - "full": (f"{case_data.get('first_name', '')} {case_data.get('last_name', '')}").strip(), - }, - "address": case_data.get("address", ""), - "role": role_name, - } - payload["parties"].append(party) - - # Add name sought information as additional case details - if case_data.get("new_first_name") or case_data.get("new_last_name"): - payload["name_change_details"] = { - "new_name": { - "first": case_data.get("new_first_name", ""), - "last": case_data.get("new_last_name", ""), - "full": (f"{case_data.get('new_first_name', '')} {case_data.get('new_last_name', '')}").strip(), - } - } - - # Add case metadata - payload["metadata"] = { - "created_via": "illinois_efile_system", - "case_classification": { - "court": case_data.get("court"), - "category": case_data.get("case_category"), - "case_type": case_data.get("case_type"), - "filing_type": case_data.get("filing_type"), - "document_type": case_data.get("document_type"), - }, - } - - return payload - - -def determine_party_type_for_new_case(case_data): - """ - Determine the appropriate party type for a new case. - This fetches actual party type codes from the API. - """ - court_code = case_data.get("court") - case_type_code = case_data.get("case_type") - case_type = case_data.get("case_type", "").lower() - jurisdiction = case_data.get("jurisdiction") - - if not court_code or not case_type_code: - logger.warning("Missing court or case_type for party type determination") - return "PET" # Default fallback code for petitioner - - # Determine target party type name based on case type - target_party_name = None - - if "name change" in case_type: - target_party_name = "petitioner" - elif "civil" in case_type: - target_party_name = "plaintiff" - elif "family" in case_type: - target_party_name = "petitioner" - elif "probate" in case_type: - target_party_name = "petitioner" - else: - target_party_name = "petitioner" - - # Get the actual party type code from API - party_code = get_party_type_code_from_api( - court_code, case_type_code, jurisdiction, target_party_name=target_party_name - ) - - # Fallback codes if API call fails - if not party_code: - if target_party_name == "plaintiff": - return "PLA" - elif target_party_name == "petitioner": - return "PET" - else: - return "PET" - - return party_code - - -def determine_party_type_for_existing_case(case_data): - """ - Determine the appropriate party type when responding to an existing case. - This fetches actual party type codes from the API. - """ - court_code = case_data.get("court") - case_type_code = case_data.get("case_type") - case_type = case_data.get("case_type", "").lower() - filing_type = case_data.get("filing_type", "").lower() - jurisdiction = case_data.get("jurisdiction") - - if not court_code or not case_type_code: - logger.warning("Missing court or case_type for party type determination") - return "DEF" # Default fallback code - - # Determine target party type name based on case and filing type - target_party_name = None - - if "criminal" in case_type: - target_party_name = "defendant" - elif "civil" in case_type or "family" in case_type: - if "answer" in filing_type or "response" in filing_type: - target_party_name = "respondent" - else: - target_party_name = "defendant" - elif "probate" in case_type: - target_party_name = "interested party" - else: - target_party_name = "defendant" - - # Get the actual party type code from API - party_code = get_party_type_code_from_api( - court_code, case_type_code, jurisdiction, target_party_name=target_party_name - ) - - # Fallback codes if API call fails - if not party_code: - if target_party_name == "defendant": - return "DEF" - elif target_party_name == "respondent": - return "RES" - elif target_party_name == "interested party": - return "INT" - else: - return "DEF" - - return party_code - - # Individual view functions for URL mapping get_filings = FilingAPIViews.get_filings create_filing = FilingAPIViews.create_filing diff --git a/efile_app/efile/api/suffolk_api_views.py b/efile_app/efile/api/suffolk_api_views.py index a5b9efd..83e2016 100644 --- a/efile_app/efile/api/suffolk_api_views.py +++ b/efile_app/efile/api/suffolk_api_views.py @@ -12,6 +12,7 @@ from django.views.decorators.http import require_http_methods from requests.exceptions import RequestException +from efile.utils.case_data_utils import update_case_data from efile.utils.jurisdiction_stuff import get_jurisdiction_from_request from efile.utils.proxy_connection import get_headers @@ -292,17 +293,18 @@ def get_party_types_from_suffolk_api(request): f"{party_types[0].get('name', 'Unknown')} ({selected_party_type})" ) - # Save to session - case_data = request.session.get("case_data", {}) - case_data["determined_party_type"] = selected_party_type - case_data["party_type"] = selected_party_type - case_data["petitioner_party_type"] = selected_party_type - case_data["available_party_types"] = party_types - case_data["existing_case"] = existing_case # Save existing case status - request.session["case_data"] = case_data - request.session.modified = True - - logger.debug(f"Saved party type to session: {selected_party_type}") + # Persist the resolved party type onto the current draft + update_case_data( + request, + { + "determined_party_type": selected_party_type, + "party_type": selected_party_type, + "petitioner_party_type": selected_party_type, + "existing_case": existing_case, + }, + ) + + logger.debug(f"Saved party type to draft: {selected_party_type}") if only_required: filtered_party_types = [p for p in party_types if p.get("isrequired", False)] diff --git a/efile_app/efile/migrations/0002_filing_drafts.py b/efile_app/efile/migrations/0002_filing_drafts.py index aec407d..c90d4b2 100644 --- a/efile_app/efile/migrations/0002_filing_drafts.py +++ b/efile_app/efile/migrations/0002_filing_drafts.py @@ -64,9 +64,9 @@ class Migration(migrations.Migration): ("docket_number", models.CharField(blank=True, max_length=255)), ("selected_payment_account_id", models.CharField(blank=True, max_length=255)), ("selected_payment_account_name", models.CharField(blank=True, max_length=255)), + ("name_change_reason", models.TextField(blank=True)), ("optional_services", models.JSONField(blank=True, default=list)), ("extracted_guesses", models.JSONField(blank=True, default=dict)), - ("extra_case_data", models.JSONField(blank=True, default=dict)), ("submission_response", models.JSONField(blank=True, default=dict)), ("submitted_at", models.DateTimeField(blank=True, null=True)), ("created_at", models.DateTimeField(auto_now_add=True)), @@ -114,7 +114,6 @@ class Migration(migrations.Migration): ("filing_component_code", models.CharField(blank=True, max_length=100)), ("filing_component_name", models.CharField(blank=True, max_length=255)), ("courtesy_copy_email", models.EmailField(blank=True, max_length=254)), - ("metadata", models.JSONField(blank=True, default=dict)), ("created_at", models.DateTimeField(auto_now_add=True)), ("updated_at", models.DateTimeField(auto_now=True)), ( @@ -151,7 +150,6 @@ class Migration(migrations.Migration): ("state", models.CharField(blank=True, max_length=50)), ("zip_code", models.CharField(blank=True, max_length=20)), ("country", models.CharField(blank=True, default="US", max_length=2)), - ("metadata", models.JSONField(blank=True, default=dict)), ("created_at", models.DateTimeField(auto_now_add=True)), ("updated_at", models.DateTimeField(auto_now=True)), ( diff --git a/efile_app/efile/models.py b/efile_app/efile/models.py index 7e52f7c..7d144db 100644 --- a/efile_app/efile/models.py +++ b/efile_app/efile/models.py @@ -75,9 +75,10 @@ class Status(models.TextChoices): selected_payment_account_id = models.CharField(max_length=255, blank=True) selected_payment_account_name = models.CharField(max_length=255, blank=True) + name_change_reason = models.TextField(blank=True) + optional_services = models.JSONField(default=list, blank=True) extracted_guesses = models.JSONField(default=dict, blank=True) - extra_case_data = models.JSONField(default=dict, blank=True) submission_response = models.JSONField(default=dict, blank=True) submitted_at = models.DateTimeField(blank=True, null=True) @@ -134,7 +135,6 @@ class Role(models.TextChoices): filing_component_name = models.CharField(max_length=255, blank=True) courtesy_copy_email = models.EmailField(blank=True) - metadata = models.JSONField(default=dict, blank=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) @@ -177,8 +177,6 @@ class FilingParty(models.Model): zip_code = models.CharField(max_length=20, blank=True) country = models.CharField(max_length=2, default="US", blank=True) - metadata = models.JSONField(default=dict, blank=True) - created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py index 9632a6c..aea025b 100644 --- a/efile_app/efile/services/drafts.py +++ b/efile_app/efile/services/drafts.py @@ -1,8 +1,8 @@ """Operations on the durable filing draft aggregate. -This module deliberately has no dependency on HTTP requests, sessions, or the -legacy session data shapes. Request/session selection lives in -``current_drafts``; legacy data translation lives in ``legacy_draft_bridge``. +This module deliberately has no dependency on HTTP requests or sessions. Request/ +session selection lives in ``current_drafts``; the wire (de)serialization between +the browser's flat case_data/upload_data blobs and the typed model lives here. """ from __future__ import annotations @@ -72,6 +72,361 @@ def set_current_step(draft: FilingDraft, current_step: WorkflowStepKey | str) -> return draft +# --- Wire (de)serialization ------------------------------------------------- +# +# The browser speaks a flat "case_data" / "upload_data" JSON blob. These maps are +# the *entire* contract: any key not listed here is deliberately not persisted. +# Adding a field a screen needs to keep means adding a typed column/party field +# and an entry here -- there is no catch-all blob. + +_MISSING = object() + +# Draft scalar column -> the wire keys it may arrive under (first present wins). +_DRAFT_FIELD_SOURCES: dict[str, tuple[str, ...]] = { + "court_code": ("court",), + "court_name": ("court_name",), + "case_category_code": ("case_category",), + "case_category_name": ("case_category_name",), + "case_type_code": ("case_type",), + "case_type_name": ("case_type_name",), + "case_subtype_code": ("case_subtype",), + "case_subtype_name": ("case_subtype_name",), + "filing_type_code": ("filing_type", "filing_type_id"), + "filing_type_name": ("filing_type_name",), + "document_type_code": ("document_type",), + "document_type_name": ("document_type_name",), + "existing_case": ("existing_case",), + "previous_case_id": ("case_tracking_id", "previous_case_id"), + "docket_number": ("case_docket_id", "case_number", "docket_number"), + "selected_payment_account_id": ("selected_payment_account", "payment_account_id"), + "selected_payment_account_name": ("selected_payment_account_name",), + "name_change_reason": ("reason_for_name_change", "reason_for_change"), +} + +# FilingParty role -> {model field: wire key}. Petitioner party_type is special +# (three legacy aliases for one value) and handled separately. +_PARTY_SPECS: dict[str, dict[str, str]] = { + "petitioner": { + "first_name": "petitioner_first_name", + "last_name": "petitioner_last_name", + "address_line_1": "petitioner_address", + "email": "petitioner_email", + "phone": "petitioner_phone", + }, + "new_name": { + "first_name": "new_first_name", + "middle_name": "new_middle_name", + "last_name": "new_last_name", + "suffix": "new_suffix", + "party_type": "new_name_party_type", + }, + "respondent": { + "first_name": "respondent_first_name", + "middle_name": "respondent_middle_name", + "last_name": "respondent_last_name", + "suffix": "respondent_suffix", + "party_type": "respondent_name_party_type", + }, + "other": { + "first_name": "other_first_name", + "last_name": "other_last_name", + "party_type": "other_party_type", + "address_line_1": "other_address_line_1", + "address_line_2": "other_address_line_2", + "city": "other_address_city", + "state": "other_address_state", + "zip_code": "other_address_zip", + "email": "other_email", + "phone": "other_phone_number", + }, +} + +_PETITIONER_PARTY_TYPE_KEYS = ("determined_party_type", "party_type", "petitioner_party_type") + + +def _first_present(data: dict[str, Any], keys: tuple[str, ...]) -> Any: + for key in keys: + if key in data: + return data[key] + return _MISSING + + +def _as_str(value: Any) -> str: + return "" if value in (None, "") else str(value) + + +def _put(data: dict[str, Any], key: str, value: Any) -> None: + """Add ``key`` to the wire blob only when it carries a real value.""" + + if value not in (None, "", [], {}): + data[key] = value + + +@transaction.atomic +def write_case_data( + draft: FilingDraft, + case_data: dict[str, Any] | None, + *, + current_step: WorkflowStepKey | str | None = None, +) -> FilingDraft: + """Persist a (possibly partial) case_data blob onto the draft. + + Only keys present in ``case_data`` are written; omitted fields are left as-is + so per-screen partial saves don't clobber earlier steps. + """ + + data = dict(case_data or {}) + update_fields: list[str] = [] + + for field, sources in _DRAFT_FIELD_SOURCES.items(): + value = _first_present(data, sources) + if value is _MISSING: + continue + value = _as_str(value) + if getattr(draft, field) != value: + setattr(draft, field, value) + update_fields.append(field) + + if "optional_services" in data: + services = data.get("optional_services") or [] + if draft.optional_services != services: + draft.optional_services = services + update_fields.append("optional_services") + + if current_step is not None and draft.current_step != str(current_step): + draft.current_step = str(current_step) + update_fields.append("current_step") + + if update_fields: + draft.save(update_fields=sorted({*update_fields, "updated_at"})) + + _write_parties(draft, data) + return draft + + +def _write_parties(draft: FilingDraft, data: dict[str, Any]) -> None: + for role, spec in _PARTY_SPECS.items(): + values = {model_field: _as_str(data[wire_key]) for model_field, wire_key in spec.items() if wire_key in data} + if role == "petitioner": + party_type = _first_present(data, _PETITIONER_PARTY_TYPE_KEYS) + if party_type is not _MISSING: + values["party_type"] = _as_str(party_type) + if values: + FilingParty.objects.update_or_create(draft=draft, role=role, sort_order=0, defaults=values) + + +def read_case_data(draft: FilingDraft | None) -> dict[str, Any]: + """Serialize the draft back into the flat case_data blob the browser reads.""" + + if draft is None: + return {} + + data: dict[str, Any] = {} + _put(data, "jurisdiction", draft.jurisdiction) + _put(data, "jurisdiction_id", draft.jurisdiction) + _put(data, "existing_case", draft.existing_case) + _put(data, "court", draft.court_code) + _put(data, "court_name", draft.court_name) + _put(data, "case_category", draft.case_category_code) + _put(data, "case_category_name", draft.case_category_name) + _put(data, "case_type", draft.case_type_code) + _put(data, "case_type_name", draft.case_type_name) + _put(data, "case_subtype", draft.case_subtype_code) + _put(data, "case_subtype_name", draft.case_subtype_name) + _put(data, "filing_type", draft.filing_type_code) + _put(data, "filing_type_id", draft.filing_type_code) + _put(data, "filing_type_name", draft.filing_type_name) + _put(data, "document_type", draft.document_type_code) + _put(data, "document_type_name", draft.document_type_name) + _put(data, "previous_case_id", draft.previous_case_id) + _put(data, "docket_number", draft.docket_number) + _put(data, "selected_payment_account", draft.selected_payment_account_id) + _put(data, "selected_payment_account_name", draft.selected_payment_account_name) + _put(data, "optional_services", list(draft.optional_services or [])) + _put(data, "reason_for_name_change", draft.name_change_reason) + _put(data, "reason_for_change", draft.name_change_reason) + + for party in FilingParty.objects.filter(draft=draft): + spec = _PARTY_SPECS.get(party.role) + if spec is None: + continue + for model_field, wire_key in spec.items(): + _put(data, wire_key, getattr(party, model_field)) + if party.role == "petitioner" and party.party_type: + for key in _PETITIONER_PARTY_TYPE_KEYS: + _put(data, key, party.party_type) + + return data + + +def _lead_config(data: dict[str, Any]) -> dict[str, Any]: + config = { + "filing_type": data.get("lead_filing_type"), + "filing_type_name": data.get("lead_filing_type_name"), + "document_type": data.get("lead_document_type"), + "document_type_name": data.get("lead_document_type_name"), + "filing_component": data.get("lead_filing_component"), + "filing_component_name": data.get("lead_filing_component_name"), + "cc_email": data.get("lead_cc_email"), + } + return {key: value for key, value in config.items() if value not in (None, "")} + + +def _positive_int(value: Any) -> int | None: + try: + number = int(value) + except (TypeError, ValueError): + return None + return number if number >= 0 else None + + +def _apply_document(doc: FilingDocument, file_obj: dict[str, Any], config: dict[str, Any]) -> None: + if "name" in file_obj: + doc.name = _as_str(file_obj.get("name")) + if not doc.original_filename: + doc.original_filename = doc.name + if "url" in file_obj: + doc.public_url = _as_str(file_obj.get("url")) + if "s3_key" in file_obj: + doc.s3_key = _as_str(file_obj.get("s3_key")) + if "type" in file_obj or "content_type" in file_obj: + doc.content_type = _as_str(file_obj.get("type") or file_obj.get("content_type")) + if "size" in file_obj: + doc.size = _positive_int(file_obj.get("size")) + + config_fields = { + "filing_type_code": "filing_type", + "filing_type_name": "filing_type_name", + "document_type_code": "document_type", + "document_type_name": "document_type_name", + "filing_component_code": "filing_component", + "filing_component_name": "filing_component_name", + "courtesy_copy_email": "cc_email", + } + for model_field, config_key in config_fields.items(): + if config_key in config: + setattr(doc, model_field, _as_str(config.get(config_key))) + + +def _upsert_document( + draft: FilingDraft, + role: str, + sort_order: int, + file_obj: dict[str, Any], + config: dict[str, Any], +) -> None: + doc, _created = FilingDocument.objects.get_or_create(draft=draft, role=role, sort_order=sort_order) + _apply_document(doc, file_obj, config) + doc.save() + + +@transaction.atomic +def write_upload_data( + draft: FilingDraft, + upload_data: dict[str, Any] | None, + *, + current_step: WorkflowStepKey | str | None = None, +) -> FilingDraft: + """Persist a (possibly partial) upload_data blob into FilingDocument rows.""" + + data = dict(upload_data or {}) + update_fields: list[str] = [] + + if "guesses" in data: + guesses = data.get("guesses") or {} + if draft.extracted_guesses != guesses: + draft.extracted_guesses = guesses + update_fields.append("extracted_guesses") + + files = data.get("files") or {} + lead_config = _lead_config(data) + + if "lead" in files: + _upsert_document(draft, FilingDocument.Role.LEAD, 0, files.get("lead") or {}, lead_config) + elif lead_config: + lead = FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.LEAD).first() + if lead is not None: + _apply_document(lead, {}, lead_config) + lead.save() + + if "supporting" in files: + supporting_files = files.get("supporting") or [] + supporting_configs = data.get("supporting_documents") or [] + FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING).delete() + for index, file_obj in enumerate(supporting_files): + config = supporting_configs[index] if index < len(supporting_configs) else {} + _upsert_document(draft, FilingDocument.Role.SUPPORTING, index, file_obj or {}, config or {}) + + if current_step is not None and draft.current_step != str(current_step): + draft.current_step = str(current_step) + update_fields.append("current_step") + + if update_fields: + draft.save(update_fields=sorted({*update_fields, "updated_at"})) + return draft + + +def _document_file(doc: FilingDocument) -> dict[str, Any]: + file_obj: dict[str, Any] = {} + _put(file_obj, "name", doc.name) + _put(file_obj, "url", doc.public_url) + _put(file_obj, "s3_key", doc.s3_key) + _put(file_obj, "type", doc.content_type) + _put(file_obj, "size", doc.size) + return file_obj + + +def _document_config(doc: FilingDocument) -> dict[str, Any]: + config: dict[str, Any] = {} + _put(config, "filing_type", doc.filing_type_code) + _put(config, "filing_type_name", doc.filing_type_name) + _put(config, "document_type", doc.document_type_code) + _put(config, "document_type_name", doc.document_type_name) + _put(config, "filing_component", doc.filing_component_code) + _put(config, "filing_component_name", doc.filing_component_name) + _put(config, "cc_email", doc.courtesy_copy_email) + return config + + +def read_upload_data(draft: FilingDraft | None) -> dict[str, Any]: + """Reconstruct the upload_data blob the browser reads from document rows.""" + + if draft is None: + return {} + + lead = FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.LEAD).first() + supporting = list( + FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING).order_by("sort_order") + ) + + files: dict[str, Any] = {} + if lead is not None: + files["lead"] = _document_file(lead) + supporting_files = [_document_file(doc) for doc in supporting] + if supporting_files: + files["supporting"] = supporting_files + + data: dict[str, Any] = {} + if files: + data["files"] = files + _put(data, "guesses", draft.extracted_guesses) + + if lead is not None: + _put(data, "lead_filing_type", lead.filing_type_code) + _put(data, "lead_filing_type_name", lead.filing_type_name) + _put(data, "lead_document_type", lead.document_type_code) + _put(data, "lead_document_type_name", lead.document_type_name) + _put(data, "lead_filing_component", lead.filing_component_code) + _put(data, "lead_filing_component_name", lead.filing_component_name) + _put(data, "lead_cc_email", lead.courtesy_copy_email) + + supporting_configs = [_document_config(doc) for doc in supporting] + if any(supporting_configs): + data["supporting_documents"] = supporting_configs + + return data + + def draft_snapshot(draft: FilingDraft | None) -> dict[str, Any] | None: """Return the stable, JSON-safe representation exposed to the UI.""" diff --git a/efile_app/efile/services/legacy_draft_bridge.py b/efile_app/efile/services/legacy_draft_bridge.py deleted file mode 100644 index 1ef1783..0000000 --- a/efile_app/efile/services/legacy_draft_bridge.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Translate legacy session blobs into the durable filing draft aggregate. - -Only the session-backed endpoints should import this module. It can be removed -once those endpoints write typed draft fields directly. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import Any - -from django.db import transaction - -from efile.models import FilingDocument, FilingDraft, FilingParty -from efile.services.current_drafts import get_current_draft -from efile.workflow import WorkflowStepKey - -CASE_FIELD_MAPPINGS: dict[str, tuple[str, ...]] = { - "existing_case": ("existing_case",), - "court_code": ("court_code", "court"), - "court_name": ("court_name",), - "case_category_code": ("case_category_code", "case_category"), - "case_category_name": ("case_category_name",), - "case_type_code": ("case_type_code", "case_type"), - "case_type_name": ("case_type_name",), - "case_subtype_code": ("case_subtype_code", "case_subtype"), - "case_subtype_name": ("case_subtype_name",), - "filing_type_code": ("filing_type_code", "filing_type", "filing_type_id"), - "filing_type_name": ("filing_type_name",), - "document_type_code": ("document_type_code", "document_type"), - "document_type_name": ("document_type_name",), - "previous_case_id": ("previous_case_id", "case_tracking_id"), - "docket_number": ("docket_number", "case_docket_id"), - "selected_payment_account_id": ( - "selected_payment_account_id", - "selected_payment_account", - "payment_account_id", - ), - "selected_payment_account_name": ("selected_payment_account_name", "payment_account_name"), -} - -_MISSING = object() - - -def _first_present(data: Mapping[str, Any], *keys: str) -> Any: - for key in keys: - if key in data: - return data[key] - return _MISSING - - -def _first_value(data: Mapping[str, Any], *keys: str) -> Any: - for key in keys: - value = data.get(key) - if value not in (None, ""): - return value - return "" - - -def _as_string(value: Any) -> str: - return "" if value in (None, "") else str(value) - - -@transaction.atomic -def update_draft_from_case_data( - draft: FilingDraft, - case_data: Mapping[str, Any] | None, - *, - current_step: WorkflowStepKey | str | None = None, -) -> FilingDraft: - """Mirror one complete legacy ``case_data`` blob into ``draft``.""" - - data = dict(case_data or {}) - update_fields = [] - - for draft_field, source_keys in CASE_FIELD_MAPPINGS.items(): - source_value = _first_present(data, *source_keys) - if source_value is _MISSING: - continue - value = _as_string(source_value) - if getattr(draft, draft_field) != value: - setattr(draft, draft_field, value) - update_fields.append(draft_field) - - if "optional_services" in data: - optional_services = data.get("optional_services") or [] - if draft.optional_services != optional_services: - draft.optional_services = optional_services - update_fields.append("optional_services") - - if draft.extra_case_data != data: - draft.extra_case_data = data - update_fields.append("extra_case_data") - - if current_step is not None and draft.current_step != str(current_step): - draft.current_step = str(current_step) - update_fields.append("current_step") - - if update_fields: - draft.save(update_fields=sorted({*update_fields, "updated_at"})) - - _sync_parties_from_case_data(draft, data) - return draft - - -@transaction.atomic -def sync_documents_from_upload_data( - draft: FilingDraft, - upload_data: Mapping[str, Any] | None, - *, - current_step: WorkflowStepKey | str | None = None, -) -> FilingDraft: - """Mirror one complete legacy ``upload_data`` blob into ``draft``.""" - - data = dict(upload_data or {}) - files = dict(data.get("files") or {}) - lead_document = files.get("lead") - if lead_document: - _upsert_document(draft, FilingDocument.Role.LEAD, lead_document, data, sort_order=0) - else: - FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.LEAD).delete() - - supporting_documents = files.get("supporting") or [] - supporting_configs = data.get("supporting_documents") or [] - kept_orders = [] - for index, document in enumerate(supporting_documents): - config = supporting_configs[index] if index < len(supporting_configs) else {} - _upsert_document(draft, FilingDocument.Role.SUPPORTING, document, config, sort_order=index) - kept_orders.append(index) - - supporting = FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.SUPPORTING) - if kept_orders: - supporting.exclude(sort_order__in=kept_orders).delete() - else: - supporting.delete() - - guesses = data.get("guesses") or {} - update_fields = [] - if draft.extracted_guesses != guesses: - draft.extracted_guesses = guesses - update_fields.append("extracted_guesses") - if current_step is not None and draft.current_step != str(current_step): - draft.current_step = str(current_step) - update_fields.append("current_step") - draft.save(update_fields=[*update_fields, "updated_at"]) - - return draft - - -def sync_current_draft_case_data(request, case_data: Mapping[str, Any]) -> FilingDraft | None: - """Compatibility hook for a session endpoint that just saved case data.""" - - jurisdiction = _as_string( - _first_value(case_data, "jurisdiction_id", "jurisdiction") or request.session.get("jurisdiction") - ) - draft = get_current_draft(request, jurisdiction=jurisdiction or None, resume_latest=False) - if draft is not None: - update_draft_from_case_data(draft, case_data) - return draft - - -def sync_current_draft_upload_data(request, upload_data: Mapping[str, Any]) -> FilingDraft | None: - """Compatibility hook for a session endpoint that just saved upload data.""" - - jurisdiction = _as_string(request.session.get("jurisdiction")) - draft = get_current_draft(request, jurisdiction=jurisdiction or None, resume_latest=False) - if draft is not None: - sync_documents_from_upload_data(draft, upload_data) - return draft - - -def _sync_parties_from_case_data(draft: FilingDraft, case_data: Mapping[str, Any]) -> None: - petitioner = { - "party_type": _as_string( - _first_value(case_data, "petitioner_party_type", "party_type", "determined_party_type") - ), - "first_name": _as_string(_first_value(case_data, "petitioner_first_name", "first_name")), - "middle_name": _as_string(_first_value(case_data, "petitioner_middle_name", "middle_name")), - "last_name": _as_string(_first_value(case_data, "petitioner_last_name", "last_name")), - "suffix": _as_string(_first_value(case_data, "petitioner_suffix", "suffix")), - "email": _as_string(_first_value(case_data, "petitioner_email", "email")), - "phone": _as_string(_first_value(case_data, "petitioner_phone", "phone")), - "address_line_1": _as_string(_first_value(case_data, "petitioner_address", "address", "address_line_1")), - "address_line_2": _as_string(_first_value(case_data, "petitioner_address_line_2", "address_line2")), - "city": _as_string(_first_value(case_data, "petitioner_city", "city")), - "state": _as_string(_first_value(case_data, "petitioner_state", "state")), - "zip_code": _as_string(_first_value(case_data, "petitioner_zip", "zip", "zip_code")), - "metadata": {key: value for key, value in case_data.items() if key.startswith("petitioner_")}, - } - _upsert_or_delete_party(draft, "petitioner", petitioner) - - name_sought = { - "party_type": _as_string(_first_value(case_data, "new_name_party_type")), - "first_name": _as_string(_first_value(case_data, "new_first_name")), - "middle_name": _as_string(_first_value(case_data, "new_middle_name")), - "last_name": _as_string(_first_value(case_data, "new_last_name")), - "suffix": _as_string(_first_value(case_data, "new_suffix")), - "metadata": { - key: value for key, value in case_data.items() if key.startswith("new_") or key.startswith("reason_") - }, - } - _upsert_or_delete_party(draft, "name_sought", name_sought) - - -def _upsert_or_delete_party(draft: FilingDraft, role: str, values: dict[str, Any]) -> None: - meaningful_fields = set(values) - {"metadata"} - if any(values[field] for field in meaningful_fields): - FilingParty.objects.update_or_create(draft=draft, role=role, sort_order=0, defaults=values) - else: - FilingParty.objects.filter(draft=draft, role=role, sort_order=0).delete() - - -def _upsert_document( - draft: FilingDraft, - role: str, - document: Mapping[str, Any], - config: Mapping[str, Any], - *, - sort_order: int, -) -> FilingDocument: - document_data = dict(document or {}) - config_data = dict(config or {}) - defaults = { - "name": _as_string(_first_value(document_data, "name", "filename", "original_filename")), - "original_filename": _as_string(_first_value(document_data, "original_filename", "filename", "name")), - "size": _positive_int(_first_value(document_data, "size", "file_size")), - "content_type": _as_string(_first_value(document_data, "content_type", "mime_type", "type")), - "s3_key": _as_string(_first_value(document_data, "s3_key", "key")), - "public_url": _as_string(_first_value(document_data, "url", "s3_url", "file_url", "download_url")), - "filing_type_code": _as_string( - _first_value(config_data, "filing_type", "lead_filing_type", "filing_type_code") - ), - "filing_type_name": _as_string(_first_value(config_data, "filing_type_name", "lead_filing_type_name")), - "document_type_code": _as_string( - _first_value(config_data, "document_type", "lead_document_type", "document_type_code") - ), - "document_type_name": _as_string(_first_value(config_data, "document_type_name", "lead_document_type_name")), - "filing_component_code": _as_string( - _first_value(config_data, "filing_component", "lead_filing_component", "filing_component_code") - ), - "filing_component_name": _as_string( - _first_value(config_data, "filing_component_name", "lead_filing_component_name") - ), - "courtesy_copy_email": _as_string(_first_value(config_data, "cc_email", "lead_cc_email")), - "metadata": {"file": document_data, "config": config_data}, - } - document, _created = FilingDocument.objects.update_or_create( - draft=draft, - role=role, - sort_order=sort_order, - defaults=defaults, - ) - return document - - -def _positive_int(value: Any) -> int | None: - if value in (None, ""): - return None - try: - value = int(value) - except (TypeError, ValueError): - return None - return value if value >= 0 else None diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index 50ed8c6..a99d4b7 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -5,37 +5,39 @@ from efile.models import FilingDocument, FilingDraft, FilingParty from efile.services.current_drafts import CURRENT_DRAFT_SESSION_KEY, get_current_draft -from efile.services.drafts import draft_snapshot -from efile.services.legacy_draft_bridge import sync_documents_from_upload_data, update_draft_from_case_data +from efile.services.drafts import ( + draft_snapshot, + read_case_data, + read_upload_data, + write_case_data, + write_upload_data, +) from efile.workflow import WorkflowStepKey, get_workflow_step_choices @pytest.mark.django_db -def test_update_draft_from_case_data_normalizes_known_fields(django_user_model): +def test_write_case_data_normalizes_known_fields(django_user_model): user = django_user_model.objects.create_user(username="draft-owner", tyler_jurisdiction="illinois") draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") case_data = { "court": "cook:cd", "court_name": "Cook County Circuit Court", "case_category": "MR", - "case_category_name": "Miscellaneous Remedy", "case_type": "Name Change", - "case_type_name": "Change of Name", "filing_type": "motion", - "filing_type_name": "Motion", "document_type": "petition", - "document_type_name": "Petition", "selected_payment_account": "pay-123", "selected_payment_account_name": "Card ending in 4242", "optional_services": ["certified_copy"], "petitioner_first_name": "Ada", "petitioner_last_name": "Lovelace", "petitioner_email": "ada@example.com", + "petitioner_party_type": "PET", "new_first_name": "Augusta Ada", "new_last_name": "Lovelace", } - update_draft_from_case_data(draft, case_data, current_step=WorkflowStepKey.CASE_INFORMATION) + write_case_data(draft, case_data, current_step=WorkflowStepKey.CASE_INFORMATION) draft.refresh_from_db() assert draft.current_step == WorkflowStepKey.CASE_INFORMATION @@ -46,19 +48,62 @@ def test_update_draft_from_case_data_normalizes_known_fields(django_user_model): assert draft.document_type_code == "petition" assert draft.selected_payment_account_id == "pay-123" assert draft.optional_services == ["certified_copy"] - assert draft.extra_case_data["petitioner_first_name"] == "Ada" petitioner = FilingParty.objects.get(draft=draft, role="petitioner") assert petitioner.first_name == "Ada" assert petitioner.last_name == "Lovelace" assert petitioner.email == "ada@example.com" + assert petitioner.party_type == "PET" - name_sought = FilingParty.objects.get(draft=draft, role="name_sought") - assert name_sought.first_name == "Augusta Ada" + new_name = FilingParty.objects.get(draft=draft, role="new_name") + assert new_name.first_name == "Augusta Ada" @pytest.mark.django_db -def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(django_user_model): +def test_write_case_data_does_not_persist_unknown_keys(django_user_model): + """Only modelled fields survive; there is no catch-all blob for random keys.""" + user = django_user_model.objects.create_user(username="typed-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + + write_case_data(draft, {"court": "cook:cd", "totally_made_up_field": "should be dropped"}) + draft.refresh_from_db() + + assert draft.court_code == "cook:cd" + assert "totally_made_up_field" not in read_case_data(draft) + + +@pytest.mark.django_db +def test_case_data_round_trips_through_the_model(django_user_model): + user = django_user_model.objects.create_user(username="round-trip-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + case_data = { + "court": "cook:cd", + "case_category": "MR", + "case_type": "Name Change", + "filing_type": "motion", + "document_type": "petition", + "existing_case": "no", + "petitioner_first_name": "Ada", + "petitioner_party_type": "PET", + "other_first_name": "Grace", + "other_address_city": "Chicago", + } + + write_case_data(draft, case_data) + blob = read_case_data(draft) + + assert blob["court"] == "cook:cd" + assert blob["case_type"] == "Name Change" + assert blob["petitioner_first_name"] == "Ada" + # petitioner party type is echoed under all three legacy aliases the browser reads + assert blob["party_type"] == "PET" + assert blob["determined_party_type"] == "PET" + assert blob["other_first_name"] == "Grace" + assert blob["other_address_city"] == "Chicago" + + +@pytest.mark.django_db +def test_write_upload_data_creates_lead_and_supporting_documents(django_user_model): user = django_user_model.objects.create_user(username="document-owner", tyler_jurisdiction="illinois") draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") upload_data = { @@ -68,14 +113,14 @@ def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(d "url": "https://example.com/petition.pdf", "s3_key": "drafts/petition.pdf", "size": 1234, - "content_type": "application/pdf", + "type": "application/pdf", }, "supporting": [ { "name": "order.pdf", "url": "https://example.com/order.pdf", "size": 4321, - "content_type": "application/pdf", + "type": "application/pdf", } ], }, @@ -93,7 +138,7 @@ def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(d ], } - sync_documents_from_upload_data(draft, upload_data, current_step=WorkflowStepKey.DOCUMENTS) + write_upload_data(draft, upload_data, current_step=WorkflowStepKey.DOCUMENTS) draft.refresh_from_db() assert draft.current_step == WorkflowStepKey.DOCUMENTS @@ -102,6 +147,7 @@ def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(d lead = FilingDocument.objects.get(draft=draft, role=FilingDocument.Role.LEAD) assert lead.name == "petition.pdf" assert lead.s3_key == "drafts/petition.pdf" + assert lead.content_type == "application/pdf" assert lead.filing_type_code == "efile" supporting = FilingDocument.objects.get(draft=draft, role=FilingDocument.Role.SUPPORTING) @@ -110,6 +156,41 @@ def test_sync_documents_from_upload_data_creates_lead_and_supporting_documents(d assert supporting.courtesy_copy_email == "copy@example.com" +@pytest.mark.django_db +def test_upload_data_round_trips_through_the_model(django_user_model): + user = django_user_model.objects.create_user(username="upload-round-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + write_upload_data( + draft, + { + "files": {"lead": {"name": "petition.pdf", "url": "https://example.com/petition.pdf"}}, + "lead_filing_type": "efile", + "guesses": {"court": "Cook County"}, + }, + ) + + blob = read_upload_data(draft) + + assert blob["files"]["lead"]["name"] == "petition.pdf" + assert blob["files"]["lead"]["url"] == "https://example.com/petition.pdf" + assert blob["lead_filing_type"] == "efile" + assert blob["guesses"] == {"court": "Cook County"} + + +@pytest.mark.django_db +def test_supporting_documents_are_replaced_wholesale(django_user_model): + user = django_user_model.objects.create_user(username="supporting-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + write_upload_data(draft, {"files": {"supporting": [{"name": "first.pdf"}, {"name": "second.pdf"}]}}) + assert draft.documents.filter(role=FilingDocument.Role.SUPPORTING).count() == 2 + + write_upload_data(draft, {"files": {"supporting": [{"name": "only.pdf"}]}}) + + supporting = draft.documents.filter(role=FilingDocument.Role.SUPPORTING) + assert supporting.count() == 1 + assert supporting.first().name == "only.pdf" + + @pytest.mark.django_db def test_draft_snapshot_is_json_serializable(django_user_model): user = django_user_model.objects.create_user(username="snapshot-owner", tyler_jurisdiction="illinois") @@ -184,8 +265,8 @@ def test_current_draft_does_not_cross_jurisdictions(client, django_user_model): @pytest.mark.django_db -def test_legacy_case_endpoint_mirrors_into_current_draft(client, django_user_model): - user = django_user_model.objects.create_user(username="bridge-user", tyler_jurisdiction="illinois") +def test_save_case_endpoint_persists_into_current_draft(client, django_user_model): + user = django_user_model.objects.create_user(username="endpoint-user", tyler_jurisdiction="illinois") client.force_login(user) client.post( reverse("create_draft", kwargs={"jurisdiction": "illinois"}), @@ -215,6 +296,17 @@ def test_legacy_case_endpoint_mirrors_into_current_draft(client, django_user_mod assert draft.parties.get(role="petitioner").first_name == "Ada" +@pytest.mark.django_db +def test_save_case_endpoint_requires_authentication(client): + response = client.post( + reverse("save_case_data_api"), + data={"jurisdiction": "illinois", "data": {"court": "cook:cd"}}, + content_type="application/json", + ) + + assert response.status_code == 401 + + @pytest.mark.django_db def test_model_step_choices_follow_workflow_registry(): current_step = FilingDraft._meta.get_field("current_step") @@ -223,7 +315,7 @@ def test_model_step_choices_follow_workflow_registry(): @pytest.mark.django_db -def test_legacy_partial_case_update_does_not_clear_omitted_fields(django_user_model): +def test_partial_case_update_does_not_clear_omitted_fields(django_user_model): user = django_user_model.objects.create_user(username="partial-update-user", tyler_jurisdiction="illinois") draft = FilingDraft.objects.create( user=user, @@ -232,29 +324,8 @@ def test_legacy_partial_case_update_does_not_clear_omitted_fields(django_user_mo court_name="Court name to preserve", ) - update_draft_from_case_data(draft, {"court": "new-code"}) + write_case_data(draft, {"court": "new-code"}) draft.refresh_from_db() assert draft.court_code == "new-code" assert draft.court_name == "Court name to preserve" - - -@pytest.mark.django_db -def test_legacy_upload_sync_removes_state_missing_from_complete_blob(django_user_model): - user = django_user_model.objects.create_user(username="upload-replace-user", tyler_jurisdiction="illinois") - draft = FilingDraft.objects.create( - user=user, - jurisdiction="illinois", - extracted_guesses={"court": "Old guess"}, - ) - FilingDocument.objects.create( - draft=draft, - role=FilingDocument.Role.LEAD, - name="old.pdf", - ) - - sync_documents_from_upload_data(draft, {"files": {}, "guesses": {}}) - draft.refresh_from_db() - - assert draft.extracted_guesses == {} - assert not draft.documents.exists() diff --git a/efile_app/efile/tests/tests.py b/efile_app/efile/tests/tests.py index 59d68dc..941255e 100644 --- a/efile_app/efile/tests/tests.py +++ b/efile_app/efile/tests/tests.py @@ -388,6 +388,9 @@ def authenticated_client(self, client, db): """Create an authenticated client.""" user = User.objects.create_user(username="testuser", email="test@example.com", password="testpass123") client.force_login(user) + session = client.session + session["auth_tokens"] = {"TYLER-TOKEN-ILLINOIS": "token"} + session.save() return client def test_expert_form_page_loads(self, authenticated_client): diff --git a/efile_app/efile/utils/case_data_utils.py b/efile_app/efile/utils/case_data_utils.py index de399ad..50124ca 100644 --- a/efile_app/efile/utils/case_data_utils.py +++ b/efile_app/efile/utils/case_data_utils.py @@ -1,61 +1,65 @@ -""" -Utility functions for handling case data throughout the application +"""Accessors for the current filing's case/upload data. + +The durable ``FilingDraft`` aggregate is the single source of truth. These +helpers keep their old ``request``-based signatures so every view, template, and +the submission path keep working, but they now (de)serialize the draft instead of +reading a session blob. The browser still exchanges the same flat JSON shape; +only the server-side store moved. """ import logging +from efile.services.current_drafts import ensure_current_draft, get_current_draft +from efile.services.drafts import read_case_data, read_upload_data, write_case_data, write_upload_data + logger = logging.getLogger(__name__) -def get_case_data(request): - """ - Get case data from session with safe defaults - """ - return request.session.get("case_data", {}) +def _current_jurisdiction(request): + draft = get_current_draft(request, resume_latest=False) + if draft is not None: + return draft.jurisdiction + return request.session.get("jurisdiction") -def update_case_data(request, updates): - """ - Update specific fields in the case data - """ - case_data = get_case_data(request) - case_data.update(updates) - request.session["case_data"] = case_data - request.session.modified = True - return case_data +def _resolve_writable_draft(request): + """Return the draft to write to, creating one for authenticated users only.""" + if not getattr(request.user, "is_authenticated", False): + return None + jurisdiction = _current_jurisdiction(request) + if jurisdiction: + return ensure_current_draft(request, jurisdiction) + return get_current_draft(request) -def clear_case_data(request): - """ - Clear all case data from session - """ - if "case_data" in request.session: - del request.session["case_data"] - request.session.modified = True +def get_case_data(request): + """Return the current draft serialized to the case_data blob (``{}`` if none).""" + return read_case_data(get_current_draft(request)) -def get_upload_data(request): - return request.session.get("upload_data", {}) +def update_case_data(request, updates): + """Persist a partial case_data blob onto the current draft and return the merged blob.""" + draft = _resolve_writable_draft(request) + if draft is None: + return {} + write_case_data(draft, updates) + return read_case_data(draft) -def update_upload_data(request, updates): - upload_data = get_upload_data(request) - upload_data.update(updates) - request.session["upload_data"] = upload_data - request.session.modified = True - return upload_data +def get_upload_data(request): + return read_upload_data(get_current_draft(request)) -def clear_upload_data(request): - if "upload_data" in request.session: - del request.session["upload_data"] - request.session.modified = True +def update_upload_data(request, updates): + draft = _resolve_writable_draft(request) + if draft is None: + return {} + write_upload_data(draft, updates) + return read_upload_data(draft) def get_petitioner_info(request): - """ - Get petitioner information specifically - """ + """Get petitioner information specifically.""" case_data = get_case_data(request) full_name = f"{case_data.get('petitioner_first_name', '')} {case_data.get('petitioner_last_name', '')}".strip() return { @@ -67,9 +71,7 @@ def get_petitioner_info(request): def get_name_sought_info(request): - """ - Get name sought information specifically - """ + """Get name sought information specifically.""" case_data = get_case_data(request) return { "first_name": case_data.get("new_first_name", ""), @@ -79,15 +81,13 @@ def get_name_sought_info(request): def get_case_classification(request): - """ - Get case classification information - """ + """Get case classification information.""" case_data = get_case_data(request) logger.info("Case data: %s", case_data) return { "court": case_data.get("court", ""), - "case_category": case_data.get("case_category_code", case_data.get("case_category", "")), - "case_type": case_data.get("case_type_code", case_data.get("case_type", "")), + "case_category": case_data.get("case_category", ""), + "case_type": case_data.get("case_type", ""), "filing_type": case_data.get("filing_type", ""), "document_type": case_data.get("document_type", ""), "is_name_change": "name change" in case_data.get("case_type", "").lower(), @@ -95,8 +95,6 @@ def get_case_classification(request): def get_selected_services(request): - """ - Get list of selected optional services - """ + """Get list of selected optional services.""" case_data = get_case_data(request) return case_data.get("optional_services", []) diff --git a/efile_app/efile/views/api_views.py b/efile_app/efile/views/api_views.py index 905aabf..8639438 100644 --- a/efile_app/efile/views/api_views.py +++ b/efile_app/efile/views/api_views.py @@ -1,4 +1,3 @@ -import json import logging import requests @@ -8,94 +7,29 @@ from django.views import View from django.views.decorators.csrf import ensure_csrf_cookie -from efile.services.current_drafts import get_current_draft -from efile.services.drafts import draft_snapshot -from efile.services.legacy_draft_bridge import sync_current_draft_case_data +from ..utils.case_data_utils import get_case_data logger = logging.getLogger(__name__) @method_decorator(ensure_csrf_cookie, name="dispatch") class GetCaseDataView(View): - """API endpoint to retrieve case data from session.""" + """API endpoint to return the current draft as the case_data blob.""" def get(self, request): - data = {} try: - for param_to_check in ["session_id", "jurisdiction", "existing_case", "case_data"]: - value = request.session.get(param_to_check) + case_data = get_case_data(request) + data = {"case_data": case_data} + for key in ("session_id", "jurisdiction", "existing_case"): + value = request.session.get(key) or case_data.get(key) if value: - data[param_to_check] = value - - draft = get_current_draft(request, resume_latest=False) - if draft: - data["filing_draft"] = draft_snapshot(draft) - + data[key] = value return JsonResponse({"success": True, "data": data}) except Exception: logger.exception("Error retrieving case data") return JsonResponse({"success": False, "error": "Server error occurred"}, status=500) -@method_decorator(ensure_csrf_cookie, name="dispatch") -class SaveCaseDataView(View): - """API endpoint to save case data to session.""" - - def post(self, request): - try: - data = json.loads(request.body) - - # Extract case data from form submission - case_data = { - "court": data.get("court", ""), - "case_category": data.get("case_category", ""), - "case_type": data.get("case_type", ""), - "filing_type": data.get("filing_type", ""), - "document_type": data.get("document_type", ""), - "petitioner_first_name": data.get("petitioner_first_name", ""), - "petitioner_last_name": data.get("petitioner_last_name", ""), - "petitioner_address": data.get("petitioner_address", ""), - "petitioner_party_type": data.get("petitioner_party_type", ""), # Add party type - "new_first_name": data.get("new_first_name", ""), - "new_last_name": data.get("new_last_name", ""), - "new_name_party_type": data.get("new_name_party_type", ""), # Add party type - "optional_services": data.get("optional_services", []), - } - - # Validate required fields - required_fields = ["court", "case_category", "case_type", "filing_type", "document_type"] - missing_fields = [field for field in required_fields if not case_data.get(field)] - - if missing_fields: - err_missing = f"Missing required fields: {', '.join(missing_fields)}" - return JsonResponse({"success": False, "error": err_missing}, status=400) - - # For name change cases, validate party information - if "name change" in case_data.get("case_type", "").lower(): - party_fields = ["petitioner_first_name", "petitioner_last_name", "new_first_name", "new_last_name"] - missing_party_fields = [field for field in party_fields if not case_data.get(field)] - - if missing_party_fields: - err_party = f"Missing party information for name change case: {', '.join(missing_party_fields)}" - return JsonResponse({"success": False, "error": err_party}, status=400) - - # Save to session - request.session["case_data"] = case_data - request.session.modified = True - - sync_current_draft_case_data(request, case_data) - - logger.debug(f"Saved case data to session: {case_data}") - - return JsonResponse({"success": True, "message": "Case data saved successfully"}) - - except json.JSONDecodeError: - return JsonResponse({"success": False, "error": "Invalid JSON data"}, status=400) - except Exception: - logger.exception("Error saving case data") - return JsonResponse({"success": False, "error": "Server error occurred"}, status=500) - - @method_decorator(ensure_csrf_cookie, name="dispatch") class GetFilingComponentsView(View): """API endpoint to get filing components from Suffolk LIT Lab API (proxy to avoid CORS).""" @@ -107,9 +41,9 @@ def get(self, request): jurisdiction = request.GET.get("jurisdiction") filing_type_id = request.GET.get("filing_type") - # If not provided in query, try to get from session + # If not provided in query, try to get from the current draft if not court or not filing_type_id: - case_data = request.session.get("case_data", {}) + case_data = get_case_data(request) court = court or case_data.get("court", "cook:cd") filing_type_id = filing_type_id or case_data.get("filing_type") @@ -163,5 +97,4 @@ def get(self, request): # Function-based view wrapper for easy URL mapping get_case_data_api = GetCaseDataView.as_view() -save_case_data = SaveCaseDataView.as_view() get_filing_components = GetFilingComponentsView.as_view() diff --git a/efile_app/efile/views/session_api.py b/efile_app/efile/views/session_api.py index b8bbeb7..b6109c5 100644 --- a/efile_app/efile/views/session_api.py +++ b/efile_app/efile/views/session_api.py @@ -8,15 +8,11 @@ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from efile.services.legacy_draft_bridge import ( - sync_current_draft_case_data, - sync_current_draft_upload_data, -) +from efile.services.current_drafts import clear_current_draft, get_current_draft -from ..utils.case_data_utils import clear_case_data, clear_upload_data, get_upload_data +from ..utils.case_data_utils import get_case_data, get_upload_data, update_case_data, update_upload_data from ..utils.llms import LlmError, extract_fields_from_file from ..utils.proxy_connection import get_party_type_code_from_api -from ..utils.zip_to_county_il import get_county_by_zip logger = logging.getLogger(__name__) @@ -139,99 +135,6 @@ def determine_party_type_for_existing_case(case_data): return party_code -@csrf_exempt -@require_http_methods(["POST"]) -def save_form_data_to_session(request): - """Save form data (including petitioner contact info) to Django session and derive county from zip.""" - try: - data = json.loads(request.body) - form_data = data.get("data", {}) - - # Start from existing case_data so we don't clobber other fields - case_data = request.session.get("case_data", {}) - - # Update case_data fields with provided form values (preserve existing when not provided) - case_data.update( - { - "court": form_data.get("court", case_data.get("court", "")), - "case_category": form_data.get("case_category", case_data.get("case_category", "")), - "case_type": form_data.get("case_type", case_data.get("case_type", "")), - "filing_type": form_data.get("filing_type", case_data.get("filing_type", "")), - "document_type": form_data.get("document_type", case_data.get("document_type", "")), - # simplified contact/address fields - "first_name": form_data.get("first_name", case_data.get("first_name", "")), - "last_name": form_data.get("last_name", case_data.get("last_name", "")), - "address": form_data.get("address", case_data.get("address", "")), - "address_line2": form_data.get("address_line2", case_data.get("address_line2", "")), - "city": form_data.get("city", case_data.get("city", "")), - "state": form_data.get("state", case_data.get("state", "")), - "zip": form_data.get("zip", case_data.get("zip", "")), - "email": form_data.get("email", case_data.get("email", "")), - "phone": form_data.get("phone", case_data.get("phone", "")), - # optional services and friendly names - "optional_services": form_data.get("optional_services", case_data.get("optional_services", [])), - "court_name": form_data.get("court_name", case_data.get("court_name", "")), - "case_category_name": form_data.get("case_category_name", case_data.get("case_category_name", "")), - "case_type_name": form_data.get("case_type_name", case_data.get("case_type_name", "")), - "filing_type_name": form_data.get("filing_type_name", case_data.get("filing_type_name", "")), - "document_type_name": form_data.get("document_type_name", case_data.get("document_type_name", "")), - } - ) - - # Add all dynamic fields that might be present in the form data - # This includes petitioner information, name change details, etc. - dynamic_fields = [ - "petitioner_first_name", - "petitioner_last_name", - "petitioner_address", - "petitioner_phone", - "petitioner_email", - "new_first_name", - "new_last_name", - "reason_for_change", - "minor_first_name", - "minor_last_name", - "parent_first_name", - "parent_last_name", - "guardian_first_name", - "guardian_last_name", - ] - - for field in dynamic_fields: - if field in form_data: - case_data[field] = form_data[field] - - # Also save any other fields that might be dynamically added but not in our predefined list - for key, value in form_data.items(): - if key not in case_data and value: # Only add if not already handled and has a value - case_data[key] = value - - # Try to derive county from zip code and save it - zip_code = ( - case_data.get("zip") or case_data.get("zip_code") or form_data.get("zip") or form_data.get("zip_code", "") - ) - if zip_code: - try: - county = get_county_by_zip(zip_code) - if county: - # Save simplified county key and keep petitioner_county for backward compatibility - case_data["county"] = county - case_data["petitioner_county"] = county - except Exception: - # If mapping fails, ignore and continue - pass - - # Persist to session - request.session["case_data"] = case_data - request.session.modified = True - sync_current_draft_case_data(request, case_data) - - return JsonResponse({"success": True, "message": "Case data saved to session"}) - - except Exception as e: - return JsonResponse({"success": False, "error": str(e)}, status=500) - - @csrf_exempt @require_http_methods(["POST"]) def save_upload_first_data(request): @@ -239,8 +142,13 @@ def save_upload_first_data(request): logger.debug("Received POST request to save upload data") logger.debug(f"Request body: {request.body.decode('utf-8')}") + if not request.user.is_authenticated: + return JsonResponse({"success": False, "error": "Authentication required"}, status=401) + data = json.loads(request.body) jurisdiction_id = data.get("jurisdiction_id", "default") + if data.get("jurisdiction_id"): + request.session["jurisdiction"] = jurisdiction_id upload_data = {"files": data.get("files", {})} try: @@ -274,61 +182,49 @@ def save_upload_first_data(request): upload_data["guesses"]["case type"] = found_fields.get("case type") upload_data["guesses"]["docket number"] = found_fields.get("docket number") - # Save to session - request.session["upload_data"] = upload_data - request.session.modified = True - sync_current_draft_upload_data(request, upload_data) + update_upload_data(request, upload_data) - logger.info("Successfully saved upload data to session") - return JsonResponse({"success": True, "message": "Upload data saved to session"}) + logger.info("Persisted lead upload to the current draft") + return JsonResponse({"success": True, "message": "Upload data saved"}) @csrf_exempt @require_http_methods(["POST"]) def save_upload_data_to_session(request): - """Save upload data and file information to Django session for review.""" + """Persist supporting documents and lead filing config onto the current draft.""" try: - logger.debug("Received POST request to save upload data") - logger.debug(f"Request body: {request.body.decode('utf-8')}") + if not request.user.is_authenticated: + return JsonResponse({"success": False, "error": "Authentication required"}, status=401) data = json.loads(request.body) - logger.debug("Parsed upload data") - old_upload_data = request.session["upload_data"] upload_data = { - "files": {"lead": old_upload_data["files"]["lead"], "supporting": data["files"]}, - "options": data.get("options", {}), - "guesses": old_upload_data.get("guesses", {}), + "files": {"supporting": data.get("files", [])}, + # Lead document filing information (updates the already-persisted lead doc) "lead_filing_type": data.get("lead_filing_type", ""), - # Lead document filing information "lead_filing_type_name": data.get("lead_filing_type_name", ""), "lead_document_type": data.get("lead_document_type", ""), "lead_document_type_name": data.get("lead_document_type_name", ""), "lead_filing_component": data.get("lead_filing_component", ""), "lead_filing_component_name": data.get("lead_filing_component_name", ""), - "lead_cc_email": data.get("lead_cc_email"), + "lead_cc_email": data.get("lead_cc_email", ""), # Supporting documents filing information "supporting_documents": data.get("supporting_documents", []), } - logger.debug("Processed upload data") + update_upload_data(request, upload_data) - # Save to session - request.session["upload_data"] = upload_data - request.session.modified = True - sync_current_draft_upload_data(request, upload_data) - - logger.info("Successfully saved upload data to session") - return JsonResponse({"success": True, "message": "Upload data saved to session"}) + logger.info("Persisted supporting documents to the current draft") + return JsonResponse({"success": True, "message": "Upload data saved"}) except Exception as e: - logger.exception("Error saving upload data to session") + logger.exception("Error saving upload data") return JsonResponse({"success": False, "error": str(e)}, status=500) @require_http_methods(["GET"]) def get_upload_data_from_session(request): - """Get upload data from Django session.""" + """Return the current draft's documents as the upload_data blob.""" return JsonResponse(get_upload_data(request), safe=False) @@ -342,9 +238,9 @@ def submit_final_filing(request): if not data.get("confirm_submission"): return JsonResponse({"success": False, "error": "Submission confirmation is required"}, status=400) - # Get all data from session - case_data = request.session.get("case_data", {}) - jurisdiction_id = request.session.get("jurisdiction") + # Read the filing state from the current durable draft + case_data = get_case_data(request) + jurisdiction_id = request.session.get("jurisdiction") or case_data.get("jurisdiction") upload_data = get_upload_data(request) auth_tokens = request.session.get("auth_tokens", {}) @@ -450,9 +346,11 @@ def submit_final_filing(request): if response.status_code == 200 or response.status_code == 201: response_data = response.json() - # Clear session data after successful submission - clear_case_data(request) - clear_upload_data(request) + # Finalize the draft; it drops out of the active set so the flow resets + draft = get_current_draft(request) + if draft is not None: + draft.mark_submitted(response_data) + clear_current_draft(request) return JsonResponse( { @@ -532,6 +430,9 @@ def api_save_case_data(request): API endpoint to save case data to session """ try: + if not request.user.is_authenticated: + return JsonResponse({"success": False, "error": "Authentication required"}, status=401) + data = json.loads(request.body) session_id = data.get("session_id") @@ -555,40 +456,18 @@ def api_save_case_data(request): form_data = data existing_case = data.get("existing_case") - # Save existing_case to session if provided + updates = dict(form_data) if existing_case: request.session["existing_case"] = existing_case + updates["existing_case"] = existing_case - # Always save all form data to case_data for upload view compatibility - case_data = request.session.get("case_data", {}) - case_data.update(form_data) - - # Ensure existing_case status is available in case_data - if existing_case: - case_data["existing_case"] = existing_case - - if jurisdiction: - case_data["jurisdiction_id"] = jurisdiction - - # Map case details fields to standard names for eFiling - if "case_docket_id" in form_data: - case_data["docket_number"] = form_data["case_docket_id"] - if "case_tracking_id" in form_data: - case_data["previous_case_id"] = form_data["case_tracking_id"] - - # Determine party type for existing cases - typically defendant/respondent when responding to existing case - if existing_case == "yes": - # For existing cases, we need to determine the appropriate party type - # This depends on the case type and filing type + # Determine party type for existing cases - typically defendant/respondent when responding + if existing_case == "yes" and not updates.get("party_type"): party_type = determine_party_type_for_existing_case(form_data) - if not case_data.get("party_type"): - case_data["party_type"] = party_type - if not case_data.get("petitioner_party_type"): - case_data["petitioner_party_type"] = party_type + updates["party_type"] = party_type + updates.setdefault("petitioner_party_type", party_type) - request.session["case_data"] = case_data - request.session.modified = True - sync_current_draft_case_data(request, case_data) + update_case_data(request, updates) return JsonResponse( {"success": True, "data": {"existing_case": existing_case, "saved_fields": list(form_data.keys())}} @@ -608,6 +487,9 @@ def fetch_and_save_party_type(request): Fetch party type code from Suffolk LIT Lab API and save to session """ try: + if not request.user.is_authenticated: + return JsonResponse({"success": False, "error": "Authentication required"}, status=401) + data = json.loads(request.body) court_code = data.get("court") case_type_code = data.get("case_type") @@ -619,6 +501,8 @@ def fetch_and_save_party_type(request): if not jurisdiction: return JsonResponse({"success": False, "error": "Jurisdiction is required"}, status=400) + request.session["jurisdiction"] = jurisdiction + if not court_code or not case_type_code: return JsonResponse({"success": False, "error": "Court and case_type are required"}, status=400) @@ -669,16 +553,16 @@ def fetch_and_save_party_type(request): logger.debug(f"Final party type code: {party_type_code}") - # Save to session - case_data = request.session.get("case_data", {}) - case_data["determined_party_type"] = party_type_code - case_data["party_type"] = party_type_code - case_data["petitioner_party_type"] = party_type_code - request.session["case_data"] = case_data - request.session.modified = True - sync_current_draft_case_data(request, case_data) + update_case_data( + request, + { + "determined_party_type": party_type_code, + "party_type": party_type_code, + "petitioner_party_type": party_type_code, + }, + ) - logger.debug(f"Saved party type to session: {party_type_code}") + logger.debug(f"Saved party type to draft: {party_type_code}") return JsonResponse({"success": True, "party_type": party_type_code}) @@ -695,24 +579,25 @@ def save_party_type_to_session(request): Save party type code to session after it's been fetched from Suffolk API on frontend """ try: + if not request.user.is_authenticated: + return JsonResponse({"success": False, "error": "Authentication required"}, status=401) + data = json.loads(request.body) party_type = data.get("party_type") - party_types_available = data.get("party_types_available", []) if not party_type: return JsonResponse({"success": False, "error": "Party type is required"}, status=400) - # Save to session - case_data = request.session.get("case_data", {}) - case_data["determined_party_type"] = party_type - case_data["party_type"] = party_type - case_data["petitioner_party_type"] = party_type - case_data["available_party_types"] = party_types_available - request.session["case_data"] = case_data - request.session.modified = True - sync_current_draft_case_data(request, case_data) - - logger.debug(f"Saved party type to session: {party_type}") + update_case_data( + request, + { + "determined_party_type": party_type, + "party_type": party_type, + "petitioner_party_type": party_type, + }, + ) + + logger.debug(f"Saved party type to draft: {party_type}") return JsonResponse({"success": True, "party_type": party_type}) From 6c19d171feabdaa8769f6b69601c1d783f8b5180 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Tue, 21 Jul 2026 18:30:50 -0400 Subject: [PATCH 33/47] Stop caching server-owned state in the browser The localStorage cache (24h TTL, keyed only by URL) was mirroring per-user and per-draft state: case/upload data, profile, payment accounts, the Tyler auth token, and fee quotes. Now that the model is the source of truth, those stale copies could survive a submit/reset or leak on a shared browser -- and a cached auth token is its own hazard. Route all of them through the non-caching request path so they always hit the server. The cache is now reserved for static, shareable reference data (dropdowns, form config), which is what it's good for. Remove the now-dead cachedPost helper. Add js-tests/api-utils.test.js (node --test, no new deps; kept out of ./tests so Playwright ignores it) to lock in the rule: reference GETs cache, draft/profile/payment reads do not. Run with `npm run test:unit`. Co-Authored-By: Claude Opus 4.8 --- efile_app/efile/static/js/api-utils.js | 54 ++++---------------- efile_app/efile/static/js/payment.js | 8 +-- efile_app/efile/static/js/review.js | 4 +- efile_app/js-tests/api-utils.test.js | 71 ++++++++++++++++++++++++++ efile_app/package.json | 3 +- 5 files changed, 88 insertions(+), 52 deletions(-) create mode 100644 efile_app/js-tests/api-utils.test.js diff --git a/efile_app/efile/static/js/api-utils.js b/efile_app/efile/static/js/api-utils.js index a8c2247..79f3844 100644 --- a/efile_app/efile/static/js/api-utils.js +++ b/efile_app/efile/static/js/api-utils.js @@ -272,31 +272,6 @@ class ApiUtils { return response; } - async cachedPost(endpoint, body) { - // Check cache first - const cachedResponse = this.getCachedResponse(endpoint, body); - if (cachedResponse !== null) { - return cachedResponse; - } - - let headers = { - "Content-Type": "application/json", - "X-CSRFToken": apiUtils.getCSRFToken(), - }; - - // Make API request if not cached - const response = await this.makeRequest(endpoint, { - method: "POST", - headers, - data: body, - }); - - // Cache the response - this.setCachedResponse(endpoint, body, response); - - return response; - - } async post(endpoint, data = {}, params = {}) { return this.makeRequest(endpoint, { @@ -329,42 +304,31 @@ class ApiUtils { }); } + // Draft state (case/upload data) is owned by the server-side FilingDraft + // model. It must never be cached in localStorage: a stale blob could survive + // a submit/reset and leak into the next filing. These always hit the server. async getCaseData() { - return this.get("/api/get-case-data", {}, true); + return this.fetchJSON("/api/get-case-data", "GET"); } async saveCaseData(body) { - let resp = this.fetchJSON("/api/save-case-data/", "POST", {}, body) - const cacheKey = this.getCacheKey("/api/get-case-data", {}); - this.clearCache(cacheKey) - return resp; + return this.fetchJSON("/api/save-case-data/", "POST", {}, body); } - /** Calling `get-party-types` will set certain values on the case. For now, just nuke the cache.. */ async getPartyTypes(params) { - let resp = this.fetchJSON("/api/get-party-types", "GET", params); - const cacheKey = this.getCacheKey("/api/get-case-data", {}); - this.clearCache(cacheKey) - return resp; + return this.fetchJSON("/api/get-party-types", "GET", params); } async getUploadData() { - return this.get("/api/get-upload-data", {}, true); + return this.fetchJSON("/api/get-upload-data", "GET"); } async saveUploadData(body) { - return this._saveUpload("/api/save-upload-data/", body); + return this.fetchJSON("/api/save-upload-data/", "POST", {}, body); } async saveFirstUploadData(body) { - return this._saveUpload("/api/save-upload-data-first/", body); - } - - async _saveUpload(endpoint, body) { - let resp = this.fetchJSON(endpoint, "POST", {}, body); - const cacheKey = this.getCacheKey("/api/get-upload-data", {}); - this.clearCache(cacheKey) - return resp; + return this.fetchJSON("/api/save-upload-data-first/", "POST", {}, body); } // Cache management methods diff --git a/efile_app/efile/static/js/payment.js b/efile_app/efile/static/js/payment.js index 1c120d3..325facc 100644 --- a/efile_app/efile/static/js/payment.js +++ b/efile_app/efile/static/js/payment.js @@ -122,7 +122,7 @@ const APIHandlers = { const params = { jurisdiction: apiUtils.getCurrentJurisdiction() }; - const result = await apiUtils.get(CONFIG.URLS.PAYMENT_ACCOUNTS, params); + const result = await apiUtils.fetchJSON(CONFIG.URLS.PAYMENT_ACCOUNTS, "GET", params); if (result?.success && result.data) { UIUpdater.updatePaymentMethodsSection(result.data); @@ -211,7 +211,7 @@ const PaymentHandler = { const params = new URLSearchParams({ jurisdiction: apiUtils.getCurrentJurisdiction(), }); - const authData = await apiUtils.get(CONFIG.URLS.TYLER_TOKEN, params); + const authData = await apiUtils.fetchJSON(CONFIG.URLS.TYLER_TOKEN, "GET", params); if (!authData?.success || !authData.data?.tyler_token) { Messages.showError("Authentication failed. Please try again."); @@ -317,7 +317,7 @@ const FilingHandler = { const params = { jurisdiction: apiUtils.getCurrentJurisdiction() }; - const data = await apiUtils.get(CONFIG.URLS.PROFILE, params, true); + const data = await apiUtils.fetchJSON(CONFIG.URLS.PROFILE, "GET", params); if (data?.success && data.data) { const profile = data.data; @@ -365,7 +365,7 @@ const FilingHandler = { const efilingData = this.buildEFilingData(userData, caseData, uploadData, paymentAccountID); - return await apiUtils.cachedPost(CONFIG.URLS.QUERY_FEES, { + return await apiUtils.post(CONFIG.URLS.QUERY_FEES, { efile_data: efilingData, confirm_submission: true, payment_account_id: paymentAccountID diff --git a/efile_app/efile/static/js/review.js b/efile_app/efile/static/js/review.js index 714c9ed..55a8022 100644 --- a/efile_app/efile/static/js/review.js +++ b/efile_app/efile/static/js/review.js @@ -258,7 +258,7 @@ const APIHandlers = { const params = { jurisdiction: apiUtils.getCurrentJurisdiction() }; - const data = await apiUtils.get(CONFIG.URLS.PROFILE, params); + const data = await apiUtils.fetchJSON(CONFIG.URLS.PROFILE, "GET", params); if (data?.success && data.data) { const profile = data.data; @@ -477,7 +477,7 @@ const FilingHandler = { const efilingData = this.buildEFilingData(userData, caseData, uploadData, paymentAccountID); - return await apiUtils.cachedPost(CONFIG.URLS.QUERY_FEES, { + return await apiUtils.post(CONFIG.URLS.QUERY_FEES, { efile_data: efilingData, confirm_submission: true, payment_account_id: paymentAccountID diff --git a/efile_app/js-tests/api-utils.test.js b/efile_app/js-tests/api-utils.test.js new file mode 100644 index 0000000..f48522d --- /dev/null +++ b/efile_app/js-tests/api-utils.test.js @@ -0,0 +1,71 @@ +/** + * Unit tests for ApiUtils caching behavior. + * + * Runs on Node's built-in test runner (`node --test`) with no extra deps. + * Kept out of ./tests so Playwright's E2E runner does not pick it up. + * + * The rule these tests lock in: the localStorage cache is only for static, + * shareable reference data (dropdowns, form config). Per-user / per-draft + * state (case data, uploads, profile, payment accounts, tokens, fees) is + * server-owned and must always be fetched fresh. + */ + +const test = require("node:test"); +const assert = require("node:assert"); + +// ApiUtils constructs a singleton and touches window/document/localStorage at +// module load, so stub those globals before requiring it. +const store = new Map(); +globalThis.localStorage = { + getItem: (key) => (store.has(key) ? store.get(key) : null), + setItem: (key, value) => store.set(key, String(value)), + removeItem: (key) => store.delete(key), +}; +globalThis.window = { location: { origin: "http://localhost" } }; +globalThis.document = { querySelector: () => ({ value: "test-csrf-token" }), cookie: "" }; + +const { ApiUtils } = require("../efile/static/js/api-utils.js"); + +function makeClient() { + const client = new ApiUtils(); + client.clearAllCache(); + let calls = 0; + // Replace the network layer so we can count real round trips. + client.makeRequest = async (endpoint) => { + calls += 1; + return { success: true, endpoint, call: calls }; + }; + return { client, calls: () => calls }; +} + +test("reference-data GETs are cached: repeated reads hit the network once", async () => { + const { client, calls } = makeClient(); + await client.get("/api/dropdowns/courts", { jurisdiction: "illinois" }); + await client.get("/api/dropdowns/courts", { jurisdiction: "illinois" }); + assert.strictEqual(calls(), 1); +}); + +test("draft state (case + upload data) is never cached", async () => { + const { client, calls } = makeClient(); + await client.getCaseData(); + await client.getCaseData(); + await client.getUploadData(); + await client.getUploadData(); + assert.strictEqual(calls(), 4); +}); + +test("per-user reads via fetchJSON (profile, payment accounts, token) are not cached", async () => { + const { client, calls } = makeClient(); + await client.fetchJSON("/api/payment-accounts", "GET", { jurisdiction: "illinois" }); + await client.fetchJSON("/api/payment-accounts", "GET", { jurisdiction: "illinois" }); + assert.strictEqual(calls(), 2); +}); + +test("saving case data does not populate the read cache", async () => { + const { client, calls } = makeClient(); + await client.saveCaseData({ court: "cook:cd" }); + await client.getCaseData(); + await client.getCaseData(); + // 1 save + 2 uncached reads + assert.strictEqual(calls(), 3); +}); diff --git a/efile_app/package.json b/efile_app/package.json index c9e1c81..607ef2c 100644 --- a/efile_app/package.json +++ b/efile_app/package.json @@ -6,7 +6,8 @@ "test": "tests" }, "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "test:unit": "node --test js-tests/*.test.js" }, "keywords": [], "author": "", From 659740fa57d98ab890319a65d232fff3d7f70e14 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Tue, 21 Jul 2026 18:50:29 -0400 Subject: [PATCH 34/47] Formatting --- efile_app/js-tests/api-utils.test.js | 70 ++++++++++++++++++++++------ efile_app/uv.lock | 4 -- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/efile_app/js-tests/api-utils.test.js b/efile_app/js-tests/api-utils.test.js index f48522d..258eec3 100644 --- a/efile_app/js-tests/api-utils.test.js +++ b/efile_app/js-tests/api-utils.test.js @@ -21,10 +21,21 @@ globalThis.localStorage = { setItem: (key, value) => store.set(key, String(value)), removeItem: (key) => store.delete(key), }; -globalThis.window = { location: { origin: "http://localhost" } }; -globalThis.document = { querySelector: () => ({ value: "test-csrf-token" }), cookie: "" }; +globalThis.window = { + location: { + origin: "http://localhost" + } +}; +globalThis.document = { + querySelector: () => ({ + value: "test-csrf-token" + }), + cookie: "" +}; -const { ApiUtils } = require("../efile/static/js/api-utils.js"); +const { + ApiUtils +} = require("../efile/static/js/api-utils.js"); function makeClient() { const client = new ApiUtils(); @@ -33,20 +44,37 @@ function makeClient() { // Replace the network layer so we can count real round trips. client.makeRequest = async (endpoint) => { calls += 1; - return { success: true, endpoint, call: calls }; + return { + success: true, + endpoint, + call: calls + }; + }; + return { + client, + calls: () => calls }; - return { client, calls: () => calls }; } test("reference-data GETs are cached: repeated reads hit the network once", async () => { - const { client, calls } = makeClient(); - await client.get("/api/dropdowns/courts", { jurisdiction: "illinois" }); - await client.get("/api/dropdowns/courts", { jurisdiction: "illinois" }); + const { + client, + calls + } = makeClient(); + await client.get("/api/dropdowns/courts", { + jurisdiction: "illinois" + }); + await client.get("/api/dropdowns/courts", { + jurisdiction: "illinois" + }); assert.strictEqual(calls(), 1); }); test("draft state (case + upload data) is never cached", async () => { - const { client, calls } = makeClient(); + const { + client, + calls + } = makeClient(); await client.getCaseData(); await client.getCaseData(); await client.getUploadData(); @@ -55,17 +83,29 @@ test("draft state (case + upload data) is never cached", async () => { }); test("per-user reads via fetchJSON (profile, payment accounts, token) are not cached", async () => { - const { client, calls } = makeClient(); - await client.fetchJSON("/api/payment-accounts", "GET", { jurisdiction: "illinois" }); - await client.fetchJSON("/api/payment-accounts", "GET", { jurisdiction: "illinois" }); + const { + client, + calls + } = makeClient(); + await client.fetchJSON("/api/payment-accounts", "GET", { + jurisdiction: "illinois" + }); + await client.fetchJSON("/api/payment-accounts", "GET", { + jurisdiction: "illinois" + }); assert.strictEqual(calls(), 2); }); test("saving case data does not populate the read cache", async () => { - const { client, calls } = makeClient(); - await client.saveCaseData({ court: "cook:cd" }); + const { + client, + calls + } = makeClient(); + await client.saveCaseData({ + court: "cook:cd" + }); await client.getCaseData(); await client.getCaseData(); // 1 save + 2 uncached reads assert.strictEqual(calls(), 3); -}); +}); \ No newline at end of file diff --git a/efile_app/uv.lock b/efile_app/uv.lock index 042b6c6..620505d 100644 --- a/efile_app/uv.lock +++ b/efile_app/uv.lock @@ -8,10 +8,6 @@ resolution-markers = [ "python_full_version < '3.13' and sys_platform != 'win32'", ] -[options] -exclude-newer = "2026-04-19T15:47:24.300481Z" -exclude-newer-span = "P2D" - [[package]] name = "annotated-types" version = "0.7.0" From 87b5883e7bb23c2c648c3eea69a6e794fe80e1cc Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Tue, 21 Jul 2026 19:05:05 -0400 Subject: [PATCH 35/47] Fix existing-case code loss and make submission idempotent Two review findings: - Existing-case lookup dropped classification codes. The lookup posts case_category_code / case_type_code (and court_code), but the serializer only accepted the bare case_category / case_type forms, silently losing the codes so a resumed existing case had no classification. Accept both the bare and *_code wire forms. - Submission was not idempotent. The wrapper forwarded every request and only marked the draft submitted afterward, so concurrent double-clicks could each fire the external filing API. Claim the draft into SUBMITTING with a single-winner atomic update before forwarding; a losing request gets 409 and never calls the API. A precondition failure releases the claim back to DRAFT so the user can retry. The current-draft pointer lookup now resolves SUBMITTING drafts (a draft mid-submission is still the user's current draft) while resume/listings still exclude them. Co-Authored-By: Claude Opus 4.8 --- efile_app/efile/services/current_drafts.py | 11 ++++- efile_app/efile/services/drafts.py | 21 +++++++--- efile_app/efile/tests/test_durable_drafts.py | 42 ++++++++++++++++++++ efile_app/efile/views/submission.py | 41 ++++++++++++++++++- 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/efile_app/efile/services/current_drafts.py b/efile_app/efile/services/current_drafts.py index 00c84e7..2e9d756 100644 --- a/efile_app/efile/services/current_drafts.py +++ b/efile_app/efile/services/current_drafts.py @@ -5,7 +5,7 @@ from django.db import transaction from efile.models import FilingDraft -from efile.services.drafts import create_draft, get_active_draft, set_current_step +from efile.services.drafts import CURRENT_DRAFT_STATUSES, create_draft, get_active_draft, set_current_step from efile.workflow import WorkflowStepKey CURRENT_DRAFT_SESSION_KEY = "filing_draft_id" @@ -56,7 +56,14 @@ def get_current_draft( draft_id = None if draft_id is not None: - draft = get_active_draft(user=user, draft_id=draft_id, jurisdiction=jurisdiction) + # The pointed-at draft may be mid-submission (SUBMITTING); it is still the + # user's current draft, so resolve it even though resume/listings would not. + draft = get_active_draft( + user=user, + draft_id=draft_id, + jurisdiction=jurisdiction, + statuses=CURRENT_DRAFT_STATUSES, + ) if draft is not None: return draft clear_current_draft(request) diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py index aea025b..c512fbe 100644 --- a/efile_app/efile/services/drafts.py +++ b/efile_app/efile/services/drafts.py @@ -16,6 +16,9 @@ from efile.workflow import WorkflowStepKey ACTIVE_DRAFT_STATUSES = (FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR) +# A draft mid-submission is still the user's current draft (so the submit flow can +# read its own data), but it is not offered for resume in listings. +CURRENT_DRAFT_STATUSES = (*ACTIVE_DRAFT_STATUSES, FilingDraft.Status.SUBMITTING) def active_drafts_for(user, *, jurisdiction: str | None = None) -> QuerySet[FilingDraft]: @@ -32,10 +35,14 @@ def get_active_draft( user, draft_id: int | str | None = None, jurisdiction: str | None = None, + statuses: tuple[str, ...] = ACTIVE_DRAFT_STATUSES, ) -> FilingDraft | None: - """Get an owned active draft by ID, or the user's most recent draft.""" + """Get an owned draft by ID, or the user's most recent one, within ``statuses``.""" - drafts = active_drafts_for(user, jurisdiction=jurisdiction) + drafts = FilingDraft.objects.filter(user=user, status__in=statuses) + if jurisdiction is not None: + drafts = drafts.filter(jurisdiction=jurisdiction) + drafts = drafts.order_by("-updated_at") if draft_id is not None: return drafts.filter(pk=draft_id).first() return drafts.first() @@ -83,13 +90,15 @@ def set_current_step(draft: FilingDraft, current_step: WorkflowStepKey | str) -> # Draft scalar column -> the wire keys it may arrive under (first present wins). _DRAFT_FIELD_SOURCES: dict[str, tuple[str, ...]] = { - "court_code": ("court",), + "court_code": ("court", "court_code"), "court_name": ("court_name",), - "case_category_code": ("case_category",), + # The dropdown flow sends the bare name ("case_category"); the existing-case + # lookup sends the explicit "*_code" form. Accept both. + "case_category_code": ("case_category", "case_category_code"), "case_category_name": ("case_category_name",), - "case_type_code": ("case_type",), + "case_type_code": ("case_type", "case_type_code"), "case_type_name": ("case_type_name",), - "case_subtype_code": ("case_subtype",), + "case_subtype_code": ("case_subtype", "case_subtype_code"), "case_subtype_name": ("case_subtype_name",), "filing_type_code": ("filing_type", "filing_type_id"), "filing_type_name": ("filing_type_name",), diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index a6c5056..085873e 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -131,6 +131,33 @@ def test_case_data_round_trips_through_the_model(django_user_model): assert blob["other_address_city"] == "Chicago" +@pytest.mark.django_db +def test_existing_case_lookup_codes_are_persisted(django_user_model): + """The existing-case lookup sends *_code keys; they must not be dropped.""" + user = django_user_model.objects.create_user(username="existing-case-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + + write_case_data( + draft, + { + "existing_case": "yes", + "court": "cook:cd", + "case_category_code": "MR", + "case_category_name": "Miscellaneous Remedy", + "case_type_code": "Name Change", + "case_type_name": "Change of Name", + "case_tracking_id": "track-1", + "case_docket_id": "2024-MR-1", + }, + ) + draft.refresh_from_db() + + assert draft.case_category_code == "MR" + assert draft.case_type_code == "Name Change" + assert draft.previous_case_id == "track-1" + assert draft.docket_number == "2024-MR-1" + + @pytest.mark.django_db def test_write_upload_data_creates_lead_and_supporting_documents(django_user_model): user = django_user_model.objects.create_user(username="document-owner", tyler_jurisdiction="illinois") @@ -403,6 +430,21 @@ def fake_post(*_args, **_kwargs): assert draft.submission_response["response"]["api_status_code"] == 400 +@pytest.mark.django_db +def test_submission_claim_prevents_duplicate_filing(django_user_model): + """The SUBMITTING claim is single-winner, so concurrent submits can't both file.""" + from efile.views.submission import _claim_for_submission + + user = django_user_model.objects.create_user(username="claim-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + + assert _claim_for_submission(draft) is True + draft.refresh_from_db() + assert draft.status == FilingDraft.Status.SUBMITTING + # A second attempt on the same draft is refused. + assert _claim_for_submission(draft) is False + + @pytest.mark.django_db def test_model_step_choices_follow_workflow_registry(): current_step = FilingDraft._meta.get_field("current_step") diff --git a/efile_app/efile/views/submission.py b/efile_app/efile/views/submission.py index 459c7bb..0912a34 100644 --- a/efile_app/efile/views/submission.py +++ b/efile_app/efile/views/submission.py @@ -2,15 +2,43 @@ import logging from django.http import JsonResponse +from django.utils import timezone from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods +from efile.models import FilingDraft from efile.services.current_drafts import clear_current_draft, get_current_draft from .session_api import submit_final_filing as legacy_submit_final_filing logger = logging.getLogger(__name__) +_CLAIMABLE_STATUSES = (FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR) + + +def _claim_for_submission(draft: FilingDraft) -> bool: + """Atomically move a submittable draft into SUBMITTING. + + Only one request can win this transition, so concurrent double-clicks or + retries cannot each forward to the external filing API and create duplicates. + """ + claimed = FilingDraft.objects.filter(pk=draft.pk, status__in=_CLAIMABLE_STATUSES).update( + status=FilingDraft.Status.SUBMITTING, + updated_at=timezone.now(), + ) + if claimed: + draft.status = FilingDraft.Status.SUBMITTING + return bool(claimed) + + +def _release_claim(draft: FilingDraft) -> None: + """Return a claimed draft to DRAFT when no external submission was attempted.""" + FilingDraft.objects.filter(pk=draft.pk, status=FilingDraft.Status.SUBMITTING).update( + status=FilingDraft.Status.DRAFT, + updated_at=timezone.now(), + ) + draft.status = FilingDraft.Status.DRAFT + def _json_payload(response: JsonResponse) -> dict: try: @@ -33,11 +61,18 @@ def _submission_attempt_failed(response: JsonResponse, payload: dict) -> bool: @csrf_exempt @require_http_methods(["POST"]) def submit_final_filing(request): - """Submit through the legacy session path and mirror the result to the durable draft.""" + """Submit through the session path, guarding against duplicate external filings.""" jurisdiction = request.session.get("jurisdiction") draft = get_current_draft(request, jurisdiction=jurisdiction, resume_latest=False) + # Claim the draft before forwarding so a concurrent request can't file twice. + if draft is not None and not _claim_for_submission(draft): + return JsonResponse( + {"success": False, "error": "This filing is already being submitted."}, + status=409, + ) + response = legacy_submit_final_filing(request) payload = _json_payload(response) @@ -54,5 +89,9 @@ def submit_final_filing(request): "response": payload, } ) + else: + # A precondition failed before any external call (e.g. missing data); + # release the claim so the user can fix it and retry. + _release_claim(draft) return response From a06376b2ab93424b6bd418993203ed702c0cbb9e Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Tue, 21 Jul 2026 19:18:10 -0400 Subject: [PATCH 36/47] Harden submission recovery and enforce per-route draft isolation Three review findings: - A submission could stick permanently in SUBMITTING if a request died after claiming but before mark_submitted/mark_error. The claim now also takes over a SUBMITTING draft whose claim is older than SUBMISSION_CLAIM_TIMEOUT (15m), so a crashed submit is recoverable. - Ambiguous post-external-call failures no longer reset the draft to DRAFT. Only failures the submit view raises *before* calling the filing API (missing data, missing efile_data, etc.) release the claim; anything else is recorded as ERROR so a blind retry can't silently double-file. - Draft accessors now enforce the route/payload jurisdiction. get_case_data / get_upload_data / update_* (and the derived getters) take an optional jurisdiction, and the flow views and jurisdiction-aware API writers pass it, so a request for jurisdiction A can no longer read or write a draft pointed to from jurisdiction B. Co-Authored-By: Claude Opus 4.8 --- efile_app/efile/api/filing_views.py | 2 +- efile_app/efile/api/suffolk_api_views.py | 1 + efile_app/efile/tests/test_durable_drafts.py | 84 ++++++++++++++++++++ efile_app/efile/utils/case_data_utils.py | 53 +++++++----- efile_app/efile/views/expert_form.py | 2 +- efile_app/efile/views/filing_statuses.py | 2 +- efile_app/efile/views/options.py | 2 +- efile_app/efile/views/payment.py | 2 +- efile_app/efile/views/review.py | 8 +- efile_app/efile/views/session_api.py | 5 +- efile_app/efile/views/submission.py | 66 +++++++++------ efile_app/efile/views/upload.py | 10 +-- efile_app/efile/views/upload_first.py | 8 +- 13 files changed, 180 insertions(+), 65 deletions(-) diff --git a/efile_app/efile/api/filing_views.py b/efile_app/efile/api/filing_views.py index 5e6cddd..6a8410c 100644 --- a/efile_app/efile/api/filing_views.py +++ b/efile_app/efile/api/filing_views.py @@ -172,7 +172,7 @@ def payment_fees(request): jurisdiction_id = request.session.get("jurisdiction") auth_tokens = request.session.get("auth_tokens", {}) - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction_id) court_id = case_data.get("court", "") url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filing/fees" diff --git a/efile_app/efile/api/suffolk_api_views.py b/efile_app/efile/api/suffolk_api_views.py index 83e2016..5342d98 100644 --- a/efile_app/efile/api/suffolk_api_views.py +++ b/efile_app/efile/api/suffolk_api_views.py @@ -302,6 +302,7 @@ def get_party_types_from_suffolk_api(request): "petitioner_party_type": selected_party_type, "existing_case": existing_case, }, + jurisdiction, ) logger.debug(f"Saved party type to draft: {selected_party_type}") diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index 085873e..adaee8c 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -445,6 +445,90 @@ def test_submission_claim_prevents_duplicate_filing(django_user_model): assert _claim_for_submission(draft) is False +@pytest.mark.django_db +def test_stale_submission_claim_is_recoverable(django_user_model): + """A claim left behind by a crashed request can be taken over after the timeout.""" + from datetime import timedelta + + from django.utils import timezone + + from efile.views.submission import SUBMISSION_CLAIM_TIMEOUT, _claim_for_submission + + user = django_user_model.objects.create_user(username="stale-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + + assert _claim_for_submission(draft) is True + assert _claim_for_submission(draft) is False # fresh claim is not recoverable + + stale = timezone.now() - SUBMISSION_CLAIM_TIMEOUT - timedelta(minutes=1) + FilingDraft.objects.filter(pk=draft.pk).update(updated_at=stale) + + assert _claim_for_submission(draft) is True # stale claim is recovered + + +@pytest.mark.django_db +def test_precondition_failure_releases_claim_to_draft(client, django_user_model): + """A failure before the external call frees the draft for a safe retry.""" + user = django_user_model.objects.create_user(username="precondition-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois", current_step=WorkflowStepKey.REVIEW) + client.force_login(user) + _prepare_submission(client, draft) + + response = client.post( + reverse("submit_final_filing"), + data={"confirm_submission": True}, # no efile_data -> pre-call validation failure + content_type="application/json", + ) + + assert response.status_code == 400 + draft.refresh_from_db() + assert draft.status == FilingDraft.Status.DRAFT + + +@pytest.mark.django_db +def test_ambiguous_failure_does_not_release_to_draft(client, django_user_model, monkeypatch): + """An error after requests.post may mean the filing went through: never reset to DRAFT.""" + + def boom(*_args, **_kwargs): + raise ValueError("crashed after sending") + + monkeypatch.setattr("requests.post", boom) + user = django_user_model.objects.create_user(username="ambiguous-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois", current_step=WorkflowStepKey.REVIEW) + client.force_login(user) + _prepare_submission(client, draft) + + response = client.post( + reverse("submit_final_filing"), + data={"confirm_submission": True, "efile_data": {"al_court_bundle": {}}}, + content_type="application/json", + ) + + assert response.status_code >= 400 + draft.refresh_from_db() + assert draft.status == FilingDraft.Status.ERROR + + +@pytest.mark.django_db +def test_route_jurisdiction_isolates_reads(client, django_user_model): + """A request served for jurisdiction A must not read a draft pointed to from B.""" + from efile.utils.case_data_utils import get_case_data + + user = django_user_model.objects.create_user(username="multi-jur-user", tyler_jurisdiction="illinois") + illinois_draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + massachusetts_draft = FilingDraft.objects.create(user=user, jurisdiction="massachusetts") + write_case_data(illinois_draft, {"court": "cook:cd"}) + write_case_data(massachusetts_draft, {"court": "suffolk:ma"}) + + client.force_login(user) + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = massachusetts_draft.pk + session.save() + request = type("Request", (), {"user": user, "session": client.session})() + + assert get_case_data(request, jurisdiction="illinois").get("court") == "cook:cd" + + @pytest.mark.django_db def test_model_step_choices_follow_workflow_registry(): current_step = FilingDraft._meta.get_field("current_step") diff --git a/efile_app/efile/utils/case_data_utils.py b/efile_app/efile/utils/case_data_utils.py index 50124ca..9c279d8 100644 --- a/efile_app/efile/utils/case_data_utils.py +++ b/efile_app/efile/utils/case_data_utils.py @@ -22,45 +22,54 @@ def _current_jurisdiction(request): return request.session.get("jurisdiction") -def _resolve_writable_draft(request): - """Return the draft to write to, creating one for authenticated users only.""" +def _resolve_writable_draft(request, jurisdiction=None): + """Return the draft to write to, creating one for authenticated users only. + + When ``jurisdiction`` is given (the route/payload the caller is acting on), it + is enforced so a request for jurisdiction A can never write jurisdiction B's + currently-pointed draft. + """ if not getattr(request.user, "is_authenticated", False): return None - jurisdiction = _current_jurisdiction(request) - if jurisdiction: - return ensure_current_draft(request, jurisdiction) + target = jurisdiction or _current_jurisdiction(request) + if target: + return ensure_current_draft(request, target) return get_current_draft(request) -def get_case_data(request): - """Return the current draft serialized to the case_data blob (``{}`` if none).""" - return read_case_data(get_current_draft(request)) +def get_case_data(request, jurisdiction=None): + """Return the current draft serialized to the case_data blob (``{}`` if none). + + Passing ``jurisdiction`` (the route the caller is serving) enforces isolation: + a draft pointed to from a different jurisdiction is not returned. + """ + return read_case_data(get_current_draft(request, jurisdiction=jurisdiction)) -def update_case_data(request, updates): +def update_case_data(request, updates, jurisdiction=None): """Persist a partial case_data blob onto the current draft and return the merged blob.""" - draft = _resolve_writable_draft(request) + draft = _resolve_writable_draft(request, jurisdiction) if draft is None: return {} write_case_data(draft, updates) return read_case_data(draft) -def get_upload_data(request): - return read_upload_data(get_current_draft(request)) +def get_upload_data(request, jurisdiction=None): + return read_upload_data(get_current_draft(request, jurisdiction=jurisdiction)) -def update_upload_data(request, updates): - draft = _resolve_writable_draft(request) +def update_upload_data(request, updates, jurisdiction=None): + draft = _resolve_writable_draft(request, jurisdiction) if draft is None: return {} write_upload_data(draft, updates) return read_upload_data(draft) -def get_petitioner_info(request): +def get_petitioner_info(request, jurisdiction=None): """Get petitioner information specifically.""" - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) full_name = f"{case_data.get('petitioner_first_name', '')} {case_data.get('petitioner_last_name', '')}".strip() return { "first_name": case_data.get("petitioner_first_name", ""), @@ -70,9 +79,9 @@ def get_petitioner_info(request): } -def get_name_sought_info(request): +def get_name_sought_info(request, jurisdiction=None): """Get name sought information specifically.""" - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) return { "first_name": case_data.get("new_first_name", ""), "last_name": case_data.get("new_last_name", ""), @@ -80,9 +89,9 @@ def get_name_sought_info(request): } -def get_case_classification(request): +def get_case_classification(request, jurisdiction=None): """Get case classification information.""" - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) logger.info("Case data: %s", case_data) return { "court": case_data.get("court", ""), @@ -94,7 +103,7 @@ def get_case_classification(request): } -def get_selected_services(request): +def get_selected_services(request, jurisdiction=None): """Get list of selected optional services.""" - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) return case_data.get("optional_services", []) diff --git a/efile_app/efile/views/expert_form.py b/efile_app/efile/views/expert_form.py index bfff5ce..a77d3a8 100644 --- a/efile_app/efile/views/expert_form.py +++ b/efile_app/efile/views/expert_form.py @@ -40,7 +40,7 @@ def efile_expert_form(request, jurisdiction): logger.debug(f"All session data keys: {list(request.session.keys())}") logger.debug(f"Clear session parameter received: {request.GET.get('clear_session', 'not present')}") - upload_data = get_upload_data(request) + upload_data = get_upload_data(request, jurisdiction) # Check if we have all required data for upload required_fields = ["court", "case_category", "case_type", "filing_type", "document_type"] diff --git a/efile_app/efile/views/filing_statuses.py b/efile_app/efile/views/filing_statuses.py index 11a22da..945ff2b 100644 --- a/efile_app/efile/views/filing_statuses.py +++ b/efile_app/efile/views/filing_statuses.py @@ -11,7 +11,7 @@ def filing_statuses(request, jurisdiction): return redirect("efile_login", jurisdiction=jurisdiction) # Get case data from session - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) is_logged_in = request.user.is_authenticated if not get_tyler_token(request, jurisdiction): diff --git a/efile_app/efile/views/options.py b/efile_app/efile/views/options.py index b376cef..af14559 100644 --- a/efile_app/efile/views/options.py +++ b/efile_app/efile/views/options.py @@ -15,7 +15,7 @@ def efile_options(request, jurisdiction): # Get case data from session if request.user.is_authenticated: - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) active_draft = get_current_draft(request, jurisdiction=jurisdiction) else: case_data = {} diff --git a/efile_app/efile/views/payment.py b/efile_app/efile/views/payment.py index ed38bfe..e5c90ab 100644 --- a/efile_app/efile/views/payment.py +++ b/efile_app/efile/views/payment.py @@ -23,7 +23,7 @@ def efile_payment(request, jurisdiction): return redirect("efile_login", jurisdiction=jurisdiction) # Get case data from session - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) logger.debug("Review view case_data %s", case_data) # Add user email from session if available and not already in case_data diff --git a/efile_app/efile/views/review.py b/efile_app/efile/views/review.py index 982d2b9..ce062d9 100644 --- a/efile_app/efile/views/review.py +++ b/efile_app/efile/views/review.py @@ -23,7 +23,7 @@ def case_review(request, jurisdiction): return redirect("efile_login", jurisdiction=jurisdiction) # Get case data from session - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) logger.debug("Review view case_data %s", case_data) # Add user email from session if available and not already in case_data @@ -39,9 +39,9 @@ def case_review(request, jurisdiction): filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.REVIEW) # Get organized case information - petitioner_info = get_petitioner_info(request) - name_sought_info = get_name_sought_info(request) - case_classification = get_case_classification(request) + petitioner_info = get_petitioner_info(request, jurisdiction) + name_sought_info = get_name_sought_info(request, jurisdiction) + case_classification = get_case_classification(request, jurisdiction) # Use friendly names if available, otherwise fallback to raw values friendly_case_type = case_data.get("case_type_name", case_classification["case_type"]) diff --git a/efile_app/efile/views/session_api.py b/efile_app/efile/views/session_api.py index 974a5b8..0ffa7da 100644 --- a/efile_app/efile/views/session_api.py +++ b/efile_app/efile/views/session_api.py @@ -180,7 +180,7 @@ def save_upload_first_data(request): upload_data["guesses"]["case type"] = found_fields.get("case type") upload_data["guesses"]["docket number"] = found_fields.get("docket number") - update_upload_data(request, upload_data) + update_upload_data(request, upload_data, jurisdiction_id) logger.info("Persisted lead upload to the current draft") return JsonResponse({"success": True, "message": "Upload data saved"}) @@ -461,7 +461,7 @@ def api_save_case_data(request): updates["party_type"] = party_type updates.setdefault("petitioner_party_type", party_type) - update_case_data(request, updates) + update_case_data(request, updates, jurisdiction) return JsonResponse( {"success": True, "data": {"existing_case": existing_case, "saved_fields": list(form_data.keys())}} @@ -554,6 +554,7 @@ def fetch_and_save_party_type(request): "party_type": party_type_code, "petitioner_party_type": party_type_code, }, + jurisdiction, ) logger.debug(f"Saved party type to draft: {party_type_code}") diff --git a/efile_app/efile/views/submission.py b/efile_app/efile/views/submission.py index 0912a34..38fc6ae 100644 --- a/efile_app/efile/views/submission.py +++ b/efile_app/efile/views/submission.py @@ -1,6 +1,8 @@ import json import logging +from datetime import timedelta +from django.db.models import Q from django.http import JsonResponse from django.utils import timezone from django.views.decorators.csrf import csrf_exempt @@ -15,19 +17,42 @@ _CLAIMABLE_STATUSES = (FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR) +# A claim older than this is assumed to belong to a crashed request and may be +# taken over, so a draft can never be stuck in SUBMITTING forever. +SUBMISSION_CLAIM_TIMEOUT = timedelta(minutes=15) + +# Errors the session submit view returns *before* it calls the external filing +# API. Only these are safe to release back to DRAFT; any other failure may have +# left a filing submitted, so the draft must not be freed for a blind retry. +_PRE_SUBMIT_ERROR_PREFIXES = ( + "Submission confirmation is required", + "No case data found", + "No upload data found", + "No efile data provided", + "Missing required fields in efile_data", + "Court ID is required", +) + def _claim_for_submission(draft: FilingDraft) -> bool: - """Atomically move a submittable draft into SUBMITTING. + """Atomically move a submittable (or stale-claimed) draft into SUBMITTING. - Only one request can win this transition, so concurrent double-clicks or - retries cannot each forward to the external filing API and create duplicates. + Only one request can win this transition, so concurrent double-clicks cannot + each forward to the external API. A claim left behind by a crashed request + becomes recoverable once it is older than ``SUBMISSION_CLAIM_TIMEOUT``. """ - claimed = FilingDraft.objects.filter(pk=draft.pk, status__in=_CLAIMABLE_STATUSES).update( - status=FilingDraft.Status.SUBMITTING, - updated_at=timezone.now(), + now = timezone.now() + stale_before = now - SUBMISSION_CLAIM_TIMEOUT + claimed = ( + FilingDraft.objects.filter(pk=draft.pk) + .filter( + Q(status__in=_CLAIMABLE_STATUSES) | Q(status=FilingDraft.Status.SUBMITTING, updated_at__lt=stale_before) + ) + .update(status=FilingDraft.Status.SUBMITTING, updated_at=now) ) if claimed: draft.status = FilingDraft.Status.SUBMITTING + draft.updated_at = now return bool(claimed) @@ -47,15 +72,12 @@ def _json_payload(response: JsonResponse) -> dict: return {} -def _submission_attempt_failed(response: JsonResponse, payload: dict) -> bool: +def _failed_before_external_call(payload: dict) -> bool: + """True only when the submit view rejected the request before calling the API.""" if "api_status_code" in payload: - return True - - error = payload.get("error") - if not isinstance(error, str): return False - - return error.startswith("Filing submission failed:") or error.startswith("Network error during filing submission:") + error = payload.get("error") + return isinstance(error, str) and error.startswith(_PRE_SUBMIT_ERROR_PREFIXES) @csrf_exempt @@ -82,16 +104,14 @@ def submit_final_filing(request): if response.status_code < 400 and payload.get("success") is True: draft.mark_submitted(payload.get("api_response") or {}) clear_current_draft(request) - elif _submission_attempt_failed(response, payload): - draft.mark_error( - { - "status_code": response.status_code, - "response": payload, - } - ) - else: - # A precondition failed before any external call (e.g. missing data); - # release the claim so the user can fix it and retry. + elif _failed_before_external_call(payload): + # No external call happened, so it is safe to free the draft for retry. _release_claim(draft) + else: + # The API failed, or the outcome is ambiguous (an error after requests.post + # may mean the filing went through). Record an error and leave it OUT of + # DRAFT so a blind retry cannot silently double-file. (True exactly-once + # would require an idempotency key on the external filing API.) + draft.mark_error({"status_code": response.status_code, "response": payload}) return response diff --git a/efile_app/efile/views/upload.py b/efile_app/efile/views/upload.py index b2f7d4a..dea6b21 100644 --- a/efile_app/efile/views/upload.py +++ b/efile_app/efile/views/upload.py @@ -29,7 +29,7 @@ def efile_upload(request, jurisdiction): if not get_tyler_token(request, jurisdiction): return redirect("efile_login", jurisdiction=jurisdiction) - case_data = get_case_data(request) + case_data = get_case_data(request, jurisdiction) if not case_data: messages.error(request, gettext("Please complete the case details first.")) @@ -37,15 +37,15 @@ def efile_upload(request, jurisdiction): filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.DOCUMENTS) - petitioner_info = get_petitioner_info(request) - name_sought_info = get_name_sought_info(request) - case_classification = get_case_classification(request) + petitioner_info = get_petitioner_info(request, jurisdiction) + name_sought_info = get_name_sought_info(request, jurisdiction) + case_classification = get_case_classification(request, jurisdiction) friendly_case_type = case_data.get("case_type_name", case_classification["case_type"]) friendly_filing_type = case_data.get("filing_type_name", case_classification["filing_type"]) friendly_court = case_data.get("court_name", case_classification["court"]) - upload_data = get_upload_data(request) + upload_data = get_upload_data(request, jurisdiction) context = { "is_logged_in": True, diff --git a/efile_app/efile/views/upload_first.py b/efile_app/efile/views/upload_first.py index bf29a56..1a04ef0 100644 --- a/efile_app/efile/views/upload_first.py +++ b/efile_app/efile/views/upload_first.py @@ -46,12 +46,12 @@ def efile_upload_first(request, jurisdiction): filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.UPLOAD_FIRST) # Could visit here from a back button press, so use upload data if any - upload_data = get_upload_data(request) + upload_data = get_upload_data(request, jurisdiction) # Get organized case information - petitioner_info = get_petitioner_info(request) - name_sought_info = get_name_sought_info(request) - case_classification = get_case_classification(request) + petitioner_info = get_petitioner_info(request, jurisdiction) + name_sought_info = get_name_sought_info(request, jurisdiction) + case_classification = get_case_classification(request, jurisdiction) context = { "is_logged_in": True, From 377e20e75175d56c8440f5955ed16b6085881968 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Tue, 21 Jul 2026 19:53:39 -0400 Subject: [PATCH 37/47] Updated per additional PR review --- compose.yml | 2 +- .../efile/migrations/0002_filing_drafts.py | 1 + efile_app/efile/models.py | 5 ++ efile_app/efile/services/drafts.py | 18 +++++ efile_app/efile/tests/test_durable_drafts.py | 65 ++++++++++++++----- efile_app/efile/views/submission.py | 57 ++++++++-------- 6 files changed, 101 insertions(+), 47 deletions(-) diff --git a/compose.yml b/compose.yml index 86c159a..04d71f8 100644 --- a/compose.yml +++ b/compose.yml @@ -5,7 +5,7 @@ services: dockerfile: Dockerfile image: form-submission-mvp:web command: >- - sh -c "uv run python efile_app/manage.py migrate && + sh -c "uv run python efile_app/manage.py migrate --noinput --fake-initial && uv run python efile_app/manage.py runserver 0.0.0.0:8000" ports: - "8000:8000" diff --git a/efile_app/efile/migrations/0002_filing_drafts.py b/efile_app/efile/migrations/0002_filing_drafts.py index c90d4b2..8bdacfd 100644 --- a/efile_app/efile/migrations/0002_filing_drafts.py +++ b/efile_app/efile/migrations/0002_filing_drafts.py @@ -67,6 +67,7 @@ class Migration(migrations.Migration): ("name_change_reason", models.TextField(blank=True)), ("optional_services", models.JSONField(blank=True, default=list)), ("extracted_guesses", models.JSONField(blank=True, default=dict)), + ("supplemental_fields", models.JSONField(blank=True, default=dict)), ("submission_response", models.JSONField(blank=True, default=dict)), ("submitted_at", models.DateTimeField(blank=True, null=True)), ("created_at", models.DateTimeField(auto_now_add=True)), diff --git a/efile_app/efile/models.py b/efile_app/efile/models.py index 7d144db..0b37f82 100644 --- a/efile_app/efile/models.py +++ b/efile_app/efile/models.py @@ -79,6 +79,11 @@ class Status(models.TextChoices): optional_services = models.JSONField(default=list, blank=True) extracted_guesses = models.JSONField(default=dict, blank=True) + # Area-of-law questionnaire answers (e.g. divorce children questions). These are + # driven by the per-state/case-type config, not a fixed schema, so they live in a + # structured JSON field rather than a column each. Only config-defined keys are + # stored here -- it is not a catch-all for arbitrary case data. + supplemental_fields = models.JSONField(default=dict, blank=True) submission_response = models.JSONField(default=dict, blank=True) submitted_at = models.DateTimeField(blank=True, null=True) diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py index c512fbe..15d660f 100644 --- a/efile_app/efile/services/drafts.py +++ b/efile_app/efile/services/drafts.py @@ -152,6 +152,12 @@ def set_current_step(draft: FilingDraft, current_step: WorkflowStepKey | str) -> _PETITIONER_PARTY_TYPE_KEYS = ("determined_party_type", "party_type", "petitioner_party_type") +# Area-of-law questionnaire fields that persist in ``supplemental_fields`` instead +# of a dedicated column. Keep this allowlist aligned with fields currently rendered +# from the state configuration. Values are stored as received rather than being +# string-coerced like the typed columns. +_SUPPLEMENTAL_FIELDS = ("has_children", "child_count") + def _first_present(data: dict[str, Any], keys: tuple[str, ...]) -> Any: for key in keys: @@ -202,6 +208,13 @@ def write_case_data( draft.optional_services = services update_fields.append("optional_services") + supplemental_updates = {key: data[key] for key in _SUPPLEMENTAL_FIELDS if key in data} + if supplemental_updates: + merged = {**draft.supplemental_fields, **supplemental_updates} + if merged != draft.supplemental_fields: + draft.supplemental_fields = merged + update_fields.append("supplemental_fields") + if current_step is not None and draft.current_step != str(current_step): draft.current_step = str(current_step) update_fields.append("current_step") @@ -265,6 +278,10 @@ def read_case_data(draft: FilingDraft | None) -> dict[str, Any]: for key in _PETITIONER_PARTY_TYPE_KEYS: _put(data, key, party.party_type) + # Supplemental answers are emitted as stored (a False/0 answer is meaningful). + for key, value in (draft.supplemental_fields or {}).items(): + data[key] = value + return data @@ -465,6 +482,7 @@ def draft_snapshot(draft: FilingDraft | None) -> dict[str, Any] | None: "selected_payment_account_name": draft.selected_payment_account_name, "optional_services": draft.optional_services, "extracted_guesses": draft.extracted_guesses, + "supplemental_fields": draft.supplemental_fields, "document_count": FilingDocument.objects.filter(draft=draft).count(), "party_count": FilingParty.objects.filter(draft=draft).count(), "created_at": draft.created_at.isoformat() if draft.created_at else None, diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index adaee8c..6fb3f31 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -131,6 +131,22 @@ def test_case_data_round_trips_through_the_model(django_user_model): assert blob["other_address_city"] == "Chicago" +@pytest.mark.django_db +def test_supplemental_case_fields_round_trip(django_user_model): + """Config-driven questionnaire answers survive a durable-draft round trip.""" + user = django_user_model.objects.create_user(username="supplemental-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + + write_case_data(draft, {"has_children": "false", "child_count": "2", "unknown_answer": "drop me"}) + draft.refresh_from_db() + + assert draft.supplemental_fields == {"has_children": "false", "child_count": "2"} + assert read_case_data(draft)["has_children"] == "false" + assert read_case_data(draft)["child_count"] == "2" + assert "unknown_answer" not in read_case_data(draft) + assert draft_snapshot(draft)["supplemental_fields"] == {"has_children": "false", "child_count": "2"} + + @pytest.mark.django_db def test_existing_case_lookup_codes_are_persisted(django_user_model): """The existing-case lookup sends *_code keys; they must not be dropped.""" @@ -407,7 +423,7 @@ def fake_post(*_args, **_kwargs): @pytest.mark.django_db -def test_final_submission_marks_current_draft_error_on_api_failure(client, django_user_model, monkeypatch): +def test_confirmed_api_rejection_releases_draft_for_retry(client, django_user_model, monkeypatch): def fake_post(*_args, **_kwargs): return FakeApiResponse(400, {"error": "Rejected", "validation_errors": ["bad bundle"]}) @@ -425,9 +441,17 @@ def fake_post(*_args, **_kwargs): assert response.status_code == 400 draft.refresh_from_db() - assert draft.status == FilingDraft.Status.ERROR - assert draft.submission_response["status_code"] == 400 - assert draft.submission_response["response"]["api_status_code"] == 400 + assert draft.status == FilingDraft.Status.DRAFT + + +@pytest.mark.parametrize( + ("status_code", "is_confirmed_rejection"), + [(400, True), (422, True), (408, False), (409, False), (500, False)], +) +def test_confirmed_api_rejection_excludes_ambiguous_statuses(status_code, is_confirmed_rejection): + from efile.views.submission import _confirmed_api_rejection + + assert _confirmed_api_rejection({"api_status_code": status_code}) is is_confirmed_rejection @pytest.mark.django_db @@ -446,24 +470,18 @@ def test_submission_claim_prevents_duplicate_filing(django_user_model): @pytest.mark.django_db -def test_stale_submission_claim_is_recoverable(django_user_model): - """A claim left behind by a crashed request can be taken over after the timeout.""" - from datetime import timedelta - - from django.utils import timezone - - from efile.views.submission import SUBMISSION_CLAIM_TIMEOUT, _claim_for_submission +def test_ambiguous_submission_states_are_not_automatically_reclaimed(django_user_model): + """SUBMITTING and ERROR require review because retrying either may double-file.""" + from efile.views.submission import _claim_for_submission - user = django_user_model.objects.create_user(username="stale-user", tyler_jurisdiction="illinois") + user = django_user_model.objects.create_user(username="ambiguous-state-user", tyler_jurisdiction="illinois") draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") assert _claim_for_submission(draft) is True - assert _claim_for_submission(draft) is False # fresh claim is not recoverable - - stale = timezone.now() - SUBMISSION_CLAIM_TIMEOUT - timedelta(minutes=1) - FilingDraft.objects.filter(pk=draft.pk).update(updated_at=stale) + assert _claim_for_submission(draft) is False - assert _claim_for_submission(draft) is True # stale claim is recovered + draft.mark_error({"error": "outcome unknown"}) + assert _claim_for_submission(draft) is False @pytest.mark.django_db @@ -489,7 +507,11 @@ def test_precondition_failure_releases_claim_to_draft(client, django_user_model) def test_ambiguous_failure_does_not_release_to_draft(client, django_user_model, monkeypatch): """An error after requests.post may mean the filing went through: never reset to DRAFT.""" + calls = 0 + def boom(*_args, **_kwargs): + nonlocal calls + calls += 1 raise ValueError("crashed after sending") monkeypatch.setattr("requests.post", boom) @@ -508,6 +530,15 @@ def boom(*_args, **_kwargs): draft.refresh_from_db() assert draft.status == FilingDraft.Status.ERROR + retry = client.post( + reverse("submit_final_filing"), + data={"confirm_submission": True, "efile_data": {"al_court_bundle": {}}}, + content_type="application/json", + ) + + assert retry.status_code == 409 + assert calls == 1 + @pytest.mark.django_db def test_route_jurisdiction_isolates_reads(client, django_user_model): diff --git a/efile_app/efile/views/submission.py b/efile_app/efile/views/submission.py index 38fc6ae..62a8a6f 100644 --- a/efile_app/efile/views/submission.py +++ b/efile_app/efile/views/submission.py @@ -1,8 +1,6 @@ import json import logging -from datetime import timedelta -from django.db.models import Q from django.http import JsonResponse from django.utils import timezone from django.views.decorators.csrf import csrf_exempt @@ -15,15 +13,15 @@ logger = logging.getLogger(__name__) -_CLAIMABLE_STATUSES = (FilingDraft.Status.DRAFT, FilingDraft.Status.ERROR) - -# A claim older than this is assumed to belong to a crashed request and may be -# taken over, so a draft can never be stuck in SUBMITTING forever. -SUBMISSION_CLAIM_TIMEOUT = timedelta(minutes=15) +# Only a clean DRAFT may be submitted. A draft is deliberately never auto-recovered +# out of SUBMITTING or ERROR: without an idempotency key on the external filing API, +# a retry after an ambiguous outcome could file a second time. Those states require +# manual review instead. +_CLAIMABLE_STATUSES = (FilingDraft.Status.DRAFT,) # Errors the session submit view returns *before* it calls the external filing -# API. Only these are safe to release back to DRAFT; any other failure may have -# left a filing submitted, so the draft must not be freed for a blind retry. +# API. Only these -- and a confirmed API rejection -- are safe to release back to +# DRAFT; any other failure may have left a filing submitted. _PRE_SUBMIT_ERROR_PREFIXES = ( "Submission confirmation is required", "No case data found", @@ -35,29 +33,23 @@ def _claim_for_submission(draft: FilingDraft) -> bool: - """Atomically move a submittable (or stale-claimed) draft into SUBMITTING. + """Atomically move a DRAFT into SUBMITTING. Only one request can win this transition, so concurrent double-clicks cannot - each forward to the external API. A claim left behind by a crashed request - becomes recoverable once it is older than ``SUBMISSION_CLAIM_TIMEOUT``. + each forward to the external API. A draft already SUBMITTING or ERROR is not + reclaimed here -- those are not safe to retry automatically. """ - now = timezone.now() - stale_before = now - SUBMISSION_CLAIM_TIMEOUT - claimed = ( - FilingDraft.objects.filter(pk=draft.pk) - .filter( - Q(status__in=_CLAIMABLE_STATUSES) | Q(status=FilingDraft.Status.SUBMITTING, updated_at__lt=stale_before) - ) - .update(status=FilingDraft.Status.SUBMITTING, updated_at=now) + claimed = FilingDraft.objects.filter(pk=draft.pk, status__in=_CLAIMABLE_STATUSES).update( + status=FilingDraft.Status.SUBMITTING, + updated_at=timezone.now(), ) if claimed: draft.status = FilingDraft.Status.SUBMITTING - draft.updated_at = now return bool(claimed) def _release_claim(draft: FilingDraft) -> None: - """Return a claimed draft to DRAFT when no external submission was attempted.""" + """Return a claimed draft to DRAFT when nothing was filed (pre-call or rejection).""" FilingDraft.objects.filter(pk=draft.pk, status=FilingDraft.Status.SUBMITTING).update( status=FilingDraft.Status.DRAFT, updated_at=timezone.now(), @@ -80,6 +72,13 @@ def _failed_before_external_call(payload: dict) -> bool: return isinstance(error, str) and error.startswith(_PRE_SUBMIT_ERROR_PREFIXES) +def _confirmed_api_rejection(payload: dict) -> bool: + """True when the filing API answered with a 4xx: the filing was definitely not accepted.""" + code = payload.get("api_status_code") + # A timeout is ambiguous, and a conflict can mean the filing already exists. + return isinstance(code, int) and 400 <= code < 500 and code not in (408, 409) + + @csrf_exempt @require_http_methods(["POST"]) def submit_final_filing(request): @@ -91,7 +90,7 @@ def submit_final_filing(request): # Claim the draft before forwarding so a concurrent request can't file twice. if draft is not None and not _claim_for_submission(draft): return JsonResponse( - {"success": False, "error": "This filing is already being submitted."}, + {"success": False, "error": "This filing can't be submitted again automatically."}, status=409, ) @@ -104,14 +103,14 @@ def submit_final_filing(request): if response.status_code < 400 and payload.get("success") is True: draft.mark_submitted(payload.get("api_response") or {}) clear_current_draft(request) - elif _failed_before_external_call(payload): - # No external call happened, so it is safe to free the draft for retry. + elif _failed_before_external_call(payload) or _confirmed_api_rejection(payload): + # Nothing was filed (rejected before the call, or the API refused it), + # so it is safe to return the draft to DRAFT for a corrected retry. _release_claim(draft) else: - # The API failed, or the outcome is ambiguous (an error after requests.post - # may mean the filing went through). Record an error and leave it OUT of - # DRAFT so a blind retry cannot silently double-file. (True exactly-once - # would require an idempotency key on the external filing API.) + # Ambiguous: a network error or an error after requests.post may mean the + # filing went through. Leave it ERROR for manual review -- ERROR is not + # claimable, so a click cannot silently re-file. draft.mark_error({"status_code": response.status_code, "response": payload}) return response From d2558b3a1b4593278adba1a46314f44461b9674c Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Wed, 22 Jul 2026 11:36:30 -0400 Subject: [PATCH 38/47] Typing --- efile_app/.env.example | 1 + efile_app/efile/tests/test_durable_drafts.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/efile_app/.env.example b/efile_app/.env.example index 3064fba..bda32bf 100644 --- a/efile_app/.env.example +++ b/efile_app/.env.example @@ -6,6 +6,7 @@ DJANGO_LOG_LEVEL=DEBUG DATABASE_URL=postgresql://user:password@localhost:5432/dbname SUFFOLK_EFILE_API_KEY = "your-suffolk-efile-api-key-here" +EFSP_URL = "https://efile-test.suffolklitlab.org" # AWS S3 Configuration AWS_S3_ENDPOINT_URL = "http://host.docker.internal:4566" # if mocking S3 locally, see [testing README](../testing/README.md). diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index 6fb3f31..80ff5b0 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -143,8 +143,8 @@ def test_supplemental_case_fields_round_trip(django_user_model): assert draft.supplemental_fields == {"has_children": "false", "child_count": "2"} assert read_case_data(draft)["has_children"] == "false" assert read_case_data(draft)["child_count"] == "2" - assert "unknown_answer" not in read_case_data(draft) - assert draft_snapshot(draft)["supplemental_fields"] == {"has_children": "false", "child_count": "2"} + assert (snapshot := draft_snapshot(draft)) is not None + assert snapshot["supplemental_fields"] == {"has_children": "false", "child_count": "2"} @pytest.mark.django_db From 4bef4245980cbd5008fcd33ba5fe80cfa82f6d1e Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Wed, 22 Jul 2026 15:01:49 -0400 Subject: [PATCH 39/47] WIP - get end to end filing to work (need to cleanup some localstack/cloudflare stuff still) --- compose.yml | 56 ++++++++- efile_app/efile/api/filing_views.py | 92 ++++++++++++++- efile_app/efile/api/s3_upload.py | 37 +++++- efile_app/efile/api/urls.py | 2 + efile_app/efile/services/drafts.py | 14 ++- efile_app/efile/settings_base.py | 9 +- efile_app/efile/settings_dev.py | 2 +- .../efile/static/js/cascading-dropdowns.js | 23 +++- efile_app/efile/static/js/payment.js | 15 ++- efile_app/efile/static/js/review.js | 15 ++- .../efile/static/js/upload-handler-first.js | 85 +++++++++++++- efile_app/efile/static/js/upload-handler.js | 66 ++++++++--- .../efile/templates/efile/expert_form.html | 3 +- efile_app/efile/templates/efile/options.html | 3 +- efile_app/efile/templates/efile/payment.html | 2 +- efile_app/efile/templates/efile/review.html | 2 +- efile_app/efile/templates/efile/upload.html | 11 +- .../efile/templates/efile/upload_first.html | 2 +- efile_app/efile/tests/test_durable_drafts.py | 79 +++++++++++++ .../efile/tests/test_local_document_urls.py | 110 ++++++++++++++++++ efile_app/efile/utils/s3_upload_handler.py | 56 +++++++++ efile_app/efile/views/options.py | 25 +++- efile_app/efile/views/session_api.py | 9 +- efile_app/efile/views/upload.py | 7 +- .../js-tests/cascading-dropdowns.test.js | 50 ++++++++ .../ready.d/01-create-s3-bucket.sh | 8 ++ 26 files changed, 728 insertions(+), 55 deletions(-) create mode 100644 efile_app/efile/tests/test_local_document_urls.py create mode 100644 efile_app/js-tests/cascading-dropdowns.test.js create mode 100755 testing/localstack-init/ready.d/01-create-s3-bucket.sh diff --git a/compose.yml b/compose.yml index 04d71f8..3c5160f 100644 --- a/compose.yml +++ b/compose.yml @@ -1,27 +1,79 @@ services: + localstack: + # Keep the pre-auth Community Edition for local development. Newer + # calendar-versioned images require a LocalStack account token. + image: localstack/localstack:3.8.0 + ports: + - "127.0.0.1:4566:4566" + - "127.0.0.1:4510-4559:4510-4559" + environment: + DEBUG: "${DEBUG:-0}" + SERVICES: s3 + AWS_DEFAULT_REGION: us-east-1 + AWS_S3_BUCKET_NAME: forms-mvp-xf6361 + volumes: + - localstack:/var/lib/localstack + - /var/run/docker.sock:/var/run/docker.sock + - ./testing/localstack-init:/etc/localstack/init/ready.d:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"] + interval: 5s + timeout: 5s + retries: 20 + web: build: context: . dockerfile: Dockerfile image: form-submission-mvp:web command: >- - sh -c "uv run python efile_app/manage.py migrate --noinput --fake-initial && - uv run python efile_app/manage.py runserver 0.0.0.0:8000" + sh -c "uv run python manage.py migrate --noinput --fake-initial && + uv run python manage.py runserver 0.0.0.0:8000" + depends_on: + localstack: + condition: service_healthy ports: - "8000:8000" environment: PYTHONDONTWRITEBYTECODE: "1" PYTHONUNBUFFERED: "1" + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_S3_BUCKET_NAME: forms-mvp-xf6361 + AWS_S3_REGION_NAME: us-east-1 + AWS_S3_ENDPOINT_URL: http://localstack:4566 + AWS_ACCOUNT_ID_ENDPOINT_MODE: disabled + DJANGO_ALLOWED_HOSTS: "*" + LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG: /shared/cloudflared.log + LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS: "30" # DJANGO_SETTINGS_MODULE: efile.settings # manage.py sets this; uncomment to override volumes: - .:/app:delegated - venv:/app/.venv + - public-tunnel:/shared:ro # healthcheck: # test: ["CMD-SHELL", "wget -qO- http://localhost:8000/ || exit 1"] # interval: 10s # timeout: 5s # retries: 5 + # The remote EFSP server fetches document URLs from outside Docker. A + # Cloudflare quick tunnel makes the local document proxy reachable without + # an account or per-developer configuration. + public-tunnel: + image: cloudflare/cloudflared:latest + command: tunnel --no-autoupdate --url http://web:8000 --logfile /shared/cloudflared.log + user: "0:0" + depends_on: + web: + condition: service_started + volumes: + - public-tunnel:/shared + volumes: + localstack: + driver: local venv: driver: local + public-tunnel: + driver: local diff --git a/efile_app/efile/api/filing_views.py b/efile_app/efile/api/filing_views.py index 6a8410c..6804e3e 100644 --- a/efile_app/efile/api/filing_views.py +++ b/efile_app/efile/api/filing_views.py @@ -4,6 +4,7 @@ import json import logging +from urllib.parse import unquote, urlparse import requests from django.conf import settings @@ -15,10 +16,88 @@ from ..utils.case_data_utils import get_case_data from ..utils.proxy_connection import get_headers +from ..utils.s3_upload_handler import S3UploadHandler from .base import APIResponseMixin logger = logging.getLogger(__name__) + +def rewrite_local_document_urls(efile_data): + """Refresh stale Docker-only document URLs before an EFSP request. + + Drafts can outlive a local tunnel restart. Rebuild URLs from their S3 keys + when a draft still contains a LocalStack or host-only URL. + """ + if not getattr(settings, "LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG", ""): + return + + for bundle in efile_data.get("al_court_bundle", []): + document_url = bundle.get("data_url") + if not document_url: + continue + + parsed_url = urlparse(document_url) + if not parsed_url.scheme and document_url.startswith("efile-documents/"): + public_url = S3UploadHandler.get_local_public_url(document_url) + if public_url: + bundle["data_url"] = public_url + continue + + is_quick_tunnel = bool(parsed_url.hostname and parsed_url.hostname.endswith(".trycloudflare.com")) + if parsed_url.hostname not in {"localstack", "localhost", "127.0.0.1", "host.docker.internal"} and not is_quick_tunnel: + continue + + marker = "/efile-documents/" + marker_position = parsed_url.path.find(marker) + if marker_position == -1: + continue + + s3_key = unquote(parsed_url.path[marker_position + 1 :]) + public_url = S3UploadHandler.get_local_public_url(s3_key) + if public_url: + bundle["data_url"] = public_url + + +def rewrite_fallback_filing_components(efile_data, jurisdiction_id, court_id): + """Replace legacy supporting-document labels with the court's code.""" + bundles_needing_component = [ + bundle + for bundle in efile_data.get("al_court_bundle", []) + if str(bundle.get("filing_component", "")).lower() in {"", "supporting", "attachment", "attachments"} + ] + if not bundles_needing_component: + return + + for bundle in bundles_needing_component: + filing_type = bundle.get("filing_type") + if not filing_type: + continue + url = ( + f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/codes/courts/{court_id}/" + f"filing_types/{filing_type}/filing_components" + ) + try: + response = requests.get(url, timeout=10) + if response.status_code != 200: + continue + components = response.json() + if not isinstance(components, list): + continue + attachment = next( + ( + component + for component in components + if str(component.get("name", "")).lower() in {"attachment", "attachments"} + ), + None, + ) + component = attachment or (components[0] if components else None) + component_code = component.get("code") if isinstance(component, dict) else None + if component_code: + bundle["filing_component"] = component_code + except (requests.RequestException, ValueError, TypeError): + logger.exception("Could not resolve filing component for filing type %s", filing_type) + # TODO(brycew): this file doesn't work in it's current state. Keeping # around for later refactors, when we inevitably want to start letting users # handle filings themselves / see current status, etc. @@ -170,10 +249,15 @@ def payment_fees(request): if not efile_data: return JsonResponse({"success": False, "error": "No efile data provided in request"}, status=400) + if efile_data.get("cross_references") is None or efile_data.get("cross_references") == "": + efile_data.pop("cross_references", None) + rewrite_local_document_urls(efile_data) + jurisdiction_id = request.session.get("jurisdiction") auth_tokens = request.session.get("auth_tokens", {}) case_data = get_case_data(request, jurisdiction_id) court_id = case_data.get("court", "") + rewrite_fallback_filing_components(efile_data, jurisdiction_id, court_id) url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filing/fees" headers = get_headers() @@ -190,7 +274,7 @@ def payment_fees(request): logger.warning(f"No Tyler token found for jurisdiction '{jurisdiction_id}' in filing submission") logger.info(f"Making request!: {url}") - response = requests.post(url, json=efile_data, headers=headers) + response = requests.post(url, json=efile_data, headers=headers, timeout=60) logger.info(f"Made request: {response.status_code}") if response.status_code == 200 or response.status_code == 201: @@ -205,9 +289,13 @@ def payment_fees(request): } ) else: + logger.info("EFSP fee response body: %s", response.text[:2000]) try: error_data = response.json() - error_message = error_data.get("error", f"API returned status {response.status_code}") + if isinstance(error_data, dict): + error_message = error_data.get("error", f"API returned status {response.status_code}") + else: + error_message = error_data logger.info(f"Sending back: {error_data}, {error_message}") # For 400 errors, include more details diff --git a/efile_app/efile/api/s3_upload.py b/efile_app/efile/api/s3_upload.py index 5c12032..17255e2 100644 --- a/efile_app/efile/api/s3_upload.py +++ b/efile_app/efile/api/s3_upload.py @@ -1,7 +1,8 @@ import logging import uuid -from django.http import JsonResponse +from botocore.exceptions import ClientError +from django.http import FileResponse, HttpResponseNotFound, JsonResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods @@ -133,6 +134,40 @@ def simple_s3_upload(request): return JsonResponse({"success": False, "error": f"Upload error: {str(e)}"}, status=500) +@require_http_methods(["GET", "HEAD"]) +def public_s3_upload(request, key): + """Serve a LocalStack document to the remote EFSP during local testing. + + The route is limited to files generated by this application. Production + deployments do not configure the tunnel, so their uploads continue using + direct S3 URLs. + """ + if not key.startswith("efile-documents/") or ".." in key.split("/"): + return HttpResponseNotFound() + + from ..utils.s3_upload_handler import S3UploadHandler + + handler = S3UploadHandler() + if not handler._ensure_initialized() or handler.s3_client is None: + return HttpResponseNotFound() + + try: + object_data = handler.s3_client.get_object(Bucket=handler.bucket_name, Key=key) + except ClientError as error: + error_code = error.response.get("Error", {}).get("Code") + if error_code not in {"NoSuchKey", "404", "NoSuchBucket"}: + logger.exception("Could not read public local upload %s", key) + return HttpResponseNotFound() + + response = FileResponse( + object_data["Body"], + content_type=object_data.get("ContentType", "application/octet-stream"), + ) + if object_data.get("ContentLength") is not None: + response["Content-Length"] = str(object_data["ContentLength"]) + return response + + @csrf_exempt @require_http_methods(["POST"]) def mock_s3_upload(request): diff --git a/efile_app/efile/api/urls.py b/efile_app/efile/api/urls.py index 605b1e1..0d9aa80 100644 --- a/efile_app/efile/api/urls.py +++ b/efile_app/efile/api/urls.py @@ -26,6 +26,7 @@ from .filing_views import create_filing, delete_filing, get_filing_detail, get_filings, payment_fees, update_filing from .s3_upload import ( mock_s3_upload, + public_s3_upload, simple_s3_upload, test_s3_connection, ) @@ -36,6 +37,7 @@ urlpatterns = [ path("get-party-types/", get_party_types_from_suffolk_api, name="get_party_types"), path("simple-s3-upload/", simple_s3_upload, name="simple_s3_upload"), + path("public-upload/", public_s3_upload, name="public_s3_upload"), path("mock-s3-upload/", mock_s3_upload, name="mock_s3_upload"), path("test-s3-connection/", test_s3_connection, name="test_s3_connection"), # path("api/create-filing/", create_filing, name="create_filing"), diff --git a/efile_app/efile/services/drafts.py b/efile_app/efile/services/drafts.py index 15d660f..0967151 100644 --- a/efile_app/efile/services/drafts.py +++ b/efile_app/efile/services/drafts.py @@ -331,7 +331,19 @@ def _apply_document(doc: FilingDocument, file_obj: dict[str, Any], config: dict[ } for model_field, config_key in config_fields.items(): if config_key in config: - setattr(doc, model_field, _as_str(config.get(config_key))) + value = config.get(config_key) + if isinstance(value, dict): + value = value.get("id") or value.get("code") or "" + setattr(doc, model_field, _as_str(value)) + + # Older upload clients stored the selected component on the file object as + # {id, name}, while the durable draft config was empty. Preserve that code + # so the payment payload never falls back to the literal "supporting". + if not doc.filing_component_code and file_obj.get("filing_component"): + value = file_obj["filing_component"] + if isinstance(value, dict): + value = value.get("id") or value.get("code") or "" + doc.filing_component_code = _as_str(value) def _upsert_document( diff --git a/efile_app/efile/settings_base.py b/efile_app/efile/settings_base.py index 3a13a32..86efa8a 100644 --- a/efile_app/efile/settings_base.py +++ b/efile_app/efile/settings_base.py @@ -16,7 +16,8 @@ DEBUG = True # Override in env-specific settings -ALLOWED_HOSTS: list[str] = [] +_configured_allowed_hosts = os.getenv("DJANGO_ALLOWED_HOSTS", "") +ALLOWED_HOSTS: list[str] = [host.strip() for host in _configured_allowed_hosts.split(",") if host.strip()] # Override in env-specific settings CSRF_TRUSTED_ORIGINS: list[str] = [] @@ -114,6 +115,12 @@ AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME", "us-east-1") AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", None) +# LocalStack URLs are only resolvable inside Docker. Local compose starts a +# public quick tunnel and shares its log with the web container so the remote +# EFSP server can fetch uploaded documents during local development. +LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG = os.getenv("LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG", "") +LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS = float(os.getenv("LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS", "30")) + # File Upload Settings MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB ALLOWED_FILE_TYPES = [".pdf", ".doc", ".docx"] diff --git a/efile_app/efile/settings_dev.py b/efile_app/efile/settings_dev.py index f97d173..dd3e06e 100644 --- a/efile_app/efile/settings_dev.py +++ b/efile_app/efile/settings_dev.py @@ -17,7 +17,7 @@ DATABASES = BASE_DATABASES DEBUG = True -ALLOWED_HOSTS = ["localhost", "127.0.0.1", ".localhost", "[::1]", "testserver"] +ALLOWED_HOSTS = ALLOWED_HOSTS or ["localhost", "127.0.0.1", ".localhost", "[::1]", "testserver"] CSRF_TRUSTED_ORIGINS = [ "http://localhost", "http://127.0.0.1", diff --git a/efile_app/efile/static/js/cascading-dropdowns.js b/efile_app/efile/static/js/cascading-dropdowns.js index 2b27086..23eb0c5 100644 --- a/efile_app/efile/static/js/cascading-dropdowns.js +++ b/efile_app/efile/static/js/cascading-dropdowns.js @@ -30,6 +30,9 @@ class CascadingDropdowns { case_type: null, party_type: null, }; + // A new filing has no uploaded document yet, so the upload-data + // response may not contain document classification guesses. + this.guesses = {}; this.optionalServicesLoaded = false; this.isAutomaticSelection = false; // Track if selection is automatic } @@ -135,8 +138,15 @@ class CascadingDropdowns { } async loadGuesses() { - const data = await apiUtils.getUploadData(); - this.guesses = data['guesses'] + try { + const data = await apiUtils.getUploadData(); + this.guesses = data?.guesses || {}; + } catch (error) { + // Guesses are optional and should never prevent a new filing from + // using the cascading dropdowns. + console.warn("Could not load upload guesses:", error); + this.guesses = {}; + } } async loadDropdownData(fieldId, endpoint, params = {}) { @@ -190,6 +200,9 @@ class CascadingDropdowns { } handleDropdownChange(dropdown) { + // Upload guesses are optional. Keep the change handler safe even if a + // stale/partial response or another caller clears the property. + this.guesses = this.guesses || {}; const fieldId = dropdown.id; const selectedValue = dropdown.value; const mapping = this.dropdownMapping[fieldId]; @@ -259,8 +272,8 @@ class CascadingDropdowns { if (this.validateParameters(fieldId, params)) { // Load data for the next dropdown only if there is a next dropdown if (mapping.next && mapping.endpoint) { - params.guessed_case_category = this.guesses['case category']; - params.guessed_case_type = this.guesses['case type']; + params.guessed_case_category = this.guesses?.['case category']; + params.guessed_case_type = this.guesses?.['case type']; params.only_required = true; this.loadDropdownData(mapping.next, mapping.endpoint, params); } @@ -1015,4 +1028,4 @@ if (typeof module !== "undefined" && module.exports) { module.exports = CascadingDropdowns; } else { window.CascadingDropdowns = CascadingDropdowns; -} \ No newline at end of file +} diff --git a/efile_app/efile/static/js/payment.js b/efile_app/efile/static/js/payment.js index 325facc..df6686f 100644 --- a/efile_app/efile/static/js/payment.js +++ b/efile_app/efile/static/js/payment.js @@ -400,7 +400,10 @@ const FilingHandler = { email: userData.email, party_type: partyType, date_of_birth: "", - is_form_filler: true, + // The authenticated account is a firm filer, not a self-represented + // party. Marking the filer as the party makes Tyler reject fees with + // "Cannot specify self as party for a firm filer." + is_form_filler: false, name: { first: firstName, middle: middleName, @@ -473,7 +476,6 @@ const FilingHandler = { other_parties, user_started_case: !caseData?.previous_case_id, al_court_bundle: [], - cross_references: "", comments_to_clerk: "", tyler_payment_id: paymentAccountID, lead_contact: { @@ -520,11 +522,16 @@ const FilingHandler = { if (uploadData?.files?.supporting?.length > 0) { uploadData.files.supporting.forEach((doc, index) => { const config = uploadData.supporting_documents?.[index] || {}; + const configuredComponent = config.filing_component; + const documentComponent = doc.filing_component; + const filingComponent = typeof configuredComponent === "object" + ? configuredComponent.id + : configuredComponent || (typeof documentComponent === "object" ? documentComponent.id : documentComponent); const bundle = this.createDocumentBundle( doc, config.filing_type || caseData.filing_type_id, config.document_type || caseData.document_type, - config.filing_component || "supporting", + filingComponent || caseData.filing_component, users, config.filing_type_name || `Supporting Document ${index + 1}`, config.document_type_name || "", @@ -623,4 +630,4 @@ window.PaymentHandler = PaymentHandler; window.Navigation = Navigation; // Initialize app when DOM is ready -document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); \ No newline at end of file +document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); diff --git a/efile_app/efile/static/js/review.js b/efile_app/efile/static/js/review.js index 55a8022..b64b6e0 100644 --- a/efile_app/efile/static/js/review.js +++ b/efile_app/efile/static/js/review.js @@ -560,7 +560,10 @@ const FilingHandler = { email: userData.email, party_type: partyType, date_of_birth: "", - is_form_filler: true, + // The authenticated account is a firm filer, not a self-represented + // party. Marking the filer as the party makes Tyler reject fees with + // "Cannot specify self as party for a firm filer." + is_form_filler: false, name: { first: firstName, middle: middleName, @@ -633,7 +636,6 @@ const FilingHandler = { other_parties, user_started_case: !caseData?.previous_case_id, al_court_bundle: [], - cross_references: "", comments_to_clerk: "", tyler_payment_id: paymentAccountID, lead_contact: { @@ -680,11 +682,16 @@ const FilingHandler = { if (uploadData?.files?.supporting?.length > 0) { uploadData.files.supporting.forEach((doc, index) => { const config = uploadData.supporting_documents?.[index] || {}; + const configuredComponent = config.filing_component; + const documentComponent = doc.filing_component; + const filingComponent = typeof configuredComponent === "object" + ? configuredComponent.id + : configuredComponent || (typeof documentComponent === "object" ? documentComponent.id : documentComponent); const bundle = this.createDocumentBundle( doc, config.filing_type || caseData.filing_type_id, config.document_type || caseData.document_type, - config.filing_component || "supporting", + filingComponent || caseData.filing_component, users, config.filing_type_name || `Supporting Document ${index + 1}`, config.document_type_name || "", @@ -793,4 +800,4 @@ window.queryFees = FilingHandler.queryFees.bind(FilingHandler); window.Navigation = Navigation; // Initialize app when DOM is ready -document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); \ No newline at end of file +document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); diff --git a/efile_app/efile/static/js/upload-handler-first.js b/efile_app/efile/static/js/upload-handler-first.js index de6d5ff..54dfc58 100644 --- a/efile_app/efile/static/js/upload-handler-first.js +++ b/efile_app/efile/static/js/upload-handler-first.js @@ -14,6 +14,8 @@ class UploadHandler { this.successAlert = document.getElementById('successAlert'); this.uploadedFile = null; + this.uploadPromise = null; + this.leadPersisted = false; this.initialized = false; @@ -52,6 +54,7 @@ class UploadHandler { const lead = response?.files?.lead; if (lead) { this.uploadedFile = lead; + this.leadPersisted = true; this.updateFilePreview(this.leadDocumentArea, lead); document.getElementById("leadDocument").removeAttribute("required"); } @@ -59,7 +62,11 @@ class UploadHandler { async saveUploadDataToSession(uploadData) { try { - const result = await apiUtils.saveFirstUploadData(uploadData); + const payload = { + ...uploadData, + jurisdiction_id: uploadData.jurisdiction_id || apiUtils.getCurrentJurisdiction() + }; + const result = await apiUtils.saveFirstUploadData(payload); if (!result.success) { throw new Error(result.error || 'Failed to save upload data to session'); } @@ -165,7 +172,7 @@ class UploadHandler { } // Automatically upload lead document - this.uploadFileImmediately(validFiles[0], 0); + this.uploadPromise = this.uploadFileImmediately(validFiles[0], 0); this.updateSubmitButton(); } @@ -293,6 +300,32 @@ class UploadHandler { // Store the upload result for later use during form submission this.uploadedFile.uploadResult = result; + // Persist the lead as soon as S3 accepts it. This keeps a refresh + // or navigation from losing the document before the user clicks + // Continue. + const uploadedLead = result.files?.[0]; + if (uploadedLead) { + try { + await this.saveUploadDataToSession({ + files: { + lead: { + name: this.uploadedFile.name, + size: this.uploadedFile.size, + type: this.uploadedFile.type, + url: uploadedLead.public_url || uploadedLead.url, + s3_key: uploadedLead.key, + }, + }, + options: { lead: {} }, + }); + this.leadPersisted = true; + } catch (error) { + // Continue still retries the same save, so an interim + // persistence failure should not discard the S3 result. + console.warn('Could not persist lead upload immediately:', error); + } + } + } catch (error) { console.error('Error uploading file immediately:', error); this.showFileUploadError(file.name, index, error.message); @@ -344,6 +377,45 @@ class UploadHandler { return; } + // File selection starts an asynchronous S3 upload. Wait for both the + // upload and its durable-draft save before navigating away. + if (this.uploadPromise) { + await this.uploadPromise; + this.uploadPromise = null; + } + + if (!this.uploadedFile.uploadResult && !(this.uploadedFile.url && this.uploadedFile.s3_key)) { + this.showError('The lead document upload did not finish. Please try again.'); + return; + } + + if (this.uploadedFile.uploadResult && !this.leadPersisted) { + this.showWaiting("Saving your document..."); + try { + const uploadedLead = this.uploadedFile.uploadResult.files?.[0]; + if (!uploadedLead) { + throw new Error('The lead document upload did not return a file.'); + } + await this.saveUploadDataToSession({ + files: { + lead: { + name: this.uploadedFile.name, + size: this.uploadedFile.size, + type: this.uploadedFile.type, + url: uploadedLead.public_url || uploadedLead.url, + s3_key: uploadedLead.key, + }, + }, + options: { lead: {} }, + }); + this.leadPersisted = true; + } catch (error) { + console.error('Error persisting lead upload:', error); + this.showError(error.message); + return; + } + } + if (this.uploadedFile.url && this.uploadedFile.s3_key) { // We've already uploaded the file previously. Just continue. const jurisdiction = apiUtils.getCurrentJurisdiction(); @@ -376,8 +448,11 @@ class UploadHandler { }; } - // Save the complete upload data to session - await this.saveUploadDataToSession(uploadDataWithUrls); + // The lead was already saved after S3 accepted it. Only retry the + // legacy submit-time save if an upload result is unexpectedly absent. + if (!this.leadPersisted) { + await this.saveUploadDataToSession(uploadDataWithUrls); + } // Redirect to next page @@ -440,4 +515,4 @@ document.addEventListener('DOMContentLoaded', function() { } else { console.warn('UploadHandler already exists, skipping initialization'); } -}); \ No newline at end of file +}); diff --git a/efile_app/efile/static/js/upload-handler.js b/efile_app/efile/static/js/upload-handler.js index 4973487..e1c5b8b 100644 --- a/efile_app/efile/static/js/upload-handler.js +++ b/efile_app/efile/static/js/upload-handler.js @@ -115,8 +115,15 @@ class UploadHandler { console.error('Error syncing form data:', error); } } - // Take stuff from server and show on page - let upload_data = await apiUtils.getUploadData(); + // Take stuff from server and show on page. Upload metadata is optional + // while a draft is being created, and the page should still initialize + // if the metadata request temporarily fails. + let upload_data = {}; + try { + upload_data = await apiUtils.getUploadData() || {}; + } catch (error) { + console.warn('Could not load saved upload data:', error); + } await this.prepLeadFileSelection(upload_data); await this.prepSupportingFileSelection(upload_data); @@ -273,7 +280,14 @@ class UploadHandler { } async prepLeadFileSelection(upload_data) { - let lead = upload_data.files.lead; + const lead = upload_data?.files?.lead; + + // A draft may not have a lead document yet (for example after an + // interrupted upload). Leave the options hidden and let the page + // recover without throwing during initialization. + if (!lead) { + return; + } // Add file previews const preview = this.createFilePreviewNoRemove(lead.name, lead.size); @@ -306,12 +320,13 @@ class UploadHandler { } async prepSupportingFileSelection(upload_data) { - this.uploadedFiles = upload_data.files.supporting || []; + this.uploadedFiles = upload_data?.files?.supporting || []; if (this.uploadedFiles && this.uploadedFiles.length > 0) { this.uploadedFileStatuses = this.uploadedFiles.map(f => FileStatus.SUCCESS); this.updateFilePreview(this.supportingDocumentsArea, this.uploadedFiles, this.uploadedFileStatuses); - for (let index = 0; index < upload_data.supporting_documents.length; index++) { - let d = upload_data.supporting_documents[index]; + const supportingDocuments = upload_data?.supporting_documents || []; + for (let index = 0; index < supportingDocuments.length; index++) { + let d = supportingDocuments[index]; if (d.filing_type) { this.initializeFilingTypeDropdown(document.getElementById(`supportingFilingType${index}_search`)); @@ -681,6 +696,11 @@ class UploadHandler { this.showError(`Please select a filing component for supporting document: ${this.uploadedFiles[i].name}`); return; } + const supportingDocumentType = document.getElementById(`supportingDocumentType${i}`)?.value; + if (!supportingDocumentType) { + this.showError(`Please select a document type for supporting document: ${this.uploadedFiles[i].name}`); + return; + } } try { @@ -692,6 +712,12 @@ class UploadHandler { const leadFilingTypeName = leadFilingTypeSelect && leadFilingTypeSelect.selectedOptions[0] ? leadFilingTypeSelect.selectedOptions[0].text : ''; const leadDocumentType = leadDocumentTypeSelect ? leadDocumentTypeSelect.value : ''; const leadDocumentTypeName = leadDocumentTypeSelect && leadDocumentTypeSelect.selectedOptions[0] ? leadDocumentTypeSelect.selectedOptions[0].text : ''; + + if (!leadFilingType || !leadDocumentType) { + this.showError('Please select a filing type and document type for the lead document.'); + return; + } + const leadFilingComponentValue = this.globalFilingComponentLead.id; const leadFilingComponentName = this.globalFilingComponentLead.name; @@ -759,8 +785,8 @@ class UploadHandler { this.uploadedFiles.forEach((file, index) => { const supportingFilingComponent = this.globalFilingComponentSupport; if (file.uploadResult) { - file.s3_key = file.uploadResult.files[0]?.public_url; - file.url = file.uploadResult.files[0]?.key; + file.url = file.uploadResult.files[0]?.public_url; + file.s3_key = file.uploadResult.files[0]?.key; } if (file.s3_key && file.url) { uploadDataWithUrls.files.push({ @@ -894,6 +920,16 @@ class UploadHandler { }; } }); + + // Some filing types expose only one filing component. In + // that case the EFSP still expects that component's code for + // supporting documents; never send the UI label "supporting". + if (!this.globalFilingComponentSupport.id && this.globalFilingComponentLead.id) { + this.globalFilingComponentSupport = { + id: this.globalFilingComponentLead.id, + name: this.globalFilingComponentLead.name + }; + } } else { console.error("API returned error:", result.error); } @@ -919,7 +955,13 @@ class UploadHandler { return; } - let guesses = (await apiUtils.getUploadData())['guesses']; + let uploadData = {}; + try { + uploadData = await apiUtils.getUploadData() || {}; + } catch (error) { + console.warn('Could not load upload guesses:', error); + } + const guesses = uploadData.guesses || {}; // Fetch filing types data only once if (this.globalFilingTypes.length === 0) { @@ -1078,14 +1120,12 @@ function createSupportingDocumentOptions(index, fileName) { name="supportingFilingType${index}_search" placeholder="Search filing types..." autocomplete="off" - required /> @@ -1102,7 +1142,7 @@ function createSupportingDocumentOptions(index, fileName) {
-
@@ -1135,4 +1175,4 @@ document.addEventListener('DOMContentLoaded', function() { } else { console.warn('UploadHandler already exists, skipping initialization'); } -}); \ No newline at end of file +}); diff --git a/efile_app/efile/templates/efile/expert_form.html b/efile_app/efile/templates/efile/expert_form.html index fe8c385..c08f6b5 100644 --- a/efile_app/efile/templates/efile/expert_form.html +++ b/efile_app/efile/templates/efile/expert_form.html @@ -694,7 +694,8 @@
Case Information Found
- + + diff --git a/efile_app/efile/templates/efile/options.html b/efile_app/efile/templates/efile/options.html index 6eed8ea..bfd66f8 100644 --- a/efile_app/efile/templates/efile/options.html +++ b/efile_app/efile/templates/efile/options.html @@ -101,7 +101,8 @@

{% translate "View past filings" %}

window.location.href = result.redirect_url || `/jurisdiction/{{jurisdiction}}/upload_first/`; }); } else { - // TODO(brycew): go straight to the page where the existing session was, reload everything + const resumeUrl = "{{ resume_url|default:'' }}"; + window.location.href = resumeUrl || `/jurisdiction/{{jurisdiction}}/upload_first/`; } } diff --git a/efile_app/efile/templates/efile/payment.html b/efile_app/efile/templates/efile/payment.html index 552148e..1cbbd89 100644 --- a/efile_app/efile/templates/efile/payment.html +++ b/efile_app/efile/templates/efile/payment.html @@ -73,7 +73,7 @@

{% translate "Filing Fees" %}

{{ new_toga_url|json_script:"new-toga-url" }} - + diff --git a/efile_app/efile/templates/efile/review.html b/efile_app/efile/templates/efile/review.html index af4a917..34ec99a 100644 --- a/efile_app/efile/templates/efile/review.html +++ b/efile_app/efile/templates/efile/review.html @@ -239,7 +239,7 @@

{% translate "Submit my filing" %}

{{ friendly_names|json_script:"friendly-names" }} - + - + - + diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index 80ff5b0..f32c60b 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -1,4 +1,5 @@ import json +from unittest.mock import patch import pytest from django.urls import reverse @@ -319,6 +320,84 @@ def test_create_draft_view_requires_jurisdiction_token(client, django_user_model assert not FilingDraft.objects.exists() +@pytest.mark.django_db +def test_options_page_points_resume_to_draft_workflow_step(client, django_user_model): + user = django_user_model.objects.create_user( + username="resume-user", + password="testpass123", + tyler_jurisdiction="illinois", + ) + draft = FilingDraft.objects.create( + user=user, + jurisdiction="illinois", + current_step=WorkflowStepKey.DOCUMENTS, + ) + client.force_login(user) + _authorize_jurisdiction_session(client) + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = draft.pk + session.save() + + response = client.get(reverse("efile_options", kwargs={"jurisdiction": "illinois"})) + + assert response.status_code == 200 + assert reverse("upload", kwargs={"jurisdiction": "illinois"}).encode() in response.content + + +@pytest.mark.django_db +def test_documents_page_returns_to_lead_upload_when_lead_is_missing(client, django_user_model): + user = django_user_model.objects.create_user(username="missing-lead-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + write_case_data(draft, {"court": "cook:cd", "case_type": "Name Change"}) + client.force_login(user) + _authorize_jurisdiction_session(client) + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = draft.pk + session.save() + + response = client.get(reverse("upload", kwargs={"jurisdiction": "illinois"})) + + assert response.status_code == 302 + assert response.url == reverse("upload_first", kwargs={"jurisdiction": "illinois"}) + + +@pytest.mark.django_db +def test_first_upload_save_uses_current_draft_jurisdiction(client, django_user_model): + """A first-upload request without a jurisdiction must not fall into ``default``.""" + user = django_user_model.objects.create_user(username="first-upload-owner", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + client.force_login(user) + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = draft.pk + session["jurisdiction"] = "illinois" + session.save() + + with ( + patch("efile.views.session_api.requests.get") as get_file, + patch("efile.views.session_api.extract_fields_from_file", return_value={}), + ): + get_file.return_value.content = b"%PDF-1.7" + response = client.post( + reverse("save_upload_data_to_session"), + data=json.dumps( + { + "files": { + "lead": { + "name": "petition.pdf", + "url": "http://localstack:4566/forms/petition.pdf", + "s3_key": "efile-documents/lead/petition.pdf", + } + } + } + ), + content_type="application/json", + ) + + assert response.status_code == 200 + assert FilingDocument.objects.filter(draft=draft, role=FilingDocument.Role.LEAD).exists() + assert not FilingDraft.objects.filter(user=user, jurisdiction="default").exists() + + @pytest.mark.django_db def test_current_draft_enforces_owner(client, django_user_model): illinois_user = django_user_model.objects.create_user(username="illinois-user", tyler_jurisdiction="illinois") diff --git a/efile_app/efile/tests/test_local_document_urls.py b/efile_app/efile/tests/test_local_document_urls.py new file mode 100644 index 0000000..dbf10b2 --- /dev/null +++ b/efile_app/efile/tests/test_local_document_urls.py @@ -0,0 +1,110 @@ +from django.test import override_settings + +from efile.api.filing_views import rewrite_fallback_filing_components, rewrite_local_document_urls +from efile.utils.s3_upload_handler import S3UploadHandler + + +def test_local_public_url_uses_current_tunnel_after_restart(tmp_path): + tunnel_log = tmp_path / "cloudflared.log" + tunnel_log.write_text( + "Requesting new quick Tunnel...\n" + "https://old-tunnel.trycloudflare.com\n" + "Requesting new quick Tunnel...\n" + "https://current-tunnel.trycloudflare.com\n", + encoding="utf-8", + ) + + with override_settings( + LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), + LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, + ): + assert ( + S3UploadHandler.get_local_public_url("efile-documents/lead/document.pdf") + == "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" + ) + + +def test_stale_local_document_url_is_rewritten(tmp_path): + tunnel_log = tmp_path / "cloudflared.log" + tunnel_log.write_text( + "Requesting new quick Tunnel...\nhttps://current-tunnel.trycloudflare.com\n", + encoding="utf-8", + ) + efile_data = { + "al_court_bundle": [ + { + "data_url": "http://localstack:4566/forms-mvp-xf6361/efile-documents/lead/document.pdf?signature=old" + } + ] + } + + with override_settings( + LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), + LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, + ): + rewrite_local_document_urls(efile_data) + + assert efile_data["al_court_bundle"][0]["data_url"] == ( + "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" + ) + + +def test_raw_s3_key_is_rewritten_to_local_public_url(tmp_path): + tunnel_log = tmp_path / "cloudflared.log" + tunnel_log.write_text( + "Requesting new quick Tunnel...\nhttps://current-tunnel.trycloudflare.com\n", + encoding="utf-8", + ) + efile_data = {"al_court_bundle": [{"data_url": "efile-documents/lead/document.pdf"}]} + + with override_settings( + LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), + LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, + ): + rewrite_local_document_urls(efile_data) + + assert efile_data["al_court_bundle"][0]["data_url"] == ( + "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" + ) + + +def test_stale_quick_tunnel_document_url_is_rewritten(tmp_path): + tunnel_log = tmp_path / "cloudflared.log" + tunnel_log.write_text( + "Requesting new quick Tunnel...\nhttps://current-tunnel.trycloudflare.com\n", + encoding="utf-8", + ) + efile_data = { + "al_court_bundle": [ + {"data_url": "https://old-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf"} + ] + } + + with override_settings( + LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), + LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, + ): + rewrite_local_document_urls(efile_data) + + assert efile_data["al_court_bundle"][0]["data_url"] == ( + "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" + ) + + +def test_legacy_supporting_label_is_resolved_to_court_code(monkeypatch, settings): + class Response: + status_code = 200 + + @staticmethod + def json(): + return [{"code": "331", "name": "Lead Document"}, {"code": "332", "name": "Attachments"}] + + monkeypatch.setattr("efile.api.filing_views.requests.get", lambda *args, **kwargs: Response()) + settings.EFSP_URL = "https://efile-test.example" + efile_data = { + "al_court_bundle": [{"filing_type": "27965", "filing_component": "supporting"}] + } + + rewrite_fallback_filing_components(efile_data, "illinois", "adams") + + assert efile_data["al_court_bundle"][0]["filing_component"] == "332" diff --git a/efile_app/efile/utils/s3_upload_handler.py b/efile_app/efile/utils/s3_upload_handler.py index f43f849..3a99af8 100644 --- a/efile_app/efile/utils/s3_upload_handler.py +++ b/efile_app/efile/utils/s3_upload_handler.py @@ -4,6 +4,8 @@ import logging import mimetypes +import re +import time import uuid from urllib.parse import quote @@ -191,6 +193,10 @@ def get_public_url(self, s3_key, expiration=604800): # 7 days default Get a presigned URL for an S3 object (for efile submission) Uses presigned URLs since bucket policy makes files private """ + local_url = self.get_local_public_url(s3_key) + if local_url: + return local_url + if self.s3_client: try: return self.s3_client.generate_presigned_url( @@ -202,6 +208,56 @@ def get_public_url(self, s3_key, expiration=604800): # 7 days default # Fallback to direct URL (will return 403 with current bucket policy) return f"https://{self.bucket_name}.s3.{self.region_name}.amazonaws.com/{s3_key}" + @classmethod + def get_local_public_url(cls, s3_key): + """Return a public proxy URL for a local S3 key, if configured.""" + local_tunnel_url = cls._get_local_tunnel_url() + if not local_tunnel_url: + return None + return f"{local_tunnel_url}/api/public-upload/{quote(s3_key, safe='/')}" + + @staticmethod + def _get_local_tunnel_url(): + """Read the automatic local-development tunnel URL. + + cloudflared writes the URL after it starts. Waiting here makes the + first upload reliable even if it happens while the tunnel is booting. + The shared log setting is absent outside local Docker development. + """ + log_path = getattr(settings, "LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG", "") + if not log_path: + return None + + wait_seconds = max(0, getattr(settings, "LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS", 30)) + deadline = time.monotonic() + wait_seconds + tunnel_pattern = re.compile(r"https://[a-z0-9-]+\.trycloudflare\.com", re.IGNORECASE) + + while True: + try: + with open(log_path, encoding="utf-8") as log_file: + log_contents = log_file.read() + # A named Docker volume keeps the log across container + # restarts. Ignore the previous tunnel until cloudflared has + # announced a new one. + latest_start = log_contents.rfind("Requesting new quick Tunnel") + current_log = log_contents[latest_start:] if latest_start != -1 else log_contents + matches = tunnel_pattern.findall(current_log) + if matches: + return matches[-1].rstrip("/") + except FileNotFoundError: + pass + except OSError as error: + logger.warning("Could not read local public upload tunnel log: %s", error) + return None + + if time.monotonic() >= deadline: + logger.error( + "Local public upload tunnel did not become ready within %ss", + wait_seconds, + ) + return None + time.sleep(0.25) + def delete_file(self, s3_key): """ Delete a file from S3 diff --git a/efile_app/efile/views/options.py b/efile_app/efile/views/options.py index af14559..416ded2 100644 --- a/efile_app/efile/views/options.py +++ b/efile_app/efile/views/options.py @@ -6,7 +6,29 @@ from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import get_case_data -from ..workflow import WorkflowStepKey, get_workflow_context +from ..workflow import WorkflowStepKey, get_step_url, get_workflow_context + + +def _resume_url(active_draft, jurisdiction): + """Return the safest workflow URL for an active draft. + + ``OPTIONS`` is the model default for older drafts, but resuming there would + only send the user back to this page. Start those drafts at the first filing + step instead. The fallback also keeps an invalid legacy value from breaking + the options page. + """ + if active_draft is None: + return None + + try: + current_step = WorkflowStepKey(active_draft.current_step) + except ValueError: + current_step = WorkflowStepKey.UPLOAD_FIRST + + if current_step == WorkflowStepKey.OPTIONS: + current_step = WorkflowStepKey.UPLOAD_FIRST + + return get_step_url(current_step, jurisdiction) def efile_options(request, jurisdiction): @@ -30,6 +52,7 @@ def efile_options(request, jurisdiction): "is_logged_in": is_logged_in, "case_data": case_data, "filing_draft": draft_snapshot(active_draft), + "resume_url": _resume_url(active_draft, jurisdiction), "has_case_data": bool(case_data or active_draft), } context.update(get_workflow_context(WorkflowStepKey.OPTIONS, jurisdiction)) diff --git a/efile_app/efile/views/session_api.py b/efile_app/efile/views/session_api.py index 0ffa7da..24c91e9 100644 --- a/efile_app/efile/views/session_api.py +++ b/efile_app/efile/views/session_api.py @@ -8,6 +8,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods +from ..services.current_drafts import get_current_draft from ..utils.case_data_utils import get_case_data, get_upload_data, update_case_data, update_upload_data from ..utils.llms import LlmError, extract_fields_from_file from ..utils.proxy_connection import get_party_type_code_from_api @@ -144,7 +145,13 @@ def save_upload_first_data(request): return JsonResponse({"success": False, "error": "Authentication required"}, status=401) data = json.loads(request.body) - jurisdiction_id = data.get("jurisdiction_id", "default") + current_draft = get_current_draft(request, resume_latest=False) + jurisdiction_id = ( + data.get("jurisdiction_id") + or (current_draft.jurisdiction if current_draft is not None else None) + or request.session.get("jurisdiction") + or "default" + ) if data.get("jurisdiction_id"): request.session["jurisdiction"] = jurisdiction_id upload_data = {"files": data.get("files", {})} diff --git a/efile_app/efile/views/upload.py b/efile_app/efile/views/upload.py index dea6b21..2573f75 100644 --- a/efile_app/efile/views/upload.py +++ b/efile_app/efile/views/upload.py @@ -35,6 +35,11 @@ def efile_upload(request, jurisdiction): messages.error(request, gettext("Please complete the case details first.")) return redirect("efile_options", jurisdiction=jurisdiction) + upload_data = get_upload_data(request, jurisdiction) + if not upload_data.get("files", {}).get("lead"): + messages.error(request, gettext("Please upload a lead document before continuing.")) + return redirect("upload_first", jurisdiction=jurisdiction) + filing_draft = ensure_current_draft(request, jurisdiction, current_step=WorkflowStepKey.DOCUMENTS) petitioner_info = get_petitioner_info(request, jurisdiction) @@ -45,8 +50,6 @@ def efile_upload(request, jurisdiction): friendly_filing_type = case_data.get("filing_type_name", case_classification["filing_type"]) friendly_court = case_data.get("court_name", case_classification["court"]) - upload_data = get_upload_data(request, jurisdiction) - context = { "is_logged_in": True, "case_data": case_data, diff --git a/efile_app/js-tests/cascading-dropdowns.test.js b/efile_app/js-tests/cascading-dropdowns.test.js new file mode 100644 index 0000000..bbe0301 --- /dev/null +++ b/efile_app/js-tests/cascading-dropdowns.test.js @@ -0,0 +1,50 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const CascadingDropdowns = require("../efile/static/js/cascading-dropdowns.js"); + +test("missing upload guesses do not break a new filing", async () => { + globalThis.apiUtils = { + getUploadData: async () => ({}), + getCurrentJurisdiction: () => "illinois" + }; + + const dropdowns = new CascadingDropdowns(); + await dropdowns.loadGuesses(); + + assert.deepEqual(dropdowns.guesses, {}); + + globalThis.document = { + getElementById: () => null + }; + + const requested = []; + dropdowns.resetDependentDropdowns = () => {}; + dropdowns.clearAllRecommendationNotices = () => {}; + dropdowns.clearAllDropdownVisualIndicators = () => {}; + dropdowns.validateParameters = () => true; + dropdowns.loadDropdownData = async (...args) => requested.push(args); + + assert.doesNotThrow(() => dropdowns.handleDropdownChange({ + id: "court", + value: "adams" + })); + + assert.equal(requested.length, 1); + assert.equal(requested[0][0], "case_category"); + assert.equal(requested[0][2].guessed_case_category, undefined); + assert.equal(requested[0][2].guessed_case_type, undefined); +}); + +test("failed upload guess requests fall back to an empty guess set", async () => { + globalThis.apiUtils = { + getUploadData: async () => { + throw new Error("network unavailable"); + } + }; + + const dropdowns = new CascadingDropdowns(); + await dropdowns.loadGuesses(); + + assert.deepEqual(dropdowns.guesses, {}); +}); diff --git a/testing/localstack-init/ready.d/01-create-s3-bucket.sh b/testing/localstack-init/ready.d/01-create-s3-bucket.sh new file mode 100755 index 0000000..d632fb1 --- /dev/null +++ b/testing/localstack-init/ready.d/01-create-s3-bucket.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +awslocal s3api create-bucket \ + --bucket "${AWS_S3_BUCKET_NAME:-forms-mvp-xf6361}" \ + --region "${AWS_DEFAULT_REGION:-us-east-1}" \ + 2>/dev/null || true From 0344c0d08fd71ff1faaca438f7a94cd58a0a8d1a Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Wed, 22 Jul 2026 16:21:20 -0400 Subject: [PATCH 40/47] Consolidate test vars; duplicate JS; redundant cache busting --- README.md | 5 +- compose.yml | 30 +-- efile_app/.env.example | 15 +- efile_app/efile/api/filing_views.py | 93 +------ efile_app/efile/api/s3_upload.py | 49 +--- efile_app/efile/api/urls.py | 2 - efile_app/efile/settings_base.py | 6 - efile_app/efile/settings_dev.py | 16 +- efile_app/efile/settings_prod.py | 20 ++ efile_app/efile/static/js/payment.js | 240 +----------------- efile_app/efile/static/js/review.js | 239 +---------------- .../efile/static/js/upload-handler-first.js | 47 ++-- efile_app/efile/static/js/upload-handler.js | 4 + .../efile/templates/efile/expert_form.html | 3 +- efile_app/efile/templates/efile/options.html | 3 +- efile_app/efile/templates/efile/payment.html | 4 +- efile_app/efile/templates/efile/review.html | 4 +- efile_app/efile/templates/efile/upload.html | 7 +- .../efile/templates/efile/upload_first.html | 2 +- efile_app/efile/tests/test_integration.py | 22 +- .../efile/tests/test_local_document_urls.py | 110 -------- efile_app/efile/tests/test_workflow.py | 25 ++ efile_app/efile/tests/tests.py | 22 +- efile_app/efile/utils/s3_upload_handler.py | 60 ----- efile_app/efile/views/options.py | 26 +- efile_app/efile/views/session_api.py | 4 + efile_app/efile/workflow.py | 22 ++ efile_app/tests/setup.js | 4 +- efile_app/tests/test-utils.js | 8 +- testing/README.md | 72 +++++- testing/compose.yml | 13 - 31 files changed, 286 insertions(+), 891 deletions(-) delete mode 100644 efile_app/efile/tests/test_local_document_urls.py delete mode 100644 testing/compose.yml diff --git a/README.md b/README.md index c8c8519..ca7ca45 100644 --- a/README.md +++ b/README.md @@ -242,8 +242,9 @@ of the current end-to-end testing. - __Environment variables__: Create a `.env` file in the `efile_app/` directory with: ```bash - E2E_TEST_USERNAME=your_test_email@example.com - E2E_TEST_PASSWORD=your_test_password + # Tyler test-EFM login. The Python test suite reads the same two names. + TESTS_TYLER_USERNAME=your_test_email@example.com + TESTS_TYLER_PASSWORD=your_test_password E2E_TEST_BASE_URL=http://localhost:8000 # optional, defaults to localhost:8000 ``` diff --git a/compose.yml b/compose.yml index 3c5160f..bf470bc 100644 --- a/compose.yml +++ b/compose.yml @@ -42,38 +42,28 @@ services: AWS_S3_BUCKET_NAME: forms-mvp-xf6361 AWS_S3_REGION_NAME: us-east-1 AWS_S3_ENDPOINT_URL: http://localstack:4566 + # botocore >= 1.35 routes S3 through account-ID-specific endpoints, which + # LocalStack does not serve. Disable that routing. AWS_ACCOUNT_ID_ENDPOINT_MODE: disabled - DJANGO_ALLOWED_HOSTS: "*" - LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG: /shared/cloudflared.log - LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS: "30" + # Optional: a publicly readable PDF to hand the EFSP proxy in place of the + # uploaded document, so fee quotes work without exposing this machine. + # See testing/README.md. + EFSP_TEST_DOCUMENT_URL: "${EFSP_TEST_DOCUMENT_URL:-}" # DJANGO_SETTINGS_MODULE: efile.settings # manage.py sets this; uncomment to override volumes: - .:/app:delegated - - venv:/app/.venv - - public-tunnel:/shared:ro + # The image builds its venv at /app/efile_app/.venv (Dockerfile WORKDIR). + # This must shadow that exact path, or the bind mount above exposes the + # host's venv to the container and root-owned files land in the checkout. + - venv:/app/efile_app/.venv # healthcheck: # test: ["CMD-SHELL", "wget -qO- http://localhost:8000/ || exit 1"] # interval: 10s # timeout: 5s # retries: 5 - # The remote EFSP server fetches document URLs from outside Docker. A - # Cloudflare quick tunnel makes the local document proxy reachable without - # an account or per-developer configuration. - public-tunnel: - image: cloudflare/cloudflared:latest - command: tunnel --no-autoupdate --url http://web:8000 --logfile /shared/cloudflared.log - user: "0:0" - depends_on: - web: - condition: service_started - volumes: - - public-tunnel:/shared - volumes: localstack: driver: local venv: driver: local - public-tunnel: - driver: local diff --git a/efile_app/.env.example b/efile_app/.env.example index 0ae5871..c6f63a0 100644 --- a/efile_app/.env.example +++ b/efile_app/.env.example @@ -8,13 +8,26 @@ DATABASE_URL=postgresql://user:password@localhost:5432/dbname SUFFOLK_EFILE_API_KEY = "your-suffolk-efile-api-key-here" EFSP_URL = "https://efile-test.suffolklitlab.org" +# Local development only. The EFSP proxy downloads each document itself -- for +# fee quotes as well as filings -- and cannot reach a LocalStack URL. Set this to +# any publicly readable PDF to run fees end-to-end without exposing this machine. +# Uploads still go to S3 for real; only the URL sent to the proxy changes. +# See ../testing/README.md. Leave unset to send the real document URL. +EFSP_TEST_DOCUMENT_URL = "" + # AWS S3 Configuration -AWS_S3_ENDPOINT_URL = "http://host.docker.internal:4566" # if mocking S3 locally, see [testing README](../testing/README.md). +AWS_S3_ENDPOINT_URL = "http://localstack:4566" # if mocking S3 locally, see [testing README](../testing/README.md). AWS_ACCESS_KEY_ID = "your-aws-access-key-id-here" AWS_SECRET_ACCESS_KEY = r"your-aws-secret-access-key-here" # Raw string to handle special chars AWS_S3_BUCKET_NAME = "forms-mvp-xf6361" AWS_S3_REGION_NAME = "us-east-1" +# Tyler test-EFM login, shared by the Python suite (efile/tests/) and the +# Playwright suite (tests/). CI supplies the same two names from repo secrets. +TESTS_TYLER_USERNAME = "your_test_email@example.com" +TESTS_TYLER_PASSWORD = "your_test_password" +E2E_TEST_BASE_URL = "http://localhost:8000" # Playwright only; optional + # File Upload Settings MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB ALLOWED_FILE_TYPES = ['.pdf', '.doc', '.docx'] diff --git a/efile_app/efile/api/filing_views.py b/efile_app/efile/api/filing_views.py index 6804e3e..175593f 100644 --- a/efile_app/efile/api/filing_views.py +++ b/efile_app/efile/api/filing_views.py @@ -4,7 +4,6 @@ import json import logging -from urllib.parse import unquote, urlparse import requests from django.conf import settings @@ -14,90 +13,13 @@ from efile.utils.jurisdiction_stuff import get_jurisdiction_from_request +from ..services.efsp_payload import prepare_efile_payload from ..utils.case_data_utils import get_case_data from ..utils.proxy_connection import get_headers -from ..utils.s3_upload_handler import S3UploadHandler from .base import APIResponseMixin logger = logging.getLogger(__name__) - -def rewrite_local_document_urls(efile_data): - """Refresh stale Docker-only document URLs before an EFSP request. - - Drafts can outlive a local tunnel restart. Rebuild URLs from their S3 keys - when a draft still contains a LocalStack or host-only URL. - """ - if not getattr(settings, "LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG", ""): - return - - for bundle in efile_data.get("al_court_bundle", []): - document_url = bundle.get("data_url") - if not document_url: - continue - - parsed_url = urlparse(document_url) - if not parsed_url.scheme and document_url.startswith("efile-documents/"): - public_url = S3UploadHandler.get_local_public_url(document_url) - if public_url: - bundle["data_url"] = public_url - continue - - is_quick_tunnel = bool(parsed_url.hostname and parsed_url.hostname.endswith(".trycloudflare.com")) - if parsed_url.hostname not in {"localstack", "localhost", "127.0.0.1", "host.docker.internal"} and not is_quick_tunnel: - continue - - marker = "/efile-documents/" - marker_position = parsed_url.path.find(marker) - if marker_position == -1: - continue - - s3_key = unquote(parsed_url.path[marker_position + 1 :]) - public_url = S3UploadHandler.get_local_public_url(s3_key) - if public_url: - bundle["data_url"] = public_url - - -def rewrite_fallback_filing_components(efile_data, jurisdiction_id, court_id): - """Replace legacy supporting-document labels with the court's code.""" - bundles_needing_component = [ - bundle - for bundle in efile_data.get("al_court_bundle", []) - if str(bundle.get("filing_component", "")).lower() in {"", "supporting", "attachment", "attachments"} - ] - if not bundles_needing_component: - return - - for bundle in bundles_needing_component: - filing_type = bundle.get("filing_type") - if not filing_type: - continue - url = ( - f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/codes/courts/{court_id}/" - f"filing_types/{filing_type}/filing_components" - ) - try: - response = requests.get(url, timeout=10) - if response.status_code != 200: - continue - components = response.json() - if not isinstance(components, list): - continue - attachment = next( - ( - component - for component in components - if str(component.get("name", "")).lower() in {"attachment", "attachments"} - ), - None, - ) - component = attachment or (components[0] if components else None) - component_code = component.get("code") if isinstance(component, dict) else None - if component_code: - bundle["filing_component"] = component_code - except (requests.RequestException, ValueError, TypeError): - logger.exception("Could not resolve filing component for filing type %s", filing_type) - # TODO(brycew): this file doesn't work in it's current state. Keeping # around for later refactors, when we inevitably want to start letting users # handle filings themselves / see current status, etc. @@ -249,15 +171,15 @@ def payment_fees(request): if not efile_data: return JsonResponse({"success": False, "error": "No efile data provided in request"}, status=400) - if efile_data.get("cross_references") is None or efile_data.get("cross_references") == "": - efile_data.pop("cross_references", None) - rewrite_local_document_urls(efile_data) - jurisdiction_id = request.session.get("jurisdiction") auth_tokens = request.session.get("auth_tokens", {}) case_data = get_case_data(request, jurisdiction_id) court_id = case_data.get("court", "") - rewrite_fallback_filing_components(efile_data, jurisdiction_id, court_id) + + # Must match what submit_final_filing sends, or fees are quoted + # against a payload that differs from the one actually filed. + prepare_efile_payload(efile_data, jurisdiction_id, court_id) + url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filing/fees" headers = get_headers() @@ -289,7 +211,8 @@ def payment_fees(request): } ) else: - logger.info("EFSP fee response body: %s", response.text[:2000]) + # Debug, not info: fee responses echo party names and case details. + logger.debug("EFSP fee response body: %s", response.text[:2000]) try: error_data = response.json() if isinstance(error_data, dict): diff --git a/efile_app/efile/api/s3_upload.py b/efile_app/efile/api/s3_upload.py index 17255e2..4da5336 100644 --- a/efile_app/efile/api/s3_upload.py +++ b/efile_app/efile/api/s3_upload.py @@ -1,12 +1,11 @@ import logging import uuid -from botocore.exceptions import ClientError -from django.http import FileResponse, HttpResponseNotFound, JsonResponse +from django.http import JsonResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -from ..utils.s3_upload_handler import s3_handler # noqa: F401 +from ..utils.s3_upload_handler import S3UploadHandler logger = logging.getLogger(__name__) @@ -16,10 +15,8 @@ def test_s3_connection(request): """Test S3 connection and bucket access.""" try: - # Reinitialize the global handler to pick up the corrected credentials - global s3_handler - from ..utils.s3_upload_handler import S3UploadHandler - + # A fresh handler per request, so a credential change takes effect + # without a restart. s3_handler = S3UploadHandler() # Test S3 connection @@ -65,10 +62,6 @@ def simple_s3_upload(request): if not uploaded_files: return JsonResponse({"success": False, "error": "No documents provided."}, status=400) - # Reinitialize S3 handler - global s3_handler - from ..utils.s3_upload_handler import S3UploadHandler - s3_handler = S3UploadHandler() if not s3_handler._ensure_initialized(): @@ -134,40 +127,6 @@ def simple_s3_upload(request): return JsonResponse({"success": False, "error": f"Upload error: {str(e)}"}, status=500) -@require_http_methods(["GET", "HEAD"]) -def public_s3_upload(request, key): - """Serve a LocalStack document to the remote EFSP during local testing. - - The route is limited to files generated by this application. Production - deployments do not configure the tunnel, so their uploads continue using - direct S3 URLs. - """ - if not key.startswith("efile-documents/") or ".." in key.split("/"): - return HttpResponseNotFound() - - from ..utils.s3_upload_handler import S3UploadHandler - - handler = S3UploadHandler() - if not handler._ensure_initialized() or handler.s3_client is None: - return HttpResponseNotFound() - - try: - object_data = handler.s3_client.get_object(Bucket=handler.bucket_name, Key=key) - except ClientError as error: - error_code = error.response.get("Error", {}).get("Code") - if error_code not in {"NoSuchKey", "404", "NoSuchBucket"}: - logger.exception("Could not read public local upload %s", key) - return HttpResponseNotFound() - - response = FileResponse( - object_data["Body"], - content_type=object_data.get("ContentType", "application/octet-stream"), - ) - if object_data.get("ContentLength") is not None: - response["Content-Length"] = str(object_data["ContentLength"]) - return response - - @csrf_exempt @require_http_methods(["POST"]) def mock_s3_upload(request): diff --git a/efile_app/efile/api/urls.py b/efile_app/efile/api/urls.py index 0d9aa80..605b1e1 100644 --- a/efile_app/efile/api/urls.py +++ b/efile_app/efile/api/urls.py @@ -26,7 +26,6 @@ from .filing_views import create_filing, delete_filing, get_filing_detail, get_filings, payment_fees, update_filing from .s3_upload import ( mock_s3_upload, - public_s3_upload, simple_s3_upload, test_s3_connection, ) @@ -37,7 +36,6 @@ urlpatterns = [ path("get-party-types/", get_party_types_from_suffolk_api, name="get_party_types"), path("simple-s3-upload/", simple_s3_upload, name="simple_s3_upload"), - path("public-upload/", public_s3_upload, name="public_s3_upload"), path("mock-s3-upload/", mock_s3_upload, name="mock_s3_upload"), path("test-s3-connection/", test_s3_connection, name="test_s3_connection"), # path("api/create-filing/", create_filing, name="create_filing"), diff --git a/efile_app/efile/settings_base.py b/efile_app/efile/settings_base.py index 86efa8a..2c2b2eb 100644 --- a/efile_app/efile/settings_base.py +++ b/efile_app/efile/settings_base.py @@ -115,12 +115,6 @@ AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME", "us-east-1") AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", None) -# LocalStack URLs are only resolvable inside Docker. Local compose starts a -# public quick tunnel and shares its log with the web container so the remote -# EFSP server can fetch uploaded documents during local development. -LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG = os.getenv("LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG", "") -LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS = float(os.getenv("LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS", "30")) - # File Upload Settings MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB ALLOWED_FILE_TYPES = [".pdf", ".doc", ".docx"] diff --git a/efile_app/efile/settings_dev.py b/efile_app/efile/settings_dev.py index dd3e06e..7f1402b 100644 --- a/efile_app/efile/settings_dev.py +++ b/efile_app/efile/settings_dev.py @@ -11,13 +11,27 @@ load_dotenv(dotenv_path=_BASE_DIR / ".env", override=False) from efile.settings_base import * # noqa: E402,F401,F403 +from efile.settings_base import ALLOWED_HOSTS as BASE_ALLOWED_HOSTS # noqa: E402 from efile.settings_base import DATABASES as BASE_DATABASES # noqa: E402 # Bind DATABASES explicitly to avoid F405 and make linter aware of the symbol DATABASES = BASE_DATABASES DEBUG = True -ALLOWED_HOSTS = ALLOWED_HOSTS or ["localhost", "127.0.0.1", ".localhost", "[::1]", "testserver"] +ALLOWED_HOSTS = BASE_ALLOWED_HOSTS or ["localhost", "127.0.0.1", ".localhost", "[::1]", "testserver"] + +# Opt-in: send this URL to the EFSP proxy as every document's `data_url` instead +# of the real S3 URL. +# +# The proxy downloads each `data_url` itself -- for a fee quote as well as for a +# filing -- and only accepts http(s), so a LocalStack URL is unreachable to it. +# Setting this to any publicly readable PDF lets fee quotes and submissions run +# end-to-end from a laptop with no ingress, tunnel, or public bucket. +# +# Uploads still go to S3/LocalStack for real and drafts still store the real keys +# and URLs; only the URL handed to the proxy changes. Defined here rather than in +# settings_base so no environment variable can enable it outside development. +EFSP_TEST_DOCUMENT_URL = os.getenv("EFSP_TEST_DOCUMENT_URL", "").strip() CSRF_TRUSTED_ORIGINS = [ "http://localhost", "http://127.0.0.1", diff --git a/efile_app/efile/settings_prod.py b/efile_app/efile/settings_prod.py index 145c7fe..19f83fc 100644 --- a/efile_app/efile/settings_prod.py +++ b/efile_app/efile/settings_prod.py @@ -3,7 +3,11 @@ import dj_database_url from efile.settings_base import * # noqa: F401,F403 + +# Import specific names with aliases to avoid F405 from star import usage +from efile.settings_base import BASE_DIR as BASE_SETTINGS_DIR from efile.settings_base import DATABASES as BASE_DATABASES +from efile.settings_base import MIDDLEWARE as BASE_MIDDLEWARE # Bind DATABASES explicitly to avoid F405 and make linter aware of the symbol DATABASES = BASE_DATABASES @@ -12,6 +16,22 @@ ALLOWED_HOSTS = ["forms-mvp-prod.fly.dev"] CSRF_TRUSTED_ORIGINS = ["https://forms-mvp-prod.fly.dev"] +# Static files (WhiteNoise), mirroring settings_staging. +# +# ManifestStaticFilesStorage content-hashes each filename (payment.js -> +# payment.4a3f9c2b.js), so a changed asset gets a new URL and an unchanged one +# keeps its cache entry. This is why templates carry no manual ?v= cache-buster. +STATIC_ROOT = BASE_SETTINGS_DIR / "staticfiles" +STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" + +# Build MIDDLEWARE from base and insert WhiteNoise right after SecurityMiddleware +MIDDLEWARE = list(BASE_MIDDLEWARE) +try: + security_index = MIDDLEWARE.index("django.middleware.security.SecurityMiddleware") +except ValueError: + security_index = -1 +MIDDLEWARE.insert(security_index + 1 if security_index >= 0 else 0, "whitenoise.middleware.WhiteNoiseMiddleware") + # Security hardening SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") if not SECRET_KEY or SECRET_KEY.startswith("django-insecure-"): diff --git a/efile_app/efile/static/js/payment.js b/efile_app/efile/static/js/payment.js index df6686f..1138b49 100644 --- a/efile_app/efile/static/js/payment.js +++ b/efile_app/efile/static/js/payment.js @@ -370,243 +370,15 @@ const FilingHandler = { confirm_submission: true, payment_account_id: paymentAccountID }); - }, - - - buildEFilingData(userData, caseData, uploadData, paymentAccountID) { - const nameParts = userData.fullName.split(" "); - const firstName = nameParts[0] || ""; - const lastName = nameParts.length > 1 ? nameParts[nameParts.length - 1] : ""; - const middleName = nameParts.length > 2 ? nameParts.slice(1, -1).join(" ") : ""; - - const partyType = caseData.determined_party_type || caseData.petitioner_party_type || caseData.party_type; - - if (!partyType) { - throw new Error('Party type could not be determined. This is required for eFiling.'); - } - - // Build user object - const mainUser = { - mobile_number: userData.phone, - phone_number: userData.phone, - address: { - address: userData.address, - unit: userData.addressLine2, - city: userData.city, - state: userData.state, - zip: userData.zip, - country: "US" - }, - email: userData.email, - party_type: partyType, - date_of_birth: "", - // The authenticated account is a firm filer, not a self-represented - // party. Marking the filer as the party makes Tyler reject fees with - // "Cannot specify self as party for a firm filer." - is_form_filler: false, - name: { - first: firstName, - middle: middleName, - last: lastName, - suffix: "" - }, - is_new: true - }; - - const users = [mainUser]; - - // Add second user if needed for name changes - if (caseData.new_name_party_type) { - users.push({ - ...mainUser, - party_type: caseData.new_name_party_type, - name: { - first: caseData.new_first_name || firstName, - middle: caseData.new_middle_name || middleName, - last: caseData.new_last_name || lastName, - suffix: caseData.new_suffix || "" - } - }); - } - - // Add second user if needed for name changes - if (caseData.respondent_name_party_type) { - users.push({ - party_type: caseData.respondent_name_party_type, - name: { - first: caseData.respondent_first_name || "", - middle: caseData.respondent_middle_name || "", - last: caseData.respondent_last_name || "", - suffix: caseData.respondent_suffix || "" - }, - is_new: true, - }); - } - - let other_parties = []; - - if (caseData.other_first_name && caseData.other_party_type) { - other_parties.push({ - party_type: caseData.other_party_type, - name: { - first: caseData.other_first_name, - last: caseData.other_last_name - }, - address: { - address: caseData.other_address_line_1, - unit: caseData.other_address_line_2, - city: caseData.other_address_city, - state: caseData.other_address_state, - zip: caseData.other_address_zip, - country: "US" - }, - email: caseData.other_email, - phone_number: caseData.other_phone_number, - is_new: true, - }); - } - - const efilingData = { - efile_case_category: caseData.case_category, - efile_case_type: caseData.case_type, - efile_case_subtype: caseData.case_subtype, - previous_case_id: caseData?.previous_case_id, - docket_number: caseData?.docket_number, - users, - other_parties, - user_started_case: !caseData?.previous_case_id, - al_court_bundle: [], - comments_to_clerk: "", - tyler_payment_id: paymentAccountID, - lead_contact: { - name: { - first: firstName, - middle: middleName, - last: lastName - }, - email: userData.email - }, - return_date: "" - }; - - // Add court bundles for documents - this.addCourtBundles(efilingData, uploadData, caseData, users); - - return efilingData; - }, - - addCourtBundles(efilingData, uploadData, caseData, users) { - const courtName = caseData.court_name || caseData.court || ""; - if (courtName.toLowerCase().includes("cook") || courtName.toLowerCase().includes("dupage")) { - efilingData.cross_references = { - 254500: "254500" - }; - } - - // Add lead document - if (uploadData?.files?.lead) { - const leadBundle = this.createDocumentBundle( - uploadData.files.lead, - uploadData.lead_filing_type || caseData.filing_type, - uploadData.lead_document_type || caseData.document_type, - uploadData.lead_filing_component || caseData.filing_component, - users, - uploadData.lead_filing_type_name || caseData.case_type_name, - uploadData.lead_document_type_name || "", - uploadData.lead_cc_email - ); - efilingData.al_court_bundle.push(leadBundle); - } - - // Add supporting documents - if (uploadData?.files?.supporting?.length > 0) { - uploadData.files.supporting.forEach((doc, index) => { - const config = uploadData.supporting_documents?.[index] || {}; - const configuredComponent = config.filing_component; - const documentComponent = doc.filing_component; - const filingComponent = typeof configuredComponent === "object" - ? configuredComponent.id - : configuredComponent || (typeof documentComponent === "object" ? documentComponent.id : documentComponent); - const bundle = this.createDocumentBundle( - doc, - config.filing_type || caseData.filing_type_id, - config.document_type || caseData.document_type, - filingComponent || caseData.filing_component, - users, - config.filing_type_name || `Supporting Document ${index + 1}`, - config.document_type_name || "", - config.cc_email - ); - efilingData.al_court_bundle.push(bundle); - }); - } - }, - - createDocumentBundle(doc, filingType, documentType, filingComponent, users, description, docDescription, cc_email) { - if (cc_email) { - courtesy_copies = [cc_email] - } else { - courtesy_copies = [] - } - return { - proxy_enabled: true, - filing_type: filingType, - optional_services: [], - due_date: null, - filing_description: description, - reference_number: "", - filing_attorney: "", - filing_comment: "", - courtesy_copies: courtesy_copies, - preliminary_copies: [], - filing_parties: users.length === 1 ? ["users[0]"] : ["users[0]", "users[1]"], - filing_action: "efile", - tyler_merge_attachments: false, - document_type: documentType, - filing_component: filingComponent, - filename: doc.name, - document_description: docDescription, - data_url: doc.url || doc.s3_url || doc.file_url || doc.download_url - }; - }, - - handleSubmissionResult(result) { - if (result?.success) { - Messages.showSuccess(gettext("Filing submitted successfully! You will be redirected to the confirmation page.")); - setTimeout(() => { - const jurisdiction = apiUtils.getCurrentJurisdiction(); - window.location.href = result.redirect_url || `/jurisdiction/${jurisdiction}/filing-confirmation/`; - }, 2000); - } else { - Messages.showError(result?.error || "An error occurred during submission."); - this.setSubmissionState(false); - } - }, - - handleFeesResponse(result) { - if (result?.success) { - let htmlStr = ` - Total: $${result.api_response.feesCalculationAmount.value} - -
    - `; - for (let specificFee of result.api_response.allowanceCharge) { - if (specificFee.chargeIndicator.value) { - htmlStr += `
  • ${specificFee.allowanceChargeReason.value}: $${specificFee.amount.value}
  • `; - } - } - htmlStr += "
"; - - let infoElem = document.getElementById("paymentInfo"); - infoElem.innerHTML = htmlStr; - document.getElementById("paymentSection").removeAttribute("hidden"); - } else { - Messages.showError(result?.error || "An error occurred when calculating fees."); - } - this.setFeesState(false); } }; +// buildEFilingData / addCourtBundles / createDocumentBundle / +// handleSubmissionResult / handleFeesResponse are shared with the other +// filing page. See filing-payload.js -- keep payload changes there so the +// fee quote and the submitted filing cannot drift apart. +Object.assign(FilingHandler, FilingPayload); + // Main application initialization const ReviewApp = { async init() { diff --git a/efile_app/efile/static/js/review.js b/efile_app/efile/static/js/review.js index b64b6e0..ea8bb8f 100644 --- a/efile_app/efile/static/js/review.js +++ b/efile_app/efile/static/js/review.js @@ -531,242 +531,15 @@ const FilingHandler = { payment_account_id: paymentAccountID }) }); - }, - - buildEFilingData(userData, caseData, uploadData, paymentAccountID) { - const nameParts = userData.fullName.split(" "); - const firstName = nameParts[0] || ""; - const lastName = nameParts.length > 1 ? nameParts[nameParts.length - 1] : ""; - const middleName = nameParts.length > 2 ? nameParts.slice(1, -1).join(" ") : ""; - - const partyType = caseData.determined_party_type || caseData.petitioner_party_type || caseData.party_type; - - if (!partyType) { - throw new Error('Party type could not be determined. This is required for eFiling.'); - } - - // Build user object - const mainUser = { - mobile_number: userData.phone, - phone_number: userData.phone, - address: { - address: userData.address, - unit: userData.addressLine2, - city: userData.city, - state: userData.state, - zip: userData.zip, - country: "US" - }, - email: userData.email, - party_type: partyType, - date_of_birth: "", - // The authenticated account is a firm filer, not a self-represented - // party. Marking the filer as the party makes Tyler reject fees with - // "Cannot specify self as party for a firm filer." - is_form_filler: false, - name: { - first: firstName, - middle: middleName, - last: lastName, - suffix: "" - }, - is_new: true - }; - - const users = [mainUser]; - - // Add second user if needed for name changes - if (caseData.new_name_party_type) { - users.push({ - ...mainUser, - party_type: caseData.new_name_party_type, - name: { - first: caseData.new_first_name || firstName, - middle: caseData.new_middle_name || middleName, - last: caseData.new_last_name || lastName, - suffix: caseData.new_suffix || "" - } - }); - } - - // Add second user if needed for name changes - if (caseData.respondent_name_party_type) { - users.push({ - party_type: caseData.respondent_name_party_type, - name: { - first: caseData.respondent_first_name || "", - middle: caseData.respondent_middle_name || "", - last: caseData.respondent_last_name || "", - suffix: caseData.respondent_suffix || "" - }, - is_new: true, - }); - } - - let other_parties = []; - - if (caseData.other_first_name && caseData.other_party_type) { - other_parties.push({ - party_type: caseData.other_party_type, - name: { - first: caseData.other_first_name, - last: caseData.other_last_name - }, - address: { - address: caseData.other_address_line_1, - unit: caseData.other_address_line_2, - city: caseData.other_address_city, - state: caseData.other_address_state, - zip: caseData.other_address_zip, - country: "US" - }, - email: caseData.other_email, - phone_number: caseData.other_phone_number, - is_new: true, - }); - } - - const efilingData = { - efile_case_category: caseData.case_category, - efile_case_type: caseData.case_type, - efile_case_subtype: caseData.case_subtype, - previous_case_id: caseData?.previous_case_id, - docket_number: caseData?.docket_number, - users, - other_parties, - user_started_case: !caseData?.previous_case_id, - al_court_bundle: [], - comments_to_clerk: "", - tyler_payment_id: paymentAccountID, - lead_contact: { - name: { - first: firstName, - middle: middleName, - last: lastName - }, - email: userData.email - }, - return_date: "" - }; - - // Add court bundles for documents - this.addCourtBundles(efilingData, uploadData, caseData, users); - - return efilingData; - }, - - addCourtBundles(efilingData, uploadData, caseData, users) { - const courtName = caseData.court_name || caseData.court || ""; - if (courtName.toLowerCase().includes("cook") || courtName.toLowerCase().includes("dupage")) { - efilingData.cross_references = { - 254500: "254500" - }; - } - - // Add lead document - if (uploadData?.files?.lead) { - const leadBundle = this.createDocumentBundle( - uploadData.files.lead, - uploadData.lead_filing_type || caseData.filing_type, - uploadData.lead_document_type || caseData.document_type, - uploadData.lead_filing_component || caseData.filing_component, - users, - uploadData.lead_filing_type_name || caseData.case_type_name, - uploadData.lead_document_type_name || "", - uploadData.lead_cc_email - ); - efilingData.al_court_bundle.push(leadBundle); - } - - // Add supporting documents - if (uploadData?.files?.supporting?.length > 0) { - uploadData.files.supporting.forEach((doc, index) => { - const config = uploadData.supporting_documents?.[index] || {}; - const configuredComponent = config.filing_component; - const documentComponent = doc.filing_component; - const filingComponent = typeof configuredComponent === "object" - ? configuredComponent.id - : configuredComponent || (typeof documentComponent === "object" ? documentComponent.id : documentComponent); - const bundle = this.createDocumentBundle( - doc, - config.filing_type || caseData.filing_type_id, - config.document_type || caseData.document_type, - filingComponent || caseData.filing_component, - users, - config.filing_type_name || `Supporting Document ${index + 1}`, - config.document_type_name || "", - config.cc_email - ); - efilingData.al_court_bundle.push(bundle); - }); - } - }, - - createDocumentBundle(doc, filingType, documentType, filingComponent, users, description, docDescription, cc_email) { - if (cc_email) { - courtesy_copies = [cc_email] - } else { - courtesy_copies = [] - } - return { - proxy_enabled: true, - filing_type: filingType, - optional_services: [], - due_date: null, - filing_description: description, - reference_number: "", - filing_attorney: "", - filing_comment: "", - courtesy_copies: courtesy_copies, - preliminary_copies: [], - filing_parties: users.length === 1 ? ["users[0]"] : ["users[0]", "users[1]"], - filing_action: "efile", - tyler_merge_attachments: false, - document_type: documentType, - filing_component: filingComponent, - filename: doc.name, - document_description: docDescription, - data_url: doc.url || doc.s3_url || doc.file_url || doc.download_url - }; - }, - - handleSubmissionResult(result) { - if (result?.success) { - Messages.showSuccess(gettext("Filing submitted successfully! You will be redirected to the confirmation page.")); - setTimeout(() => { - const jurisdiction = apiUtils.getCurrentJurisdiction(); - window.location.href = result.redirect_url || `/jurisdiction/${jurisdiction}/filing-confirmation/`; - }, 2000); - } else { - Messages.showError(result?.error || "An error occurred during submission."); - this.setSubmissionState(false); - } - }, - - handleFeesResponse(result) { - if (result?.success) { - let htmlStr = ` - Total: $${result.api_response.feesCalculationAmount.value} - -
    - `; - for (let specificFee of result.api_response.allowanceCharge) { - if (specificFee.chargeIndicator.value) { - htmlStr += `
  • ${specificFee.allowanceChargeReason.value}: $${specificFee.amount.value}
  • `; - } - } - htmlStr += "
"; - - let infoElem = document.getElementById("paymentInfo"); - infoElem.innerHTML = htmlStr; - document.getElementById("paymentSection").removeAttribute("hidden"); - } else { - Messages.showError(result?.error || "An error occurred when calculating fees."); - } - this.setFeesState(false); } }; +// buildEFilingData / addCourtBundles / createDocumentBundle / +// handleSubmissionResult / handleFeesResponse are shared with the other +// filing page. See filing-payload.js -- keep payload changes there so the +// fee quote and the submitted filing cannot drift apart. +Object.assign(FilingHandler, FilingPayload); + // Main application initialization const ReviewApp = { async init() { diff --git a/efile_app/efile/static/js/upload-handler-first.js b/efile_app/efile/static/js/upload-handler-first.js index 54dfc58..6ed47ff 100644 --- a/efile_app/efile/static/js/upload-handler-first.js +++ b/efile_app/efile/static/js/upload-handler-first.js @@ -60,6 +60,27 @@ class UploadHandler { } } + /** + * Build the durable-draft payload for a lead document that S3 has accepted. + * Used both by the save-on-upload path and by the save-on-Continue retry, so + * the two can never drift. + * @param {Object} uploadedLead - one entry from the upload response's `files` + */ + buildLeadPayload(uploadedLead) { + return { + files: { + lead: { + name: this.uploadedFile.name, + size: this.uploadedFile.size, + type: this.uploadedFile.type, + url: uploadedLead.public_url || uploadedLead.url, + s3_key: uploadedLead.key, + }, + }, + options: { lead: {} }, + }; + } + async saveUploadDataToSession(uploadData) { try { const payload = { @@ -306,18 +327,7 @@ class UploadHandler { const uploadedLead = result.files?.[0]; if (uploadedLead) { try { - await this.saveUploadDataToSession({ - files: { - lead: { - name: this.uploadedFile.name, - size: this.uploadedFile.size, - type: this.uploadedFile.type, - url: uploadedLead.public_url || uploadedLead.url, - s3_key: uploadedLead.key, - }, - }, - options: { lead: {} }, - }); + await this.saveUploadDataToSession(this.buildLeadPayload(uploadedLead)); this.leadPersisted = true; } catch (error) { // Continue still retries the same save, so an interim @@ -396,18 +406,7 @@ class UploadHandler { if (!uploadedLead) { throw new Error('The lead document upload did not return a file.'); } - await this.saveUploadDataToSession({ - files: { - lead: { - name: this.uploadedFile.name, - size: this.uploadedFile.size, - type: this.uploadedFile.type, - url: uploadedLead.public_url || uploadedLead.url, - s3_key: uploadedLead.key, - }, - }, - options: { lead: {} }, - }); + await this.saveUploadDataToSession(this.buildLeadPayload(uploadedLead)); this.leadPersisted = true; } catch (error) { console.error('Error persisting lead upload:', error); diff --git a/efile_app/efile/static/js/upload-handler.js b/efile_app/efile/static/js/upload-handler.js index e1c5b8b..24f422b 100644 --- a/efile_app/efile/static/js/upload-handler.js +++ b/efile_app/efile/static/js/upload-handler.js @@ -1109,6 +1109,10 @@ function createSupportingDocumentOptions(index, fileName) { const html = `
Options for: ${fileName}
+
diff --git a/efile_app/efile/templates/efile/expert_form.html b/efile_app/efile/templates/efile/expert_form.html index c08f6b5..fe8c385 100644 --- a/efile_app/efile/templates/efile/expert_form.html +++ b/efile_app/efile/templates/efile/expert_form.html @@ -694,8 +694,7 @@
Case Information Found
- - + diff --git a/efile_app/efile/templates/efile/options.html b/efile_app/efile/templates/efile/options.html index bfd66f8..e77453b 100644 --- a/efile_app/efile/templates/efile/options.html +++ b/efile_app/efile/templates/efile/options.html @@ -92,6 +92,7 @@

{% translate "View past filings" %}

+ {{ resume_url|json_script:"resume-url" }} - + + + diff --git a/efile_app/efile/templates/efile/review.html b/efile_app/efile/templates/efile/review.html index 34ec99a..e7c9f2d 100644 --- a/efile_app/efile/templates/efile/review.html +++ b/efile_app/efile/templates/efile/review.html @@ -239,7 +239,9 @@

{% translate "Submit my filing" %}

{{ friendly_names|json_script:"friendly-names" }} - + + + - + - + diff --git a/efile_app/efile/tests/test_integration.py b/efile_app/efile/tests/test_integration.py index 74fbbc3..eb8e1d9 100644 --- a/efile_app/efile/tests/test_integration.py +++ b/efile_app/efile/tests/test_integration.py @@ -12,15 +12,26 @@ from efile.utils.config_loader import config_loader # ===================================================== -# Secrets loaded at runtime +# Secrets loaded at runtime. +# +# These tests log in to the live Tyler test EFM. Without credentials they skip +# rather than fail, so a missing local .env stays distinguishable from a real +# regression. CI supplies both from repo secrets, so coverage there is unchanged. + + +def _require_env(name): + value = os.environ.get(name) + if not value: + pytest.skip(f"{name} is not set; skipping test that needs the live Tyler EFM") + return value def get_test_username(): - return os.environ["TESTS_TYLER_USERNAME"] + return _require_env("TESTS_TYLER_USERNAME") def get_test_password(): - return os.environ["TESTS_TYLER_PASSWORD"] + return _require_env("TESTS_TYLER_PASSWORD") # ======================= @@ -74,7 +85,10 @@ def test_profile_api_basic_functionality(self): if data["success"]: assert "data" in data assert data["data"]["username"] == username - assert data["data"]["first_name"] == "Bryce" + # Assert the shape, not one developer's Tyler account: whoever's + # credentials are configured, the profile must carry a real first name. + assert isinstance(data["data"]["first_name"], str) + assert data["data"]["first_name"] @pytest.mark.django_db def test_case_categories_api_basic_functionality(self): diff --git a/efile_app/efile/tests/test_local_document_urls.py b/efile_app/efile/tests/test_local_document_urls.py deleted file mode 100644 index dbf10b2..0000000 --- a/efile_app/efile/tests/test_local_document_urls.py +++ /dev/null @@ -1,110 +0,0 @@ -from django.test import override_settings - -from efile.api.filing_views import rewrite_fallback_filing_components, rewrite_local_document_urls -from efile.utils.s3_upload_handler import S3UploadHandler - - -def test_local_public_url_uses_current_tunnel_after_restart(tmp_path): - tunnel_log = tmp_path / "cloudflared.log" - tunnel_log.write_text( - "Requesting new quick Tunnel...\n" - "https://old-tunnel.trycloudflare.com\n" - "Requesting new quick Tunnel...\n" - "https://current-tunnel.trycloudflare.com\n", - encoding="utf-8", - ) - - with override_settings( - LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), - LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, - ): - assert ( - S3UploadHandler.get_local_public_url("efile-documents/lead/document.pdf") - == "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" - ) - - -def test_stale_local_document_url_is_rewritten(tmp_path): - tunnel_log = tmp_path / "cloudflared.log" - tunnel_log.write_text( - "Requesting new quick Tunnel...\nhttps://current-tunnel.trycloudflare.com\n", - encoding="utf-8", - ) - efile_data = { - "al_court_bundle": [ - { - "data_url": "http://localstack:4566/forms-mvp-xf6361/efile-documents/lead/document.pdf?signature=old" - } - ] - } - - with override_settings( - LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), - LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, - ): - rewrite_local_document_urls(efile_data) - - assert efile_data["al_court_bundle"][0]["data_url"] == ( - "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" - ) - - -def test_raw_s3_key_is_rewritten_to_local_public_url(tmp_path): - tunnel_log = tmp_path / "cloudflared.log" - tunnel_log.write_text( - "Requesting new quick Tunnel...\nhttps://current-tunnel.trycloudflare.com\n", - encoding="utf-8", - ) - efile_data = {"al_court_bundle": [{"data_url": "efile-documents/lead/document.pdf"}]} - - with override_settings( - LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), - LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, - ): - rewrite_local_document_urls(efile_data) - - assert efile_data["al_court_bundle"][0]["data_url"] == ( - "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" - ) - - -def test_stale_quick_tunnel_document_url_is_rewritten(tmp_path): - tunnel_log = tmp_path / "cloudflared.log" - tunnel_log.write_text( - "Requesting new quick Tunnel...\nhttps://current-tunnel.trycloudflare.com\n", - encoding="utf-8", - ) - efile_data = { - "al_court_bundle": [ - {"data_url": "https://old-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf"} - ] - } - - with override_settings( - LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG=str(tunnel_log), - LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS=0, - ): - rewrite_local_document_urls(efile_data) - - assert efile_data["al_court_bundle"][0]["data_url"] == ( - "https://current-tunnel.trycloudflare.com/api/public-upload/efile-documents/lead/document.pdf" - ) - - -def test_legacy_supporting_label_is_resolved_to_court_code(monkeypatch, settings): - class Response: - status_code = 200 - - @staticmethod - def json(): - return [{"code": "331", "name": "Lead Document"}, {"code": "332", "name": "Attachments"}] - - monkeypatch.setattr("efile.api.filing_views.requests.get", lambda *args, **kwargs: Response()) - settings.EFSP_URL = "https://efile-test.example" - efile_data = { - "al_court_bundle": [{"filing_type": "27965", "filing_component": "supporting"}] - } - - rewrite_fallback_filing_components(efile_data, "illinois", "adams") - - assert efile_data["al_court_bundle"][0]["filing_component"] == "332" diff --git a/efile_app/efile/tests/test_workflow.py b/efile_app/efile/tests/test_workflow.py index 51b7cf6..ed9ceac 100644 --- a/efile_app/efile/tests/test_workflow.py +++ b/efile_app/efile/tests/test_workflow.py @@ -6,6 +6,7 @@ WorkflowStepKey, get_next_step, get_previous_step, + get_resume_step_url, get_step, get_step_url, get_workflow_context, @@ -80,6 +81,30 @@ def test_get_step_url_reverses_workflow_route(): assert get_step_url(WorkflowStepKey.PAYMENT, "illinois") == expected_url +def test_get_resume_step_url_returns_the_drafts_own_step(): + expected_url = reverse("upload", kwargs={"jurisdiction": "illinois"}) + + assert get_resume_step_url(WorkflowStepKey.DOCUMENTS, "illinois") == expected_url + + +def test_get_resume_step_url_skips_options_so_resuming_does_not_loop(): + # OPTIONS is the model default for older drafts; resuming there would just + # return the user to the page that offered to resume. + expected_url = reverse("upload_first", kwargs={"jurisdiction": "illinois"}) + + assert get_resume_step_url(WorkflowStepKey.OPTIONS, "illinois") == expected_url + + +def test_get_resume_step_url_falls_back_for_an_unrecognised_step(): + expected_url = reverse("upload_first", kwargs={"jurisdiction": "illinois"}) + + assert get_resume_step_url("a_step_that_was_removed", "illinois") == expected_url + + +def test_get_resume_step_url_returns_none_without_a_draft(): + assert get_resume_step_url(None, "illinois") is None + + def test_get_workflow_context_includes_current_previous_and_next_urls(): context = get_workflow_context(WorkflowStepKey.PAYMENT, "illinois") previous_url = reverse("upload", kwargs={"jurisdiction": "illinois"}) diff --git a/efile_app/efile/tests/tests.py b/efile_app/efile/tests/tests.py index 941255e..652c078 100644 --- a/efile_app/efile/tests/tests.py +++ b/efile_app/efile/tests/tests.py @@ -11,15 +11,26 @@ User = get_user_model() # ===================================================== -# Secrets loaded at runtime +# Secrets loaded at runtime. +# +# These tests log in to the live Tyler test EFM. Without credentials they skip +# rather than fail, so a missing local .env stays distinguishable from a real +# regression. CI supplies both from repo secrets, so coverage there is unchanged. + + +def _require_env(name): + value = os.environ.get(name) + if not value: + pytest.skip(f"{name} is not set; skipping test that needs the live Tyler EFM") + return value def get_test_username(): - return os.environ["TESTS_TYLER_USERNAME"] + return _require_env("TESTS_TYLER_USERNAME") def get_test_password(): - return os.environ["TESTS_TYLER_PASSWORD"] + return _require_env("TESTS_TYLER_PASSWORD") # ============================================================================ @@ -208,7 +219,10 @@ def test_profile_api_authenticated_user(self, client, user): data = json.loads(response.content) assert data["success"] is True assert data["data"]["username"] == username - assert data["data"]["first_name"] == "Bryce" + # Assert the shape, not one developer's Tyler account: whoever's + # credentials are configured, the profile must carry a real first name. + assert isinstance(data["data"]["first_name"], str) + assert data["data"]["first_name"] assert data["data"]["email"] == username def test_profile_api_includes_location_data(self, client, user): diff --git a/efile_app/efile/utils/s3_upload_handler.py b/efile_app/efile/utils/s3_upload_handler.py index 3a99af8..6c63d77 100644 --- a/efile_app/efile/utils/s3_upload_handler.py +++ b/efile_app/efile/utils/s3_upload_handler.py @@ -4,8 +4,6 @@ import logging import mimetypes -import re -import time import uuid from urllib.parse import quote @@ -193,10 +191,6 @@ def get_public_url(self, s3_key, expiration=604800): # 7 days default Get a presigned URL for an S3 object (for efile submission) Uses presigned URLs since bucket policy makes files private """ - local_url = self.get_local_public_url(s3_key) - if local_url: - return local_url - if self.s3_client: try: return self.s3_client.generate_presigned_url( @@ -208,56 +202,6 @@ def get_public_url(self, s3_key, expiration=604800): # 7 days default # Fallback to direct URL (will return 403 with current bucket policy) return f"https://{self.bucket_name}.s3.{self.region_name}.amazonaws.com/{s3_key}" - @classmethod - def get_local_public_url(cls, s3_key): - """Return a public proxy URL for a local S3 key, if configured.""" - local_tunnel_url = cls._get_local_tunnel_url() - if not local_tunnel_url: - return None - return f"{local_tunnel_url}/api/public-upload/{quote(s3_key, safe='/')}" - - @staticmethod - def _get_local_tunnel_url(): - """Read the automatic local-development tunnel URL. - - cloudflared writes the URL after it starts. Waiting here makes the - first upload reliable even if it happens while the tunnel is booting. - The shared log setting is absent outside local Docker development. - """ - log_path = getattr(settings, "LOCAL_PUBLIC_UPLOAD_TUNNEL_LOG", "") - if not log_path: - return None - - wait_seconds = max(0, getattr(settings, "LOCAL_PUBLIC_UPLOAD_WAIT_SECONDS", 30)) - deadline = time.monotonic() + wait_seconds - tunnel_pattern = re.compile(r"https://[a-z0-9-]+\.trycloudflare\.com", re.IGNORECASE) - - while True: - try: - with open(log_path, encoding="utf-8") as log_file: - log_contents = log_file.read() - # A named Docker volume keeps the log across container - # restarts. Ignore the previous tunnel until cloudflared has - # announced a new one. - latest_start = log_contents.rfind("Requesting new quick Tunnel") - current_log = log_contents[latest_start:] if latest_start != -1 else log_contents - matches = tunnel_pattern.findall(current_log) - if matches: - return matches[-1].rstrip("/") - except FileNotFoundError: - pass - except OSError as error: - logger.warning("Could not read local public upload tunnel log: %s", error) - return None - - if time.monotonic() >= deadline: - logger.error( - "Local public upload tunnel did not become ready within %ss", - wait_seconds, - ) - return None - time.sleep(0.25) - def delete_file(self, s3_key): """ Delete a file from S3 @@ -318,7 +262,3 @@ def validate_file(self, file_obj, max_size_mb=10, allowed_types=None): pass return {"valid": True} - - -# Global instance -s3_handler = S3UploadHandler() diff --git a/efile_app/efile/views/options.py b/efile_app/efile/views/options.py index 416ded2..1916296 100644 --- a/efile_app/efile/views/options.py +++ b/efile_app/efile/views/options.py @@ -6,29 +6,7 @@ from efile.services.drafts import draft_snapshot from ..utils.case_data_utils import get_case_data -from ..workflow import WorkflowStepKey, get_step_url, get_workflow_context - - -def _resume_url(active_draft, jurisdiction): - """Return the safest workflow URL for an active draft. - - ``OPTIONS`` is the model default for older drafts, but resuming there would - only send the user back to this page. Start those drafts at the first filing - step instead. The fallback also keeps an invalid legacy value from breaking - the options page. - """ - if active_draft is None: - return None - - try: - current_step = WorkflowStepKey(active_draft.current_step) - except ValueError: - current_step = WorkflowStepKey.UPLOAD_FIRST - - if current_step == WorkflowStepKey.OPTIONS: - current_step = WorkflowStepKey.UPLOAD_FIRST - - return get_step_url(current_step, jurisdiction) +from ..workflow import WorkflowStepKey, get_resume_step_url, get_workflow_context def efile_options(request, jurisdiction): @@ -52,7 +30,7 @@ def efile_options(request, jurisdiction): "is_logged_in": is_logged_in, "case_data": case_data, "filing_draft": draft_snapshot(active_draft), - "resume_url": _resume_url(active_draft, jurisdiction), + "resume_url": get_resume_step_url(active_draft.current_step if active_draft else None, jurisdiction), "has_case_data": bool(case_data or active_draft), } context.update(get_workflow_context(WorkflowStepKey.OPTIONS, jurisdiction)) diff --git a/efile_app/efile/views/session_api.py b/efile_app/efile/views/session_api.py index 24c91e9..a6a5810 100644 --- a/efile_app/efile/views/session_api.py +++ b/efile_app/efile/views/session_api.py @@ -9,6 +9,7 @@ from django.views.decorators.http import require_http_methods from ..services.current_drafts import get_current_draft +from ..services.efsp_payload import prepare_efile_payload from ..utils.case_data_utils import get_case_data, get_upload_data, update_case_data, update_upload_data from ..utils.llms import LlmError, extract_fields_from_file from ..utils.proxy_connection import get_party_type_code_from_api @@ -300,6 +301,9 @@ def submit_final_filing(request): if not court_id: return JsonResponse({"success": False, "error": "Court ID is required for filing submission"}, status=400) + # Same fixups the fee quote applied, so the filing matches the quote. + prepare_efile_payload(efile_data, jurisdiction_id, court_id) + # Construct the Suffolk LIT Lab API endpoint api_url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filings" diff --git a/efile_app/efile/workflow.py b/efile_app/efile/workflow.py index 9213899..2f4eddd 100644 --- a/efile_app/efile/workflow.py +++ b/efile_app/efile/workflow.py @@ -100,6 +100,28 @@ def get_step_url(step_key: WorkflowStepKey | str, jurisdiction: str) -> str: return reverse(step.url_name, kwargs={"jurisdiction": jurisdiction}) +def get_resume_step_url(current_step: WorkflowStepKey | str | None, jurisdiction: str) -> str | None: + """Return the URL to send someone back to when they resume a draft. + + OPTIONS is the model default for drafts saved before ``current_step`` was + tracked, but resuming there just returns the user to the page offering to + resume. Those start at the first real filing step instead. An unrecognised + value is treated the same way rather than breaking the options page. + """ + if current_step is None: + return None + + try: + step_key = WorkflowStepKey(current_step) + except ValueError: + step_key = WorkflowStepKey.UPLOAD_FIRST + + if step_key == WorkflowStepKey.OPTIONS: + step_key = WorkflowStepKey.UPLOAD_FIRST + + return get_step_url(step_key, jurisdiction) + + def get_workflow_context(current_step: WorkflowStepKey | str, jurisdiction: str) -> dict: previous_step = get_previous_step(current_step) next_step = get_next_step(current_step) diff --git a/efile_app/tests/setup.js b/efile_app/tests/setup.js index 48c3537..10053f1 100644 --- a/efile_app/tests/setup.js +++ b/efile_app/tests/setup.js @@ -20,7 +20,7 @@ require('dotenv').config({ async function globalSetup(config) { // Validate required environment variables - const requiredEnvVars = ['E2E_TEST_USERNAME', 'E2E_TEST_PASSWORD']; + const requiredEnvVars = ['TESTS_TYLER_USERNAME', 'TESTS_TYLER_PASSWORD']; const missingVars = requiredEnvVars.filter(varName => !process.env[varName]); if (missingVars.length > 0) { @@ -29,7 +29,7 @@ async function globalSetup(config) { console.log('✓ Environment variables loaded successfully'); console.log(`✓ Test base URL: ${process.env.E2E_TEST_BASE_URL || 'http://localhost:8000'}`); - console.log(`✓ Test username: ${process.env.E2E_TEST_USERNAME}`); + console.log(`✓ Test username: ${process.env.TESTS_TYLER_USERNAME}`); // You can add more global setup here: // - Database setup/cleanup diff --git a/efile_app/tests/test-utils.js b/efile_app/tests/test-utils.js index bef2375..f9dd964 100644 --- a/efile_app/tests/test-utils.js +++ b/efile_app/tests/test-utils.js @@ -10,12 +10,14 @@ * @returns {Object} Test configuration object */ function getTestConfig() { - const username = process.env.E2E_TEST_USERNAME; - const password = process.env.E2E_TEST_PASSWORD; + // Same Tyler test-EFM login the Python suite uses (efile/tests/) and that + // CI supplies from secrets -- one name, so a working .env works everywhere. + const username = process.env.TESTS_TYLER_USERNAME; + const password = process.env.TESTS_TYLER_PASSWORD; const baseUrl = process.env.E2E_TEST_BASE_URL || 'http://localhost:8000'; if (!username || !password) { - throw new Error('E2E_TEST_USERNAME and E2E_TEST_PASSWORD must be set in .env file'); + throw new Error('TESTS_TYLER_USERNAME and TESTS_TYLER_PASSWORD must be set in .env file'); } return { diff --git a/testing/README.md b/testing/README.md index 8eb8957..d43665c 100644 --- a/testing/README.md +++ b/testing/README.md @@ -1,21 +1,71 @@ # Testing Help -For local testing, we recommend using [LocalStack](https://docs.localstack.cloud/aws/getting-started/installation/#docker-compose) -to mock S3 locally, and [awslocal](https://github.com/localstack/awscli-local) -to use it. +## Local S3 (LocalStack) -LocalStack will be used through Docker, but to install `awslocal`, run the following: +`compose.yml` in the repo root starts [LocalStack](https://docs.localstack.cloud/) +alongside the web container and creates the bucket automatically +(`testing/localstack-init/ready.d/01-create-s3-bucket.sh`), so uploads work out +of the box with `docker compose up`. + +Uploads exercise the real code path: real boto3 client, real bucket, real keys, +real presigned URLs. Nothing about the upload flow is mocked. + +To poke at the bucket by hand, install +[awslocal](https://github.com/localstack/awscli-local): ```bash -pip install awscli-local[ver1] -`` +pip install 'awscli-local[ver1]' +awslocal s3 ls s3://forms-mvp-xf6361/efile-documents/ --recursive +``` + +## Calculating fees and filing end-to-end -You can start the S3 mock using the `docker-compose.yml` here in this folder with the following commands. +The EFSP proxy **downloads each document itself** from the `data_url` in the +filing payload, and it does this for a fee quote exactly as it does for a real +filing (`FilingReviewService.calculateFilingFees` runs the same deserializer, +which calls `inStream.readAllBytes()` inline). It also refuses any scheme other +than `http://` or `https://` — there is no way to POST the file bytes, and no +base64 or `data:` URI support. + +A LocalStack presigned URL points at `http://localstack:4566`, which only +resolves inside the Docker network. So a hosted EFSP proxy cannot fetch your +locally uploaded document, and fee quotes fail. + +The opt-in workaround is `EFSP_TEST_DOCUMENT_URL`: set it to any publicly +readable PDF, and the app sends *that* URL to the proxy as every document's +`data_url`. ```bash -docker compose up -d -# Create a bucket with the name that should be in .env as `AWS_S3_BUCKET_NAME`. `AWS_S3_REGION_NAME` defaults to "us-east-1". -awslocal s3api create-bucket --bucket efile-form-submission-bucket +# in efile_app/.env, or the environment you run compose from +EFSP_TEST_DOCUMENT_URL="https://example.org/some/public/blank.pdf" ``` -Also make sure that `AWS_S3_ENDPOINT_URL = "http://host.docker.internal:4566"` and `AWS_ACCOUNT_ID_ENDPOINT_MODE = "disabled"` in your env. +What this does and does not change: + +- **Unchanged:** the file is uploaded to LocalStack for real, the draft stores + the real S3 key and URL, and the review/payment screens show the real + document. Everything up to the EFSP call is exercised normally. +- **Changed:** only the `data_url` field in the payload sent to the proxy. + +Fees are calculated from filing codes, party counts, and optional services +rather than from the document's contents, and `page_count` is taken from the +JSON payload rather than parsed out of the PDF — so a stand-in PDF returns the +same fees as the real one. It does need to be a real, fetchable PDF, because the +proxy forwards the bytes on to Tyler. + +The app logs a warning on every request where a substitution happens, so a +stand-in filing is never mistaken for a real one. + +The setting is defined in `settings_dev.py` only. `settings_staging` and +`settings_prod` never read it, so no environment variable can enable this +outside local development. + +### If you need the proxy to fetch the real document + +Leave `EFSP_TEST_DOCUMENT_URL` unset and choose one of: + +- Point `AWS_S3_ENDPOINT_URL` at a real dev S3 bucket. Presigned URLs are the + production path, so this exercises exactly what ships. +- Run [EfileProxyServer](https://github.com/SuffolkLITLab/EfileProxyServer) in + your own compose stack. It deliberately does not block private addresses, so a + `data_url` on the Docker network works and no public ingress is needed. diff --git a/testing/compose.yml b/testing/compose.yml deleted file mode 100644 index 6d70da6..0000000 --- a/testing/compose.yml +++ /dev/null @@ -1,13 +0,0 @@ -services: - localstack: - container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" - image: localstack/localstack - ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - - "127.0.0.1:4510-4559:4510-4559" # external services port range - environment: - # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ - - DEBUG=${DEBUG:-0} - volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" From 73d615e5797470afb1cac59b94ddf0eb429a0b9e Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Wed, 22 Jul 2026 16:28:13 -0400 Subject: [PATCH 41/47] Add EFSP payload preparation service and frontend module with tests --- efile_app/efile/services/efsp_payload.py | 115 +++++++++ efile_app/efile/static/js/filing-payload.js | 270 ++++++++++++++++++++ efile_app/efile/tests/test_efsp_payload.py | 191 ++++++++++++++ efile_app/js-tests/filing-payload.test.js | 92 +++++++ 4 files changed, 668 insertions(+) create mode 100644 efile_app/efile/services/efsp_payload.py create mode 100644 efile_app/efile/static/js/filing-payload.js create mode 100644 efile_app/efile/tests/test_efsp_payload.py create mode 100644 efile_app/js-tests/filing-payload.test.js diff --git a/efile_app/efile/services/efsp_payload.py b/efile_app/efile/services/efsp_payload.py new file mode 100644 index 0000000..baac0ca --- /dev/null +++ b/efile_app/efile/services/efsp_payload.py @@ -0,0 +1,115 @@ +"""Prepare a client-built filing payload before it is sent to the EFSP proxy. + +The fee quote and the final submission post the *same* ``efile_data`` blob to +two different EFSP endpoints. Every server-side adjustment to that blob belongs +here so the two cannot drift: a fee quoted against one payload and a filing made +with another is a bug that only shows up as a wrong number on a real filing. +""" + +import logging + +import requests +from django.conf import settings + +logger = logging.getLogger(__name__) + +# Labels the UI has historically stored in place of a real court filing-component +# code. None of these are meaningful to the EFSP. +_PLACEHOLDER_COMPONENT_LABELS = {"", "supporting", "attachment", "attachments"} + + +def prepare_efile_payload(efile_data, jurisdiction_id, court_id): + """Apply every server-side fixup an EFSP request needs. Mutates ``efile_data``.""" + _drop_empty_cross_references(efile_data) + substitute_test_document_urls(efile_data) + resolve_placeholder_filing_components(efile_data, jurisdiction_id, court_id) + return efile_data + + +def _drop_empty_cross_references(efile_data): + """Omit ``cross_references`` entirely rather than sending an empty value.""" + if not efile_data.get("cross_references"): + efile_data.pop("cross_references", None) + + +def substitute_test_document_urls(efile_data): + """Point the proxy at a stand-in PDF when ``EFSP_TEST_DOCUMENT_URL`` is set. + + The proxy downloads every ``data_url`` itself -- for a fee quote just as much + as for a real filing -- and refuses any scheme other than http(s). A + LocalStack URL is not reachable from outside Docker, so exercising fees + locally would otherwise require public ingress to the dev machine. + + Documents are still uploaded to S3 for real and drafts still store their real + keys and URLs; only the URL handed to the proxy is replaced. The setting is + defined in ``settings_dev`` alone, so no environment variable can turn this on + in staging or production. + """ + test_url = getattr(settings, "EFSP_TEST_DOCUMENT_URL", "") + if not test_url: + return + + substituted = 0 + for bundle in efile_data.get("al_court_bundle", []): + if bundle.get("data_url") == test_url: + continue + bundle["data_url"] = test_url + substituted += 1 + + if substituted: + logger.warning( + "EFSP_TEST_DOCUMENT_URL is set: sent %d stand-in document(s) to the EFSP instead of the " + "uploaded file(s). Fees and filings from this request do not reflect real documents.", + substituted, + ) + + +def resolve_placeholder_filing_components(efile_data, jurisdiction_id, court_id): + """Replace leftover UI labels such as ``"supporting"`` with the court's code. + + Newer uploads store the real code, so this only fires for drafts saved by an + older client. Results are cached per filing type because a bundle commonly + repeats the same one. + """ + bundles = [ + bundle + for bundle in efile_data.get("al_court_bundle", []) + if str(bundle.get("filing_component", "")).lower() in _PLACEHOLDER_COMPONENT_LABELS + ] + if not bundles: + return + + resolved_codes: dict[str, str | None] = {} + for bundle in bundles: + filing_type = bundle.get("filing_type") + if not filing_type: + continue + if filing_type not in resolved_codes: + resolved_codes[filing_type] = _lookup_attachment_component(jurisdiction_id, court_id, filing_type) + code = resolved_codes[filing_type] + if code: + bundle["filing_component"] = code + + +def _lookup_attachment_component(jurisdiction_id, court_id, filing_type): + """Return the court's attachment filing-component code, or None if unavailable.""" + url = ( + f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/codes/courts/{court_id}/" + f"filing_types/{filing_type}/filing_components" + ) + try: + response = requests.get(url, timeout=10) + if response.status_code != 200: + return None + components = response.json() + except (requests.RequestException, ValueError) as error: + logger.warning("Could not resolve filing component for filing type %s: %s", filing_type, error) + return None + + if not isinstance(components, list): + return None + + for component in components: + if isinstance(component, dict) and str(component.get("name", "")).lower() in {"attachment", "attachments"}: + return component.get("code") + return None diff --git a/efile_app/efile/static/js/filing-payload.js b/efile_app/efile/static/js/filing-payload.js new file mode 100644 index 0000000..acf70d5 --- /dev/null +++ b/efile_app/efile/static/js/filing-payload.js @@ -0,0 +1,270 @@ +/** + * Shared filing-payload construction. + * + * The review page and the payment page each build the exact same `efile_data` + * blob for the EFSP -- the payment page to quote fees, the review page to quote + * fees and to submit. These functions were duplicated verbatim in payment.js and + * review.js; a fix applied to one and not the other produced a fee quote that + * disagreed with the filing, so they live here once. + * + * Mixed into each page's `FilingHandler` with Object.assign, so `this` still + * refers to the host handler and page-specific hooks (`setFeesState`, + * `setSubmissionState`) resolve normally. + * + * Depends on the page-level globals `Messages`, `apiUtils`, and `gettext`. + */ +/** + * Normalise a filing component to its bare code. + * + * The upload page stores the selection as `{id, name}` on the file record while + * the durable draft stores a plain code string, so both shapes reach here. Note + * `typeof null === "object"`, hence the explicit truthiness check -- an older + * session with `filing_component: null` would otherwise throw on `.id`. + * + * @param {Object|string|null|undefined} value + * @returns {string} the code, or "" when there isn't one + */ +function componentCode(value) { + if (value && typeof value === "object") { + return value.id || value.code || ""; + } + return value || ""; +} + +const FilingPayload = { + buildEFilingData(userData, caseData, uploadData, paymentAccountID) { + const nameParts = userData.fullName.split(" "); + const firstName = nameParts[0] || ""; + const lastName = nameParts.length > 1 ? nameParts[nameParts.length - 1] : ""; + const middleName = nameParts.length > 2 ? nameParts.slice(1, -1).join(" ") : ""; + + const partyType = caseData.determined_party_type || caseData.petitioner_party_type || caseData.party_type; + + if (!partyType) { + throw new Error('Party type could not be determined. This is required for eFiling.'); + } + + // Build user object + const mainUser = { + mobile_number: userData.phone, + phone_number: userData.phone, + address: { + address: userData.address, + unit: userData.addressLine2, + city: userData.city, + state: userData.state, + zip: userData.zip, + country: "US" + }, + email: userData.email, + party_type: partyType, + date_of_birth: "", + // The authenticated account is a firm filer, not a self-represented + // party. Marking the filer as the party makes Tyler reject fees with + // "Cannot specify self as party for a firm filer." + is_form_filler: false, + name: { + first: firstName, + middle: middleName, + last: lastName, + suffix: "" + }, + is_new: true + }; + + const users = [mainUser]; + + // Add second user if needed for name changes + if (caseData.new_name_party_type) { + users.push({ + ...mainUser, + party_type: caseData.new_name_party_type, + name: { + first: caseData.new_first_name || firstName, + middle: caseData.new_middle_name || middleName, + last: caseData.new_last_name || lastName, + suffix: caseData.new_suffix || "" + } + }); + } + + // Add second user if needed for name changes + if (caseData.respondent_name_party_type) { + users.push({ + party_type: caseData.respondent_name_party_type, + name: { + first: caseData.respondent_first_name || "", + middle: caseData.respondent_middle_name || "", + last: caseData.respondent_last_name || "", + suffix: caseData.respondent_suffix || "" + }, + is_new: true, + }); + } + + let other_parties = []; + + if (caseData.other_first_name && caseData.other_party_type) { + other_parties.push({ + party_type: caseData.other_party_type, + name: { + first: caseData.other_first_name, + last: caseData.other_last_name + }, + address: { + address: caseData.other_address_line_1, + unit: caseData.other_address_line_2, + city: caseData.other_address_city, + state: caseData.other_address_state, + zip: caseData.other_address_zip, + country: "US" + }, + email: caseData.other_email, + phone_number: caseData.other_phone_number, + is_new: true, + }); + } + + const efilingData = { + efile_case_category: caseData.case_category, + efile_case_type: caseData.case_type, + efile_case_subtype: caseData.case_subtype, + previous_case_id: caseData?.previous_case_id, + docket_number: caseData?.docket_number, + users, + other_parties, + user_started_case: !caseData?.previous_case_id, + al_court_bundle: [], + comments_to_clerk: "", + tyler_payment_id: paymentAccountID, + lead_contact: { + name: { + first: firstName, + middle: middleName, + last: lastName + }, + email: userData.email + }, + return_date: "" + }; + + // Add court bundles for documents + this.addCourtBundles(efilingData, uploadData, caseData, users); + + return efilingData; + }, + + addCourtBundles(efilingData, uploadData, caseData, users) { + const courtName = caseData.court_name || caseData.court || ""; + if (courtName.toLowerCase().includes("cook") || courtName.toLowerCase().includes("dupage")) { + efilingData.cross_references = { + 254500: "254500" + }; + } + + // Add lead document + if (uploadData?.files?.lead) { + const leadBundle = this.createDocumentBundle( + uploadData.files.lead, + uploadData.lead_filing_type || caseData.filing_type, + uploadData.lead_document_type || caseData.document_type, + uploadData.lead_filing_component || caseData.filing_component, + users, + uploadData.lead_filing_type_name || caseData.case_type_name, + uploadData.lead_document_type_name || "", + uploadData.lead_cc_email + ); + efilingData.al_court_bundle.push(leadBundle); + } + + // Add supporting documents + if (uploadData?.files?.supporting?.length > 0) { + uploadData.files.supporting.forEach((doc, index) => { + const config = uploadData.supporting_documents?.[index] || {}; + const filingComponent = componentCode(config.filing_component) + || componentCode(doc.filing_component); + const bundle = this.createDocumentBundle( + doc, + config.filing_type || caseData.filing_type_id, + config.document_type || caseData.document_type, + filingComponent || caseData.filing_component, + users, + config.filing_type_name || `Supporting Document ${index + 1}`, + config.document_type_name || "", + config.cc_email + ); + efilingData.al_court_bundle.push(bundle); + }); + } + }, + + createDocumentBundle(doc, filingType, documentType, filingComponent, users, description, docDescription, cc_email) { + if (cc_email) { + courtesy_copies = [cc_email] + } else { + courtesy_copies = [] + } + return { + proxy_enabled: true, + filing_type: filingType, + optional_services: [], + due_date: null, + filing_description: description, + reference_number: "", + filing_attorney: "", + filing_comment: "", + courtesy_copies: courtesy_copies, + preliminary_copies: [], + filing_parties: users.length === 1 ? ["users[0]"] : ["users[0]", "users[1]"], + filing_action: "efile", + tyler_merge_attachments: false, + document_type: documentType, + filing_component: filingComponent, + filename: doc.name, + document_description: docDescription, + data_url: doc.url || doc.s3_url || doc.file_url || doc.download_url + }; + }, + + handleSubmissionResult(result) { + if (result?.success) { + Messages.showSuccess(gettext("Filing submitted successfully! You will be redirected to the confirmation page.")); + setTimeout(() => { + const jurisdiction = apiUtils.getCurrentJurisdiction(); + window.location.href = result.redirect_url || `/jurisdiction/${jurisdiction}/filing-confirmation/`; + }, 2000); + } else { + Messages.showError(result?.error || "An error occurred during submission."); + this.setSubmissionState(false); + } + }, + + handleFeesResponse(result) { + if (result?.success) { + let htmlStr = ` + Total: $${result.api_response.feesCalculationAmount.value} + +
    + `; + for (let specificFee of result.api_response.allowanceCharge) { + if (specificFee.chargeIndicator.value) { + htmlStr += `
  • ${specificFee.allowanceChargeReason.value}: $${specificFee.amount.value}
  • `; + } + } + htmlStr += "
"; + + let infoElem = document.getElementById("paymentInfo"); + infoElem.innerHTML = htmlStr; + document.getElementById("paymentSection").removeAttribute("hidden"); + } else { + Messages.showError(result?.error || "An error occurred when calculating fees."); + } + this.setFeesState(false); + } +}; + +if (typeof module !== "undefined" && module.exports) { + module.exports = FilingPayload; +} else { + window.FilingPayload = FilingPayload; +} diff --git a/efile_app/efile/tests/test_efsp_payload.py b/efile_app/efile/tests/test_efsp_payload.py new file mode 100644 index 0000000..8fa7597 --- /dev/null +++ b/efile_app/efile/tests/test_efsp_payload.py @@ -0,0 +1,191 @@ +"""Tests for the shared EFSP payload preparation. + +The safety-critical property is that EFSP_TEST_DOCUMENT_URL is inert unless a +developer opts in, so the "off" cases are tested as carefully as the "on" ones. +""" + +import logging + +import pytest +from django.test import override_settings + +from efile.services.efsp_payload import ( + prepare_efile_payload, + resolve_placeholder_filing_components, + substitute_test_document_urls, +) + +REAL_S3_URL = "https://forms-mvp-xf6361.s3.us-east-1.amazonaws.com/efile-documents/lead/abc.pdf?X-Amz-Signature=x" +STAND_IN_URL = "https://example.org/fixtures/blank.pdf" + + +@pytest.fixture +def efile_logs(caplog): + """Capture records from the ``efile`` logger. + + ``caplog`` only attaches to the root logger, and settings_base.LOGGING sets + ``propagate: False`` on ``efile``, so its records never reach root. + """ + logger = logging.getLogger("efile") + logger.addHandler(caplog.handler) + caplog.set_level(logging.WARNING, logger="efile") + yield caplog + logger.removeHandler(caplog.handler) + + +def _bundle(**overrides): + bundle = {"data_url": REAL_S3_URL, "filing_type": "27965", "filing_component": "331"} + bundle.update(overrides) + return {"al_court_bundle": [bundle]} + + +# --- EFSP_TEST_DOCUMENT_URL off (the production configuration) --------------- + + +def test_real_document_url_is_untouched_when_stand_in_is_not_configured(): + efile_data = _bundle() + + with override_settings(EFSP_TEST_DOCUMENT_URL=""): + substitute_test_document_urls(efile_data) + + assert efile_data["al_court_bundle"][0]["data_url"] == REAL_S3_URL + + +def test_substitution_is_inert_when_setting_is_absent_entirely(): + """settings_prod/settings_staging never define the setting at all.""" + efile_data = _bundle() + + substitute_test_document_urls(efile_data) + + assert efile_data["al_court_bundle"][0]["data_url"] == REAL_S3_URL + + +# --- EFSP_TEST_DOCUMENT_URL on ---------------------------------------------- + + +def test_stand_in_url_replaces_every_document_url(): + efile_data = { + "al_court_bundle": [ + {"data_url": REAL_S3_URL}, + {"data_url": "https://forms-mvp-xf6361.s3.us-east-1.amazonaws.com/efile-documents/supporting/d.pdf"}, + ] + } + + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL): + substitute_test_document_urls(efile_data) + + assert [b["data_url"] for b in efile_data["al_court_bundle"]] == [STAND_IN_URL, STAND_IN_URL] + + +def test_substitution_warns_so_the_filing_is_not_mistaken_for_a_real_one(efile_logs): + efile_data = _bundle() + + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL): + substitute_test_document_urls(efile_data) + + assert any("stand-in document" in record.getMessage() for record in efile_logs.records) + + +def test_substitution_leaves_other_bundle_fields_alone(): + efile_data = _bundle(filing_component="331", filing_type="27965") + + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL): + substitute_test_document_urls(efile_data) + + bundle = efile_data["al_court_bundle"][0] + assert bundle["filing_component"] == "331" + assert bundle["filing_type"] == "27965" + + +# --- cross_references -------------------------------------------------------- + + +@pytest.mark.parametrize("empty_value", ["", None, [], {}]) +def test_empty_cross_references_is_dropped_rather_than_sent(empty_value): + efile_data = {"al_court_bundle": [], "cross_references": empty_value} + + with override_settings(EFSP_TEST_DOCUMENT_URL=""): + prepare_efile_payload(efile_data, "illinois", "adams") + + assert "cross_references" not in efile_data + + +def test_populated_cross_references_is_preserved(): + efile_data = {"al_court_bundle": [], "cross_references": [{"code": "1", "value": "abc"}]} + + with override_settings(EFSP_TEST_DOCUMENT_URL=""): + prepare_efile_payload(efile_data, "illinois", "adams") + + assert efile_data["cross_references"] == [{"code": "1", "value": "abc"}] + + +# --- placeholder filing components ------------------------------------------- + + +class _ComponentsResponse: + status_code = 200 + + @staticmethod + def json(): + return [{"code": "331", "name": "Lead Document"}, {"code": "332", "name": "Attachments"}] + + +def test_placeholder_component_label_is_resolved_to_the_court_code(monkeypatch, settings): + monkeypatch.setattr("efile.services.efsp_payload.requests.get", lambda *args, **kwargs: _ComponentsResponse()) + settings.EFSP_URL = "https://efile-test.example" + efile_data = _bundle(filing_component="supporting") + + resolve_placeholder_filing_components(efile_data, "illinois", "adams") + + assert efile_data["al_court_bundle"][0]["filing_component"] == "332" + + +def test_real_component_code_is_not_looked_up(monkeypatch, settings): + def fail(*args, **kwargs): + raise AssertionError("should not call the EFSP when the code is already real") + + monkeypatch.setattr("efile.services.efsp_payload.requests.get", fail) + settings.EFSP_URL = "https://efile-test.example" + efile_data = _bundle(filing_component="331") + + resolve_placeholder_filing_components(efile_data, "illinois", "adams") + + assert efile_data["al_court_bundle"][0]["filing_component"] == "331" + + +def test_repeated_filing_type_is_looked_up_once(monkeypatch, settings): + calls = [] + + def record(*args, **kwargs): + calls.append(args) + return _ComponentsResponse() + + monkeypatch.setattr("efile.services.efsp_payload.requests.get", record) + settings.EFSP_URL = "https://efile-test.example" + efile_data = { + "al_court_bundle": [ + {"filing_type": "27965", "filing_component": "supporting"}, + {"filing_type": "27965", "filing_component": "attachment"}, + ] + } + + resolve_placeholder_filing_components(efile_data, "illinois", "adams") + + assert len(calls) == 1 + assert [b["filing_component"] for b in efile_data["al_court_bundle"]] == ["332", "332"] + + +def test_unresolvable_component_is_left_for_the_efsp_to_reject(monkeypatch, settings): + import requests + + def boom(*args, **kwargs): + raise requests.RequestException("EFSP unreachable") + + monkeypatch.setattr("efile.services.efsp_payload.requests.get", boom) + settings.EFSP_URL = "https://efile-test.example" + efile_data = _bundle(filing_component="supporting") + + resolve_placeholder_filing_components(efile_data, "illinois", "adams") + + # Guessing a code would file under the wrong component; leave it visibly wrong. + assert efile_data["al_court_bundle"][0]["filing_component"] == "supporting" diff --git a/efile_app/js-tests/filing-payload.test.js b/efile_app/js-tests/filing-payload.test.js new file mode 100644 index 0000000..1355e5d --- /dev/null +++ b/efile_app/js-tests/filing-payload.test.js @@ -0,0 +1,92 @@ +const test = require("node:test"); +const assert = require("node:assert"); + +const FilingPayload = require("../efile/static/js/filing-payload.js"); + +// The module reads these page-level globals at call time. +global.Messages = { showError() {}, showSuccess() {} }; +global.gettext = (s) => s; +global.apiUtils = { getCurrentJurisdiction: () => "illinois" }; + +/** A FilingHandler stand-in, mixed the same way the real pages mix it. */ +function makeHandler() { + const handler = { setFeesState() {}, setSubmissionState() {} }; + return Object.assign(handler, FilingPayload); +} + +const CASE_DATA = { filing_component: "999", filing_type_id: "27965", document_type: "dt" }; + +function bundlesFor(uploadData) { + const handler = makeHandler(); + const efilingData = { al_court_bundle: [] }; + handler.addCourtBundles(efilingData, uploadData, CASE_DATA, []); + return efilingData.al_court_bundle; +} + +test("supporting component given as a plain code string is used as-is", () => { + const bundles = bundlesFor({ + files: { supporting: [{ name: "a.pdf" }] }, + supporting_documents: [{ filing_component: "332" }], + }); + + assert.strictEqual(bundles[0].filing_component, "332"); +}); + +test("supporting component given as an {id, name} object is flattened to its id", () => { + const bundles = bundlesFor({ + files: { supporting: [{ name: "a.pdf" }] }, + supporting_documents: [{ filing_component: { id: "332", name: "Attachments" } }], + }); + + assert.strictEqual(bundles[0].filing_component, "332"); +}); + +test("component stored on the file record is used when the config has none", () => { + const bundles = bundlesFor({ + files: { supporting: [{ name: "a.pdf", filing_component: { id: "332", name: "Attachments" } }] }, + supporting_documents: [{}], + }); + + assert.strictEqual(bundles[0].filing_component, "332"); +}); + +test("a null component falls back instead of throwing on .id", () => { + // typeof null === "object" -- the previous ternary crashed here. + const bundles = bundlesFor({ + files: { supporting: [{ name: "a.pdf", filing_component: null }] }, + supporting_documents: [{ filing_component: null }], + }); + + assert.strictEqual(bundles[0].filing_component, CASE_DATA.filing_component); +}); + +test("with no component anywhere, the case default is used and never the label 'supporting'", () => { + const bundles = bundlesFor({ + files: { supporting: [{ name: "a.pdf" }] }, + supporting_documents: [{}], + }); + + assert.strictEqual(bundles[0].filing_component, "999"); + assert.notStrictEqual(bundles[0].filing_component, "supporting"); +}); + +test("lead and supporting documents both land in the bundle", () => { + const bundles = bundlesFor({ + files: { lead: { name: "lead.pdf" }, supporting: [{ name: "a.pdf" }] }, + lead_filing_component: "331", + supporting_documents: [{ filing_component: "332" }], + }); + + assert.strictEqual(bundles.length, 2); + assert.strictEqual(bundles[0].filing_component, "331"); + assert.strictEqual(bundles[1].filing_component, "332"); +}); + +test("the same module object serves both pages, so payloads cannot drift", () => { + const paymentHandler = makeHandler(); + const reviewHandler = makeHandler(); + + assert.strictEqual(paymentHandler.buildEFilingData, reviewHandler.buildEFilingData); + assert.strictEqual(paymentHandler.addCourtBundles, reviewHandler.addCourtBundles); + assert.strictEqual(paymentHandler.createDocumentBundle, reviewHandler.createDocumentBundle); +}); From 88561a0d82be37e1f085adf173cc0cf51578c380 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Wed, 22 Jul 2026 17:47:33 -0400 Subject: [PATCH 42/47] Payment screen robustness on dev; guards against sample PDF filing --- compose.yml | 14 ++- efile_app/.env.example | 13 ++- efile_app/efile/apps.py | 4 + efile_app/efile/checks.py | 52 ++++++++++ efile_app/efile/services/efsp_payload.py | 26 ++++- efile_app/efile/settings_dev.py | 23 +++++ efile_app/efile/tests/conftest.py | 22 +++++ efile_app/efile/tests/test_efsp_payload.py | 59 ++++++++++-- .../tests/test_stand_in_document_guards.py | 95 +++++++++++++++++++ testing/README.md | 37 +++++++- 10 files changed, 322 insertions(+), 23 deletions(-) create mode 100644 efile_app/efile/checks.py create mode 100644 efile_app/efile/tests/conftest.py create mode 100644 efile_app/efile/tests/test_stand_in_document_guards.py diff --git a/compose.yml b/compose.yml index bf470bc..f1cd635 100644 --- a/compose.yml +++ b/compose.yml @@ -45,10 +45,16 @@ services: # botocore >= 1.35 routes S3 through account-ID-specific endpoints, which # LocalStack does not serve. Disable that routing. AWS_ACCOUNT_ID_ENDPOINT_MODE: disabled - # Optional: a publicly readable PDF to hand the EFSP proxy in place of the - # uploaded document, so fee quotes work without exposing this machine. - # See testing/README.md. - EFSP_TEST_DOCUMENT_URL: "${EFSP_TEST_DOCUMENT_URL:-}" + # A publicly readable PDF handed to the EFSP proxy in place of the uploaded + # document, so fee quotes work without exposing this machine. The proxy + # fetches every data_url itself and cannot resolve `localstack`, so a real + # presigned URL fails here with a 400. See testing/README.md. + # + # Read from the shell or the compose-level .env, NOT efile_app/.env: this + # value lands in the container environment, and settings_dev loads that + # .env with override=False, so whatever is set here wins. `-` rather than + # `:-` so an explicit empty value still turns the substitution off. + EFSP_TEST_DOCUMENT_URL: "${EFSP_TEST_DOCUMENT_URL-https://raw.githubusercontent.com/SuffolkLITLab/LITEFile/main/testing/sample_test.pdf}" # DJANGO_SETTINGS_MODULE: efile.settings # manage.py sets this; uncomment to override volumes: - .:/app:delegated diff --git a/efile_app/.env.example b/efile_app/.env.example index aa32ccd..244ea35 100644 --- a/efile_app/.env.example +++ b/efile_app/.env.example @@ -12,13 +12,12 @@ EFSP_URL = "https://efile-test.suffolklitlab.org" # fee quotes as well as filings -- and cannot reach a LocalStack URL. Set this to # any publicly readable PDF to run fees end-to-end without exposing this machine. # Uploads still go to S3 for real; only the URL sent to the proxy changes. -# See ../testing/README.md. Leave unset to send the real document URL. -EFSP_TEST_DOCUMENT_URL = "" - -# Local development only. The EFSP proxy downloads each document itself -- for -# fee quotes as well as filings -- and cannot reach a LocalStack URL. Set this to -# any publicly readable PDF to run fees end-to-end without exposing this machine. -# Defaults to the sample test PDF hosted on GitHub. +# Set to "" to send the real document URL. See ../testing/README.md. +# +# Only read when running the app directly (manage.py runserver). Under +# `docker compose`, compose.yml sets this in the container environment and +# settings_dev loads this file with override=False, so set it in the shell or +# the repo-root .env instead. EFSP_TEST_DOCUMENT_URL = "https://raw.githubusercontent.com/SuffolkLITLab/LITEFile/main/testing/sample_test.pdf" # AWS S3 Configuration diff --git a/efile_app/efile/apps.py b/efile_app/efile/apps.py index 4131156..bb25fcb 100644 --- a/efile_app/efile/apps.py +++ b/efile_app/efile/apps.py @@ -4,3 +4,7 @@ class EfileConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "efile" + + def ready(self): + # Registers the checks in efile/checks.py by importing them. + from efile import checks # noqa: F401 diff --git a/efile_app/efile/checks.py b/efile_app/efile/checks.py new file mode 100644 index 0000000..8ac7c0d --- /dev/null +++ b/efile_app/efile/checks.py @@ -0,0 +1,52 @@ +"""Startup system checks. + +These run on every management command -- including the ``manage.py migrate`` that +fly.toml uses as its release command -- so a misconfiguration fails the deploy +instead of waiting to be discovered by a filing. +""" + +from django.conf import settings +from django.core.checks import Error, Warning, register + + +@register() +def efsp_stand_in_document_is_development_only(app_configs, **kwargs): + """EFSP_TEST_DOCUMENT_URL must never be active outside local development. + + When set, every document in a filing payload is replaced by a stand-in PDF + before the payload reaches the EFSP proxy. That is indispensable locally, + where the proxy cannot fetch a LocalStack URL, and unacceptable anywhere a + filing might reach a real court. + + Reported as an Error (which blocks the command) rather than a Warning: the + correct response to finding this in production is to stop, not to continue + and hope the request-time guard in efsp_payload catches it. + """ + stand_in_url = getattr(settings, "EFSP_TEST_DOCUMENT_URL", "") + if not stand_in_url: + return [] + + if not settings.DEBUG: + return [ + Error( + "EFSP_TEST_DOCUMENT_URL is set while DEBUG is False.", + hint=( + "This replaces every filed document with a stand-in PDF. It is a " + "local-development affordance only. Unset EFSP_TEST_DOCUMENT_URL, and " + "confirm DJANGO_SETTINGS_MODULE points at efile.settings_staging or " + "efile.settings_prod rather than falling back to efile.settings (dev)." + ), + id="efile.E001", + ) + ] + + return [ + Warning( + f"EFSP_TEST_DOCUMENT_URL is active: filings will send {stand_in_url} in place of every uploaded document.", + hint=( + "Fee quotes and submissions from this process do not reflect real " + "documents. Unset it to send real S3 URLs to the EFSP proxy." + ), + id="efile.W001", + ) + ] diff --git a/efile_app/efile/services/efsp_payload.py b/efile_app/efile/services/efsp_payload.py index baac0ca..03f875a 100644 --- a/efile_app/efile/services/efsp_payload.py +++ b/efile_app/efile/services/efsp_payload.py @@ -10,6 +10,7 @@ import requests from django.conf import settings +from django.core.exceptions import ImproperlyConfigured logger = logging.getLogger(__name__) @@ -41,14 +42,33 @@ def substitute_test_document_urls(efile_data): locally would otherwise require public ingress to the dev machine. Documents are still uploaded to S3 for real and drafts still store their real - keys and URLs; only the URL handed to the proxy is replaced. The setting is - defined in ``settings_dev`` alone, so no environment variable can turn this on - in staging or production. + keys and URLs; only the URL handed to the proxy is replaced. + + Three independent things have to hold for a substitution to happen, so that no + single mistake can put a stand-in document in front of a real court: + + 1. ``settings_dev`` refuses to load on a deployed host at all. + 2. The setting is defined in ``settings_dev`` alone -- staging and production + never read the environment variable, so exporting it there does nothing. + 3. This function refuses to substitute when ``DEBUG`` is off, and raises + rather than quietly falling back, because a production process holding a + stand-in URL is a broken deploy and should not file anything. """ test_url = getattr(settings, "EFSP_TEST_DOCUMENT_URL", "") if not test_url: return + if not settings.DEBUG: + # Fail closed and loud. Silently sending the real document would leave a + # misconfigured deploy running and undiagnosed until it did something + # worse; silently sending the stand-in would file a placeholder PDF as + # though it were the filer's document. + raise ImproperlyConfigured( + "EFSP_TEST_DOCUMENT_URL is set while DEBUG is False. The stand-in filing " + "document is a local-development affordance and must never reach a real " + "court. Unset EFSP_TEST_DOCUMENT_URL in this environment." + ) + substituted = 0 for bundle in efile_data.get("al_court_bundle", []): if bundle.get("data_url") == test_url: diff --git a/efile_app/efile/settings_dev.py b/efile_app/efile/settings_dev.py index 7f1402b..4b939ae 100644 --- a/efile_app/efile/settings_dev.py +++ b/efile_app/efile/settings_dev.py @@ -2,8 +2,31 @@ from pathlib import Path import dj_database_url +from django.core.exceptions import ImproperlyConfigured from dotenv import load_dotenv +# Refuse to be the settings module on a deployed host. +# +# `efile/settings.py` is a bare re-export of this module, and manage.py, wsgi.py +# and asgi.py all `setdefault("DJANGO_SETTINGS_MODULE", "efile.settings")`. So a +# deploy that loses DJANGO_SETTINGS_MODULE -- an edit to fly.toml's [env] block, +# a new machine started without it -- silently falls back to *development* +# settings: DEBUG=True, and EFSP_TEST_DOCUMENT_URL live, which would file +# stand-in PDFs against a real court. Every other guard on that setting keys off +# DEBUG, so none of them would fire in exactly this case. +# +# Fly always sets FLY_APP_NAME in the runtime environment, so its presence means +# "deployed" regardless of what DJANGO_SETTINGS_MODULE says. Failing at import +# turns a silent misconfiguration into a boot failure. Deliberately no override +# env var: an escape hatch here is the same footgun again. +if os.getenv("FLY_APP_NAME"): + raise ImproperlyConfigured( + "efile.settings_dev was loaded on a deployed host (FLY_APP_NAME=" + f"{os.environ['FLY_APP_NAME']!r}). Development settings enable DEBUG and the " + "EFSP stand-in document. Set DJANGO_SETTINGS_MODULE to efile.settings_staging " + "or efile.settings_prod." + ) + # Determine BASE_DIR without importing base to load .env first _BASE_DIR = Path(__file__).resolve().parent.parent diff --git a/efile_app/efile/tests/conftest.py b/efile_app/efile/tests/conftest.py new file mode 100644 index 0000000..dd7111d --- /dev/null +++ b/efile_app/efile/tests/conftest.py @@ -0,0 +1,22 @@ +"""Shared fixtures for the Python suite.""" + +import pytest +from django.test import override_settings + + +@pytest.fixture(autouse=True) +def stand_in_document_disabled_by_default(): + """Run every test in the production-shaped configuration. + + EFSP_TEST_DOCUMENT_URL is a local-development affordance -- compose sets it by + default so fee quotes work against LocalStack uploads -- but pytest-django + forces DEBUG=False, and "stand-in URL set, DEBUG off" is precisely what + substitute_test_document_urls refuses to run under. Without this fixture the + suite would pass in CI and fail on a developer's machine, for any test that + exercises a filing path. + + Tests that want the substitution opt into it with override_settings, so the + dependency is visible in the test that relies on it. + """ + with override_settings(EFSP_TEST_DOCUMENT_URL=""): + yield diff --git a/efile_app/efile/tests/test_efsp_payload.py b/efile_app/efile/tests/test_efsp_payload.py index 8fa7597..7017ef5 100644 --- a/efile_app/efile/tests/test_efsp_payload.py +++ b/efile_app/efile/tests/test_efsp_payload.py @@ -7,6 +7,8 @@ import logging import pytest +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.test import override_settings from efile.services.efsp_payload import ( @@ -51,16 +53,61 @@ def test_real_document_url_is_untouched_when_stand_in_is_not_configured(): assert efile_data["al_court_bundle"][0]["data_url"] == REAL_S3_URL -def test_substitution_is_inert_when_setting_is_absent_entirely(): - """settings_prod/settings_staging never define the setting at all.""" +def test_substitution_is_inert_when_setting_is_absent_entirely(monkeypatch): + """settings_prod/settings_staging never define the setting at all. + + Deleted rather than left to the ambient value: the suite runs under + settings_dev, which defaults the stand-in URL to a real PDF so local fee + quotes work, and `override_settings` cannot express "no such setting". + """ efile_data = _bundle() + # Through the LazySettings wrapper, not settings._wrapped: __getattr__ caches + # each setting on the wrapper, so deleting only from the inner object leaves + # the cached value visible to getattr. + monkeypatch.delattr(settings, "EFSP_TEST_DOCUMENT_URL", raising=False) substitute_test_document_urls(efile_data) assert efile_data["al_court_bundle"][0]["data_url"] == REAL_S3_URL -# --- EFSP_TEST_DOCUMENT_URL on ---------------------------------------------- +# --- EFSP_TEST_DOCUMENT_URL set outside development -------------------------- +# +# Django's test environment forces DEBUG=False, so every "on" case below has to +# opt into DEBUG=True explicitly -- which is the precondition the substitution +# actually requires. + + +def test_stand_in_url_is_refused_when_debug_is_off(): + """A non-development process holding a stand-in URL must not file anything.""" + efile_data = _bundle() + + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=False): + with pytest.raises(ImproperlyConfigured, match="DEBUG is False"): + substitute_test_document_urls(efile_data) + + +def test_refusal_happens_before_any_url_is_rewritten(): + """The raise must not leave a half-substituted payload behind.""" + efile_data = {"al_court_bundle": [{"data_url": REAL_S3_URL}, {"data_url": REAL_S3_URL}]} + + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=False): + with pytest.raises(ImproperlyConfigured): + substitute_test_document_urls(efile_data) + + assert [b["data_url"] for b in efile_data["al_court_bundle"]] == [REAL_S3_URL, REAL_S3_URL] + + +def test_full_payload_preparation_refuses_too(): + """The guard holds on the path both the fee quote and the submission call.""" + efile_data = _bundle() + + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=False): + with pytest.raises(ImproperlyConfigured): + prepare_efile_payload(efile_data, "illinois", "adams") + + +# --- EFSP_TEST_DOCUMENT_URL on in development -------------------------------- def test_stand_in_url_replaces_every_document_url(): @@ -71,7 +118,7 @@ def test_stand_in_url_replaces_every_document_url(): ] } - with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL): + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=True): substitute_test_document_urls(efile_data) assert [b["data_url"] for b in efile_data["al_court_bundle"]] == [STAND_IN_URL, STAND_IN_URL] @@ -80,7 +127,7 @@ def test_stand_in_url_replaces_every_document_url(): def test_substitution_warns_so_the_filing_is_not_mistaken_for_a_real_one(efile_logs): efile_data = _bundle() - with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL): + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=True): substitute_test_document_urls(efile_data) assert any("stand-in document" in record.getMessage() for record in efile_logs.records) @@ -89,7 +136,7 @@ def test_substitution_warns_so_the_filing_is_not_mistaken_for_a_real_one(efile_l def test_substitution_leaves_other_bundle_fields_alone(): efile_data = _bundle(filing_component="331", filing_type="27965") - with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL): + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=True): substitute_test_document_urls(efile_data) bundle = efile_data["al_court_bundle"][0] diff --git a/efile_app/efile/tests/test_stand_in_document_guards.py b/efile_app/efile/tests/test_stand_in_document_guards.py new file mode 100644 index 0000000..a312095 --- /dev/null +++ b/efile_app/efile/tests/test_stand_in_document_guards.py @@ -0,0 +1,95 @@ +"""The guards that keep the EFSP stand-in document out of a real filing. + +`substitute_test_document_urls` replaces every document in a filing payload with +a fixed PDF so fee quotes work locally, where the EFSP proxy cannot reach a +LocalStack URL. Reaching a real court, it would file a placeholder in place of +someone's actual document, so three independent layers have to fail before that +can happen. Each is tested here; test_efsp_payload.py covers the substitution +behaviour itself. +""" + +import importlib +from pathlib import Path + +import pytest +from django.core.exceptions import ImproperlyConfigured +from django.test import override_settings + +from efile.checks import efsp_stand_in_document_is_development_only + +STAND_IN_URL = "https://example.org/fixtures/blank.pdf" + + +# --- Layer 1: settings_dev refuses to load on a deployed host ---------------- + + +def test_dev_settings_refuse_to_load_on_a_deployed_host(monkeypatch): + """manage.py/wsgi.py/asgi.py all fall back to efile.settings, which is dev. + + A deploy that loses DJANGO_SETTINGS_MODULE would otherwise come up with + DEBUG=True, where every DEBUG-keyed guard below is inert. + """ + monkeypatch.setenv("FLY_APP_NAME", "forms-mvp-staging") + + import efile.settings_dev + + with pytest.raises(ImproperlyConfigured, match="deployed host"): + importlib.reload(efile.settings_dev) + + +def test_dev_settings_load_normally_off_a_deployed_host(monkeypatch): + monkeypatch.delenv("FLY_APP_NAME", raising=False) + + import efile.settings_dev + + importlib.reload(efile.settings_dev) # must not raise + + +# --- Layer 2: the startup system check --------------------------------------- + + +def test_check_errors_when_stand_in_is_set_outside_debug(): + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=False): + messages = efsp_stand_in_document_is_development_only(None) + + assert [m.id for m in messages] == ["efile.E001"] + + +def test_check_is_silent_when_stand_in_is_unset(): + """The production configuration must produce no noise at all.""" + with override_settings(EFSP_TEST_DOCUMENT_URL="", DEBUG=False): + assert efsp_stand_in_document_is_development_only(None) == [] + + +def test_check_warns_while_the_stand_in_is_active_in_development(): + """Active substitution is announced at startup rather than only per-request.""" + with override_settings(EFSP_TEST_DOCUMENT_URL=STAND_IN_URL, DEBUG=True): + messages = efsp_stand_in_document_is_development_only(None) + + assert [m.id for m in messages] == ["efile.W001"] + assert STAND_IN_URL in messages[0].msg + + +def test_check_is_registered_so_it_runs_on_management_commands(): + from django.core.checks import registry + + assert efsp_stand_in_document_is_development_only in registry.registry.get_checks() + + +# --- Layer 3: deployed settings modules never read the variable --------------- + + +def test_the_setting_is_only_ever_read_in_dev_settings(): + """Exporting EFSP_TEST_DOCUMENT_URL in staging or production must do nothing. + + Checked against the source rather than by importing settings_prod and + settings_staging, which refuse to import without a production secret key and + DATABASE_URL. The property being protected is exactly a textual one: no + settings module other than settings_dev may read this environment variable. + """ + settings_dir = Path(__file__).resolve().parent.parent + readers = sorted( + path.name for path in settings_dir.glob("settings*.py") if "EFSP_TEST_DOCUMENT_URL" in path.read_text() + ) + + assert readers == ["settings_dev.py"] diff --git a/testing/README.md b/testing/README.md index 6012abd..e9197fb 100644 --- a/testing/README.md +++ b/testing/README.md @@ -62,9 +62,40 @@ proxy forwards the bytes on to Tyler. The app logs a warning on every request where a substitution happens, so a stand-in filing is never mistaken for a real one. -The setting is defined in `settings_dev.py` only. `settings_staging` and -`settings_prod` never read it, so no environment variable can enable this -outside local development. +### Why this cannot reach a real court + +Filing a placeholder PDF in place of someone's actual document is the worst thing +this repository could do, so the stand-in is fenced in four independent ways. No +single mistake -- a stray environment variable, a dropped setting, a bad deploy -- +gets past all of them, and each one fails loudly rather than quietly. + +1. **`settings_dev` refuses to load on a deployed host.** `efile/settings.py` is + a bare re-export of `settings_dev`, and `manage.py`, `wsgi.py` and `asgi.py` + all `setdefault("DJANGO_SETTINGS_MODULE", "efile.settings")`. A deploy that + loses `DJANGO_SETTINGS_MODULE` -- an edit to `fly.toml`'s `[env]`, a machine + started without it -- would otherwise come up on *development* settings, with + `DEBUG=True` and every guard below inert. `settings_dev` now raises + `ImproperlyConfigured` at import when `FLY_APP_NAME` is present. There is no + override flag: an escape hatch would be the same footgun again. +2. **Only `settings_dev` reads the environment variable.** `settings_staging` and + `settings_prod` never define `EFSP_TEST_DOCUMENT_URL`, so exporting it there + does nothing. A test asserts no other settings module starts reading it. +3. **A startup system check** (`efile.E001`, in `efile/checks.py`) fails any + management command when the setting is non-empty while `DEBUG` is off. + `fly.toml` runs `manage.py migrate` as its release command, so a + misconfiguration fails the *deploy* rather than the first filing. In + development the same check emits `efile.W001` at startup, naming the stand-in + URL, so an active substitution is visible before anything is filed. +4. **The substitution itself refuses to run outside `DEBUG`**, raising + `ImproperlyConfigured` rather than falling back to the real document. A + production process holding a stand-in URL is a broken deploy and should file + nothing at all until it is fixed. + +`efile/tests/test_stand_in_document_guards.py` covers all four. The suite also +disables the stand-in for every test by default (`efile/tests/conftest.py`), so +tests run in the production-shaped configuration and a developer who has the +variable exported gets the same results as CI; tests that want the substitution +opt in with `override_settings`. ### If you need the proxy to fetch the real document From 7643339e07441ec923e6d059f0f6eafa3c12cac9 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Thu, 23 Jul 2026 10:09:15 -0400 Subject: [PATCH 43/47] Improve cache busting to mirror staging/prod methods; better validation; safety checks --- efile_app/efile/api/filing_views.py | 33 ++-- efile_app/efile/api/suffolk_api_views.py | 48 ++++-- efile_app/efile/services/efsp_payload.py | 125 +++++++++++++- efile_app/efile/settings_dev.py | 31 ++++ efile_app/efile/static/js/api-utils.js | 58 ++++++- efile_app/efile/static/js/payment.js | 6 +- efile_app/efile/static/js/review.js | 17 +- efile_app/efile/tests/test_durable_drafts.py | 37 +++++ efile_app/efile/tests/test_efsp_payload.py | 163 +++++++++++++++++++ efile_app/efile/views/session_api.py | 28 ++-- efile_app/efile/views/submission.py | 7 + efile_app/js-tests/api-utils.test.js | 96 ++++++++++- 12 files changed, 581 insertions(+), 68 deletions(-) diff --git a/efile_app/efile/api/filing_views.py b/efile_app/efile/api/filing_views.py index 175593f..4fce663 100644 --- a/efile_app/efile/api/filing_views.py +++ b/efile_app/efile/api/filing_views.py @@ -13,7 +13,8 @@ from efile.utils.jurisdiction_stuff import get_jurisdiction_from_request -from ..services.efsp_payload import prepare_efile_payload +from ..services.efsp_errors import describe_efsp_error +from ..services.efsp_payload import PayloadValidationError, prepare_efile_payload from ..utils.case_data_utils import get_case_data from ..utils.proxy_connection import get_headers from .base import APIResponseMixin @@ -178,7 +179,12 @@ def payment_fees(request): # Must match what submit_final_filing sends, or fees are quoted # against a payload that differs from the one actually filed. - prepare_efile_payload(efile_data, jurisdiction_id, court_id) + try: + prepare_efile_payload(efile_data, jurisdiction_id, court_id) + except PayloadValidationError as error: + # Known-bad payload: answer with the specific reason rather than + # letting the EFSP reply with a code-list error no filer can act on. + return JsonResponse({"success": False, "error": str(error)}, status=400) url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filing/fees" @@ -213,31 +219,12 @@ def payment_fees(request): else: # Debug, not info: fee responses echo party names and case details. logger.debug("EFSP fee response body: %s", response.text[:2000]) - try: - error_data = response.json() - if isinstance(error_data, dict): - error_message = error_data.get("error", f"API returned status {response.status_code}") - else: - error_message = error_data - logger.info(f"Sending back: {error_data}, {error_message}") - - # For 400 errors, include more details - if response.status_code == 400: - validation_errors = error_data.get("validation_errors", error_data.get("errors", [])) - if validation_errors: - error_message += f" - Validation errors: {validation_errors}" - - except json.JSONDecodeError: - error_message = f"API returned status {response.status_code} - Response: {response.text}" - except Exception as parse_error: - error_message = ( - f"API returned status {response.status_code} - Could not parse response: {str(parse_error)}" - ) + error_message = describe_efsp_error(response) return JsonResponse( { "success": False, - "error": f"Filing submission failed: {error_message}", + "error": f"Could not get filing fees: {error_message}", "api_status_code": response.status_code, "api_response": response.text[:500] if response.text else "No response body", }, diff --git a/efile_app/efile/api/suffolk_api_views.py b/efile_app/efile/api/suffolk_api_views.py index 5342d98..de75c5a 100644 --- a/efile_app/efile/api/suffolk_api_views.py +++ b/efile_app/efile/api/suffolk_api_views.py @@ -12,7 +12,7 @@ from django.views.decorators.http import require_http_methods from requests.exceptions import RequestException -from efile.utils.case_data_utils import update_case_data +from efile.utils.case_data_utils import get_case_data, update_case_data from efile.utils.jurisdiction_stuff import get_jurisdiction_from_request from efile.utils.proxy_connection import get_headers @@ -293,19 +293,43 @@ def get_party_types_from_suffolk_api(request): f"{party_types[0].get('name', 'Unknown')} ({selected_party_type})" ) - # Persist the resolved party type onto the current draft - update_case_data( - request, - { - "determined_party_type": selected_party_type, - "party_type": selected_party_type, - "petitioner_party_type": selected_party_type, - "existing_case": existing_case, - }, - jurisdiction, + # Seed the draft's party type, but never overwrite one it already + # has. Everything above is a *guess* from the case type's name + # ("civil" -> plaintiff), and this endpoint is a GET called on + # page load -- including from review, long after the filer chose. + # Overwriting there silently replaced a filer who had picked + # Defendant/Respondent with the guessed Plaintiff/Petitioner, + # leaving both sides the same party type and the filing rejected + # by the court for a required party that no longer had anyone in + # it. The guess is a default for an empty draft, not an answer. + stored_case_data = get_case_data(request, jurisdiction) or {} + stored_party_type = next( + ( + stored_case_data.get(key) + for key in ("determined_party_type", "party_type", "petitioner_party_type") + if stored_case_data.get(key) + ), + "", ) - logger.debug(f"Saved party type to draft: {selected_party_type}") + if stored_party_type: + # Answer with what the draft holds, so the page cannot show a + # different party type from the one that will be filed. + selected_party_type = stored_party_type + update_case_data(request, {"existing_case": existing_case}, jurisdiction) + logger.debug("Kept the party type already on the draft: %s", stored_party_type) + else: + update_case_data( + request, + { + "determined_party_type": selected_party_type, + "party_type": selected_party_type, + "petitioner_party_type": selected_party_type, + "existing_case": existing_case, + }, + jurisdiction, + ) + logger.debug(f"Saved party type to draft: {selected_party_type}") if only_required: filtered_party_types = [p for p in party_types if p.get("isrequired", False)] diff --git a/efile_app/efile/services/efsp_payload.py b/efile_app/efile/services/efsp_payload.py index 03f875a..c31af6a 100644 --- a/efile_app/efile/services/efsp_payload.py +++ b/efile_app/efile/services/efsp_payload.py @@ -19,11 +19,24 @@ _PLACEHOLDER_COMPONENT_LABELS = {"", "supporting", "attachment", "attachments"} +class PayloadValidationError(Exception): + """The payload cannot succeed at the EFSP, with a reason worth showing a filer. + + Raised only for conditions the court's own code lists already prove wrong, so + the message can be specific. Views turn this into a 400 carrying the message. + """ + + def prepare_efile_payload(efile_data, jurisdiction_id, court_id): - """Apply every server-side fixup an EFSP request needs. Mutates ``efile_data``.""" + """Apply every server-side fixup an EFSP request needs. Mutates ``efile_data``. + + Raises ``PayloadValidationError`` when the payload is knowably invalid. + """ _drop_empty_cross_references(efile_data) substitute_test_document_urls(efile_data) + validate_document_selections(efile_data) resolve_placeholder_filing_components(efile_data, jurisdiction_id, court_id) + validate_required_party_types(efile_data, jurisdiction_id, court_id) return efile_data @@ -84,6 +97,43 @@ def substitute_test_document_urls(efile_data): ) +def validate_document_selections(efile_data): + """Reject a document that carries no filing type. + + The filing type is chosen per document on the upload screen, and a draft can + reach the fee quote without one: an interrupted session, or an upload page + running a stale script whose own check was missing. The blank travels as + ``"filing_type": ""`` and the EFSP answers with a `wrong_vars` entry naming + ``al_court_bundle.elements[0].filing_type`` and a bare list of code numbers. + + Only the filing type is checked. It is required for every court -- the code + lists are keyed on it -- while document type and the rest vary by court, and + this module refuses payloads only where the court's own rules already prove + them wrong. + """ + missing = [ + _document_label(bundle, index) + for index, bundle in enumerate(efile_data.get("al_court_bundle", [])) + if not str(bundle.get("filing_type") or "").strip() + ] + if not missing: + return + + raise PayloadValidationError( + f"No filing type is set for: {', '.join(missing)}. Go back to the documents step " + f"and choose a filing type for each document." + ) + + +def _document_label(bundle, index): + """Name a bundle the way the filer would recognise it.""" + for key in ("filename", "filing_description", "document_description"): + label = str(bundle.get(key) or "").strip() + if label: + return label + return f"document {index + 1}" + + def resolve_placeholder_filing_components(efile_data, jurisdiction_id, court_id): """Replace leftover UI labels such as ``"supporting"`` with the court's code. @@ -111,6 +161,79 @@ def resolve_placeholder_filing_components(efile_data, jurisdiction_id, court_id) bundle["filing_component"] = code +def validate_required_party_types(efile_data, jurisdiction_id, court_id): + """Reject a payload that leaves one of the court's required party types empty. + + A case type declares which party types are mandatory -- a Cook County civil + case needs both a Plaintiff and a Defendant. Nothing in the UI stops a filer + from choosing the same type for themselves and for the other side, and the + EFSP answers that with an opaque 400 ("All required parties not covered by + existing party types. ([173174]. Missing [173180]") that reaches the filer as + a generic failure with no way to act on it. + + Checked here rather than in the browser so the fee quote and the submission + share one answer, and so it holds regardless of what the client sends. + + Fails open: if the court's party-type list cannot be fetched, the payload goes + through and the EFSP stays the authority. This check exists to produce a + better message, not to be a second gatekeeper. + """ + case_type = efile_data.get("efile_case_type") + if not case_type: + return + + required = _lookup_required_party_types(jurisdiction_id, court_id, case_type) + if not required: + return + + present = { + str(party.get("party_type")) + for party in [*efile_data.get("users", []), *efile_data.get("other_parties", [])] + if party.get("party_type") + } + missing = {code: name for code, name in required.items() if code not in present} + if not missing: + return + + missing_names = ", ".join(sorted(missing.values())) + raise PayloadValidationError( + f"This case type requires a party of every required type, and none was given for: " + f"{missing_names}. Go back to the case details and give each side a different party type." + ) + + +def _lookup_required_party_types(jurisdiction_id, court_id, case_type): + """Return ``{code: name}`` for the case type's required party types. + + Empty when the list cannot be fetched or the court marks nothing required, so + callers treat "unknown" and "nothing required" alike -- both mean "do not + block on this". + """ + url = ( + f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/codes/courts/{court_id}/" + f"case_types/{case_type}/party_types" + ) + try: + response = requests.get(url, timeout=10) + if response.status_code != 200: + return {} + party_types = response.json() + except (requests.RequestException, ValueError) as error: + logger.warning("Could not resolve required party types for case type %s: %s", case_type, error) + return {} + + if not isinstance(party_types, list): + return {} + + return { + str(party_type.get("code")): party_type.get("name") or str(party_type.get("code")) + for party_type in party_types + # The EFSP renders this as a JSON boolean, but Tyler has been seen sending + # the string "true" for the same field elsewhere in the code lists. + if isinstance(party_type, dict) and str(party_type.get("isrequired", "")).lower() == "true" + } + + def _lookup_attachment_component(jurisdiction_id, court_id, filing_type): """Return the court's attachment filing-component code, or None if unavailable.""" url = ( diff --git a/efile_app/efile/settings_dev.py b/efile_app/efile/settings_dev.py index 4b939ae..e154d30 100644 --- a/efile_app/efile/settings_dev.py +++ b/efile_app/efile/settings_dev.py @@ -36,6 +36,8 @@ from efile.settings_base import * # noqa: E402,F401,F403 from efile.settings_base import ALLOWED_HOSTS as BASE_ALLOWED_HOSTS # noqa: E402 from efile.settings_base import DATABASES as BASE_DATABASES # noqa: E402 +from efile.settings_base import INSTALLED_APPS as BASE_INSTALLED_APPS # noqa: E402 +from efile.settings_base import MIDDLEWARE as BASE_MIDDLEWARE # noqa: E402 # Bind DATABASES explicitly to avoid F405 and make linter aware of the symbol DATABASES = BASE_DATABASES @@ -43,6 +45,35 @@ DEBUG = True ALLOWED_HOSTS = BASE_ALLOWED_HOSTS or ["localhost", "127.0.0.1", ".localhost", "[::1]", "testserver"] +# Static files: serve dev assets through WhiteNoise, as staging and production do. +# +# `runserver`'s own static handler sends Last-Modified and an ETag but no +# Cache-Control, which leaves the browser free to guess a freshness lifetime +# (Chrome: a fraction of the file's age) and reuse a script for hours without +# revalidating. A JS change then reaches the tests and never reaches the browser +# -- which is how an upload page shipped a validation check that the running app +# did not have, and let a filing go out with no filing type selected. +# +# ManifestStaticFilesStorage covers the deployed environments by content-hashing +# each filename, so the fix belongs here rather than in a ?v= query string in the +# templates: a hand-maintained cache-buster only helps the files someone +# remembered to bump. +# +# `whitenoise.runserver_nostatic` must precede django.contrib.staticfiles to stop +# runserver from installing its handler ahead of the middleware. USE_FINDERS +# serves straight from STATICFILES_DIRS (no collectstatic in dev), AUTOREFRESH +# re-stats each file per request, and MAX_AGE=0 makes the browser revalidate +# every time. +INSTALLED_APPS = ["whitenoise.runserver_nostatic", *BASE_INSTALLED_APPS] +MIDDLEWARE = list(BASE_MIDDLEWARE) +MIDDLEWARE.insert( + MIDDLEWARE.index("django.middleware.security.SecurityMiddleware") + 1, + "whitenoise.middleware.WhiteNoiseMiddleware", +) +WHITENOISE_USE_FINDERS = True +WHITENOISE_AUTOREFRESH = True +WHITENOISE_MAX_AGE = 0 + # Opt-in: send this URL to the EFSP proxy as every document's `data_url` instead # of the real S3 URL. # diff --git a/efile_app/efile/static/js/api-utils.js b/efile_app/efile/static/js/api-utils.js index 79f3844..f7266e4 100644 --- a/efile_app/efile/static/js/api-utils.js +++ b/efile_app/efile/static/js/api-utils.js @@ -3,6 +3,20 @@ * Features: CSRF handling, request building, error handling */ class ApiUtils { + /** + * Timeout for calls that reach the EFSP proxy, which in turn calls Tyler: + * fee quotes and filing submissions. Round trips over 40s have been observed + * on real courts (Adams County order-of-protection fees), against a 30s + * default that reported "Request timed out" for a request the server went on + * to answer with a valid $0.00 quote. + * + * Deliberately longer than the server's own 60s timeout on that call, so the + * server is what decides a request has failed. The browser giving up first + * only hides the outcome: this timeout does not abort the request, so the + * filing continues regardless of what the filer is shown. + */ + static FILING_TIMEOUT_MS = 120000; + constructor() { this.baseUrl = window.location.origin; this.csrfToken = this.getCSRFToken(); @@ -181,19 +195,39 @@ class ApiUtils { requestOptions.body = JSON.stringify(data); } - // Create a timeout promise + // Create a timeout promise. The timer is cleared once the race + // settles: it is not cancelled by losing, and an uncleared one keeps + // a pending task alive for the full budget after the response is + // already in hand -- up to two minutes on a filing call. + let timeoutId; const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => reject(new Error('Request timeout')), timeout); + timeoutId = setTimeout(() => reject(new Error('Request timeout')), timeout); }); // Race between fetch and timeout - const response = await Promise.race([ - fetch(url, requestOptions), - timeoutPromise - ]); + let response; + try { + response = await Promise.race([ + fetch(url, requestOptions), + timeoutPromise + ]); + } finally { + clearTimeout(timeoutId); + } if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); + // Prefer the server's own message. Our API answers failures with + // {success: false, error: "..."}, and that text is often the only + // actionable thing the filer can be told -- which required party + // is missing, which document the EFSP could not fetch. Collapsing + // every 400 to "Invalid request" throws exactly that away. + const serverMessage = await response.clone().json() + .then(body => body?.error) + .catch(() => null); + const error = new Error(serverMessage || `HTTP error! status: ${response.status}`); + error.status = response.status; + error.serverMessage = serverMessage || null; + throw error; } const result = await response.json(); @@ -218,6 +252,11 @@ class ApiUtils { } handleApiError(error) { + // A message the server wrote is always more useful than the status-code + // wording below, so it passes through untouched. + if (error.serverMessage) { + return error; + } if (error.message === 'Request timeout') { return new Error('Request timed out. Please check your connection and try again.'); } else if (error.message.includes('Failed to fetch')) { @@ -273,11 +312,12 @@ class ApiUtils { } - async post(endpoint, data = {}, params = {}) { + async post(endpoint, data = {}, params = {}, options = {}) { return this.makeRequest(endpoint, { method: 'POST', data, - params + params, + ...options }); } diff --git a/efile_app/efile/static/js/payment.js b/efile_app/efile/static/js/payment.js index 1138b49..d081498 100644 --- a/efile_app/efile/static/js/payment.js +++ b/efile_app/efile/static/js/payment.js @@ -308,7 +308,9 @@ const FilingHandler = { this.handleFeesResponse(result); } catch (error) { console.warn("Error on submission: %o", error) - Messages.showError(gettext("An unexpected error occurred. Please try again.")); + // Show what the server said when it said anything: "no party of type + // Plaintiff" is fixable by the filer, "an unexpected error" is not. + Messages.showError(error?.serverMessage || gettext("An unexpected error occurred. Please try again.")); this.setFeesState(false); } }, @@ -369,6 +371,8 @@ const FilingHandler = { efile_data: efilingData, confirm_submission: true, payment_account_id: paymentAccountID + }, {}, { + timeout: ApiUtils.FILING_TIMEOUT_MS }); } }; diff --git a/efile_app/efile/static/js/review.js b/efile_app/efile/static/js/review.js index ea8bb8f..32dec19 100644 --- a/efile_app/efile/static/js/review.js +++ b/efile_app/efile/static/js/review.js @@ -103,7 +103,14 @@ const DataManager = { }, ...options }); - return response.ok ? await response.json() : null; + // Parse the body on failures too. This is the filing-submission call, + // and the API answers a rejection with {success: false, error: "..."} + // naming what the filer has to correct -- a missing required party, + // a document the EFSP could not fetch. Returning null on !ok threw + // that away and left "An error occurred during submission." + // handleSubmissionResult already keys off `success`, so a non-ok body + // routes to the same error branch, now with the reason in it. + return await response.json().catch(() => null); } catch (error) { console.error(`Fetch error for ${url}:`, error); return null; @@ -446,7 +453,9 @@ const FilingHandler = { this.handleSubmissionResult(result); } catch (error) { console.error("Error on submission: %o", error) - Messages.showError(gettext("An unexpected error occurred. Please try again.")); + // See payment.js: a message the server wrote names something the + // filer can actually correct. + Messages.showError(error?.serverMessage || gettext("An unexpected error occurred. Please try again.")); this.setSubmissionState(false); } }, @@ -462,7 +471,7 @@ const FilingHandler = { this.handleFeesResponse(result); } catch (error) { console.error("Error on submission: %o", error) - Messages.showError(gettext("An unexpected error occurred. Please try again.")); + Messages.showError(error?.serverMessage || gettext("An unexpected error occurred. Please try again.")); this.setFeesState(false); } }, @@ -481,6 +490,8 @@ const FilingHandler = { efile_data: efilingData, confirm_submission: true, payment_account_id: paymentAccountID + }, {}, { + timeout: ApiUtils.FILING_TIMEOUT_MS }); }, diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index f32c60b..f818871 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -13,6 +13,7 @@ write_case_data, write_upload_data, ) +from efile.services.efsp_payload import PayloadValidationError from efile.workflow import WorkflowStepKey, get_workflow_step_choices @@ -661,3 +662,39 @@ def test_partial_case_update_does_not_clear_omitted_fields(django_user_model): assert draft.court_code == "new-code" assert draft.court_name == "Court name to preserve" + + +@pytest.mark.django_db +def test_payload_validation_failure_releases_draft_for_a_corrected_retry(client, django_user_model, monkeypatch): + """A pre-flight rejection never reached the court, so the filer may fix and retry. + + The submission wrapper decides between "release to DRAFT" and "park in ERROR" + largely by matching error text, and ERROR is not claimable. A validation + rejection landing in ERROR would lock a filer out of their own filing over a + party type they could have corrected in a minute, so the response carries an + explicit pre_submit marker instead of relying on its wording. + """ + + def reject(*_args, **_kwargs): + raise PayloadValidationError("This case type requires a party of every required type") + + def unreachable_post(*_args, **_kwargs): + raise AssertionError("the filing API must not be called for an invalid payload") + + monkeypatch.setattr("efile.views.session_api.prepare_efile_payload", reject) + monkeypatch.setattr("requests.post", unreachable_post) + user = django_user_model.objects.create_user(username="invalid-payload-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois", current_step=WorkflowStepKey.REVIEW) + client.force_login(user) + _prepare_submission(client, draft) + + response = client.post( + reverse("submit_final_filing"), + data={"confirm_submission": True, "efile_data": {"al_court_bundle": {}}}, + content_type="application/json", + ) + + assert response.status_code == 400 + assert "requires a party of every required type" in response.json()["error"] + draft.refresh_from_db() + assert draft.status == FilingDraft.Status.DRAFT diff --git a/efile_app/efile/tests/test_efsp_payload.py b/efile_app/efile/tests/test_efsp_payload.py index 7017ef5..b614090 100644 --- a/efile_app/efile/tests/test_efsp_payload.py +++ b/efile_app/efile/tests/test_efsp_payload.py @@ -12,9 +12,12 @@ from django.test import override_settings from efile.services.efsp_payload import ( + PayloadValidationError, prepare_efile_payload, resolve_placeholder_filing_components, substitute_test_document_urls, + validate_document_selections, + validate_required_party_types, ) REAL_S3_URL = "https://forms-mvp-xf6361.s3.us-east-1.amazonaws.com/efile-documents/lead/abc.pdf?X-Amz-Signature=x" @@ -166,6 +169,57 @@ def test_populated_cross_references_is_preserved(): assert efile_data["cross_references"] == [{"code": "1", "value": "abc"}] +# --- missing filing type ------------------------------------------------------ +# +# A draft can reach the fee quote with no filing type on a document. The EFSP +# answers with a wrong_vars entry and a list of bare code numbers, so the blank +# is caught here while the document can still be named. + + +@pytest.mark.parametrize("blank", ["", " ", None]) +def test_document_without_a_filing_type_is_rejected(blank): + efile_data = _bundle(filing_type=blank, filename="petition.pdf") + + with pytest.raises(PayloadValidationError, match="petition.pdf"): + validate_document_selections(efile_data) + + +def test_missing_filing_type_names_every_affected_document(): + efile_data = { + "al_court_bundle": [ + {"filing_type": "27965", "filename": "petition.pdf"}, + {"filing_type": "", "filename": "exhibit-a.pdf"}, + {"filing_type": "", "filename": "exhibit-b.pdf"}, + ] + } + + with pytest.raises(PayloadValidationError) as rejection: + validate_document_selections(efile_data) + + assert "exhibit-a.pdf" in str(rejection.value) + assert "exhibit-b.pdf" in str(rejection.value) + assert "petition.pdf" not in str(rejection.value) + + +def test_unnamed_document_is_identified_by_position(): + with pytest.raises(PayloadValidationError, match="document 2"): + validate_document_selections({"al_court_bundle": [{"filing_type": "27965"}, {"filing_type": ""}]}) + + +def test_document_type_is_left_to_the_court(): + """Only the filing type is universally required; the rest varies by court.""" + validate_document_selections({"al_court_bundle": [{"filing_type": "27965", "document_type": ""}]}) + + +def test_preparation_rejects_a_blank_filing_type_end_to_end(): + """The exact payload the payment screen sent when its filing type was blank.""" + efile_data = _bundle(filing_type="", filename="petition.pdf") + + with override_settings(EFSP_TEST_DOCUMENT_URL=""): + with pytest.raises(PayloadValidationError, match="filing type"): + prepare_efile_payload(efile_data, "illinois", "edgar") + + # --- placeholder filing components ------------------------------------------- @@ -236,3 +290,112 @@ def boom(*args, **kwargs): # Guessing a code would file under the wrong component; leave it visibly wrong. assert efile_data["al_court_bundle"][0]["filing_component"] == "supporting" + + +# --- required party types ---------------------------------------------------- +# +# A case type names the party types a filing must include. Nothing in the UI stops +# a filer from choosing the same type for themselves and the other side, and the +# EFSP answers that with a code-list error ("Missing [173180]") that means nothing +# to a filer, so the combination is caught here while it can still be explained. + +PLAINTIFF = {"code": "173180", "name": "Plaintiff", "isrequired": True} +DEFENDANT = {"code": "173174", "name": "Defendant", "isrequired": True} +OPTIONAL_GUARDIAN = {"code": "173999", "name": "Guardian", "isrequired": False} + + +class _PartyTypesResponse: + status_code = 200 + + def __init__(self, party_types): + self._party_types = party_types + + def json(self): + return self._party_types + + +def _party_type_api(monkeypatch, settings, party_types): + settings.EFSP_URL = "https://efile-test.example" + monkeypatch.setattr( + "efile.services.efsp_payload.requests.get", + lambda *args, **kwargs: _PartyTypesResponse(party_types), + ) + + +def _payload(user_types, other_types=()): + return { + "efile_case_type": "186542", + "al_court_bundle": [], + "users": [{"party_type": code} for code in user_types], + "other_parties": [{"party_type": code} for code in other_types], + } + + +def test_missing_required_party_type_is_rejected_with_the_missing_name(monkeypatch, settings): + """The exact case a filer hits by picking Defendant for both sides.""" + _party_type_api(monkeypatch, settings, [PLAINTIFF, DEFENDANT]) + + with pytest.raises(PayloadValidationError, match="Plaintiff"): + validate_required_party_types(_payload(["173174"], ["173174"]), "illinois", "cook:chd1") + + +def test_every_required_party_type_present_passes(monkeypatch, settings): + _party_type_api(monkeypatch, settings, [PLAINTIFF, DEFENDANT]) + + validate_required_party_types(_payload(["173180"], ["173174"]), "illinois", "cook:chd1") + + +def test_other_parties_count_toward_coverage(monkeypatch, settings): + """The filer is one side; the other side is only ever in other_parties.""" + _party_type_api(monkeypatch, settings, [PLAINTIFF, DEFENDANT]) + + validate_required_party_types(_payload(["173174"], ["173180"]), "illinois", "cook:chd1") + + +def test_optional_party_types_are_not_required(monkeypatch, settings): + _party_type_api(monkeypatch, settings, [PLAINTIFF, DEFENDANT, OPTIONAL_GUARDIAN]) + + validate_required_party_types(_payload(["173180"], ["173174"]), "illinois", "cook:chd1") + + +def test_string_valued_isrequired_is_honoured(monkeypatch, settings): + """Tyler's code lists are inconsistent about JSON booleans vs "true".""" + _party_type_api(monkeypatch, settings, [{"code": "173180", "name": "Plaintiff", "isrequired": "true"}]) + + with pytest.raises(PayloadValidationError, match="Plaintiff"): + validate_required_party_types(_payload(["173174"]), "illinois", "cook:chd1") + + +def test_unreachable_party_type_list_does_not_block_the_filing(monkeypatch, settings): + """Fails open: the EFSP stays the authority, this check only improves the message.""" + import requests + + settings.EFSP_URL = "https://efile-test.example" + + def boom(*args, **kwargs): + raise requests.RequestException("EFSP unreachable") + + monkeypatch.setattr("efile.services.efsp_payload.requests.get", boom) + + validate_required_party_types(_payload(["173174"], ["173174"]), "illinois", "cook:chd1") + + +def test_payload_without_a_case_type_is_not_checked(monkeypatch, settings): + """No case type means no code list to check against -- and no API call.""" + settings.EFSP_URL = "https://efile-test.example" + + def fail(*args, **kwargs): + raise AssertionError("should not call the EFSP without a case type") + + monkeypatch.setattr("efile.services.efsp_payload.requests.get", fail) + + validate_required_party_types({"al_court_bundle": [], "users": []}, "illinois", "cook:chd1") + + +def test_preparation_rejects_the_payload_end_to_end(monkeypatch, settings): + """prepare_efile_payload is what both the fee quote and the submission call.""" + _party_type_api(monkeypatch, settings, [PLAINTIFF, DEFENDANT]) + + with override_settings(EFSP_TEST_DOCUMENT_URL=""): + with pytest.raises(PayloadValidationError): + prepare_efile_payload(_payload(["173174"], ["173174"]), "illinois", "cook:chd1") diff --git a/efile_app/efile/views/session_api.py b/efile_app/efile/views/session_api.py index a6a5810..e31e2d4 100644 --- a/efile_app/efile/views/session_api.py +++ b/efile_app/efile/views/session_api.py @@ -9,7 +9,8 @@ from django.views.decorators.http import require_http_methods from ..services.current_drafts import get_current_draft -from ..services.efsp_payload import prepare_efile_payload +from ..services.efsp_errors import describe_efsp_error +from ..services.efsp_payload import PayloadValidationError, prepare_efile_payload from ..utils.case_data_utils import get_case_data, get_upload_data, update_case_data, update_upload_data from ..utils.llms import LlmError, extract_fields_from_file from ..utils.proxy_connection import get_party_type_code_from_api @@ -302,7 +303,13 @@ def submit_final_filing(request): return JsonResponse({"success": False, "error": "Court ID is required for filing submission"}, status=400) # Same fixups the fee quote applied, so the filing matches the quote. - prepare_efile_payload(efile_data, jurisdiction_id, court_id) + try: + prepare_efile_payload(efile_data, jurisdiction_id, court_id) + except PayloadValidationError as error: + # `pre_submit` tells the submission wrapper that nothing reached the + # filing API, so the draft is released for a corrected retry instead + # of being parked in ERROR. + return JsonResponse({"success": False, "error": str(error), "pre_submit": True}, status=400) # Construct the Suffolk LIT Lab API endpoint api_url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filings" @@ -370,22 +377,7 @@ def submit_final_filing(request): logger.error(f"API Error - Status: {response.status_code}") logger.error(f"API Error - Response: {response.text}") - try: - error_data = response.json() - error_message = error_data.get("error", f"API returned status {response.status_code}") - - # For 400 errors, include more details - if response.status_code == 400: - validation_errors = error_data.get("validation_errors", error_data.get("errors", [])) - if validation_errors: - error_message += f" - Validation errors: {validation_errors}" - - except json.JSONDecodeError: - error_message = f"API returned status {response.status_code} - Response: {response.text}" - except Exception as parse_error: - error_message = ( - f"API returned status {response.status_code} - Could not parse response: {str(parse_error)}" - ) + error_message = describe_efsp_error(response) return JsonResponse( { diff --git a/efile_app/efile/views/submission.py b/efile_app/efile/views/submission.py index 62a8a6f..939c794 100644 --- a/efile_app/efile/views/submission.py +++ b/efile_app/efile/views/submission.py @@ -22,6 +22,11 @@ # Errors the session submit view returns *before* it calls the external filing # API. Only these -- and a confirmed API rejection -- are safe to release back to # DRAFT; any other failure may have left a filing submitted. +# +# Matching on message text is why new pre-call rejections should set +# `"pre_submit": True` on their response instead of being added here: a reworded +# message would otherwise silently start parking drafts in ERROR, locking the +# filer out of a retry for something that never reached the court. _PRE_SUBMIT_ERROR_PREFIXES = ( "Submission confirmation is required", "No case data found", @@ -68,6 +73,8 @@ def _failed_before_external_call(payload: dict) -> bool: """True only when the submit view rejected the request before calling the API.""" if "api_status_code" in payload: return False + if payload.get("pre_submit") is True: + return True error = payload.get("error") return isinstance(error, str) and error.startswith(_PRE_SUBMIT_ERROR_PREFIXES) diff --git a/efile_app/js-tests/api-utils.test.js b/efile_app/js-tests/api-utils.test.js index 258eec3..17c3dd7 100644 --- a/efile_app/js-tests/api-utils.test.js +++ b/efile_app/js-tests/api-utils.test.js @@ -108,4 +108,98 @@ test("saving case data does not populate the read cache", async () => { await client.getCaseData(); // 1 save + 2 uncached reads assert.strictEqual(calls(), 3); -}); \ No newline at end of file +}); + +// --- error reporting --------------------------------------------------------- +// +// The server answers a rejected filing with {success: false, error: "..."} that +// names what the filer must correct -- a missing required party type, a document +// the EFSP could not fetch. Collapsing every 400 to "Invalid request. Please +// check your input." hid two real, separately-diagnosable failures behind one +// unactionable sentence. + +function stubFetch(response) { + const client = new ApiUtils(); + globalThis.fetch = async () => response; + return client; +} + +function jsonResponse({ ok, status, body }) { + const response = { + ok, + status, + json: async () => body, + }; + response.clone = () => response; + return response; +} + +test("a failed request surfaces the server's own error message", async () => { + const client = stubFetch(jsonResponse({ + ok: false, + status: 400, + body: { success: false, error: "This case type requires a Plaintiff." }, + })); + + await assert.rejects( + () => client.post("/api/payment-fees/", {}), + (error) => { + assert.strictEqual(error.message, "This case type requires a Plaintiff."); + assert.strictEqual(error.serverMessage, "This case type requires a Plaintiff."); + assert.strictEqual(error.status, 400); + return true; + }, + ); +}); + +test("a failure with no parseable body still reports the status", async () => { + const response = { + ok: false, + status: 500, + json: async () => { + throw new Error("not JSON"); + }, + }; + response.clone = () => response; + const client = stubFetch(response); + + await assert.rejects( + () => client.post("/api/payment-fees/", {}), + (error) => { + assert.match(error.message, /Server error/); + return true; + }, + ); +}); + +test("a successful response is returned unchanged", async () => { + const client = stubFetch(jsonResponse({ + ok: true, + status: 200, + body: { success: true, api_response: { feesCalculationAmount: { value: 0 } } }, + })); + + const result = await client.post("/api/payment-fees/", {}); + + assert.strictEqual(result.success, true); +}); + + +// --- long-running filing calls ---------------------------------------------- + +test("fee quotes and submissions may raise the request timeout", async () => { + // The generic 30s default reported a timeout for an Adams County fee quote + // the server answered successfully after 43s. Callers pass the longer budget + // explicitly rather than raising it for every request on the page. + const client = new ApiUtils(); + let seenTimeout = null; + client.makeRequest = async (endpoint, options) => { + seenTimeout = options.timeout; + return { success: true }; + }; + + await client.post("/api/payment-fees/", {}, {}, { timeout: ApiUtils.FILING_TIMEOUT_MS }); + + assert.strictEqual(seenTimeout, ApiUtils.FILING_TIMEOUT_MS); + assert.ok(ApiUtils.FILING_TIMEOUT_MS > 60000, "must outlast the server's own 60s timeout"); +}); From 71a5e039a067bfcb9291aa55e7f98f56404cd2b5 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Thu, 23 Jul 2026 10:13:55 -0400 Subject: [PATCH 44/47] Add missing files --- efile_app/efile/services/efsp_errors.py | 100 +++++++++++++ efile_app/efile/tests/test_efsp_errors.py | 138 ++++++++++++++++++ .../tests/test_party_type_persistence.py | 96 ++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 efile_app/efile/services/efsp_errors.py create mode 100644 efile_app/efile/tests/test_efsp_errors.py create mode 100644 efile_app/efile/tests/test_party_type_persistence.py diff --git a/efile_app/efile/services/efsp_errors.py b/efile_app/efile/services/efsp_errors.py new file mode 100644 index 0000000..9e36c6f --- /dev/null +++ b/efile_app/efile/services/efsp_errors.py @@ -0,0 +1,100 @@ +"""Turn an EFSP error response into one line worth showing a filer. + +The proxy answers a bad payload with a machine-readable description of which +fields are wrong rather than with an ``error`` string:: + + {"required_vars": [], + "optional_vars": [...], + "wrong_vars": [{"name": "al_court_bundle.elements[0].filing_type", + "description": "What filing type is this??", + "datatype": "choice", "currentVal": "", "choices": [...]}]} + +Reading only ``error`` out of that leaves the filer with "API returned status +400" and no way to act, while the response says exactly which document is +missing exactly which field. Shared by the fee quote and the submission so both +describe the same rejection the same way. +""" + +import json +import re + +# Field names the EFSP uses, in the words the UI uses for them. +_FIELD_LABELS = { + "document_type": "document type", + "efile_case_category": "case category", + "efile_case_subtype": "case subtype", + "efile_case_type": "case type", + "filing_component": "filing component", + "filing_parties": "filing parties", + "filing_type": "filing type", + "party_type": "party type", + "previous_case_id": "case number", + "tyler_payment_id": "payment account", +} + +# "al_court_bundle.elements[0].filing_type" -> document 1, field filing_type +_BUNDLE_FIELD = re.compile(r"^al_court_bundle\.elements\[(\d+)\]\.(.+)$") + +_MAX_RAW_BODY = 300 + + +def describe_efsp_error(response) -> str: + """Describe why the EFSP refused ``response``'s request. + + Never raises: a bad message is still better than a traceback on a path whose + only job is reporting someone else's failure. + """ + try: + body = response.json() + except (ValueError, json.JSONDecodeError): + text = (response.text or "").strip() + if text: + return f"the court's filing service returned status {response.status_code}: {text[:_MAX_RAW_BODY]}" + return f"the court's filing service returned status {response.status_code}" + + if not isinstance(body, dict): + return f"the court's filing service returned status {response.status_code}: {str(body)[:_MAX_RAW_BODY]}" + + problems = [ + *(_describe_var(var, missing=False) for var in _var_list(body, "wrong_vars")), + *(_describe_var(var, missing=True) for var in _var_list(body, "required_vars")), + ] + problems = [problem for problem in problems if problem] + if problems: + return "the court could not accept this filing: " + "; ".join(problems) + + # Some errors do arrive as a plain message. + error = body.get("error") or body.get("message") or body.get("detail") + if error: + message = str(error) + validation_errors = body.get("validation_errors") or body.get("errors") + if validation_errors: + message += f" - Validation errors: {validation_errors}" + return message + + return f"the court's filing service returned status {response.status_code}" + + +def _var_list(body, key): + value = body.get(key) + return [var for var in value if isinstance(var, dict)] if isinstance(value, list) else [] + + +def _describe_var(var, *, missing: bool) -> str: + name = str(var.get("name") or "").strip() + if not name: + return "" + + match = _BUNDLE_FIELD.match(name) + if match: + index, name = match.groups() + where = f" on document {int(index) + 1}" + else: + where = "" + + label = _FIELD_LABELS.get(name, name.replace("_", " ")) + current = str(var.get("currentVal") or "").strip() + + if missing or not current: + return f"no {label} was given{where}" + return f"{current!r} is not a {label} this court accepts{where}" diff --git a/efile_app/efile/tests/test_efsp_errors.py b/efile_app/efile/tests/test_efsp_errors.py new file mode 100644 index 0000000..de7ea33 --- /dev/null +++ b/efile_app/efile/tests/test_efsp_errors.py @@ -0,0 +1,138 @@ +"""Tests for turning an EFSP rejection into a message a filer can act on. + +The body in ``WRONG_FILING_TYPE`` is the real one the fee endpoint returned for a +document saved with no filing type, which reached the filer as "Filing +submission failed: API returned status 400". +""" + +import json + +import pytest + +from efile.services.efsp_errors import describe_efsp_error + +WRONG_FILING_TYPE = { + "required_vars": [], + "optional_vars": [ + { + "name": "efile_case_subtype", + "description": "subtype (not always present)", + "datatype": "text", + "currentVal": "", + "choices": [], + } + ], + "wrong_vars": [ + { + "name": "al_court_bundle.elements[0].filing_type", + "description": "What filing type is this??", + "datatype": "choice", + "currentVal": "", + "choices": ["60384", "60409", "123566"], + } + ], +} + + +class FakeResponse: + """Enough of requests.Response for the describer.""" + + def __init__(self, status_code, body=None, text=None): + self.status_code = status_code + self._body = body + self.text = text if text is not None else json.dumps(body) + + def json(self): + if self._body is None: + raise json.JSONDecodeError("Expecting value", self.text, 0) + return self._body + + +def test_wrong_var_names_the_field_and_the_document(): + message = describe_efsp_error(FakeResponse(400, WRONG_FILING_TYPE)) + + assert "filing type" in message + assert "document 1" in message + assert "400" not in message + + +def test_optional_vars_are_not_reported_as_problems(): + """The same body lists an empty optional field; saying so would only confuse.""" + assert "subtype" not in describe_efsp_error(FakeResponse(400, WRONG_FILING_TYPE)) + + +def test_a_wrong_value_is_quoted_rather_than_called_missing(): + body = {"wrong_vars": [{"name": "al_court_bundle.elements[1].filing_type", "currentVal": "99999"}]} + + message = describe_efsp_error(FakeResponse(400, body)) + + assert "99999" in message + assert "document 2" in message + + +def test_required_var_is_reported_as_missing(): + message = describe_efsp_error(FakeResponse(400, {"required_vars": [{"name": "efile_case_type"}]})) + + assert "no case type was given" in message + + +def test_several_problems_are_all_reported(): + body = { + "wrong_vars": [ + {"name": "al_court_bundle.elements[0].filing_type", "currentVal": ""}, + {"name": "al_court_bundle.elements[1].filing_component", "currentVal": ""}, + ] + } + + message = describe_efsp_error(FakeResponse(400, body)) + + assert "filing type" in message + assert "filing component" in message + + +def test_unmapped_field_name_is_still_readable(): + message = describe_efsp_error(FakeResponse(400, {"required_vars": [{"name": "lead_contact"}]})) + + assert "lead contact" in message + + +def test_plain_error_message_is_passed_through(): + body = {"error": "All required parties not covered by existing party types."} + + assert describe_efsp_error(FakeResponse(400, body)) == body["error"] + + +def test_validation_errors_are_appended_to_a_plain_message(): + body = {"error": "Rejected", "validation_errors": ["bad bundle"]} + + message = describe_efsp_error(FakeResponse(400, body)) + + assert "Rejected" in message + assert "bad bundle" in message + + +def test_non_json_body_falls_back_to_the_status_and_text(): + message = describe_efsp_error(FakeResponse(502, text="Bad Gateway")) + + assert "502" in message + assert "Bad Gateway" in message + + +def test_empty_body_reports_only_the_status(): + assert "503" in describe_efsp_error(FakeResponse(503, text="")) + + +def test_unrecognised_json_shape_reports_the_status(): + assert "418" in describe_efsp_error(FakeResponse(418, {"something": "else"})) + + +@pytest.mark.parametrize("body", [["a list"], "a string", 42]) +def test_non_object_json_does_not_raise(body): + assert describe_efsp_error(FakeResponse(400, body)) + + +def test_nameless_var_is_ignored_rather_than_described_as_blank(): + """A var with no name says nothing; the status line is more useful.""" + message = describe_efsp_error(FakeResponse(400, {"wrong_vars": [{"currentVal": ""}]})) + + assert "400" in message diff --git a/efile_app/efile/tests/test_party_type_persistence.py b/efile_app/efile/tests/test_party_type_persistence.py new file mode 100644 index 0000000..2f7d1ab --- /dev/null +++ b/efile_app/efile/tests/test_party_type_persistence.py @@ -0,0 +1,96 @@ +"""The filer's own party type must survive the pages that guess at it. + +`/api/get-party-types/` picks a likely party type from the case type's *name* +("civil" -> plaintiff) and stores it on the draft. It is a GET, called on page +load, including from the review page -- so before this was guarded it replaced a +filer who had chosen Defendant/Respondent with the guessed Plaintiff/Petitioner. +Both sides then held the same party type, and the court rejected the filing for a +required party that no longer had anyone in it. +""" + +import pytest +from django.urls import reverse + +from efile.models import FilingDraft +from efile.services.current_drafts import CURRENT_DRAFT_SESSION_KEY +from efile.services.drafts import read_case_data, write_case_data + +PLAINTIFF = {"code": "20646", "name": "Plaintiff/Petitioner", "isrequired": True} +DEFENDANT = {"code": "20641", "name": "Defendant/Respondent", "isrequired": True} + + +class _PartyTypesResponse: + status_code = 200 + + @staticmethod + def json(): + return [PLAINTIFF, DEFENDANT] + + +@pytest.fixture +def draft_session(client, django_user_model): + user = django_user_model.objects.create_user(username="party-type-user", tyler_jurisdiction="illinois") + draft = FilingDraft.objects.create(user=user, jurisdiction="illinois") + client.force_login(user) + session = client.session + session[CURRENT_DRAFT_SESSION_KEY] = draft.pk + session["jurisdiction"] = "illinois" + session["auth_tokens"] = {"TYLER-TOKEN-ILLINOIS": "token"} + session.save() + return draft + + +def _fetch_party_types(client, monkeypatch): + monkeypatch.setattr( + "efile.api.suffolk_api_views.requests.get", + lambda *args, **kwargs: _PartyTypesResponse(), + ) + return client.get( + reverse("api:get_party_types"), + {"jurisdiction": "illinois", "court": "adams", "case_type": "76015", "existing_case": "no"}, + ) + + +@pytest.mark.django_db +def test_a_chosen_party_type_is_not_replaced_by_the_guess(client, draft_session, monkeypatch): + """The exact regression: self as Defendant, guess says Plaintiff.""" + write_case_data(draft_session, {"court": "adams", "case_type": "76015", "party_type": DEFENDANT["code"]}) + + response = _fetch_party_types(client, monkeypatch) + + assert response.status_code == 200 + assert read_case_data(draft_session)["determined_party_type"] == DEFENDANT["code"] + + +@pytest.mark.django_db +def test_the_response_reports_the_stored_party_type_not_the_guess(client, draft_session, monkeypatch): + """Otherwise the page shows one party type while the draft files another.""" + write_case_data(draft_session, {"court": "adams", "case_type": "76015", "party_type": DEFENDANT["code"]}) + + response = _fetch_party_types(client, monkeypatch) + + assert response.json()["selected_party_type"] == DEFENDANT["code"] + + +@pytest.mark.django_db +def test_an_empty_draft_is_still_seeded_with_the_guess(client, draft_session, monkeypatch): + """The guess remains a useful default -- it just stops being an override.""" + write_case_data(draft_session, {"court": "adams", "case_type": "76015"}) + + response = _fetch_party_types(client, monkeypatch) + + assert response.json()["selected_party_type"] == PLAINTIFF["code"] + assert read_case_data(draft_session)["determined_party_type"] == PLAINTIFF["code"] + + +@pytest.mark.django_db +def test_repeated_page_loads_do_not_drift(client, draft_session, monkeypatch): + """Seed once, then hold: the review page reloading must not change the filing.""" + write_case_data(draft_session, {"court": "adams", "case_type": "76015"}) + + _fetch_party_types(client, monkeypatch) + write_case_data(draft_session, {"party_type": DEFENDANT["code"]}) # the filer corrects it + _fetch_party_types(client, monkeypatch) + _fetch_party_types(client, monkeypatch) + + assert read_case_data(draft_session)["determined_party_type"] == DEFENDANT["code"] From 504ea37932656ef0b5217609d2918fc69569afe6 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Thu, 23 Jul 2026 10:20:32 -0400 Subject: [PATCH 45/47] Formatting w/ ruff and js-beautify --- .../efile/static/js/cascading-dropdowns.js | 2 +- efile_app/efile/static/js/filing-payload.js | 6 +- efile_app/efile/static/js/payment.js | 2 +- efile_app/efile/static/js/review.js | 2 +- .../efile/static/js/upload-handler-first.js | 6 +- efile_app/efile/static/js/upload-handler.js | 2 +- efile_app/efile/tests/test_llms.py | 2 + efile_app/js-tests/api-utils.test.js | 30 +++++-- .../js-tests/cascading-dropdowns.test.js | 2 +- efile_app/js-tests/filing-payload.test.js | 89 +++++++++++++++---- 10 files changed, 111 insertions(+), 32 deletions(-) diff --git a/efile_app/efile/static/js/cascading-dropdowns.js b/efile_app/efile/static/js/cascading-dropdowns.js index 23eb0c5..64f1c40 100644 --- a/efile_app/efile/static/js/cascading-dropdowns.js +++ b/efile_app/efile/static/js/cascading-dropdowns.js @@ -1028,4 +1028,4 @@ if (typeof module !== "undefined" && module.exports) { module.exports = CascadingDropdowns; } else { window.CascadingDropdowns = CascadingDropdowns; -} +} \ No newline at end of file diff --git a/efile_app/efile/static/js/filing-payload.js b/efile_app/efile/static/js/filing-payload.js index acf70d5..2b81443 100644 --- a/efile_app/efile/static/js/filing-payload.js +++ b/efile_app/efile/static/js/filing-payload.js @@ -181,8 +181,8 @@ const FilingPayload = { if (uploadData?.files?.supporting?.length > 0) { uploadData.files.supporting.forEach((doc, index) => { const config = uploadData.supporting_documents?.[index] || {}; - const filingComponent = componentCode(config.filing_component) - || componentCode(doc.filing_component); + const filingComponent = componentCode(config.filing_component) || + componentCode(doc.filing_component); const bundle = this.createDocumentBundle( doc, config.filing_type || caseData.filing_type_id, @@ -267,4 +267,4 @@ if (typeof module !== "undefined" && module.exports) { module.exports = FilingPayload; } else { window.FilingPayload = FilingPayload; -} +} \ No newline at end of file diff --git a/efile_app/efile/static/js/payment.js b/efile_app/efile/static/js/payment.js index d081498..068ecdf 100644 --- a/efile_app/efile/static/js/payment.js +++ b/efile_app/efile/static/js/payment.js @@ -406,4 +406,4 @@ window.PaymentHandler = PaymentHandler; window.Navigation = Navigation; // Initialize app when DOM is ready -document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); +document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); \ No newline at end of file diff --git a/efile_app/efile/static/js/review.js b/efile_app/efile/static/js/review.js index 32dec19..24d6938 100644 --- a/efile_app/efile/static/js/review.js +++ b/efile_app/efile/static/js/review.js @@ -584,4 +584,4 @@ window.queryFees = FilingHandler.queryFees.bind(FilingHandler); window.Navigation = Navigation; // Initialize app when DOM is ready -document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); +document.addEventListener("DOMContentLoaded", () => ReviewApp.init()); \ No newline at end of file diff --git a/efile_app/efile/static/js/upload-handler-first.js b/efile_app/efile/static/js/upload-handler-first.js index 6ed47ff..23d08fc 100644 --- a/efile_app/efile/static/js/upload-handler-first.js +++ b/efile_app/efile/static/js/upload-handler-first.js @@ -77,7 +77,9 @@ class UploadHandler { s3_key: uploadedLead.key, }, }, - options: { lead: {} }, + options: { + lead: {} + }, }; } @@ -514,4 +516,4 @@ document.addEventListener('DOMContentLoaded', function() { } else { console.warn('UploadHandler already exists, skipping initialization'); } -}); +}); \ No newline at end of file diff --git a/efile_app/efile/static/js/upload-handler.js b/efile_app/efile/static/js/upload-handler.js index 24f422b..06a559f 100644 --- a/efile_app/efile/static/js/upload-handler.js +++ b/efile_app/efile/static/js/upload-handler.js @@ -1179,4 +1179,4 @@ document.addEventListener('DOMContentLoaded', function() { } else { console.warn('UploadHandler already exists, skipping initialization'); } -}); +}); \ No newline at end of file diff --git a/efile_app/efile/tests/test_llms.py b/efile_app/efile/tests/test_llms.py index 1779298..945d406 100644 --- a/efile_app/efile/tests/test_llms.py +++ b/efile_app/efile/tests/test_llms.py @@ -1,5 +1,7 @@ from unittest.mock import MagicMock, patch + from django.conf import settings + from efile.utils.llms import ( LlmError, chat_completion, diff --git a/efile_app/js-tests/api-utils.test.js b/efile_app/js-tests/api-utils.test.js index 17c3dd7..6f5964a 100644 --- a/efile_app/js-tests/api-utils.test.js +++ b/efile_app/js-tests/api-utils.test.js @@ -124,7 +124,11 @@ function stubFetch(response) { return client; } -function jsonResponse({ ok, status, body }) { +function jsonResponse({ + ok, + status, + body +}) { const response = { ok, status, @@ -138,7 +142,10 @@ test("a failed request surfaces the server's own error message", async () => { const client = stubFetch(jsonResponse({ ok: false, status: 400, - body: { success: false, error: "This case type requires a Plaintiff." }, + body: { + success: false, + error: "This case type requires a Plaintiff." + }, })); await assert.rejects( @@ -176,7 +183,14 @@ test("a successful response is returned unchanged", async () => { const client = stubFetch(jsonResponse({ ok: true, status: 200, - body: { success: true, api_response: { feesCalculationAmount: { value: 0 } } }, + body: { + success: true, + api_response: { + feesCalculationAmount: { + value: 0 + } + } + }, })); const result = await client.post("/api/payment-fees/", {}); @@ -195,11 +209,15 @@ test("fee quotes and submissions may raise the request timeout", async () => { let seenTimeout = null; client.makeRequest = async (endpoint, options) => { seenTimeout = options.timeout; - return { success: true }; + return { + success: true + }; }; - await client.post("/api/payment-fees/", {}, {}, { timeout: ApiUtils.FILING_TIMEOUT_MS }); + await client.post("/api/payment-fees/", {}, {}, { + timeout: ApiUtils.FILING_TIMEOUT_MS + }); assert.strictEqual(seenTimeout, ApiUtils.FILING_TIMEOUT_MS); assert.ok(ApiUtils.FILING_TIMEOUT_MS > 60000, "must outlast the server's own 60s timeout"); -}); +}); \ No newline at end of file diff --git a/efile_app/js-tests/cascading-dropdowns.test.js b/efile_app/js-tests/cascading-dropdowns.test.js index bbe0301..dfe84fb 100644 --- a/efile_app/js-tests/cascading-dropdowns.test.js +++ b/efile_app/js-tests/cascading-dropdowns.test.js @@ -47,4 +47,4 @@ test("failed upload guess requests fall back to an empty guess set", async () => await dropdowns.loadGuesses(); assert.deepEqual(dropdowns.guesses, {}); -}); +}); \ No newline at end of file diff --git a/efile_app/js-tests/filing-payload.test.js b/efile_app/js-tests/filing-payload.test.js index 1355e5d..498aaef 100644 --- a/efile_app/js-tests/filing-payload.test.js +++ b/efile_app/js-tests/filing-payload.test.js @@ -4,29 +4,49 @@ const assert = require("node:assert"); const FilingPayload = require("../efile/static/js/filing-payload.js"); // The module reads these page-level globals at call time. -global.Messages = { showError() {}, showSuccess() {} }; +global.Messages = { + showError() {}, + showSuccess() {} +}; global.gettext = (s) => s; -global.apiUtils = { getCurrentJurisdiction: () => "illinois" }; +global.apiUtils = { + getCurrentJurisdiction: () => "illinois" +}; /** A FilingHandler stand-in, mixed the same way the real pages mix it. */ function makeHandler() { - const handler = { setFeesState() {}, setSubmissionState() {} }; + const handler = { + setFeesState() {}, + setSubmissionState() {} + }; return Object.assign(handler, FilingPayload); } -const CASE_DATA = { filing_component: "999", filing_type_id: "27965", document_type: "dt" }; +const CASE_DATA = { + filing_component: "999", + filing_type_id: "27965", + document_type: "dt" +}; function bundlesFor(uploadData) { const handler = makeHandler(); - const efilingData = { al_court_bundle: [] }; + const efilingData = { + al_court_bundle: [] + }; handler.addCourtBundles(efilingData, uploadData, CASE_DATA, []); return efilingData.al_court_bundle; } test("supporting component given as a plain code string is used as-is", () => { const bundles = bundlesFor({ - files: { supporting: [{ name: "a.pdf" }] }, - supporting_documents: [{ filing_component: "332" }], + files: { + supporting: [{ + name: "a.pdf" + }] + }, + supporting_documents: [{ + filing_component: "332" + }], }); assert.strictEqual(bundles[0].filing_component, "332"); @@ -34,8 +54,17 @@ test("supporting component given as a plain code string is used as-is", () => { test("supporting component given as an {id, name} object is flattened to its id", () => { const bundles = bundlesFor({ - files: { supporting: [{ name: "a.pdf" }] }, - supporting_documents: [{ filing_component: { id: "332", name: "Attachments" } }], + files: { + supporting: [{ + name: "a.pdf" + }] + }, + supporting_documents: [{ + filing_component: { + id: "332", + name: "Attachments" + } + }], }); assert.strictEqual(bundles[0].filing_component, "332"); @@ -43,7 +72,15 @@ test("supporting component given as an {id, name} object is flattened to its id" test("component stored on the file record is used when the config has none", () => { const bundles = bundlesFor({ - files: { supporting: [{ name: "a.pdf", filing_component: { id: "332", name: "Attachments" } }] }, + files: { + supporting: [{ + name: "a.pdf", + filing_component: { + id: "332", + name: "Attachments" + } + }] + }, supporting_documents: [{}], }); @@ -53,8 +90,15 @@ test("component stored on the file record is used when the config has none", () test("a null component falls back instead of throwing on .id", () => { // typeof null === "object" -- the previous ternary crashed here. const bundles = bundlesFor({ - files: { supporting: [{ name: "a.pdf", filing_component: null }] }, - supporting_documents: [{ filing_component: null }], + files: { + supporting: [{ + name: "a.pdf", + filing_component: null + }] + }, + supporting_documents: [{ + filing_component: null + }], }); assert.strictEqual(bundles[0].filing_component, CASE_DATA.filing_component); @@ -62,7 +106,11 @@ test("a null component falls back instead of throwing on .id", () => { test("with no component anywhere, the case default is used and never the label 'supporting'", () => { const bundles = bundlesFor({ - files: { supporting: [{ name: "a.pdf" }] }, + files: { + supporting: [{ + name: "a.pdf" + }] + }, supporting_documents: [{}], }); @@ -72,9 +120,18 @@ test("with no component anywhere, the case default is used and never the label ' test("lead and supporting documents both land in the bundle", () => { const bundles = bundlesFor({ - files: { lead: { name: "lead.pdf" }, supporting: [{ name: "a.pdf" }] }, + files: { + lead: { + name: "lead.pdf" + }, + supporting: [{ + name: "a.pdf" + }] + }, lead_filing_component: "331", - supporting_documents: [{ filing_component: "332" }], + supporting_documents: [{ + filing_component: "332" + }], }); assert.strictEqual(bundles.length, 2); @@ -89,4 +146,4 @@ test("the same module object serves both pages, so payloads cannot drift", () => assert.strictEqual(paymentHandler.buildEFilingData, reviewHandler.buildEFilingData); assert.strictEqual(paymentHandler.addCourtBundles, reviewHandler.addCourtBundles); assert.strictEqual(paymentHandler.createDocumentBundle, reviewHandler.createDocumentBundle); -}); +}); \ No newline at end of file From d72497a15727cf998f0fe0efe4b04d1da767d636 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Thu, 23 Jul 2026 10:24:27 -0400 Subject: [PATCH 46/47] More complete pre-commit config --- .pre-commit-config.yaml | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43fae6d..41a24c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,49 @@ repos: # Ty: Rust-based Python type checker (configured in pyproject.toml under [tool.ty]) - repo: local hooks: + # Keep the non-Python assets in sync with the formatters used by CI. + - id: djlint-format + name: djlint (format) + entry: bash -c + language: system + files: ^efile_app/.*\.html$ + args: + - | + files=() + for file in "$@"; do + files+=("${file#efile_app/}") + done + (cd efile_app && uv run djlint --reformat "${files[@]}") + - -- + + - id: css-beautify + name: css-beautify (format) + entry: bash -c + language: system + files: ^efile_app/.*\.css$ + args: + - | + files=() + for file in "$@"; do + files+=("${file#efile_app/}") + done + (cd efile_app && uv run css-beautify -r "${files[@]}") + - -- + + - id: js-beautify + name: js-beautify (format) + entry: bash -c + language: system + files: ^efile_app/.*\.js$ + args: + - | + files=() + for file in "$@"; do + files+=("${file#efile_app/}") + done + (cd efile_app && uv run js-beautify -r "${files[@]}") + - -- + - id: ty name: ty (type check) entry: bash -lc From 1a7bc0cad9c5bbfd3cabaf075305a4553532d897 Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Thu, 23 Jul 2026 10:42:46 -0400 Subject: [PATCH 47/47] use stable error codes per review feedback --- efile_app/efile/services/submission_errors.py | 26 ++++++++++ efile_app/efile/tests/test_durable_drafts.py | 8 ++- efile_app/efile/views/session_api.py | 49 ++++++++++++++++--- efile_app/efile/views/submission.py | 23 +-------- 4 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 efile_app/efile/services/submission_errors.py diff --git a/efile_app/efile/services/submission_errors.py b/efile_app/efile/services/submission_errors.py new file mode 100644 index 0000000..d9a0121 --- /dev/null +++ b/efile_app/efile/services/submission_errors.py @@ -0,0 +1,26 @@ +"""Stable error codes used by the final submission flow.""" + + +class SubmissionErrorCode: + """Machine-readable codes for errors returned before filing submission.""" + + CONFIRMATION_REQUIRED = "submission_confirmation_required" + CASE_DATA_MISSING = "submission_case_data_missing" + UPLOAD_DATA_MISSING = "submission_upload_data_missing" + EFILE_DATA_MISSING = "submission_efile_data_missing" + EFILE_DATA_INVALID = "submission_efile_data_invalid" + COURT_ID_MISSING = "submission_court_id_missing" + PAYLOAD_VALIDATION_FAILED = "submission_payload_validation_failed" + + +PRE_SUBMIT_ERROR_CODES = frozenset( + { + SubmissionErrorCode.CONFIRMATION_REQUIRED, + SubmissionErrorCode.CASE_DATA_MISSING, + SubmissionErrorCode.UPLOAD_DATA_MISSING, + SubmissionErrorCode.EFILE_DATA_MISSING, + SubmissionErrorCode.EFILE_DATA_INVALID, + SubmissionErrorCode.COURT_ID_MISSING, + SubmissionErrorCode.PAYLOAD_VALIDATION_FAILED, + } +) diff --git a/efile_app/efile/tests/test_durable_drafts.py b/efile_app/efile/tests/test_durable_drafts.py index f818871..2c70b26 100644 --- a/efile_app/efile/tests/test_durable_drafts.py +++ b/efile_app/efile/tests/test_durable_drafts.py @@ -668,11 +668,8 @@ def test_partial_case_update_does_not_clear_omitted_fields(django_user_model): def test_payload_validation_failure_releases_draft_for_a_corrected_retry(client, django_user_model, monkeypatch): """A pre-flight rejection never reached the court, so the filer may fix and retry. - The submission wrapper decides between "release to DRAFT" and "park in ERROR" - largely by matching error text, and ERROR is not claimable. A validation - rejection landing in ERROR would lock a filer out of their own filing over a - party type they could have corrected in a minute, so the response carries an - explicit pre_submit marker instead of relying on its wording. + The response carries a stable error code so changing the human-readable + validation message cannot accidentally park the draft in ERROR. """ def reject(*_args, **_kwargs): @@ -695,6 +692,7 @@ def unreachable_post(*_args, **_kwargs): ) assert response.status_code == 400 + assert response.json()["error_code"] == "submission_payload_validation_failed" assert "requires a party of every required type" in response.json()["error"] draft.refresh_from_db() assert draft.status == FilingDraft.Status.DRAFT diff --git a/efile_app/efile/views/session_api.py b/efile_app/efile/views/session_api.py index e31e2d4..32fa11d 100644 --- a/efile_app/efile/views/session_api.py +++ b/efile_app/efile/views/session_api.py @@ -11,6 +11,7 @@ from ..services.current_drafts import get_current_draft from ..services.efsp_errors import describe_efsp_error from ..services.efsp_payload import PayloadValidationError, prepare_efile_payload +from ..services.submission_errors import SubmissionErrorCode from ..utils.case_data_utils import get_case_data, get_upload_data, update_case_data, update_upload_data from ..utils.llms import LlmError, extract_fields_from_file from ..utils.proxy_connection import get_party_type_code_from_api @@ -243,7 +244,14 @@ def submit_final_filing(request): data = json.loads(request.body) if not data.get("confirm_submission"): - return JsonResponse({"success": False, "error": "Submission confirmation is required"}, status=400) + return JsonResponse( + { + "success": False, + "error_code": SubmissionErrorCode.CONFIRMATION_REQUIRED, + "error": "Submission confirmation is required", + }, + status=400, + ) # Read the filing state from the current durable draft case_data = get_case_data(request) @@ -260,6 +268,7 @@ def submit_final_filing(request): return JsonResponse( { "success": False, + "error_code": SubmissionErrorCode.CASE_DATA_MISSING, "error": "No case data found in session. Please go back and resubmit your case information.", "debug_info": "Session case_data is empty", }, @@ -270,6 +279,7 @@ def submit_final_filing(request): return JsonResponse( { "success": False, + "error_code": SubmissionErrorCode.UPLOAD_DATA_MISSING, "error": "No upload data found in session. Please go back and resubmit your documents.", "debug_info": f"Upload data: {upload_data}", }, @@ -279,7 +289,14 @@ def submit_final_filing(request): # Extract efile_data from the request efile_data = data.get("efile_data", {}) if not efile_data: - return JsonResponse({"success": False, "error": "No efile data provided in request"}, status=400) + return JsonResponse( + { + "success": False, + "error_code": SubmissionErrorCode.EFILE_DATA_MISSING, + "error": "No efile data provided in request", + }, + status=400, + ) # Log the complete request data for debugging logger.debug("Complete request data received") @@ -293,23 +310,39 @@ def submit_final_filing(request): missing_fields = [field for field in required_fields if field not in efile_data] if missing_fields: return JsonResponse( - {"success": False, "error": f"Missing required fields in efile_data: {missing_fields}"}, status=400 + { + "success": False, + "error_code": SubmissionErrorCode.EFILE_DATA_INVALID, + "error": f"Missing required fields in efile_data: {missing_fields}", + }, + status=400, ) # Get jurisdiction and court info from case data court_id = case_data.get("court", "") if not court_id: - return JsonResponse({"success": False, "error": "Court ID is required for filing submission"}, status=400) + return JsonResponse( + { + "success": False, + "error_code": SubmissionErrorCode.COURT_ID_MISSING, + "error": "Court ID is required for filing submission", + }, + status=400, + ) # Same fixups the fee quote applied, so the filing matches the quote. try: prepare_efile_payload(efile_data, jurisdiction_id, court_id) except PayloadValidationError as error: - # `pre_submit` tells the submission wrapper that nothing reached the - # filing API, so the draft is released for a corrected retry instead - # of being parked in ERROR. - return JsonResponse({"success": False, "error": str(error), "pre_submit": True}, status=400) + return JsonResponse( + { + "success": False, + "error_code": SubmissionErrorCode.PAYLOAD_VALIDATION_FAILED, + "error": str(error), + }, + status=400, + ) # Construct the Suffolk LIT Lab API endpoint api_url = f"{settings.EFSP_URL}/jurisdictions/{jurisdiction_id}/filingreview/courts/{court_id}/filings" diff --git a/efile_app/efile/views/submission.py b/efile_app/efile/views/submission.py index 939c794..4e447f8 100644 --- a/efile_app/efile/views/submission.py +++ b/efile_app/efile/views/submission.py @@ -8,6 +8,7 @@ from efile.models import FilingDraft from efile.services.current_drafts import clear_current_draft, get_current_draft +from efile.services.submission_errors import PRE_SUBMIT_ERROR_CODES from .session_api import submit_final_filing as legacy_submit_final_filing @@ -19,23 +20,6 @@ # manual review instead. _CLAIMABLE_STATUSES = (FilingDraft.Status.DRAFT,) -# Errors the session submit view returns *before* it calls the external filing -# API. Only these -- and a confirmed API rejection -- are safe to release back to -# DRAFT; any other failure may have left a filing submitted. -# -# Matching on message text is why new pre-call rejections should set -# `"pre_submit": True` on their response instead of being added here: a reworded -# message would otherwise silently start parking drafts in ERROR, locking the -# filer out of a retry for something that never reached the court. -_PRE_SUBMIT_ERROR_PREFIXES = ( - "Submission confirmation is required", - "No case data found", - "No upload data found", - "No efile data provided", - "Missing required fields in efile_data", - "Court ID is required", -) - def _claim_for_submission(draft: FilingDraft) -> bool: """Atomically move a DRAFT into SUBMITTING. @@ -73,10 +57,7 @@ def _failed_before_external_call(payload: dict) -> bool: """True only when the submit view rejected the request before calling the API.""" if "api_status_code" in payload: return False - if payload.get("pre_submit") is True: - return True - error = payload.get("error") - return isinstance(error, str) and error.startswith(_PRE_SUBMIT_ERROR_PREFIXES) + return payload.get("error_code") in PRE_SUBMIT_ERROR_CODES def _confirmed_api_rejection(payload: dict) -> bool: