diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/lending-poc/.dockerignore b/lending-poc/.dockerignore new file mode 100644 index 0000000..02dfa58 --- /dev/null +++ b/lending-poc/.dockerignore @@ -0,0 +1,14 @@ +.git +.venv +**/.venv +**/.venv-windows +**/surya-env +**/node_modules +**/__pycache__ +document_processing +field_mapping_poc +gateway +frontend +scripts +docs +*.log diff --git a/lending-poc/.env.example b/lending-poc/.env.example new file mode 100644 index 0000000..5419649 --- /dev/null +++ b/lending-poc/.env.example @@ -0,0 +1,18 @@ +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=lending_poc +POSTGRES_HOST_PORT=55439 +DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:55439/lending_poc +ENCRYPTION_KEY=changeme-generate-a-base64-fernet-key +DEBUG=true + +# Selects the surya-inference/ocr variant docker-compose.yml runs: "cpu" +# (default, always works) or "gpu" (requires an NVIDIA GPU + NVIDIA +# Container Toolkit / WSL GPU passthrough on the host). +COMPOSE_PROFILES=cpu + +# Model used by the translation and field_mapping services via Ollama. +# Must be pulled into the ollama container first: +# docker compose exec ollama ollama pull +OLLAMA_MODEL=gemma4:e4b-it-qat +# OLLAMA_HOST=http://ollama:11434 diff --git a/lending-poc/.gitignore b/lending-poc/.gitignore index a9f203b..e159458 100644 --- a/lending-poc/.gitignore +++ b/lending-poc/.gitignore @@ -3,10 +3,15 @@ __pycache__/ *.pyo .venv/ .env +**/.env *.egg-info/ dist/ build/ .mypy_cache/ .pytest_cache/ .ruff_cache/ -venv/ \ No newline at end of file +venv/ +**/.venv/ +.venv-windows/ +local.env +*.log \ No newline at end of file diff --git a/lending-poc/Database_setup.md b/lending-poc/Database_setup.md index 35d443e..d8304a3 100644 --- a/lending-poc/Database_setup.md +++ b/lending-poc/Database_setup.md @@ -20,7 +20,7 @@ pip install -e ".[dev]" Create a `.env` file in the project root: ```bash -DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:55432/lending_poc +DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:55439/lending_poc ENCRYPTION_KEY=<32-byte base64 key> DEBUG=true ``` @@ -39,7 +39,7 @@ python3 -c "import base64, os; print(base64.b64encode(os.urandom(32)).decode())" docker compose up -d db ``` -This starts Postgres with pgvector on host port `55432` (mapped from container port `5432`), and waits until it reports healthy. +This starts Postgres with pgvector on host port `55439` (mapped from container port `5432`), and waits until it reports healthy. ## 4. Run database migrations diff --git a/lending-poc/README.md b/lending-poc/README.md new file mode 100644 index 0000000..88e8ac8 --- /dev/null +++ b/lending-poc/README.md @@ -0,0 +1,140 @@ +# Lending POC + +Lending POC — FastAPI backend + PostgreSQL (pgvector), plus a document +processing pipeline (OCR, translation, field mapping) fronted by a gateway, +and a React frontend. This guide covers running the **entire stack in +Docker**. + +For running the `app` service natively against a containerized DB only +(e.g. for backend development with hot-reload outside Docker), see +[Database_setup.md](Database_setup.md). + +## Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) and Docker Compose v2 + (`docker compose version`) +- Git +- **Optional, for GPU acceleration**: an NVIDIA GPU, the + [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html), + and (on Windows) WSL2 with GPU passthrough enabled + +## 1. Clone and configure environment + +```bash +git clone +cd lending-poc +cp .env.example .env +``` + +Generate a real `ENCRYPTION_KEY` — the app uses it to encrypt sensitive +database fields, and the placeholder value in `.env.example` is not a valid +key: + +```bash +python3 -c "import base64, os; print(base64.b64encode(os.urandom(32)).decode())" +``` + +Paste the result into `ENCRYPTION_KEY=` in `.env`. + +Leave `COMPOSE_PROFILES=cpu` as-is unless you have a working GPU setup — +see [Using a GPU](#using-a-gpu) below. + +## 2. Start the stack + +```bash +docker compose up --build -d +``` + +This builds and starts every service: `db`, `app`, `ollama`, `field_mapping`, +`translation`, `surya-inference` + `ocr`, `gateway`, and `frontend`. + +The first run takes a while — Ollama and Surya both download models on +first use. Watch progress with: + +```bash +docker compose logs -f +``` + +## 3. Run database migrations + +The `app` container doesn't run migrations automatically on startup: + +```bash +docker compose exec app alembic -c db/alembic.ini upgrade head +``` + +## 4. Pull the Ollama model + +Needed by the `translation` and `field_mapping` services: + +```bash +docker compose exec ollama ollama pull gemma4:e4b-it-qat +``` + +(Substitute whatever `OLLAMA_MODEL` is set to in `.env` if you changed it. +If you're running the GPU profile, use `ollama-gpu` instead of `ollama` in +the command above.) + +## 5. Verify it's running + +| Service | URL | Notes | +|---|---|---| +| Frontend | http://localhost:5173 | Main UI | +| Gateway | http://localhost:8080 | Fronts OCR / translation / field-mapping | +| App (backend API) | http://localhost:8000 | Docs at `/docs`; health at `/health` | +| Postgres | localhost:55439 | pgvector-enabled | +| OCR | http://localhost:8010 | Not normally called directly | +| Translation | http://localhost:8001 | Not normally called directly | +| Field mapping | http://localhost:8002 | Not normally called directly | +| Surya inference | http://localhost:8500 | OCR's inference backend | + +There are effectively two subsystems sharing this compose file: the +`app` + `db` lending backend, and a separate OCR/translation/field-mapping +pipeline fronted by `gateway`. The frontend talks to the gateway for +document processing and to the app for everything else. + +## Using a GPU + +`surya-inference`/`ocr` and `ollama` each come in a CPU and a GPU variant, +selected by `COMPOSE_PROFILES` in `.env`: + +- `COMPOSE_PROFILES=cpu` (default) — always works, no GPU required. +- `COMPOSE_PROFILES=gpu` — requires an NVIDIA GPU on the host plus the + NVIDIA Container Toolkit (and, on Windows, WSL2 GPU passthrough). + +To switch: + +```bash +# in .env +COMPOSE_PROFILES=gpu +``` + +```bash +docker compose up --build -d +``` + +Both GPU containers detect GPU access at startup and fall back to CPU +automatically if it isn't actually usable — but `docker compose up` will +fail to create the containers at all if the toolkit isn't installed, +since the GPU device reservation can't be satisfied. + +Ollama's own image auto-detects CUDA at runtime with no separate build, so +switching the profile is enough for it; `surya-inference`/`ocr` are built +from CUDA base images specifically for the `gpu` profile (see +[docker-compose.yml](docker-compose.yml) and +[document_processing/ocr/README.md](document_processing/ocr/README.md) +for details). + +## Stopping and cleanup + +```bash +docker compose down +``` + +Add `-v` to also delete the named volumes (`pgdata`, `ollama_models`, +`surya_models`) — this wipes the database and downloaded models, so only +do this if you want a clean slate: + +```bash +docker compose down -v +``` \ No newline at end of file diff --git a/lending-poc/app/config.py b/lending-poc/app/config.py index 90c1779..ce7fab6 100644 --- a/lending-poc/app/config.py +++ b/lending-poc/app/config.py @@ -4,7 +4,7 @@ class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") APP_NAME: str = "lending-poc" APP_VERSION: str = "0.1.0" diff --git a/lending-poc/docker-compose.yml b/lending-poc/docker-compose.yml index 38239e6..965aaea 100644 --- a/lending-poc/docker-compose.yml +++ b/lending-poc/docker-compose.yml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} POSTGRES_DB: ${POSTGRES_DB:-lending_poc} ports: - - "${POSTGRES_HOST_PORT:-55432}:5432" + - "${POSTGRES_HOST_PORT:-55439}:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: @@ -32,5 +32,194 @@ services: db: condition: service_healthy + # Ollama's own image already auto-detects CUDA at runtime and falls back + # to CPU on its own — no custom build needed here, just the same cpu/gpu + # profile split used for surya-inference/ocr below, so GPU access is only + # requested when COMPOSE_PROFILES=gpu. + ollama: + &ollama + image: ollama/ollama:latest + profiles: ["cpu"] + ports: + - "11434:11434" + environment: + # Keep the model resident once loaded instead of unloading after the + # default 5m idle timeout — cold-loading this model takes 2+ minutes, + # which otherwise blows past callers' request timeouts on every call + # after a short idle gap. + OLLAMA_KEEP_ALIVE: -1 + volumes: + - ollama_models:/root/.ollama + + ollama-gpu: + <<: *ollama + profiles: ["gpu"] + networks: + default: + aliases: + - ollama + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + field_mapping: + build: ./field_mapping_poc + command: uvicorn api:app --host 0.0.0.0 --port 8002 --reload + ports: + - "8002:8002" + environment: + OLLAMA_HOST: ${OLLAMA_HOST:-http://ollama:11434} + OLLAMA_MODEL: ${OLLAMA_MODEL:-gemma4:e4b-it-qat} + # Cold model load alone can take 2+ minutes; give the first call after + # an idle gap enough headroom instead of timing out mid-load. + OLLAMA_TIMEOUT_SECONDS: ${OLLAMA_TIMEOUT_SECONDS:-300} + volumes: + - ./field_mapping_poc:/app + depends_on: + ollama: + condition: service_started + required: false + ollama-gpu: + condition: service_started + required: false + + translation: + build: ./document_processing/translation + command: uvicorn api_server:app --host 0.0.0.0 --port 8001 --reload + ports: + - "8001:8001" + environment: + OLLAMA_HOST: ${OLLAMA_HOST:-http://ollama:11434} + OLLAMA_MODEL: ${OLLAMA_MODEL:-gemma4:e4b-it-qat} + volumes: + - ./document_processing/translation:/app + depends_on: + ollama: + condition: service_started + required: false + ollama-gpu: + condition: service_started + required: false + + # surya-inference and ocr each come in a "cpu" and "gpu" variant, selected + # via COMPOSE_PROFILES in .env (defaults to "cpu" so plain `docker compose + # up` always works). Both variants of a pair share a network alias so + # downstream services (SURYA_INFERENCE_URL, OCR_BASE_URL) don't need to + # know which one is active. See document_processing/ocr/README.md for GPU + # prerequisites (NVIDIA Container Toolkit / WSL GPU passthrough). + surya-inference: + &surya-inference + build: ./surya-inference + profiles: ["cpu"] + ports: + - "8500:8000" + environment: + SURYA_INFERENCE_PARALLEL: ${SURYA_INFERENCE_PARALLEL:-4} + SURYA_INFERENCE_CTX_SIZE: ${SURYA_INFERENCE_CTX_SIZE:-49152} + volumes: + - surya_models:/models + + surya-inference-gpu: + <<: *surya-inference + profiles: ["gpu"] + build: + context: ./surya-inference + args: + BASE_IMAGE: nvidia/cuda:12.4.1-devel-ubuntu22.04 + RUNTIME_IMAGE: nvidia/cuda:12.4.1-runtime-ubuntu22.04 + GGML_CUDA: "ON" + networks: + default: + aliases: + - surya-inference + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + ocr: + &ocr + build: ./document_processing/ocr + profiles: ["cpu"] + command: uvicorn api:app --host 0.0.0.0 --port 8010 --reload + ports: + - "8010:8010" + environment: + SURYA_INFERENCE_URL: http://surya-inference:8000/v1 + SURYA_INFERENCE_AUTOSTART: "false" + volumes: + - ./document_processing/ocr:/app + depends_on: + surya-inference: + condition: service_started + required: false + surya-inference-gpu: + condition: service_started + required: false + + ocr-gpu: + <<: *ocr + profiles: ["gpu"] + build: + context: ./document_processing/ocr + args: + GPU: "1" + networks: + default: + aliases: + - ocr + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + gateway: + build: ./gateway + command: uvicorn main:app --host 0.0.0.0 --port 8000 --reload + ports: + - "8080:8000" + environment: + FIELD_MAPPING_BASE_URL: http://field_mapping:8002 + OCR_BASE_URL: http://ocr:8010 + TRANSLATION_BASE_URL: http://translation:8001 + volumes: + - ./gateway:/app + depends_on: + field_mapping: + condition: service_started + ocr: + condition: service_started + required: false + ocr-gpu: + condition: service_started + required: false + translation: + condition: service_started + + frontend: + build: ./frontend + ports: + - "5173:5173" + environment: + # Browser-facing: must be host-reachable (localhost + published port), + # not a Docker-internal service name. + VITE_API_BASE_URL: http://localhost:8080 + VITE_TRANSLATION_API_BASE_URL: http://localhost:8080 + volumes: + - ./frontend:/app + - /app/node_modules + volumes: pgdata: + ollama_models: + surya_models: diff --git a/lending-poc/document_processing/ocr/.dockerignore b/lending-poc/document_processing/ocr/.dockerignore new file mode 100644 index 0000000..3071a79 --- /dev/null +++ b/lending-poc/document_processing/ocr/.dockerignore @@ -0,0 +1,10 @@ +__pycache__ +*.pyc +.venv +.venv-windows +surya-env +.env +.git +extraction_output +*.stdout.log +*.stderr.log diff --git a/lending-poc/document_processing/ocr/Dockerfile b/lending-poc/document_processing/ocr/Dockerfile new file mode 100644 index 0000000..0c068b3 --- /dev/null +++ b/lending-poc/document_processing/ocr/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Surya's recognition/layout models run in-process here. The LLM half runs +# in the separate surya-inference container, reached via SURYA_INFERENCE_URL +# — this image does not need llama-server. +# +# CPU by default: pin the CPU-only torch wheel so the image doesn't pull +# CUDA deps it can't use. Paired with the "gpu" compose profile, build with +# --build-arg GPU=1 to skip the pin and let requirements.txt's surya-ocr +# pull the default (CUDA-enabled) torch wheel instead — PyTorch's GPU wheels +# bundle their own CUDA runtime libs, so no CUDA base image is needed here, +# only GPU device access from the container (which the compose profile +# grants) and a host NVIDIA driver. +ARG GPU=0 +RUN if [ "$GPU" = "0" ]; then \ + pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; \ + fi + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 8010 + +CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8010"] diff --git a/lending-poc/document_processing/ocr/README.md b/lending-poc/document_processing/ocr/README.md index 210ca46..c0a24b5 100644 --- a/lending-poc/document_processing/ocr/README.md +++ b/lending-poc/document_processing/ocr/README.md @@ -200,6 +200,28 @@ make status # Show project status ### Prerequisites - Python 3.11+ - macOS/Linux (Surya requires local inference backend) +- A Surya inference backend: on CPU-only WSL install `llama.cpp` so that + `llama-server` is on `PATH`; on an NVIDIA WSL setup configure Surya's vLLM + backend and Docker/GPU passthrough. The API now verifies this at startup, + before reporting `/health` as healthy. + +### Running via Docker Compose (GPU or CPU) + +The top-level `docker-compose.yml` runs this service (and its `surya-inference` +backend) in a CPU or GPU variant, picked by `COMPOSE_PROFILES` in `.env`: + +- `COMPOSE_PROFILES=cpu` (default) — always works, no GPU required. +- `COMPOSE_PROFILES=gpu` — requires an NVIDIA GPU on the host plus the + [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) + (and, on Windows, WSL2 GPU passthrough). Both containers detect GPU access + at startup and fall back to CPU automatically if it isn't actually there, + so setting the profile without a working toolkit degrades to CPU rather + than failing outright — but `docker compose up` itself will fail to create + the containers if the toolkit isn't installed at all, since the GPU device + reservation can't be satisfied. + +Switch profiles by editing `COMPOSE_PROFILES` in `lending-poc/.env`, then +`docker compose up --build`. ### Dependencies diff --git a/lending-poc/document_processing/ocr/api.py b/lending-poc/document_processing/ocr/api.py index bba47e4..4ea26e4 100644 --- a/lending-poc/document_processing/ocr/api.py +++ b/lending-poc/document_processing/ocr/api.py @@ -5,7 +5,7 @@ Built as a thin wrapper around the existing Extractor pipeline. Usage: - uvicorn api:app --host 0.0.0.0 --port 8000 --reload + uvicorn api:app --host 0.0.0.0 --port 8010 --reload Endpoints: POST /extract - Upload and process a document @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any, Dict import os +from contextlib import asynccontextmanager from fastapi import FastAPI, File, UploadFile, HTTPException from fastapi.concurrency import run_in_threadpool @@ -25,16 +26,40 @@ from extractor import Extractor, DEFAULT_ENGINE from extractor.loader import SUPPORTED_EXTENSIONS +# Initialize extractor (reused across requests for efficiency) +extractor = Extractor(engine=DEFAULT_ENGINE) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Start Surya without making the HTTP service unavailable during warm-up.""" + app.state.ocr_ready = False + app.state.ocr_error = None + + async def warm_up() -> None: + try: + await run_in_threadpool(extractor.engine.warm_up) + except Exception as exc: + # Keep the API available for diagnostics. /extract will return a + # useful 503 instead of holding an upload open while Surya retries + # a missing/misconfigured WSL inference runtime. + app.state.ocr_error = str(exc) + else: + app.state.ocr_ready = True + + warm_up_task = asyncio.create_task(warm_up()) + yield + warm_up_task.cancel() + + # Initialize FastAPI app app = FastAPI( title="OCR Text Extraction API", description="Upload documents (PDF, PNG, JPEG) for OCR text extraction using Surya", - version="1.0.0" + version="1.0.0", + lifespan=lifespan, ) -# Initialize extractor (reused across requests for efficiency) -extractor = Extractor(engine=DEFAULT_ENGINE) - # File size limit (50MB) MAX_FILE_SIZE = 50 * 1024 * 1024 @@ -43,13 +68,18 @@ _extract_lock = asyncio.Lock() @app.get("/health") -async def health_check() -> Dict[str, str]: - """Health check endpoint to verify API is running.""" - return { - "status": "healthy", +async def health_check() -> Dict[str, Any]: + """Health check that distinguishes an online API from ready OCR inference.""" + response: Dict[str, Any] = { + "status": "healthy" if app.state.ocr_ready else "initializing", "service": "OCR Text Extraction API", - "engine": DEFAULT_ENGINE + "engine": DEFAULT_ENGINE, + "ocr_ready": app.state.ocr_ready, } + if app.state.ocr_error: + response["status"] = "unhealthy" + response["ocr_error"] = app.state.ocr_error + return response @app.post("/extract") async def extract_text(file: UploadFile = File(...)) -> Dict[str, Any]: @@ -60,6 +90,14 @@ async def extract_text(file: UploadFile = File(...)) -> Dict[str, Any]: Returns: Extracted text, HTML representation, and metadata """ + # Never accept an upload when the OCR runtime is not available. Without + # this guard Surya can block the request for its full backend timeout. + if not app.state.ocr_ready: + detail = "OCR inference is still initializing" + if app.state.ocr_error: + detail = f"OCR inference is unavailable: {app.state.ocr_error}" + raise HTTPException(status_code=503, detail=detail) + # Validate file type if not file.filename: raise HTTPException(status_code=400, detail="No filename provided") @@ -126,7 +164,7 @@ async def extract_text(file: UploadFile = File(...)) -> Dict[str, Any]: finally: # Clean up temporary file - if temp_file and os.path.exists(temp_file_path): + if temp_file_path and os.path.exists(temp_file_path): try: os.unlink(temp_file_path) except OSError: @@ -152,6 +190,6 @@ async def root() -> Dict[str, Any]: uvicorn.run( "api:app", host="0.0.0.0", - port=8000, + port=8010, reload=True - ) \ No newline at end of file + ) diff --git a/lending-poc/document_processing/ocr/extraction_input/test-image.jpg b/lending-poc/document_processing/ocr/extraction_input/test-image.jpg new file mode 100644 index 0000000..84c5ec2 Binary files /dev/null and b/lending-poc/document_processing/ocr/extraction_input/test-image.jpg differ diff --git a/lending-poc/document_processing/ocr/extractor/engines/surya_engine.py b/lending-poc/document_processing/ocr/extractor/engines/surya_engine.py index 2e53db6..689af49 100644 --- a/lending-poc/document_processing/ocr/extractor/engines/surya_engine.py +++ b/lending-poc/document_processing/ocr/extractor/engines/surya_engine.py @@ -34,9 +34,18 @@ def __init__(self) -> None: def _ensure_ready(self) -> None: if self._manager is None: - self._manager = SuryaInferenceManager() # auto-spawns vllm or llama-server + self._manager = SuryaInferenceManager() + # Surya creates its inference manager lazily. Starting it here + # means an API startup check can fail fast when WSL is missing its + # backend (llama-server on CPU, or vLLM/Docker on CUDA), instead + # of making the first uploaded document appear to hang. + self._manager.start() self._recognizer = RecognitionPredictor(self._manager) + def warm_up(self) -> None: + """Start and validate Surya's inference backend without processing a file.""" + self._ensure_ready() + def run(self, images: List[Image.Image]) -> List[PageResult]: self._ensure_ready() raw_predictions = self._recognizer(images) diff --git a/lending-poc/document_processing/translation/.dockerignore b/lending-poc/document_processing/translation/.dockerignore new file mode 100644 index 0000000..677b168 --- /dev/null +++ b/lending-poc/document_processing/translation/.dockerignore @@ -0,0 +1,7 @@ +__pycache__ +*.pyc +.venv +.venv-windows +.env +.git +output diff --git a/lending-poc/document_processing/translation/Dockerfile b/lending-poc/document_processing/translation/Dockerfile new file mode 100644 index 0000000..819520c --- /dev/null +++ b/lending-poc/document_processing/translation/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 8001 + +CMD ["uvicorn", "api_server:app", "--host", "0.0.0.0", "--port", "8001"] diff --git a/lending-poc/document_processing/translation/translation_service/config.py b/lending-poc/document_processing/translation/translation_service/config.py index 79f938d..9bc7be7 100644 --- a/lending-poc/document_processing/translation/translation_service/config.py +++ b/lending-poc/document_processing/translation/translation_service/config.py @@ -8,6 +8,7 @@ Nothing outside this file needs to change for those operations. """ +import os from pathlib import Path # --------------------------------------------------------------------------- @@ -67,7 +68,7 @@ def get_kb_path(domain: str) -> Path: MODEL_ADAPTER = "ollama" # Model identifier passed to the chosen adapter. -MODEL_NAME = "gemma4:e4b" +MODEL_NAME = os.getenv("OLLAMA_MODEL", "gemma4:e4b-it-qat") # --------------------------------------------------------------------------- # Model options (adapter-specific — passed through as-is) diff --git a/lending-poc/field_mapping_poc/.dockerignore b/lending-poc/field_mapping_poc/.dockerignore new file mode 100644 index 0000000..d372f23 --- /dev/null +++ b/lending-poc/field_mapping_poc/.dockerignore @@ -0,0 +1,6 @@ +__pycache__ +*.pyc +.venv +.env +samples +.git diff --git a/lending-poc/field_mapping_poc/.env.example b/lending-poc/field_mapping_poc/.env.example new file mode 100644 index 0000000..12ae354 --- /dev/null +++ b/lending-poc/field_mapping_poc/.env.example @@ -0,0 +1,3 @@ +DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:55432/lending_poc +ENCRYPTION_KEY=changeme-generate-a-base64-fernet-key +DEBUG=true diff --git a/lending-poc/field_mapping_poc/Dockerfile b/lending-poc/field_mapping_poc/Dockerfile new file mode 100644 index 0000000..e98c885 --- /dev/null +++ b/lending-poc/field_mapping_poc/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 8002 + +CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8002"] diff --git a/lending-poc/field_mapping_poc/api.py b/lending-poc/field_mapping_poc/api.py index 29a7a01..d3e7b24 100644 --- a/lending-poc/field_mapping_poc/api.py +++ b/lending-poc/field_mapping_poc/api.py @@ -5,7 +5,7 @@ Built as a wrapper around the existing FieldMapper. Usage: - uvicorn api:app --host 0.0.0.0 --port 8000 --reload + uvicorn api:app --host 0.0.0.0 --port 8002 --reload Endpoints: POST /map - Map OCR text to the provided JSON schema @@ -109,6 +109,6 @@ async def root() -> Dict[str, Any]: uvicorn.run( "api:app", host="0.0.0.0", - port=8000, + port=8002, reload=True ) diff --git a/lending-poc/field_mapping_poc/config.py b/lending-poc/field_mapping_poc/config.py index cda02f2..7a19041 100644 --- a/lending-poc/field_mapping_poc/config.py +++ b/lending-poc/field_mapping_poc/config.py @@ -12,7 +12,7 @@ @dataclass(frozen=True) class OllamaConfig: host: str = os.getenv("OLLAMA_HOST", "http://localhost:11434") - model: str = os.getenv("OLLAMA_MODEL", "gemma4:e4b") + model: str = os.getenv("OLLAMA_MODEL", "gemma4:e4b-it-qat") temperature: float = float(os.getenv("OLLAMA_TEMPERATURE", "0.0")) num_ctx: int = int(os.getenv("OLLAMA_NUM_CTX", "8192")) request_timeout: int = int(os.getenv("OLLAMA_TIMEOUT_SECONDS", "120")) diff --git a/lending-poc/frontend/.dockerignore b/lending-poc/frontend/.dockerignore new file mode 100644 index 0000000..c071f14 --- /dev/null +++ b/lending-poc/frontend/.dockerignore @@ -0,0 +1,6 @@ +node_modules +dist +.env +.env.local +*.log +.git diff --git a/lending-poc/frontend/.env.example b/lending-poc/frontend/.env.example new file mode 100644 index 0000000..fe75629 --- /dev/null +++ b/lending-poc/frontend/.env.example @@ -0,0 +1,2 @@ +VITE_API_BASE_URL=http://localhost:8000 +VITE_TRANSLATION_API_BASE_URL=http://localhost:8000 diff --git a/lending-poc/frontend/Dockerfile b/lending-poc/frontend/Dockerfile new file mode 100644 index 0000000..0163475 --- /dev/null +++ b/lending-poc/frontend/Dockerfile @@ -0,0 +1,12 @@ +FROM node:22-slim + +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci + +COPY . . + +EXPOSE 5173 + +CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"] diff --git a/lending-poc/frontend/package-lock.json b/lending-poc/frontend/package-lock.json index 57e38c8..7ebcb70 100644 --- a/lending-poc/frontend/package-lock.json +++ b/lending-poc/frontend/package-lock.json @@ -626,9 +626,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -646,9 +643,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -666,9 +660,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -686,9 +677,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -706,9 +694,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -726,9 +711,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -955,9 +937,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -979,9 +958,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1003,9 +979,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1027,9 +1000,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1202,9 +1172,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1222,9 +1189,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1242,9 +1206,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1262,9 +1223,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2877,9 +2835,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -2901,9 +2856,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -2925,9 +2877,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -2949,9 +2898,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ diff --git a/lending-poc/frontend/src/api/extract.ts b/lending-poc/frontend/src/api/extract.ts index 10cbcc5..4c787cb 100644 --- a/lending-poc/frontend/src/api/extract.ts +++ b/lending-poc/frontend/src/api/extract.ts @@ -4,8 +4,9 @@ import { extractResponseSchema, type ExtractResponse } from '@/schemas/extract.s // OCR extraction is CPU-bound and can take well over the client's default // 30s timeout, especially for multi-page documents or several concurrent // uploads (the backend also serializes concurrent extractions, so later -// documents in a batch wait on earlier ones). 5 minutes gives real -// documents room to finish instead of erroring out mid-processing. +// documents in a batch wait on earlier ones). CPU-only Surya can take +// several minutes per handwritten page, so this must stay aligned with the +// gateway's OCR_REQUEST_TIMEOUT_SECONDS default. const EXTRACT_TIMEOUT_MS = 5 * 60 * 1000 /** diff --git a/lending-poc/frontend/src/config/env.ts b/lending-poc/frontend/src/config/env.ts index 322b5c3..19ad126 100644 --- a/lending-poc/frontend/src/config/env.ts +++ b/lending-poc/frontend/src/config/env.ts @@ -1,5 +1,6 @@ import { z } from 'zod' + const envSchema = z.object({ VITE_API_BASE_URL: z.string().url().or(z.string().startsWith('/')), VITE_TRANSLATION_API_BASE_URL: z.string().url().or(z.string().startsWith('/')), diff --git a/lending-poc/gateway/.dockerignore b/lending-poc/gateway/.dockerignore new file mode 100644 index 0000000..2c46cf5 --- /dev/null +++ b/lending-poc/gateway/.dockerignore @@ -0,0 +1,5 @@ +__pycache__ +*.pyc +.venv +.env +.git diff --git a/lending-poc/gateway/Dockerfile b/lending-poc/gateway/Dockerfile new file mode 100644 index 0000000..b3a66cf --- /dev/null +++ b/lending-poc/gateway/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 8000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/lending-poc/scripts/start-backend.sh b/lending-poc/scripts/start-backend.sh index 6b33855..645b480 100755 --- a/lending-poc/scripts/start-backend.sh +++ b/lending-poc/scripts/start-backend.sh @@ -10,7 +10,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" PYTHON_BIN="python3.12" -OLLAMA_MODEL="${OLLAMA_MODEL:-gemma4:e4b}" +OLLAMA_MODEL="${OLLAMA_MODEL:-gemma4:e4b-it-qat}" GATEWAY_PORT=8000 OCR_PORT=8010 diff --git a/lending-poc/surya-inference/Dockerfile b/lending-poc/surya-inference/Dockerfile new file mode 100644 index 0000000..100edbe --- /dev/null +++ b/lending-poc/surya-inference/Dockerfile @@ -0,0 +1,56 @@ +# Builds llama.cpp's `llama-server` from source and serves it standalone, so +# ocr-api can point SURYA_INFERENCE_URL here instead of spawning its own copy +# in-process (see surya/inference/backends/llamacpp.py in the ocr-api image +# for the equivalent in-process spawn logic this mirrors). +# +# CPU by default. For a CUDA-capable build (paired with the "gpu" compose +# profile), pass: +# --build-arg BASE_IMAGE=nvidia/cuda:12.4.1-devel-ubuntu22.04 +# --build-arg RUNTIME_IMAGE=nvidia/cuda:12.4.1-runtime-ubuntu22.04 +# --build-arg GGML_CUDA=ON +# entrypoint.sh still probes for a visible GPU at container start and falls +# back to -ngl 0 (CPU) if none is found, so this image works either way. +ARG BASE_IMAGE=python:3.12-slim +ARG RUNTIME_IMAGE=python:3.12-slim + +FROM ${BASE_IMAGE} AS build + +ARG GGML_CUDA=OFF + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --depth 1 https://github.com/ggml-org/llama.cpp /llama.cpp +WORKDIR /llama.cpp +RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_CUDA=${GGML_CUDA} -DLLAMA_CURL=OFF -DCMAKE_BUILD_TYPE=Release \ + && cmake --build build -j"$(nproc)" --target llama-server + +FROM ${RUNTIME_IMAGE} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +# llama-server is dynamically linked against the other .so files built +# alongside it (libllama-server-impl.so, libllama-common.so, libmtmd.so, ...), +# so the whole bin/ directory needs to come along, not just the executable. +COPY --from=build /llama.cpp/build/bin /opt/llama.cpp/bin +ENV LD_LIBRARY_PATH=/opt/llama.cpp/bin +RUN ln -s /opt/llama.cpp/bin/llama-server /usr/local/bin/llama-server + +ENV SURYA_GGUF_REPO=datalab-to/surya-ocr-2-gguf \ + SURYA_GGUF_MODEL_FILE=surya-2.gguf \ + SURYA_GGUF_MMPROJ_FILE=surya-2-mmproj.gguf \ + SURYA_MODEL_ALIAS=datalab-to/surya-ocr-2 \ + MODEL_DIR=/models \ + PORT=8000 + +RUN mkdir -p /models + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 8000 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/lending-poc/surya-inference/entrypoint.sh b/lending-poc/surya-inference/entrypoint.sh new file mode 100644 index 0000000..84a64c1 --- /dev/null +++ b/lending-poc/surya-inference/entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Downloads the same GGUF files Surya's own llamacpp backend fetches when it +# spawns its own server (surya/inference/backends/llamacpp.py: +# SURYA_GGUF_REPO / SURYA_GGUF_MODEL_FILE / SURYA_GGUF_MMPROJ_FILE), then +# starts llama-server with the equivalent flags so ocr-api can attach to it +# via SURYA_INFERENCE_URL instead of spawning its own. +set -e + +MODEL_PATH="${MODEL_DIR}/${SURYA_GGUF_MODEL_FILE}" +MMPROJ_PATH="${MODEL_DIR}/${SURYA_GGUF_MMPROJ_FILE}" + +if [ ! -f "$MODEL_PATH" ]; then + echo "Downloading ${SURYA_GGUF_MODEL_FILE} from ${SURYA_GGUF_REPO}..." + curl -fL -o "$MODEL_PATH" "https://huggingface.co/${SURYA_GGUF_REPO}/resolve/main/${SURYA_GGUF_MODEL_FILE}" +fi + +if [ ! -f "$MMPROJ_PATH" ]; then + echo "Downloading ${SURYA_GGUF_MMPROJ_FILE} from ${SURYA_GGUF_REPO}..." + curl -fL -o "$MMPROJ_PATH" "https://huggingface.co/${SURYA_GGUF_REPO}/resolve/main/${SURYA_GGUF_MMPROJ_FILE}" +fi + +# nvidia-smi only shows up here if the container was actually started with +# GPU access (via the "gpu" compose profile + NVIDIA Container Toolkit) - +# so this doubles as the CPU/GPU decision regardless of how the image was +# built, and fails safe to CPU if GPU access was requested but isn't there. +if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then + echo "GPU detected - offloading layers to GPU" + NGL="${SURYA_INFERENCE_NGL:-999}" +else + echo "No GPU detected - running on CPU" + NGL=0 +fi + +exec llama-server \ + -m "$MODEL_PATH" \ + --mmproj "$MMPROJ_PATH" \ + -ngl "$NGL" \ + --host 0.0.0.0 \ + --port "${PORT:-8000}" \ + --parallel "${SURYA_INFERENCE_PARALLEL:-4}" \ + --ctx-size "${SURYA_INFERENCE_CTX_SIZE:-49152}" \ + --alias "$SURYA_MODEL_ALIAS" \ + --jinja