+ The enterprise architecture JSON did not include any nodes to render.
+
+
+ );
+ }
+
+ return (
+
+ {/* Dot-grid background */}
+
+
+
+
+ {/* Header row: title + download buttons */}
+
+
+
+ {displayTitle}
+
+
+
+
+
+
+
+
+ {/* ── Poster canvas ─────────────────────────────────────────────────────
+ Single SVG with viewBox="0 0 1600 900" + preserveAspectRatio="xMidYMid meet".
+ The browser scales and centers the entire diagram automatically.
+ Node cards are rendered as so they keep HTML styling.
+ Edges are plain SVG paths drawn in the same coordinate space.
+ ─────────────────────────────────────────────────────────────────────── */}
+
- Convert any UI screenshot into
- production-ready HTML code
-
-
- Upload a design screenshot. The backend processes the image, generates semantic HTML
- incorporating Tailwind CSS, and verifies against visual diff metrics.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Drag and drop a screenshot here
-
or click to select a file · PNG, JPG, WEBP
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Pipeline Status
-
-
-
-
-
-
-
-
-
Extracting Design Tokens
-
-
-
-
-
-
-
-
-
-
-
-
-
Generating Component Structure
-
-
-
-
-
-
-
-
-
-
-
-
-
Running Visual Diff and Self-Correcting
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Pixel Match: —
-
-
Based on visual diff analysis
-
-
-
-
-
-
-
-
-
-
- Result
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/projects/template-project/.env.example b/projects/template-project/.env.example
deleted file mode 100644
index 0133b73..0000000
--- a/projects/template-project/.env.example
+++ /dev/null
@@ -1,35 +0,0 @@
-# =============================================================================
-# Environment Variables Template
-# =============================================================================
-#
-# IMPORTANT: This file is committed to version control.
-# DO NOT put real credentials here.
-#
-# Setup:
-# cp .env.example .env
-# Then open .env and fill in your actual values.
-#
-# Your real .env file is listed in .gitignore and will never be committed.
-# =============================================================================
-
-# -----------------------------------------------------------------------
-# Oxlo API — Required
-# -----------------------------------------------------------------------
-# Get your API key at https://oxlo.ai/dashboard
-OXLO_API_KEY=
-
-# -----------------------------------------------------------------------
-# Application settings
-# -----------------------------------------------------------------------
-# The port the application listens on inside the Docker container.
-# This must match the port_number in oxlo-manifest.json and the
-# EXPOSE instruction in your Dockerfile.
-PORT=3000
-
-# -----------------------------------------------------------------------
-# Add any other variables your project needs below this line.
-# Keep variable names in UPPER_SNAKE_CASE and include a short comment
-# explaining what each one is for.
-# -----------------------------------------------------------------------
-# DATABASE_URL=
-# REDIS_URL=
\ No newline at end of file
diff --git a/projects/template-project/Dockerfile b/projects/template-project/Dockerfile
deleted file mode 100644
index dc13b18..0000000
--- a/projects/template-project/Dockerfile
+++ /dev/null
@@ -1,69 +0,0 @@
-# =============================================================================
-# Oxtools Project Dockerfile Template
-# =============================================================================
-#
-# Instructions:
-# 1. Replace the base image with whatever runtime your project needs.
-# Common choices:
-# - node:20-alpine (Node.js)
-# - python:3.12-slim (Python)
-# - golang:1.22-alpine (Go)
-# - rust:1.78-slim (Rust)
-#
-# 2. Update the WORKDIR, COPY, and RUN commands to match your project's
-# build process.
-#
-# 3. Expose the port your application listens on. This must match the
-# port_number in your oxlo-manifest.json.
-#
-# 4. Set a non-root USER for security — do not run as root in production.
-#
-# This example uses Node.js. Adapt it to your stack.
-# =============================================================================
-
-# --- Stage 1: Install dependencies ---
-FROM node:20-alpine AS deps
-
-WORKDIR /app
-
-# Copy dependency manifests first so Docker can cache this layer.
-# If your package files haven't changed, Docker skips this step on rebuild.
-COPY package.json package-lock.json* ./
-
-RUN npm ci --omit=dev
-
-
-# --- Stage 2: Build the application (only needed for compiled/bundled stacks) ---
-# Remove this stage entirely if your stack doesn't have a build step.
-FROM node:20-alpine AS builder
-
-WORKDIR /app
-COPY --from=deps /app/node_modules ./node_modules
-COPY . .
-
-# Add your build command here, e.g. npm run build, cargo build --release, etc.
-# RUN npm run build
-
-
-# --- Stage 3: Production runtime ---
-FROM node:20-alpine AS runner
-
-# Create a non-root user to run the application.
-RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser
-
-WORKDIR /app
-
-# Copy only what the application needs to run.
-COPY --from=deps /app/node_modules ./node_modules
-COPY --from=builder /app/src ./src
-COPY package.json ./
-
-# Switch to the non-root user.
-USER appuser
-
-# Expose the port your application listens on.
-# Update this if your app uses a different port.
-EXPOSE 3000
-
-# Start the application.
-CMD ["node", "src/index.js"]
diff --git a/projects/template-project/README.md b/projects/template-project/README.md
deleted file mode 100644
index 3901b47..0000000
--- a/projects/template-project/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# [Your Project Name]
-
-> Powered by [Oxlo.ai](https://oxlo.ai)
-
-Replace this paragraph with a 2–3 sentence description of your tool. Explain what problem it solves and how it interacts with the Oxlo API. Be specific — "summarizes PDFs using the Oxlo inference API" is better than "an AI-powered document tool."
-
-## Prerequisites
-
-- [Docker](https://docs.docker.com/get-docker/) and Docker Compose
-- An active Oxlo API key — get one at [oxlo.ai/dashboard](https://oxlo.ai/dashboard)
-
-## Local setup
-
-**1. Clone the repo and navigate to your project:**
-```bash
-git clone https://github.com/Cyborg-Network/Oxtools.git
-cd Oxtools/projects/your-project-name
-```
-
-**2. Create your environment file:**
-```bash
-cp .env.example .env
-```
-Open `.env` and paste your `OXLO_API_KEY`.
-
-**3. Start the container:**
-```bash
-docker compose up --build
-```
-
-The application will be available at `http://localhost:3000` (or whichever port you configured).
-
-**4. Stop the container:**
-```bash
-docker compose down
-```
-
-## Demo
-
-[Insert a link to a Loom or YouTube recording of the tool running here]
-
-## Tech stack
-
-List the language(s) and frameworks used in this project.
\ No newline at end of file
diff --git a/projects/template-project/docker-compose.yml b/projects/template-project/docker-compose.yml
deleted file mode 100644
index 69bfa18..0000000
--- a/projects/template-project/docker-compose.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-# =============================================================================
-# Oxtools docker-compose.yml Template
-# =============================================================================
-#
-# Usage:
-# docker compose up — build and start the container
-# docker compose up --build — force a rebuild before starting
-# docker compose down — stop and remove containers
-#
-# Instructions:
-# - Update `ports` if your application uses a port other than 3000.
-# Format: "host_port:container_port"
-# - The `env_file` directive loads your local `.env` file into the container.
-# Make sure you've created `.env` from `.env.example` before running.
-# - Add any additional services your project needs (databases, caches, etc.)
-# as separate entries under `services`.
-# =============================================================================
-
-version: "3.9"
-
-services:
- app:
- # Build the image from the Dockerfile in the current directory.
- build:
- context: .
- dockerfile: Dockerfile
-
- # Map the host port to the container port.
- # Change 3000 to match your EXPOSE instruction in the Dockerfile.
- ports:
- - "3000:3000"
-
- # Load environment variables from your local .env file.
- # This file is NOT committed to the repository — create it from .env.example.
- env_file:
- - .env
-
- # Restart the container if it crashes unexpectedly.
- restart: unless-stopped
-
- # --- Example: Add a database service ---
- # Uncomment and configure if your project needs one.
- #
- # db:
- # image: postgres:16-alpine
- # environment:
- # POSTGRES_USER: ${DB_USER}
- # POSTGRES_PASSWORD: ${DB_PASSWORD}
- # POSTGRES_DB: ${DB_NAME}
- # ports:
- # - "5432:5432"
- # volumes:
- # - pg_data:/var/lib/postgresql/data
-
-# volumes:
-# pg_data:
diff --git a/projects/template-project/oxlo-manifest.json b/projects/template-project/oxlo-manifest.json
deleted file mode 100644
index 8494d10..0000000
--- a/projects/template-project/oxlo-manifest.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "name": "template-project",
- "description": "Replace this with a one-sentence description of what your tool does.",
- "author": "your-github-handle",
- "tech_stack": ["nodejs"],
- "port_number": 3000,
- "oxlo_api_used": true,
- "oxlo_api_endpoint": "https://api.oxlo.ai/v1/chat/completions",
- "repository_url": "https://github.com/Cyborg-Network/Oxtools/tree/main/projects/template-project",
- "demo_url": "",
- "version": "1.0.0"
-}
diff --git a/projects/template-project/package.json b/projects/template-project/package.json
deleted file mode 100644
index d41c2e6..0000000
--- a/projects/template-project/package.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "name": "template-project",
- "version": "1.0.0",
- "description": "A template for Oxtools contributions",
- "main": "src/index.js",
- "scripts": {
- "start": "node src/index.js"
- },
- "dependencies": {}
-}
\ No newline at end of file
diff --git a/services/python-tools/Dockerfile b/services/python-tools/Dockerfile
new file mode 100644
index 0000000..a3410be
--- /dev/null
+++ b/services/python-tools/Dockerfile
@@ -0,0 +1,46 @@
+# Use Debian 12 (Bookworm) to maintain Python 3.12
+# while ensuring full, native Playwright OS support
+FROM python:3.12-bookworm
+
+WORKDIR /app
+
+# 1. Install Tesseract and OpenCV system dependencies
+RUN apt-get update && apt-get install -y \
+ tesseract-ocr \
+ tesseract-ocr-eng \
+ libtesseract-dev \
+ libgl1 \
+ libglib2.0-0 \
+ && rm -rf /var/lib/apt/lists/*
+
+# 2. Install base runner requirements
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# 3. Install Playwright Chromium with dependencies
+# (Installed explicitly here to cache the 300MB+ browser before copying tool source code)
+RUN pip install --no-cache-dir playwright==1.44.0 && playwright install --with-deps chromium
+
+# 4. Copy all tool directories
+COPY tools/ tools/
+
+# 5. Install each tool's requirements.txt automatically
+RUN for dir in tools/*/; do \
+ req="${dir}requirements.txt"; \
+ if [ -f "$req" ]; then \
+ echo "Installing deps for $(basename $dir)..."; \
+ pip install --no-cache-dir -r "$req" || true; \
+ fi; \
+ done
+
+
+
+# 6. Copy the runner script
+COPY runner.py .
+
+# Set tools directory
+ENV TOOLS_DIR=/app/tools
+
+EXPOSE 9080
+
+CMD ["uvicorn", "runner:app", "--host", "0.0.0.0", "--port", "9080"]
\ No newline at end of file
diff --git a/services/python-tools/requirements.txt b/services/python-tools/requirements.txt
new file mode 100644
index 0000000..568805b
--- /dev/null
+++ b/services/python-tools/requirements.txt
@@ -0,0 +1,5 @@
+# Base requirements for the unified tool runner
+fastapi==0.115.0
+uvicorn[standard]==0.30.6
+httpx==0.27.0
+pydantic==2.9.0
diff --git a/services/python-tools/runner.py b/services/python-tools/runner.py
new file mode 100644
index 0000000..cb02ade
--- /dev/null
+++ b/services/python-tools/runner.py
@@ -0,0 +1,335 @@
+"""
+Oxtools Unified Python Tool Runner
+====================================
+A single FastAPI service that auto-discovers and runs ALL Python tools.
+
+Tool Structure:
+ services/python-tools/tools/
+ ├── my-tool/
+ │ ├── tool.py ← Entry point (must have MANIFEST + run)
+ │ ├── helpers.py ← Any supporting files
+ │ ├── prompts/ ← Any subdirectories
+ │ └── requirements.txt ← This tool's pip deps
+ └── another-tool/
+ ├── tool.py
+ └── requirements.txt
+
+Contributors:
+ 1. Create a directory: tools/my-tool/
+ 2. Add tool.py with MANIFEST dict + async run(data) function
+ 3. Add requirements.txt for pip deps
+ 4. Register in Next.js frontend
+ 5. Done! No Docker knowledge needed.
+
+The runner auto-discovers all tools/{name}/tool.py at startup.
+"""
+
+import os
+import sys
+import json
+import time
+import logging
+import importlib
+import importlib.util
+import traceback
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse, JSONResponse
+
+logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+)
+logger = logging.getLogger("tool-runner")
+
+
+# ---------------------------------------------------------------------------
+# Tool Registry
+# ---------------------------------------------------------------------------
+
+class ToolRegistry:
+ """
+ Discovers and manages Python tools from the tools/ directory.
+
+ Each tool is a DIRECTORY containing at minimum a tool.py file with:
+ - MANIFEST: dict with {id, name, description, ...}
+ - async def run(request_data: dict) -> dict | AsyncGenerator
+
+ Contributors can add any number of supporting files, subdirectories,
+ configs, etc. inside their tool directory.
+ """
+
+ def __init__(self, tools_dir: str):
+ self.tools_dir = Path(tools_dir)
+ self.tools: dict[str, dict[str, Any]] = {}
+
+ def discover(self):
+ """Scan tools/ for directories containing tool.py."""
+ if not self.tools_dir.exists():
+ logger.warning(f"Tools directory not found: {self.tools_dir}")
+ return
+
+ for tool_dir in sorted(self.tools_dir.iterdir()):
+ # Skip files and hidden/underscore dirs
+ if not tool_dir.is_dir():
+ continue
+ if tool_dir.name.startswith(("_", ".")):
+ continue
+
+ entry_point = tool_dir / "tool.py"
+ if not entry_point.exists():
+ logger.warning(f"Skipping {tool_dir.name}/: no tool.py found")
+ continue
+
+ self._load_tool(tool_dir, entry_point)
+
+ logger.info(f"═══ Tool discovery complete: {len(self.tools)} tools loaded ═══")
+
+ def _load_tool(self, tool_dir: Path, entry_point: Path):
+ """Load a single tool from its directory."""
+ tool_name = tool_dir.name
+ module_name = f"tools.{tool_name}.tool"
+
+ try:
+ # Add the tool directory to sys.path so it can import its own modules
+ tool_path = str(tool_dir)
+ if tool_path not in sys.path:
+ sys.path.insert(0, tool_path)
+
+ # Also add parent tools/ dir for cross-tool imports
+ tools_path = str(self.tools_dir)
+ if tools_path not in sys.path:
+ sys.path.insert(0, tools_path)
+
+ # Load the module from file path directly
+ spec = importlib.util.spec_from_file_location(module_name, str(entry_point))
+ if spec is None or spec.loader is None:
+ logger.error(f"✗ Cannot load {tool_name}/tool.py: invalid module spec")
+ return
+
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[module_name] = module
+ spec.loader.exec_module(module)
+
+ # Validate required interface
+ if not hasattr(module, "MANIFEST"):
+ logger.warning(f"✗ {tool_name}/tool.py: missing MANIFEST dict")
+ return
+ if not hasattr(module, "run"):
+ logger.warning(f"✗ {tool_name}/tool.py: missing run() function")
+ return
+
+ manifest = module.MANIFEST
+ tool_id = manifest.get("id", tool_name)
+
+ self.tools[tool_id] = {
+ "id": tool_id,
+ "module": module,
+ "manifest": manifest,
+ "directory": str(tool_dir),
+ "entry_point": str(entry_point),
+ }
+
+ logger.info(f" ✓ {tool_id} — {manifest.get('name', tool_name)}")
+
+ except Exception as e:
+ logger.error(f" ✗ {tool_name}: {e}")
+ logger.debug(traceback.format_exc())
+
+ def get_tool(self, tool_id: str) -> dict | None:
+ return self.tools.get(tool_id)
+
+ def list_tools(self) -> list[dict]:
+ return [
+ {
+ "id": t["id"],
+ "name": t["manifest"].get("name", t["id"]),
+ "description": t["manifest"].get("description", ""),
+ "author": t["manifest"].get("author", "Community"),
+ "version": t["manifest"].get("version", "1.0.0"),
+ }
+ for t in self.tools.values()
+ ]
+
+
+# ---------------------------------------------------------------------------
+# FastAPI Application
+# ---------------------------------------------------------------------------
+
+TOOLS_DIR = os.getenv("TOOLS_DIR", "/app/tools")
+registry = ToolRegistry(TOOLS_DIR)
+
+app = FastAPI(
+ title="Oxtools Python Tool Runner",
+ description="Unified service that auto-discovers and runs all Python tools",
+ version="2.0.0",
+)
+
+app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"],
+ allow_credentials=False,
+ allow_methods=["*"],
+ allow_headers=["*"],
+)
+
+
+@app.on_event("startup")
+async def startup():
+ """Discover all tools on startup."""
+ logger.info(f"Scanning for tools in: {TOOLS_DIR}")
+ registry.discover()
+
+
+@app.get("/health")
+async def health():
+ return {
+ "status": "ok",
+ "service": "python-tool-runner",
+ "version": "2.0.0",
+ "tools_loaded": len(registry.tools),
+ "tool_ids": list(registry.tools.keys()),
+ }
+
+
+@app.get("/api/tools")
+async def list_tools():
+ return {"tools": registry.list_tools()}
+
+
+@app.post("/api/tools/{tool_id}")
+async def run_tool(tool_id: str, request: Request):
+ """Execute a Python tool by ID."""
+ tool = registry.get_tool(tool_id)
+ if not tool:
+ raise HTTPException(
+ status_code=404,
+ detail=f"Tool '{tool_id}' not found. Available: {list(registry.tools.keys())}"
+ )
+
+ try:
+ body = await request.json()
+ except Exception:
+ body = {}
+
+ try:
+ start = time.time()
+ logger.info(f"[{tool_id}] Executing...")
+
+ result = await tool["module"].run(body)
+
+ # Async generator → stream response with keepalive
+ if hasattr(result, "__aiter__"):
+ async def stream():
+ import asyncio
+ async for chunk in result:
+ if isinstance(chunk, dict):
+ yield json.dumps(chunk) + "\n"
+ else:
+ yield str(chunk)
+
+ async def stream_with_keepalive():
+ """Wrap stream with keepalive pings to prevent timeout."""
+ import asyncio
+ queue = asyncio.Queue()
+
+ async def producer():
+ try:
+ logger.info(f"[Producer] Task started")
+ async for chunk in result:
+ if isinstance(chunk, dict):
+ await queue.put(json.dumps(chunk) + "\n")
+ else:
+ await queue.put(str(chunk))
+ logger.info(f"[Producer] LangGraph generator finished normally!")
+ except asyncio.CancelledError:
+ logger.error(f"[Producer] Task was CANCELLED!")
+ raise
+ except Exception as e:
+ logger.error(f"Stream producer error: {e}")
+ except BaseException as e:
+ logger.error(f"[Producer] BaseException: {e}")
+ raise
+ finally:
+ logger.info(f"[Producer] Task exiting, sending EOF")
+ await queue.put(None) # EOF marker
+
+ # Start the producer in the background
+ producer_task = asyncio.create_task(producer())
+
+ # CRITICAL: Keep a strong reference in a GLOBAL scope.
+ # Previously, storing it on the queue created a reference cycle
+ # (queue -> task -> producer coroutine -> queue) which Python's
+ # cyclic garbage collector would silently destroy!
+ if not hasattr(app, "_active_tasks"):
+ app._active_tasks = set()
+ app._active_tasks.add(producer_task)
+ producer_task.add_done_callback(app._active_tasks.discard)
+
+ get_task = None
+ while True:
+ try:
+ if get_task is None:
+ get_task = asyncio.create_task(queue.get())
+
+ # Wait for either the queue item or the timeout
+ done, pending = await asyncio.wait(
+ [get_task],
+ timeout=10.0,
+ return_when=asyncio.FIRST_COMPLETED
+ )
+
+ if get_task in done:
+ chunk = get_task.result()
+ get_task = None # Reset for next iteration
+
+ if chunk is None:
+ logger.info("[Stream] Received EOF from queue, breaking loop")
+ break
+ yield chunk
+ else:
+ # Timeout occurred, get_task is still pending
+ logger.info("[Stream] Keepalive timeout, yielding dot")
+ yield ".\n"
+ except asyncio.CancelledError:
+ logger.error("[Stream] stream_with_keepalive was CANCELLED by Starlette!")
+ raise
+ except Exception as e:
+ logger.error(f"[Stream] Unexpected error in stream loop: {e}")
+ break
+
+ logger.info("[Stream] stream_with_keepalive completely finished")
+
+ logger.info("Exited stream_with_keepalive loop.")
+
+ return StreamingResponse(
+ stream_with_keepalive(),
+ media_type="text/plain",
+ headers={
+ "X-Content-Type-Options": "nosniff",
+ "X-Accel-Buffering": "no",
+ "Cache-Control": "no-cache",
+ },
+ )
+
+ elapsed = time.time() - start
+ logger.info(f"[{tool_id}] Completed in {elapsed:.1f}s")
+
+ if isinstance(result, dict):
+ return JSONResponse(result)
+ return JSONResponse({"result": str(result)})
+
+ except Exception as e:
+ logger.error(f"[{tool_id}] Error: {e}")
+ logger.debug(traceback.format_exc())
+ raise HTTPException(status_code=500, detail=str(e))
+
+
+if __name__ == "__main__":
+ import uvicorn
+ # reload=True ensures that changes to Python files in the mounted volume
+ # automatically restart the server without needing to restart Docker!
+ uvicorn.run("runner:app", host="0.0.0.0", port=9080, reload=True)
diff --git a/services/python-tools/tools/_template/requirements.txt b/services/python-tools/tools/_template/requirements.txt
new file mode 100644
index 0000000..41763b4
--- /dev/null
+++ b/services/python-tools/tools/_template/requirements.txt
@@ -0,0 +1,7 @@
+# Add your pip dependencies here, one per line.
+# These are auto-installed when the Docker container builds.
+#
+# Example:
+# openai==1.30.0
+# beautifulsoup4==4.12.0
+# numpy==1.26.0
diff --git a/services/python-tools/tools/_template/tool.py b/services/python-tools/tools/_template/tool.py
new file mode 100644
index 0000000..1f627d9
--- /dev/null
+++ b/services/python-tools/tools/_template/tool.py
@@ -0,0 +1,69 @@
+"""
+Oxtools Tool Template
+======================
+Copy this entire directory to create a new Python tool:
+
+ cp -r tools/_template tools/my-tool
+
+Then edit tool.py with your logic. That's it!
+
+Directory structure:
+ my-tool/
+ ├── tool.py ← Entry point (MANIFEST + run) — YOU EDIT THIS
+ ├── requirements.txt ← Your pip dependencies
+ └── (any other files) ← Helpers, configs, data, etc.
+"""
+
+# ─── MANIFEST (required) ─────────────────────────────────────────────
+MANIFEST = {
+ "id": "my-tool", # URL-safe ID (used in /api/tools/{id})
+ "name": "My Awesome Tool", # Human-readable name
+ "description": "One-line description of what this tool does",
+ "author": "Your Name",
+ "version": "1.0.0",
+}
+
+
+# ─── RUN FUNCTION (required) ─────────────────────────────────────────
+async def run(data: dict) -> dict:
+ """
+ Execute the tool.
+
+ Args:
+ data: Request body from the frontend form.
+ Example: {"query": "user input", "model": "llama-3.3-70b"}
+
+ Returns:
+ dict with a "result" key containing the output.
+ """
+ query = data.get("query", "")
+
+ # ── Your tool logic goes here! ──
+ # You can:
+ # - Import helper modules from this same directory
+ # - Use os.getenv("OXLO_API_KEY") for the Oxlo API
+ # - Use any packages listed in requirements.txt
+ # - Return a dict (JSON response) or an async generator (streaming)
+
+ result = f"Processed: {query}"
+
+ return {
+ "result": result,
+ "metadata": {
+ "model_used": "none",
+ "processing_time": "0.1s",
+ },
+ }
+
+
+# ─── STREAMING EXAMPLE (optional) ────────────────────────────────────
+# For long-running tools (AI agents, research, etc.), return an async
+# generator instead of a dict. The runner streams it to the frontend.
+#
+# async def run(data: dict):
+# yield "[step-1] Planning...\n"
+# # ... do work ...
+# yield "[step-2] Searching...\n"
+# # ... do work ...
+# yield "\n---RESULT---\n"
+# yield "Final output here"
diff --git a/services/python-tools/tools/code-security-scanner-v2/requirements.txt b/services/python-tools/tools/code-security-scanner-v2/requirements.txt
new file mode 100644
index 0000000..55e06c9
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/requirements.txt
@@ -0,0 +1,5 @@
+langgraph>=0.2.0
+langchain-openai>=0.2.0
+langchain-core>=0.3.0
+bandit>=1.7.0
+httpx>=0.27.0
diff --git a/services/python-tools/tools/code-security-scanner-v2/sec_agents.py b/services/python-tools/tools/code-security-scanner-v2/sec_agents.py
new file mode 100644
index 0000000..c6cfd41
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/sec_agents.py
@@ -0,0 +1,651 @@
+"""
+Code Security Scanner V2 — Agent Nodes & Graph
+================================================
+LangGraph StateGraph with typed state and 4 nodes.
+
+KEY ARCHITECTURE (v3.0 — Multi-File):
+ Pipeline: Scanner → Auditor → Fixer → Reporter
+
+ Scanner runs ALL deterministic tools across ALL files:
+ - Per-file: bandit, AST, secrets, patterns
+ - Cross-file: import graph, taint tracking, config correlation
+ - Obfuscation: base64/rot13/unicode/hex decoding
+ - Dependencies: CVE lookup via osv.dev
+
+ Auditor receives FULL structured intelligence + ALL source code.
+ This is STRICTLY BETTER than V1 because it has cross-file context.
+"""
+
+import hashlib
+import json
+import logging
+from typing import TypedDict
+
+from langgraph.graph import StateGraph, END
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from sec_config import (
+ OXLO_API_KEY, OXLO_BASE_URL,
+ AUDITOR_MODEL, FIXER_MODEL, REPORTER_MODEL,
+ SEVERITY_WEIGHTS,
+)
+from sec_prompts import AUDITOR_PROMPT, FIXER_PROMPT, REPORTER_PROMPT
+from sec_scanners import run_all_scanners
+from sec_file_parser import parse_multi_file_input, ParsedFile
+from sec_cross_file import (
+ build_import_graph, track_taint_flow, correlate_configs,
+ decode_obfuscation, scan_dependencies_for_cves,
+ scan_exec_eval_dangers, scan_timing_attacks,
+ scan_toctou, scan_mass_assignment,
+)
+
+logger = logging.getLogger("security-scanner")
+
+
+# ─── State Schema ─────────────────────────────────────────────────────
+
+class SecurityState(TypedDict):
+ """Shared state flowing through the security pipeline."""
+ # Input
+ code: str
+ files_data: str
+ language: str
+ user_model: str
+
+ # Parser output
+ parsed_files: list[dict]
+ file_count: int
+
+ # Scanner output (deterministic)
+ raw_findings: list[dict]
+ scanner_summary: str
+
+ # Cross-file intelligence (V2-exclusive)
+ import_graph: list[dict]
+ taint_chains: list[dict]
+ config_risks: list[dict]
+ decoded_payloads: list[dict]
+ cve_findings: list[dict]
+ exec_eval_findings: list[dict]
+ timing_findings: list[dict]
+
+ # Auditor output
+ triage_results: list[dict]
+ deep_findings: list[dict]
+ severity_score: int
+
+ # Fixer output
+ fixes: str
+
+ # Reporter output
+ final_report: str
+
+ # Metadata
+ status: str
+
+
+# ─── Helpers ──────────────────────────────────────────────────────────
+
+def get_llm(model: str, temperature: float = 0.1, max_tokens: int = 8192) -> ChatOpenAI:
+ return ChatOpenAI(
+ model=model, api_key=OXLO_API_KEY, base_url=OXLO_BASE_URL,
+ temperature=temperature, max_tokens=max_tokens,
+ timeout=900, # 15 min for reasoning models like DeepSeek R1
+ )
+
+
+def _parse_json_from_llm(content: str) -> dict:
+ content = content.strip()
+ if "```" in content:
+ blocks = content.split("```")
+ for block in blocks:
+ cleaned = block.strip()
+ if cleaned.startswith("json"):
+ cleaned = cleaned[4:].strip()
+ if cleaned.startswith("{") or cleaned.startswith("["):
+ try:
+ return json.loads(cleaned)
+ except json.JSONDecodeError:
+ continue
+ try:
+ return json.loads(content)
+ except json.JSONDecodeError:
+ start = content.find("{")
+ end = content.rfind("}")
+ if start != -1 and end != -1:
+ try:
+ return json.loads(content[start:end + 1])
+ except json.JSONDecodeError:
+ pass
+ return {}
+
+
+# ─── Node 1: Scanner (Deterministic — ALL files) ─────────────────────
+
+def scanner_node(state: SecurityState) -> dict:
+ """
+ Parse multi-file input, then run ALL deterministic scanners
+ on every file + cross-file analysis.
+ """
+ code = state.get("code", "")
+ files_data = state.get("files_data", "")
+ language = state.get("language", "python")
+
+ # Parse into individual files
+ parsed_files = parse_multi_file_input(code, files_data)
+
+ # If no files parsed, create a single-file fallback
+ if not parsed_files:
+ if code.strip():
+ parsed_files = [ParsedFile(
+ path="input.py", filename="input.py",
+ content=code, language=language,
+ is_config=False, size_bytes=len(code),
+ )]
+ else:
+ return {"status": "error", "scanner_summary": "No code provided."}
+
+ file_count = len(parsed_files)
+ logger.info(f"[Scanner] Processing {file_count} files")
+
+ # Run per-file scanners on each file
+ all_findings = []
+ for f in parsed_files:
+ # NOTE: Config files (settings.py, etc.) are ALSO scanned by per-file
+ # scanners — they often contain hardcoded secrets, insecure settings,
+ # and dangerous patterns. The config correlator adds ADDITIONAL analysis.
+ findings = run_all_scanners(f.content, f.language)
+ # Tag each finding with its file path and source
+ for finding in findings:
+ finding["file"] = f.path
+ if "source" not in finding:
+ finding["source"] = "deterministic_scanner"
+ all_findings.extend(findings)
+
+ # Deduplicate findings by (file, line, category) — prevents triple-count noise
+ seen = set()
+ deduped = []
+ for finding in all_findings:
+ key = (finding.get("file", ""), finding.get("line", 0), finding.get("category", ""))
+ if key not in seen:
+ seen.add(key)
+ deduped.append(finding)
+ all_findings = deduped
+
+ # Run cross-file scanners
+ import_graph = [e.to_dict() for e in build_import_graph(parsed_files)]
+ taint_chains = [c.to_dict() for c in track_taint_flow(parsed_files)]
+ config_risks = [r.to_dict() for r in correlate_configs(parsed_files)]
+ decoded_payloads = [p.to_dict() for p in decode_obfuscation(parsed_files)]
+
+ # Config risks are DETERMINISTIC — add to findings so they cannot be dismissed
+ for cr in config_risks:
+ all_findings.append({
+ "file": cr.get("file", ""),
+ "line": 0,
+ "severity": cr.get("severity", "HIGH"),
+ "category": "Configuration Risk",
+ "title": cr.get("risk", cr.get("setting", "")),
+ "description": f"{cr.get('setting', '')}: {cr.get('risk', '')}",
+ "cwe_id": "CWE-16",
+ "source": "deterministic_scanner",
+ })
+
+ # Taint chains are DETERMINISTIC — add to findings
+ for tc in taint_chains:
+ all_findings.append({
+ "file": tc.get("sink_file", tc.get("source_file", "")),
+ "line": tc.get("sink_line", 0),
+ "severity": tc.get("severity", "CRITICAL"),
+ "category": f"Cross-File {tc.get('vulnerability', 'Taint Flow')}",
+ "title": f"{tc.get('vulnerability', 'Taint flow')}: {tc.get('source', '')} → {tc.get('sink', '')}",
+ "description": f"User input from {tc.get('source', '')} ({tc.get('source_file', '')}:{tc.get('source_line', '')}) reaches {tc.get('sink', '')} ({tc.get('sink_file', '')}:{tc.get('sink_line', '')})",
+ "cwe_id": "",
+ "source": "deterministic_scanner",
+ })
+
+ # Run CVE scanner (may call external API)
+ cve_findings = []
+ try:
+ cve_findings = [c.to_dict() for c in scan_dependencies_for_cves(parsed_files)]
+ except Exception as e:
+ logger.warning(f"[Scanner] CVE scan failed: {e}")
+
+ # Run exec/eval danger scanner (Drawback 1 fix)
+ exec_eval_findings = scan_exec_eval_dangers(parsed_files)
+ for ef in exec_eval_findings:
+ ef["source"] = "deterministic_scanner"
+ all_findings.extend(exec_eval_findings)
+ logger.info(f"[Scanner] exec/eval scanner: {len(exec_eval_findings)} findings")
+ for ef in exec_eval_findings:
+ logger.info(f" → {ef.get('file','')}:{ef.get('line','')} {ef.get('title','')}")
+
+ # Run timing attack scanner (Drawback 5 fix)
+ timing_findings = scan_timing_attacks(parsed_files)
+ for tf in timing_findings:
+ tf["source"] = "deterministic_scanner"
+ all_findings.extend(timing_findings)
+ logger.info(f"[Scanner] timing attack scanner: {len(timing_findings)} findings")
+ for tf in timing_findings:
+ logger.info(f" → {tf.get('file','')}:{tf.get('line','')} {tf.get('title','')}")
+
+ # Run TOCTOU race condition scanner
+ toctou_findings = scan_toctou(parsed_files)
+ for tf in toctou_findings:
+ tf["source"] = "deterministic_scanner"
+ all_findings.extend(toctou_findings)
+ logger.info(f"[Scanner] TOCTOU scanner: {len(toctou_findings)} findings")
+
+ # Run mass assignment scanner
+ mass_assign_findings = scan_mass_assignment(parsed_files)
+ for mf in mass_assign_findings:
+ mf["source"] = "deterministic_scanner"
+ all_findings.extend(mass_assign_findings)
+ logger.info(f"[Scanner] mass assignment scanner: {len(mass_assign_findings)} findings")
+
+ # Advanced deduplication: same file + same CWE + lines within 5
+ # Tracks confirmed_by (which scanners caught it) + sha256 fingerprints
+ deduped = []
+ for finding in all_findings:
+ is_dup = False
+ for existing in deduped:
+ same_file = existing.get("file") == finding.get("file")
+ same_cwe = (existing.get("cwe_id") == finding.get("cwe_id") and existing.get("cwe_id"))
+ very_close = abs(existing.get("line", 0) - finding.get("line", 0)) <= 2
+ same_title = existing.get("title", "").lower() == finding.get("title", "").lower()
+
+ if same_file and ((same_cwe and very_close) or (same_title and very_close)):
+ # Keep higher severity version
+ sev_order = {"CRITICAL": 4, "HIGH": 3, "MEDIUM": 2, "LOW": 1}
+ if sev_order.get(finding.get("severity"), 0) > sev_order.get(existing.get("severity"), 0):
+ existing["severity"] = finding["severity"]
+ existing["title"] = finding.get("title", existing.get("title"))
+ # Track which scanners confirmed this
+ scanner_source = finding.get("source", "unknown")
+ if "confirmed_by" not in existing:
+ existing["confirmed_by"] = [existing.get("source", "unknown")]
+ if scanner_source not in existing["confirmed_by"]:
+ existing["confirmed_by"].append(scanner_source)
+ existing["duplicate_count"] = existing.get("duplicate_count", 1) + 1
+ # Set confidence based on agreement count
+ existing["confidence"] = "high" if existing["duplicate_count"] >= 2 else "medium"
+ is_dup = True
+ break
+ if not is_dup:
+ deduped.append(finding)
+ all_findings = deduped
+
+ # Generate sha256 fingerprints for every finding (Drawback 2 fix)
+ # Fingerprints lock findings — same fingerprint = same finding across runs
+ for finding in all_findings:
+ fp_data = f"{finding.get('file','')}{finding.get('line',0)}{finding.get('cwe_id','')}{finding.get('category','')}"
+ finding["fingerprint"] = hashlib.sha256(fp_data.encode()).hexdigest()[:16]
+
+ # Build summary
+ cross_file_stats = []
+ if import_graph:
+ cross_file_stats.append(f"{len(import_graph)} import edges")
+ if taint_chains:
+ cross_file_stats.append(f"{len(taint_chains)} taint chains")
+ if config_risks:
+ cross_file_stats.append(f"{len(config_risks)} config risks")
+ if decoded_payloads:
+ cross_file_stats.append(f"{len(decoded_payloads)} decoded payloads")
+ if cve_findings:
+ cross_file_stats.append(f"{len(cve_findings)} known CVEs")
+ if exec_eval_findings:
+ cross_file_stats.append(f"{len(exec_eval_findings)} exec/eval dangers")
+ if timing_findings:
+ cross_file_stats.append(f"{len(timing_findings)} timing attacks")
+
+ # Count deterministic vs LLM findings
+ det_count = sum(1 for f in all_findings if f.get("source") == "deterministic_scanner")
+ summary = (
+ f"Scanned {file_count} files. "
+ f"Found {len(all_findings)} findings ({det_count} deterministic, {len(all_findings) - det_count} pattern-based). "
+ )
+ if cross_file_stats:
+ summary += f"Cross-file analysis: {', '.join(cross_file_stats)}."
+
+ logger.info(f"[Scanner] {summary}")
+
+ return {
+ "parsed_files": [{"path": f.path, "language": f.language,
+ "is_config": f.is_config, "size": f.size_bytes}
+ for f in parsed_files],
+ "file_count": file_count,
+ "raw_findings": all_findings,
+ "import_graph": import_graph,
+ "taint_chains": taint_chains,
+ "config_risks": config_risks,
+ "decoded_payloads": decoded_payloads,
+ "cve_findings": cve_findings,
+ "exec_eval_findings": exec_eval_findings,
+ "timing_findings": timing_findings,
+ "scanner_summary": summary,
+ "status": "scanning_complete",
+ }
+
+
+# ─── Node 2: Auditor (ONE powerful LLM call, FULL context) ───────────
+
+def auditor_node(state: SecurityState) -> dict:
+ """
+ Single comprehensive security analysis with ALL cross-file intelligence.
+ The LLM sees: all source code + scanner findings + import graph +
+ taint chains + config risks + decoded payloads + CVEs.
+ """
+ code = state.get("code", "")
+ files_data = state.get("files_data", "")
+ findings = state.get("raw_findings", [])
+
+ model = state.get("user_model") or AUDITOR_MODEL
+ logger.info(f"[Auditor] Comprehensive analysis with model={model}")
+
+ # FIX Drawback 2: temperature=0 for deterministic triage
+ # Use 8192 max_tokens — auditor only returns JSON triage, not full report
+ llm = get_llm(model, 0.0, max_tokens=8192)
+
+ # Build source code section — truncated to prevent timeout
+ parsed_files = parse_multi_file_input(code, files_data)
+ code_sections = []
+ for f in parsed_files:
+ # Dynamic budget: fewer files = more space per file (up to 6KB)
+ char_budget = max(3000, 20000 // max(len(parsed_files), 1))
+ char_budget = min(char_budget, 6000)
+ truncated = f.content[:char_budget]
+ if len(f.content) > char_budget:
+ truncated += f"\n# ... [{len(f.content) - char_budget} more chars truncated]"
+ code_sections.append(f"### {f.path}\n```{f.language}\n{truncated}\n```")
+ all_code = "\n\n".join(code_sections) if code_sections else f"```\n{code[:5000]}\n```"
+
+ # Build cross-file intelligence — COMPACT format to reduce token count
+ cross_file_intel = ""
+ taint_chains = state.get("taint_chains", [])
+ if taint_chains:
+ cross_file_intel += f"\n## Cross-File Taint Chains ({len(taint_chains)}):\n"
+ # Compact: one-line per chain instead of full JSON
+ for tc in taint_chains[:8]:
+ cross_file_intel += f"- {tc.get('source','')} @ {tc.get('source_file','')}:{tc.get('source_line','')} → {tc.get('sink','')} @ {tc.get('sink_file','')}:{tc.get('sink_line','')} = **{tc.get('vulnerability','')}** [{tc.get('severity','')}]\n"
+
+ import_graph = state.get("import_graph", [])
+ if import_graph:
+ # Summary only — don't dump 35 edges as JSON
+ cross_file_intel += f"\n## Import Graph: {len(import_graph)} import edges across files\n"
+ # Just list unique modules
+ modules = set(e.get('target_module', '') for e in import_graph)
+ cross_file_intel += f"Imported modules: {', '.join(list(modules)[:15])}\n"
+
+ config_risks = state.get("config_risks", [])
+ if config_risks:
+ cross_file_intel += f"\n## Config Risks ({len(config_risks)}):\n"
+ for cr in config_risks:
+ cross_file_intel += f"- {cr.get('file','')}: {cr.get('setting','')} — {cr.get('risk','')} [{cr.get('severity','')}]\n"
+
+ decoded_payloads = state.get("decoded_payloads", [])
+ if decoded_payloads:
+ cross_file_intel += f"\n## Decoded Obfuscated Payloads ({len(decoded_payloads)}):\n"
+ for dp in decoded_payloads:
+ cross_file_intel += f"- {dp.get('file','')}:{dp.get('line_number','')} [{dp.get('encoding_type','')}] → {dp.get('decoded_value','')[:80]} {'**MALICIOUS**' if dp.get('is_malicious') else ''} [{dp.get('severity','')}]\n"
+
+ cve_findings = state.get("cve_findings", [])
+ if cve_findings:
+ cross_file_intel += f"\n## Known CVEs ({len(cve_findings)}):\n"
+ # Top 10 only — full list in reporter
+ for cve in cve_findings[:10]:
+ cross_file_intel += f"- {cve.get('package','')}=={cve.get('version','')}: {cve.get('cve_id','')} — {cve.get('summary','')[:60]} [{cve.get('severity','')}]\n"
+ if len(cve_findings) > 10:
+ cross_file_intel += f"- ... and {len(cve_findings) - 10} more CVEs\n"
+
+ # CRITICAL: Sort deterministic findings FIRST so they're always within the LLM's view window
+ # Without this, exec/eval and timing findings appended at the end get cut off
+ findings_deterministic = [f for f in findings if f.get("source") == "deterministic_scanner"]
+ findings_llm = [f for f in findings if f.get("source") != "deterministic_scanner"]
+ sorted_findings = findings_deterministic + findings_llm
+
+ # Include exec/eval and timing findings as explicit cross-file intelligence
+ exec_eval = state.get("exec_eval_findings", [])
+ if exec_eval:
+ cross_file_intel += f"\n## 🚨 Dangerous exec/eval Calls ({len(exec_eval)}):\n"
+ for ef in exec_eval:
+ cross_file_intel += f"- **{ef.get('severity','CRITICAL')}** {ef.get('file','')}:{ef.get('line','')} — {ef.get('title','')} [{ef.get('cwe_id','')}]\n"
+
+ timing = state.get("timing_findings", [])
+ if timing:
+ cross_file_intel += f"\n## ⏱️ Timing Attack Vulnerabilities ({len(timing)}):\n"
+ for tf in timing:
+ cross_file_intel += f"- **{tf.get('severity','HIGH')}** {tf.get('file','')}:{tf.get('line','')} — {tf.get('title','')} [{tf.get('cwe_id','')}]\n"
+
+ # Send up to 30 findings (deterministic always first)
+ findings_text = json.dumps(sorted_findings[:30], separators=(',', ':')) if sorted_findings else "[]"
+
+ user_content = (
+ f"## Files: {state.get('file_count', 1)}\n\n"
+ f"## Source Code:\n{all_code}\n\n"
+ f"## Scanner Findings ({len(findings)}):\n```json\n{findings_text}\n```\n\n"
+ f"{cross_file_intel}\n"
+ f"Perform a comprehensive security audit. USE the cross-file intelligence above.\n"
+ f"Return ONLY the JSON object. Be thorough but concise."
+ )
+
+ logger.info(f"[Auditor] Prompt size: {len(user_content)} chars")
+
+ response = llm.invoke([
+ SystemMessage(content=AUDITOR_PROMPT),
+ HumanMessage(content=user_content),
+ ])
+
+ result = _parse_json_from_llm(response.content)
+ triage = result.get("triage", [])
+ deep_findings = result.get("deep_findings", [])
+
+ # FIX Drawback 2: Deterministic scanner findings are NEVER dismissed
+ # Use FINGERPRINTS for matching (not fragile integer indices)
+ deterministic_fps = {}
+ for finding in findings:
+ if finding.get("source") == "deterministic_scanner" and finding.get("fingerprint"):
+ deterministic_fps[finding["fingerprint"]] = finding
+
+ # Override any LLM dismissals of deterministic findings
+ for item in triage:
+ fidx = item.get("finding_index", -1)
+ # Match by index (if LLM used it) or by fingerprint
+ matched_finding = None
+ if 0 <= fidx < len(findings):
+ matched_finding = findings[fidx]
+
+ if matched_finding and matched_finding.get("source") == "deterministic_scanner":
+ if item.get("classification") == "false_positive":
+ logger.warning(f"[Auditor] LLM tried to dismiss deterministic finding #{fidx} '{matched_finding.get('title','')}' — overriding to true_positive")
+ item["classification"] = "true_positive"
+ item["rationale"] = f"[ENFORCED] Deterministic scanner finding — cannot be dismissed. Original LLM reason: {item.get('rationale', 'N/A')}"
+
+ # Ensure ALL deterministic findings appear in triage, even if LLM ignored them
+ triaged_fps = set()
+ for item in triage:
+ fidx = item.get("finding_index", -1)
+ if 0 <= fidx < len(findings) and findings[fidx].get("fingerprint"):
+ triaged_fps.add(findings[fidx]["fingerprint"])
+
+ for fp, finding in deterministic_fps.items():
+ if fp not in triaged_fps:
+ # Find this finding's index in the sorted list
+ f_idx = next((i for i, f in enumerate(findings) if f.get("fingerprint") == fp), -1)
+ triage.append({
+ "finding_index": f_idx,
+ "classification": "true_positive",
+ "rationale": f"[AUTO-CONFIRMED] Deterministic finding: {finding.get('title','')}",
+ "adjusted_severity": finding.get("severity", "HIGH"),
+ "file": finding.get("file", ""),
+ "line": finding.get("line", 0),
+ "title": finding.get("title", ""),
+ "cwe_id": finding.get("cwe_id", ""),
+ "fingerprint": fp,
+ })
+
+ severity_score = 0
+ for item in triage:
+ if item.get("classification") in ("true_positive", "needs_investigation"):
+ sev = item.get("adjusted_severity", "MEDIUM")
+ severity_score += SEVERITY_WEIGHTS.get(sev, 4)
+ for item in deep_findings:
+ severity_score += SEVERITY_WEIGHTS.get(item.get("severity", "MEDIUM"), 4)
+ # Add CVE severity
+ for cve in cve_findings:
+ severity_score += SEVERITY_WEIGHTS.get(cve.get("severity", "HIGH"), 7)
+
+ logger.info(f"[Auditor] Triage: {len(triage)}, Deep: {len(deep_findings)}, Score: {severity_score}")
+
+ return {
+ "triage_results": triage,
+ "deep_findings": deep_findings,
+ "severity_score": severity_score,
+ "status": "audit_complete",
+ }
+
+
+# ─── Node 3: Fixer ───────────────────────────────────────────────────
+def fixer_node(state: SecurityState) -> dict:
+ logger.info("[Fixer] Node entered.")
+
+ triage = state.get("triage_results", [])
+ deep = state.get("deep_findings", [])
+ model = state.get("user_model") or FIXER_MODEL
+
+ # Add error catching in case triage results are malformed strings instead of dicts
+ try:
+ confirmed = [t for t in triage if isinstance(t, dict) and t.get("classification") == "true_positive"]
+ except Exception as e:
+ logger.error(f"[Fixer] Failed to parse triage results: {e}")
+ confirmed = []
+
+ all_vulns = confirmed + deep
+
+ if not all_vulns:
+ logger.info("[Fixer] No vulnerabilities require fixes. Exiting node.")
+ return {"fixes": "No vulnerabilities require fixes.", "status": "fixes_complete"}
+
+ logger.info(f"[Fixer] Generating patches for {len(all_vulns)} vulnerabilities")
+ llm = get_llm(model, 0.2)
+
+ try:
+ # Reconstruct code for fixer
+ code = state.get("code", "")
+ files_data = state.get("files_data", "")
+ parsed = parse_multi_file_input(code, files_data)
+ code_text = "\n\n".join(f"### {f.path}\n```{f.language}\n{f.content[:4000]}\n```" for f in parsed[:5])
+
+ response = llm.invoke([
+ SystemMessage(content=FIXER_PROMPT),
+ HumanMessage(content=(
+ f"## Source Code:\n{code_text}\n\n"
+ f"## Vulnerabilities:\n```json\n{json.dumps(all_vulns[:15], indent=2)}\n```\n"
+ f"Generate secure code patches for each vulnerability."
+ )),
+ ])
+ return {"fixes": response.content, "status": "fixes_complete"}
+ except Exception as e:
+ logger.error(f"[Fixer] LLM call failed: {e}")
+ return {"fixes": "Fix generation timed out. See findings above for remediation guidance.", "status": "fixes_complete"}
+
+
+# ─── Node 4: Reporter ────────────────────────────────────────────────
+
+def reporter_node(state: SecurityState) -> dict:
+ logger.info("[Reporter] Compiling final report...")
+ llm = get_llm(REPORTER_MODEL, 0.3)
+
+ triage = state.get("triage_results", [])
+ deep = state.get("deep_findings", [])
+ confirmed = [t for t in triage if t.get("classification") == "true_positive"]
+ dismissed = [t for t in triage if t.get("classification") == "false_positive"]
+
+ # Cross-file sections for the report
+ cross_file_summary = ""
+ taint_chains = state.get("taint_chains", [])
+ if taint_chains:
+ cross_file_summary += f"\n## Cross-File Taint Chains ({len(taint_chains)}):\n```json\n{json.dumps(taint_chains[:5], indent=2)}\n```\n"
+ cve_findings = state.get("cve_findings", [])
+ if cve_findings:
+ cross_file_summary += f"\n## Known CVEs ({len(cve_findings)}):\n```json\n{json.dumps(cve_findings, indent=2)}\n```\n"
+ config_risks = state.get("config_risks", [])
+ if config_risks:
+ cross_file_summary += f"\n## Config Risks ({len(config_risks)}):\n```json\n{json.dumps(config_risks, indent=2)}\n```\n"
+ decoded = state.get("decoded_payloads", [])
+ if decoded:
+ cross_file_summary += f"\n## Decoded Obfuscated Payloads ({len(decoded)}):\n```json\n{json.dumps(decoded, indent=2)}\n```\n"
+
+ # Include exec/eval and timing findings directly in report
+ exec_eval = state.get("exec_eval_findings", [])
+ if exec_eval:
+ cross_file_summary += f"\n## Dangerous exec/eval Calls ({len(exec_eval)}):\n```json\n{json.dumps(exec_eval, indent=2)}\n```\n"
+ timing = state.get("timing_findings", [])
+ if timing:
+ cross_file_summary += f"\n## Timing Attack Vulnerabilities ({len(timing)}):\n```json\n{json.dumps(timing, indent=2)}\n```\n"
+
+ # Manual review section for the last 5%
+ manual_review = "\n## 🔍 Manual Review Required\n"
+ manual_review += "These areas require human security engineer review:\n"
+ manual_review += "- All exec/eval calls (even if flagged above)\n"
+ manual_review += "- All authentication/authorization flows\n"
+ manual_review += "- All payment/financial logic\n"
+ manual_review += "- Any file with low LLM confidence\n"
+
+ try:
+ response = llm.invoke([
+ SystemMessage(content=REPORTER_PROMPT),
+ HumanMessage(content=(
+ f"## Scanner Summary\n{state.get('scanner_summary', 'N/A')}\n"
+ f"## Files Scanned: {state.get('file_count', 1)}\n\n"
+ f"## Confirmed Vulnerabilities ({len(confirmed)}):\n```json\n{json.dumps(confirmed, indent=2)}\n```\n\n"
+ f"## Deep Findings ({len(deep)}):\n```json\n{json.dumps(deep, indent=2)}\n```\n\n"
+ f"## Dismissed ({len(dismissed)}):\n```json\n{json.dumps(dismissed, indent=2)}\n```\n\n"
+ f"{cross_file_summary}\n"
+ f"{manual_review}\n"
+ f"## Fixes:\n{state.get('fixes', 'None')}\n\n"
+ f"## Severity Score: {state.get('severity_score', 0)}\n"
+ f"Compile the final security audit report. INCLUDE the Manual Review Required section at the end."
+ )),
+ ])
+ return {"final_report": response.content, "status": "complete"}
+ except Exception as e:
+ logger.error(f"[Reporter] LLM call failed: {e}")
+ # Fallback: build a basic report from the data we have
+ fallback = f"# 🛡️ Security Audit Report\n\n"
+ fallback += f"## Summary\nScanned {state.get('file_count', 0)} files. Found {len(confirmed)} confirmed vulnerabilities.\n"
+ fallback += f"Severity Score: {state.get('severity_score', 0)}\n\n"
+ fallback += f"## Confirmed Vulnerabilities\n"
+ for i, v in enumerate(confirmed[:30], 1):
+ fallback += f"{i}. **{v.get('title', 'Unknown')}** — {v.get('file', '')}:{v.get('line', '')} ({v.get('cwe_id', '')})\n"
+ if deep:
+ fallback += f"\n## Deep Analysis Findings\n"
+ for i, d in enumerate(deep, 1):
+ fallback += f"{i}. **{d.get('title', 'Unknown')}** — {d.get('severity', '')}\n"
+ fallback += f"\n{cross_file_summary}\n"
+ return {"final_report": fallback, "status": "complete"}
+
+
+# ─── Graph Builder ────────────────────────────────────────────────────
+
+def build_graph():
+ logger.info("[Graph] Building StateGraph")
+ wf = StateGraph(SecurityState)
+ wf.add_node("scanner", scanner_node)
+ wf.add_node("auditor", auditor_node)
+ wf.add_node("fixer", fixer_node)
+ wf.add_node("reporter", reporter_node)
+
+ wf.set_entry_point("scanner")
+ wf.add_edge("scanner", "auditor")
+
+ def after_auditor(state: SecurityState):
+ logger.info(f"[Graph] Edge from auditor. Moving to fixer.")
+ return "fixer"
+
+ wf.add_edge("auditor", "fixer")
+ wf.add_edge("fixer", "reporter")
+ wf.add_edge("reporter", END)
+
+ logger.info("[Graph] Graph compiled.")
+ return wf.compile()
diff --git a/services/python-tools/tools/code-security-scanner-v2/sec_config.py b/services/python-tools/tools/code-security-scanner-v2/sec_config.py
new file mode 100644
index 0000000..f318447
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/sec_config.py
@@ -0,0 +1,61 @@
+"""
+Code Security Scanner V2 — Configuration
+==========================================
+Model assignments per agent role, API config, and severity thresholds.
+"""
+
+import os
+
+# ─── API Configuration ─────────────────────────────────────────────────
+OXLO_API_KEY = os.getenv("OXLO_API_KEY", "")
+OXLO_BASE_URL = os.getenv("OXLO_BASE_URL", "https://api.oxlo.ai/v1")
+
+# ─── OSV (CVE) API ────────────────────────────────────────────────────
+OSV_API_URL = "https://api.osv.dev/v1/query"
+
+# ─── Model Assignments ────────────────────────────────────────────────
+SCANNER_MODEL = None # No LLM — pure deterministic
+AUDITOR_MODEL = "deepseek-r1-0528"
+FIXER_MODEL = "llama-3.3-70b" # Fast model — fixer just formats patches, no deep reasoning needed
+REPORTER_MODEL = "llama-3.3-70b"
+
+# ─── Severity Thresholds ──────────────────────────────────────────────
+SEVERITY_WEIGHTS = {
+ "CRITICAL": 10,
+ "HIGH": 7,
+ "MEDIUM": 4,
+ "LOW": 1,
+}
+
+# ─── Multi-File Limits ────────────────────────────────────────────────
+MAX_FILES = 50
+MAX_TOTAL_SIZE_MB = 10
+MAX_SINGLE_FILE_BYTES = 500_000 # 500KB per file
+
+# ─── Supported Languages ──────────────────────────────────────────────
+LANGUAGE_EXTENSIONS = {
+ "python": ".py",
+ "javascript": ".js",
+ "typescript": ".ts",
+ "go": ".go",
+ "java": ".java",
+ "c": ".c",
+ "cpp": ".cpp",
+ "ruby": ".rb",
+ "php": ".php",
+ "rust": ".rs",
+}
+
+# Extensions that are config/dependency files (not code)
+CONFIG_EXTENSIONS = {
+ ".env", ".ini", ".cfg", ".toml", ".yaml", ".yml", ".json",
+ ".lock", ".txt", # requirements.txt, package-lock.json, etc.
+}
+
+CONFIG_FILENAMES = {
+ "settings.py", "config.py", "manage.py",
+ ".env", ".env.local", ".env.production",
+ "requirements.txt", "Pipfile", "setup.py", "pyproject.toml",
+ "package.json", "package-lock.json", "yarn.lock",
+ "Gemfile", "composer.json", "go.mod", "Cargo.toml",
+}
diff --git a/services/python-tools/tools/code-security-scanner-v2/sec_cross_file.py b/services/python-tools/tools/code-security-scanner-v2/sec_cross_file.py
new file mode 100644
index 0000000..7fcba86
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/sec_cross_file.py
@@ -0,0 +1,1055 @@
+"""
+Code Security Scanner V2 — Cross-File Analysis
+================================================
+This module contains the scanners that make V2 STRICTLY BETTER than V1:
+- Import graph building (cross-file dependency mapping)
+- Cross-file taint tracking (user input → sanitize → execute)
+- Config correlator (settings.py + views.py = combined risk)
+- Obfuscation decoder (base64, rot13, unicode, hex)
+- Dependency CVE scanner (osv.dev API lookup)
+
+These capabilities are IMPOSSIBLE in a single-prompt V1 tool.
+"""
+
+import ast
+import base64
+import codecs
+import json
+import logging
+import re
+from dataclasses import dataclass, asdict
+from typing import Optional
+
+from sec_config import OSV_API_URL
+from sec_file_parser import ParsedFile
+
+logger = logging.getLogger("security-scanner")
+
+
+# ─── Data Structures ──────────────────────────────────────────────────
+
+@dataclass
+class ImportEdge:
+ """A single import relationship between two files."""
+ source_file: str # File that imports
+ target_module: str # Module being imported
+ imported_names: list[str] # Specific names imported (or ["*"])
+ line_number: int
+
+ def to_dict(self) -> dict:
+ return asdict(self)
+
+
+@dataclass
+class TaintChain:
+ """A traced path of tainted data across files."""
+ source: str # Where user input enters (e.g. "request.GET")
+ source_file: str
+ source_line: int
+ steps: list[dict] # [{file, line, description}, ...]
+ sink: str # Where it ends up (e.g. "cursor.execute()")
+ sink_file: str
+ sink_line: int
+ vulnerability: str # What this enables (e.g. "SQL Injection")
+ severity: str
+
+ def to_dict(self) -> dict:
+ return asdict(self)
+
+
+@dataclass
+class DecodedPayload:
+ """A decoded obfuscated payload."""
+ file: str
+ line_number: int
+ encoding_type: str # "base64", "rot13", "unicode_escape", "hex"
+ original_code: str
+ decoded_value: str
+ is_malicious: bool
+ severity: str
+
+ def to_dict(self) -> dict:
+ return asdict(self)
+
+
+@dataclass
+class ConfigRisk:
+ """A risky configuration setting."""
+ file: str
+ setting: str
+ value: str
+ risk: str
+ severity: str
+
+ def to_dict(self) -> dict:
+ return asdict(self)
+
+
+@dataclass
+class CVEFinding:
+ """A known CVE in a dependency."""
+ package: str
+ version: str
+ cve_id: str
+ summary: str
+ severity: str
+ file: str # requirements.txt / package.json
+
+ def to_dict(self) -> dict:
+ return asdict(self)
+
+
+# ─── Scanner: Import Graph Builder ───────────────────────────────────
+
+def build_import_graph(files: list[ParsedFile]) -> list[ImportEdge]:
+ """
+ Build a cross-file import dependency graph using Python AST.
+ Maps which files import from which modules.
+ """
+ edges = []
+
+ for f in files:
+ if f.language != "python":
+ continue
+ try:
+ tree = ast.parse(f.content)
+ except SyntaxError:
+ continue
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Import):
+ for alias in node.names:
+ edges.append(ImportEdge(
+ source_file=f.path,
+ target_module=alias.name,
+ imported_names=[alias.asname or alias.name],
+ line_number=node.lineno,
+ ))
+ elif isinstance(node, ast.ImportFrom):
+ module = node.module or ""
+ names = [alias.name for alias in node.names]
+ edges.append(ImportEdge(
+ source_file=f.path,
+ target_module=module,
+ imported_names=names,
+ line_number=node.lineno,
+ ))
+
+ logger.info(f"[ImportGraph] Built {len(edges)} import edges across {len(files)} files")
+ return edges
+
+
+# ─── Scanner: Cross-File Taint Tracker ────────────────────────────────
+
+# User input sources (Python-focused)
+TAINT_SOURCES = {
+ "request.GET", "request.POST", "request.args", "request.form",
+ "request.data", "request.json", "request.body", "request.params",
+ "request.query_params", "request.FILES", "request.headers",
+ "request.META",
+ "sys.argv", "input(", "os.environ",
+}
+
+# Celery/async task decorators — function params are untrusted input
+TASK_DECORATORS = {
+ "app.task", "shared_task", "celery.task",
+ "dramatiq.actor", "huey.task", "rq.job",
+}
+
+# Dangerous sinks
+TAINT_SINKS = {
+ "cursor.execute": "SQL Injection",
+ "os.system": "Command Injection",
+ "os.popen": "Command Injection",
+ "subprocess.call": "Command Injection",
+ "subprocess.run": "Command Injection",
+ "subprocess.Popen": "Command Injection",
+ "eval(": "Code Injection",
+ "exec(": "Code Injection",
+ "render_template_string": "SSTI (RCE)",
+ "pickle.loads": "Deserialization RCE",
+ "yaml.load": "YAML Deserialization",
+ "innerHTML": "XSS",
+ "document.write": "XSS",
+ # SSRF sinks
+ "urllib.request.urlopen": "SSRF",
+ "urlopen(": "SSRF",
+ "requests.get(": "SSRF",
+ "requests.post(": "SSRF",
+ "httpx.get(": "SSRF",
+ "httpx.post(": "SSRF",
+ "http.client.HTTPConnection": "SSRF",
+ "aiohttp.ClientSession": "SSRF",
+ # Path traversal sinks
+ "open(": "Path Traversal",
+}
+
+# Functions that look like sanitizers but aren't
+FAKE_SANITIZERS = {
+ "strip", "lower", "upper", "title", "replace",
+ "split", "join", "format", "encode", "decode",
+ "startswith", "endswith", "len", "str", "int", "float",
+}
+
+# REAL sanitizers — context-specific taint clearing
+# Key = function/method name, Value = set of contexts it actually clears
+REAL_SANITIZERS = {
+ # SQL sanitizers
+ "parameterize": {"sql"},
+ "execute": set(), # execute itself is a sink, not sanitizer
+
+ # HTML/XSS sanitizers
+ "escape": {"html"},
+ "html_escape": {"html"},
+ "markupsafe": {"html"},
+ "bleach_clean": {"html"},
+ "clean": {"html"}, # bleach.clean()
+
+ # Shell sanitizers
+ "quote": {"shell"}, # shlex.quote()
+ "shlex_quote": {"shell"},
+
+ # Path sanitizers (only if combined with startswith check)
+ "abspath": set(), # needs startswith check too
+ "realpath": set(), # needs startswith check too
+
+ # LDAP sanitizers
+ "escape_filter_chars": {"ldap"},
+}
+
+# Map vulnerability types to their taint contexts
+VULN_CONTEXT_MAP = {
+ "SQL Injection": "sql",
+ "Command Injection": "shell",
+ "Code Injection": "code",
+ "SSTI (RCE)": "html",
+ "XSS": "html",
+ "SSRF": "url",
+ "Deserialization RCE": "code",
+ "YAML Deserialization": "code",
+ "Path Traversal": "path",
+}
+
+# Security-sensitive variable name fragments for timing attack detection
+TIMING_SENSITIVE_NAMES = {
+ "password", "passwd", "pwd", "token", "secret", "key", "hash",
+ "signature", "sig", "hmac", "auth", "credential", "nonce",
+ "api_key", "apikey", "access_token",
+}
+
+
+def track_taint_flow(files: list[ParsedFile]) -> list[TaintChain]:
+ """
+ Trace user input across files to find cross-file vulnerabilities.
+
+ This is the CORE capability that makes V2 superior to V1.
+ V1 can never see two files at the same time.
+
+ FIX: Now also treats Celery/async task parameters as taint sources.
+ """
+ chains = []
+
+ # First pass: find all taint sources and sinks across all files
+ sources = [] # [(file, line, variable, source_type)]
+ sinks = [] # [(file, line, code, sink_type, vuln_type)]
+ functions = {} # {func_name: (file, line, is_named_sanitizer, is_real)}
+
+ for f in files:
+ if f.language != "python" or f.is_config:
+ continue
+
+ lines = f.content.split("\n")
+ for i, line in enumerate(lines, 1):
+ stripped = line.strip()
+
+ # Find taint sources (request objects)
+ for source in TAINT_SOURCES:
+ if source in stripped:
+ var_match = re.match(r'(\w+)\s*=\s*.*' + re.escape(source), stripped)
+ var_name = var_match.group(1) if var_match else None
+ sources.append((f.path, i, var_name, source))
+
+ # Find taint sinks
+ for sink, vuln in TAINT_SINKS.items():
+ if sink in stripped:
+ sinks.append((f.path, i, stripped[:120], sink, vuln))
+
+ # AST pass: find task decorators, sanitizers, class methods
+ try:
+ tree = ast.parse(f.content)
+ for node in ast.walk(tree):
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ # Check if function is a Celery/async task
+ for dec in node.decorator_list:
+ dec_name = ""
+ if isinstance(dec, ast.Attribute):
+ dec_name = f"{ast.dump(dec.value)}.{dec.attr}" if hasattr(dec, 'attr') else ""
+ elif isinstance(dec, ast.Name):
+ dec_name = dec.id
+ elif isinstance(dec, ast.Call):
+ if isinstance(dec.func, ast.Attribute):
+ dec_name = dec.func.attr
+ elif isinstance(dec.func, ast.Name):
+ dec_name = dec.func.id
+
+ # If this is a task decorator, treat ALL params as taint sources
+ if any(td in dec_name.lower() for td in ("task", "shared_task", "actor", "job")):
+ for arg in node.args.args:
+ if arg.arg != "self":
+ sources.append((f.path, node.lineno, arg.arg, f"@task parameter '{arg.arg}'"))
+
+ # Check if function name suggests sanitization
+ sanitizer_names = {"sanitize", "clean", "escape", "validate", "filter", "safe"}
+ is_named_sanitizer = any(s in node.name.lower() for s in sanitizer_names)
+
+ is_real = False
+ clears_contexts = set() # Which taint contexts this function clears
+ for child in ast.walk(node):
+ if isinstance(child, ast.Call):
+ func_name_inner = ""
+ if isinstance(child.func, ast.Attribute):
+ func_name_inner = child.func.attr
+ elif isinstance(child.func, ast.Name):
+ func_name_inner = child.func.id
+
+ # Check if this call is a real sanitizer
+ if func_name_inner in REAL_SANITIZERS:
+ contexts = REAL_SANITIZERS[func_name_inner]
+ if contexts: # Non-empty = actually clears something
+ is_real = True
+ clears_contexts.update(contexts)
+ elif func_name_inner not in FAKE_SANITIZERS and func_name_inner:
+ # Unknown function — could be real, mark as partial
+ pass
+
+ # Check for parameterized queries (SQL sanitization)
+ func_src = ast.get_source_segment(f.content, node) or ""
+ if any(p in func_src for p in ["%s", "$1", "?", ":param"]):
+ is_real = True
+ clears_contexts.add("sql")
+
+ functions[node.name] = (f.path, node.lineno, is_named_sanitizer, is_real, clears_contexts)
+ except SyntaxError:
+ pass
+
+ # Second pass: connect sources to sinks
+ for src_file, src_line, src_var, src_type in sources:
+ for sink_file, sink_line, sink_code, sink_type, vuln_type in sinks:
+ if src_var and src_var in sink_code:
+ steps = []
+ taint_blocked = False
+
+ # Determine what context this sink needs
+ sink_context = VULN_CONTEXT_MAP.get(vuln_type, "")
+
+ for func_name, (func_file, func_line, is_named, is_real, clears) in functions.items():
+ if is_named and not is_real:
+ steps.append({
+ "file": func_file, "line": func_line,
+ "description": f"⚠️ Function '{func_name}' is named like a sanitizer but provides NO real protection (only uses {', '.join(FAKE_SANITIZERS & set(['strip','lower','upper']))})"
+ })
+ elif is_named and is_real:
+ # Check if it clears the RIGHT context
+ if sink_context and sink_context in clears:
+ taint_blocked = True # Genuinely sanitized for this context
+ elif sink_context and sink_context not in clears:
+ steps.append({
+ "file": func_file, "line": func_line,
+ "description": f"⚠️ Function '{func_name}' sanitizes for {clears} but NOT for {sink_context} — taint still active for {vuln_type}"
+ })
+
+ if taint_blocked:
+ continue # Genuinely sanitized — skip this chain
+
+ chains.append(TaintChain(
+ source=src_type, source_file=src_file, source_line=src_line,
+ steps=steps, sink=sink_type, sink_file=sink_file, sink_line=sink_line,
+ vulnerability=vuln_type,
+ severity="CRITICAL" if vuln_type in ("SQL Injection", "Command Injection", "Code Injection", "SSTI (RCE)", "SSRF") else "HIGH",
+ ))
+
+ logger.info(f"[TaintTracker] Found {len(chains)} taint chains across {len(files)} files")
+ return chains
+
+
+# ─── Scanner: exec/eval on Non-Literal Variables (Drawback 1 fix) ─────
+
+def scan_exec_eval_dangers(files: list[ParsedFile]) -> list[dict]:
+ """
+ Flag exec()/eval() on ANY non-literal variable.
+ Rule: if exec(x) exists and x is not a string literal, it's CRITICAL.
+ Also detects: b64decode + exec in same function, __import__, compile+exec.
+ try/except does NOT reduce severity — it INCREASES it.
+ """
+ findings = []
+
+ for f in files:
+ if f.language != "python" or f.is_config:
+ continue
+
+ try:
+ tree = ast.parse(f.content)
+ except SyntaxError:
+ continue
+
+ for node in ast.walk(tree):
+ # Check functions/methods for exec/eval patterns
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ has_b64decode = False
+ has_compile = False
+ has_exec_eval = False
+ exec_line = 0
+ in_try = False
+
+ for child in ast.walk(node):
+ # Track b64decode, compile calls
+ if isinstance(child, ast.Call):
+ call_name = _get_call_name(child)
+ if "b64decode" in call_name or "base64" in call_name:
+ has_b64decode = True
+ if call_name == "compile":
+ has_compile = True
+
+ # Track try/except blocks
+ if isinstance(child, ast.Try):
+ in_try = True
+
+ # Find exec/eval calls specifically
+ for child in ast.walk(node):
+ if isinstance(child, ast.Call):
+ call_name = _get_call_name(child)
+ if call_name in ("exec", "eval"):
+ has_exec_eval = True
+ exec_line = getattr(child, 'lineno', node.lineno)
+
+ # Check if argument is a non-literal
+ is_literal = False
+ if child.args:
+ arg = child.args[0]
+ if isinstance(arg, (ast.Constant, ast.Str)):
+ is_literal = True
+
+ if not is_literal:
+ desc = f"exec/eval on non-literal variable in {node.name}()"
+ if in_try:
+ desc += " — HIDDEN inside try/except (errors silently swallowed)"
+ if has_b64decode:
+ desc += " — combined with base64 decoding (OBFUSCATED BACKDOOR)"
+
+ findings.append({
+ "file": f.path,
+ "line": exec_line,
+ "severity": "CRITICAL",
+ "category": "Code Injection / Backdoor",
+ "title": f"{call_name}() on externally-controlled input",
+ "description": desc,
+ "cwe_id": "CWE-95",
+ "source": "deterministic_scanner",
+ })
+
+ # b64decode + exec/eval in same function = CRITICAL
+ if has_b64decode and has_exec_eval and not any(
+ fd.get("description", "").endswith("(OBFUSCATED BACKDOOR)")
+ for fd in findings if fd.get("file") == f.path
+ ):
+ findings.append({
+ "file": f.path,
+ "line": exec_line or node.lineno,
+ "severity": "CRITICAL",
+ "category": "Obfuscated Backdoor",
+ "title": f"base64 decode + exec/eval in {node.name}()",
+ "description": f"Function {node.name}() combines base64 decoding with code execution — obfuscated backdoor pattern",
+ "cwe_id": "CWE-506",
+ "source": "deterministic_scanner",
+ })
+
+ # compile + exec in same function = CRITICAL
+ if has_compile and has_exec_eval:
+ findings.append({
+ "file": f.path,
+ "line": exec_line or node.lineno,
+ "severity": "CRITICAL",
+ "category": "Dynamic Code Execution",
+ "title": f"compile() + exec() in {node.name}()",
+ "description": "compile() + exec() pattern enables arbitrary code execution",
+ "cwe_id": "CWE-95",
+ "source": "deterministic_scanner",
+ })
+
+ # __import__ anywhere = CRITICAL
+ if isinstance(node, ast.Call):
+ call_name = _get_call_name(node)
+ if call_name == "__import__":
+ findings.append({
+ "file": f.path,
+ "line": getattr(node, 'lineno', 0),
+ "severity": "CRITICAL",
+ "category": "Dynamic Import",
+ "title": "__import__() call detected",
+ "description": "__import__() enables dynamic module loading — often used in backdoors",
+ "cwe_id": "CWE-95",
+ "source": "deterministic_scanner",
+ })
+
+ # Second pass: catch ALL exec/eval anywhere (including top-level and
+ # class methods that the function-level scan might have missed)
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Call):
+ call_name = _get_call_name(node)
+ if call_name in ("exec", "eval"):
+ exec_line = getattr(node, 'lineno', 0)
+ # Skip if already found at this line
+ if any(fd.get("file") == f.path and fd.get("line") == exec_line for fd in findings):
+ continue
+
+ is_literal = False
+ if node.args:
+ arg = node.args[0]
+ if isinstance(arg, (ast.Constant, ast.Str)):
+ is_literal = True
+
+ if not is_literal:
+ findings.append({
+ "file": f.path,
+ "line": exec_line,
+ "severity": "CRITICAL",
+ "category": "Code Injection / Backdoor",
+ "title": f"{call_name}() on non-literal input (module level)",
+ "description": f"{call_name}() called with dynamic variable — potential code injection",
+ "cwe_id": "CWE-95",
+ "source": "deterministic_scanner",
+ })
+
+ logger.info(f"[ExecScanner] Found {len(findings)} exec/eval dangers")
+ return findings
+
+
+def _get_call_name(node: ast.Call) -> str:
+ """Extract function name from an AST Call node."""
+ if isinstance(node.func, ast.Name):
+ return node.func.id
+ if isinstance(node.func, ast.Attribute):
+ return node.func.attr
+ return ""
+
+
+# ─── Scanner: Timing Attack Detection (Drawback 5 fix) ───────────────
+
+def scan_timing_attacks(files: list[ParsedFile]) -> list[dict]:
+ """
+ Detect == or != comparisons on security-sensitive variables.
+ Works in class methods, top-level functions, anywhere.
+ """
+ findings = []
+
+ for f in files:
+ if f.language != "python" or f.is_config:
+ continue
+
+ try:
+ tree = ast.parse(f.content)
+ except SyntaxError:
+ continue
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Compare):
+ # Check if using == or !=
+ has_eq = any(isinstance(op, (ast.Eq, ast.NotEq)) for op in node.ops)
+ if not has_eq:
+ continue
+
+ # Get all variable names in the comparison
+ names = set()
+ for child in ast.walk(node):
+ if isinstance(child, ast.Name):
+ names.add(child.id.lower())
+ elif isinstance(child, ast.Attribute):
+ names.add(child.attr.lower())
+
+ # Check if any name is security-sensitive
+ for name in names:
+ if any(sensitive in name for sensitive in TIMING_SENSITIVE_NAMES):
+ findings.append({
+ "file": f.path,
+ "line": getattr(node, 'lineno', 0),
+ "severity": "HIGH",
+ "category": "Timing Attack",
+ "title": f"Timing-unsafe comparison on '{name}'",
+ "description": f"Direct == comparison on security-sensitive value '{name}'. Use hmac.compare_digest() instead.",
+ "cwe_id": "CWE-208",
+ "source": "deterministic_scanner",
+ })
+ break # One finding per comparison
+
+ logger.info(f"[TimingScanner] Found {len(findings)} timing attack risks")
+ return findings
+
+
+# ─── Scanner: TOCTOU Race Condition Detection ─────────────────────────
+
+def scan_toctou(files: list[ParsedFile]) -> list[dict]:
+ """
+ Detect Time-of-Check/Time-of-Use race conditions.
+ Pattern: os.path.exists(x) followed by open(x) in the same function.
+ Also catches: os.access() + open(), os.stat() + open().
+ """
+ findings = []
+ CHECK_FUNCS = {"exists", "isfile", "isdir", "access", "stat"}
+ USE_FUNCS = {"open", "read", "write"}
+
+ for f in files:
+ if f.language != "python" or f.is_config:
+ continue
+ try:
+ tree = ast.parse(f.content)
+ except SyntaxError:
+ continue
+
+ for node in ast.walk(tree):
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ check_lines = []
+ use_lines = []
+ has_sleep = False
+
+ for child in ast.walk(node):
+ if isinstance(child, ast.Call):
+ call_name = _get_call_name(child)
+ # os.path.exists(), os.access(), etc.
+ if call_name in CHECK_FUNCS:
+ check_lines.append(getattr(child, 'lineno', 0))
+ # time.sleep() between check and use
+ if call_name == "sleep":
+ has_sleep = True
+
+ # open() call or with open() statement
+ if isinstance(child, ast.Call):
+ call_name = _get_call_name(child)
+ if call_name in USE_FUNCS:
+ use_lines.append(getattr(child, 'lineno', 0))
+ if isinstance(child, ast.withitem) and isinstance(child.context_expr, ast.Call):
+ call_name = _get_call_name(child.context_expr)
+ if call_name in USE_FUNCS:
+ use_lines.append(getattr(child.context_expr, 'lineno', 0))
+
+ # If we found both check and use in the same function
+ if check_lines and use_lines:
+ for cl in check_lines:
+ for ul in use_lines:
+ if ul > cl: # Use happens after check
+ desc = f"TOCTOU in {node.name}(): file checked at line {cl}, used at line {ul}"
+ if has_sleep:
+ desc += " — time.sleep() between check and use INCREASES exploitation window"
+ findings.append({
+ "file": f.path,
+ "line": cl,
+ "severity": "HIGH",
+ "category": "Race Condition",
+ "title": f"TOCTOU race condition in {node.name}()",
+ "description": desc,
+ "cwe_id": "CWE-367",
+ "source": "deterministic_scanner",
+ })
+ break # One finding per function
+ else:
+ continue
+ break
+
+ logger.info(f"[TOCTOUScanner] Found {len(findings)} TOCTOU risks")
+ return findings
+
+
+# ─── Scanner: Mass Assignment Detection ───────────────────────────────
+
+def scan_mass_assignment(files: list[ParsedFile]) -> list[dict]:
+ """
+ Detect mass assignment via setattr() in loops, __dict__.update(), etc.
+ Pattern: for key, value in data.items(): setattr(obj, key, value)
+ """
+ findings = []
+
+ for f in files:
+ if f.language != "python" or f.is_config:
+ continue
+ try:
+ tree = ast.parse(f.content)
+ except SyntaxError:
+ continue
+
+ for node in ast.walk(tree):
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ # Look for for loops containing setattr
+ for child in ast.walk(node):
+ if isinstance(child, ast.For):
+ # Check if loop body contains setattr()
+ for body_node in ast.walk(child):
+ if isinstance(body_node, ast.Call):
+ call_name = _get_call_name(body_node)
+ if call_name == "setattr":
+ findings.append({
+ "file": f.path,
+ "line": getattr(child, 'lineno', node.lineno),
+ "severity": "CRITICAL",
+ "category": "Mass Assignment",
+ "title": f"Mass assignment via setattr() loop in {node.name}()",
+ "description": f"setattr() called in a for loop in {node.name}() — attacker can set arbitrary attributes (e.g. is_admin=True). Use an explicit whitelist of allowed fields.",
+ "cwe_id": "CWE-915",
+ "source": "deterministic_scanner",
+ })
+ break # One finding per loop
+
+ logger.info(f"[MassAssignmentScanner] Found {len(findings)} mass assignment risks")
+ return findings
+
+
+# ─── Scanner: Config Correlator ───────────────────────────────────────
+
+DANGEROUS_CONFIGS = {
+ # Python/Django
+ "DEBUG = True": ("Debug mode enabled in production", "HIGH"),
+ "DEBUG=True": ("Debug mode enabled in production", "HIGH"),
+ 'ALLOWED_HOSTS = ["*"]': ("All hosts allowed — enables host header attacks", "HIGH"),
+ "ALLOWED_HOSTS = ['*']": ("All hosts allowed — enables host header attacks", "HIGH"),
+ "CORS_ALLOW_ALL_ORIGINS = True": ("CORS allows all origins — enables CSRF", "HIGH"),
+ "CORS_ORIGIN_ALLOW_ALL = True": ("CORS allows all origins — enables CSRF", "HIGH"),
+ "SESSION_COOKIE_HTTPONLY = False": ("Session cookies accessible via JavaScript — enables session theft", "HIGH"),
+ "SESSION_COOKIE_SECURE = False": ("Session cookies sent over HTTP — enables interception", "HIGH"),
+ "CSRF_COOKIE_HTTPONLY = False": ("CSRF cookie accessible via JavaScript", "MEDIUM"),
+
+ # Generic
+ "SECRET_KEY": None, # Special handling below
+}
+
+# Regex patterns for configs that need fuzzy matching
+CONFIG_REGEX_PATTERNS = [
+ (re.compile(r'#.*CsrfViewMiddleware', re.I),
+ "CSRF middleware commented out / removed", "CRITICAL"),
+ (re.compile(r"SESSION_COOKIE_HTTPONLY\s*=\s*False", re.I),
+ "Session cookie HttpOnly disabled — XSS can steal sessions", "HIGH"),
+ (re.compile(r"SESSION_COOKIE_SECURE\s*=\s*False", re.I),
+ "Session cookie Secure flag disabled", "MEDIUM"),
+ (re.compile(r"CORS_ALLOW_CREDENTIALS\s*=\s*True", re.I),
+ "CORS allows credentials — combined with permissive origins this enables session theft", "HIGH"),
+ # Celery pickle serializer = deserialization RCE
+ (re.compile(r'''task_serializer\s*=\s*['"]pickle['"]''', re.I),
+ "Celery task serializer set to pickle — enables deserialization RCE", "CRITICAL"),
+ (re.compile(r"accept_content\s*=.*pickle", re.I),
+ "Celery accepts pickle content — enables deserialization RCE", "CRITICAL"),
+ (re.compile(r'''result_serializer\s*=\s*['"]pickle['"]''', re.I),
+ "Celery result serializer set to pickle — enables deserialization RCE", "CRITICAL"),
+ # JWT algorithm confusion — only match when near algorithm context
+ (re.compile(r'''ALLOWED.*ALGORITHMS?\s*=\s*\[.*['"]none['"]''', re.I),
+ "JWT algorithm list includes 'none' — attackers can forge tokens", "CRITICAL"),
+ (re.compile(r'''algorithms?\s*=\s*\[.*['"]none['"]''', re.I),
+ "JWT algorithms parameter includes 'none' — enables token forgery", "CRITICAL"),
+]
+
+SECRET_PATTERNS = [
+ (re.compile(r"""(?:SECRET_KEY|JWT_SECRET|API_SECRET)\s*=\s*['"]([^'"]{4,40})['"]""", re.I),
+ "Hardcoded secret key", "CRITICAL"),
+ (re.compile(r"""(?:SECRET_KEY|JWT_SECRET)\s*=\s*['"](?:django-insecure|changeme|secret|password|test)""", re.I),
+ "Weak/default secret key", "CRITICAL"),
+ (re.compile(r"""(?:DB_PASSWORD|DATABASE_PASSWORD|DATABASES.*PASSWORD)\s*[:=]\s*['"]([^'"]{3,})""", re.I),
+ "Hardcoded database password", "CRITICAL"),
+ (re.compile(r"""(?:AWS_SECRET_ACCESS_KEY|aws_secret_access_key)\s*=\s*['"]([^'"]{16,})""", re.I),
+ "Hardcoded AWS credentials", "CRITICAL"),
+]
+
+
+def correlate_configs(files: list[ParsedFile]) -> list[ConfigRisk]:
+ """
+ Analyze config files for dangerous settings that amplify
+ vulnerabilities in code files.
+ Also scans Python code files for inline config (settings.py etc).
+ """
+ risks = []
+
+ # Scan config files AND Python settings files
+ scannable = [f for f in files if f.is_config or f.filename in ('settings.py', 'config.py', 'manage.py')]
+
+ for f in scannable:
+ # Check for dangerous config patterns (exact match)
+ for pattern, risk_info in DANGEROUS_CONFIGS.items():
+ if risk_info and pattern in f.content:
+ desc, severity = risk_info
+ risks.append(ConfigRisk(
+ file=f.path, setting=pattern.split("=")[0].strip(),
+ value=pattern, risk=desc, severity=severity,
+ ))
+
+ # Check for regex config patterns (fuzzy match)
+ for regex, desc, severity in CONFIG_REGEX_PATTERNS:
+ for match in regex.finditer(f.content):
+ risks.append(ConfigRisk(
+ file=f.path, setting=match.group(0)[:60],
+ value="[matched]", risk=desc, severity=severity,
+ ))
+
+ # Check for secret patterns
+ for regex, desc, severity in SECRET_PATTERNS:
+ for match in regex.finditer(f.content):
+ risks.append(ConfigRisk(
+ file=f.path, setting=match.group(0)[:40],
+ value="[REDACTED]", risk=desc, severity=severity,
+ ))
+
+ logger.info(f"[ConfigCorrelator] Found {len(risks)} config risks in {len(scannable)} files")
+ return risks
+
+
+# ─── Scanner: Obfuscation Decoder ─────────────────────────────────────
+
+# Keywords that indicate a decoded payload is malicious
+_MALICIOUS_KEYWORDS = [
+ "os.system", "rm -rf", "curl", "wget", "import os", "eval(", "exec(",
+ "subprocess", "__import__", "socket", "connect", "reverse", "shell",
+ "chmod", "passwd", "shadow", "ssh", "nc ", "ncat",
+]
+
+
+def decode_obfuscation(files: list[ParsedFile]) -> list[DecodedPayload]:
+ """
+ Detect and decode obfuscated payloads that V1 would completely miss.
+ Handles: base64, rot13, unicode escapes, hex strings.
+
+ CRITICAL FIX: Now detects exec/eval + b64decode(VARIABLE) patterns,
+ not just string literals. This catches the backdoor pattern:
+ code = base64.b64decode(some_var).decode()
+ exec(code)
+ """
+ payloads = []
+
+ for f in files:
+ if f.is_config:
+ continue
+
+ lines = f.content.split("\n")
+
+ # Multi-line scan: track b64decode results used in exec/eval
+ b64_vars = {} # {var_name: line_number}
+
+ for i, line in enumerate(lines, 1):
+ stripped = line.strip()
+
+ # Pattern 1: b64decode("STRING_LITERAL")
+ b64_match = re.search(r'b64decode\s*\(\s*["\']([A-Za-z0-9+/=]{16,})["\']', stripped)
+ if b64_match:
+ try:
+ decoded = base64.b64decode(b64_match.group(1)).decode("utf-8", errors="replace")
+ is_bad = any(kw in decoded.lower() for kw in _MALICIOUS_KEYWORDS)
+ payloads.append(DecodedPayload(
+ file=f.path, line_number=i, encoding_type="base64",
+ original_code=stripped[:120], decoded_value=decoded[:200],
+ is_malicious=is_bad, severity="CRITICAL" if is_bad else "HIGH",
+ ))
+ except Exception:
+ pass
+
+ # Pattern 2: var = b64decode(ANYTHING) — track for exec/eval
+ b64_var_match = re.search(r'(\w+)\s*=\s*.*b64decode\s*\(', stripped)
+ if b64_var_match:
+ b64_vars[b64_var_match.group(1)] = i
+ # Also track: var = ...decode() after b64decode
+ b64_var_match2 = re.search(r'(\w+)\s*=\s*.*b64decode\s*\(.+?\.decode\(', stripped)
+ if b64_var_match2:
+ b64_vars[b64_var_match2.group(1)] = i
+
+ # Pattern 3: exec(VAR) or eval(VAR) where VAR came from b64decode
+ exec_match = re.search(r'(?:exec|eval)\s*\(\s*(\w+)', stripped)
+ if exec_match:
+ var_name = exec_match.group(1)
+ if var_name in b64_vars:
+ payloads.append(DecodedPayload(
+ file=f.path, line_number=i, encoding_type="base64_exec",
+ original_code=f"exec/eval of base64-decoded variable '{var_name}' (decoded at line {b64_vars[var_name]})",
+ decoded_value=f"OBFUSCATED BACKDOOR: Variable '{var_name}' is base64-decoded then executed. This is a remote code execution backdoor.",
+ is_malicious=True, severity="CRITICAL",
+ ))
+ # Also flag exec/eval with inline b64decode
+ if 'b64decode' in stripped or 'base64' in stripped:
+ payloads.append(DecodedPayload(
+ file=f.path, line_number=i, encoding_type="base64_exec",
+ original_code=stripped[:120],
+ decoded_value="OBFUSCATED BACKDOOR: exec/eval with base64-decoded input — remote code execution",
+ is_malicious=True, severity="CRITICAL",
+ ))
+
+ # Pattern 4: exec/eval inside try/except (hidden backdoor)
+ if ('exec(' in stripped or 'eval(' in stripped) and 'except' not in stripped:
+ # Check if we're inside a try/except block (look at surrounding lines)
+ in_try = False
+ for j in range(max(0, i-5), i):
+ if 'try:' in lines[j-1] if j > 0 else '':
+ in_try = True
+ for j in range(i, min(len(lines), i+3)):
+ if 'except' in lines[j-1] if j > 0 else '':
+ in_try = True
+
+ if in_try and any(kw in f.content[max(0, f.content.find(stripped)-200):f.content.find(stripped)+50]
+ for kw in ['b64decode', 'base64', 'decode(']):
+ payloads.append(DecodedPayload(
+ file=f.path, line_number=i, encoding_type="hidden_exec",
+ original_code=stripped[:120],
+ decoded_value="HIDDEN BACKDOOR: exec/eval with encoded input inside try/except — errors silently swallowed",
+ is_malicious=True, severity="CRITICAL",
+ ))
+
+ # Pattern 5: codecs.decode("...", "rot13")
+ rot13_match = re.search(r'codecs\.decode\s*\(\s*["\'](.+?)["\']\s*,\s*["\']rot.?13', stripped)
+ if rot13_match:
+ try:
+ decoded = codecs.decode(rot13_match.group(1), "rot13")
+ is_bad = any(kw in decoded.lower() for kw in _MALICIOUS_KEYWORDS)
+ payloads.append(DecodedPayload(
+ file=f.path, line_number=i, encoding_type="rot13",
+ original_code=stripped[:120], decoded_value=decoded[:200],
+ is_malicious=is_bad, severity="CRITICAL" if is_bad else "HIGH",
+ ))
+ except Exception:
+ pass
+
+ # Pattern 6: Unicode escape sequences
+ unicode_match = re.search(r'["\']((\\x[0-9a-fA-F]{2}){4,})["\']', stripped)
+ if unicode_match:
+ try:
+ decoded = bytes(unicode_match.group(1), "utf-8").decode("unicode_escape")
+ is_bad = any(kw in decoded.lower() for kw in _MALICIOUS_KEYWORDS)
+ payloads.append(DecodedPayload(
+ file=f.path, line_number=i, encoding_type="unicode_escape",
+ original_code=stripped[:120], decoded_value=decoded[:200],
+ is_malicious=is_bad, severity="CRITICAL" if is_bad else "MEDIUM",
+ ))
+ except Exception:
+ pass
+
+ # Pattern 7: Hex-encoded strings
+ hex_match = re.search(r'fromhex\s*\(\s*["\']([0-9a-fA-F]{8,})["\']', stripped)
+ if hex_match:
+ try:
+ decoded = bytes.fromhex(hex_match.group(1)).decode("utf-8", errors="replace")
+ is_bad = any(kw in decoded.lower() for kw in _MALICIOUS_KEYWORDS)
+ payloads.append(DecodedPayload(
+ file=f.path, line_number=i, encoding_type="hex",
+ original_code=stripped[:120], decoded_value=decoded[:200],
+ is_malicious=is_bad, severity="CRITICAL" if is_bad else "MEDIUM",
+ ))
+ except Exception:
+ pass
+
+ logger.info(f"[ObfuscationDecoder] Decoded {len(payloads)} obfuscated payloads")
+ return payloads
+
+
+# ─── Scanner: Dependency CVE Lookup (osv.dev) ─────────────────────────
+
+def scan_dependencies_for_cves(files: list[ParsedFile]) -> list[CVEFinding]:
+ """
+ Check requirements.txt / package.json against the OSV.dev API
+ for known CVEs. This is IMPOSSIBLE for V1.
+ """
+ findings = []
+
+ for f in files:
+ if f.filename == "requirements.txt":
+ findings.extend(_check_python_deps(f))
+ elif f.filename == "package.json":
+ findings.extend(_check_npm_deps(f))
+
+ logger.info(f"[CVEScanner] Found {len(findings)} known CVEs")
+ return findings
+
+
+def _check_python_deps(f: ParsedFile) -> list[CVEFinding]:
+ """Query OSV for Python package CVEs."""
+ import httpx
+
+ findings = []
+ for line in f.content.split("\n"):
+ line = line.strip()
+ if not line or line.startswith("#") or line.startswith("-"):
+ continue
+
+ # Parse: package==version or package>=version
+ match = re.match(r'^([a-zA-Z0-9_.-]+)\s*[=<>!~]+\s*([0-9][0-9a-zA-Z.*-]*)', line)
+ if not match:
+ continue
+
+ pkg, version = match.group(1), match.group(2)
+
+ try:
+ resp = httpx.post(
+ OSV_API_URL,
+ json={"package": {"name": pkg, "ecosystem": "PyPI"}, "version": version},
+ timeout=5.0,
+ )
+ if resp.status_code == 200:
+ data = resp.json()
+ for vuln in data.get("vulns", [])[:3]: # Limit to 3 per package
+ cve_ids = [a for a in vuln.get("aliases", []) if a.startswith("CVE-")]
+ severity = _osv_severity(vuln)
+ findings.append(CVEFinding(
+ package=pkg, version=version,
+ cve_id=cve_ids[0] if cve_ids else vuln.get("id", "Unknown"),
+ summary=vuln.get("summary", "Known vulnerability")[:200],
+ severity=severity, file=f.path,
+ ))
+ except Exception as e:
+ logger.warning(f"[CVE] Failed to query OSV for {pkg}=={version}: {e}")
+
+ return findings
+
+
+def _check_npm_deps(f: ParsedFile) -> list[CVEFinding]:
+ """Query OSV for npm package CVEs."""
+ import httpx
+
+ findings = []
+ try:
+ pkg_json = json.loads(f.content)
+ except json.JSONDecodeError:
+ return findings
+
+ all_deps = {**pkg_json.get("dependencies", {}), **pkg_json.get("devDependencies", {})}
+
+ for pkg, version_spec in list(all_deps.items())[:20]: # Limit to 20 packages
+ version = re.sub(r'[^0-9.]', '', version_spec) # Strip ^, ~, etc.
+ if not version:
+ continue
+
+ try:
+ resp = httpx.post(
+ OSV_API_URL,
+ json={"package": {"name": pkg, "ecosystem": "npm"}, "version": version},
+ timeout=5.0,
+ )
+ if resp.status_code == 200:
+ data = resp.json()
+ for vuln in data.get("vulns", [])[:2]:
+ cve_ids = [a for a in vuln.get("aliases", []) if a.startswith("CVE-")]
+ severity = _osv_severity(vuln)
+ findings.append(CVEFinding(
+ package=pkg, version=version,
+ cve_id=cve_ids[0] if cve_ids else vuln.get("id", "Unknown"),
+ summary=vuln.get("summary", "Known vulnerability")[:200],
+ severity=severity, file=f.path,
+ ))
+ except Exception as e:
+ logger.warning(f"[CVE] Failed to query OSV for {pkg}@{version}: {e}")
+
+ return findings
+
+
+def _osv_severity(vuln: dict) -> str:
+ """Extract severity from OSV vulnerability data."""
+ for sev_obj in vuln.get("severity", []):
+ score = sev_obj.get("score", "")
+ if ":" in score:
+ # CVSS vector — extract base score
+ cvss_match = re.search(r'AV:[NL].*', score)
+ if cvss_match:
+ return "CRITICAL" if "AV:N" in score else "HIGH"
+
+ # Fallback based on database severity
+ db_severity = vuln.get("database_specific", {}).get("severity", "").upper()
+ if db_severity in ("CRITICAL", "HIGH", "MEDIUM", "LOW"):
+ return db_severity
+ return "HIGH" # Default to HIGH for known CVEs
diff --git a/services/python-tools/tools/code-security-scanner-v2/sec_file_parser.py b/services/python-tools/tools/code-security-scanner-v2/sec_file_parser.py
new file mode 100644
index 0000000..8ba5942
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/sec_file_parser.py
@@ -0,0 +1,156 @@
+"""
+Code Security Scanner V2 — Multi-File Parser
+==============================================
+Handles splitting multi-file input, extracting ZIP archives,
+and organizing files for cross-file analysis.
+"""
+
+import base64
+import io
+import os
+import re
+import zipfile
+import logging
+from dataclasses import dataclass
+
+from sec_config import MAX_FILES, MAX_SINGLE_FILE_BYTES, CONFIG_FILENAMES, CONFIG_EXTENSIONS
+
+logger = logging.getLogger("security-scanner")
+
+
+@dataclass
+class ParsedFile:
+ """A single file extracted from multi-file input."""
+ path: str # e.g. "src/auth.py" or "utils.py"
+ filename: str # e.g. "auth.py"
+ content: str
+ language: str # Inferred from extension
+ is_config: bool # True for settings.py, .env, requirements.txt, etc.
+ size_bytes: int
+
+
+def infer_language(filename: str) -> str:
+ """Infer programming language from filename extension."""
+ ext_map = {
+ ".py": "python", ".js": "javascript", ".ts": "typescript",
+ ".go": "go", ".java": "java", ".c": "c", ".cpp": "c",
+ ".rb": "ruby", ".php": "php", ".rs": "rust",
+ ".jsx": "javascript", ".tsx": "typescript", ".mjs": "javascript",
+ }
+ _, ext = os.path.splitext(filename.lower())
+ return ext_map.get(ext, "unknown")
+
+
+def is_config_file(filepath: str) -> bool:
+ """Check if a file is a configuration/dependency file."""
+ basename = os.path.basename(filepath).lower()
+ if basename in CONFIG_FILENAMES:
+ return True
+ _, ext = os.path.splitext(basename)
+ if ext in CONFIG_EXTENSIONS:
+ return True
+ return False
+
+
+def parse_multi_file_input(code: str = "", files_data: str = "") -> list[ParsedFile]:
+ """
+ Parse input into individual files.
+
+ Supports three input modes:
+ 1. Single code paste (backward compatible) → one file
+ 2. Multi-file paste with --- FILE: path --- markers → multiple files
+ 3. ZIP archive (base64 with __ZIP__: prefix) → extracted files
+ """
+ # Priority: files_data > code
+ raw = files_data.strip() if files_data.strip() else code.strip()
+
+ if not raw:
+ return []
+
+ # Mode 3: ZIP archive
+ if raw.startswith("__ZIP__:"):
+ return _extract_zip(raw[8:])
+
+ # Mode 2: Multi-file with markers
+ if "--- FILE:" in raw:
+ return _parse_file_markers(raw)
+
+ # Mode 1: Single code paste
+ return [ParsedFile(
+ path="input.py",
+ filename="input.py",
+ content=raw,
+ language="python",
+ is_config=False,
+ size_bytes=len(raw.encode("utf-8")),
+ )]
+
+
+def _extract_zip(base64_data: str) -> list[ParsedFile]:
+ """Extract files from a base64-encoded ZIP archive."""
+ files = []
+ try:
+ zip_bytes = base64.b64decode(base64_data)
+ with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf:
+ for info in zf.infolist():
+ # Skip directories, hidden files, __pycache__, node_modules
+ if info.is_dir():
+ continue
+ basename = os.path.basename(info.filename)
+ if basename.startswith(".") or "__pycache__" in info.filename:
+ continue
+ if "node_modules" in info.filename or ".git/" in info.filename:
+ continue
+ if info.file_size > MAX_SINGLE_FILE_BYTES:
+ logger.warning(f"[ZIP] Skipping {info.filename}: too large ({info.file_size} bytes)")
+ continue
+ if len(files) >= MAX_FILES:
+ logger.warning(f"[ZIP] Max file limit ({MAX_FILES}) reached, stopping extraction")
+ break
+
+ try:
+ content = zf.read(info.filename).decode("utf-8", errors="replace")
+ files.append(ParsedFile(
+ path=info.filename,
+ filename=basename,
+ content=content,
+ language=infer_language(basename),
+ is_config=is_config_file(info.filename),
+ size_bytes=info.file_size,
+ ))
+ except Exception as e:
+ logger.warning(f"[ZIP] Failed to read {info.filename}: {e}")
+ except Exception as e:
+ logger.error(f"[ZIP] Failed to extract archive: {e}")
+
+ logger.info(f"[Parser] Extracted {len(files)} files from ZIP")
+ return files
+
+
+def _parse_file_markers(raw: str) -> list[ParsedFile]:
+ """Parse multi-file input separated by --- FILE: path --- markers."""
+ files = []
+ pattern = re.compile(r"--- FILE:\s*(.+?)\s*---\n?", re.MULTILINE)
+
+ parts = pattern.split(raw)
+ # parts = [before_first_marker, path1, content1, path2, content2, ...]
+
+ i = 1 # Skip any content before the first marker
+ while i < len(parts) - 1:
+ filepath = parts[i].strip()
+ content = parts[i + 1].strip()
+ basename = os.path.basename(filepath)
+
+ if content and content != "[Binary file — skipped]":
+ files.append(ParsedFile(
+ path=filepath,
+ filename=basename,
+ content=content,
+ language=infer_language(basename),
+ is_config=is_config_file(filepath),
+ size_bytes=len(content.encode("utf-8")),
+ ))
+ i += 2
+
+ logger.info(f"[Parser] Parsed {len(files)} files from markers")
+ return files
diff --git a/services/python-tools/tools/code-security-scanner-v2/sec_prompts.py b/services/python-tools/tools/code-security-scanner-v2/sec_prompts.py
new file mode 100644
index 0000000..dfb60c9
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/sec_prompts.py
@@ -0,0 +1,179 @@
+"""
+Code Security Scanner V2 — Agent Prompts
+==========================================
+v3.0: Enhanced for multi-file cross-file analysis.
+"""
+
+AUDITOR_PROMPT = """You are a world-class Application Security Auditor performing a comprehensive security audit.
+
+You are part of an AGENTIC pipeline with capabilities a single-prompt tool CANNOT have:
+- Deterministic scanners have already analyzed the code (findings provided below)
+- Cross-file import graphs show how modules depend on each other
+- Taint chains trace user input across multiple files to dangerous sinks
+- Config correlator has flagged risky settings that amplify code vulnerabilities
+- Obfuscation decoder has pre-decoded base64/rot13/unicode/hex payloads
+- CVE scanner has checked dependencies against the OSV vulnerability database
+
+YOUR JOB IS THREEFOLD:
+
+## 1. TRIAGE the scanner findings
+For each scanner finding, classify it:
+- `true_positive` — genuine vulnerability
+- `false_positive` — safe pattern, not exploitable
+- `needs_investigation` — unclear without runtime context
+
+### CRITICAL TRIAGE RULES:
+- Findings tagged with `"source": "deterministic_scanner"` are ALWAYS true_positive. You CANNOT dismiss them.
+- To classify anything as false_positive, you MUST cite a specific line of code proving it's safe (e.g. "parameterized query at line 42" or "sanitized by bleach.clean() at line 18").
+- Vague reasons like "low risk" or "unlikely to be exploited" are NOT valid dismissal evidence.
+
+## 2. USE THE CROSS-FILE INTELLIGENCE
+This is what makes you BETTER than a single-prompt scanner:
+- **Taint chains**: If a taint chain shows user input flowing through a fake sanitizer to a dangerous sink across files, that is a CRITICAL cross-file vulnerability. Call it out.
+- **Config risks**: If DEBUG=True + ALLOWED_HOSTS=["*"] + CORS_ALLOW_ALL=True, the combined attack surface is catastrophic even if individual code looks safe.
+- **Decoded payloads**: If an obfuscated payload decodes to `os.system('rm -rf /')`, that is CRITICAL malware regardless of how it's encoded.
+- **CVEs**: If a dependency has a known RCE CVE, flag it with the CVE ID and CVSS score.
+- **Import graph**: Use it to understand which files are connected and how vulnerabilities propagate.
+
+## 3. DEEP ANALYSIS — find what scanners CANNOT find
+Go BEYOND the scanner output. You MUST specifically check for ALL of these:
+
+### A. FALSE SANITIZER DETECTION (HIGHEST PRIORITY)
+Look for functions named sanitize/clean/escape/filter that do NOT actually sanitize:
+- If a "sanitize" function only does strip(), lower(), replace() — it's a FALSE SANITIZER
+- Trace every call to that function: if its output reaches cursor.execute(), os.system(), subprocess, or HttpResponse — that's a CRITICAL cross-file vulnerability
+- FALSE SANITIZER → SQLi chain: input → fake sanitize → cursor.execute(f"...{sanitized}...")
+- FALSE SANITIZER → CMDi chain: input → fake sanitize → os.system(f"...{sanitized}...")
+
+### B. IDOR + SENSITIVE FIELD EXPOSURE
+- Look for .objects.get(id=request.GET/POST) WITHOUT checking request.user ownership
+- If an IDOR endpoint returns sensitive fields (api_token, password, is_admin, reset_token, private_key) in JsonResponse — flag as Information Exposure (CWE-200)
+- Payment endpoints without ownership check = payment IDOR
+
+### C. REFLECTED XSS IN AUTH FLOWS
+- login() functions that put username/error in HttpResponse(f"...{username}...") without escaping
+- Error messages reflecting user input without HTML encoding
+
+### D. PREDICTABLE TOKEN GENERATION
+- If generate_token/create_token uses random.choice() or random.randint() instead of secrets module — CRITICAL
+- Trace token generation to password_reset, API key generation, session creation
+
+### E. BUSINESS LOGIC FLAWS
+- Payment/transfer functions without amount validation (negative amounts)
+- Type confusion: no int/float validation on financial amounts
+- Missing rate limiting on authentication endpoints
+
+### F. MISSING SECURITY HARDENING
+- Security headers not set (X-Frame-Options, CSP, HSTS, X-Content-Type-Options)
+- Missing CSRF protection (commented out middleware)
+- Session cookie without HttpOnly/Secure/SameSite flags
+
+### G. TIMING ATTACKS
+- Any == comparison on passwords, tokens, API keys, signatures
+- check_password(), verify_token() using direct string comparison instead of hmac.compare_digest()
+
+### H. LOGGING SENSITIVE DATA
+- log/print statements containing password, token, secret, session, credit_card variables
+
+Return JSON:
+```json
+{
+ "triage": [
+ {"finding_index": 0, "classification": "true_positive", "rationale": "...", "adjusted_severity": "CRITICAL"}
+ ],
+ "deep_findings": [
+ {
+ "severity": "CRITICAL",
+ "category": "Cross-File Vulnerability",
+ "title": "...",
+ "description": "...",
+ "file": "path/to/file.py",
+ "line_numbers": [30],
+ "attack_scenario": "...",
+ "cwe_id": "CWE-89"
+ }
+ ]
+}
+```
+Return ONLY the JSON object. Be exhaustive — miss nothing."""
+
+
+FIXER_PROMPT = """You are a Security Engineer writing secure code patches.
+
+For EACH confirmed vulnerability, generate a minimal, focused fix:
+1. Show EXACT lines to change with before/after
+2. Use the MOST SECURE approach in the language's standard library
+3. Include which FILE the fix applies to (for multi-file projects)
+4. Include a brief comment explaining WHY the fix is secure
+
+Format each fix as:
+### Fix for: [vulnerability title] (File: path, Line X)
+
+**Before (vulnerable):**
+```python
+[original code]
+```
+
+**After (secure):**
+```python
+[fixed code]
+```
+
+**Why:** [one-line explanation]
+
+Generate fixes for ALL vulnerabilities, ordered by severity (CRITICAL first)."""
+
+
+REPORTER_PROMPT = """You are a Security Report Writer. Compile all findings into a professional audit report.
+
+Format:
+
+# 🛡️ Security Audit Report
+
+## Executive Summary
+Total files scanned, total findings, critical count, overall risk score (0-100).
+
+## Scan Methodology
+List ALL tools used:
+- **Deterministic Scanners**: Bandit, Python AST, Secret Detection, Pattern Matching
+- **Cross-File Analysis**: Import Graph, Taint Tracking, Config Correlation
+- **Obfuscation Decoding**: Base64, ROT13, Unicode, Hex
+- **Dependency CVE Check**: OSV.dev API
+- **LLM Deep Audit**: DeepSeek R1 comprehensive analysis
+
+## Findings by Severity
+
+### 🔴 CRITICAL
+(List each with title, CWE, file location, description, attack scenario, fix)
+
+### 🟠 HIGH / 🟡 MEDIUM / 🟢 LOW
+(Same format)
+
+## Cross-File Attack Chains
+If taint chains were found, show the full path:
+```
+File A (line X): user input enters via request.GET
+ → File B (line Y): passed through fake sanitizer (strip() only)
+ → File A (line Z): reaches cursor.execute() — SQL INJECTION
+```
+
+## Known CVEs in Dependencies
+List each CVE with package, version, CVE ID, and description.
+
+## Configuration Risks
+List dangerous config settings and their combined impact.
+
+## Decoded Obfuscated Payloads
+Show what each obfuscated payload actually does when decoded.
+
+## Recommended Fixes
+Include all secure code patches.
+
+## Disclaimer
+⚠️ This automated scan provides a starting point for security review. It may produce false positives and cannot guarantee detection of all vulnerabilities. Always conduct manual penetration testing and code review for production systems.
+
+Rules:
+- Be ACTIONABLE — every finding needs a clear fix
+- Include CWE IDs
+- For multi-file projects, always specify which FILE each finding is in
+- Distinguish SSTI (RCE) from XSS clearly"""
diff --git a/services/python-tools/tools/code-security-scanner-v2/sec_scanners.py b/services/python-tools/tools/code-security-scanner-v2/sec_scanners.py
new file mode 100644
index 0000000..4562b71
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/sec_scanners.py
@@ -0,0 +1,652 @@
+"""
+Code Security Scanner V2 — Deterministic Scanners
+===================================================
+Pure programmatic security analysis — NO LLM involvement.
+These scanners use AST parsing, regex pattern matching, and
+subprocess execution to find vulnerabilities deterministically.
+
+This is what makes this tool a REAL agent, not a prompt wrapper:
+the LLM only sees pre-validated, structured findings from these
+scanners, not raw code.
+"""
+
+import ast
+import os
+import re
+import json
+import logging
+import subprocess
+import tempfile
+from dataclasses import dataclass, field, asdict
+from typing import Optional
+
+from sec_config import LANGUAGE_EXTENSIONS
+
+logger = logging.getLogger("security-scanner")
+
+
+# ─── Finding Data Structure ───────────────────────────────────────────
+
+@dataclass
+class Finding:
+ """A single security finding from any scanner."""
+ scanner: str # Which scanner found it ("bandit", "ast", "secrets", "patterns")
+ severity: str # "CRITICAL" | "HIGH" | "MEDIUM" | "LOW"
+ category: str # OWASP category or custom category
+ title: str # Short title
+ description: str # What the issue is
+ line_number: Optional[int] = None
+ code_snippet: Optional[str] = None
+ cwe_id: Optional[str] = None # CWE reference if applicable
+ confidence: str = "HIGH" # "HIGH" | "MEDIUM" | "LOW"
+
+ def to_dict(self) -> dict:
+ return asdict(self)
+
+
+# ─── Scanner 1: Bandit (Python-specific) ──────────────────────────────
+
+def run_bandit(code: str) -> list[Finding]:
+ """
+ Execute Bandit static analyzer on Python code via subprocess.
+ Returns structured findings, not raw text.
+ """
+ findings = []
+
+ with tempfile.NamedTemporaryFile(
+ suffix=".py", delete=False, mode="w", encoding="utf-8"
+ ) as f:
+ f.write(code)
+ temp_path = f.name
+
+ try:
+ result = subprocess.run(
+ ["bandit", "-r", temp_path, "-f", "json", "--severity-level", "all"],
+ capture_output=True,
+ text=True,
+ timeout=30,
+ check=False,
+ )
+
+ if result.stdout:
+ data = json.loads(result.stdout)
+ for issue in data.get("results", []):
+ severity_map = {"LOW": "LOW", "MEDIUM": "MEDIUM", "HIGH": "HIGH"}
+ findings.append(Finding(
+ scanner="bandit",
+ severity=severity_map.get(issue.get("issue_severity", ""), "MEDIUM"),
+ category=f"Bandit {issue.get('test_id', 'Unknown')}",
+ title=issue.get("issue_text", "Unknown issue"),
+ description=(
+ f"Test: {issue.get('test_name', 'unknown')}\n"
+ f"Confidence: {issue.get('issue_confidence', 'unknown')}"
+ ),
+ line_number=issue.get("line_number"),
+ code_snippet=issue.get("code", ""),
+ cwe_id=issue.get("cwe", {}).get("id") if isinstance(issue.get("cwe"), dict) else None,
+ confidence=issue.get("issue_confidence", "MEDIUM"),
+ ))
+
+ logger.info(f"[Bandit] Found {len(findings)} issues")
+
+ except FileNotFoundError:
+ logger.warning("[Bandit] Not installed — skipping (pip install bandit)")
+ except subprocess.TimeoutExpired:
+ logger.warning("[Bandit] Timed out after 30s")
+ except json.JSONDecodeError:
+ logger.warning("[Bandit] Failed to parse output")
+ except Exception as e:
+ logger.warning(f"[Bandit] Unexpected error: {e}")
+ finally:
+ os.unlink(temp_path)
+
+ return findings
+
+
+# ─── Scanner 2: Secret Detection (All Languages) ─────────────────────
+
+# Patterns compiled once at module load for performance
+SECRET_PATTERNS = [
+ # API Keys & Tokens
+ (re.compile(r"""(?:api[_-]?key|apikey|api_secret)\s*[:=]\s*['"]([a-zA-Z0-9_\-]{16,})['"]""", re.I),
+ "Hardcoded API Key", "CRITICAL", "CWE-798"),
+
+ # AWS Access Keys
+ (re.compile(r"""AKIA[0-9A-Z]{16}"""),
+ "AWS Access Key ID", "CRITICAL", "CWE-798"),
+
+ # AWS Secret Keys
+ (re.compile(r"""(?:aws_secret|secret_access_key)\s*[:=]\s*['"]([a-zA-Z0-9/+=]{40})['"]""", re.I),
+ "AWS Secret Access Key", "CRITICAL", "CWE-798"),
+
+ # Generic Secrets/Passwords
+ (re.compile(r"""(?:password|passwd|pwd|secret|token)\s*[:=]\s*['"]([^'"]{8,})['"]""", re.I),
+ "Hardcoded Secret/Password", "HIGH", "CWE-798"),
+
+ # Private Keys
+ (re.compile(r"""-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----"""),
+ "Embedded Private Key", "CRITICAL", "CWE-321"),
+
+ # JWT Tokens
+ (re.compile(r"""eyJ[a-zA-Z0-9_-]{10,}\.eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_\-]+"""),
+ "Hardcoded JWT Token", "HIGH", "CWE-798"),
+
+ # Database Connection Strings
+ (re.compile(r"""(?:mongodb|postgres|mysql|redis):\/\/[^\s'"]+""", re.I),
+ "Hardcoded Database Connection String", "HIGH", "CWE-798"),
+
+ # Generic Bearer Tokens
+ (re.compile(r"""(?:bearer|authorization)\s*[:=]\s*['"]([a-zA-Z0-9_\-.]{20,})['"]""", re.I),
+ "Hardcoded Bearer/Auth Token", "HIGH", "CWE-798"),
+]
+
+def scan_secrets(code: str) -> list[Finding]:
+ """
+ Regex-based secret detection across ALL languages.
+ This catches what bandit misses in non-Python code.
+ """
+ findings = []
+ lines = code.split("\n")
+
+ for pattern, title, severity, cwe in SECRET_PATTERNS:
+ for i, line in enumerate(lines, 1):
+ # Skip comments
+ stripped = line.strip()
+ if stripped.startswith("#") or stripped.startswith("//") or stripped.startswith("*"):
+ continue
+
+ if pattern.search(line):
+ findings.append(Finding(
+ scanner="secrets",
+ severity=severity,
+ category="Data Exposure",
+ title=title,
+ description=f"Detected pattern matching {title.lower()} in source code.",
+ line_number=i,
+ code_snippet=line.strip()[:120],
+ cwe_id=cwe,
+ confidence="HIGH",
+ ))
+
+ logger.info(f"[Secrets] Found {len(findings)} potential secrets")
+ return findings
+
+
+# ─── Scanner 3: Python AST Analysis ──────────────────────────────────
+
+def scan_python_ast(code: str) -> list[Finding]:
+ """
+ Deep AST-based analysis for Python code.
+ Catches dangerous function calls, unsafe deserialization,
+ SQL injection patterns, and command injection risks.
+ """
+ findings = []
+
+ try:
+ tree = ast.parse(code)
+ except SyntaxError as e:
+ findings.append(Finding(
+ scanner="ast",
+ severity="LOW",
+ category="Code Quality",
+ title="Syntax Error in Code",
+ description=f"Python AST parser failed: {str(e)}",
+ line_number=e.lineno,
+ ))
+ return findings
+
+ # Dangerous function calls
+ DANGEROUS_CALLS = {
+ "eval": ("CRITICAL", "CWE-95", "Code Injection via eval()", "Arbitrary code execution"),
+ "exec": ("CRITICAL", "CWE-95", "Code Injection via exec()", "Arbitrary code execution"),
+ "compile": ("HIGH", "CWE-95", "Dynamic code compilation", "Potential code injection"),
+ "__import__": ("MEDIUM", "CWE-95", "Dynamic import", "May load arbitrary modules"),
+ }
+
+ DANGEROUS_ATTRS = {
+ ("pickle", "loads"): ("CRITICAL", "CWE-502", "Unsafe Deserialization (pickle.loads)", "Arbitrary code execution via crafted pickle data — attacker can inject __reduce__ RCE gadgets"),
+ ("pickle", "load"): ("CRITICAL", "CWE-502", "Unsafe Deserialization (pickle.load)", "Arbitrary code execution via crafted pickle data — attacker can inject __reduce__ RCE gadgets"),
+ ("pickle", "dumps"): ("MEDIUM", "CWE-502", "Pickle Serialization", "If combined with loads(), enables RCE via __reduce__ gadget chains"),
+ ("yaml", "load"): ("CRITICAL", "CWE-502", "Unsafe YAML Deserialization", "yaml.load without Loader allows arbitrary Python object construction. Use yaml.safe_load()"),
+ ("subprocess", "call"): ("HIGH", "CWE-78", "Subprocess Call", "Potential command injection if user input flows in"),
+ ("subprocess", "Popen"): ("HIGH", "CWE-78", "Subprocess Popen", "Potential command injection if user input flows in"),
+ ("subprocess", "run"): ("MEDIUM", "CWE-78", "Subprocess Run", "Check for shell=True with user input — enables command injection"),
+ ("os", "system"): ("CRITICAL", "CWE-78", "OS Command Execution", "Direct shell command execution — high injection risk"),
+ ("os", "popen"): ("HIGH", "CWE-78", "OS Pipe Execution", "Direct shell pipe — high injection risk"),
+ ("marshal", "loads"): ("HIGH", "CWE-502", "Unsafe Deserialization (marshal)", "Can execute arbitrary bytecode"),
+ ("shelve", "open"): ("MEDIUM", "CWE-502", "Shelve Deserialization", "Uses pickle under the hood"),
+ # SSTI
+ ("render_template_string", None): ("CRITICAL", "CWE-1336", "Server-Side Template Injection (SSTI)", "render_template_string with user input enables RCE via Jinja2 template expressions"),
+ }
+
+ # Special check: render_template_string as a direct call (not attribute)
+ DANGEROUS_CALLS["render_template_string"] = ("CRITICAL", "CWE-1336", "Server-Side Template Injection (SSTI)", "User input in Jinja2 templates enables full RCE via {{config.__class__.__init__.__globals__}}")
+
+ for node in ast.walk(tree):
+ # Check direct function calls: eval(), exec(), etc.
+ if isinstance(node, ast.Call):
+ if isinstance(node.func, ast.Name) and node.func.id in DANGEROUS_CALLS:
+ sev, cwe, title, desc = DANGEROUS_CALLS[node.func.id]
+ findings.append(Finding(
+ scanner="ast",
+ severity=sev,
+ category="Code Injection",
+ title=title,
+ description=desc,
+ line_number=node.lineno,
+ cwe_id=cwe,
+ ))
+
+ # Check attribute calls: pickle.loads(), os.system(), etc.
+ if isinstance(node.func, ast.Attribute):
+ if isinstance(node.func.value, ast.Name):
+ key = (node.func.value.id, node.func.attr)
+ if key in DANGEROUS_ATTRS:
+ sev, cwe, title, desc = DANGEROUS_ATTRS[key]
+ findings.append(Finding(
+ scanner="ast",
+ severity=sev,
+ category="Dangerous API Usage",
+ title=title,
+ description=desc,
+ line_number=node.lineno,
+ cwe_id=cwe,
+ ))
+
+ # Check for bare except clauses (swallows all errors including SystemExit)
+ if isinstance(node, ast.ExceptHandler) and node.type is None:
+ findings.append(Finding(
+ scanner="ast",
+ severity="LOW",
+ category="Error Handling",
+ title="Bare except clause",
+ description="Catches all exceptions including SystemExit and KeyboardInterrupt. Use specific exception types.",
+ line_number=node.lineno,
+ cwe_id="CWE-396",
+ ))
+
+ # Check for assert statements (removed in optimized bytecode)
+ if isinstance(node, ast.Assert):
+ findings.append(Finding(
+ scanner="ast",
+ severity="MEDIUM",
+ category="Authentication",
+ title="Assert used for validation",
+ description="assert statements are removed when Python runs with -O flag. Never use assert for security checks.",
+ line_number=node.lineno,
+ cwe_id="CWE-617",
+ ))
+
+ # Check for __reduce__ method (RCE gadget for pickle deserialization)
+ if isinstance(node, ast.FunctionDef) and node.name == "__reduce__":
+ findings.append(Finding(
+ scanner="ast",
+ severity="CRITICAL",
+ category="Deserialization",
+ title="__reduce__ RCE Gadget",
+ description="Class defines __reduce__ which can execute arbitrary code when pickled/unpickled. This is a deserialization attack vector.",
+ line_number=node.lineno,
+ cwe_id="CWE-502",
+ ))
+
+ # Check for subprocess with shell=True
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ if isinstance(node.func.value, ast.Name) and node.func.value.id == "subprocess":
+ for kw in node.keywords:
+ if kw.arg == "shell" and isinstance(kw.value, ast.Constant) and kw.value.value is True:
+ findings.append(Finding(
+ scanner="ast",
+ severity="CRITICAL",
+ category="Command Injection",
+ title="Subprocess with shell=True",
+ description="shell=True passes command through the shell, enabling injection via semicolons, pipes, and backticks.",
+ line_number=node.lineno,
+ cwe_id="CWE-78",
+ ))
+
+ # Check for XML parsers without defuse
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ if node.func.attr in ("parse", "fromstring", "iterparse"):
+ if isinstance(node.func.value, ast.Name) and node.func.value.id in ("etree", "ET", "ElementTree", "minidom", "xml"):
+ findings.append(Finding(
+ scanner="ast",
+ severity="HIGH",
+ category="XML External Entity",
+ title="XML Parser without defuse (XXE / Billion Laughs)",
+ description="XML parsing without defusedxml enables XXE attacks and Billion Laughs DoS.",
+ line_number=node.lineno,
+ cwe_id="CWE-611",
+ ))
+
+ # Check for timing-vulnerable comparisons (== on secrets)
+ if isinstance(node, ast.Compare) and len(node.ops) == 1 and isinstance(node.ops[0], ast.Eq):
+ # Check if variable names suggest secrets
+ secret_names = {"key", "token", "secret", "password", "sig", "signature", "api_key", "provided", "expected"}
+ names_involved = set()
+ if isinstance(node.left, ast.Name):
+ names_involved.add(node.left.id.lower())
+ for comp in node.comparators:
+ if isinstance(comp, ast.Name):
+ names_involved.add(comp.id.lower())
+ if names_involved & secret_names:
+ findings.append(Finding(
+ scanner="ast",
+ severity="HIGH",
+ category="Timing Attack",
+ title="Timing-vulnerable comparison on secret",
+ description="Using == to compare secrets enables timing attacks. Use hmac.compare_digest() instead.",
+ line_number=node.lineno,
+ cwe_id="CWE-208",
+ ))
+
+ logger.info(f"[AST] Found {len(findings)} issues")
+ return findings
+
+
+# ─── Scanner 4: Language-Agnostic Pattern Matching ────────────────────
+
+VULN_PATTERNS = {
+ "sql_injection": [
+ (re.compile(r"""(?:execute|query|raw)\s*\(\s*(?:f['"]|['"].*%s|['"].*\+|['"].*\.format)""", re.I),
+ "SQL Injection Risk", "CRITICAL", "CWE-89",
+ "String concatenation or f-strings in SQL queries enable injection attacks"),
+ (re.compile(r"""(?:execute|cursor)\s*\(\s*f['"](?:SELECT|INSERT|UPDATE|DELETE)""", re.I),
+ "SQL Injection via f-string", "CRITICAL", "CWE-89",
+ "f-string in SQL statement — use parameterized queries"),
+ ],
+ "xss": [
+ (re.compile(r"""innerHTML\s*=\s*""", re.I),
+ "Potential XSS via innerHTML", "HIGH", "CWE-79",
+ "Setting innerHTML with unsanitized input enables cross-site scripting"),
+ (re.compile(r"""dangerouslySetInnerHTML""", re.I),
+ "React dangerouslySetInnerHTML", "MEDIUM", "CWE-79",
+ "Renders raw HTML — ensure input is sanitized"),
+ ],
+ "ssti": [
+ (re.compile(r"""render_template_string\s*\(""", re.I),
+ "Server-Side Template Injection (SSTI)", "CRITICAL", "CWE-1336",
+ "render_template_string with user input enables full RCE via Jinja2"),
+ (re.compile(r"""Environment\s*\(\)\s*\.\s*from_string""", re.I),
+ "SSTI via Jinja2 Environment", "CRITICAL", "CWE-1336",
+ "Unsandboxed Jinja2 Environment().from_string allows template injection RCE"),
+ (re.compile(r"""Template\s*\(\s*(?:request|user|input|data)""", re.I),
+ "Potential Template Injection", "HIGH", "CWE-1336",
+ "User input flowing into template constructor"),
+ ],
+ "path_traversal": [
+ (re.compile(r"""(?:open|read|write)\s*\(.*(?:request|req|params|query|input|argv)""", re.I),
+ "Potential Path Traversal", "HIGH", "CWE-22",
+ "File operations with user input — validate and sanitize paths"),
+ (re.compile(r"""f\.save\s*\(""", re.I),
+ "Unvalidated File Upload Save", "HIGH", "CWE-434",
+ "File saved without validating content — check magic bytes, not just Content-Type"),
+ (re.compile(r"""content_type\s*in\s*\[""", re.I),
+ "Content-Type Only Validation", "HIGH", "CWE-434",
+ "Validating file type by Content-Type header only — trivially spoofed by attacker"),
+ ],
+ "crypto": [
+ (re.compile(r"""hashlib\.(?:md5|sha1)\s*\(""", re.I),
+ "Weak Password Hashing (MD5/SHA1)", "HIGH", "CWE-328",
+ "MD5/SHA1 for password hashing is cryptographically broken — rainbow tables can crack in seconds. Use bcrypt/argon2/scrypt"),
+ (re.compile(r"""(?:md5|sha1)\s*\(""", re.I),
+ "Weak Hashing Algorithm", "MEDIUM", "CWE-328",
+ "MD5/SHA1 are cryptographically broken — use SHA-256 or bcrypt"),
+ (re.compile(r"""hexdigest\s*\(\).*==|==.*hexdigest\s*\(\)""", re.I),
+ "Unsalted Hash Comparison", "HIGH", "CWE-916",
+ "Hash comparison without salt — vulnerable to rainbow table attacks. Use bcrypt with per-user salt"),
+ (re.compile(r"""(?:DES|RC4|Blowfish)""", re.I),
+ "Weak Encryption Algorithm", "HIGH", "CWE-327",
+ "DES/RC4/Blowfish are deprecated — use AES-256"),
+ (re.compile(r"""MODE_ECB"""),
+ "ECB Mode Encryption", "HIGH", "CWE-327",
+ "ECB mode reveals patterns in plaintext — use CBC or GCM"),
+ (re.compile(r"""b'\\x00'\s*\*\s*\d+"""),
+ "Static/Zero IV", "HIGH", "CWE-329",
+ "Hardcoded or zero IV defeats CBC security — generate random IV per encryption"),
+ (re.compile(r"""nonce.*(?:never|reuse|static|counter.*0|same)""", re.I),
+ "Cryptographic Nonce Reuse", "CRITICAL", "CWE-323",
+ "Reusing nonce in CTR/GCM mode enables XOR attacks to recover plaintext"),
+ ],
+ "ldap_injection": [
+ (re.compile(r"""(?:search_s|search)\s*\(.*f['"]\s*\(""", re.I),
+ "LDAP Injection", "CRITICAL", "CWE-90",
+ "User input in LDAP filter without sanitization — bypass: *)(&(objectClass=*"),
+ (re.compile(r"""f['"]\(&\(uid=\{|f['"]\(uid=\{""", re.I),
+ "LDAP Filter Injection", "CRITICAL", "CWE-90",
+ "F-string in LDAP filter with user input enables authentication bypass"),
+ ],
+ "nosql_injection": [
+ (re.compile(r"""find_one\s*\(\s*\{.*(?:request|password|user|input)""", re.I),
+ "NoSQL Injection", "CRITICAL", "CWE-943",
+ "User input in MongoDB query — attacker can pass {\"$gt\": \"\"} to bypass auth"),
+ (re.compile(r"""\$(?:gt|ne|lt|gte|lte|regex|where|exists)"""),
+ "NoSQL Operator in Data", "HIGH", "CWE-943",
+ "MongoDB operators in data suggest NoSQL injection risk"),
+ ],
+ "xml_attack": [
+ (re.compile(r"""['\"]""", re.I),
+ "Reflected XSS in Return Value", "HIGH", "CWE-79",
+ "User input embedded in HTML string — use template escaping"),
+ (re.compile(r"""HttpResponse\s*\(.*(?:request\.GET|request\.POST|request\.META)""", re.I),
+ "Reflected XSS via Request Data in Response", "HIGH", "CWE-79",
+ "Request data directly in HttpResponse without escaping — reflected XSS"),
+ ],
+ "zip_slip": [
+ (re.compile(r"""extractall\s*\(""", re.I),
+ "Zip Slip via extractall()", "CRITICAL", "CWE-22",
+ "zipfile.extractall() doesn't validate member paths — attacker can write files to ../../etc/passwd"),
+ (re.compile(r"""ZipFile.*extract\s*\(""", re.I),
+ "Potential Zip Slip via extract()", "HIGH", "CWE-22",
+ "Individual zip extract without path validation — check for ../ in member names"),
+ ],
+ "toctou": [
+ (re.compile(r"""os\.path\.exists\s*\(.*\).*(?:open|read|write)\s*\(""", re.I | re.S),
+ "TOCTOU Race Condition", "HIGH", "CWE-367",
+ "Time-of-check/time-of-use: file existence checked then used later — attacker can swap file between check and use"),
+ ],
+ "idor": [
+ (re.compile(r"""\.get\s*\(\s*(?:id|pk)\s*=.*(?:request|params|args|data)""", re.I),
+ "Potential IDOR (Insecure Direct Object Reference)", "HIGH", "CWE-639",
+ "Object retrieved by user-supplied ID without ownership verification — any user can access any object"),
+ (re.compile(r"""\.objects\.get\s*\(\s*(?:id|pk)\s*=""", re.I),
+ "Direct Object Access Without Authorization", "HIGH", "CWE-639",
+ "Database object fetched by ID — verify the requesting user owns or is authorized to access this object"),
+ (re.compile(r"""JsonResponse\s*\(.*(?:api_token|token|secret|password|hash|reset_token|private_key|ssn|credit_card|is_admin)""", re.I),
+ "Sensitive Field Exposure in API Response", "HIGH", "CWE-200",
+ "Sensitive fields (tokens, passwords, admin flags) included in API response — information disclosure risk"),
+ ],
+ "jwt_misconfiguration": [
+ (re.compile(r"""['\"]none['\"]""", re.I),
+ "JWT 'none' Algorithm Allowed", "CRITICAL", "CWE-327",
+ "Allowing 'none' algorithm in JWT lets attackers forge tokens without a signature — full auth bypass"),
+ (re.compile(r"""algorithms?\s*=\s*\[.*['\"]none['\"]""", re.I),
+ "JWT Algorithm List Includes 'none'", "CRITICAL", "CWE-327",
+ "JWT decode accepts 'none' algorithm — attacker can create unsigned tokens to bypass authentication"),
+ ],
+ "ssrf": [
+ (re.compile(r"""urllib\.request\.urlopen\s*\(""", re.I),
+ "SSRF via urllib.request.urlopen()", "CRITICAL", "CWE-918",
+ "urlopen() with user-controlled URL enables SSRF — attacker can access internal services (169.254.169.254, localhost, etc.)"),
+ (re.compile(r"""requests\.get\s*\(.*(?:url|callback|webhook|endpoint|target)""", re.I),
+ "SSRF via requests.get() with user input", "HIGH", "CWE-918",
+ "HTTP request with user-controlled URL — validate against allowlist to prevent SSRF"),
+ ],
+ "header_trust": [
+ (re.compile(r"""X-Forwarded-For""", re.I),
+ "Trusting X-Forwarded-For Header", "MEDIUM", "CWE-290",
+ "X-Forwarded-For is trivially spoofable by clients — rate limiters using it can be bypassed"),
+ ],
+ "file_upload": [
+ (re.compile(r"""uploaded.*\.name|request\.FILES""", re.I),
+ "File Upload Without Sanitization", "HIGH", "CWE-434",
+ "File upload without filename sanitization — attacker can use ../../etc/passwd as filename for path traversal"),
+ (re.compile(r"""os\.path\.join\s*\(\s*\w+\s*,\s*(?:filename|uploaded|file\.name)""", re.I),
+ "Path Traversal in File Upload", "CRITICAL", "CWE-22",
+ "os.path.join with unsanitized filename — attacker filename '../../../etc/passwd' escapes upload directory"),
+ (re.compile(r"""(?:open|write)\s*\(.*(?:filename|uploaded_file|file_path).*['"]w""", re.I),
+ "File Write Without Extension Validation", "HIGH", "CWE-434",
+ "File written without validating extension — attacker can upload .php/.py/.jsp for remote code execution"),
+ ],
+ "insecure_tempfile": [
+ (re.compile(r"""tempfile\.mk(?:temp|stemp)\s*\(""", re.I),
+ "Insecure Temporary File Creation", "HIGH", "CWE-377",
+ "mktemp/mkstemp creates predictable temp files — use tempfile.NamedTemporaryFile or tempfile.mkdtemp"),
+ (re.compile(r"""['\"](?:/tmp/|C:\\temp\\).*(?:filename|name|uploaded)""", re.I),
+ "Predictable Temp Path with User Input", "HIGH", "CWE-377",
+ "Hardcoded temp directory with user-influenced filename — race condition and path traversal risk"),
+ ],
+ "false_sanitizer": [
+ (re.compile(r"""def\s+(?:sanitize|clean|escape|filter|validate)_?\w*\s*\(.*\).*:\s*$""", re.I),
+ "Custom Sanitizer Function Detected", "MEDIUM", "CWE-20",
+ "Custom sanitizer detected — verify it actually neutralizes dangerous characters, not just strip()/lower(). False sanitizers are a critical cross-file vulnerability source"),
+ (re.compile(r"""(?:strip|lower|upper|title|capitalize)\s*\(\)\s*$""", re.I),
+ "Ineffective Input Sanitization", "HIGH", "CWE-20",
+ "strip()/lower() does NOT sanitize against injection attacks (SQLi, XSS, CMDi). These are formatting functions, not security functions"),
+ (re.compile(r"""return\s+\w+\.strip\s*\(\)(?:\.lower\s*\(\))?\s*$""", re.I),
+ "False Sanitizer: strip()/lower() Only", "CRITICAL", "CWE-20",
+ "Function returns input.strip().lower() claiming to sanitize — this provides ZERO protection against SQL injection, XSS, or command injection"),
+ ],
+ "business_logic": [
+ (re.compile(r"""(?:amount|price|quantity|balance)\s*=.*(?:request|params|data|body)""", re.I),
+ "Unvalidated Financial Amount from User Input", "MEDIUM", "CWE-20",
+ "Financial amount taken directly from user input — validate: positive number, reasonable range, correct type"),
+ (re.compile(r"""(?:payment|charge|transfer|withdraw).*(?:amount|price)""", re.I),
+ "Payment Logic Without Amount Validation", "HIGH", "CWE-20",
+ "Payment processing without validating amount — negative amounts can credit attacker's account"),
+ ],
+ "missing_security_headers": [
+ (re.compile(r"""(?:SecurityMiddleware|XFrameOptionsMiddleware|ContentSecurityPolicy)""", re.I),
+ "Security Header Middleware Reference", "LOW", "CWE-693",
+ "Verify these security middlewares are enabled, not commented out, and properly configured"),
+ ],
+}
+
+def scan_patterns(code: str, language: str) -> list[Finding]:
+ """
+ Language-agnostic vulnerability pattern matching.
+ Catches SQL injection, XSS, path traversal, and crypto issues.
+ """
+ findings = []
+ lines = code.split("\n")
+
+ for category, patterns in VULN_PATTERNS.items():
+ for pattern, title, severity, cwe, description in patterns:
+ for i, line in enumerate(lines, 1):
+ if pattern.search(line):
+ findings.append(Finding(
+ scanner="patterns",
+ severity=severity,
+ category=category.replace("_", " ").title(),
+ title=title,
+ description=description,
+ line_number=i,
+ code_snippet=line.strip()[:120],
+ cwe_id=cwe,
+ confidence="MEDIUM",
+ ))
+
+ logger.info(f"[Patterns] Found {len(findings)} issues for {language}")
+ return findings
+
+
+# ─── Orchestrator: Run All Scanners ───────────────────────────────────
+
+def run_all_scanners(code: str, language: str) -> list[dict]:
+ """
+ Execute all applicable scanners and return deduplicated findings.
+ This is the single entry point called by the Scanner agent node.
+ """
+ all_findings: list[Finding] = []
+
+ # Always run language-agnostic scanners
+ all_findings.extend(scan_secrets(code))
+ all_findings.extend(scan_patterns(code, language))
+
+ # Run Python-specific scanners
+ if language == "python":
+ all_findings.extend(run_bandit(code))
+ all_findings.extend(scan_python_ast(code))
+
+ # Deduplicate by (line_number, title)
+ seen = set()
+ unique: list[Finding] = []
+ for f in all_findings:
+ key = (f.line_number, f.title)
+ if key not in seen:
+ seen.add(key)
+ unique.append(f)
+
+ # Sort by severity (critical first)
+ severity_order = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3}
+ unique.sort(key=lambda f: severity_order.get(f.severity, 99))
+
+ logger.info(f"[Scanners] Total unique findings: {len(unique)}")
+ return [f.to_dict() for f in unique]
diff --git a/services/python-tools/tools/code-security-scanner-v2/tool.py b/services/python-tools/tools/code-security-scanner-v2/tool.py
new file mode 100644
index 0000000..6907c61
--- /dev/null
+++ b/services/python-tools/tools/code-security-scanner-v2/tool.py
@@ -0,0 +1,177 @@
+"""
+Code Security Scanner V2 — Tool Entry Point
+=============================================
+v3.0: Multi-file support with ZIP upload.
+
+Architecture:
+ sec_file_parser.py — Multi-file input parsing (ZIP, markers, single)
+ sec_scanners.py — 4 per-file deterministic scanners
+ sec_cross_file.py — 5 cross-file scanners (import graph, taint, config, obfuscation, CVE)
+ sec_agents.py — LangGraph: Scanner → Auditor → Fixer → Reporter
+ sec_prompts.py — Agent prompts with cross-file awareness
+ sec_config.py — Model fallbacks, limits, API config
+"""
+
+from sec_agents import build_graph, SecurityState
+from sec_config import OXLO_API_KEY
+
+MANIFEST = {
+ "id": "code-security-scanner-v2",
+ "name": "Code Security Scanner (Agentic V2)",
+ "description": (
+ "Multi-file agentic security pipeline: cross-file taint tracking, "
+ "import chain analysis, obfuscation decoding, dependency CVE checks "
+ "(osv.dev), config correlation + comprehensive LLM deep audit"
+ ),
+ "author": "Oxlo Team",
+ "version": "3.0.0",
+ "requires": ["langgraph", "langchain-openai", "langchain-core", "bandit", "httpx"],
+}
+
+
+async def run(data: dict):
+ """Execute the security audit pipeline."""
+ if not OXLO_API_KEY:
+ return {"error": "OXLO_API_KEY not configured. Set it in .env"}
+
+ code = data.get("code", "")
+ files_data = data.get("files", "")
+
+ if not code.strip() and not files_data.strip():
+ return {"error": "No code provided. Upload files or paste code to scan."}
+
+ language = data.get("language", "auto").lower()
+ user_model = data.get("model", "")
+
+ graph = build_graph()
+ initial: SecurityState = {
+ "code": code,
+ "files_data": files_data,
+ "language": language,
+ "user_model": user_model,
+ "parsed_files": [],
+ "file_count": 0,
+ "raw_findings": [],
+ "scanner_summary": "",
+ "import_graph": [],
+ "taint_chains": [],
+ "config_risks": [],
+ "decoded_payloads": [],
+ "cve_findings": [],
+ "exec_eval_findings": [],
+ "timing_findings": [],
+ "triage_results": [],
+ "deep_findings": [],
+ "severity_score": 0,
+ "fixes": "",
+ "final_report": "",
+ "status": "starting",
+ }
+
+ # Track state across the stream so we can build a fallback report on crash
+ last_state = {}
+
+ async def stream():
+ nonlocal last_state
+ try:
+ async for event in graph.astream(initial):
+ for node_name, node_output in event.items():
+ last_state.update(node_output)
+ status = node_output.get("status", "processing")
+ yield f"[{node_name}] {status}\n"
+
+ if node_name == "scanner":
+ file_count = node_output.get("file_count", 0)
+ yield f" > Scanned {file_count} files\n"
+ if node_output.get("scanner_summary"):
+ yield f" > {node_output['scanner_summary']}\n"
+
+ # Cross-file intelligence summary
+ tc = len(node_output.get("taint_chains", []))
+ ig = len(node_output.get("import_graph", []))
+ dp = len(node_output.get("decoded_payloads", []))
+ cve = len(node_output.get("cve_findings", []))
+ cr = len(node_output.get("config_risks", []))
+ if any([tc, ig, dp, cve, cr]):
+ yield f" > Cross-file: {tc} taint chains, {ig} imports, {dp} decoded payloads, {cve} CVEs, {cr} config risks\n"
+
+ if node_name == "auditor":
+ triage = node_output.get("triage_results", [])
+ deep = node_output.get("deep_findings", [])
+ confirmed = sum(1 for t in triage if t.get("classification") == "true_positive")
+ dismissed = sum(1 for t in triage if t.get("classification") == "false_positive")
+ yield f" > Triage: {confirmed} confirmed, {dismissed} dismissed\n"
+ yield f" > Deep analysis: {len(deep)} additional vulnerabilities\n"
+ yield f" > Severity score: {node_output.get('severity_score', 0)}\n"
+
+ if node_name == "fixer":
+ yield f" > Secure patches generated\n"
+
+ if node_name == "reporter" and "final_report" in node_output:
+ yield "\n---REPORT_START---\n"
+ yield node_output["final_report"]
+
+ except Exception as e:
+ import traceback
+ import logging
+ logger = logging.getLogger("security-scanner")
+ logger.error(f"[Pipeline] Stream crashed: {e}")
+ logger.error(traceback.format_exc())
+
+ # Emit error info to the user
+ yield f"\n[error] Pipeline error: {str(e)}\n"
+
+ # Build a fallback report from whatever state we accumulated
+ triage = last_state.get("triage_results", [])
+ deep = last_state.get("deep_findings", [])
+ confirmed = [t for t in triage if t.get("classification") == "true_positive"]
+ score = last_state.get("severity_score", 0)
+ file_count = last_state.get("file_count", 0)
+
+ if confirmed or deep:
+ yield "\n---REPORT_START---\n"
+ yield f"# 🛡️ Security Audit Report\n\n"
+ yield f"⚠️ *Note: Report compilation encountered an error. Showing raw findings.*\n\n"
+ yield f"## Summary\n"
+ yield f"Scanned **{file_count}** files. Found **{len(confirmed)}** confirmed vulnerabilities"
+ if deep:
+ yield f" and **{len(deep)}** additional deep findings"
+ yield f".\n\nSeverity Score: **{score}**\n\n"
+
+ if confirmed:
+ yield "## Confirmed Vulnerabilities\n\n"
+ yield "| # | Severity | Title | File | Line | CWE |\n"
+ yield "|---|---|---|---|---|---|\n"
+ for i, v in enumerate(confirmed, 1):
+ yield f"| {i} | {v.get('adjusted_severity', 'N/A')} | {v.get('title', 'Unknown')} | {v.get('file', '')} | {v.get('line', '')} | {v.get('cwe_id', '')} |\n"
+ yield "\n"
+
+ if deep:
+ yield "## Deep Analysis Findings\n\n"
+ for i, d in enumerate(deep, 1):
+ yield f"{i}. **{d.get('title', 'Unknown')}** — Severity: {d.get('severity', 'N/A')}\n"
+ if d.get("description"):
+ yield f" {d['description']}\n"
+ yield "\n"
+
+ # CVEs
+ cve_findings = last_state.get("cve_findings", [])
+ if cve_findings:
+ yield f"## Known CVEs ({len(cve_findings)})\n\n"
+ for cve in cve_findings[:20]:
+ yield f"- **{cve.get('cve_id', 'N/A')}**: {cve.get('package', '')} — {cve.get('summary', '')}\n"
+ yield "\n"
+
+ # Config risks
+ config_risks = last_state.get("config_risks", [])
+ if config_risks:
+ yield f"## Configuration Risks ({len(config_risks)})\n\n"
+ for cr in config_risks[:15]:
+ if isinstance(cr, dict):
+ yield f"- {cr.get('risk', str(cr))}\n"
+ else:
+ yield f"- {cr}\n"
+ yield "\n"
+
+ return stream()
+
diff --git a/services/python-tools/tools/deep-research/agents.py b/services/python-tools/tools/deep-research/agents.py
new file mode 100644
index 0000000..75da4f2
--- /dev/null
+++ b/services/python-tools/tools/deep-research/agents.py
@@ -0,0 +1,254 @@
+"""
+Deep Research Agent — Agent Nodes & Graph
+==========================================
+Contains the LangGraph state, agent nodes, and graph builder.
+Each node is one specialist agent in the research pipeline:
+ Planner → Searcher → Analyzer → Verifier → Writer
+"""
+
+import json
+import logging
+from typing import TypedDict, Annotated
+from operator import add
+
+from langgraph.graph import StateGraph, END
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from config import (
+ OXLO_API_KEY, OXLO_BASE_URL, TAVILY_API_KEY,
+ PLANNER_MODEL, SEARCHER_MODEL, ANALYZER_MODEL, VERIFIER_MODEL, WRITER_MODEL,
+)
+from prompts import (
+ PLANNER_PROMPT, SEARCHER_PROMPT, ANALYZER_PROMPT, VERIFIER_PROMPT, WRITER_PROMPT,
+)
+
+logger = logging.getLogger("deep-research")
+
+
+# ─── State Schema ──────────────────────────────────────────────────────
+
+class ResearchState(TypedDict):
+ """Shared state flowing through the research graph."""
+ query: str
+ sub_questions: list[str]
+ search_results: Annotated[list[dict], add]
+ analysis: str
+ verification: str
+ gaps: list[str]
+ iteration: int
+ max_iterations: int
+ final_report: str
+ status: str
+
+
+# ─── Helpers ───────────────────────────────────────────────────────────
+
+def get_llm(model: str, temperature: float = 0.3) -> ChatOpenAI:
+ """Create an LLM instance pointing to Oxlo API."""
+ return ChatOpenAI(
+ model=model,
+ api_key=OXLO_API_KEY,
+ base_url=OXLO_BASE_URL,
+ temperature=temperature,
+ max_tokens=4096,
+ )
+
+
+def get_tavily():
+ """Create Tavily client for web search. Returns None if no API key."""
+ if not TAVILY_API_KEY:
+ logger.warning("[Tavily] No API key — web search disabled")
+ return None
+ from tavily import TavilyClient
+ return TavilyClient(api_key=TAVILY_API_KEY)
+
+
+# ─── Agent Nodes ───────────────────────────────────────────────────────
+
+def planner_node(state: ResearchState) -> dict:
+ """Break down the query into focused sub-questions."""
+ logger.info(f"[Planner] Breaking down: {state['query'][:80]}...")
+ llm = get_llm(PLANNER_MODEL, 0.2)
+
+ response = llm.invoke([
+ SystemMessage(content=PLANNER_PROMPT),
+ HumanMessage(content=f"Research query: {state['query']}"),
+ ])
+
+ try:
+ content = response.content.strip()
+ if "```" in content:
+ content = content.split("```")[1]
+ if content.startswith("json"):
+ content = content[4:]
+ sub_questions = json.loads(content)
+ if not isinstance(sub_questions, list):
+ sub_questions = [state["query"]]
+ except (json.JSONDecodeError, IndexError):
+ sub_questions = [state["query"]]
+
+ logger.info(f"[Planner] Generated {len(sub_questions)} sub-questions")
+ return {"sub_questions": sub_questions, "status": "planning_complete"}
+
+
+def searcher_node(state: ResearchState) -> dict:
+ """Search the web for real-time data, then synthesize with LLM."""
+ logger.info(f"[Searcher] Researching {len(state['sub_questions'])} questions...")
+ tavily = get_tavily()
+ llm = get_llm(SEARCHER_MODEL, 0.3)
+ results = []
+
+ for i, question in enumerate(state["sub_questions"]):
+ web_context = _search_web(tavily, question, i)
+
+ response = llm.invoke([
+ SystemMessage(content=SEARCHER_PROMPT),
+ HumanMessage(
+ content=f"Question: {question}\n\n--- WEB RESULTS ---\n{web_context}\n--- END ---\n\n"
+ "Analyze these results and provide a comprehensive, well-sourced answer."
+ ),
+ ])
+
+ results.append({
+ "question": question,
+ "answer": response.content,
+ "web_sources": web_context[:1000],
+ "index": i,
+ })
+
+ logger.info(f"[Searcher] Gathered {len(results)} results")
+ return {"search_results": results, "status": "search_complete"}
+
+
+def _search_web(tavily, question: str, index: int) -> str:
+ """Execute Tavily web search for a single question."""
+ if not tavily:
+ return "\n(No web search configured — using model knowledge only)\n"
+
+ try:
+ logger.info(f"[Searcher] Web search Q{index+1}: {question[:60]}...")
+ response = tavily.search(
+ query=question,
+ search_depth="advanced",
+ max_results=5,
+ include_answer=True,
+ )
+
+ context = ""
+ if response.get("answer"):
+ context += f"\n**Tavily AI Answer:**\n{response['answer']}\n"
+
+ for j, result in enumerate(response.get("results", [])[:5]):
+ title = result.get("title", "Unknown")
+ url = result.get("url", "")
+ content = result.get("content", "")[:500]
+ score = result.get("score", 0)
+ context += f"\n**Source {j+1}** [{title}]({url}) (relevance: {score:.2f}):\n{content}\n"
+
+ logger.info(f"[Searcher] Found {len(response.get('results', []))} results for Q{index+1}")
+ return context
+
+ except Exception as e:
+ logger.warning(f"[Searcher] Tavily failed for Q{index+1}: {e}")
+ return "\n(Web search failed — using model knowledge only)\n"
+
+
+def analyzer_node(state: ResearchState) -> dict:
+ """Analyze results, extract insights, identify gaps."""
+ logger.info("[Analyzer] Analyzing research results...")
+ llm = get_llm(ANALYZER_MODEL, 0.2)
+
+ results_text = "\n\n".join([
+ f"### Q{r['index']+1}: {r['question']}\n{r['answer']}"
+ for r in state["search_results"]
+ ])
+
+ response = llm.invoke([
+ SystemMessage(content=ANALYZER_PROMPT),
+ HumanMessage(content=f"Original query: {state['query']}\n\nResearch results:\n{results_text}"),
+ ])
+
+ gaps = []
+ try:
+ if '```json' in response.content:
+ json_block = response.content.split('```json')[1].split('```')[0]
+ gaps = json.loads(json_block).get("gaps", [])
+ except Exception:
+ pass
+
+ logger.info(f"[Analyzer] Found {len(gaps)} knowledge gaps")
+ return {"analysis": response.content, "gaps": gaps, "status": "analysis_complete"}
+
+
+def verifier_node(state: ResearchState) -> dict:
+ """Cross-check facts and verify claims."""
+ logger.info("[Verifier] Cross-checking claims...")
+ llm = get_llm(VERIFIER_MODEL, 0.1)
+
+ response = llm.invoke([
+ SystemMessage(content=VERIFIER_PROMPT),
+ HumanMessage(content=f"Original query: {state['query']}\n\nAnalysis to verify:\n{state['analysis']}"),
+ ])
+
+ logger.info("[Verifier] Verification complete")
+ return {
+ "verification": response.content,
+ "status": "verification_complete",
+ "iteration": state["iteration"] + 1,
+ }
+
+
+def writer_node(state: ResearchState) -> dict:
+ """Synthesize all research into a final report."""
+ logger.info("[Writer] Generating final report...")
+ llm = get_llm(WRITER_MODEL, 0.4)
+
+ response = llm.invoke([
+ SystemMessage(content=WRITER_PROMPT),
+ HumanMessage(content=(
+ f"Original query: {state['query']}\n\n"
+ f"Research Results:\n{json.dumps([{'q': r['question'], 'a': r['answer'][:800], 'sources': r.get('web_sources', '')[:300]} for r in state['search_results']], indent=2)}\n\n"
+ f"Analysis:\n{state['analysis']}\n\n"
+ f"Verification:\n{state['verification']}"
+ )),
+ ])
+
+ logger.info("[Writer] Report generated")
+ return {"final_report": response.content, "status": "complete"}
+
+
+# ─── Routing ───────────────────────────────────────────────────────────
+
+def should_iterate(state: ResearchState) -> str:
+ """Decide: do another research pass or proceed to writing."""
+ if state["iteration"] >= state["max_iterations"]:
+ return "write"
+ if len(state.get("gaps", [])) > 0:
+ return "search"
+ return "write"
+
+
+# ─── Graph Builder ─────────────────────────────────────────────────────
+
+def build_graph():
+ """Build the LangGraph research workflow."""
+ wf = StateGraph(ResearchState)
+
+ wf.add_node("planner", planner_node)
+ wf.add_node("searcher", searcher_node)
+ wf.add_node("analyzer", analyzer_node)
+ wf.add_node("verifier", verifier_node)
+ wf.add_node("writer", writer_node)
+
+ wf.set_entry_point("planner")
+ wf.add_edge("planner", "searcher")
+ wf.add_edge("searcher", "analyzer")
+ wf.add_edge("analyzer", "verifier")
+ wf.add_conditional_edges(
+ "verifier", should_iterate,
+ {"search": "searcher", "write": "writer"},
+ )
+ wf.add_edge("writer", END)
+
+ return wf.compile()
diff --git a/services/python-tools/tools/deep-research/config.py b/services/python-tools/tools/deep-research/config.py
new file mode 100644
index 0000000..01748f0
--- /dev/null
+++ b/services/python-tools/tools/deep-research/config.py
@@ -0,0 +1,19 @@
+"""
+Deep Research Agent — Configuration
+"""
+
+import os
+
+# Oxlo API
+OXLO_API_KEY = os.getenv("OXLO_API_KEY", "")
+OXLO_BASE_URL = os.getenv("OXLO_BASE_URL", "https://api.oxlo.ai/v1")
+
+# Tavily web search
+TAVILY_API_KEY = os.getenv("TAVILY_API_KEY", "")
+
+# Model assignments per agent role
+PLANNER_MODEL = "deepseek-r1-0528"
+SEARCHER_MODEL = "llama-3.3-70b"
+ANALYZER_MODEL = "deepseek-r1-0528"
+VERIFIER_MODEL = "deepseek-r1-0528"
+WRITER_MODEL = "llama-3.3-70b"
diff --git a/services/python-tools/tools/deep-research/prompts.py b/services/python-tools/tools/deep-research/prompts.py
new file mode 100644
index 0000000..7215315
--- /dev/null
+++ b/services/python-tools/tools/deep-research/prompts.py
@@ -0,0 +1,80 @@
+"""
+Deep Research Agent — Prompt Templates
+========================================
+All system prompts for each agent in the research pipeline.
+Separated for easy editing and A/B testing.
+"""
+
+PLANNER_PROMPT = """You are a research planner. Break down a complex research query
+into 3-5 specific, focused sub-questions that together will comprehensively answer
+the original query.
+
+For each sub-question, think about:
+- What specific data or evidence is needed?
+- What sources would be most relevant?
+- How does this sub-question relate to the others?
+
+Return ONLY a JSON array of strings (the sub-questions). No explanation."""
+
+SEARCHER_PROMPT = """You are a research assistant with access to real-time web search results.
+Analyze the web search results provided and synthesize a comprehensive answer.
+
+Important:
+1. Cite specific data points, numbers, and facts from the web results
+2. Include the source URLs when referencing specific claims
+3. If the web results contain conflicting information, note all perspectives
+4. Confidence: HIGH if supported by multiple sources, MEDIUM if single source, LOW if inferred
+5. Always specify the date/recency of information when available
+
+Be thorough, factual, and cite your sources."""
+
+ANALYZER_PROMPT = """You are a research analyst. Analyze the gathered research and provide:
+
+1. **Key Findings**: The most important insights discovered (with source citations)
+2. **Cross-References**: Where different sources agree or disagree
+3. **Knowledge Gaps**: What important information is still missing
+4. **Data Quality**: Rate the quality of sources (official/academic vs. informal)
+5. **Confidence Assessment**: How reliable is the gathered information
+
+Also return a JSON block at the end with gaps:
+```json
+{"gaps": ["gap1", "gap2"]}
+```"""
+
+VERIFIER_PROMPT = """You are a fact-checker and research verifier. Review the analysis and:
+
+1. **Verify Claims**: Check each major claim for internal consistency and source backing
+2. **Flag Issues**: Identify any contradictions, unsupported claims, or logical fallacies
+3. **Rate Confidence**: Give each major finding a confidence score (1-5)
+4. **Source Quality**: Evaluate whether claims are backed by reliable sources
+5. **Suggest Improvements**: What additional verification would strengthen the research
+
+Be rigorous but constructive. Pay special attention to numerical claims and dates."""
+
+WRITER_PROMPT = """You are a research report writer. Synthesize all research into a well-structured,
+comprehensive report. Use the following format:
+
+# Research Report: [Topic]
+
+## Executive Summary
+Brief overview of key findings with the most important data points.
+
+## Key Findings
+### Finding 1: [Title]
+Details with specific data, numbers, and source citations.
+
+## Analysis & Discussion
+Deeper analysis with cross-references between sources.
+
+## Confidence Assessment
+What we're confident about (with sources), what needs more research.
+
+## Sources & References
+List all web sources cited in the research with URLs.
+
+Important guidelines:
+- Include specific numbers, dates, and data points
+- Cite sources with URLs where available
+- Note when information is real-time vs. from training data
+- Use markdown formatting for readability
+- Be precise about recency of data"""
diff --git a/services/python-tools/tools/deep-research/requirements.txt b/services/python-tools/tools/deep-research/requirements.txt
new file mode 100644
index 0000000..c28267f
--- /dev/null
+++ b/services/python-tools/tools/deep-research/requirements.txt
@@ -0,0 +1,5 @@
+# Requirements for: Deep Research Agent
+langgraph==0.4.1
+langchain-openai==0.3.12
+langchain-core==0.3.50
+tavily-python==0.5.0
diff --git a/services/python-tools/tools/deep-research/tool.py b/services/python-tools/tools/deep-research/tool.py
new file mode 100644
index 0000000..f76cb30
--- /dev/null
+++ b/services/python-tools/tools/deep-research/tool.py
@@ -0,0 +1,61 @@
+"""
+Deep Research Agent — Tool Entry Point
+=======================================
+This is the entry point loaded by the unified runner.
+The actual agent logic lives in agents.py and prompts.py.
+"""
+
+from agents import build_graph, ResearchState
+from config import OXLO_API_KEY
+
+# ─── MANIFEST ──────────────────────────────────────────────────────────
+MANIFEST = {
+ "id": "deep-research",
+ "name": "Deep Research Agent",
+ "description": "Multi-agent research system with real-time web search via Tavily",
+ "author": "Oxlo Team",
+ "version": "2.0.0",
+ "requires": ["langgraph", "langchain-openai", "langchain-core", "tavily-python"],
+}
+
+
+# ─── RUN (called by the unified runner) ──────────────────────────────
+async def run(data: dict):
+ """
+ Execute the deep research pipeline.
+ Returns an async generator that streams progress + final report.
+ """
+ if not OXLO_API_KEY:
+ return {"error": "OXLO_API_KEY not configured"}
+
+ query = data.get("query", "")
+ if not query.strip():
+ return {"error": "Query cannot be empty"}
+
+ depth = data.get("depth", "standard")
+ max_iter = {"quick": 1, "standard": 2, "deep": 3}.get(depth, 2)
+
+ graph = build_graph()
+ initial: ResearchState = {
+ "query": query,
+ "sub_questions": [],
+ "search_results": [],
+ "analysis": "",
+ "verification": "",
+ "gaps": [],
+ "iteration": 0,
+ "max_iterations": max_iter,
+ "final_report": "",
+ "status": "starting",
+ }
+
+ async def stream():
+ async for event in graph.astream(initial):
+ for node_name, node_output in event.items():
+ status = node_output.get("status", "processing")
+ yield f"[{node_name}] {status}\n"
+ if node_name == "writer" and "final_report" in node_output:
+ yield "\n---REPORT_START---\n"
+ yield node_output["final_report"]
+
+ return stream()
diff --git a/services/python-tools/tools/image-palette-extractor/color_extractor.py b/services/python-tools/tools/image-palette-extractor/color_extractor.py
new file mode 100644
index 0000000..edecb82
--- /dev/null
+++ b/services/python-tools/tools/image-palette-extractor/color_extractor.py
@@ -0,0 +1,258 @@
+"""
+Color Extraction Module
+========================
+Extracts dominant colors from images using KMeans clustering.
+Handles preprocessing, noise removal, and color sorting.
+"""
+
+import io
+import logging
+from base64 import b64decode
+from typing import List, Tuple
+
+import numpy as np
+from PIL import Image
+from sklearn.cluster import KMeans
+from colorsys import rgb_to_hsv
+
+logger = logging.getLogger(__name__)
+
+
+class ColorExtractor:
+ """Extracts and processes dominant colors from images."""
+
+ def __init__(self, num_colors: int = 8, resize_size: int = 150):
+ """
+ Initialize color extractor.
+
+ Args:
+ num_colors: Number of dominant colors to extract (default 8)
+ resize_size: Size to resize image for processing (smaller = faster)
+ """
+ self.num_colors = num_colors
+ self.resize_size = resize_size
+
+ def extract_from_base64(self, image_base64: str) -> List[str]:
+ """
+ Extract colors from base64-encoded image.
+
+ Args:
+ image_base64: Base64 string with data URI prefix (e.g., "data:image/png;base64,...")
+
+ Returns:
+ List of dominant colors as hex strings
+ """
+ try:
+ # Remove data URI prefix if present
+ if image_base64.startswith("data:"):
+ image_base64 = image_base64.split(",", 1)[1]
+
+ # Decode base64 to bytes
+ image_bytes = b64decode(image_base64)
+ image = Image.open(io.BytesIO(image_bytes))
+
+ return self.extract_from_image(image)
+ except Exception as e:
+ logger.error(f"Failed to extract from base64: {e}")
+ raise ValueError(f"Invalid image data: {str(e)}")
+
+ def get_pixel_map(
+ self,
+ image_base64: str,
+ max_pixels: int = 15000,
+ max_dim: int = 800,
+ ) -> Tuple[str, List[dict], int, int]:
+ """
+ Get a safely-sized pixel map and a downscaled image data URI suitable for frontend preview.
+
+ Args:
+ image_base64: Base64 string of image (may include data: prefix)
+ max_pixels: Maximum number of pixels to return in the pixel map (safety cap)
+ max_dim: Maximum longest edge (width or height) for the returned image
+
+ Returns:
+ Tuple of (data_uri_image, pixel_list, width, height)
+ """
+ try:
+ # Normalize base64 payload
+ if image_base64.startswith("data:"):
+ header, image_base64 = image_base64.split(",", 1)
+ else:
+ header = "data:image/png;base64"
+
+ image_bytes = b64decode(image_base64)
+ image = Image.open(io.BytesIO(image_bytes))
+
+ if image.mode != "RGB":
+ image = image.convert("RGB")
+
+ # Resize image to limit dimensions for frontend preview and sampling
+ width, height = image.size
+ if max(width, height) > max_dim:
+ ratio = min(max_dim / width, max_dim / height)
+ new_w = int(width * ratio)
+ new_h = int(height * ratio)
+ image = image.resize((new_w, new_h), Image.LANCZOS)
+ width, height = image.size
+
+ image_array = np.array(image)
+
+ # Determine sampling step to keep pixel count <= max_pixels
+ total = width * height
+ pixels = []
+ if max_pixels and max_pixels > 0:
+ if total <= max_pixels:
+ step = 1
+ else:
+ # sample roughly uniformly using a square step
+ step = int(max(1, (total / max_pixels) ** 0.5))
+
+ for y in range(0, height, step):
+ for x in range(0, width, step):
+ rgb = image_array[y, x]
+ hex_color = self._rgb_to_hex(tuple(rgb))
+ pixels.append({"x": int(x), "y": int(y), "color": hex_color})
+ else:
+ step = 0
+
+ # Re-encode resized image to data URI (jpeg to reduce size)
+ buffer = io.BytesIO()
+ image.save(buffer, format="JPEG", quality=85)
+ buffer.seek(0)
+ resized_b64 = buffer.getvalue()
+ from base64 import b64encode
+
+ data_uri = f"{header};base64,{b64encode(resized_b64).decode('utf-8')}"
+
+ logger.info(f"Generated pixel map: {len(pixels)} sampled pixels from {width}x{height} (step={step})")
+ return data_uri, pixels, width, height
+
+ except Exception as e:
+ logger.error(f"Failed to get pixel map: {e}")
+ raise
+
+
+ def extract_from_image(self, image: Image.Image) -> List[str]:
+ """
+ Extract dominant colors from PIL Image.
+
+ Args:
+ image: PIL Image object
+
+ Returns:
+ List of dominant colors as hex strings, sorted by brightness
+ """
+ try:
+ # Convert to RGB if needed
+ if image.mode != "RGB":
+ image = image.convert("RGB")
+
+ # Resize for faster processing
+ image = image.resize((self.resize_size, self.resize_size))
+
+ # Convert to array and reshape
+ image_array = np.array(image)
+ pixels = image_array.reshape(-1, 3)
+
+ # Remove grayscale/near-grayscale pixels (low saturation)
+ pixels = self._filter_grayscale(pixels)
+
+ # Cluster colors
+ kmeans = KMeans(n_clusters=min(self.num_colors, len(pixels)),
+ n_init=10, random_state=42)
+ kmeans.fit(pixels)
+
+ # Get cluster centers and sort by brightness
+ colors = kmeans.cluster_centers_.astype(int)
+ colors = self._sort_by_luminance(colors)
+
+ # Convert to hex
+ hex_colors = [self._rgb_to_hex(rgb) for rgb in colors]
+
+ logger.info(f"Extracted {len(hex_colors)} colors from image")
+ return hex_colors
+
+ except Exception as e:
+ logger.error(f"Color extraction failed: {e}")
+ raise
+
+ @staticmethod
+ def _filter_grayscale(pixels: np.ndarray, sat_threshold: float = 0.15) -> np.ndarray:
+ """
+ Remove grayscale pixels (low saturation) to focus on colored regions.
+
+ Args:
+ pixels: Array of RGB pixels
+ sat_threshold: Minimum saturation to keep (0-1)
+
+ Returns:
+ Filtered pixel array
+ """
+ # Convert to HSV and check saturation
+ hsv_pixels = []
+ for rgb in pixels:
+ h, s, v = rgb_to_hsv(rgb[0]/255, rgb[1]/255, rgb[2]/255)
+ hsv_pixels.append((h, s, v))
+
+ hsv_pixels = np.array(hsv_pixels)
+ mask = hsv_pixels[:, 1] > sat_threshold # saturation channel
+
+ return pixels[mask] if mask.sum() > 0 else pixels
+
+ @staticmethod
+ def _sort_by_luminance(colors: np.ndarray) -> np.ndarray:
+ """
+ Sort colors by perceived luminance (brightness).
+
+ Args:
+ colors: Array of RGB colors
+
+ Returns:
+ Sorted array
+ """
+ # Perceived luminance formula
+ luminance = 0.299 * colors[:, 0] + 0.587 * colors[:, 1] + 0.114 * colors[:, 2]
+ indices = np.argsort(luminance)[::-1] # Descending order
+ return colors[indices]
+
+ @staticmethod
+ def _rgb_to_hex(rgb: Tuple[int, int, int]) -> str:
+ """Convert RGB tuple to hex string."""
+ return f"#{int(rgb[0]):02x}{int(rgb[1]):02x}{int(rgb[2]):02x}"
+
+ @staticmethod
+ def _hex_to_rgb(hex_str: str) -> Tuple[int, int, int]:
+ """Convert hex string to RGB tuple."""
+ hex_str = hex_str.lstrip("#")
+ return tuple(int(hex_str[i:i+2], 16) for i in (0, 2, 4))
+
+ @staticmethod
+ def calculate_contrast(hex1: str, hex2: str) -> float:
+ """
+ Calculate WCAG contrast ratio between two colors.
+
+ Args:
+ hex1: First color in hex format
+ hex2: Second color in hex format
+
+ Returns:
+ Contrast ratio (1-21)
+ """
+ def relative_luminance(rgb):
+ """Calculate relative luminance per WCAG formula."""
+ r, g, b = [x / 255 for x in rgb]
+ r = r / 12.92 if r <= 0.03928 else ((r + 0.055) / 1.055) ** 2.4
+ g = g / 12.92 if g <= 0.03928 else ((g + 0.055) / 1.055) ** 2.4
+ b = b / 12.92 if b <= 0.03928 else ((b + 0.055) / 1.055) ** 2.4
+ return 0.2126 * r + 0.7152 * g + 0.0722 * b
+
+ rgb1 = ColorExtractor._hex_to_rgb(hex1)
+ rgb2 = ColorExtractor._hex_to_rgb(hex2)
+
+ l1 = relative_luminance(rgb1)
+ l2 = relative_luminance(rgb2)
+
+ lighter = max(l1, l2)
+ darker = min(l1, l2)
+
+ return (lighter + 0.05) / (darker + 0.05)
diff --git a/services/python-tools/tools/image-palette-extractor/config.py b/services/python-tools/tools/image-palette-extractor/config.py
new file mode 100644
index 0000000..5f9491c
--- /dev/null
+++ b/services/python-tools/tools/image-palette-extractor/config.py
@@ -0,0 +1,69 @@
+"""
+Configuration and LLM setup for Image Palette Extractor
+"""
+
+import os
+import logging
+from langchain_openai import ChatOpenAI
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Get Oxlo API key from environment
+OXLO_API_KEY = os.getenv("OXLO_API_KEY", "")
+
+# Initialize LLM with Oxlo API
+# Using Oxlo's OpenAI-compatible API
+LLM = ChatOpenAI(
+ api_key=OXLO_API_KEY,
+ model="kimi-k2.5",
+ base_url="https://api.oxlo.ai/v1",
+ temperature=0.7,
+ max_tokens=4000,
+) if OXLO_API_KEY else None
+
+# System prompt for LLM palette refinement
+REFINEMENT_SYSTEM_PROMPT = """You are a professional UI/UX designer and color theorist specializing in color harmony and accessibility.
+
+CRITICAL: Your response MUST be ONLY valid JSON. Do NOT include any text before or after the JSON object.
+
+Your task:
+1. Analyze the provided extracted colors
+2. Harmonize them into a cohesive palette
+3. Assign semantic UI roles to each color
+4. Ensure WCAG AA contrast compliance (4.5:1 minimum)
+5. Generate CSS variables and Tailwind config
+6. Provide color theory explanation
+
+Return ONLY this JSON structure (no other text):
+{
+ "palette": {
+ "primary": "#XXXXXX",
+ "secondary": "#XXXXXX",
+ "accent": "#XXXXXX",
+ "background": "#XXXXXX",
+ "surface": "#XXXXXX",
+ "text": "#XXXXXX",
+ "muted": "#XXXXXX",
+ "success": "#XXXXXX",
+ "error": "#XXXXXX"
+ },
+ "roles": {
+ "primary": "Main brand color for CTAs and highlights",
+ "secondary": "Supporting brand color for secondary actions",
+ "accent": "Highlights and attention-drawing elements",
+ "background": "Page/screen background",
+ "surface": "Cards, panels, and elevated surfaces",
+ "text": "Primary text color",
+ "muted": "Secondary text and disabled states",
+ "success": "Success messages and positive feedback",
+ "error": "Error messages and alerts"
+ },
+ "colorTheory": "Explanation of color harmony and accessibility",
+ "cssVariables": ":root { --primary: #XXXXXX; ... }",
+ "tailwindConfig": "colors: { primary: { 50: '...', ... } }",
+ "wcagCompliance": "Contrast analysis and accessibility notes"
+}
+
+IMPORTANT: Return ONLY the JSON object. No markdown, no code blocks, no explanations.
+"""
diff --git a/services/python-tools/tools/image-palette-extractor/llm_refiner.py b/services/python-tools/tools/image-palette-extractor/llm_refiner.py
new file mode 100644
index 0000000..40819d6
--- /dev/null
+++ b/services/python-tools/tools/image-palette-extractor/llm_refiner.py
@@ -0,0 +1,225 @@
+"""
+LLM Palette Refiner Module
+===========================
+Uses LLM to refine extracted colors, assign UI roles, and ensure accessibility.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Any
+
+from config import LLM, REFINEMENT_SYSTEM_PROMPT
+from color_extractor import ColorExtractor
+
+logger = logging.getLogger(__name__)
+
+_TOOL_DIR = Path(__file__).parent
+
+
+class PaletteRefiner:
+ """Refines extracted colors using LLM for UI role assignment and accessibility."""
+
+ def __init__(self, llm=None):
+ """Initialize refiner with LLM."""
+ if llm is None:
+ llm = LLM
+
+ self.llm = llm
+ if not self.llm:
+ raise RuntimeError("LLM not initialized. Ensure OXLO_API_KEY is set.")
+
+ async def refine_palette(self, colors: list, user_preferences: str = "") -> Dict[str, Any]:
+ """
+ Refine extracted colors into a production-ready palette.
+
+ Args:
+ colors: List of hex color codes
+ user_preferences: Optional user preferences for palette
+
+ Returns:
+ Structured palette with roles, theory, CSS, etc.
+ """
+ try:
+ # Prepare color summary for LLM
+ color_summary = self._prepare_color_summary(colors)
+
+ # Build user prompt
+ user_prompt = f"""
+Here are the extracted dominant colors from the uploaded image:
+{color_summary}
+
+User preferences: {user_preferences or 'None specified'}
+
+Please:
+1. Refine and harmonize these colors into a cohesive palette
+2. Map each color to an appropriate UI role
+3. Ensure WCAG AA contrast compliance for text
+4. Provide CSS variables and Tailwind configuration
+5. Explain the color theory and accessibility
+
+Return a valid JSON object following the structure specified in your system prompt.
+"""
+
+ # Call LLM
+ logger.info("Calling LLM for palette refinement...")
+ response = await self.llm.ainvoke([
+ {"role": "system", "content": REFINEMENT_SYSTEM_PROMPT},
+ {"role": "user", "content": user_prompt}
+ ])
+
+ # Parse response
+ content = response.content
+
+ # Extract JSON from response
+ palette_data = self._extract_json(content)
+
+ logger.info("Palette refinement completed successfully")
+ return palette_data
+
+ except Exception as e:
+ logger.error(f"Palette refinement failed: {e}")
+ raise
+
+ def _prepare_color_summary(self, colors: list) -> str:
+ """Format extracted colors for LLM."""
+ summary = "Dominant colors (from dark to light):\n"
+ for i, color in enumerate(colors, 1):
+ summary += f"{i}. {color}\n"
+ return summary
+
+ def _extract_json(self, text: str) -> Dict[str, Any]:
+ """Extract JSON object from LLM response text."""
+ import re
+
+ logger.info(f"LLM Response (first 500 chars): {text[:500]}")
+
+ try:
+ # Try direct parsing first
+ return json.loads(text)
+ except json.JSONDecodeError as e:
+ logger.debug(f"Direct parse failed: {e}")
+
+ # Try to find JSON in markdown code blocks (various formats)
+ patterns = [
+ r'```json\s*\n(.*?)\n```', # ```json\n...\n```
+ r'```\s*\n(.*?)\n```', # ```\n...\n```
+ r'```(.*?)```', # ```...``` (any spacing)
+ ]
+
+ for pattern in patterns:
+ match = re.search(pattern, text, re.DOTALL)
+ if match:
+ json_str = match.group(1).strip()
+ try:
+ return json.loads(json_str)
+ except json.JSONDecodeError:
+ logger.debug(f"Failed to parse JSON from markdown: {json_str[:100]}")
+ # Try to fix incomplete JSON in code block
+ try:
+ fixed_json = self._fix_truncated_json(json_str)
+ return json.loads(fixed_json)
+ except Exception:
+ continue
+
+ # Try to extract nested JSON: find first { and matching }
+ start = text.find('{')
+ if start != -1:
+ # Find matching closing brace with proper nesting
+ depth = 0
+ for i in range(start, len(text)):
+ if text[i] == '{':
+ depth += 1
+ elif text[i] == '}':
+ depth -= 1
+ if depth == 0:
+ try:
+ json_str = text[start:i+1]
+ result = json.loads(json_str)
+ logger.info("Successfully extracted JSON from nested braces")
+ return result
+ except json.JSONDecodeError:
+ logger.debug(f"Failed to parse extracted JSON: {json_str[:100]}")
+ pass
+
+ # If we couldn't find matching braces, try to fix truncated JSON
+ if start != -1:
+ try:
+ json_str = text[start:]
+ fixed_json = self._fix_truncated_json(json_str)
+ result = json.loads(fixed_json)
+ logger.info("Successfully parsed truncated JSON after fixing")
+ return result
+ except Exception as e:
+ logger.debug(f"Could not fix truncated JSON: {e}")
+
+ # If all parsing fails, log the response and raise
+ logger.error(f"Could not extract valid JSON. Full response:\n{text}")
+ raise ValueError(f"Could not extract valid JSON from LLM response. Got: {text[:200]}")
+
+ @staticmethod
+ def _fix_truncated_json(json_str: str) -> str:
+ """Attempt to fix truncated JSON by closing open braces/brackets."""
+ # Count open/close braces and brackets
+ open_braces = json_str.count('{') - json_str.count('}')
+ open_brackets = json_str.count('[') - json_str.count(']')
+ open_quotes = json_str.count('"') % 2 # If odd, there's an unclosed quote
+
+ # Close any open structures
+ fixed = json_str.rstrip()
+
+ # If we're in a string, close it
+ if open_quotes:
+ fixed += '"'
+
+ # Close open arrays
+ for _ in range(open_brackets):
+ fixed += ']'
+
+ # Close open braces
+ for _ in range(open_braces):
+ fixed += '}'
+
+ return fixed
+
+ @staticmethod
+ def ensure_wcag_compliance(palette: Dict[str, str]) -> Dict[str, str]:
+ """
+ Ensure WCAG AA contrast between text colors and backgrounds.
+ Adjusts colors if needed to meet minimum 4.5:1 ratio for normal text.
+
+ Args:
+ palette: Dictionary with color roles
+
+ Returns:
+ Adjusted palette ensuring contrast compliance
+ """
+ extractor = ColorExtractor()
+ min_contrast = 4.5 # WCAG AA for normal text
+
+ # Check critical pairs
+ pairs_to_check = [
+ ("text", "background"),
+ ("text", "surface"),
+ ("muted", "background"),
+ ("muted", "surface"),
+ ]
+
+ adjusted = palette.copy()
+
+ for text_role, bg_role in pairs_to_check:
+ if text_role in adjusted and bg_role in adjusted:
+ contrast = extractor.calculate_contrast(
+ adjusted[text_role],
+ adjusted[bg_role]
+ )
+
+ if contrast < min_contrast:
+ logger.warning(
+ f"Low contrast between {text_role} and {bg_role}: {contrast:.1f}"
+ )
+ # Adjust text color for better contrast
+ # This is a simplified approach - LLM should handle most cases
+ # For now, we just log the issue
+
+ return adjusted
diff --git a/services/python-tools/tools/image-palette-extractor/pipeline.py b/services/python-tools/tools/image-palette-extractor/pipeline.py
new file mode 100644
index 0000000..6c2e75d
--- /dev/null
+++ b/services/python-tools/tools/image-palette-extractor/pipeline.py
@@ -0,0 +1,228 @@
+"""
+LangGraph Pipeline
+==================
+Orchestrates the image-to-palette workflow using LangGraph.
+
+Pipeline Flow:
+1. Validate Input → 2. Extract Colors → 3. Refine Palette → 4. Output Formatting
+"""
+
+import logging
+from typing import TypedDict, Any
+from langgraph.graph import StateGraph, END
+import sys
+from pathlib import Path
+
+# Ensure this tool's directory is in sys.path for imports
+_TOOL_DIR = Path(__file__).parent
+if str(_TOOL_DIR) not in sys.path:
+ sys.path.insert(0, str(_TOOL_DIR))
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineState(TypedDict):
+ """State object for the pipeline."""
+ image_base64: str
+ user_preferences: str
+ num_colors: int
+ extracted_colors: list
+ refined_palette: dict
+ final_output: dict
+ error: str
+ status: str
+
+
+async def validate_input(state: PipelineState) -> PipelineState:
+ """Validate input image and parameters."""
+ try:
+ logger.info("Validating input...")
+
+ if not state.get("image_base64"):
+ state["error"] = "No image provided"
+ state["status"] = "failed"
+ return state
+
+ state["status"] = "input_validated"
+ logger.info("Input validation passed")
+ return state
+
+ except Exception as e:
+ logger.error(f"Input validation failed: {e}")
+ state["error"] = str(e)
+ state["status"] = "failed"
+ return state
+
+
+async def extract_colors(state: PipelineState) -> PipelineState:
+ """Extract dominant colors from image."""
+ try:
+ from color_extractor import ColorExtractor
+
+ if state.get("error"):
+ return state
+
+ logger.info("Extracting colors from image...")
+
+ num_colors = state.get("num_colors", 8)
+ extractor = ColorExtractor(num_colors=num_colors)
+
+ colors = extractor.extract_from_base64(state["image_base64"])
+
+ state["extracted_colors"] = colors
+ state["status"] = "colors_extracted"
+ logger.info(f"Extracted {len(colors)} colors: {colors}")
+
+ return state
+
+ except Exception as e:
+ logger.error(f"Color extraction failed: {e}")
+ state["error"] = f"Color extraction failed: {str(e)}"
+ state["status"] = "failed"
+ return state
+
+
+async def refine_palette(state: PipelineState) -> PipelineState:
+ """Refine extracted colors using LLM."""
+ try:
+ from llm_refiner import PaletteRefiner
+
+ if state.get("error"):
+ return state
+
+ logger.info("Refining palette with LLM...")
+
+ refiner = PaletteRefiner()
+ refined = await refiner.refine_palette(
+ state["extracted_colors"],
+ state.get("user_preferences", "")
+ )
+
+ # Ensure WCAG compliance
+ if "palette" in refined:
+ refined["palette"] = PaletteRefiner.ensure_wcag_compliance(refined["palette"])
+
+ state["refined_palette"] = refined
+ state["status"] = "palette_refined"
+ logger.info("Palette refinement completed")
+
+ return state
+
+ except Exception as e:
+ logger.error(f"Palette refinement failed: {e}")
+ state["error"] = f"Palette refinement failed: {str(e)}"
+ state["status"] = "failed"
+ return state
+
+
+async def format_output(state: PipelineState) -> PipelineState:
+ """Format final output for frontend."""
+ try:
+ if state.get("error"):
+ return state
+
+ logger.info("Formatting output...")
+
+ refined = state.get("refined_palette", {})
+
+ # Build comprehensive output
+ final_output = {
+ "success": True,
+ "palette": refined.get("palette", {}),
+ "roles": refined.get("roles", {}),
+ "colorTheory": refined.get("colorTheory", ""),
+ "cssVariables": refined.get("cssVariables", ""),
+ "tailwindConfig": refined.get("tailwindConfig", ""),
+ "wcagCompliance": refined.get("wcagCompliance", ""),
+ "extractedColors": state.get("extracted_colors", []),
+ }
+
+ state["final_output"] = final_output
+ state["status"] = "complete"
+ logger.info("Output formatting completed")
+
+ return state
+
+ except Exception as e:
+ logger.error(f"Output formatting failed: {e}")
+ state["error"] = f"Output formatting failed: {str(e)}"
+ state["status"] = "failed"
+ return state
+
+
+def build_pipeline_graph(*, skip_extract: bool = False):
+ """Build the LangGraph workflow.
+
+ If skip_extract is True, the graph assumes `extracted_colors` is already present
+ in the state and jumps directly from validate → refine.
+ """
+
+ # Create graph
+ graph = StateGraph(PipelineState)
+
+ # Add nodes
+ graph.add_node("validate", validate_input)
+ graph.add_node("extract", extract_colors)
+ graph.add_node("refine", refine_palette)
+ graph.add_node("format", format_output)
+
+ # Add edges
+ if skip_extract:
+ graph.add_edge("validate", "refine")
+ else:
+ graph.add_edge("validate", "extract")
+ graph.add_edge("extract", "refine")
+ graph.add_edge("refine", "format")
+ graph.add_edge("format", END)
+
+ # Set entry point
+ graph.set_entry_point("validate")
+
+ return graph.compile()
+
+
+async def execute_pipeline(image_base64: str,
+ user_preferences: str = "",
+ num_colors: int = 8,
+ extracted_colors: list | None = None) -> dict:
+ """
+ Execute the complete palette extraction pipeline.
+
+ Args:
+ image_base64: Base64-encoded image with data URI prefix
+ user_preferences: Optional user preferences string
+ num_colors: Number of colors to extract
+
+ Returns:
+ Final formatted output
+ """
+
+ # Build pipeline
+ pipeline = build_pipeline_graph(skip_extract=bool(extracted_colors))
+
+ # Initialize state
+ initial_state: PipelineState = {
+ "image_base64": image_base64,
+ "user_preferences": user_preferences,
+ "num_colors": num_colors,
+ "extracted_colors": extracted_colors or [],
+ "refined_palette": {},
+ "final_output": {},
+ "error": "",
+ "status": "started",
+ }
+
+ # Execute pipeline
+ logger.info("Starting image palette extraction pipeline...")
+ result = await pipeline.ainvoke(initial_state)
+
+ # Handle errors
+ if result.get("error"):
+ return {
+ "success": False,
+ "error": result["error"],
+ "status": result.get("status", "failed")
+ }
+
+ logger.info(f"Pipeline completed with status: {result.get('status')}")
+ return result.get("final_output", {})
diff --git a/services/python-tools/tools/image-palette-extractor/requirements.txt b/services/python-tools/tools/image-palette-extractor/requirements.txt
new file mode 100644
index 0000000..d4f5cf7
--- /dev/null
+++ b/services/python-tools/tools/image-palette-extractor/requirements.txt
@@ -0,0 +1,5 @@
+langgraph>=0.1.0
+langchain-openai>=0.1.0
+Pillow>=10.0.0
+scikit-learn>=1.3.0
+numpy>=1.24.0
diff --git a/services/python-tools/tools/image-palette-extractor/tool.py b/services/python-tools/tools/image-palette-extractor/tool.py
new file mode 100644
index 0000000..b0d7ede
--- /dev/null
+++ b/services/python-tools/tools/image-palette-extractor/tool.py
@@ -0,0 +1,217 @@
+"""
+Image Palette Extractor Tool
+=============================
+Entry point for the unified Python tool runner.
+Extracts dominant colors from images and generates production-ready palettes.
+
+Agentic Architecture:
+- LangGraph for orchestration
+- LangChain for LLM integration
+- KMeans clustering for color extraction
+- WCAG compliance validation
+
+Processing Flow:
+Image (base64) → Preprocessing → Color Extraction (KMeans)
+→ Palette Refinement (LLM) → WCAG Validation → Structured Output
+"""
+
+import json
+import logging
+import sys
+from pathlib import Path
+import hashlib
+
+logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s [%(name)s] %(levelname)s: %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+# Ensure tool directory is in sys.path BEFORE any imports
+_TOOL_DIR = Path(__file__).parent
+if str(_TOOL_DIR) not in sys.path:
+ sys.path.insert(0, str(_TOOL_DIR))
+
+
+# ─── MANIFEST ──────────────────────────────────────────────────────────
+MANIFEST = {
+ "id": "color-palette",
+ "name": "Color Palette Generator (Agentic)",
+ "description": "Extract dominant colors from images using KMeans clustering and LLM-powered refinement with WCAG compliance",
+ "author": "Oxlo Team",
+ "version": "1.0.0",
+ "requires": [
+ "langgraph>=0.1.0",
+ "langchain-openai>=0.1.0",
+ "Pillow>=10.0.0",
+ "scikit-learn>=1.3.0",
+ "numpy>=1.24.0",
+ ],
+}
+
+
+# ─── RUN (called by the unified runner) ──────────────────────────
+async def run(data: dict):
+ """
+ Execute the image palette extraction pipeline.
+
+ Inputs (from frontend):
+ - image: base64-encoded image (required)
+ - description: text description (optional)
+ - style: style preference (optional)
+ - count: number of colors (optional, default 8)
+
+ Returns:
+ Async generator streaming status updates + final JSON output
+ """
+
+ try:
+ # Import here to avoid module loading issues at startup
+ from pipeline import execute_pipeline
+
+ # Validate inputs
+ image_base64 = data.get("image", "")
+ description = data.get("description", "")
+ style = data.get("style", "")
+ count = data.get("count", "8")
+
+ logger.info("Processing image mode with color extraction")
+
+ # Parse color count
+ try:
+ num_colors = min(int(count), 12) # Cap at 12 colors
+ num_colors = max(num_colors, 4) # Min 4 colors
+ except (ValueError, TypeError):
+ num_colors = 8
+
+ # Validate image is provided
+ if not image_base64:
+ error_msg = "Image is required for color extraction"
+ logger.error(error_msg)
+ return {"success": False, "error": error_msg}
+
+ logger.info("Processing image mode with agentic pipeline...")
+
+ # Build user preferences from optional fields
+ user_prefs = []
+ if style:
+ user_prefs.append(f"Style: {style}")
+ if description:
+ user_prefs.append(f"Additional notes: {description}")
+ user_preferences = ". ".join(user_prefs) if user_prefs else ""
+
+ # Execute pipeline with streaming
+ async def stream():
+ yield "[VALIDATE] Validating image input...\n"
+ yield "[EXTRACT] Extracting dominant colors with KMeans...\n"
+
+ try:
+ # Extract first, so the UI can render quickly
+ from color_extractor import ColorExtractor
+
+ extractor = ColorExtractor(num_colors=num_colors)
+ extracted_colors = extractor.extract_from_base64(image_base64)
+
+ # Generate a downscaled preview image on the server but DO NOT include
+ # the full pixel map in the response (frontend will sample pixels from canvas).
+ image_base64_clean, pixel_map, img_width, img_height = extractor.get_pixel_map(
+ image_base64, max_pixels=0, max_dim=800
+ )
+
+ # Ensure image is formatted as data URI for immediate frontend display
+ if not image_base64_clean.startswith("data:"):
+ image_data_uri = f"data:image/jpeg;base64,{image_base64_clean}"
+ else:
+ image_data_uri = image_base64_clean
+
+ # Stream a lightweight, machine-parsable block inside logs with image and colors
+ # (frontend reads this before the final OUTPUT_START JSON for immediate display)
+ yield "\n---EXTRACTED_COLORS_START---\n"
+ yield json.dumps(
+ {
+ "success": True,
+ "extractedColors": extracted_colors,
+ "numColors": num_colors,
+ "image": image_data_uri,
+ "imageWidth": img_width,
+ "imageHeight": img_height,
+ # stable key for potential frontend caching
+ "imageKey": hashlib.sha256(image_base64.encode("utf-8")).hexdigest()[:16],
+ }
+ )
+ yield "\n---EXTRACTED_COLORS_END---\n\n"
+
+ yield "[REFINE] Refining palette with LLM...\n"
+ yield "[FORMAT] Formatting output...\n"
+
+ try:
+ result = await execute_pipeline(
+ image_base64=image_base64,
+ user_preferences=user_preferences,
+ num_colors=num_colors,
+ extracted_colors=extracted_colors,
+ )
+ except Exception as llm_error:
+ logger.warning(f"LLM refinement failed ({llm_error}), using extracted colors as fallback")
+ # CREATE FALLBACK PALETTE from extracted colors
+ result = {
+ "success": True,
+ "palette": {color: color for color in extracted_colors[:min(num_colors, len(extracted_colors))]},
+ "roles": {
+ "primary": extracted_colors[0] if extracted_colors else "#000000",
+ "secondary": extracted_colors[1] if len(extracted_colors) > 1 else extracted_colors[0],
+ "accent": extracted_colors[2] if len(extracted_colors) > 2 else extracted_colors[0],
+ "background": "#ffffff",
+ "surface": "#f5f5f5",
+ "text": "#333333",
+ "muted": "#999999",
+ },
+ "colorTheory": "Extracted from image using KMeans clustering",
+ "extractedColors": extracted_colors,
+ }
+
+ # Ensure image data is properly formatted as data URI and include resized preview
+ result["image"] = image_base64_clean if image_base64_clean.startswith("data:") else f"data:image/png;base64,{image_base64_clean}"
+ result["imageWidth"] = img_width
+ result["imageHeight"] = img_height
+
+ # Safety: cap final JSON size; drop optional heavy fields if too large
+ serialized = json.dumps(result)
+ if len(serialized.encode("utf-8")) > 2_000_000:
+ # Remove pixel map if present
+ if "pixels" in result:
+ del result["pixels"]
+ serialized = json.dumps(result)
+ if len(serialized.encode("utf-8")) > 2_000_000:
+ # As a last resort, remove the image preview to keep payload small
+ if "image" in result:
+ del result["image"]
+
+ logger.info(f"Result prepared: image preview present={ 'image' in result }, pixels_sent={ 'pixels' in result }, dimensions={img_width}x{img_height}")
+
+ yield "\n---OUTPUT_START---\n"
+ yield json.dumps(result, indent=2)
+ yield "\n---OUTPUT_END---\n"
+
+ except Exception as e:
+ logger.error(f"Pipeline execution failed: {e}")
+ import traceback
+ traceback.print_exc()
+ error_response = {
+ "success": False,
+ "error": str(e),
+ "status": "failed"
+ }
+ yield json.dumps(error_response, indent=2)
+
+ return stream()
+
+ except Exception as e:
+ logger.error(f"Tool execution failed: {e}")
+ import traceback
+ traceback.print_exc()
+ return {
+ "success": False,
+ "error": str(e),
+ "status": "error"
+ }
diff --git a/services/python-tools/tools/json-to-schema-v2/requirements.txt b/services/python-tools/tools/json-to-schema-v2/requirements.txt
new file mode 100644
index 0000000..d3a2b49
--- /dev/null
+++ b/services/python-tools/tools/json-to-schema-v2/requirements.txt
@@ -0,0 +1,5 @@
+# JSON to Schema V2 — Dependencies
+# Core LangGraph agent framework
+langgraph>=0.2.0
+langchain-openai>=0.2.0
+langchain-core>=0.3.0
diff --git a/services/python-tools/tools/json-to-schema-v2/schema_agents.py b/services/python-tools/tools/json-to-schema-v2/schema_agents.py
new file mode 100644
index 0000000..9c8d417
--- /dev/null
+++ b/services/python-tools/tools/json-to-schema-v2/schema_agents.py
@@ -0,0 +1,657 @@
+"""
+JSON to Schema V2 — Agent Nodes & Graph
+=========================================
+LangGraph StateGraph with typed state and 5 specialist nodes.
+
+KEY ARCHITECTURAL FIX (v2.2):
+ Old approach: Parser does analysis → Architect designs schema from scratch
+ Problem: LLM creates generic EAV meta-schemas instead of specific tables
+
+ New approach: Parser BUILDS a draft schema deterministically → Architect REFINES it
+ This gives the LLM a much simpler task (adjust types, handle edge cases)
+ while the deterministic builder ensures every JSON key gets its own table/column.
+
+ Pipeline:
+ Parser (builds draft schema) → Architect (refines) → Reviewer → Compiler → Documenter
+"""
+
+import json
+import logging
+import re
+from typing import TypedDict
+
+from langgraph.graph import StateGraph, END
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+from schema_config import (
+ OXLO_API_KEY, OXLO_BASE_URL,
+ ARCHITECT_MODEL, REVIEWER_MODEL, DOCUMENTER_MODEL,
+ MAX_REVIEW_ITERATIONS,
+)
+from schema_prompts import ARCHITECT_PROMPT, REVIEWER_PROMPT, DOCUMENTER_PROMPT
+from schema_compilers import compile_schema
+
+logger = logging.getLogger("json-to-schema")
+
+
+# ─── State Schema ─────────────────────────────────────────────────────
+
+class SchemaState(TypedDict):
+ """Shared state flowing through the schema generation pipeline."""
+ # Input
+ raw_json: str
+ parsed_data: dict
+ output_format: str
+ user_model: str
+
+ # Parser output (deterministic)
+ structure_analysis: str
+ draft_schema: dict # NEW: deterministic draft schema
+
+ # Architect output
+ proposed_schema: dict
+ design_decisions: list[str]
+
+ # Reviewer output
+ review_approved: bool
+ review_issues: list[dict]
+ review_refinements: list[dict]
+ review_iteration: int
+
+ # Compiler output (deterministic)
+ compiled_output: str
+
+ # Documenter output
+ documentation: str
+
+ # Metadata
+ status: str
+
+
+# ─── Helpers ──────────────────────────────────────────────────────────
+
+def get_llm(model: str, temperature: float = 0.1) -> ChatOpenAI:
+ """Create an LLM instance pointing to Oxlo API."""
+ return ChatOpenAI(
+ model=model,
+ api_key=OXLO_API_KEY,
+ base_url=OXLO_BASE_URL,
+ temperature=temperature,
+ max_tokens=16384,
+ )
+
+
+def _parse_json_from_llm(content: str) -> dict:
+ """Robustly extract a JSON object from LLM output."""
+ content = content.strip()
+
+ if "```" in content:
+ blocks = content.split("```")
+ for block in blocks:
+ cleaned = block.strip()
+ if cleaned.startswith("json"):
+ cleaned = cleaned[4:].strip()
+ if cleaned.startswith("{"):
+ try:
+ return json.loads(cleaned)
+ except json.JSONDecodeError:
+ continue
+
+ try:
+ return json.loads(content)
+ except json.JSONDecodeError:
+ start = content.find("{")
+ end = content.rfind("}")
+ if start != -1 and end != -1:
+ try:
+ return json.loads(content[start:end + 1])
+ except json.JSONDecodeError:
+ pass
+
+ return {}
+
+
+def _get_depth(obj, depth=0) -> int:
+ """Calculate the maximum nesting depth of a JSON structure."""
+ if isinstance(obj, dict):
+ if not obj:
+ return depth
+ return max(_get_depth(v, depth + 1) for v in obj.values())
+ elif isinstance(obj, list):
+ if not obj:
+ return depth
+ return max(_get_depth(item, depth + 1) for item in obj[:5])
+ return depth
+
+
+def _sanitize_name(name: str) -> str:
+ """Convert a JSON key to a safe SQL table/column name."""
+ # Remove dangerous characters, path traversal, special prefixes
+ cleaned = re.sub(r'[^a-zA-Z0-9_]', '_', name)
+ cleaned = re.sub(r'_+', '_', cleaned).strip('_')
+ # Don't start with a number
+ if cleaned and cleaned[0].isdigit():
+ cleaned = f"col_{cleaned}"
+ return cleaned.lower() or "unnamed"
+
+
+# ─── Dangerous Key Detection ─────────────────────────────────────────
+
+DANGEROUS_KEYS = {
+ # Prototype pollution
+ "__proto__", "constructor", "prototype", "__class__",
+ "__subclasses__", "__globals__", "__builtins__",
+ # NoSQL operators
+ "$gt", "$ne", "$lt", "$gte", "$lte", "$regex", "$where", "$exists",
+ "$in", "$nin", "$or", "$and", "$not", "$set", "$unset",
+}
+
+SHELL_METACHARACTERS = set(';|&$`><')
+
+
+def _is_dangerous_key(key: str) -> tuple[bool, str]:
+ """Check if a JSON key is an attack payload. Returns (is_dangerous, threat_type)."""
+ lower = key.lower()
+ if lower in DANGEROUS_KEYS:
+ if lower.startswith("$"):
+ return True, "nosql_operator"
+ return True, "prototype_pollution"
+ # Path traversal
+ if ".." in key or key.startswith("/") or "%2f" in lower or "%2e" in lower:
+ return True, "path_traversal"
+ # Shell metacharacters in keys
+ if any(c in key for c in SHELL_METACHARACTERS):
+ return True, "shell_injection"
+ return False, ""
+
+
+def _is_type_confused(value) -> bool:
+ """Check if a string value LOOKS like a typed primitive but contains injection."""
+ if not isinstance(value, str):
+ return False
+ v = value.strip().lower()
+ # String "true"/"false" → type confusion
+ if v in ("true", "false"):
+ return True
+ # String number with injection payload
+ if re.match(r'^-?\d+', v) and not v.replace('-', '').replace('.', '').isdigit():
+ return True
+ return False
+
+
+def _infer_safe_type(value) -> str:
+ """Infer a safe SQL type from a JSON value. Defaults to TEXT for safety."""
+ if isinstance(value, bool):
+ return "boolean"
+ if isinstance(value, int):
+ if abs(value) > 2_147_483_647:
+ return "bigint"
+ return "integer"
+ if isinstance(value, float):
+ return "decimal"
+ if isinstance(value, dict):
+ # Check for NoSQL operators as values
+ if any(k.startswith("$") for k in value.keys()):
+ return "text" # Don't create child table for operator dicts
+ return "jsonb"
+ if isinstance(value, list):
+ return "jsonb"
+ # For strings: always TEXT (safe default)
+ if isinstance(value, str):
+ return "text"
+ return "text"
+
+
+# ─── Deterministic Schema Builder ────────────────────────────────────
+
+def _build_draft_schema(data: dict) -> dict:
+ """
+ Build a complete schema deterministically from JSON structure.
+
+ Rules:
+ - Each top-level key → its own table
+ - Each sub-key → a column in that table
+ - Nested objects → separate child table with FK
+ - Dangerous keys → stored in a safe key_value table
+ - Deeply nested (>5 levels) → flattened with nesting_level column
+ - All string values → TEXT (safe default)
+ - _comment keys → ignored
+ """
+ tables = []
+ design_decisions = []
+ has_dangerous_keys = False
+
+ if not isinstance(data, dict):
+ # If top-level is an array, create one table
+ tables.append({
+ "name": "items",
+ "columns": [
+ {"name": "id", "type": "serial", "isPrimary": True, "isNullable": False, "isUnique": False},
+ {"name": "data", "type": "jsonb", "isPrimary": False, "isNullable": True, "isUnique": False},
+ {"name": "created_at", "type": "timestamp", "isPrimary": False, "isNullable": False, "isUnique": False, "defaultValue": "CURRENT_TIMESTAMP"},
+ ],
+ "foreignKeys": [],
+ "indexes": [],
+ })
+ return {"tables": tables, "designDecisions": ["Top-level is an array, stored as JSONB rows"]}
+
+ # Process each top-level key
+ for top_key, top_value in data.items():
+ # Skip comment/meta keys
+ if top_key.startswith("_"):
+ design_decisions.append(f"Skipped meta key '{top_key}'")
+ continue
+
+ if not isinstance(top_value, dict):
+ # Scalar top-level value — add to a general table later
+ continue
+
+ table_name = _sanitize_name(top_key)
+
+ # Check if this object contains dangerous keys (RECURSIVE)
+ dangerous_entries = []
+ safe_entries = {}
+
+ def _scan_keys_recursive(obj, prefix=""):
+ """Recursively scan ALL nesting levels for dangerous keys."""
+ nonlocal has_dangerous_keys
+ if isinstance(obj, dict):
+ for k, v in obj.items():
+ full_path = f"{prefix}.{k}" if prefix else k
+ is_bad, threat = _is_dangerous_key(k)
+ if is_bad:
+ dangerous_entries.append((full_path, v, threat))
+ has_dangerous_keys = True
+ else:
+ _scan_keys_recursive(v, full_path)
+ elif isinstance(obj, list):
+ for item in obj[:5]:
+ _scan_keys_recursive(item, prefix)
+
+ _scan_keys_recursive(top_value)
+
+ for key, value in top_value.items():
+ is_bad, _ = _is_dangerous_key(key)
+ if not is_bad:
+ safe_entries[key] = value
+
+ # Build columns for safe entries
+ columns = [
+ {"name": "id", "type": "serial", "isPrimary": True, "isNullable": False, "isUnique": False},
+ ]
+
+ child_tables = []
+
+ for key, value in safe_entries.items():
+ col_name = _sanitize_name(key)
+ # Type confusion: string "true"/"18; DROP TABLE" stays as _raw TEXT
+ if _is_type_confused(value):
+ col_name = f"{col_name}_raw"
+
+ if isinstance(value, dict):
+ # Check nesting depth
+ depth = _get_depth(value)
+ if depth > 5:
+ # Deeply nested → flatten
+ columns.append({"name": f"{col_name}_deepest_value", "type": "text", "isPrimary": False, "isNullable": True, "isUnique": False})
+ columns.append({"name": f"{col_name}_nesting_level", "type": "integer", "isPrimary": False, "isNullable": False, "isUnique": False, "defaultValue": "0"})
+ design_decisions.append(f"Flattened deeply nested '{key}' (depth={depth}) with nesting_level + deepest_value")
+ else:
+ # Nested object → child table
+ child_table_name = f"{table_name}_{col_name}"
+ child_cols = [
+ {"name": "id", "type": "serial", "isPrimary": True, "isNullable": False, "isUnique": False},
+ {"name": f"{table_name}_id", "type": "integer", "isPrimary": False, "isNullable": False, "isUnique": False},
+ ]
+ for sub_key, sub_value in value.items():
+ is_bad, threat = _is_dangerous_key(sub_key)
+ if is_bad:
+ dangerous_entries.append((f"{key}.{sub_key}", sub_value, threat))
+ else:
+ sub_col_name = _sanitize_name(sub_key)
+ if _is_type_confused(sub_value):
+ sub_col_name = f"{sub_col_name}_raw"
+ sub_type = _infer_safe_type(sub_value)
+ child_cols.append({"name": sub_col_name, "type": sub_type, "isPrimary": False, "isNullable": True, "isUnique": False})
+ child_cols.append({"name": "created_at", "type": "timestamp", "isPrimary": False, "isNullable": False, "isUnique": False, "defaultValue": "CURRENT_TIMESTAMP"})
+ child_tables.append({
+ "name": child_table_name,
+ "columns": child_cols,
+ "foreignKeys": [{"column": f"{table_name}_id", "referencesTable": table_name, "referencesColumn": "id"}],
+ "indexes": [{"columns": [f"{table_name}_id"], "unique": False}],
+ })
+ elif isinstance(value, list):
+ if value and isinstance(value[0], dict):
+ # Array of objects → child table
+ child_table_name = f"{table_name}_{col_name}"
+ child_cols = [
+ {"name": "id", "type": "serial", "isPrimary": True, "isNullable": False, "isUnique": False},
+ {"name": f"{table_name}_id", "type": "integer", "isPrimary": False, "isNullable": False, "isUnique": False},
+ ]
+ for sub_key, sub_value in value[0].items():
+ sub_col_name = _sanitize_name(sub_key)
+ sub_type = _infer_safe_type(sub_value)
+ child_cols.append({"name": sub_col_name, "type": sub_type, "isPrimary": False, "isNullable": True, "isUnique": False})
+ child_cols.append({"name": "created_at", "type": "timestamp", "isPrimary": False, "isNullable": False, "isUnique": False, "defaultValue": "CURRENT_TIMESTAMP"})
+ child_tables.append({
+ "name": child_table_name,
+ "columns": child_cols,
+ "foreignKeys": [{"column": f"{table_name}_id", "referencesTable": table_name, "referencesColumn": "id"}],
+ "indexes": [{"columns": [f"{table_name}_id"], "unique": False}],
+ })
+ else:
+ # Array of primitives → JSONB
+ columns.append({"name": col_name, "type": "jsonb", "isPrimary": False, "isNullable": True, "isUnique": False})
+ else:
+ # Scalar value → column
+ col_type = _infer_safe_type(value)
+ columns.append({"name": col_name, "type": col_type, "isPrimary": False, "isNullable": True, "isUnique": False})
+
+ # Add created_at to main table
+ columns.append({"name": "created_at", "type": "timestamp", "isPrimary": False, "isNullable": False, "isUnique": False, "defaultValue": "CURRENT_TIMESTAMP"})
+
+ tables.append({
+ "name": table_name,
+ "columns": columns,
+ "foreignKeys": [],
+ "indexes": [],
+ })
+ tables.extend(child_tables)
+
+ # If there were dangerous entries, log them
+ if dangerous_entries:
+ design_decisions.append(
+ f"Table '{table_name}': quarantined {len(dangerous_entries)} dangerous keys "
+ f"({', '.join(k for k, *_ in dangerous_entries)}) in dangerous_key_values table"
+ )
+
+ # Create a dangerous_key_values table if any dangerous keys were found
+ if has_dangerous_keys:
+ tables.append({
+ "name": "dangerous_key_values",
+ "columns": [
+ {"name": "id", "type": "serial", "isPrimary": True, "isNullable": False, "isUnique": False},
+ {"name": "source_table", "type": "text", "isPrimary": False, "isNullable": False, "isUnique": False},
+ {"name": "key_path", "type": "text", "isPrimary": False, "isNullable": False, "isUnique": False},
+ {"name": "raw_value", "type": "text", "isPrimary": False, "isNullable": True, "isUnique": False},
+ {"name": "threat_type", "type": "text", "isPrimary": False, "isNullable": False, "isUnique": False},
+ {"name": "is_confirmed_attack", "type": "boolean", "isPrimary": False, "isNullable": False, "isUnique": False, "defaultValue": "TRUE"},
+ {"name": "detected_at", "type": "timestamp", "isPrimary": False, "isNullable": False, "isUnique": False, "defaultValue": "CURRENT_TIMESTAMP"},
+ ],
+ "foreignKeys": [],
+ "indexes": [
+ {"columns": ["source_table"], "unique": False},
+ {"columns": ["threat_type"], "unique": False},
+ ],
+ })
+ design_decisions.append("Created dangerous_key_values table with threat_type classification (prototype_pollution, path_traversal, nosql_operator, shell_injection)")
+
+ return {"tables": tables, "designDecisions": design_decisions}
+
+
+# ─── Node 1: Parser (Deterministic Schema Builder) ───────────────────
+
+def parser_node(state: SchemaState) -> dict:
+ """
+ Pure Python JSON parsing + DETERMINISTIC SCHEMA BUILDING.
+
+ This is the key fix: instead of just analyzing the JSON structure,
+ we BUILD a complete draft schema that the Architect refines.
+ This ensures every JSON key gets its own table/column.
+ """
+ raw_json = state["raw_json"]
+
+ logger.info("[Parser] Analyzing JSON structure and building draft schema...")
+
+ data = json.loads(raw_json)
+
+ # Build draft schema deterministically
+ draft_schema = _build_draft_schema(data)
+
+ # Generate human-readable analysis
+ json_str = json.dumps(data, indent=2)
+ stats = (
+ f"JSON Stats: {len(json_str)} bytes, "
+ f"Max depth: {_get_depth(data)}, "
+ f"Top-level type: {type(data).__name__}, "
+ f"Top-level keys: {len(data) if isinstance(data, dict) else 'N/A'}"
+ )
+
+ draft_tables = draft_schema.get("tables", [])
+ draft_summary = f"Draft schema: {len(draft_tables)} tables built deterministically"
+ for t in draft_tables:
+ cols = len(t.get("columns", []))
+ fks = len(t.get("foreignKeys", []))
+ draft_summary += f"\n - {t['name']}: {cols} columns, {fks} FKs"
+
+ structure_analysis = stats + "\n\n" + draft_summary
+
+ logger.info(f"[Parser] Built draft schema with {len(draft_tables)} tables")
+
+ return {
+ "parsed_data": data,
+ "structure_analysis": structure_analysis,
+ "draft_schema": draft_schema,
+ "status": "parsing_complete",
+ }
+
+
+# ─── Node 2: Architect (LLM Refines Draft Schema) ────────────────────
+
+def architect_node(state: SchemaState) -> dict:
+ """
+ LLM REFINES the draft schema built by the Parser.
+
+ KEY FIX: The Architect no longer designs from scratch.
+ It receives a complete draft schema and makes intelligent adjustments:
+ - Type refinements
+ - Safety overrides
+ - Naming improvements
+ - Index suggestions
+ - Structural changes for better normalization
+ """
+ logger.info("[Architect] Refining draft schema...")
+ model = state.get("user_model") or ARCHITECT_MODEL
+ llm = get_llm(model, 0.2)
+
+ draft = state.get("draft_schema", {})
+ raw_json_full = state["raw_json"]
+
+ user_content = (
+ f"## Draft Schema (built deterministically from JSON structure):\n"
+ f"```json\n{json.dumps(draft, indent=2)}\n```\n\n"
+ f"## Raw JSON Data:\n```json\n{raw_json_full}\n```\n\n"
+ f"## Target Format: {state['output_format']}\n\n"
+ )
+
+ # Include reviewer feedback if this is a refinement pass
+ if state.get("review_issues"):
+ user_content += (
+ f"## Reviewer Feedback (address these issues):\n"
+ f"```json\n{json.dumps(state['review_issues'], indent=2)}\n```\n\n"
+ f"## Suggested Refinements:\n"
+ f"```json\n{json.dumps(state.get('review_refinements', []), indent=2)}\n```\n\n"
+ )
+
+ response = llm.invoke([
+ SystemMessage(content=ARCHITECT_PROMPT),
+ HumanMessage(content=user_content),
+ ])
+
+ result = _parse_json_from_llm(response.content)
+
+ # If the LLM returned fewer tables than draft, fall back to draft
+ # This prevents the LLM from collapsing specific tables into EAV
+ tables = result.get("tables", [])
+ draft_tables = draft.get("tables", [])
+ if len(tables) < len(draft_tables):
+ logger.warning(f"[Architect] LLM returned {len(tables)} tables vs draft's {len(draft_tables)} — using draft")
+ result = draft
+ tables = draft_tables
+
+ decisions = result.get("designDecisions", [])
+ # Merge draft decisions
+ draft_decisions = draft.get("designDecisions", [])
+ all_decisions = draft_decisions + decisions
+
+ logger.info(f"[Architect] Final schema: {len(tables)} tables, {len(all_decisions)} design decisions")
+
+ return {
+ "proposed_schema": result,
+ "design_decisions": all_decisions,
+ "status": "architecture_complete",
+ }
+
+
+# ─── Node 3: Reviewer (LLM Quality Check) ────────────────────────────
+
+def reviewer_node(state: SchemaState) -> dict:
+ """Reviews schema for normalization issues, safety, and anti-patterns."""
+ iteration = state.get("review_iteration", 0)
+ schema = state.get("proposed_schema", {})
+
+ logger.info(f"[Reviewer] Reviewing schema (iteration {iteration + 1})...")
+ model = state.get("user_model") or REVIEWER_MODEL
+ llm = get_llm(model, 0.1)
+
+ response = llm.invoke([
+ SystemMessage(content=REVIEWER_PROMPT),
+ HumanMessage(content=(
+ f"## Original JSON Structure:\n{state['structure_analysis']}\n\n"
+ f"## Proposed Schema:\n```json\n{json.dumps(schema, indent=2)}\n```\n\n"
+ f"Review this schema for correctness and quality."
+ )),
+ ])
+
+ review = _parse_json_from_llm(response.content)
+
+ approved = review.get("approved", True)
+ issues = review.get("issues", [])
+ refinements = review.get("refinements", [])
+
+ if refinements and approved:
+ schema = _apply_refinements(schema, refinements)
+
+ logger.info(
+ f"[Reviewer] {'Approved' if approved else 'Rejected'} — "
+ f"{len(issues)} issues, {len(refinements)} refinements"
+ )
+
+ return {
+ "proposed_schema": schema,
+ "review_approved": approved,
+ "review_issues": issues,
+ "review_refinements": refinements,
+ "review_iteration": iteration + 1,
+ "status": "review_complete",
+ }
+
+
+def _apply_refinements(schema: dict, refinements: list) -> dict:
+ """Apply minor reviewer refinements directly to the schema."""
+ for ref in refinements:
+ table_name = ref.get("table", "")
+ action = ref.get("action", "")
+
+ for table in schema.get("tables", []):
+ if table["name"] == table_name:
+ if action == "add_column" and ref.get("column"):
+ table.setdefault("columns", []).append(ref["column"])
+ elif action == "add_index" and ref.get("index"):
+ table.setdefault("indexes", []).append(ref["index"])
+
+ return schema
+
+
+# ─── Node 4: Compiler (Pure Deterministic) ───────────────────────────
+
+def compiler_node(state: SchemaState) -> dict:
+ """Deterministic compilation to target format. NO LLM."""
+ schema = state.get("proposed_schema", {})
+ output_format = state.get("output_format", "postgresql")
+
+ logger.info(f"[Compiler] Generating {output_format} output...")
+
+ compiled = compile_schema(schema, output_format)
+
+ logger.info(f"[Compiler] Generated {len(compiled)} chars of {output_format}")
+ return {
+ "compiled_output": compiled,
+ "status": "compilation_complete",
+ }
+
+
+# ─── Node 5: Documenter (LLM Documentation) ──────────────────────────
+
+def documenter_node(state: SchemaState) -> dict:
+ """Generates documentation, example queries, and design notes."""
+ logger.info("[Documenter] Writing documentation...")
+ llm = get_llm(DOCUMENTER_MODEL, 0.4)
+
+ response = llm.invoke([
+ SystemMessage(content=DOCUMENTER_PROMPT),
+ HumanMessage(content=(
+ f"## Schema (compiled {state.get('output_format', 'SQL')}):\n"
+ f"```\n{state.get('compiled_output', '')[:6000]}\n```\n\n"
+ f"## Design Decisions:\n"
+ + "\n".join(f"- {d}" for d in state.get("design_decisions", []))
+ )),
+ ])
+
+ logger.info("[Documenter] Documentation generated")
+ return {
+ "documentation": response.content,
+ "status": "complete",
+ }
+
+
+# ─── Routing Logic ────────────────────────────────────────────────────
+
+def should_refine(state: SchemaState) -> str:
+ """After review, decide: refine or compile."""
+ approved = state.get("review_approved", True)
+ iteration = state.get("review_iteration", 0)
+
+ if not approved and iteration < MAX_REVIEW_ITERATIONS:
+ logger.info("[Router] Reviewer rejected — sending back to Architect")
+ return "refine"
+
+ if not approved:
+ logger.info("[Router] Max iterations — proceeding despite issues")
+
+ return "compile"
+
+
+# ─── Graph Builder ────────────────────────────────────────────────────
+
+def build_graph():
+ """
+ Build the LangGraph schema generation workflow.
+
+ parser (builds draft) → architect (refines) → reviewer ─┬→ compiler → documenter → END
+ ↑ │
+ └── refine ──────────────┘
+ """
+ wf = StateGraph(SchemaState)
+
+ wf.add_node("parser", parser_node)
+ wf.add_node("architect", architect_node)
+ wf.add_node("reviewer", reviewer_node)
+ wf.add_node("compiler", compiler_node)
+ wf.add_node("documenter", documenter_node)
+
+ wf.set_entry_point("parser")
+ wf.add_edge("parser", "architect")
+ wf.add_edge("architect", "reviewer")
+
+ wf.add_conditional_edges(
+ "reviewer",
+ should_refine,
+ {"refine": "architect", "compile": "compiler"},
+ )
+
+ wf.add_edge("compiler", "documenter")
+ wf.add_edge("documenter", END)
+
+ return wf.compile()
diff --git a/services/python-tools/tools/json-to-schema-v2/schema_compilers.py b/services/python-tools/tools/json-to-schema-v2/schema_compilers.py
new file mode 100644
index 0000000..f7b45fb
--- /dev/null
+++ b/services/python-tools/tools/json-to-schema-v2/schema_compilers.py
@@ -0,0 +1,420 @@
+"""
+JSON to Schema V2 — Deterministic Compilers
+=============================================
+Pure Python code generators for each output format.
+These take a structured schema definition (from the LLM architect)
+and produce syntactically perfect, production-ready code.
+
+ZERO LLM involvement — this is deterministic compilation.
+
+KEY FIX (v2.1): All FK field accessors use _get_fk_field() which
+handles multiple key name variants from LLM output:
+ referencesTable / references_table / refTable / ref_table / table
+"""
+
+import logging
+
+logger = logging.getLogger("json-to-schema")
+
+
+# ─── Type Mapping ─────────────────────────────────────────────────────
+
+PG_TYPE_MAP = {
+ "string": "VARCHAR(255)",
+ "text": "TEXT",
+ "integer": "INTEGER",
+ "bigint": "BIGINT",
+ "float": "DOUBLE PRECISION",
+ "decimal": "DECIMAL(10, 2)",
+ "boolean": "BOOLEAN",
+ "date": "DATE",
+ "datetime": "TIMESTAMP WITH TIME ZONE",
+ "timestamp": "TIMESTAMP WITH TIME ZONE",
+ "uuid": "UUID",
+ "json": "JSONB",
+ "jsonb": "JSONB",
+ "array": "JSONB",
+ "serial": "SERIAL",
+ "bigserial": "BIGSERIAL",
+}
+
+MYSQL_TYPE_MAP = {
+ "string": "VARCHAR(255)",
+ "text": "TEXT",
+ "integer": "INT",
+ "bigint": "BIGINT",
+ "float": "DOUBLE",
+ "decimal": "DECIMAL(10, 2)",
+ "boolean": "TINYINT(1)",
+ "date": "DATE",
+ "datetime": "DATETIME",
+ "timestamp": "TIMESTAMP",
+ "uuid": "CHAR(36)",
+ "json": "JSON",
+ "jsonb": "JSON",
+ "array": "JSON",
+ "serial": "INT AUTO_INCREMENT",
+ "bigserial": "BIGINT AUTO_INCREMENT",
+}
+
+PRISMA_TYPE_MAP = {
+ "string": "String",
+ "text": "String",
+ "integer": "Int",
+ "bigint": "BigInt",
+ "float": "Float",
+ "decimal": "Decimal",
+ "boolean": "Boolean",
+ "date": "DateTime",
+ "datetime": "DateTime",
+ "timestamp": "DateTime",
+ "uuid": "String @default(uuid())",
+ "json": "Json",
+ "jsonb": "Json",
+ "array": "Json",
+ "serial": "Int @default(autoincrement())",
+}
+
+MONGOOSE_TYPE_MAP = {
+ "string": "String",
+ "text": "String",
+ "integer": "Number",
+ "bigint": "Number",
+ "float": "Number",
+ "decimal": "Number",
+ "boolean": "Boolean",
+ "date": "Date",
+ "datetime": "Date",
+ "timestamp": "Date",
+ "uuid": "String",
+ "json": "Schema.Types.Mixed",
+ "jsonb": "Schema.Types.Mixed",
+ "array": "[Schema.Types.Mixed]",
+}
+
+DRIZZLE_TYPE_MAP = {
+ "string": "varchar('name', { length: 255 })",
+ "text": "text('name')",
+ "integer": "integer('name')",
+ "bigint": "bigint('name', { mode: 'number' })",
+ "float": "doublePrecision('name')",
+ "decimal": "decimal('name', { precision: 10, scale: 2 })",
+ "boolean": "boolean('name')",
+ "date": "date('name')",
+ "datetime": "timestamp('name')",
+ "timestamp": "timestamp('name')",
+ "uuid": "uuid('name').defaultRandom()",
+ "json": "jsonb('name')",
+ "jsonb": "jsonb('name')",
+ "serial": "serial('name')",
+}
+
+
+# ─── Helpers ──────────────────────────────────────────────────────────
+
+def _resolve_type(col_type: str, type_map: dict) -> str:
+ """Resolve a column type through the type map, with fallback."""
+ normalized = col_type.lower().strip()
+ return type_map.get(normalized, col_type.upper())
+
+
+def _get_fk_ref_table(fk: dict) -> str:
+ """
+ Extract the referenced table name from a foreign key dict.
+ Handles multiple key name variants that LLMs might generate:
+ referencesTable, references_table, refTable, ref_table, table, target_table
+ """
+ for key in ("referencesTable", "references_table", "refTable", "ref_table",
+ "table", "target_table", "targetTable", "referenced_table"):
+ if key in fk:
+ return fk[key]
+ return fk.get("references", {}).get("table", "unknown_table")
+
+
+def _get_fk_ref_column(fk: dict) -> str:
+ """
+ Extract the referenced column name from a foreign key dict.
+ Handles multiple key name variants.
+ """
+ for key in ("referencesColumn", "references_column", "refColumn", "ref_column",
+ "column_ref", "targetColumn", "target_column", "referenced_column"):
+ if key in fk:
+ return fk[key]
+ ref = fk.get("references", {})
+ if isinstance(ref, dict):
+ return ref.get("column", "id")
+ return "id"
+
+
+def _get_fk_column(fk: dict) -> str:
+ """Extract the source column name from a foreign key dict."""
+ for key in ("column", "source_column", "sourceColumn", "from_column", "fromColumn"):
+ if key in fk:
+ return fk[key]
+ return "unknown_id"
+
+
+def _safe_col(col: dict, field: str, default=None):
+ """Safely get a column field, handling camelCase and snake_case."""
+ camel_map = {
+ "isPrimary": ["isPrimary", "is_primary", "primary"],
+ "isNullable": ["isNullable", "is_nullable", "nullable"],
+ "isUnique": ["isUnique", "is_unique", "unique"],
+ "defaultValue": ["defaultValue", "default_value", "default"],
+ }
+ keys = camel_map.get(field, [field])
+ for k in keys:
+ if k in col:
+ return col[k]
+ return default
+
+
+# ─── Compiler: PostgreSQL ─────────────────────────────────────────────
+
+def compile_postgresql(schema: dict) -> str:
+ """Generate PostgreSQL DDL from structured schema."""
+ output = "-- Generated by Oxtools JSON-to-Schema V2 (Agentic Pipeline)\n"
+ output += "-- Format: PostgreSQL\n\n"
+
+ tables = schema.get("tables", [])
+ if not tables:
+ return output + "-- No tables in schema. The architect may have returned an unexpected format.\n"
+
+ for table in tables:
+ name = table.get("name", "unnamed_table")
+ output += f"CREATE TABLE {name} (\n"
+
+ col_lines = []
+ for col in table.get("columns", []):
+ col_name = col.get("name", "unnamed_col")
+ pg_type = _resolve_type(col.get("type", "text"), PG_TYPE_MAP)
+ line = f" {col_name} {pg_type}"
+ if _safe_col(col, "isPrimary"):
+ line += " PRIMARY KEY"
+ if not _safe_col(col, "isNullable", True) and not _safe_col(col, "isPrimary"):
+ line += " NOT NULL"
+ if _safe_col(col, "isUnique"):
+ line += " UNIQUE"
+ default_val = _safe_col(col, "defaultValue")
+ if default_val:
+ line += f" DEFAULT {default_val}"
+ col_lines.append(line)
+
+ for fk in table.get("foreignKeys", []):
+ fk_col = _get_fk_column(fk)
+ ref_table = _get_fk_ref_table(fk)
+ ref_col = _get_fk_ref_column(fk)
+ col_lines.append(
+ f" CONSTRAINT fk_{name}_{fk_col} "
+ f"FOREIGN KEY ({fk_col}) "
+ f"REFERENCES {ref_table}({ref_col}) "
+ f"ON DELETE CASCADE"
+ )
+
+ output += ",\n".join(col_lines) + "\n);\n\n"
+
+ # Indexes
+ for idx in table.get("indexes", []):
+ cols_list = idx.get("columns", [])
+ if not cols_list:
+ continue
+ cols = ", ".join(cols_list)
+ unique = "UNIQUE " if idx.get("unique") else ""
+ idx_name = f"idx_{name}_{'_'.join(cols_list)}"
+ output += f"CREATE {unique}INDEX {idx_name} ON {name} ({cols});\n"
+
+ output += "\n"
+
+ return output
+
+
+# ─── Compiler: MySQL ──────────────────────────────────────────────────
+
+def compile_mysql(schema: dict) -> str:
+ """Generate MySQL DDL from structured schema."""
+ output = "-- Generated by Oxtools JSON-to-Schema V2 (Agentic Pipeline)\n"
+ output += "-- Format: MySQL\n\n"
+
+ for table in schema.get("tables", []):
+ name = table.get("name", "unnamed_table")
+ output += f"CREATE TABLE `{name}` (\n"
+
+ col_lines = []
+ for col in table.get("columns", []):
+ col_name = col.get("name", "unnamed_col")
+ mysql_type = _resolve_type(col.get("type", "text"), MYSQL_TYPE_MAP)
+ line = f" `{col_name}` {mysql_type}"
+ if _safe_col(col, "isPrimary"):
+ line += " PRIMARY KEY"
+ if not _safe_col(col, "isNullable", True) and not _safe_col(col, "isPrimary"):
+ line += " NOT NULL"
+ col_lines.append(line)
+
+ for fk in table.get("foreignKeys", []):
+ fk_col = _get_fk_column(fk)
+ ref_table = _get_fk_ref_table(fk)
+ ref_col = _get_fk_ref_column(fk)
+ col_lines.append(
+ f" FOREIGN KEY (`{fk_col}`) "
+ f"REFERENCES `{ref_table}`(`{ref_col}`)"
+ )
+
+ output += ",\n".join(col_lines) + "\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n\n"
+
+ return output
+
+
+# ─── Compiler: Prisma ─────────────────────────────────────────────────
+
+def compile_prisma(schema: dict) -> str:
+ """Generate Prisma schema from structured schema."""
+ output = "// Generated by Oxtools JSON-to-Schema V2 (Agentic Pipeline)\n"
+ output += "// Format: Prisma ORM\n\n"
+ output += 'generator client {\n provider = "prisma-client-js"\n}\n\n'
+ output += 'datasource db {\n provider = "postgresql"\n url = env("DATABASE_URL")\n}\n\n'
+
+ for table in schema.get("tables", []):
+ raw_name = table.get("name", "unnamed_table")
+ model_name = raw_name[0].upper() + raw_name[1:] if raw_name else "Unknown"
+ output += f"model {model_name} {{\n"
+
+ for col in table.get("columns", []):
+ col_name = col.get("name", "unnamed_col")
+ prisma_type = _resolve_type(col.get("type", "text"), PRISMA_TYPE_MAP)
+ nullable = "?" if _safe_col(col, "isNullable") and not _safe_col(col, "isPrimary") else ""
+ id_attr = " @id" if _safe_col(col, "isPrimary") else ""
+ unique_attr = " @unique" if _safe_col(col, "isUnique") else ""
+ output += f" {col_name} {prisma_type}{nullable}{id_attr}{unique_attr}\n"
+
+ # Relations from foreign keys
+ for fk in table.get("foreignKeys", []):
+ fk_col = _get_fk_column(fk)
+ ref_table = _get_fk_ref_table(fk)
+ ref_col = _get_fk_ref_column(fk)
+ ref_model = ref_table[0].upper() + ref_table[1:] if ref_table else "Unknown"
+ rel_name = fk_col.replace("_id", "").replace("Id", "")
+ output += f" {rel_name} {ref_model} @relation(fields: [{fk_col}], references: [{ref_col}])\n"
+
+ output += "}\n\n"
+
+ return output
+
+
+# ─── Compiler: Mongoose ───────────────────────────────────────────────
+
+def compile_mongoose(schema: dict) -> str:
+ """Generate Mongoose schema definitions from structured schema."""
+ output = "// Generated by Oxtools JSON-to-Schema V2 (Agentic Pipeline)\n"
+ output += "// Format: Mongoose (MongoDB)\n\n"
+ output += 'const mongoose = require("mongoose");\n'
+ output += "const { Schema } = mongoose;\n\n"
+
+ for table in schema.get("tables", []):
+ name = table.get("name", "unnamed_table")
+ var_name = name[0].lower() + name[1:] if name else "unknown"
+ output += f"const {var_name}Schema = new Schema({{\n"
+
+ for col in table.get("columns", []):
+ col_name = col.get("name", "unnamed_col")
+ if _safe_col(col, "isPrimary") and col_name == "_id":
+ continue # MongoDB handles _id automatically
+
+ mg_type = _resolve_type(col.get("type", "text"), MONGOOSE_TYPE_MAP)
+ required = "true" if not _safe_col(col, "isNullable", True) else "false"
+ unique = ", unique: true" if _safe_col(col, "isUnique") else ""
+
+ # Check for foreign key reference
+ fk_ref = None
+ for fk in table.get("foreignKeys", []):
+ if _get_fk_column(fk) == col_name:
+ fk_ref = _get_fk_ref_table(fk)
+ break
+
+ if fk_ref:
+ ref_model = fk_ref[0].upper() + fk_ref[1:] if fk_ref else "Unknown"
+ output += f' {col_name}: {{ type: Schema.Types.ObjectId, ref: "{ref_model}", required: {required} }},\n'
+ else:
+ output += f' {col_name}: {{ type: {mg_type}, required: {required}{unique} }},\n'
+
+ output += "}, { timestamps: true });\n\n"
+
+ model_name = name[0].upper() + name[1:] if name else "Unknown"
+ output += f'const {model_name} = mongoose.model("{model_name}", {var_name}Schema);\n'
+ output += f"module.exports = {model_name};\n\n"
+
+ return output
+
+
+# ─── Compiler: Drizzle ORM ────────────────────────────────────────────
+
+def compile_drizzle(schema: dict) -> str:
+ """Generate Drizzle ORM schema from structured schema."""
+ output = "// Generated by Oxtools JSON-to-Schema V2 (Agentic Pipeline)\n"
+ output += "// Format: Drizzle ORM (PostgreSQL)\n\n"
+ output += 'import { pgTable, serial, varchar, text, integer, boolean, timestamp, uuid, jsonb, bigint, doublePrecision, decimal, date } from "drizzle-orm/pg-core";\n\n'
+
+ for table in schema.get("tables", []):
+ name = table.get("name", "unnamed_table")
+ output += f"export const {name} = pgTable('{name}', {{\n"
+
+ for col in table.get("columns", []):
+ col_name = col.get("name", "unnamed_col")
+ col_type = col.get("type", "text").lower().strip()
+
+ # Map to drizzle column builder
+ if _safe_col(col, "isPrimary") and col_type in ("serial", "integer"):
+ output += f" {col_name}: serial('{col_name}').primaryKey(),\n"
+ elif col_type == "uuid" and _safe_col(col, "isPrimary"):
+ output += f" {col_name}: uuid('{col_name}').defaultRandom().primaryKey(),\n"
+ elif col_type in ("string", "varchar"):
+ not_null = ".notNull()" if not _safe_col(col, "isNullable", True) else ""
+ output += f" {col_name}: varchar('{col_name}', {{ length: 255 }}){not_null},\n"
+ elif col_type == "text":
+ not_null = ".notNull()" if not _safe_col(col, "isNullable", True) else ""
+ output += f" {col_name}: text('{col_name}'){not_null},\n"
+ elif col_type in ("integer", "int"):
+ not_null = ".notNull()" if not _safe_col(col, "isNullable", True) else ""
+ output += f" {col_name}: integer('{col_name}'){not_null},\n"
+ elif col_type == "boolean":
+ not_null = ".notNull()" if not _safe_col(col, "isNullable", True) else ""
+ output += f" {col_name}: boolean('{col_name}'){not_null},\n"
+ elif col_type in ("datetime", "timestamp"):
+ output += f" {col_name}: timestamp('{col_name}').defaultNow(),\n"
+ elif col_type in ("json", "jsonb"):
+ output += f" {col_name}: jsonb('{col_name}'),\n"
+ else:
+ output += f" {col_name}: text('{col_name}'), // unmapped type: {col_type}\n"
+
+ output += "});\n\n"
+
+ return output
+
+
+# ─── Compiler Dispatcher ─────────────────────────────────────────────
+
+COMPILERS = {
+ "postgresql": compile_postgresql,
+ "mysql": compile_mysql,
+ "prisma": compile_prisma,
+ "mongoose": compile_mongoose,
+ "drizzle": compile_drizzle,
+}
+
+def compile_schema(schema: dict, output_format: str) -> str:
+ """
+ Dispatch to the appropriate compiler.
+ Returns syntactically valid, production-ready output.
+ """
+ compiler = COMPILERS.get(output_format.lower())
+ if not compiler:
+ return f"-- Unsupported format: {output_format}. Supported: {', '.join(COMPILERS.keys())}"
+
+ try:
+ result = compiler(schema)
+ logger.info(f"[Compiler] Generated {output_format} output ({len(result)} chars)")
+ return result
+ except Exception as e:
+ logger.error(f"[Compiler] Failed: {e}")
+ import traceback
+ logger.error(traceback.format_exc())
+ return f"-- Compilation error: {str(e)}\n-- Please report this bug."
diff --git a/services/python-tools/tools/json-to-schema-v2/schema_config.py b/services/python-tools/tools/json-to-schema-v2/schema_config.py
new file mode 100644
index 0000000..e317e35
--- /dev/null
+++ b/services/python-tools/tools/json-to-schema-v2/schema_config.py
@@ -0,0 +1,27 @@
+"""
+JSON to Schema V2 — Configuration
+===================================
+Model assignments and API config.
+
+KEY FIX: Use user's selected model for Architect (main analysis node).
+"""
+
+import os
+
+# ─── API Configuration ─────────────────────────────────────────────────
+OXLO_API_KEY = os.getenv("OXLO_API_KEY", "")
+OXLO_BASE_URL = os.getenv("OXLO_BASE_URL", "https://api.oxlo.ai/v1")
+
+# ─── Model Assignments (fallbacks — user model takes priority) ─────────
+PARSER_MODEL = None # No LLM — pure Python
+ARCHITECT_MODEL = "deepseek-r1-0528" # Fallback for schema design
+REVIEWER_MODEL = "deepseek-r1-0528" # Fallback for review
+COMPILER_MODEL = None # No LLM — deterministic
+DOCUMENTER_MODEL = "llama-3.3-70b" # Fast model for docs
+
+# ─── Compiler Defaults ────────────────────────────────────────────────
+DEFAULT_OUTPUT_FORMAT = "postgresql"
+MAX_JSON_SIZE = 100_000 # 100KB max input
+
+# ─── Refinement Config ────────────────────────────────────────────────
+MAX_REVIEW_ITERATIONS = 2
diff --git a/services/python-tools/tools/json-to-schema-v2/schema_prompts.py b/services/python-tools/tools/json-to-schema-v2/schema_prompts.py
new file mode 100644
index 0000000..c7081e9
--- /dev/null
+++ b/services/python-tools/tools/json-to-schema-v2/schema_prompts.py
@@ -0,0 +1,122 @@
+"""
+JSON to Schema V2 — Agent Prompts
+====================================
+KEY FIX (v2.2): Architect REFINES a deterministic draft schema instead
+of designing from scratch. This prevents EAV meta-schemas.
+"""
+
+# ─── Architect (Refine Draft Schema) ──────────────────────────────────
+
+ARCHITECT_PROMPT = """You are a Database Schema Refiner. You will receive a DRAFT schema
+that was built deterministically from JSON data, plus the raw JSON itself.
+
+The draft schema already has:
+- One table per top-level JSON key
+- Columns for each sub-key
+- Dangerous keys (__proto__, constructor, $gt) safely stored in a separate table
+- Deeply nested structures flattened with nesting_level + deepest_value
+- All string values as TEXT (safe default)
+
+YOUR JOB: Review and REFINE this draft schema. You may:
+1. **Improve column types** — but ONLY if you are 100% certain (keep TEXT for suspicious values)
+2. **Add missing indexes** — for columns that would be commonly queried
+3. **Improve table/column names** — for clarity
+4. **Add constraints** — UNIQUE, NOT NULL, CHECK where appropriate
+5. **Merge small tables** — if two tables would be better as one
+6. **Split large tables** — if a table has too many columns
+
+DO NOT:
+- Create generic EAV (Entity-Attribute-Value) schemas
+- Create meta-schemas about "test types" or "test cases"
+- Collapse specific tables into key-value stores
+- Change TEXT to typed columns for suspicious/attack payload values
+- Remove the dangerous_key_values table
+- Remove created_at columns
+
+IMPORTANT: Preserve the one-table-per-category structure from the draft.
+Each JSON key should map to its own table with specific columns.
+
+## Output Format
+
+Return the refined schema using EXACTLY these JSON key names:
+
+```json
+{
+ "tables": [
+ {
+ "name": "table_name",
+ "columns": [
+ {"name": "id", "type": "serial", "isPrimary": true, "isNullable": false, "isUnique": false},
+ {"name": "value", "type": "text", "isPrimary": false, "isNullable": true, "isUnique": false, "defaultValue": "CURRENT_TIMESTAMP"}
+ ],
+ "foreignKeys": [
+ {"column": "parent_id", "referencesTable": "parent_table", "referencesColumn": "id"}
+ ],
+ "indexes": [
+ {"columns": ["parent_id"], "unique": false}
+ ]
+ }
+ ],
+ "designDecisions": ["reason 1", "reason 2"]
+}
+```
+
+MANDATORY key names in foreignKeys: "column", "referencesTable", "referencesColumn"
+MANDATORY key names in columns: "name", "type", "isPrimary", "isNullable", "isUnique"
+Indexes on FK columns MUST have "unique": false (NON-unique, 1:N relationship)
+
+Return ONLY the JSON object. No text outside the JSON."""
+
+
+# ─── Reviewer ──────────────────────────────────────────────────────────
+
+REVIEWER_PROMPT = """You are a Database Review Specialist. Review the proposed schema.
+
+Check for:
+1. **Specificity** — Each JSON data category should have its OWN table with specific columns.
+ If you see a generic key-value table being used for everything, flag as HIGH severity.
+2. **FK Indexes** — All FK indexes MUST be NON-unique (unique: false)
+3. **Type safety** — Suspicious values stored as TEXT? No typed columns from attack payloads?
+4. **Dangerous keys** — __proto__, constructor, $gt handled safely (not as real tables)?
+5. **Deep nesting** — Flattened with depth metadata, not raw JSONB blob?
+6. **Timestamps** — Every table has created_at?
+7. **FK key names** — Must use "referencesTable" and "referencesColumn"
+
+Return JSON:
+```json
+{
+ "approved": true,
+ "issues": [
+ {"severity": "HIGH", "table": "name", "issue": "description", "suggestion": "fix"}
+ ],
+ "refinements": []
+}
+```
+
+Return ONLY the JSON object."""
+
+
+# ─── Documenter ────────────────────────────────────────────────────────
+
+DOCUMENTER_PROMPT = """You are a Technical Writer for database documentation.
+
+Given a database schema and compiled DDL, write:
+
+1. **Schema Overview** — Data model summary in one paragraph
+2. **Table Descriptions** — One sentence per table explaining its purpose and column count
+3. **Relationship Diagram** — ASCII art showing table relationships with FK arrows.
+ ONLY draw arrows between tables with ACTUAL foreign key relationships.
+4. **Example Queries** — 5 useful SQL queries using real table and column names:
+ - Basic INSERT
+ - SELECT with WHERE
+ - JOIN query
+ - Aggregation (COUNT, GROUP BY)
+ - Complex query joining 3+ tables
+5. **Security Design Notes** (MANDATORY) — Document EVERY security decision:
+ - Every key that was quarantined and WHY (list threat_type)
+ - Every field kept as TEXT (or _raw suffix) instead of typed and WHY
+ - Any deeply nested structure that was flattened and at what depth
+ - Any NoSQL operators detected in values
+ - Explain why dangerous_key_values table exists (if present)
+
+Use clean markdown. Use the ACTUAL table and column names from the schema."""
diff --git a/services/python-tools/tools/json-to-schema-v2/tool.py b/services/python-tools/tools/json-to-schema-v2/tool.py
new file mode 100644
index 0000000..979d0cb
--- /dev/null
+++ b/services/python-tools/tools/json-to-schema-v2/tool.py
@@ -0,0 +1,113 @@
+"""
+JSON to Schema V2 — Tool Entry Point
+======================================
+This is the entry point loaded by the unified runner.
+The actual agent logic lives in agents.py, compilers.py, and prompts.py.
+
+Architecture:
+ compilers.py — Deterministic SQL/Prisma/Mongoose/Drizzle generators
+ agents.py — LangGraph StateGraph with 5 specialist nodes
+ prompts.py — Agent prompts separated for A/B testing
+ config.py — Model assignments and output format config
+"""
+
+import json
+
+from schema_agents import build_graph, SchemaState
+from schema_config import OXLO_API_KEY, MAX_JSON_SIZE
+
+# ─── MANIFEST ──────────────────────────────────────────────────────────
+MANIFEST = {
+ "id": "json-to-schema-v2",
+ "name": "JSON to DB Schema (Agentic V2)",
+ "description": (
+ "Multi-agent schema generation: programmatic JSON parsing, "
+ "LLM-powered schema design with iterative review, and "
+ "deterministic compilation to PostgreSQL/MySQL/Prisma/Mongoose/Drizzle"
+ ),
+ "author": "Oxlo Team",
+ "version": "2.0.0",
+ "requires": ["langgraph", "langchain-openai", "langchain-core"],
+}
+
+
+# ─── RUN (called by the unified runner) ──────────────────────────────
+async def run(data: dict):
+ """
+ Execute the schema generation pipeline.
+ Returns an async generator that streams per-node status + final output.
+ """
+ if not OXLO_API_KEY:
+ return {"error": "OXLO_API_KEY not configured. Set it in .env"}
+
+ raw_json = data.get("json", "")
+ if not raw_json.strip():
+ return {"error": "No JSON provided. Paste your JSON data."}
+
+ # Validate JSON before entering the pipeline
+ try:
+ parsed = json.loads(raw_json)
+ except json.JSONDecodeError as e:
+ return {"error": f"Invalid JSON: {str(e)}. Fix syntax errors before converting."}
+
+ if len(raw_json) > MAX_JSON_SIZE:
+ return {"error": f"JSON too large ({len(raw_json)} bytes). Max: {MAX_JSON_SIZE} bytes."}
+
+ output_format = data.get("outputFormat", "postgresql").lower()
+ user_model = data.get("model", "")
+
+ graph = build_graph()
+ initial: SchemaState = {
+ "raw_json": raw_json,
+ "parsed_data": {},
+ "output_format": output_format,
+ "user_model": user_model,
+ "structure_analysis": "",
+ "draft_schema": {},
+ "proposed_schema": {},
+ "design_decisions": [],
+ "review_approved": False,
+ "review_issues": [],
+ "review_refinements": [],
+ "review_iteration": 0,
+ "compiled_output": "",
+ "documentation": "",
+ "status": "starting",
+ }
+
+ async def stream():
+ async for event in graph.astream(initial):
+ for node_name, node_output in event.items():
+ status = node_output.get("status", "processing")
+ yield f"[{node_name}] {status}\n"
+
+ # Stream parser stats
+ if node_name == "parser" and "structure_analysis" in node_output:
+ lines = node_output["structure_analysis"].split("\n")
+ yield f" → {lines[0]}\n" # Stats line
+
+ # Stream architect table count
+ if node_name == "architect" and "proposed_schema" in node_output:
+ tables = node_output["proposed_schema"].get("tables", [])
+ yield f" → Designed {len(tables)} tables\n"
+
+ # Stream reviewer verdict
+ if node_name == "reviewer":
+ approved = node_output.get("review_approved", False)
+ issues = len(node_output.get("review_issues", []))
+ iteration = node_output.get("review_iteration", 0)
+ verdict = "✓ Approved" if approved else "✗ Needs refinement"
+ yield f" → {verdict} ({issues} issues, iteration {iteration})\n"
+
+ # Stream compiled output
+ if node_name == "compiler" and "compiled_output" in node_output:
+ yield "\n---SCHEMA_START---\n"
+ yield node_output["compiled_output"]
+ yield "\n---SCHEMA_END---\n"
+
+ # Stream documentation
+ if node_name == "documenter" and "documentation" in node_output:
+ yield "\n---DOCS_START---\n"
+ yield node_output["documentation"]
+
+ return stream()
diff --git a/services/python-tools/tools/screenshot-to-code/requirements.txt b/services/python-tools/tools/screenshot-to-code/requirements.txt
new file mode 100644
index 0000000..532ee02
--- /dev/null
+++ b/services/python-tools/tools/screenshot-to-code/requirements.txt
@@ -0,0 +1,8 @@
+# Requirements for: Screenshot to Code
+openai>=1.68.2,<2.0.0
+Pillow==10.4.0
+playwright==1.44.0
+scikit-image==0.24.0 # SSIM scoring
+numpy==1.26.4
+pytesseract==0.3.10
+opencv-python-headless==4.9.0.80
\ No newline at end of file
diff --git a/services/python-tools/tools/screenshot-to-code/tool.py b/services/python-tools/tools/screenshot-to-code/tool.py
new file mode 100644
index 0000000..4ab295c
--- /dev/null
+++ b/services/python-tools/tools/screenshot-to-code/tool.py
@@ -0,0 +1,1149 @@
+import os
+import asyncio
+import base64
+import json
+import tempfile
+import time
+import logging
+from io import BytesIO
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+from openai import OpenAI
+from PIL import Image
+from skimage.metrics import structural_similarity as ssim_fn
+logger = logging.getLogger("screenshot-to-code")
+
+# ─── MANIFEST ─────────────────────────────────────────────────────────────────
+MANIFEST = {
+ "id": "screenshot-to-code",
+ "name": "Screenshot to Code",
+ "description": "Upload a UI screenshot and get Tailwind/HTML code via Multi-Agent Consensus Pipeline",
+ "author": "ArunMadhavan EVR",
+ "version": "9.0.0",
+}
+
+# ─── Config ───────────────────────────────────────────────────────────────────
+COMPRESS_MAX_PX = 1920
+COMPRESS_JPEG_QUALITY = 90
+
+MODEL_CODER = "kimi-k2.5"
+MODEL_JUDGE = "kimi-k2.5"
+
+OXLO_BASE_URL = "https://api.oxlo.ai/v1"
+
+# NEW #9 — raised from 12000
+MAX_TOKENS_EXTRACT = 12000
+MAX_TOKENS_CODE = 16000
+# NEW #8 — raised from 16. Judge now has room to reason before answering.
+# The system prompt instructs it to still end with just the digit,
+# but giving it 512 tokens lets it think through the candidates properly.
+MAX_TOKENS_JUDGE = 2048
+
+# ── Strategy swarm (unchanged from v8.0) ─────────────────────────────────────
+SWARM_STRATEGIES = [
+ {
+ "name": "structure-first",
+ "temperature": 0.0,
+ "prefix": (
+ "═══ STRATEGY: STRUCTURE-FIRST ═══\n"
+ "Before writing a single HTML tag, reason through:\n"
+ " 1. What is the outermost container? (full-width, fixed-width, mobile?)\n"
+ " 2. What are the major layout sections? (header, sidebar, main, footer?)\n"
+ " 3. What CSS layout system governs each section? (flex-row, flex-col, grid?)\n"
+ " 4. What are the background colors of each section? (sample exact hex)\n"
+ "Only then write the HTML, outside-in from largest container to smallest leaf.\n\n"
+ ),
+ },
+ {
+ "name": "typography-first",
+ "temperature": 0.0,
+ "prefix": (
+ "═══ STRATEGY: TYPOGRAPHY-FIRST ═══\n"
+ "Before writing a single HTML tag, inventory every text element:\n"
+ " 1. List every visible string, its approximate px size, weight, and color.\n"
+ " 2. Identify heading hierarchy (h1/h2/h3) from visual prominence.\n"
+ " 3. Note any monospace, italic, or special-weight text.\n"
+ " 4. Mark interactive text (links, buttons, labels) separately.\n"
+ "Build the HTML by placing text elements first, then wrap them in layout containers.\n\n"
+ ),
+ },
+ {
+ "name": "component-first",
+ "temperature": 0.0,
+ "prefix": (
+ "═══ STRATEGY: COMPONENT-FIRST ═══\n"
+ "Before writing a single HTML tag, decompose the UI into components:\n"
+ " 1. Identify discrete, reusable UI components (navbar, card, badge, list-row, tab-bar).\n"
+ " 2. For each component: note its exact background, border, shadow, and padding.\n"
+ " 3. Identify which components repeat (list items, table rows, grid cards).\n"
+ " 4. Note the exact count of repeating items — do NOT truncate lists.\n"
+ "Implement each component as a self-contained HTML block, then compose the full page.\n\n"
+ ),
+ },
+]
+
+# ── NEW #12 — Healing constants (raised thresholds for higher quality bar) ────
+SSIM_SHIP_THRESHOLD = 92.0 # NEW #12: was 88 — keep healing until 92%
+SSIM_HEAL_THRESHOLD = 55.0 # abort floor — below this model is lost
+MAX_HEALING_PASSES = 3 # NEW #11: was 2
+DIFF_PIXEL_THRESHOLD = 12.0 # slightly tighter than v8.0's 15.0
+
+# ── Slice constants (unchanged) ───────────────────────────────────────────────
+SLICE_ASPECT_THRESHOLD = 2.5
+SLICE_N = 3
+
+# ── Font injection (unchanged from v7.0) ─────────────────────────────────────
+FONT_INJECT = (
+ ''
+ ''
+ ''
+ ''
+)
+
+# ─── Step 1 — Spatial Extractor system (iterative edits only) ─────────────────
+
+EXTRACTOR_USER = (
+ "Extract the COMPLETE JSON layout from this screenshot. "
+ "Include every visible list item, link, and text string — do not skip any rows. "
+ "Output ONLY the raw JSON array. No explanation, no markdown."
+)
+
+# ─── Step 2 — Coder system (base, strategies prepended per-candidate) ─────────
+CODER_SYSTEM_BASE = """You are a pixel-perfect UI compiler with vision capabilities.
+You will be shown a UI screenshot. Reproduce it as HTML using Tailwind CSS.
+
+CRITICAL RULES — violations will cause rejection:
+1. Examine the screenshot at maximum detail before writing a single line of HTML.
+2. Use Tailwind arbitrary values for EVERY color, size, spacing: bg-[#1a1a2e] text-[13px] w-[340px] gap-[12px].
+3. Copy ALL visible text character-for-character. Never invent, paraphrase, or omit any text.
+4. Reproduce exact background colors, text colors, border colors from the screenshot.
+5. Match layout structure exactly: if it is a mobile screen, use a mobile-width container. If it is a desktop, use full width.
+6. Render EVERY visible row, list item, and element. Do not truncate dense lists under any circumstances.
+7. Simple icons (back arrow, checkmark, search, hamburger): inline SVG matching the screenshot shape exactly.
+8. Profile photos, product images, logos: with correct dimensions and colors.
+9. Status bar elements (time, battery, signal): reproduce as text/SVG, never skip.
+10. Bottom navigation bars: reproduce all tabs with correct icons and labels.
+11. Include in . No other scripts.
+12. No JavaScript. No invented content whatsoever.
+13. CRITICAL: If the screenshot shows a browser window with tabs/address bar, reproduce ONLY the inner page content — not the browser chrome.
+14. Shadows, borders, border-radius: match exactly using arbitrary Tailwind values.
+15. Gradients: reproduce using Tailwind bg-gradient-to-* classes with exact from/via/to hex values.
+16. Opacity: match exactly using opacity-[N] or text-[#rrggbbAA] where relevant.
+
+OUTPUT: Raw HTML only, starting with . Zero explanation. Zero markdown fences."""
+
+CODER_USER = (
+ "Study this screenshot in full detail. "
+ "Before writing HTML, mentally note:\n"
+ " - The exact background color of the page and each section\n"
+ " - Every text string, its size, weight, and color\n"
+ " - Every UI component and its precise spacing\n"
+ " - The layout system (flex/grid) used at each level\n"
+ " - Any gradients, shadows, borders, or special effects\n\n"
+ "Then produce pixel-perfect HTML with Tailwind CSS that is indistinguishable from the screenshot.\n"
+ "Output ONLY raw HTML starting with . No explanation."
+)
+
+# ─── NEW #1/#13 — Healer system (full context, detail=high) ──────────────────
+HEALER_SYSTEM = """You are a pixel-perfect UI debugger with vision capabilities.
+You are given three images in order:
+ IMAGE 1 — The ORIGINAL UI screenshot (ground truth target)
+ IMAGE 2 — Your PREVIOUS HTML rendered in a browser
+ IMAGE 3 — A DIFF MASK: red pixels = wrong, green tint = correct
+
+Your ONLY job: fix the HTML so every red zone disappears.
+
+CRITICAL RULES:
+1. DO NOT rewrite sections that are correct (green zones). Touch only what is broken.
+2. For each red zone, compare IMAGE 1 vs IMAGE 2 and diagnose the root cause:
+ - Wrong spacing? → Fix padding/margin/gap arbitrary value precisely.
+ - Wrong color? → Sample the exact hex from IMAGE 1 and correct bg-[#xxx] or text-[#xxx].
+ - Wrong font size/weight? → Fix text-[Npx] or font-weight class.
+ - Missing element? → Add the complete missing HTML block.
+ - Wrong layout? → Fix flex-row ↔ flex-col or grid column count.
+ - Wrong border/shadow? → Correct border-[#xxx], rounded-[Npx], or shadow class.
+ - Wrong gradient? → Fix from-[#xxx] via-[#xxx] to-[#xxx] and direction.
+ - Wrong image dimensions? → Fix the placehold.co URL with correct W×H.
+3. Be surgical. The goal is zero red pixels in the next render.
+4. Preserve ALL text strings exactly — do not alter any text content.
+5. Return the COMPLETE corrected HTML starting with .
+
+OUTPUT: Raw HTML only, starting with . Zero explanation. Zero markdown fences."""
+
+# ─── NEW #6/#13 — Judge system (full HTML, reasons before deciding) ───────────
+JUDGE_SYSTEM = """You are a UI fidelity judge with vision capabilities.
+You are given the original UI screenshot and 2-3 complete HTML candidates.
+Your job: select the candidate that most faithfully reproduces the screenshot.
+
+Evaluate each candidate on these criteria IN ORDER OF IMPORTANCE:
+ 1. TEXT ACCURACY — every visible string present, verbatim, correct position
+ 2. COLOR ACCURACY — exact background, text, border, and accent colors
+ 3. LAYOUT FIDELITY — correct flex/grid structure, correct hierarchy
+ 4. COMPLETENESS — no missing rows, nav items, icons, or sections
+ 5. SPACING — correct padding, margin, gap values
+ 6. VISUAL EFFECTS — shadows, borders, gradients, border-radius
+
+Think through each candidate systematically. Then on the VERY LAST LINE of your
+response, write ONLY the single digit 1, 2, or 3 — nothing else on that line."""
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# IMAGE HELPERS (unchanged from v7.0 / v8.0)
+# ═══════════════════════════════════════════════════════════════════════════════
+def compress_image(raw_bytes: bytes) -> tuple[str, str, Image.Image]:
+ img = Image.open(BytesIO(raw_bytes))
+ if img.mode == "RGBA":
+ bg = Image.new("RGB", img.size, (255, 255, 255))
+ bg.paste(img, mask=img.split()[3])
+ img = bg
+ elif img.mode not in ("RGB",):
+ img = img.convert("RGB")
+ img.thumbnail((COMPRESS_MAX_PX, COMPRESS_MAX_PX), Image.LANCZOS)
+ buf = BytesIO()
+ img.save(buf, format="JPEG", quality=COMPRESS_JPEG_QUALITY, optimize=True)
+ encoded = base64.standard_b64encode(buf.getvalue()).decode("utf-8")
+ logger.info("Compressed image to %dx%d", *img.size)
+ return encoded, "image/jpeg", img
+
+
+def _normalise_viewport(ref_image: Image.Image) -> tuple[tuple[int, int], Image.Image]:
+ w, h = ref_image.size
+ if w > 1920:
+ w, h = w // 2, h // 2
+ ref_image = ref_image.resize((w, h), Image.LANCZOS)
+ logger.info("Retina detected — ref_image halved to %dx%d", w, h)
+ w = max(320, min(w, 1920))
+ h = max(400, h)
+ return (w, h), ref_image
+
+
+def compute_ssim(ref: Image.Image, rendered: Image.Image) -> float:
+ rendered_rgb = rendered.convert("RGB")
+ ref_resized = ref.resize(rendered_rgb.size, Image.LANCZOS).convert("RGB")
+ ref_arr = np.array(ref_resized, dtype=np.float32)
+ render_arr = np.array(rendered_rgb, dtype=np.float32)
+ try:
+ win_size = min(21, rendered_rgb.size[0], rendered_rgb.size[1])
+ win_size = win_size if win_size % 2 == 1 else win_size - 1
+ if win_size < 3:
+ return 0.0
+ score = ssim_fn(
+ ref_arr, render_arr,
+ data_range=255.0,
+ channel_axis=2,
+ win_size=win_size,
+ gaussian_weights=True,
+ )
+ return max(0.0, float(score)) * 100.0
+ except Exception as exc:
+ logger.warning("SSIM computation failed: %s", exc)
+ return 0.0
+
+
+# ─── NEW #10 — _call_api with 600s timeout, 3 attempts ───────────────────────
+def _call_api(
+ client: OpenAI,
+ model: str,
+ messages: list[dict],
+ max_tokens: int,
+ temperature: float = 0.0,
+ attempt_limit: int = 3,
+) -> str:
+ """
+ NEW #10: timeout raised to 600s per attempt.
+ The judge with full HTML context can take 4-5 minutes to respond.
+ Total max wait = 3 attempts × 600s + 2 × 8s backoff = ~30 minutes worst case.
+ That is acceptable for maximum fidelity.
+ """
+ import httpx
+ last_exc: Exception = RuntimeError("No attempts made")
+
+ for attempt in range(attempt_limit):
+ try:
+ resp = client.chat.completions.create(
+ model=model,
+ messages=messages,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ timeout=float(os.getenv("OXLO_API_TIMEOUT", "600.0")), # NEW #10: was 180s
+ )
+ return resp.choices[0].message.content or ""
+
+ except Exception as exc:
+ last_exc = exc
+ err_str = str(exc).lower()
+ is_rate = "429" in err_str
+ is_server = any(c in err_str for c in ("500", "502", "503", "504"))
+ is_timeout = "timeout" in err_str or isinstance(exc, httpx.TimeoutException)
+
+ if (is_rate or is_server or is_timeout) and attempt < attempt_limit - 1:
+ wait = 4 ** (attempt + 1) # 4s, 16s
+ logger.warning(
+ "[%s] attempt %d failed (%s...), retry in %ds",
+ model, attempt + 1, err_str[:80], wait
+ )
+ time.sleep(wait)
+ else:
+ break
+
+ raise RuntimeError(f"[{model}] failed after {attempt_limit} attempts: {last_exc}")
+
+
+def _clean_html(raw: str) -> str:
+ import re as _re
+ raw = raw.strip()
+ if raw.startswith("```"):
+ lines = raw.split("\n")[1:]
+ if lines and lines[-1].strip().startswith("```"):
+ lines = lines[:-1]
+ raw = "\n".join(lines).strip()
+ lower = raw.lower()
+ for tag in (" safely — only match the real tag, not one inside
+ # a """
+ if '' in html.lower():
+ idx = html.lower().rfind('')
+ return html[:idx] + script + html[idx:]
+ return html + script
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EDIT SCRIPT INJECTOR (postMessage-based live editing)
+# ═══════════════════════════════════════════════════════════════════════════════
+def _inject_edit_script(html: str) -> str:
+ script = """
+"""
+ if '