Revert "Production mvp"

.env.example

@@ -76,31 +76,3 @@ SCIDK_CHAT_OPENAI_API_KEY=
 
 # Anthropic key for entity extraction (optional, falls back to pattern matching)
 SCIDK_ANTHROPIC_API_KEY=
-
-# ------------------------------
-# Chat Neo4j (for chat intelligence & history)
-# ------------------------------
-# Separate Neo4j instance for chat metadata (not research data)
-CHAT_NEO4J_URI=bolt://localhost:7688
-CHAT_NEO4J_AUTH=neo4j/your-password-here
-
-# ------------------------------
-# Concept Graph Neo4j (for meta-reasoning)
-# ------------------------------
-# Separate Neo4j instance for concept graph (intents, tools, schema abstractions)
-# This layer enables semantic intent classification and graph-based planning
-SCIDK_CONCEPT_NEO4J_URI=bolt://localhost:7689
-SCIDK_CONCEPT_NEO4J_AUTH=neo4j/concept-graph-password
-SCIDK_CONCEPT_GRAPH_ENABLED=1
-
-# Chat Intelligence Settings
-SCIDK_CHAT_CONTEXT_RETRIEVAL_TOP_K=3        # messages per strategy (semantic + node overlap)
-SCIDK_CHAT_CONTEXT_MAX_TOTAL=6              # max total messages in context
-SCIDK_CHAT_REACT_MAX_STEPS=4                # ReAct loop step limit
-SCIDK_CHAT_REACT_TOKEN_BUDGET=3000          # tokens per step
-
-# Background Summarization (fast model for compression)
-SCIDK_CHAT_SUMMARIZATION_MODEL=qwen2.5:7b
-
-# Embedding Model (for semantic retrieval)
-SCIDK_CHAT_EMBEDDING_MODEL=nomic-embed-text

.github/workflows/ci.yml

@@ -20,19 +20,19 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -e .[dev]
-      - name: Run pytest with coverage (exclude E2E and integration tests)
+      - name: Run pytest with coverage (exclude E2E)
         run: |
-          python -m coverage run -m pytest -q -m "not e2e and not integration"
+          python -m coverage run -m pytest -q -m "not e2e"
           python -m coverage report
           python -m coverage xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
           file: ./coverage.xml
           fail_ci_if_error: false
-      - name: Check coverage threshold (48%)
+      - name: Check coverage threshold (50%)
         run: |
-          python -m coverage report --fail-under=48
+          python -m coverage report --fail-under=50
 
   # E2E tests temporarily disabled in CI (Feb 2026)
   # The test suite has stability issues (auth conflicts, timing, cleanup) that need dedicated attention.

.gitignore

@@ -75,13 +75,3 @@ backups/
 dev/test-runs/tmp/
 dev/code-imports/
 docs/archive/
-gunicorn.log
-
-# Scanner and debug outputs
-/ambig_paths.txt
-/ambiguous_paths.txt
-/big_dir_tree.txt
-/magic_results.txt
-/out.txt
-/out*.txt
-/out_*.txt

README.md

@@ -43,10 +43,9 @@ source scripts/init_env.fish --write-dotenv
 3) Run the server:
 ```
 scidk-serve
-# or the equivalent module form:
+# or
 python3 -m scidk.app
 ```
-> **Canonical launch command:** `scidk-serve` (entry point `scidk.app:main`). `python3 -m scidk.app` is exactly equivalent. Other docs reference this section rather than restating launch commands.
 
 4) Open the UI in your browser:
 - http://127.0.0.1:5000/
@@ -205,8 +204,8 @@ Details:
 - Add tests alongside new features in future cycles; see dev/cycles.md for cycle protocol.
 
 ## Notes
-- The graph backend is selectable via `SCIDK_GRAPH_BACKEND` (`memory` by default, `neo4j` to persist). Neo4j is fully wired: scans commit File/Folder/Scan nodes and relationships with post-commit verification (see `scidk/services/neo4j_client.py`).
-- With the in-memory backend, graph data resets on restart; use `SCIDK_GRAPH_BACKEND=neo4j` for persistence. Neo4j deployment docs reside in dev/ops/deployment-neo4j.md.
+- This MVP uses an in-memory graph; data resets on restart.
+- Neo4j deployment docs reside in dev/ops/deployment-neo4j.md, but Neo4j is not yet wired in the MVP code.
 
 ## Documentation
 - Delivery cycles and planning protocol: dev/cycles.md
@@ -215,19 +214,11 @@ Details:
 
 ## Architecture
 SciDK uses a modular Flask blueprint architecture for web routes:
-- **~27 blueprints** organize 300+ routes by functional area (UI, files, tasks, graph, Neo4j, providers, chat, labels, links, admin, interpreters, and more)
-- **Application factory**: `create_app()` lives in `scidk/app.py` (~314 lines); blueprints are registered via `register_blueprints()` in `scidk/web/routes/__init__.py`
+- **9 blueprints** organize 91 routes by functional area (UI, files, tasks, graph, Neo4j, providers, chat, admin, interpreters)
+- **Reduced footprint**: app.py reduced from 5,781 to 645 lines (89% reduction)
 - **Clean separation**: Each blueprint is self-contained with proper import scoping
 - See `scidk/web/routes/README.md` for detailed blueprint documentation
 
-## MCP Server (Model Context Protocol)
-SciDK exposes core functionality via an MCP server for external AI agents (Claude Desktop, etc.):
-- **5 core tools**: `query_knowledge_graph`, `get_schema`, `summarize_dataset`, `get_label_profile`, `list_labels`
-- **Read-only safety**: All queries are validated to block write operations
-- **Stdio transport**: Uses standard MCP protocol over stdin/stdout
-- **Setup guide**: See `docs/mcp-setup.md` for Claude Desktop configuration
-- **Run**: `python3 -m scidk.mcp_server`
-
 ## Scanning progress and background tasks (MVP)
 - Current options:
   - Synchronous: POST /api/scan runs immediately and returns when complete.
@@ -244,11 +235,13 @@ SciDK exposes core functionality via an MCP server for external AI agents (Claud
   - Preview and download instances for File, Folder, and Scan labels as CSV (XLSX if openpyxl is installed).
 
 ## Neo4j integration
-- Status: Neo4j is fully wired. The app ships with docker-compose.neo4j.yml to run a local Neo4j; set `SCIDK_GRAPH_BACKEND=neo4j` (plus `NEO4J_URI`/`NEO4J_AUTH`) to persist to it. With the default in-memory backend, graph data is not persisted across restarts.
-- How it works:
-  - The graph backend is selected at startup in `create_app()` via `SCIDK_GRAPH_BACKEND` (`memory` default, `neo4j` to persist); invalid Neo4j params fall back to in-memory.
-  - Committing a scan writes `(:File)`, `(:Folder)`, `(:Scan)` nodes with `(Folder)-[:CONTAINS]->(File|Folder)`, `(File|Folder)-[:SCANNED_IN]->(Scan)`, and `INTERPRETED_AS` relationships, via `scidk/services/neo4j_client.py`.
-  - Schema triples for `/api/graph/schema` are computed with Cypher (APOC variants available at `/api/graph/schema.apoc`).
+- Status: The app ships with docker-compose.neo4j.yml to run a local Neo4j, but the Flask app currently uses an in-memory graph.
+- Next steps to enable Neo4j writes/reads:
+  1) Add a GraphAdapter interface and a Neo4jAdapter implementing upsert_dataset, add_interpretation, commit_scan, schema_triples.
+  2) Add config/feature flag (e.g., SCIDK_GRAPH_BACKEND=neo4j) to switch adapters.
+  3) Map current in-memory structures to Neo4j schema: (:File), (:Folder), (:Scan) nodes and CONTAINS, INTERPRETED_AS, SCANNED_IN relationships.
+  4) Use Cypher or APOC to compute schema triples for /api/graph/schema.
+- Until then, data is not persisted to Neo4j. Use the CSV exports or the in-memory map for the demo.
 
 
 ## New in this cycle: Optional Neo4j schema endpoints and extra Instance exports
@@ -450,17 +443,19 @@ Local commands:
 - make e2e → pytest -m e2e tests/e2e -q
 - make check → runs unit, integration, and e2e sequentially
 
-CI is defined in `.github/workflows/ci.yml` and runs pytest with `-m "not e2e"`. **E2E tests are disabled in CI as of Feb 2026** (the E2E job is commented out) — run them locally with `npm run e2e` or `pytest -m e2e`.
+See .github/workflows/tests.yml for the CI matrix that runs each tier.
 
 ## Verify CI and Record Demo Artifacts
 
 Follow these steps to verify the full test suite and automatically capture screenshots/JSON for the demo.
 
 1) Verify CI on GitHub
-- Navigate to GitHub → Actions → the CI workflow (defined in `.github/workflows/ci.yml`).
-- Confirm the pytest job is green (runs `pytest -m "not e2e"`).
-- E2E (Playwright) is not run in CI as of Feb 2026; verify E2E locally with `npm run e2e`.
-- Click into the latest run to see logs if the job is red.
+- Navigate to GitHub → Actions → "Tests" workflow (defined in `.github/workflows/tests.yml`).
+- Confirm that all three matrix jobs are green:
+  - tier=unit
+  - tier=integration
+  - tier=e2e (installs Playwright browsers automatically)
+- Click into the latest run to see logs if any job is red.
 
 2) Run all tests locally (mirrors CI)
 ```

docker-compose.neo4j.yml

@@ -28,26 +28,3 @@ services:
       retries: 10
       start_period: 20s
 
-  neo4j-chat:
-    image: neo4j:2025.10.1
-    container_name: scidk-neo4j-chat
-    restart: unless-stopped
-    environment:
-      - NEO4J_AUTH=${CHAT_NEO4J_AUTH:-neo4j/changeme}
-      - NEO4J_server_memory_heap_initial__size=512m
-      - NEO4J_server_memory_heap_max__size=512m
-      - NEO4J_server_memory_pagecache_size=256m
-      - NEO4J_dbms_security_auth__enabled=true
-    ports:
-      - "7688:7687"   # Bolt (chat Neo4j on different port)
-      - "7475:7474"   # HTTP (chat Neo4j Browser)
-    volumes:
-      - ${CHAT_NEO4J_HOST_DATA_DIR:-./data/neo4j-chat/data}:/data
-      - ${CHAT_NEO4J_HOST_LOGS_DIR:-./data/neo4j-chat/logs}:/logs
-    healthcheck:
-      test: ["CMD-SHELL", "PASS=$${NEO4J_AUTH#neo4j/}; cypher-shell -u neo4j -p \"$${PASS}\" -a bolt://localhost:7687 \"RETURN 1;\""]
-      interval: 15s
-      timeout: 5s
-      retries: 10
-      start_period: 20s
-