diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/task_definition.json b/.github/workflows/task_definition.json new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/template.yml b/.github/workflows/template.yml new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/api/main.py b/api/main.py index d81efbc..600df58 100644 --- a/api/main.py +++ b/api/main.py @@ -14,6 +14,7 @@ from src.document_analyzer.data_analysis import DocumentAnalyzer from src.document_compare.document_comparator import DocumentComparatorLLM from src.document_chat.retrieval import ConversationalRAG +from utils.document_ops import FastAPIFileAdapter, read_pdf_via_handler FAISS_BASE = os.getenv("FAISS_BASE", "faiss_index") UPLOAD_BASE = os.getenv("UPLOAD_BASE", "data") @@ -48,7 +49,7 @@ async def serve_ui(request: Request) -> HTMLResponse: @app.get("/health") def health() -> Dict[str, str]: - return {"status": "ok", "service": "document-bot-api", "version": "0.1"} + return {"status": "ok", "service": "document-bot", "version": "0.1"} # ---------- ANALYZE ---------- @app.post("/analyze") @@ -60,7 +61,7 @@ async def analyze_document(file: UploadFile = File(...)) -> Any: try: dh = DocumentHandler() saved_path = dh.save_pdf(FastAPIFileAdapter(file)) - text = _read_pdf_via_handler(dh, saved_path) + text = read_pdf_via_handler(dh, saved_path) analyzer = DocumentAnalyzer() result = analyzer.analyze_document(text) return JSONResponse(content=result) @@ -79,10 +80,9 @@ async def compare_documents(reference: UploadFile = File(...), actual: UploadFil """ try: dc = DocumentComparator() - ref_path, act_path = dc.save_uploaded_files( + dc.save_uploaded_files( FastAPIFileAdapter(reference), FastAPIFileAdapter(actual) ) - _ = ref_path, act_path combined_text = dc.combine_documents() comp = DocumentComparatorLLM() df = comp.compare_documents(combined_text) @@ -171,25 +171,6 @@ async def chat_query( except Exception as e: raise HTTPException(status_code=500, detail=f"Query failed: {e}") - -# ---------- Helpers ---------- -class FastAPIFileAdapter: - """Adapt FastAPI UploadFile -> .name + .getbuffer() API""" - def __init__(self, uf: UploadFile): - self._uf = uf - self.name = uf.filename - - def getbuffer(self) -> bytes: - self._uf.file.seek(0) - return self._uf.file.read() - -def _read_pdf_via_handler(handler: DocumentHandler, path: str) -> str: - if hasattr(handler, "read_pdf"): - return handler.read_pdf(path) - if hasattr(handler, "read_"): - return handler.read_(path) - raise RuntimeError("DocumentHandler has neither read_pdf nor read_ method.") - # uvcorn is ASGI server for running FastAPI applications that supports async features which helps in handling multiple requests efficiently # To run the FastAPI app, use the command below in your terminal: # uvicorn api.main:app --host 0.0.0.0 --port 8080 --reload diff --git a/src/document_chat/retrieval.py b/src/document_chat/retrieval.py index 315649c..5997509 100644 --- a/src/document_chat/retrieval.py +++ b/src/document_chat/retrieval.py @@ -27,8 +27,6 @@ def __init__(self, session_id: Optional[str], retriever=None): try: self.log = CustomLogger().get_logger(__name__) self.session_id = session_id - - # Load LLM and prompts once self.llm = self._load_llm() self.contextualize_prompt: ChatPromptTemplate = PROMPT_REGISTRY[ PromptType.CONTEXTUALIZE_QUESTION.value @@ -36,20 +34,15 @@ def __init__(self, session_id: Optional[str], retriever=None): self.qa_prompt: ChatPromptTemplate = PROMPT_REGISTRY[ PromptType.CONTEXT_QA.value ] - - # Lazy pieces self.retriever = retriever self.chain = None if self.retriever is not None: self._build_lcel_chain() - self.log.info("ConversationalRAG initialized", session_id=self.session_id) except Exception as e: self.log.error("Failed to initialize ConversationalRAG", error=str(e)) raise DocumentPortalException("Initialization error in ConversationalRAG", sys) - # ---------- Public API ---------- - def load_retriever_from_faiss( self, index_path: str, diff --git a/src/document_ingestion/data_ingestion.py b/src/document_ingestion/data_ingestion.py index 56fac7b..cff5e5b 100644 --- a/src/document_ingestion/data_ingestion.py +++ b/src/document_ingestion/data_ingestion.py @@ -13,7 +13,7 @@ from utils.model_loader import ModelLoader from logger.custom_logger import CustomLogger from exception.custom_exception import DocumentPortalException -from utils.file_io import _session_id, save_uploaded_files +from utils.file_io import generate_session_id, save_uploaded_files from utils.document_ops import load_documents SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".txt"} @@ -70,18 +70,17 @@ def add_documents(self, docs: List[Document]): """ if self.vs is None: raise RuntimeError("Call load_or_create() before add_documents_idempotent().") - new_docs: List[Document] = [] - for d in docs: key = self._fingerprint(d.page_content, d.metadata or {}) if key in self._meta["rows"]: continue self._meta["rows"][key] = True new_docs.append(d) - if new_docs: - self.vs.add_documents(new_docs) + # This will add new documents to the FAISS index, which first Embeds the chunks and then adds it to the index. + # Add / Append new documents over the same existing index + self.vs.add_documents(new_docs) self.vs.save_local(str(self.index_dir)) self._save_meta() return len(new_docs) @@ -100,8 +99,8 @@ def load_or_create(self, texts: Optional[List[str]]=None, metadatas: Optional[Li if not texts: raise DocumentPortalException("No existing FAISS index and no data to create one", sys) - self.vs = FAISS.from_texts(texts=texts, embedding=self.emb, metadatas=metadatas or []) - self.vs.save_local(str(self.index_dir)) + self.vs = FAISS.from_texts(texts=texts, embedding=self.emb, metadatas=metadatas or []) # create new index + self.vs.save_local(str(self.index_dir)) # save the new index return self.vs class ChatIngestor: @@ -125,7 +124,7 @@ def __init__( self, self.model_loader = ModelLoader() self.use_session = use_session_dirs - self.session_id = session_id or _session_id() + self.session_id = session_id or generate_session_id() self.temp_base = Path(temp_base) self.temp_base.mkdir(parents=True, exist_ok=True) diff --git a/utils/document_ops.py b/utils/document_ops.py index edb0c44..06d88e6 100644 --- a/utils/document_ops.py +++ b/utils/document_ops.py @@ -5,6 +5,7 @@ from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader, TextLoader from logger.custom_logger import CustomLogger from exception.custom_exception import DocumentPortalException +from fastapi import UploadFile log = CustomLogger().get_logger(__name__) @@ -42,4 +43,21 @@ def concat_for_analysis(docs: List[Document]) -> str: def concat_for_comparison(ref_docs: List[Document], act_docs: List[Document]) -> str: left = concat_for_analysis(ref_docs) right = concat_for_analysis(act_docs) - return f"<>\n{left}\n\n<>\n{right}" \ No newline at end of file + return f"<>\n{left}\n\n<>\n{right}" + +class FastAPIFileAdapter: + """Adapt FastAPI UploadFile -> .name + .getbuffer() API""" + def __init__(self, uf: UploadFile): + self._uf = uf + self.name = uf.filename + + def getbuffer(self) -> bytes: + self._uf.file.seek(0) + return self._uf.file.read() + +def read_pdf_via_handler(handler, path: str) -> str: + if hasattr(handler, "read_pdf"): + return handler.read_pdf(path) + if hasattr(handler, "read_"): + return handler.read_(path) + raise RuntimeError("DocumentHandler has neither read_pdf nor read_ method.") diff --git a/utils/file_io.py b/utils/file_io.py index 53cfb46..818a339 100644 --- a/utils/file_io.py +++ b/utils/file_io.py @@ -10,7 +10,7 @@ SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".txt"} -def _session_id(prefix: str = "session") -> str: +def generate_session_id(prefix: str = "session") -> str: return f"{prefix}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}" def save_uploaded_files(uploaded_files: Iterable, target_dir: Path) -> List[Path]: