Dev to Main

api/main.py

@@ -14,6 +14,7 @@
 from src.document_analyzer.data_analysis import DocumentAnalyzer
 from src.document_compare.document_comparator import DocumentComparatorLLM
 from src.document_chat.retrieval import ConversationalRAG
+from utils.document_ops import FastAPIFileAdapter, read_pdf_via_handler
 
 FAISS_BASE = os.getenv("FAISS_BASE", "faiss_index")
 UPLOAD_BASE = os.getenv("UPLOAD_BASE", "data")
@@ -48,7 +49,7 @@ async def serve_ui(request: Request) -> HTMLResponse:
 
 @app.get("/health")
 def health() -> Dict[str, str]:
-    return {"status": "ok", "service": "document-bot-api", "version": "0.1"}
+    return {"status": "ok", "service": "document-bot", "version": "0.1"}
 
 # ---------- ANALYZE ----------
 @app.post("/analyze")
@@ -60,7 +61,7 @@ async def analyze_document(file: UploadFile = File(...)) -> Any:
     try:
         dh = DocumentHandler()
         saved_path = dh.save_pdf(FastAPIFileAdapter(file))
-        text = _read_pdf_via_handler(dh, saved_path)
+        text = read_pdf_via_handler(dh, saved_path)
         analyzer = DocumentAnalyzer()
         result = analyzer.analyze_document(text)
         return JSONResponse(content=result)
@@ -79,10 +80,9 @@ async def compare_documents(reference: UploadFile = File(...), actual: UploadFil
     """
     try:
         dc = DocumentComparator()
-        ref_path, act_path = dc.save_uploaded_files(
+        dc.save_uploaded_files(
             FastAPIFileAdapter(reference), FastAPIFileAdapter(actual)
         )
-        _ = ref_path, act_path
         combined_text = dc.combine_documents()
         comp = DocumentComparatorLLM()
         df = comp.compare_documents(combined_text)
@@ -171,25 +171,6 @@ async def chat_query(
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Query failed: {e}")
 
-
-# ---------- Helpers ----------
-class FastAPIFileAdapter:
-    """Adapt FastAPI UploadFile -> .name + .getbuffer() API"""
-    def __init__(self, uf: UploadFile):
-        self._uf = uf
-        self.name = uf.filename
-
-    def getbuffer(self) -> bytes:
-        self._uf.file.seek(0)
-        return self._uf.file.read()
-
-def _read_pdf_via_handler(handler: DocumentHandler, path: str) -> str:
-    if hasattr(handler, "read_pdf"):
-        return handler.read_pdf(path)
-    if hasattr(handler, "read_"):
-        return handler.read_(path)
-    raise RuntimeError("DocumentHandler has neither read_pdf nor read_ method.")
-
 # uvcorn is ASGI server for running FastAPI applications that supports async features which helps in handling multiple requests efficiently
 # To run the FastAPI app, use the command below in your terminal:
 # uvicorn api.main:app --host 0.0.0.0 --port 8080 --reload

src/document_chat/retrieval.py

@@ -27,29 +27,22 @@ def __init__(self, session_id: Optional[str], retriever=None):
         try:
             self.log = CustomLogger().get_logger(__name__)
             self.session_id = session_id
-
-            # Load LLM and prompts once
             self.llm = self._load_llm()
             self.contextualize_prompt: ChatPromptTemplate = PROMPT_REGISTRY[
                 PromptType.CONTEXTUALIZE_QUESTION.value
             ]
             self.qa_prompt: ChatPromptTemplate = PROMPT_REGISTRY[
                 PromptType.CONTEXT_QA.value
             ]
-
-            # Lazy pieces
             self.retriever = retriever
             self.chain = None
             if self.retriever is not None:
                 self._build_lcel_chain()
-
             self.log.info("ConversationalRAG initialized", session_id=self.session_id)
         except Exception as e:
             self.log.error("Failed to initialize ConversationalRAG", error=str(e))
             raise DocumentPortalException("Initialization error in ConversationalRAG", sys)
 
-    # ---------- Public API ----------
-
     def load_retriever_from_faiss(
         self,
         index_path: str,

src/document_ingestion/data_ingestion.py

@@ -13,7 +13,7 @@
 from utils.model_loader import ModelLoader
 from logger.custom_logger import CustomLogger
 from exception.custom_exception import DocumentPortalException
-from utils.file_io import _session_id, save_uploaded_files
+from utils.file_io import generate_session_id, save_uploaded_files
 from utils.document_ops import load_documents
 
 SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".txt"}
@@ -70,18 +70,17 @@ def add_documents(self, docs: List[Document]):
         """
         if self.vs is None:
             raise RuntimeError("Call load_or_create() before add_documents_idempotent().")
-
         new_docs: List[Document] = []
-
         for d in docs:
             key = self._fingerprint(d.page_content, d.metadata or {})
             if key in self._meta["rows"]:
                 continue
             self._meta["rows"][key] = True
             new_docs.append(d)
-
         if new_docs:
-            self.vs.add_documents(new_docs)
+            # This will add new documents to the FAISS index, which first Embeds the chunks and then adds it to the index.
+            # Add / Append new documents over the same existing index
+            self.vs.add_documents(new_docs) 
             self.vs.save_local(str(self.index_dir))
             self._save_meta()
         return len(new_docs)
@@ -100,8 +99,8 @@ def load_or_create(self, texts: Optional[List[str]]=None, metadatas: Optional[Li
         if not texts:
             raise DocumentPortalException("No existing FAISS index and no data to create one", sys)
 
-        self.vs = FAISS.from_texts(texts=texts, embedding=self.emb, metadatas=metadatas or [])
-        self.vs.save_local(str(self.index_dir))
+        self.vs = FAISS.from_texts(texts=texts, embedding=self.emb, metadatas=metadatas or [])  # create new index
+        self.vs.save_local(str(self.index_dir))  # save the new index
         return self.vs
 
 class ChatIngestor:
@@ -125,7 +124,7 @@ def __init__( self,
             self.model_loader = ModelLoader()
 
             self.use_session = use_session_dirs
-            self.session_id = session_id or _session_id()
+            self.session_id = session_id or generate_session_id()
 
             self.temp_base = Path(temp_base)
             self.temp_base.mkdir(parents=True, exist_ok=True)

utils/document_ops.py

@@ -5,6 +5,7 @@
 from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader, TextLoader
 from logger.custom_logger import CustomLogger
 from exception.custom_exception import DocumentPortalException
+from fastapi import UploadFile
 
 log = CustomLogger().get_logger(__name__)
 
@@ -42,4 +43,21 @@ def concat_for_analysis(docs: List[Document]) -> str:
 def concat_for_comparison(ref_docs: List[Document], act_docs: List[Document]) -> str:
     left = concat_for_analysis(ref_docs)
     right = concat_for_analysis(act_docs)
-    return f"<<REFERENCE_DOCUMENTS>>\n{left}\n\n<<ACTUAL_DOCUMENTS>>\n{right}"
+    return f"<<REFERENCE_DOCUMENTS>>\n{left}\n\n<<ACTUAL_DOCUMENTS>>\n{right}"
+
+class FastAPIFileAdapter:
+    """Adapt FastAPI UploadFile -> .name + .getbuffer() API"""
+    def __init__(self, uf: UploadFile):
+        self._uf = uf
+        self.name = uf.filename
+
+    def getbuffer(self) -> bytes:
+        self._uf.file.seek(0)
+        return self._uf.file.read()
+
+def read_pdf_via_handler(handler, path: str) -> str:
+    if hasattr(handler, "read_pdf"):
+        return handler.read_pdf(path)
+    if hasattr(handler, "read_"):
+        return handler.read_(path)
+    raise RuntimeError("DocumentHandler has neither read_pdf nor read_ method.")

utils/file_io.py

@@ -10,7 +10,7 @@
 
 SUPPORTED_EXTENSIONS = {".pdf", ".docx", ".txt"}
 
-def _session_id(prefix: str = "session") -> str:
+def generate_session_id(prefix: str = "session") -> str:
     return f"{prefix}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
 
 def save_uploaded_files(uploaded_files: Iterable, target_dir: Path) -> List[Path]: