v1.8.38: remove venv from git tracking — was bloating repo to 2.67GB

This reverts commit 825765c2c88df7b7e021596665338e2b4cbd602d.

.gitignore

@@ -5,3 +5,15 @@ __pycache__/
 versions/
 *.snapshot
 *.tar.gz
+venv/
+.venv/
+node_modules/
+*.so
+*.pyc
+__pycache__/
+.secrets
+versions/
+*.snapshot
+*.tar.gz
+static/uploads/
+mnt/

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+
+WORKDIR /opt/rag
+
+RUN pip install --no-cache-dir \
+    uvicorn fastapi python-multipart requests \
+    sentence-transformers qdrant-client \
+    psycopg2-binary redis anthropic pymupdf \
+    mem0ai ollama passlib[bcrypt] python-jose[cryptography] bcrypt PyJWT
+
+COPY . .
+
+CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8100"]

PLAN.md

@@ -0,0 +1,115 @@
+# AfakZaid RAG System — Project Plan
+
+## Architecture Overview
+
+```
+User Query
+    ↓
+API Gateway (FastAPI :8100)
+    ↓
+GPTCache (Redis) ──── HIT → Return cached answer
+    ↓ MISS
+Mem0 (Long-term Memory) ── retrieve relevant past context
+    ↓
+Qdrant (Vector DB :6333) ── retrieve relevant book chunks
+    ↓
+Brain / Router
+    ↓                    ↓
+Simple Query         Complex Query
+    ↓                    ↓
+Ollama/Qwen2        Claude API
+(Local, Free)       (Paid, High Quality)
+    ↓                    ↓
+    └──────────┬──────────┘
+               ↓
+        Store in Mem0
+        Store in Cache
+               ↓
+        Return Answer
+```
+
+## Stack
+
+| Component        | Tool                              | Port  | Status     |
+|-----------------|-----------------------------------|-------|------------|
+| Vector DB        | Qdrant                            | 6333  | ✅ Running |
+| RAG Pipeline     | LlamaIndex                        | -     | ⬜ Pending |
+| Embeddings       | CAMeL-Lab Arabic BERT             | -     | ✅ Ready   |
+| Long-term Memory | Mem0 (self-hosted)                | -     | ⬜ Pending |
+| Semantic Cache   | GPTCache + Redis                  | 6379  | ⬜ Pending |
+| Local LLM        | Ollama + Qwen2                    | 11434 | ⬜ Pending |
+| Cloud LLM        | Claude API (complex queries only) | -     | ⬜ Pending |
+| Relational DB    | PostgreSQL                        | 5432  | ⬜ Pending |
+| Cache Layer      | Redis                             | 6379  | ⬜ Pending |
+| API Gateway      | FastAPI                           | 8100  | ✅ Running |
+
+## Query Routing Logic
+
+- GPTCache HIT → return instantly (no LLM call)
+- GPTCache MISS →
+  - Retrieve memory from Mem0
+  - Retrieve context from Qdrant
+  - Route to Brain:
+    - Simple query (short, high confidence) → Ollama/Qwen2
+    - Complex query (long, low confidence, first-time topic) → Claude API
+  - Store result in Mem0 + GPTCache
+
+## Complexity Scoring (Brain Router)
+
+A query is "complex" if ANY of these are true:
+- Query word count > 30
+- Qdrant top result score < 0.75
+- No cache hit for this topic ever
+- User explicitly flags it
+
+## Directory Structure on Server
+
+```
+/opt/rag/
+├── books/          # Uploaded PDF books
+├── cache/          # GPTCache storage
+├── memory/         # Mem0 storage
+├── brain/          # Orchestrator logic
+├── api/            # FastAPI gateway
+├── logs/           # All service logs
+├── models/         # Local model cache
+├── ingest.py       # PDF ingestion pipeline
+├── api.py          # Main API
+└── venv/           # Python virtualenv
+```
+
+## Build Order
+
+1. ✅ Qdrant — vector DB
+2. ✅ CAMeL embeddings
+3. ✅ Basic ingestion pipeline
+4. ✅ Basic FastAPI
+5. ⬜ PostgreSQL — metadata + history
+6. ⬜ Redis — caching layer
+7. ⬜ Ollama + Qwen2 — local LLM
+8. ⬜ Mem0 — long-term memory
+9. ⬜ GPTCache — semantic caching
+10. ⬜ LlamaIndex — RAG pipeline upgrade
+11. ⬜ Brain / Router — hybrid LLM logic
+12. ⬜ Claude API integration
+13. ⬜ Full API Gateway
+14. ⬜ Coolify deployment for all services
+
+## Data Flow — Ingestion
+
+PDF Upload → PyMuPDF extract text → Chunk (500 words, 50 overlap)
+→ CAMeL embed → Qdrant store → PostgreSQL metadata
+
+## Data Flow — Query
+
+Query → GPTCache check → Mem0 retrieve → Qdrant search
+→ Brain route → LLM generate → Mem0 store → Cache store → Return
+
+## Environment Variables Needed
+
+- CLAUDE_API_KEY
+- POSTGRES_URL
+- REDIS_URL=redis://localhost:6379
+- QDRANT_URL=http://localhost:6333
+- OLLAMA_URL=http://localhost:11434
+- MEM0_API_KEY (if using cloud) or self-hosted

api.py

@@ -9,14 +9,14 @@ from typing import Optional
 from urllib.parse import quote
 import urllib.request
 
-from brain import process_query
+from brain import process_query, stream_query
 from meta_extract import extract_and_save_bg as _meta_bg
 from routes import router as main_router
 from worker import start_worker, stop_worker
 from qdrant_client import QdrantClient
 import psycopg2
 from psycopg2.extras import RealDictCursor
-from auth import get_current_user, get_current_user_optional, require_upload, bearer
+from auth import get_current_user, get_current_user_optional, require_upload, require_admin, bearer
 
 # ── Config ────────────────────────────────────────────────────────────────────
 BOOKS_DIR  = "/mnt/rag-media/pdfs"
@@ -52,11 +52,6 @@ async def verify_key(key: str = Depends(api_key_header)):
         raise HTTPException(status_code=401, detail="Invalid or missing API key")
     return key
 
-async def verify_admin(key: str = Depends(api_key_header)):
-    admin_key = os.getenv("ADMIN_KEY", "")
-    if not key or key != admin_key:
-        raise HTTPException(status_code=401, detail="Admin access required")
-    return key
 
 # routes.py is included FIRST so its routes take priority
 app.include_router(main_router)
@@ -90,11 +85,16 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request, call_next):
         response = await call_next(request)
         response.headers["X-Content-Type-Options"]    = "nosniff"
-        response.headers["X-Frame-Options"]           = "DENY"
         response.headers["X-XSS-Protection"]          = "1; mode=block"
         response.headers["Referrer-Policy"]           = "strict-origin-when-cross-origin"
         response.headers["Permissions-Policy"]        = "geolocation=(), microphone=(), camera=()"
         response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
+        # Allow iframes only from same origin (needed for PDF reader)
+        path = request.url.path
+        if path.endswith("/read") or "/reader" in path:
+            response.headers["X-Frame-Options"] = "SAMEORIGIN"
+        else:
+            response.headers["X-Frame-Options"] = "SAMEORIGIN"
         return response
 
 app.add_middleware(SecurityHeadersMiddleware)
@@ -127,6 +127,9 @@ class BrandingUpdate(BaseModel):
     site_subtitle: str = None
     primary_color: str = None
     secondary_color: str = None
+    logo_size: str = None
+    bg_color: str = None
+    bg_image: str = None
     logo_url: str = None
 
 class ClaudeKeyUpdate(BaseModel):
@@ -134,6 +137,7 @@ class ClaudeKeyUpdate(BaseModel):
 
 # ── Page Routes ───────────────────────────────────────────────────────────────
 @app.get("/")
+@app.head("/")
 def root():       return FileResponse("/opt/rag/static/index.html")
 
 @app.get("/admin")
@@ -142,6 +146,56 @@ def admin_ui():   return FileResponse("/opt/rag/static/admin.html")
 @app.get("/library")
 def library_ui(): return FileResponse("/opt/rag/static/library.html")
 
+
+@app.get("/admin/pages/{page_name}")
+def get_page_content(page_name: str, _=Depends(require_admin)):
+    allowed = {"privacy", "terms", "about", "index", "library"}
+    if page_name not in allowed:
+        raise HTTPException(404, "Page not found")
+    path = f"/opt/rag/static/{page_name}.html"
+    if not os.path.exists(path):
+        raise HTTPException(404, "File not found")
+    with open(path, "r") as f:
+        return {"page": page_name, "content": f.read()}
+
+@app.post("/admin/pages/{page_name}")
+def save_page_content(page_name: str, payload: dict, _=Depends(require_admin)):
+    allowed = {"privacy", "terms", "about", "index", "library"}
+    if page_name not in allowed:
+        raise HTTPException(404, "Page not found")
+    content = payload.get("content", "")
+    if not content.strip():
+        raise HTTPException(400, "Content cannot be empty")
+    path = f"/opt/rag/static/{page_name}.html"
+    with open(path, "w") as f:
+        f.write(content)
+    return {"status": "saved", "page": page_name}
+
+@app.get("/favicon.ico")
+def favicon(): return FileResponse("/opt/rag/static/favicon.ico", media_type="image/x-icon")
+
+@app.get("/apple-touch-icon.png")
+def apple_icon(): return FileResponse("/opt/rag/static/apple-touch-icon.png", media_type="image/png")
+
+@app.get("/components.js")
+def components_js():
+    from fastapi.responses import FileResponse as FR
+    return FR("/opt/rag/static/components.js", media_type="application/javascript")
+
+@app.get("/google1d7bb3283b75009e.html")
+def google_verify(): return FileResponse("/opt/rag/static/google1d7bb3283b75009e.html")
+
+@app.get("/about")
+def about(): return FileResponse("/opt/rag/static/about.html")
+
+@app.get("/privacy")
+@app.head("/privacy")
+def privacy(): return FileResponse("/opt/rag/static/privacy.html")
+
+@app.get("/terms")
+@app.head("/terms")
+def terms(): return FileResponse("/opt/rag/static/terms.html")
+
 @app.get("/reader")
 def reader_ui():  return FileResponse("/opt/rag/static/reader.html")
 
@@ -157,7 +211,7 @@ def health():
     return {"status": "ok", "version": "2.0.0"}
 
 @app.get("/admin/services")
-def admin_services(_=Depends(verify_admin)):
+def admin_services(_=Depends(require_admin)):
     """Return live health status of all services."""
     import requests as _req
 
@@ -260,9 +314,15 @@ def stats():
             with conn.cursor() as cur:
                 cur.execute("SELECT COUNT(*) FROM books");        books   = cur.fetchone()[0]
                 cur.execute("SELECT COUNT(*) FROM conversations"); convos  = cur.fetchone()[0]
-                cur.execute("SELECT COUNT(*) FROM memory");        memories = cur.fetchone()[0]
+                try:
+                    import urllib.request, json as _json
+                    with urllib.request.urlopen("http://172.16.2.1:6333/collections/mem0_memory", timeout=2) as _r:
+                        _mi = _json.loads(_r.read())
+                    memories = _mi.get("result",{}).get("points_count", 0)
+                except Exception as _me:
+                    memories = 0
     except Exception as e:
-        print(f"Stats db error: {e}")
+        print(f"Stats db error: {e}", flush=True)
         books = convos = memories = 0
     # Cache count
     cache_count = 0
@@ -279,12 +339,28 @@ def stats():
             "memories": memories, "cache": cache_count}
 
 @app.post("/query", response_model=QueryResponse)
-@limiter.limit("30/minute")
+@limiter.limit("2/minute")
 def query(request: Request, req: QueryRequest, _: str = Depends(verify_key)):
     if not req.query.strip():
         raise HTTPException(status_code=400, detail="Query cannot be empty")
     return QueryResponse(**process_query(req.query, req.session_id))
 
+
+@app.post("/query/stream")
+@limiter.limit("30/minute")
+def query_stream(request: Request, req: QueryRequest, _: str = Depends(verify_key)):
+    """SSE streaming endpoint — yields tokens as they are generated."""
+    if not req.query.strip():
+        raise HTTPException(status_code=400, detail="Query cannot be empty")
+    return StreamingResponse(
+        stream_query(req.query, req.session_id),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",   # disable Nginx/Traefik buffering
+        }
+    )
+
 @app.post("/ingest")
 @limiter.limit("20/minute")
 async def ingest_pdf(request: Request, file: UploadFile = File(...), _=Depends(verify_key)):
@@ -368,7 +444,7 @@ def clear_memory(session_id: str, _=Depends(verify_key)):
 
 # ── Admin ─────────────────────────────────────────────────────────────────────
 @app.get("/admin/overview")
-def admin_overview(_=Depends(verify_admin)):
+def admin_overview(_=Depends(require_admin)):
     cache_keys, cache_memory = 0, "?"
     try:
         import redis as _r
@@ -385,7 +461,11 @@ def admin_overview(_=Depends(verify_admin)):
             with conn.cursor(cursor_factory=RealDictCursor) as cur:
                 cur.execute("SELECT COUNT(*) as total FROM books");           books = cur.fetchone()["total"]
                 cur.execute("SELECT COUNT(*) as total FROM conversations");   convos = cur.fetchone()["total"]
-                cur.execute("SELECT COUNT(*) as total FROM memory");          memories = cur.fetchone()["total"]
+                try:
+                    import urllib.request, json as _j
+                    with urllib.request.urlopen("http://172.16.2.1:6333/collections/mem0_memory", timeout=2) as _r:
+                        memories = _j.loads(_r.read()).get("result",{}).get("points_count",0)
+                except: memories = 0
                 cur.execute("SELECT COUNT(DISTINCT session_id) as total FROM conversations"); unique_users = cur.fetchone()["total"]
                 cur.execute("SELECT llm_used, COUNT(*) as cnt FROM conversations GROUP BY llm_used ORDER BY cnt DESC")
                 llm_usage = cur.fetchall()
@@ -423,8 +503,122 @@ def admin_overview(_=Depends(verify_admin)):
         }
     }
 
+
+# ── Library Health ─────────────────────────────────────────────────────────────
+@app.get("/admin/library-health")
+def admin_library_health(_=Depends(require_admin)):
+    from qdrant_client import QdrantClient as _QC
+    import os as _os
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT COUNT(*) AS total FROM books")
+            total_books = cur.fetchone()["total"]
+            cur.execute("""
+                SELECT b.id, b.filename, b.title
+                FROM books b
+                WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.book_filename = b.filename)
+                ORDER BY b.filename
+            """)
+            missing = cur.fetchall()
+            try:
+                cur.execute("""
+                    SELECT filename, status, error_message, created_at
+                    FROM ingest_jobs WHERE status='failed'
+                    ORDER BY created_at DESC LIMIT 10
+                """)
+                failed = cur.fetchall()
+                cur.execute("SELECT COUNT(*) AS c FROM ingest_jobs WHERE status='pending' OR status='running'")
+                pending_jobs = cur.fetchone()["c"]
+            except Exception:
+                conn.rollback()
+                failed = []
+                pending_jobs = 0
+            cur.execute("SELECT COUNT(*) AS c FROM chunks")
+            total_chunks = cur.fetchone()["c"]
+
+    try:
+        qc = _QC(url=_os.getenv("QDRANT_URL","http://qdrant:6333"),
+                 api_key=_os.getenv("QDRANT_API_KEY",""))
+        info = qc.get_collection("arabic_books")
+        vector_count = info.vectors_count or info.points_count or 0
+    except Exception:
+        vector_count = 0
+
+    return {
+        "total_books":    total_books,
+        "indexed_books":  total_books - len(missing),
+        "missing_chunks": len(missing),
+        "missing_books":  [dict(r) for r in missing],
+        "failed_files":   [dict(r) for r in failed],
+        "pending_jobs":   pending_jobs,
+        "total_chunks":   total_chunks,
+        "vector_count":   vector_count,
+    }
+
+
+# ── Ingest single book ─────────────────────────────────────────────────────────
+@app.post("/admin/ingest-book/{book_id}")
+def admin_ingest_book(book_id: int, _=Depends(require_admin)):
+    import os as _os, sys as _sys
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT filename FROM books WHERE id=%s", (book_id,))
+            row = cur.fetchone()
+            if not row:
+                raise HTTPException(status_code=404, detail="Book not found")
+            filename = row["filename"]
+    path = f"/mnt/rag-media/pdfs/{filename}"
+    if not _os.path.exists(path):
+        raise HTTPException(status_code=404, detail=f"PDF file not found: {filename}")
+    _sys.path.insert(0, "/opt/rag")
+    import ingest as _ingest
+    try:
+        _ingest.ingest_pdf(path)
+        return {"status": "ok", "filename": filename}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# ── Rebuild BM25 ───────────────────────────────────────────────────────────────
+@app.post("/admin/rebuild-bm25")
+def admin_rebuild_bm25(_=Depends(require_admin)):
+    from brain import rebuild_bm25
+    rebuild_bm25()
+    return {"status": "ok", "message": "BM25 index rebuilt"}
+
+# ── Clear Cache ────────────────────────────────────────────────────────────────
+@app.post("/admin/clear-cache")
+def admin_clear_cache(_=Depends(require_admin)):
+    import redis as _redis, os as _os
+    url  = _os.getenv("REDIS_URL", "redis://rag-redis:6379")
+    pwd  = _os.getenv("REDIS_PASSWORD", "")
+    r    = _redis.from_url(url + (f"?password={pwd}" if pwd else ""), decode_responses=True)
+    keys = r.keys("rag:cache:*")
+    deleted = len(keys)
+    if keys: r.delete(*keys)
+    return {"status": "ok", "deleted_keys": deleted}
+
+# ── Re-ingest Missing ──────────────────────────────────────────────────────────
+@app.post("/admin/reingest-missing")
+def admin_reingest_missing(_=Depends(require_admin)):
+    from tasks import ingest_pdf_task
+    import os as _os
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT filename FROM books b
+                WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.book_filename = b.filename)
+            """)
+            rows = cur.fetchall()
+    queued = 0
+    for r in rows:
+        path = f"/mnt/rag-media/pdfs/{r['filename']}"
+        if _os.path.exists(path):
+            ingest_pdf_task.delay(path)
+            queued += 1
+    return {"status": "ok", "queued": queued}
+
 @app.get("/admin/books")
-def admin_books(_=Depends(verify_admin)):
+def admin_books(_=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
             cur.execute("""SELECT b.*, u.name as uploader_name
@@ -433,7 +627,7 @@ def admin_books(_=Depends(verify_admin)):
             return {"books": [dict(r) for r in cur.fetchall()]}
 
 @app.get("/admin/books/{book_id}")
-def admin_get_book(book_id: int, _=Depends(verify_admin)):
+def admin_get_book(book_id: int, _=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
             cur.execute("""SELECT b.*, u.name as uploader_name
@@ -445,7 +639,7 @@ def admin_get_book(book_id: int, _=Depends(verify_admin)):
     return dict(book)
 
 @app.post("/admin/books/{book_id}/extract-meta")
-def admin_extract_meta(book_id: int, _=Depends(verify_admin)):
+def admin_extract_meta(book_id: int, _=Depends(require_admin)):
     """Trigger Ollama metadata re-extraction for a book (background)."""
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
@@ -457,7 +651,7 @@ def admin_extract_meta(book_id: int, _=Depends(verify_admin)):
     return {"status": "extraction_started", "book_id": book_id}
 
 @app.get("/admin/conversations")
-def admin_conversations(limit: int = 50, session: str = None, _=Depends(verify_admin)):
+def admin_conversations(limit: int = 50, session: str = None, _=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
             if session:
@@ -467,7 +661,7 @@ def admin_conversations(limit: int = 50, session: str = None, _=Depends(verify_a
             return {"conversations": [dict(r) for r in cur.fetchall()]}
 
 @app.get("/admin/sessions")
-def admin_sessions(_=Depends(verify_admin)):
+def admin_sessions(_=Depends(require_admin)):
     """Active login sessions"""
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
@@ -481,17 +675,16 @@ def admin_sessions(_=Depends(verify_admin)):
             return {"sessions": [dict(r) for r in cur.fetchall()]}
 
 @app.get("/admin/settings")
-def admin_settings(_=Depends(verify_admin)):
+def admin_settings(_=Depends(require_admin)):
     return {
         "ollama_model": os.getenv("OLLAMA_MODEL", "qwen2:7b"),
         "ollama_url": os.getenv("OLLAMA_URL", "http://localhost:11434"),
         "qdrant_url": QDRANT_URL,
         "claude_api_configured": bool(os.getenv("CLAUDE_API_KEY", "")),
         "cache_ttl_hours": 24,
-        "complexity_threshold_words": 30,
         "confidence_threshold": 0.20,  # SCORE_THRESH in brain.py
         "chunk_size": 150,
-        "chunk_overlap": 200,
+        "chunk_overlap": 2,  # sentences (≈30 words)
         # ── Cloudflare R2 ─────────────────────────────────────────────────────
         "r2_endpoint":    os.getenv("R2_ENDPOINT",   ""),
         "r2_bucket":      os.getenv("R2_BUCKET",     ""),
@@ -503,7 +696,7 @@ def admin_settings(_=Depends(verify_admin)):
     }
 
 @app.post("/admin/r2-config")
-def save_r2_config(data: dict, _=Depends(verify_admin)):
+def save_r2_config(data: dict, _=Depends(require_admin)):
     """Save Cloudflare R2 config to env AND persist to /opt/rag/.secrets."""
     allowed = {"R2_ENDPOINT","R2_ACCESS_KEY","R2_SECRET_KEY","R2_BUCKET","R2_PUBLIC_URL"}
 
@@ -543,7 +736,7 @@ def save_r2_config(data: dict, _=Depends(verify_admin)):
     return {"status": "saved", "note": "Saved to runtime env and /opt/rag/.secrets"}
 
 @app.get("/admin/r2-test")
-def test_r2_connection(_=Depends(verify_admin)):
+def test_r2_connection(_=Depends(require_admin)):
     """Test R2 connection using stored credentials — never modifies anything."""
     import boto3
     from botocore.exceptions import ClientError, NoCredentialsError
@@ -572,15 +765,33 @@ def test_r2_connection(_=Depends(verify_admin)):
     except Exception as e:
         return {"ok": False, "detail": str(e)[:120]}
 
+
+# ── Upload Logo ────────────────────────────────────────────────────────────────
+@app.post("/admin/upload-logo")
+async def upload_logo(payload: dict, _=Depends(require_admin)):
+    import base64, os, uuid, mimetypes
+    data      = payload.get("data","")
+    mime_type = payload.get("mime_type","image/png")
+    filename  = payload.get("filename","logo.png")
+    if not data:
+        raise HTTPException(status_code=400, detail="No image data")
+    ext  = mimetypes.guess_extension(mime_type) or ".png"
+    name = f"logo_{uuid.uuid4().hex[:8]}{ext}"
+    path = f"/opt/rag/static/uploads/{name}"
+    os.makedirs("/opt/rag/static/uploads", exist_ok=True)
+    with open(path, "wb") as f:
+        f.write(base64.b64decode(data))
+    return {"url": f"/static/uploads/{name}"}
+
 @app.get("/admin/branding")
-def get_branding(_=Depends(verify_admin)):
+def get_branding(_=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
             cur.execute("SELECT key, value FROM branding ORDER BY key")
             return {r["key"]: r["value"] for r in cur.fetchall()}
 
 @app.post("/admin/branding")
-def update_branding(data: BrandingUpdate, _=Depends(verify_admin)):
+def update_branding(data: BrandingUpdate, _=Depends(require_admin)):
     updates = {k: v for k, v in data.dict().items() if v is not None}
     with get_db() as conn:
         with conn.cursor() as cur:
@@ -597,7 +808,7 @@ def public_branding():
             return {r["key"]: r["value"] for r in cur.fetchall()}
 
 @app.get("/admin/claude")
-def get_claude_settings(_=Depends(verify_admin)):
+def get_claude_settings(_=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
             cur.execute("SELECT value FROM api_settings WHERE key='claude_api_key'")
@@ -614,7 +825,7 @@ def get_claude_settings(_=Depends(verify_admin)):
             "totals": dict(totals) if totals else {}, "daily": [dict(r) for r in daily]}
 
 @app.post("/admin/claude/key")
-def update_claude_key(data: ClaudeKeyUpdate, _=Depends(verify_admin)):
+def update_claude_key(data: ClaudeKeyUpdate, _=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor() as cur:
             cur.execute("INSERT INTO api_settings (key,value,updated_at) VALUES ('claude_api_key',%s,NOW()) ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value, updated_at=NOW()", (data.api_key,))
@@ -623,7 +834,7 @@ def update_claude_key(data: ClaudeKeyUpdate, _=Depends(verify_admin)):
     return {"status": "saved"}
 
 @app.get("/admin/claude/usage")
-def get_claude_usage(limit: int = 50, _=Depends(verify_admin)):
+def get_claude_usage(limit: int = 50, _=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
             cur.execute("SELECT session_id, query, input_tokens, output_tokens, cost_usd, created_at FROM claude_usage ORDER BY created_at DESC LIMIT %s", (limit,))
@@ -808,6 +1019,7 @@ def delete_book(book_id: int, _=Depends(require_upload)):
 
     return {"status": "deleted", "details": results}
 
+
 # ── Analytics ─────────────────────────────────────────────────────────────────
 @app.get("/analytics/config")
 def analytics_config():
@@ -824,7 +1036,7 @@ def analytics_config():
     }
 
 @app.get("/admin/analytics")
-def get_analytics(_=Depends(verify_admin)):
+def get_analytics(_=Depends(require_admin)):
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
             cur.execute("SELECT key, value FROM analytics_config")
@@ -841,7 +1053,7 @@ class AnalyticsGAUpdate(BaseModel):
     enabled: bool = True
 
 @app.post("/admin/analytics/ga")
-def update_analytics_ga(data: AnalyticsGAUpdate, _=Depends(verify_admin)):
+def update_analytics_ga(data: AnalyticsGAUpdate, _=Depends(require_admin)):
     mid = data.measurement_id.strip()
     if mid and not mid.startswith("G-"):
         raise HTTPException(400, "Measurement ID must start with G-")
@@ -853,7 +1065,7 @@ def update_analytics_ga(data: AnalyticsGAUpdate, _=Depends(verify_admin)):
     return {"status": "saved"}
 
 @app.post("/admin/analytics/events")
-def update_analytics_events(payload: dict, _=Depends(verify_admin)):
+def update_analytics_events(payload: dict, _=Depends(require_admin)):
     import json as _json
     events = payload.get("events", [])
     with get_db() as conn:
@@ -864,7 +1076,7 @@ def update_analytics_events(payload: dict, _=Depends(verify_admin)):
     return {"status": "saved", "count": len(events)}
 
 @app.get("/admin/analytics/pixels")
-def get_pixels(_=Depends(verify_admin)):
+def get_pixels(_=Depends(require_admin)):
     import json as _json
     with get_db() as conn:
         with conn.cursor(cursor_factory=RealDictCursor) as cur:
@@ -873,7 +1085,7 @@ def get_pixels(_=Depends(verify_admin)):
     return {"pixels": _json.loads(row["value"] if row else "[]")}
 
 @app.post("/admin/analytics/pixels")
-def update_pixels(payload: dict, _=Depends(verify_admin)):
+def update_pixels(payload: dict, _=Depends(require_admin)):
     import json as _json
     pixels = payload.get("pixels", [])
     with get_db() as conn:
@@ -903,7 +1115,7 @@ def analytics_full_config():
 
 # ── Analytics Page Scanner ────────────────────────────────────────────────────
 @app.get("/admin/analytics/scan")
-def scan_page_elements(page: str = "library", _=Depends(verify_admin)):
+def scan_page_elements(page: str = "library", _=Depends(require_admin)):
     """Scan a frontend page HTML and return trackable elements grouped by type."""
     import re
     from collections import defaultdict
@@ -1033,3 +1245,91 @@ def scan_page_elements(page: str = "library", _=Depends(verify_admin)):
 
     return {"page": page, "total": len(elements), "groups": ordered}
 
+# ── LLM Provider Config ───────────────────────────────────────────────────────
+@app.get("/admin/llm-config")
+def get_llm_config_endpoint(_=Depends(require_admin)):
+    import json as _j
+    try:
+        with get_db() as conn:
+            with conn.cursor(cursor_factory=RealDictCursor) as cur:
+                cur.execute("SELECT value FROM api_settings WHERE key='llm_provider'")
+                row = cur.fetchone()
+                if row and row["value"]:
+                    cfg = _j.loads(row["value"])
+                    token = cfg.get("cf_api_token","")
+                    cfg["cf_api_token_masked"] = ("*"*(len(token)-4)+token[-4:]) if len(token)>4 else "*"*len(token)
+                    cfg["cf_api_token_configured"] = bool(token)
+                    return cfg
+    except Exception as e:
+        print(f"get_llm_config error: {e}")
+    return {"provider":"local","cf_account_id":"","cf_model":"@cf/meta/llama-3.1-8b-instruct-fp8-fast","cf_api_token_configured":False,"cf_api_token_masked":""}
+
+@app.post("/admin/llm-config")
+def save_llm_config_endpoint(data: dict, _=Depends(require_admin)):
+    import json as _j
+    provider = data.get("provider","local")
+    cf_account_id = data.get("cf_account_id","").strip()
+    cf_api_token  = data.get("cf_api_token","").strip()
+    cf_model      = data.get("cf_model","@cf/meta/llama-3.1-8b-instruct-fp8-fast").strip()
+    try:
+        with get_db() as conn:
+            with conn.cursor(cursor_factory=RealDictCursor) as cur:
+                cur.execute("SELECT value FROM api_settings WHERE key='llm_provider'")
+                row = cur.fetchone()
+                existing = _j.loads(row["value"]) if row and row["value"] else {}
+    except: existing = {}
+    if not cf_api_token:
+        cf_api_token = existing.get("cf_api_token","")
+    cfg = {"provider":provider,"cf_account_id":cf_account_id,"cf_api_token":cf_api_token,"cf_model":cf_model}
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("INSERT INTO api_settings (key,value,updated_at) VALUES ('llm_provider',%s,NOW()) ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value, updated_at=NOW()", (_j.dumps(cfg),))
+        conn.commit()
+    os.environ["LLM_PROVIDER"] = provider
+    os.environ["CF_ACCOUNT_ID"] = cf_account_id
+    os.environ["CF_API_TOKEN"]  = cf_api_token
+    os.environ["CF_MODEL"]      = cf_model
+    return {"status":"saved","provider":provider}
+
+@app.post("/admin/llm-test")
+def test_llm_config_endpoint(data: dict, _=Depends(require_admin)):
+    import requests as _req
+    provider      = data.get("provider","local")
+    cf_account_id = data.get("cf_account_id","").strip()
+    cf_api_token  = data.get("cf_api_token","").strip()
+    cf_model      = data.get("cf_model","@cf/meta/llama-3.1-8b-instruct-fp8-fast").strip()
+    # Load saved token if blank
+    if not cf_api_token:
+        import json as _j
+        try:
+            with get_db() as conn:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT value FROM api_settings WHERE key='llm_provider'")
+                    row = cur.fetchone()
+                    if row and row[0]:
+                        saved = _j.loads(row[0])
+                        cf_api_token  = saved.get("cf_api_token","")
+                        if not cf_account_id:
+                            cf_account_id = saved.get("cf_account_id","")
+        except: pass
+    if provider == "cloudflare":
+        if not cf_account_id or not cf_api_token:
+            return {"ok":False,"detail":"Account ID and API Token required"}
+        url = f"https://api.cloudflare.com/client/v4/accounts/{cf_account_id}/ai/run/{cf_model}"
+        try:
+            resp = _req.post(url, headers={"Authorization":f"Bearer {cf_api_token}","Content-Type":"application/json"},
+                json={"messages":[{"role":"user","content":"Reply with only the word: OK"}],"max_tokens":10}, timeout=20)
+            d = resp.json()
+            if not resp.ok: return {"ok":False,"detail":f"HTTP {resp.status_code}: {d}"}
+            answer = (d.get("result",{}).get("response") or "").strip()
+            return {"ok":True,"detail":f"Response: {answer or '(empty)'}"}
+        except Exception as e:
+            return {"ok":False,"detail":str(e)}
+    else:
+        try:
+            r = _req.get(f"{os.getenv('OLLAMA_URL','http://localhost:11434')}/api/tags", timeout=5)
+            models = [m["name"] for m in r.json().get("models",[])]
+            return {"ok":True,"detail":f"Local Ollama — models: {', '.join(models[:3])}"}
+        except Exception as e:
+            return {"ok":False,"detail":str(e)}
+

api.py.bak

@@ -0,0 +1,927 @@
+import os
+import shutil
+import subprocess
+from fastapi import FastAPI, UploadFile, File, HTTPException, Depends
+from fastapi.security import APIKeyHeader
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import Optional
+from brain import process_query
+from routes import router as main_router
+from qdrant_client import QdrantClient
+import psycopg2
+from psycopg2.extras import RealDictCursor
+
+BOOKS_DIR    = "/opt/rag/books"
+QDRANT_URL   = os.getenv("QDRANT_URL", "http://localhost:6333")
+POSTGRES_URL = os.getenv("POSTGRES_URL", "postgresql://raguser:rag_secure_2025@localhost:5432/ragdb")
+COLLECTION   = "arabic_books"
+
+os.makedirs(BOOKS_DIR, exist_ok=True)
+
+app = FastAPI(title="AfakZaid RAG API", version="2.0.0")
+
+API_KEY = os.getenv("RAG_API_KEY", "")
+api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
+
+async def verify_key(key: str = Depends(api_key_header)):
+    if API_KEY:
+        if not key or key != API_KEY:
+            raise HTTPException(status_code=401, detail="Invalid or missing API key")
+    return key
+
+app.include_router(main_router)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"]
+)
+
+qdrant = QdrantClient(url=QDRANT_URL)
+
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+# ── Models ────────────────────────────────────────────────────────────────────
+class QueryRequest(BaseModel):
+    query: str
+    session_id: str = "default"
+    top_k: int = 5
+
+class QueryResponse(BaseModel):
+    answer: str
+    llm: str
+    source: str
+    confidence: float = 0.0
+    books_used: list = []
+    chunks: list = []
+
+# ── Routes ────────────────────────────────────────────────────────────────────
+@app.get("/health")
+def health():
+    return {"status": "ok", "version": "2.0.0"}
+
+@app.get("/stats")
+def stats():
+    try:
+        info = qdrant.get_collection(COLLECTION)
+        vectors = info.points_count
+    except:
+        vectors = 0
+    try:
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("SELECT COUNT(*) FROM books")
+                books = cur.fetchone()[0]
+                cur.execute("SELECT COUNT(*) FROM conversations")
+                convos = cur.fetchone()[0]
+                cur.execute("SELECT COUNT(*) FROM memory")
+                memories = cur.fetchone()[0]
+    except:
+        books = convos = memories = 0
+    return {
+        "vectors": vectors,
+        "books": books,
+        "conversations": convos,
+        "memories": memories
+    }
+
+@app.post("/query", response_model=QueryResponse)
+def query(req: QueryRequest, _: str = Depends(verify_key)):
+    if not req.query.strip():
+        raise HTTPException(status_code=400, detail="Query cannot be empty")
+    result = process_query(req.query, req.session_id)
+    return QueryResponse(**result)
+
+@app.post("/ingest")
+async def ingest_pdf(file: UploadFile = File(...), _=Depends(verify_key)):
+    if not file.filename.endswith(".pdf"):
+        raise HTTPException(status_code=400, detail="Only PDF files are allowed")
+    dest = os.path.join(BOOKS_DIR, file.filename)
+    with open(dest, "wb") as f:
+        shutil.copyfileobj(file.file, f)
+    result = subprocess.run(
+        ["python3", "/opt/rag/ingest.py", dest],
+        capture_output=True, text=True, timeout=300, env={**__import__("os").environ}
+    )
+    return {"status": "done", "file": file.filename, "log": result.stdout[-500:]}
+
+@app.get("/books")
+def list_books():
+    try:
+        with get_db() as conn:
+            with conn.cursor(cursor_factory=RealDictCursor) as cur:
+                cur.execute("SELECT * FROM books ORDER BY uploaded_at DESC")
+                return {"books": cur.fetchall()}
+    except:
+        books = [f for f in os.listdir(BOOKS_DIR) if f.endswith(".pdf")]
+        return {"books": books}
+
+@app.get("/history/{session_id}")
+def get_history(session_id: str, limit: int = 20):
+    try:
+        with get_db() as conn:
+            with conn.cursor(cursor_factory=RealDictCursor) as cur:
+                cur.execute("""
+                    SELECT user_query, assistant_response, llm_used, created_at
+                    FROM conversations WHERE session_id = %s
+                    ORDER BY created_at DESC LIMIT %s
+                """, (session_id, limit))
+                return {"session_id": session_id, "history": cur.fetchall()}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.delete("/memory/{session_id}")
+def clear_memory(session_id: str, _=Depends(verify_key)):
+    try:
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("DELETE FROM memory WHERE session_id = %s", (session_id,))
+            conn.commit()
+        return {"status": "cleared", "session_id": session_id}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+import urllib.request
+
+app.mount("/static", StaticFiles(directory="/opt/rag/static"), name="static")
+
+@app.get("/")
+def root():
+    return FileResponse("/opt/rag/static/index.html")
+
+@app.get("/admin")
+def admin_ui():
+    return FileResponse("/opt/rag/static/admin.html")
+
+@app.get("/reader")
+def reader_ui():
+    return FileResponse("/opt/rag/static/reader.html")
+
+@app.get("/login")
+def login_ui():
+    return FileResponse("/opt/rag/static/login.html")
+
+@app.get("/library")
+def library_page():
+    return FileResponse("/opt/rag/static/library.html")
+
+@app.get("/read")
+def reader_page():
+    return FileResponse("/opt/rag/static/reader.html")
+
+@app.get("/library")
+def library_page():
+    return FileResponse("/opt/rag/static/library.html")
+
+@app.get("/read")
+def reader_page():
+    return FileResponse("/opt/rag/static/reader.html")
+
+@app.get("/reader")
+def reader_ui():
+    return FileResponse("/opt/rag/static/reader.html")
+
+@app.get("/login")
+def login_ui():
+    return FileResponse("/opt/rag/static/login.html")
+
+@app.get("/library")
+def library_page():
+    return FileResponse("/opt/rag/static/library.html")
+
+@app.get("/read")
+def reader_page():
+    return FileResponse("/opt/rag/static/reader.html")
+
+@app.get("/library")
+def library_page():
+    return FileResponse("/opt/rag/static/library.html")
+
+@app.get("/read")
+def reader_page():
+    return FileResponse("/opt/rag/static/reader.html")
+
+@app.get("/library")
+def library_ui():
+    return FileResponse("/opt/rag/static/library.html")
+
+class URLIngestRequest(BaseModel):
+    url: str
+
+@app.post("/ingest-url")
+async def ingest_url(req: URLIngestRequest, _=Depends(verify_key)):
+    if not req.url.endswith(".pdf"):
+        raise HTTPException(status_code=400, detail="URL must point to a PDF file")
+    filename = req.url.split("/")[-1].split("?")[0]
+    if not filename.endswith(".pdf"):
+        filename = "book_" + str(int(__import__('time').time())) + ".pdf"
+    dest = os.path.join(BOOKS_DIR, filename)
+    urllib.request.urlretrieve(req.url, dest)
+    result = subprocess.run(
+        ["python3", "/opt/rag/ingest.py", dest],
+        capture_output=True, text=True, timeout=300, env={**__import__("os").environ}
+    )
+    return {"status": "done", "file": filename, "log": result.stdout[-500:]}
+
+# ── Admin Routes ──────────────────────────────────────────────────────────────
+ADMIN_KEY = os.getenv("ADMIN_KEY", "admin-" + os.getenv("RAG_API_KEY", "")[:16])
+
+async def verify_admin(key: str = Depends(api_key_header)):
+    admin_key = os.getenv("ADMIN_KEY", "")
+    if not key or key != admin_key:
+        raise HTTPException(status_code=401, detail="Admin access required")
+    return key
+
+@app.get("/admin/overview")
+def admin_overview(_=Depends(verify_admin)):
+    try:
+        import redis as r
+        rc = r.from_url(os.getenv("REDIS_URL", "redis://localhost:6379"), decode_responses=True)
+        cache_keys = len(rc.keys("rag:cache:*"))
+        cache_memory = rc.info("memory").get("used_memory_human", "?")
+    except:
+        cache_keys = 0
+        cache_memory = "?"
+    try:
+        with get_db() as conn:
+            with conn.cursor(cursor_factory=RealDictCursor) as cur:
+                cur.execute("SELECT COUNT(*) as total FROM books")
+                books = cur.fetchone()["total"]
+                cur.execute("SELECT COUNT(*) as total FROM conversations")
+                convos = cur.fetchone()["total"]
+                cur.execute("SELECT COUNT(*) as total FROM memory")
+                memories = cur.fetchone()["total"]
+                cur.execute("SELECT COUNT(DISTINCT session_id) as total FROM conversations")
+                unique_users = cur.fetchone()["total"]
+                cur.execute("SELECT llm_used, COUNT(*) as cnt FROM conversations GROUP BY llm_used ORDER BY cnt DESC")
+                llm_usage = cur.fetchall()
+                cur.execute("SELECT DATE(created_at) as day, COUNT(*) as cnt FROM conversations GROUP BY day ORDER BY day DESC LIMIT 7")
+                daily = cur.fetchall()
+                cur.execute("SELECT user_query, created_at, llm_used FROM conversations ORDER BY created_at DESC LIMIT 10")
+                recent = cur.fetchall()
+    except Exception as e:
+        return {"error": str(e)}
+    try:
+        info = qdrant.get_collection(COLLECTION)
+        vectors = info.points_count
+    except:
+        vectors = 0
+    try:
+        with open("/proc/meminfo") as f:
+            minfo = {l.split(":")[0]: int(l.split()[1]) for l in f if ":" in l}
+        mem_used_gb = round((minfo.get("MemTotal",0)-minfo.get("MemAvailable",0))/1024/1024,1)
+        mem_total_gb = round(minfo.get("MemTotal",0)/1024/1024,1)
+    except:
+        mem_used_gb = mem_total_gb = 0
+    try:
+        import shutil
+        du = shutil.disk_usage("/")
+        disk_used = round(du.used/1024**3,1)
+        disk_total = round(du.total/1024**3,1)
+        disk_pct = round(du.used/du.total*100)
+    except:
+        disk_used = disk_total = disk_pct = 0
+    return {
+        "books": books,
+        "vectors": vectors,
+        "conversations": convos,
+        "memories": memories,
+        "unique_users": unique_users,
+        "cache_keys": cache_keys,
+        "cache_memory": cache_memory,
+        "llm_usage": [dict(r) for r in llm_usage],
+        "daily_queries": [dict(r) for r in daily],
+        "recent_queries": [dict(r) for r in recent],
+        "system": {
+            "disk_used": str(disk_used) + "G",
+            "disk_total": str(disk_total) + "G",
+            "disk_pct": str(disk_pct) + "%",
+            "ram_used": str(mem_used_gb) + "G",
+            "ram_total": str(mem_total_gb) + "G",
+        }
+    }
+
+@app.get("/admin/books")
+def admin_books(_=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT * FROM books ORDER BY uploaded_at DESC")
+            return {"books": [dict(r) for r in cur.fetchall()]}
+
+@app.delete("/admin/books/{filename}")
+def admin_delete_book(filename: str, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("DELETE FROM books WHERE filename = %s", (filename,))
+        conn.commit()
+    return {"status": "deleted", "filename": filename}
+
+@app.get("/admin/conversations")
+def admin_conversations(limit: int = 50, session: str = None, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            if session:
+                cur.execute("SELECT * FROM conversations WHERE session_id = %s ORDER BY created_at DESC LIMIT %s", (session, limit))
+            else:
+                cur.execute("SELECT * FROM conversations ORDER BY created_at DESC LIMIT %s", (limit,))
+            return {"conversations": [dict(r) for r in cur.fetchall()]}
+
+@app.get("/admin/users")
+def admin_users(_=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT session_id,
+                       COUNT(*) as query_count,
+                       MIN(created_at) as first_seen,
+                       MAX(created_at) as last_seen
+                FROM conversations GROUP BY session_id ORDER BY query_count DESC
+            """)
+            return {"users": [dict(r) for r in cur.fetchall()]}
+
+@app.get("/admin/settings")
+def admin_get_settings(_=Depends(verify_admin)):
+    return {
+        "ollama_model": os.getenv("OLLAMA_MODEL", "qwen2:7b"),
+        "ollama_url": os.getenv("OLLAMA_URL", "http://localhost:11434"),
+        "qdrant_url": os.getenv("QDRANT_URL", "http://localhost:6333"),
+        "claude_api_configured": bool(os.getenv("CLAUDE_API_KEY", "")),
+        "cache_ttl_hours": 24,
+        "complexity_threshold_words": 30,
+        "confidence_threshold": 0.75,
+        "chunk_size": 500,
+        "chunk_overlap": 50,
+    }
+
+# ── Branding ──────────────────────────────────────────────────────────────────
+@app.get("/admin/branding")
+def get_branding(_=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT key, value FROM branding ORDER BY key")
+            rows = cur.fetchall()
+            return {r["key"]: r["value"] for r in rows}
+
+class BrandingUpdate(BaseModel):
+    site_name: str = None
+    site_subtitle: str = None
+    primary_color: str = None
+    secondary_color: str = None
+    logo_url: str = None
+
+@app.post("/admin/branding")
+def update_branding(data: BrandingUpdate, _=Depends(verify_admin)):
+    updates = {k: v for k, v in data.dict().items() if v is not None}
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            for key, value in updates.items():
+                cur.execute(
+                    "INSERT INTO branding (key, value) VALUES (%s, %s) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()",
+                    (key, value)
+                )
+        conn.commit()
+    return {"status": "updated", "fields": list(updates.keys())}
+
+@app.get("/branding")
+def public_branding():
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT key, value FROM branding ORDER BY key")
+            rows = cur.fetchall()
+            return {r["key"]: r["value"] for r in rows}
+
+# ── Auth & Users ──────────────────────────────────────────────────────────────
+from auth import (hash_password, verify_password, create_token,
+                  get_current_user, get_current_user_optional,
+                  require_upload, bearer)
+
+class RegisterRequest(BaseModel):
+    name: str
+    email: str
+    password: str
+    phone: str = None
+    country_code: str = None
+    country_name: str = None
+
+class LoginRequest(BaseModel):
+    email: str
+    password: str
+
+class UpdateRoleRequest(BaseModel):
+    user_id: int
+    role_id: int
+
+@app.post("/auth/register")
+def do_register(data: RegisterRequest):
+    if len(data.password) < 6:
+        raise HTTPException(status_code=400, detail="Password must be at least 6 characters")
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT id FROM users WHERE email=%s", (data.email.lower(),))
+            if cur.fetchone():
+                raise HTTPException(status_code=400, detail="Email already registered")
+            cur.execute(
+                "INSERT INTO users (name, email, phone, country_code, country_name, password_hash, role) VALUES (%s,%s,%s,%s,%s,%s,%s) RETURNING id",
+                (data.name, data.email.lower(), data.phone, data.country_code, data.country_name,
+                 hash_password(data.password), "reader")
+            )
+            user_id = cur.fetchone()["id"]
+        conn.commit()
+    token = create_token(user_id, "reader")
+    return {"status": "ok", "token": token, "message": "Account created successfully"}
+
+@app.post("/auth/login")
+def do_login(data: LoginRequest):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute(
+                "SELECT u.*, r.permissions, r.display_name as role_display FROM users u JOIN roles r ON r.name=u.role WHERE u.email=%s AND u.is_active=TRUE",
+                (data.email.lower(),)
+            )
+            user = cur.fetchone()
+            if not user or not verify_password(data.password, user["password_hash"]):
+                raise HTTPException(status_code=401, detail="Invalid email or password")
+            cur.execute("UPDATE users SET last_login=NOW() WHERE id=%s", (user["id"],))
+        conn.commit()
+    token = create_token(user["id"], user["role"])
+    return {
+        "status": "ok", "token": token,
+        "user": {"id": user["id"], "name": user["name"], "email": user["email"],
+                 "role": user["role"], "role_display": user["role_display"],
+                 "permissions": list(user["permissions"])}
+    }
+
+@app.post("/auth/logout")
+def do_logout(user=Depends(get_current_user)):
+    return {"status": "ok"}
+
+@app.get("/auth/me")
+def get_me(user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute(
+                "SELECT u.*, r.permissions, r.display_name as role_display FROM users u JOIN roles r ON r.name=u.role WHERE u.id=%s",
+                (user["id"],)
+            )
+            full = cur.fetchone()
+    if not full:
+        raise HTTPException(status_code=404, detail="User not found")
+    return {"id": full["id"], "name": full["name"], "email": full["email"],
+            "phone": full["phone"], "country_name": full["country_name"],
+            "role": full["role"], "role_display": full["role_display"],
+            "permissions": list(full["permissions"]), "created_at": str(full["created_at"])}
+
+
+# ── Public Library ────────────────────────────────────────────────────────────
+@app.get("/api/library")
+def public_library():
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT id, filename, total_chunks, uploaded_at, language FROM books ORDER BY uploaded_at DESC")
+            books = cur.fetchall()
+    return {"books": [dict(b) for b in books]}
+
+# ── Admin User Management ─────────────────────────────────────────────────────
+@app.get("/admin/users-list")
+def admin_users_list(_=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT u.id, u.name, u.email, u.phone, u.country_name,
+                       u.is_active, u.created_at, u.last_login,
+                       r.name as role_name, r.display_name as role_display, r.id as role_id
+                FROM users u JOIN roles r ON r.id=u.role_id
+                ORDER BY u.created_at DESC
+            """)
+            return {"users": [dict(r) for r in cur.fetchall()]}
+
+@app.get("/admin/roles")
+def admin_roles(_=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT * FROM roles ORDER BY id")
+            return {"roles": [dict(r) for r in cur.fetchall()]}
+
+@app.post("/admin/users/{user_id}/role")
+def admin_set_role(user_id: int, data: UpdateRoleRequest, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT name FROM roles WHERE id=%s", (data.role_id,))
+            role = cur.fetchone()
+            if not role: raise HTTPException(status_code=404, detail="Role not found")
+            cur.execute("UPDATE users SET role=%s WHERE id=%s", (role[0], user_id))
+        conn.commit()
+    return {"status": "ok"}
+
+@app.post("/admin/users/{user_id}/toggle")
+def admin_toggle_user(user_id: int, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("UPDATE users SET is_active = NOT is_active WHERE id=%s RETURNING is_active", (user_id,))
+            state = cur.fetchone()["is_active"]
+        conn.commit()
+    return {"status": "ok", "is_active": state}
+
+@app.delete("/admin/users/{user_id}")
+def admin_delete_user(user_id: int, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("DELETE FROM users WHERE id=%s", (user_id,))
+        conn.commit()
+    return {"status": "deleted"}
+
+@app.post("/admin/roles/{role_id}/permissions")
+def admin_update_permissions(role_id: int, permissions: dict, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            import json
+            cur.execute("UPDATE roles SET permissions=%s WHERE id=%s", (json.dumps(permissions), role_id))
+        conn.commit()
+    return {"status": "ok"}
+
+# ── Claude API Key Management ─────────────────────────────────────────────────
+class ClaudeKeyUpdate(BaseModel):
+    api_key: str
+
+@app.get("/admin/claude")
+def get_claude_settings(_=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT value FROM api_settings WHERE key='claude_api_key'")
+            row = cur.fetchone()
+            key = row["value"] if row else ""
+            masked = ("*" * (len(key)-4) + key[-4:]) if len(key) > 4 else ("*" * len(key))
+            cur.execute("""
+                SELECT
+                    COUNT(*) as total_calls,
+                    COALESCE(SUM(input_tokens),0) as total_input_tokens,
+                    COALESCE(SUM(output_tokens),0) as total_output_tokens,
+                    COALESCE(SUM(cost_usd),0) as total_cost_usd,
+                    DATE(created_at) as day,
+                    COUNT(*) as calls
+                FROM claude_usage
+                GROUP BY DATE(created_at)
+                ORDER BY day DESC LIMIT 7
+            """)
+            daily = cur.fetchall()
+            cur.execute("SELECT COALESCE(SUM(input_tokens),0) as ti, COALESCE(SUM(output_tokens),0) as to_, COALESCE(SUM(cost_usd),0) as tc, COUNT(*) as calls FROM claude_usage")
+            totals = cur.fetchone()
+    return {
+        "key_configured": bool(key),
+        "key_masked": masked,
+        "totals": dict(totals) if totals else {},
+        "daily": [dict(r) for r in daily]
+    }
+
+@app.post("/admin/claude/key")
+def update_claude_key(data: ClaudeKeyUpdate, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                "INSERT INTO api_settings (key, value, updated_at) VALUES ('claude_api_key', %s, NOW()) ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value, updated_at=NOW()",
+                (data.api_key,)
+            )
+        conn.commit()
+    # Update in-memory env
+    import os
+    os.environ["CLAUDE_API_KEY"] = data.api_key
+    return {"status": "saved"}
+
+@app.get("/admin/claude/usage")
+def get_claude_usage(limit: int = 50, _=Depends(verify_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT session_id, query, input_tokens, output_tokens, cost_usd, created_at
+                FROM claude_usage ORDER BY created_at DESC LIMIT %s
+            """, (limit,))
+            return {"usage": [dict(r) for r in cur.fetchall()]}
+
+# ── PDF Metadata Extraction ───────────────────────────────────────────────────
+@app.post("/api/library/extract-meta")
+async def extract_meta(file: UploadFile = File(...), _=Depends(require_upload)):
+    import fitz, requests as http_req, tempfile, os
+    content_bytes = await file.read()
+    with tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) as tmp:
+        tmp.write(content_bytes)
+        tmp_path = tmp.name
+    try:
+        doc = fitz.open(tmp_path)
+        # Extract text from first 3 pages
+        text = ""
+        for i in range(min(3, len(doc))):
+            text += doc[i].get_text()
+        text = text[:3000]
+        doc.close()
+    finally:
+        os.unlink(tmp_path)
+
+    prompt = f"""أنت مساعد متخصص في استخراج معلومات الكتب. استخرج المعلومات التالية من النص أدناه:
+- عنوان الكتاب
+- اسم المؤلف
+- التصنيف (مثل: فقه، حديث، تفسير، أدب، تاريخ، علوم)
+- وصف مختصر (جملة أو جملتان)
+
+النص:
+{text}
+
+أجب بصيغة JSON فقط بدون أي نص إضافي:
+{{"title": "...", "author": "...", "category": "...", "description": "..."}}"""
+
+    try:
+        ollama_url = os.getenv("OLLAMA_URL", "http://localhost:11434")
+        r = http_req.post(f"{ollama_url}/api/generate",
+            json={"model": "qwen2:7b", "prompt": prompt, "stream": False},
+            timeout=60)
+        response_text = r.json().get("response", "")
+        import json, re
+        json_match = re.search(r'\{[^}]+\}', response_text, re.DOTALL)
+        if json_match:
+            meta = json.loads(json_match.group())
+            return {"status": "ok", "meta": meta}
+    except Exception as e:
+        pass
+
+    # Fallback: extract from filename
+    name = file.filename.replace('.pdf','').strip()
+    return {"status": "ok", "meta": {"title": name, "author": "", "category": "عام", "description": ""}}
+
+# ── Book Reading, Download & Favorites ───────────────────────────────────────
+from fastapi.responses import FileResponse as FR, StreamingResponse
+import os
+
+@app.get("/api/library/{book_id}/read")
+def read_book(book_id: int, user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT * FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
+            book = cur.fetchone()
+    if not book:
+        raise HTTPException(404, "Book not found")
+    file_path = book.get("file_path") or f"/opt/rag/books/{book['filename']}"
+    if not os.path.exists(file_path):
+        raise HTTPException(404, "PDF file not available. Please re-upload the book.")
+    return FR(file_path, media_type="application/pdf",
+              headers={"Content-Disposition": f"inline; filename=\"{book['filename']}\""})
+
+@app.get("/api/library/{book_id}/download")
+def download_book(book_id: int, user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT * FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
+            book = cur.fetchone()
+    if not book:
+        raise HTTPException(404, "Book not found")
+    file_path = book.get("file_path") or f"/opt/rag/books/{book['filename']}"
+    if not os.path.exists(file_path):
+        raise HTTPException(404, "PDF file not available. Please re-upload the book.")
+    name = (book.get("title") or book["filename"]).replace(" ", "_") + ".pdf"
+    return FR(file_path, media_type="application/pdf",
+              headers={"Content-Disposition": f"attachment; filename=\"{name}\""})
+
+@app.post("/api/library/{book_id}/favorite")
+def toggle_favorite(book_id: int, user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT id FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
+            existing = cur.fetchone()
+            if existing:
+                cur.execute("DELETE FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
+                status = "removed"
+            else:
+                cur.execute("INSERT INTO favorites (user_id, book_id) VALUES (%s,%s)", (user["id"], book_id))
+                status = "added"
+        conn.commit()
+    return {"status": status}
+
+@app.get("/api/library/favorites/list")
+def get_favorites(user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT b.*, f.created_at as fav_at
+                FROM favorites f JOIN books b ON b.id=f.book_id
+                WHERE f.user_id=%s ORDER BY f.created_at DESC
+            """, (user["id"],))
+            return {"favorites": [dict(r) for r in cur.fetchall()]}
+
+@app.get("/api/library/{book_id}/is-favorite")
+def is_favorite(book_id: int, user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT id FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
+            return {"is_favorite": bool(cur.fetchone())}
+
+@app.get("/reader")
+def reader_page():
+    return FR("/opt/rag/static/reader.html")
+
+# ── Temp PDF Token ────────────────────────────────────────────────────────────
+import time, hashlib
+
+PDF_TOKENS = {}  # {token: (user_id, book_id, expires)}
+
+@app.get("/api/library/{book_id}/token")
+def get_pdf_token(book_id: int, user=Depends(get_current_user)):
+    token = hashlib.md5(f"{user['id']}-{book_id}-{time.time()}".encode()).hexdigest()
+    PDF_TOKENS[token] = (user['id'], book_id, time.time() + 3600)  # 1 hour
+    return {"token": token}
+
+# ── PDF Serve & Download ──────────────────────────────────────────────────────
+from fastapi.responses import StreamingResponse
+import mimetypes
+
+@app.get("/api/library/{book_id}/pdf")
+def serve_pdf(book_id: int):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
+            book = cur.fetchone()
+    if not book or not book["pdf_path"]:
+        raise HTTPException(status_code=404, detail="PDF not available")
+    import os
+    if not os.path.exists(book["pdf_path"]):
+        raise HTTPException(status_code=404, detail="PDF file not found")
+    from urllib.parse import quote
+    safe_name = quote(book["filename"], safe="")
+    file_size = os.path.getsize(book["pdf_path"])
+    def iter_file():
+        with open(book["pdf_path"], "rb") as f:
+            yield from f
+    return StreamingResponse(iter_file(), media_type="application/pdf", headers={
+        "Content-Disposition": f"inline; filename*=UTF-8''{safe_name}",
+        "Content-Length": str(file_size),
+        "Cache-Control": "private, max-age=3600"
+    })
+
+@app.get("/api/library/{book_id}/download")
+def download_pdf(book_id: int):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
+            book = cur.fetchone()
+    if not book or not book["pdf_path"]:
+        raise HTTPException(status_code=404, detail="PDF not available")
+    import os
+    if not os.path.exists(book["pdf_path"]):
+        raise HTTPException(status_code=404, detail="PDF file not found")
+    from urllib.parse import quote
+    safe_name = quote(book["filename"], safe="")
+    file_size = os.path.getsize(book["pdf_path"])
+    def iter_file():
+        with open(book["pdf_path"], "rb") as f:
+            yield from f
+    return StreamingResponse(iter_file(), media_type="application/pdf", headers={
+        "Content-Disposition": f"attachment; filename*=UTF-8''{safe_name}",
+        "Content-Length": str(file_size),
+        "Cache-Control": "private, max-age=3600"
+    })
+
+# ── Favorites ─────────────────────────────────────────────────────────────────
+@app.get("/api/favorites")
+def get_favorites(user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT b.id, b.filename, b.title, b.author, b.category, b.total_chunks,
+                       b.uploaded_at, b.pdf_path, f.created_at as fav_date
+                FROM favorites f JOIN books b ON b.id=f.book_id
+                WHERE f.user_id=%s ORDER BY f.created_at DESC
+            """, (user["id"],))
+            return {"favorites": [dict(r) for r in cur.fetchall()]}
+
+@app.post("/api/favorites/{book_id}")
+def add_favorite(book_id: int, user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("INSERT INTO favorites (user_id, book_id) VALUES (%s,%s) ON CONFLICT DO NOTHING", (user["id"], book_id))
+        conn.commit()
+    return {"status": "added"}
+
+@app.delete("/api/favorites/{book_id}")
+def remove_favorite(book_id: int, user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("DELETE FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
+        conn.commit()
+    return {"status": "removed"}
+
+@app.get("/api/favorites/check/{book_id}")
+def check_favorite(book_id: int, user=Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT 1 FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
+            return {"is_favorite": bool(cur.fetchone())}
+
+# ── Delete Book (full reversal) ────────────────────────────────────────────────
+@app.delete("/api/library/{book_id}")
+def delete_book(book_id: int, user=Depends(require_upload)):
+    import os
+    from qdrant_client.models import Filter, FieldCondition, MatchValue
+
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s", (book_id,))
+            book = cur.fetchone()
+    if not book:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    filename = book["filename"]
+    removed = {"vectors": 0, "pdf": False, "db": False}
+
+    # 1. Remove vectors from Qdrant
+    try:
+        from qdrant_client.models import Filter, FieldCondition, MatchValue
+        result = qdrant.delete(
+            collection_name=COLLECTION,
+            points_selector=Filter(
+                must=[FieldCondition(key="book", match=MatchValue(value=filename))]
+            )
+        )
+        removed["vectors"] = True
+    except Exception as e:
+        removed["vectors_error"] = str(e)
+
+    # 2. Delete PDF file
+    try:
+        pdf = book.get("pdf_path") or ""
+        if pdf and os.path.exists(pdf):
+            os.remove(pdf)
+            removed["pdf"] = True
+        # Also check books dir
+        books_path = os.path.join("/opt/rag/books", filename)
+        if os.path.exists(books_path):
+            os.remove(books_path)
+    except Exception as e:
+        removed["pdf_error"] = str(e)
+
+    # 3. Delete from Postgres
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("DELETE FROM favorites WHERE book_id=%s", (book_id,))
+            cur.execute("DELETE FROM books WHERE id=%s", (book_id,))
+        conn.commit()
+        removed["db"] = True
+
+    return {"status": "deleted", "filename": filename, "removed": removed}
+
+# ── Delete Book (full reversal) ───────────────────────────────────────────────
+@app.delete("/api/library/{book_id}")
+def delete_book(book_id: int, _=Depends(require_upload)):
+    import os
+    from qdrant_client.models import Filter, FieldCondition, MatchValue
+
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s", (book_id,))
+            book = cur.fetchone()
+
+    if not book:
+        raise HTTPException(status_code=404, detail="Book not found")
+
+    filename = book["filename"]
+    results = {"book": filename, "vectors_deleted": 0, "pdf_deleted": False, "db_deleted": False}
+
+    # 1. Delete vectors from Qdrant
+    try:
+        from qdrant_client.models import Filter, FieldCondition, MatchValue
+        qdrant_url = os.getenv("QDRANT_URL", "http://localhost:6333")
+        from qdrant_client import QdrantClient
+        qclient = QdrantClient(url=qdrant_url)
+        cols = [c.name for c in qclient.get_collections().collections]
+        if "arabic_books" in cols:
+            result = qclient.delete(
+                collection_name="arabic_books",
+                points_selector=Filter(
+                    must=[FieldCondition(key="book", match=MatchValue(value=filename))]
+                )
+            )
+            results["vectors_deleted"] = True
+    except Exception as e:
+        results["qdrant_error"] = str(e)
+
+    # 2. Delete PDF file
+    try:
+        if book["pdf_path"] and os.path.exists(book["pdf_path"]):
+            os.remove(book["pdf_path"])
+            results["pdf_deleted"] = True
+        # Also check books dir
+        books_path = os.path.join("/opt/rag/books", filename)
+        if os.path.exists(books_path):
+            os.remove(books_path)
+    except Exception as e:
+        results["file_error"] = str(e)
+
+    # 3. Delete from Postgres
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("DELETE FROM favorites WHERE book_id=%s", (book_id,))
+            cur.execute("DELETE FROM books WHERE id=%s", (book_id,))
+        conn.commit()
+        results["db_deleted"] = True
+
+    return {"status": "deleted", "details": results}

books/test.pdf

@@ -0,0 +1 @@
+%PDF-1.4 test content

brain.py

@@ -1,3 +1,19 @@
+
+def normalize_arabic(text: str) -> str:
+    import re, unicodedata
+    replacements = {
+        '\u06cc': 'ي', '\u0643': 'ك',
+        '\u06a9': 'ك', '\u0647': 'ه',
+        '\u06be': 'ه', '\u06c1': 'ه',
+    }
+    for src, dst in replacements.items():
+        text = text.replace(src, dst)
+    # Fix broken Allah ligature — "هللا" → "الله"
+    text = text.replace('هللا', 'الله')
+    # Fix broken Alef + Lam prefix — "ال " → "الـ" (keep spacing)
+    text = unicodedata.normalize('NFC', text)
+    return text
+
 """
 brain.py — Hybrid RAG: Dense (semantic) + BM25 (keyword) + RRF fusion + CrossEncoder rerank
 Pipeline:
@@ -15,7 +31,7 @@ from psycopg2.extras import RealDictCursor
 import redis
 import numpy as np
 from qdrant_client import QdrantClient
-from sentence_transformers import SentenceTransformer, CrossEncoder
+from sentence_transformers import CrossEncoder
 from rank_bm25 import BM25Okapi
 from mem0 import Memory
 
@@ -25,22 +41,27 @@ REDIS_URL     = os.getenv("REDIS_URL",    "redis://localhost:6379")
 POSTGRES_URL = os.getenv("POSTGRES_URL", "")
 OLLAMA_URL    = os.getenv("OLLAMA_URL",   "http://localhost:11434")
 COLLECTION    = "arabic_books"
-MODEL_NAME    = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
+MODEL_NAME    = "bge-m3"
+OLLAMA_EMBED_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") + "/api/embed"
 OLLAMA_MODEL  = "qwen2:7b"
 CACHE_TTL     = 60 * 60 * 24   # 24h
 
 DENSE_K       = 15   # retrieve from Qdrant
 BM25_K        = 15   # retrieve from BM25
 RRF_K         = 60   # RRF constant (higher = less aggressive rank differences)
-RERANK_INPUT  = 10   # cross-encoder input size (top N from RRF)
+RERANK_INPUT  = 15   # top N from RRF to keep
-FINAL_K       = 5    # final chunks after MMR
+FINAL_K       = 6    # final chunks to return
 SCORE_THRESH  = 0.20 # discard dense results below this
 MMR_LAMBDA    = 0.6  # relevance vs diversity trade-off
 COMPLEX_WORDS = 30   # query word count → use Claude
 
 # ── Clients ────────────────────────────────────────────────────────────────────
-print("Loading embedding model...")
+print(f"Embedding model: {MODEL_NAME} via Ollama")
-embedder = SentenceTransformer(MODEL_NAME)
+import requests as _breq
+def _embed(texts):
+    if isinstance(texts, str): texts = [texts]
+    r = _breq.post(OLLAMA_EMBED_URL, json={"model": MODEL_NAME, "input": texts}, timeout=60)
+    return r.json()["embeddings"]
 
 print("Loading cross-encoder reranker...")
 reranker = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", max_length=512)
@@ -196,12 +217,8 @@ def set_cache(q, resp):
 
 # ── Memory ─────────────────────────────────────────────────────────────────────
 def get_memory(session_id, query):
-    try:
+    """Memory retrieval disabled — store only to prevent poisoning answers."""
-        results  = mem0.search(query=query, filters={"user_id": session_id}, limit=3)
+    return ""
-        memories = results if isinstance(results, list) else results.get("results", [])
-        return "\n".join(m.get("memory", m.get("text","")) for m in memories if m)
-    except Exception as e:
-        print(f"Mem0 search error: {e}"); return ""
 
 def store_memory(session_id, query, response):
     try:
@@ -231,7 +248,8 @@ def retrieve_dense(query: str, top_k: int = DENSE_K):
     except:
         return [], 0.0
 
-    emb      = embedder.encode([query])[0].tolist()
+    query = normalize_arabic(query)
+    emb      = _embed([query])[0]
     response = qdrant.query_points(
         collection_name=COLLECTION,
         query=emb,
@@ -311,21 +329,20 @@ def rrf_fuse(dense_chunks, bm25_chunks, k: int = RRF_K):
 
 # ── CrossEncoder rerank ────────────────────────────────────────────────────────
 def rerank(query: str, chunks: list, keep: int = RERANK_INPUT):
+    """Skip CrossEncoder rerank — rely on bge-m3 + RRF scores which are more accurate for Arabic."""
     if not chunks:
         return chunks
-    pairs  = [(query, c["text"]) for c in chunks]
+    for c in chunks:
-    scores = reranker.predict(pairs)
+        c["rerank_score"] = c.get("rrf_score", 0.0)
-    for i, c in enumerate(chunks):
+    return chunks[:keep]
-        c["rerank_score"] = float(scores[i])
-    return sorted(chunks, key=lambda x: x["rerank_score"], reverse=True)[:keep]
 
 # ── MMR diversity ──────────────────────────────────────────────────────────────
 def mmr(query: str, chunks: list, keep: int = FINAL_K, lam: float = MMR_LAMBDA):
     if len(chunks) <= keep:
         return chunks
     texts     = [c["text"] for c in chunks]
-    q_emb     = embedder.encode([query])[0]
+    q_emb     = _embed([query])[0]
-    c_embs    = embedder.encode(texts, show_progress_bar=False)
+    c_embs    = _embed(texts)
     norm      = lambda v: v / (np.linalg.norm(v) or 1)
     q_emb     = norm(q_emb)
     c_embs    = [norm(e) for e in c_embs]
@@ -348,15 +365,25 @@ def mmr(query: str, chunks: list, keep: int = FINAL_K, lam: float = MMR_LAMBDA):
 def build_prompt(query: str, chunks: list, memory: str):
     source_blocks = []
     for i, c in enumerate(chunks, 1):
+        import re as _re
         book_short = c["book"].replace(".pdf","")
+        # Remove date prefix and lecture number e.g. "1419-07-06 ... (119)" → clean title
+        book_short = _re.sub(r'^\d{4}-\d{2}-\d{2}\s*', '', book_short)
+        book_short = _re.sub(r'\s*\(\d+\)\s*(\(\d+\))?$', '', book_short).strip()
         source_tag = f"{'🔗 ' if c.get('source')=='both' else ''}"
+        import re as _re2
+        raw_text = clean_chunk_text(normalize_arabic(c['text']))
+        # Strip lecture number patterns like "(119) جواز التداوي" from chunk body
+        raw_text = _re2.sub(r'\(\d+\)\s*[^\n]{0,40}\n?', ' ', raw_text).strip()
+        raw_text = _re2.sub(r' {2,}', ' ', raw_text)
+        chunk_text = raw_text[:400] + ('...' if len(raw_text) > 400 else '')
         source_blocks.append(
-            f"[{i}] «{book_short}» — صفحة {c['page']}:\n{c['text']}"
+            f"[{i}] «{book_short}» — صفحة {c['page']}:\n{chunk_text}"
         )
     context    = "\n\n".join(source_blocks)
     mem_block  = f"\n\nمن محادثات سابقة:\n{memory}" if memory else ""
 
-    return f"""أنت باحث إسلامي متخصص في الفقه والحديث والعلوم الشرعية. أجب على الأسئلة اعتماداً على المصادر المقدمة فقط.
+    return f"""أنت باحث إسلامي متخصص. أجب على السؤال اعتماداً على المصادر المقدمة فقط.
 {mem_block}
 
 المصادر:
@@ -364,17 +391,80 @@ def build_prompt(query: str, chunks: list, memory: str):
 
 السؤال: {query}
 
-تعليمات:
+تعليمات صارمة:
-١. حلّل المصادر وحدد الأجزاء المتعلقة بالسؤال مباشرةً.
+١. أجب فقط بناءً على ما ورد في المصادر أعلاه — لا تضف معلومات من خارجها.
-٢. أجب بشكل دقيق ومفصل، ووثّق كل معلومة بالمصدر [١][٢][٣].
+٢. لا تخترع عناوين كتب أو أرقاماً غير موجودة في المصادر.
-٣. إذا تعارضت المصادر، أشر إلى ذلك وقدّم الأقوى دليلاً.
+٣. وثّق كل معلومة بالمصدر [١][٢][٣].
-٤. إن لم تكفِ المصادر للإجابة الكاملة، صرّح بذلك بوضوح.
+٤. إن لم تكفِ المصادر، قل ذلك صراحةً دون اختراع إجابة.
-٥. الإجابة بالعربية الفصحى.
+٥. أجب بالعربية الفصحى في نص عادي — بدون markdown أو ### أو **.
+٦. لا تذكر أرقاماً أو إصدارات للكتب إلا إذا وردت حرفياً في المصادر.
+٧. أجب بشكل مفصّل وشامل — استخدم جميع المعلومات المتاحة من المصادر ولا تختصر.
 
 الإجابة:"""
 
 # ── LLM ────────────────────────────────────────────────────────────────────────
-def call_ollama(prompt):
+
+def clean_answer(text):
+    import re as _r
+    text = _r.sub(u"[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF]", "", text)
+    text = _r.sub(u"[\U0001D400-\U0001D7FF\U0001F000-\U0001FFFF]", "", text)
+    text = _r.sub("[*]+", "", text)
+    text = _r.sub("[#]+\\s*", "", text)
+    text = _r.sub("\\n{3,}", "\n\n", text)
+    text = _r.sub(" {2,}", " ", text)
+    return text.strip()
+
+def clean_chunk_text(text):
+    import re as _r, unicodedata
+    text = _r.sub(u"[\U0001D400-\U0001D7FF\U0001F000-\U0001FFFF]", "", text)
+    text = _r.sub(u"[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF]", "", text)
+    text = unicodedata.normalize("NFC", text)
+    text = _r.sub(" {2,}", " ", text)
+    return text.strip()
+
+
+def get_llm_config():
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT value FROM api_settings WHERE key='llm_provider'")
+        row = cur.fetchone(); cur.close(); conn.close()
+        if row and row[0]:
+            import json as _j
+            return _j.loads(row[0])
+    except: pass
+    return {
+        "provider":      os.getenv("LLM_PROVIDER", "local"),
+        "cf_account_id": os.getenv("CF_ACCOUNT_ID", ""),
+        "cf_api_token":  os.getenv("CF_API_TOKEN", ""),
+        "cf_model":      os.getenv("CF_MODEL", "@cf/meta/llama-3.1-8b-instruct-fp8-fast"),
+    }
+
+def call_cloudflare(prompt, cfg):
+    account_id = cfg.get("cf_account_id", "")
+    api_token  = cfg.get("cf_api_token", "")
+    model      = cfg.get("cf_model", "@cf/meta/llama-3.1-8b-instruct-fp8-fast")
+    if not account_id or not api_token:
+        return call_ollama_local(prompt)
+    url = f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/{model}"
+    try:
+        resp   = http_requests.post(url,
+                     headers={"Authorization": f"Bearer {api_token}", "Content-Type": "application/json"},
+                     json={"messages": [{"role":"user","content":prompt}], "max_tokens": 1200},
+                     timeout=60)
+        data = resp.json()
+        result = data.get("result", {})
+        # Handle both Ollama-style (result.response) and OpenAI-style (result.choices[0].message.content)
+        if "response" in result:
+            answer = (result.get("response") or "").strip()
+        elif "choices" in result:
+            answer = (result["choices"][0].get("message",{}).get("content") or "").strip()
+        else:
+            answer = ""
+        return answer if answer else call_ollama_local(prompt)
+    except Exception as e:
+        print(f"[LLM] CF error: {e}"); return call_ollama_local(prompt)
+
+def call_ollama_local(prompt):
     resp = http_requests.post(
         f"{OLLAMA_URL}/api/generate",
         json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":False,
@@ -383,6 +473,18 @@ def call_ollama(prompt):
     )
     return resp.json().get("response","").strip()
 
+def get_active_llm_label():
+    cfg = get_llm_config(); provider = cfg.get("provider","local")
+    if provider == "cloudflare":
+        model = cfg.get("cf_model","@cf/meta/llama-3.1-8b-instruct-fp8-fast")
+        return f"cloudflare/{model.split('/')[-1]}"
+    return f"ollama/{OLLAMA_MODEL}"
+
+def call_ollama(prompt):
+    cfg = get_llm_config(); provider = cfg.get("provider","local")
+    if provider == "cloudflare": return call_cloudflare(prompt, cfg)
+    return call_ollama_local(prompt)
+
 def call_claude(prompt, session_id="", query=""):
     key = get_claude_key()
     if not key:
@@ -394,34 +496,109 @@ def call_claude(prompt, session_id="", query=""):
         messages=[{"role":"user","content":prompt}]
     )
     log_claude_usage(session_id, query, msg.usage.input_tokens, msg.usage.output_tokens)
-    return msg.content[0].text.strip()
+    return clean_answer(msg.content[0].text.strip())
 
 # ── Main ────────────────────────────────────────────────────────────────────────
+def validate_query(query: str) -> tuple:
+    """Validate query before processing. Returns (is_valid, reason)."""
+    import re
+    q = query.strip()
+
+    # ── Length ────────────────────────────────────────────────────────────────
+    if len(q) < 3:
+        return False, "السؤال قصير جداً"
+    if len(q) > 1000:
+        return False, "السؤال طويل جداً"
+
+    # ── PII detection ─────────────────────────────────────────────────────────
+    pii_patterns = [
+        (r"\b[\w.+-]+@[\w-]+\.[\w.]+\b",           "البريد الإلكتروني"),
+        (r"\b(\+?\d[\s.-]?){8,15}\b",               "رقم الهاتف"),
+        (r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b", "رقم بطاقة"),
+        (r"\b\d{9,12}\b",                              "رقم هوية"),
+    ]
+    for pattern, label in pii_patterns:
+        if re.search(pattern, q):
+            return False, f"يبدو أن السؤال يحتوي على {label} — يرجى عدم مشاركة معلومات شخصية"
+
+    # ── Prompt injection ──────────────────────────────────────────────────────
+    injection_patterns = [
+        r"ignore (all |previous |above )?instructions",
+        r"you are now",
+        r"system prompt",
+        r"jailbreak",
+        r"<\|.*?\|>",
+        r"\[INST\]",
+    ]
+    for pattern in injection_patterns:
+        if re.search(pattern, q, re.IGNORECASE):
+            return False, "لا يمكن معالجة هذا الطلب"
+
+    # ── Language detection ────────────────────────────────────────────────────
+    try:
+        from langdetect import detect, LangDetectException
+        lang = detect(q)
+        if lang not in ("ar", "en", "fa", "ur"):
+            return False, "يرجى طرح السؤال باللغة العربية أو الإنجليزية"
+    except Exception:
+        pass  # if langdetect fails, allow through
+
+    # ── Meaningful content ────────────────────────────────────────────────────
+    arabic_words = re.findall(r"[؀-ۿ]{3,}", q)
+    latin_words  = re.findall(r"[a-zA-Z]{3,}", q)
+    if not arabic_words and not latin_words:
+        return False, "لم يتم التعرف على محتوى مفيد في السؤال"
+
+    # Gibberish Latin — consonant clusters
+    if not arabic_words and latin_words:
+        consonant_clusters = re.findall(r"[bcdfghjklmnpqrstvwxyz]{5,}", q.lower())
+        if len(consonant_clusters) > 1:
+            return False, "السؤال يبدو غير مفهوم"
+
+    # Gibberish Arabic — very long word with no spaces
+    if arabic_words:
+        longest = max(len(w) for w in arabic_words)
+        if longest > 15 and len(arabic_words) < 2:
+            return False, "السؤال يبدو غير مفهوم"
+
+    return True, ""
+
+
+def _retrieve(query: str):
+    """Shared retrieval pipeline: dense + BM25 → RRF → rerank → MMR → final chunks."""
+    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+    fused                   = rrf_fuse(dense_chunks, bm25_chunks)
+    if not fused:
+        return [], 0.0
+    # Use top chunks from RRF — limit to RERANK_INPUT
+    top_fused = fused[:FINAL_K]
+    reranked  = rerank(query, top_fused, keep=FINAL_K)
+    return reranked, top_score
+
+
 def process_query(query: str, session_id: str = "default"):
+    # 0. Validate
+    is_valid, reason = validate_query(query)
+    if not is_valid:
+        return {"answer": f"عذراً، {reason}. يرجى إعادة صياغة السؤال.",
+                "source":"invalid","chunks":[],"llm":"none","confidence":0.0,"books_used":[]}
+    # 0.5 Sanitize — strip potential injection patterns
+    import re as _san
+    query = _san.sub(r"[;<>{}\[\]|`]", "", query).strip()
     # 1. Cache
     cached = get_cache(query)
     if cached:
         return {"answer":cached,"source":"cache","chunks":[],"llm":"cache","confidence":1.0,"books_used":[]}
 
-    # 2. Dense + BM25 retrieval (parallel concept, sequential execution)
+    # 2-5. Retrieval pipeline (shared)
-    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    final_chunks, top_score = _retrieve(query)
-    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+    if not final_chunks or top_score < 0.25:
-
-    # 3. RRF fusion
-    fused = rrf_fuse(dense_chunks, bm25_chunks)
-    if not fused:
         return {"answer":"لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة.",
                 "source":"none","chunks":[],"llm":"none","confidence":0.0,"books_used":[]}
 
-    # 4. Cross-encoder rerank (top RERANK_INPUT from fused)
-    top_fused = fused[:RERANK_INPUT]
-    reranked  = rerank(query, top_fused, keep=RERANK_INPUT)
-
-    # 5. MMR diversity filter → final FINAL_K chunks
-    final_chunks = mmr(query, reranked, keep=FINAL_K)
-
     # 6. Memory
-    memory = get_memory(session_id, query)
+    memory = ""  # Cross-session memory disabled — prevents contamination
 
     # 7. Prompt + LLM
     prompt     = build_prompt(query, final_chunks, memory)
@@ -432,12 +609,15 @@ def process_query(query: str, session_id: str = "default"):
         llm_used = "claude"
     else:
         answer   = call_ollama(prompt)
-        llm_used = f"ollama/{OLLAMA_MODEL}"
+        llm_used = get_active_llm_label()
 
-    # 8. Store
+    # 8. Store — memory is async (Mem0 calls Ollama internally, ~100s blocking)
     books_used = list(set(c["book"] for c in final_chunks))
-    store_memory(session_id, query, answer)
+    import threading as _t
-    store_conversation(session_id, query, answer, llm_used, books_used)
+    if len(answer) > 20 and books_used:
+        _t.Thread(target=store_memory, args=(session_id, query, answer), daemon=True).start()
+    if len(answer) > 20 and books_used:
+        store_conversation(session_id, query, answer, llm_used, books_used)
     set_cache(query, answer)
 
     return {
@@ -448,13 +628,106 @@ def process_query(query: str, session_id: str = "default"):
         "chunks":       final_chunks,
         "books_used":   books_used,
         "retrieval": {
-            "dense":  len(dense_chunks),
+            "dense":  len(final_chunks),
-            "bm25":   len(bm25_chunks),
+            "bm25":   len(final_chunks),
-            "fused":  len(fused),
+            "fused":  len(final_chunks),
-            "both":   sum(1 for c in fused if c.get("source") == "both"),
+            "both":   sum(1 for c in final_chunks if c.get("source") == "both"),
         }
     }
 
+def stream_ollama(prompt):
+    """Generator: yields token strings from local Ollama."""
+    import json as _json
+    with http_requests.post(
+        f"{OLLAMA_URL}/api/generate",
+        json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":True,
+              "options":{"num_predict":700,"temperature":0.2,"top_p":0.9}},
+        stream=True, timeout=180
+    ) as resp:
+        for line in resp.iter_lines():
+            if not line: continue
+            try:
+                chunk = _json.loads(line)
+                token = chunk.get("response","")
+                if token: yield token
+                if chunk.get("done"): break
+            except: continue
+
+def stream_ollama_or_cf(prompt):
+    """Route streaming to Cloudflare (non-streaming fallback) or local Ollama."""
+    cfg      = get_llm_config()
+    provider = cfg.get("provider","local")
+    if provider == "cloudflare":
+        # Cloudflare doesn't support true streaming via our setup — yield full answer at once
+        answer = call_cloudflare(prompt, cfg)
+        yield answer
+    else:
+        yield from stream_ollama(prompt)
+
+
+def stream_query(query: str, session_id: str = "default"):
+    """
+    Generator version of process_query.
+    Yields SSE-formatted lines:
+      data: {"t":"token","v":"..."}   — one per Ollama token
+      data: {"t":"meta","v":{...}}    — once at the end (llm, confidence, books_used)
+      data: [DONE]
+    """
+    import json as _j, re as _r
+
+    # 1. Cache hit → stream the cached answer char by char in one shot
+    cached = get_cache(query)
+    if cached:
+        yield f'data: {_j.dumps({"t":"token","v":cached})}\n\n'
+        yield f'data: {_j.dumps({"t":"meta","v":{"llm":"cache","confidence":1.0,"books_used":[]}})}\n\n'
+        yield "data: [DONE]\n\n"
+        return
+
+    # 2-5. Retrieval pipeline (shared)
+    final_chunks, top_score = _retrieve(query)
+    if not final_chunks:
+        msg = "لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة."
+        yield f'data: {_j.dumps({"t":"token","v":msg})}\n\n'
+        yield f'data: {_j.dumps({"t":"meta","v":{"llm":"none","confidence":0.0,"books_used":[]}})}\n\n'
+        yield "data: [DONE]\n\n"
+        return
+    memory       = get_memory(session_id, query)
+    prompt       = build_prompt(query, final_chunks, memory)
+    books_used   = list(set(c["book"] for c in final_chunks))
+
+    # 3. Stream tokens
+    use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
+    full_answer = []
+
+    if use_claude:
+        # Claude doesn't easily stream here — fall back to single call
+        answer = call_claude(prompt, session_id=session_id, query=query)
+        yield f'data: {_j.dumps({"t":"token","v":answer})}\n\n'
+        full_answer = [answer]
+        llm_used = "claude"
+    else:
+        for token in stream_ollama_or_cf(prompt):
+            if token:
+                full_answer.append(token)
+                yield f'data: {_j.dumps({"t":"token","v":token})}\n\n'
+        llm_used = get_active_llm_label()
+
+    # Clean the complete answer (not individual tokens — strip() kills inter-word spaces)
+    answer = clean_answer("".join(full_answer))
+
+    # 4. Meta frame
+    yield f'data: {_j.dumps({"t":"meta","v":{"llm":llm_used,"confidence":top_score,"books_used":books_used}})}\n\n'
+    yield "data: [DONE]\n\n"
+
+    # 5. Store async (non-blocking)
+    import threading as _t
+    if len(answer) > 20 and books_used:
+        _t.Thread(target=store_memory,    args=(session_id, query, answer), daemon=True).start()
+    if len(answer) > 20 and books_used:
+        store_conversation(session_id, query, answer, llm_used, books_used)
+    set_cache(query, answer)
+
+
 def rebuild_bm25():
     """Called by worker after new ingest."""
     bm25_index.build(POSTGRES_URL)

brain.py.bak2

@@ -0,0 +1,483 @@
+"""
+brain.py — Hybrid RAG: Dense (semantic) + BM25 (keyword) + RRF fusion + CrossEncoder rerank
+Pipeline:
+  1. Dense retrieval  → top 15 from Qdrant (semantic similarity)
+  2. BM25 retrieval   → top 15 from in-memory index (keyword matching)
+  3. RRF fusion       → combine ranked lists into unified score
+  4. CrossEncoder     → rerank fused top 10 by (query, chunk) relevance
+  5. MMR              → diversity filter → final 5 chunks
+  6. LLM              → build prompt with numbered citations → call Ollama/Claude
+"""
+import os, hashlib, threading
+import requests as http_requests
+import psycopg2
+from psycopg2.extras import RealDictCursor
+import redis
+import numpy as np
+from qdrant_client import QdrantClient
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from rank_bm25 import BM25Okapi
+from mem0 import Memory
+
+# ── Config ─────────────────────────────────────────────────────────────────────
+QDRANT_URL    = os.getenv("QDRANT_URL",   "http://localhost:6333")
+REDIS_URL     = os.getenv("REDIS_URL",    "redis://localhost:6379")
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+OLLAMA_URL    = os.getenv("OLLAMA_URL",   "http://localhost:11434")
+COLLECTION    = "arabic_books"
+MODEL_NAME    = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
+OLLAMA_MODEL  = "qwen2:7b"
+CACHE_TTL     = 60 * 60 * 24   # 24h
+
+DENSE_K       = 15   # retrieve from Qdrant
+BM25_K        = 15   # retrieve from BM25
+RRF_K         = 60   # RRF constant (higher = less aggressive rank differences)
+RERANK_INPUT  = 10   # cross-encoder input size (top N from RRF)
+FINAL_K       = 5    # final chunks after MMR
+SCORE_THRESH  = 0.20 # discard dense results below this
+MMR_LAMBDA    = 0.6  # relevance vs diversity trade-off
+COMPLEX_WORDS = 30   # query word count → use Claude
+
+# ── Clients ────────────────────────────────────────────────────────────────────
+print("Loading embedding model...")
+embedder = SentenceTransformer(MODEL_NAME)
+
+print("Loading cross-encoder reranker...")
+reranker = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", max_length=512)
+
+_qdrant_key = os.getenv("QDRANT_API_KEY", "")
+qdrant       = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
+_redis_password = os.getenv("REDIS_PASSWORD", "")
+_redis_url_auth = REDIS_URL
+if _redis_password and "://:@" not in _redis_url_auth and "@" not in _redis_url_auth.split("://")[1]:
+    _redis_url_auth = REDIS_URL.replace("redis://", f"redis://:{_redis_password}@")
+redis_client = redis.from_url(_redis_url_auth, decode_responses=True)
+
+# ── BM25 Index (in-memory, rebuilt from PostgreSQL on startup) ──────────────────
+class BM25Index:
+    """Thread-safe BM25 index over all chunks. Rebuilt when new content is ingested."""
+
+    def __init__(self):
+        self._lock   = threading.RLock()
+        self._index  = None
+        self._chunks = []   # list of {text, book, page, qdrant_id}
+        self._corpus = []   # tokenized
+
+    def _tokenize(self, text: str):
+        """
+        Arabic-aware tokenization:
+        - Remove diacritics (tashkeel)
+        - Normalize alef variants
+        - Split on whitespace/punctuation
+        """
+        import re, unicodedata
+        # Remove Arabic diacritics (harakat)
+        text = re.sub(r'[\u064B-\u065F\u0670]', '', text)
+        # Normalize alef variants → plain alef
+        text = re.sub(r'[إأآ]', 'ا', text)
+        # Normalize teh marbuta
+        text = re.sub(r'ة', 'ه', text)
+        # Remove punctuation
+        text = re.sub(r'[^\w\s]', ' ', text)
+        tokens = text.lower().split()
+        return [t for t in tokens if len(t) > 1]
+
+    def build(self, db_url: str):
+        """Load all chunks from PostgreSQL and build BM25 index."""
+        try:
+            conn = psycopg2.connect(db_url)
+            cur  = conn.cursor(cursor_factory=RealDictCursor)
+            cur.execute("""
+                SELECT c.qdrant_id, c.book_filename, c.page, c.text, c.word_count
+                FROM chunks c
+                ORDER BY c.book_filename, c.page
+            """)
+            rows = cur.fetchall()
+            cur.close(); conn.close()
+        except Exception as e:
+            print(f"BM25 index build failed: {e}")
+            return 0
+
+        with self._lock:
+            self._chunks = [dict(r) for r in rows]
+            self._corpus = [self._tokenize(c['text']) for c in self._chunks]
+            if self._corpus:
+                self._index = BM25Okapi(self._corpus)
+            else:
+                self._index = None
+
+        print(f"BM25 index built: {len(self._chunks)} chunks")
+        return len(self._chunks)
+
+    def search(self, query: str, top_k: int = BM25_K):
+        """Return list of (chunk_dict, bm25_score) sorted by score desc."""
+        with self._lock:
+            if self._index is None or not self._chunks:
+                return []
+            tokens = self._tokenize(query)
+            if not tokens:
+                return []
+            scores = self._index.get_scores(tokens)
+            # Get top_k indices
+            top_idx = np.argsort(scores)[::-1][:top_k]
+            return [
+                {**self._chunks[i], "bm25_score": float(scores[i])}
+                for i in top_idx
+                if scores[i] > 0
+            ]
+
+    def add_chunks(self, chunks: list, db_url: str):
+        """Rebuild after new ingest."""
+        self.build(db_url)
+
+
+# Singleton index
+bm25_index = BM25Index()
+
+# ── Mem0 ───────────────────────────────────────────────────────────────────────
+print("Initialising Mem0...")
+# Use url= (not host+port) so Mem0's QdrantClient stays on HTTP even with api_key
+_mem0_qdrant_cfg = {
+    "url":                  QDRANT_URL,
+    "collection_name":      "mem0_memory",
+    "embedding_model_dims": 768,   # nomic-embed-text output dimension
+}
+if _qdrant_key:
+    _mem0_qdrant_cfg["api_key"] = _qdrant_key
+
+mem0 = Memory.from_config({
+    "llm":     {"provider":"ollama","config":{"model":OLLAMA_MODEL,"ollama_base_url":OLLAMA_URL}},
+    "embedder":{"provider":"ollama","config":{"model":"nomic-embed-text","ollama_base_url":OLLAMA_URL,"embedding_dims":768}},
+    "vector_store":{"provider":"qdrant","config": _mem0_qdrant_cfg},
+})
+print("Mem0 ready.")
+
+# Build BM25 index on startup
+print("Building BM25 index...")
+n = bm25_index.build(POSTGRES_URL)
+print(f"BM25 ready: {n} chunks indexed")
+if n == 0:
+    print("[WARNING] BM25 index empty — check POSTGRES_URL and chunks table")
+
+# ── DB helpers ─────────────────────────────────────────────────────────────────
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+def get_claude_key():
+    key = os.getenv("CLAUDE_API_KEY", "")
+    if key: return key
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT value FROM api_settings WHERE key='claude_api_key'")
+        row = cur.fetchone(); cur.close(); conn.close()
+        return row[0] if row and row[0] else ""
+    except: return ""
+
+def log_claude_usage(session_id, query, in_tok, out_tok):
+    cost = (in_tok * 0.000003) + (out_tok * 0.000015)
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("INSERT INTO claude_usage (session_id,query,input_tokens,output_tokens,cost_usd) VALUES (%s,%s,%s,%s,%s)",
+                    (session_id, query[:200], in_tok, out_tok, cost))
+        conn.commit(); cur.close(); conn.close()
+    except: pass
+
+# ── Cache ──────────────────────────────────────────────────────────────────────
+def cache_key(q):
+    return "rag:cache:" + hashlib.md5(q.strip().lower().encode()).hexdigest()
+
+def get_cache(q):
+    try: return redis_client.get(cache_key(q))
+    except: return None
+
+def set_cache(q, resp):
+    try: redis_client.setex(cache_key(q), CACHE_TTL, resp)
+    except Exception as e: print(f"Cache write skipped: {e}")
+
+# ── Memory ─────────────────────────────────────────────────────────────────────
+def get_memory(session_id, query):
+    try:
+        results  = mem0.search(query=query, filters={"user_id": session_id}, limit=3)
+        memories = results if isinstance(results, list) else results.get("results", [])
+        return "\n".join(m.get("memory", m.get("text","")) for m in memories if m)
+    except Exception as e:
+        print(f"Mem0 search error: {e}"); return ""
+
+def store_memory(session_id, query, response):
+    try:
+        mem0.add(
+            messages=[{"role":"user","content":query},{"role":"assistant","content":response}],
+            user_id=str(session_id)
+        )
+    except Exception as e:
+        print(f"Mem0 store error: {e}")
+
+def store_conversation(session_id, query, response, llm, books):
+    try:
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("INSERT INTO conversations (session_id,user_query,assistant_response,llm_used,books_used) VALUES (%s,%s,%s,%s,%s)",
+                            (session_id, query, response, llm, books))
+            conn.commit()
+    except Exception as e:
+        print(f"Conversation store error: {e}")
+
+# ── Dense retrieval ────────────────────────────────────────────────────────────
+def retrieve_dense(query: str, top_k: int = DENSE_K):
+    try:
+        cols = [c.name for c in qdrant.get_collections().collections]
+        if COLLECTION not in cols:
+            return [], 0.0
+    except:
+        return [], 0.0
+
+    emb      = embedder.encode([query])[0].tolist()
+    response = qdrant.query_points(
+        collection_name=COLLECTION,
+        query=emb,
+        limit=top_k,
+        with_payload=True,
+        score_threshold=SCORE_THRESH,
+    )
+    pts = response.points
+    if not pts:
+        return [], 0.0
+
+    chunks = [
+        {"text":  p.payload["text"],
+         "book":  p.payload["book"],
+         "page":  p.payload.get("page", 1),
+         "score": p.score,
+         "qdrant_id": str(p.id)}
+        for p in pts
+    ]
+    return chunks, pts[0].score
+
+# ── BM25 retrieval ─────────────────────────────────────────────────────────────
+def retrieve_bm25(query: str, top_k: int = BM25_K):
+    results = bm25_index.search(query, top_k=top_k)
+    return [
+        {"text":  r["text"],
+         "book":  r["book_filename"],
+         "page":  r["page"],
+         "bm25_score": r["bm25_score"],
+         "qdrant_id": r.get("qdrant_id","")}
+        for r in results
+    ]
+
+# ── Reciprocal Rank Fusion ─────────────────────────────────────────────────────
+def rrf_fuse(dense_chunks, bm25_chunks, k: int = RRF_K):
+    """
+    Combine two ranked lists with RRF.
+    score(d) = Σ 1/(k + rank_i(d))
+
+    Returns unified list of chunks with rrf_score, deduped by qdrant_id.
+    """
+    scores  = {}  # qdrant_id → rrf_score
+    payload = {}  # qdrant_id → chunk dict
+
+    for rank, c in enumerate(dense_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        payload[cid] = {**c, "dense_rank": rank + 1}
+
+    for rank, c in enumerate(bm25_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        if cid not in payload:
+            payload[cid] = {**c}
+        else:
+            payload[cid]["bm25_rank"] = rank + 1
+        payload[cid]["rrf_score"] = scores[cid]
+
+    # Tag each chunk with which source retrieved it
+    fused = []
+    for cid, chunk in payload.items():
+        chunk["rrf_score"] = scores[cid]
+        has_dense = "dense_rank" in chunk
+        has_bm25  = "bm25_rank"  in chunk
+        chunk["source"] = (
+            "both"    if has_dense and has_bm25 else
+            "dense"   if has_dense else
+            "bm25"
+        )
+        fused.append(chunk)
+
+    fused.sort(key=lambda x: x["rrf_score"], reverse=True)
+    # Boost chunks found by both sources
+    both_hits = [c for c in fused if c["source"] == "both"]
+    print(f"  RRF: {len(dense_chunks)} dense + {len(bm25_chunks)} BM25 → {len(fused)} fused ({len(both_hits)} in both)")
+    return fused
+
+# ── CrossEncoder rerank ────────────────────────────────────────────────────────
+def rerank(query: str, chunks: list, keep: int = RERANK_INPUT):
+    if not chunks:
+        return chunks
+    pairs  = [(query, c["text"]) for c in chunks]
+    scores = reranker.predict(pairs)
+    for i, c in enumerate(chunks):
+        c["rerank_score"] = float(scores[i])
+    return sorted(chunks, key=lambda x: x["rerank_score"], reverse=True)[:keep]
+
+# ── MMR diversity ──────────────────────────────────────────────────────────────
+def mmr(query: str, chunks: list, keep: int = FINAL_K, lam: float = MMR_LAMBDA):
+    if len(chunks) <= keep:
+        return chunks
+    texts     = [c["text"] for c in chunks]
+    q_emb     = embedder.encode([query])[0]
+    c_embs    = embedder.encode(texts, show_progress_bar=False)
+    norm      = lambda v: v / (np.linalg.norm(v) or 1)
+    q_emb     = norm(q_emb)
+    c_embs    = [norm(e) for e in c_embs]
+    selected  = []
+    remaining = list(range(len(chunks)))
+    for _ in range(keep):
+        if not remaining: break
+        rel = np.array([c_embs[i].dot(q_emb) for i in remaining])
+        if not selected:
+            best = remaining[int(np.argmax(rel))]
+        else:
+            sel_embs = [c_embs[i] for i in selected]
+            red = np.array([max(c_embs[i].dot(s) for s in sel_embs) for i in remaining])
+            best = remaining[int(np.argmax(lam * rel - (1 - lam) * red))]
+        selected.append(best)
+        remaining.remove(best)
+    return [chunks[i] for i in selected]
+
+# ── Prompt ─────────────────────────────────────────────────────────────────────
+def build_prompt(query: str, chunks: list, memory: str):
+    source_blocks = []
+    for i, c in enumerate(chunks, 1):
+        book_short = c["book"].replace(".pdf","")
+        source_tag = f"{'🔗 ' if c.get('source')=='both' else ''}"
+        source_blocks.append(
+            f"[{i}] «{book_short}» — صفحة {c['page']}:\n{c['text']}"
+        )
+    context    = "\n\n".join(source_blocks)
+    mem_block  = f"\n\nمن محادثات سابقة:\n{memory}" if memory else ""
+
+    return f"""أنت باحث إسلامي متخصص في الفقه والحديث والعلوم الشرعية. أجب على الأسئلة اعتماداً على المصادر المقدمة فقط.
+{mem_block}
+
+المصادر:
+{context}
+
+السؤال: {query}
+
+تعليمات:
+١. حلّل المصادر وحدد الأجزاء المتعلقة بالسؤال مباشرةً.
+٢. أجب بشكل دقيق ومفصل، ووثّق كل معلومة بالمصدر [١][٢][٣].
+٣. إذا تعارضت المصادر، أشر إلى ذلك وقدّم الأقوى دليلاً.
+٤. إن لم تكفِ المصادر للإجابة الكاملة، صرّح بذلك بوضوح.
+٥. الإجابة بالعربية الفصحى.
+
+الإجابة:"""
+
+# ── LLM ────────────────────────────────────────────────────────────────────────
+
+def clean_answer(text: str) -> str:
+    """
+    Remove artifacts Ollama/Claude occasionally inject into Arabic responses:
+    - CJK / foreign Unicode characters mid-Arabic-word
+    - Markdown bold/italic markers (**text**, *text*)
+    - Repeated whitespace
+    """
+    import re as _re
+    # Strip CJK and other non-Arabic/Latin foreign scripts injected mid-text
+    text = _re.sub(r'[⺀-鿿㐀-䶿豈-﫿Ͱ-ϿЀ-ӿ]', '', text)
+    # Strip markdown bold/italic markers
+    text = _re.sub(r'\*{1,3}([^*
+]+?)\*{1,3}', r'\1', text)
+    # Clean up double spaces left by removal
+    text = _re.sub(r'  +', ' ', text)
+    text = _re.sub(r'
+{3,}', '
+
+', text)
+    return text.strip()
+
+
+def call_ollama(prompt):
+    resp = http_requests.post(
+        f"{OLLAMA_URL}/api/generate",
+        json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":False,
+              "options":{"num_predict":700,"temperature":0.2,"top_p":0.9}},
+        timeout=180
+    )
+    return clean_answer(resp.json().get("response","").strip())
+
+def call_claude(prompt, session_id="", query=""):
+    key = get_claude_key()
+    if not key:
+        return call_ollama(prompt)
+    import anthropic
+    c = anthropic.Anthropic(api_key=key)
+    msg = c.messages.create(
+        model="claude-sonnet-4-20250514", max_tokens=1024,
+        messages=[{"role":"user","content":prompt}]
+    )
+    log_claude_usage(session_id, query, msg.usage.input_tokens, msg.usage.output_tokens)
+    return clean_answer(msg.content[0].text.strip())
+
+# ── Main ────────────────────────────────────────────────────────────────────────
+def process_query(query: str, session_id: str = "default"):
+    # 1. Cache
+    cached = get_cache(query)
+    if cached:
+        return {"answer":cached,"source":"cache","chunks":[],"llm":"cache","confidence":1.0,"books_used":[]}
+
+    # 2. Dense + BM25 retrieval (parallel concept, sequential execution)
+    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+
+    # 3. RRF fusion
+    fused = rrf_fuse(dense_chunks, bm25_chunks)
+    if not fused:
+        return {"answer":"لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة.",
+                "source":"none","chunks":[],"llm":"none","confidence":0.0,"books_used":[]}
+
+    # 4. Cross-encoder rerank (top RERANK_INPUT from fused)
+    top_fused = fused[:RERANK_INPUT]
+    reranked  = rerank(query, top_fused, keep=RERANK_INPUT)
+
+    # 5. MMR diversity filter → final FINAL_K chunks
+    final_chunks = mmr(query, reranked, keep=FINAL_K)
+
+    # 6. Memory
+    memory = get_memory(session_id, query)
+
+    # 7. Prompt + LLM
+    prompt     = build_prompt(query, final_chunks, memory)
+    use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
+
+    if use_claude:
+        answer   = call_claude(prompt, session_id=session_id, query=query)
+        llm_used = "claude"
+    else:
+        answer   = call_ollama(prompt)
+        llm_used = f"ollama/{OLLAMA_MODEL}"
+
+    # 8. Store
+    books_used = list(set(c["book"] for c in final_chunks))
+    store_memory(session_id, query, answer)
+    store_conversation(session_id, query, answer, llm_used, books_used)
+    set_cache(query, answer)
+
+    return {
+        "answer":       answer,
+        "source":       "hybrid",
+        "llm":          llm_used,
+        "confidence":   top_score,
+        "chunks":       final_chunks,
+        "books_used":   books_used,
+        "retrieval": {
+            "dense":  len(dense_chunks),
+            "bm25":   len(bm25_chunks),
+            "fused":  len(fused),
+            "both":   sum(1 for c in fused if c.get("source") == "both"),
+        }
+    }
+
+def rebuild_bm25():
+    """Called by worker after new ingest."""
+    bm25_index.build(POSTGRES_URL)

brain.py.bak3

@@ -0,0 +1,562 @@
+"""
+brain.py — Hybrid RAG: Dense (semantic) + BM25 (keyword) + RRF fusion + CrossEncoder rerank
+Pipeline:
+  1. Dense retrieval  → top 15 from Qdrant (semantic similarity)
+  2. BM25 retrieval   → top 15 from in-memory index (keyword matching)
+  3. RRF fusion       → combine ranked lists into unified score
+  4. CrossEncoder     → rerank fused top 10 by (query, chunk) relevance
+  5. MMR              → diversity filter → final 5 chunks
+  6. LLM              → build prompt with numbered citations → call Ollama/Claude
+"""
+import os, hashlib, threading
+import requests as http_requests
+import psycopg2
+from psycopg2.extras import RealDictCursor
+import redis
+import numpy as np
+from qdrant_client import QdrantClient
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from rank_bm25 import BM25Okapi
+from mem0 import Memory
+
+# ── Config ─────────────────────────────────────────────────────────────────────
+QDRANT_URL    = os.getenv("QDRANT_URL",   "http://localhost:6333")
+REDIS_URL     = os.getenv("REDIS_URL",    "redis://localhost:6379")
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+OLLAMA_URL    = os.getenv("OLLAMA_URL",   "http://localhost:11434")
+COLLECTION    = "arabic_books"
+MODEL_NAME    = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
+OLLAMA_MODEL  = "qwen2:7b"
+CACHE_TTL     = 60 * 60 * 24   # 24h
+
+DENSE_K       = 15   # retrieve from Qdrant
+BM25_K        = 15   # retrieve from BM25
+RRF_K         = 60   # RRF constant (higher = less aggressive rank differences)
+RERANK_INPUT  = 10   # cross-encoder input size (top N from RRF)
+FINAL_K       = 4    # final chunks after MMR
+SCORE_THRESH  = 0.20 # discard dense results below this
+MMR_LAMBDA    = 0.6  # relevance vs diversity trade-off
+COMPLEX_WORDS = 30   # query word count → use Claude
+
+# ── Clients ────────────────────────────────────────────────────────────────────
+print("Loading embedding model...")
+embedder = SentenceTransformer(MODEL_NAME)
+
+print("Loading cross-encoder reranker...")
+reranker = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", max_length=512)
+
+_qdrant_key = os.getenv("QDRANT_API_KEY", "")
+qdrant       = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
+_redis_password = os.getenv("REDIS_PASSWORD", "")
+_redis_url_auth = REDIS_URL
+if _redis_password and "://:@" not in _redis_url_auth and "@" not in _redis_url_auth.split("://")[1]:
+    _redis_url_auth = REDIS_URL.replace("redis://", f"redis://:{_redis_password}@")
+redis_client = redis.from_url(_redis_url_auth, decode_responses=True)
+
+# ── BM25 Index (in-memory, rebuilt from PostgreSQL on startup) ──────────────────
+class BM25Index:
+    """Thread-safe BM25 index over all chunks. Rebuilt when new content is ingested."""
+
+    def __init__(self):
+        self._lock   = threading.RLock()
+        self._index  = None
+        self._chunks = []   # list of {text, book, page, qdrant_id}
+        self._corpus = []   # tokenized
+
+    def _tokenize(self, text: str):
+        """
+        Arabic-aware tokenization:
+        - Remove diacritics (tashkeel)
+        - Normalize alef variants
+        - Split on whitespace/punctuation
+        """
+        import re, unicodedata
+        # Remove Arabic diacritics (harakat)
+        text = re.sub(r'[\u064B-\u065F\u0670]', '', text)
+        # Normalize alef variants → plain alef
+        text = re.sub(r'[إأآ]', 'ا', text)
+        # Normalize teh marbuta
+        text = re.sub(r'ة', 'ه', text)
+        # Remove punctuation
+        text = re.sub(r'[^\w\s]', ' ', text)
+        tokens = text.lower().split()
+        return [t for t in tokens if len(t) > 1]
+
+    def build(self, db_url: str):
+        """Load all chunks from PostgreSQL and build BM25 index."""
+        try:
+            conn = psycopg2.connect(db_url)
+            cur  = conn.cursor(cursor_factory=RealDictCursor)
+            cur.execute("""
+                SELECT c.qdrant_id, c.book_filename, c.page, c.text, c.word_count
+                FROM chunks c
+                ORDER BY c.book_filename, c.page
+            """)
+            rows = cur.fetchall()
+            cur.close(); conn.close()
+        except Exception as e:
+            print(f"BM25 index build failed: {e}")
+            return 0
+
+        with self._lock:
+            self._chunks = [dict(r) for r in rows]
+            self._corpus = [self._tokenize(c['text']) for c in self._chunks]
+            if self._corpus:
+                self._index = BM25Okapi(self._corpus)
+            else:
+                self._index = None
+
+        print(f"BM25 index built: {len(self._chunks)} chunks")
+        return len(self._chunks)
+
+    def search(self, query: str, top_k: int = BM25_K):
+        """Return list of (chunk_dict, bm25_score) sorted by score desc."""
+        with self._lock:
+            if self._index is None or not self._chunks:
+                return []
+            tokens = self._tokenize(query)
+            if not tokens:
+                return []
+            scores = self._index.get_scores(tokens)
+            # Get top_k indices
+            top_idx = np.argsort(scores)[::-1][:top_k]
+            return [
+                {**self._chunks[i], "bm25_score": float(scores[i])}
+                for i in top_idx
+                if scores[i] > 0
+            ]
+
+    def add_chunks(self, chunks: list, db_url: str):
+        """Rebuild after new ingest."""
+        self.build(db_url)
+
+
+# Singleton index
+bm25_index = BM25Index()
+
+# ── Mem0 ───────────────────────────────────────────────────────────────────────
+print("Initialising Mem0...")
+# Use url= (not host+port) so Mem0's QdrantClient stays on HTTP even with api_key
+_mem0_qdrant_cfg = {
+    "url":                  QDRANT_URL,
+    "collection_name":      "mem0_memory",
+    "embedding_model_dims": 768,   # nomic-embed-text output dimension
+}
+if _qdrant_key:
+    _mem0_qdrant_cfg["api_key"] = _qdrant_key
+
+mem0 = Memory.from_config({
+    "llm":     {"provider":"ollama","config":{"model":OLLAMA_MODEL,"ollama_base_url":OLLAMA_URL}},
+    "embedder":{"provider":"ollama","config":{"model":"nomic-embed-text","ollama_base_url":OLLAMA_URL,"embedding_dims":768}},
+    "vector_store":{"provider":"qdrant","config": _mem0_qdrant_cfg},
+})
+print("Mem0 ready.")
+
+# Build BM25 index on startup
+print("Building BM25 index...")
+n = bm25_index.build(POSTGRES_URL)
+print(f"BM25 ready: {n} chunks indexed")
+if n == 0:
+    print("[WARNING] BM25 index empty — check POSTGRES_URL and chunks table")
+
+# ── DB helpers ─────────────────────────────────────────────────────────────────
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+def get_claude_key():
+    key = os.getenv("CLAUDE_API_KEY", "")
+    if key: return key
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT value FROM api_settings WHERE key='claude_api_key'")
+        row = cur.fetchone(); cur.close(); conn.close()
+        return row[0] if row and row[0] else ""
+    except: return ""
+
+def log_claude_usage(session_id, query, in_tok, out_tok):
+    cost = (in_tok * 0.000003) + (out_tok * 0.000015)
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("INSERT INTO claude_usage (session_id,query,input_tokens,output_tokens,cost_usd) VALUES (%s,%s,%s,%s,%s)",
+                    (session_id, query[:200], in_tok, out_tok, cost))
+        conn.commit(); cur.close(); conn.close()
+    except: pass
+
+# ── Cache ──────────────────────────────────────────────────────────────────────
+def cache_key(q):
+    return "rag:cache:" + hashlib.md5(q.strip().lower().encode()).hexdigest()
+
+def get_cache(q):
+    try: return redis_client.get(cache_key(q))
+    except: return None
+
+def set_cache(q, resp):
+    try: redis_client.setex(cache_key(q), CACHE_TTL, resp)
+    except Exception as e: print(f"Cache write skipped: {e}")
+
+# ── Memory ─────────────────────────────────────────────────────────────────────
+def get_memory(session_id, query):
+    try:
+        results  = mem0.search(query=query, filters={"user_id": session_id}, limit=3)
+        memories = results if isinstance(results, list) else results.get("results", [])
+        return "\n".join(m.get("memory", m.get("text","")) for m in memories if m)
+    except Exception as e:
+        print(f"Mem0 search error: {e}"); return ""
+
+def store_memory(session_id, query, response):
+    try:
+        mem0.add(
+            messages=[{"role":"user","content":query},{"role":"assistant","content":response}],
+            user_id=str(session_id)
+        )
+    except Exception as e:
+        print(f"Mem0 store error: {e}")
+
+def store_conversation(session_id, query, response, llm, books):
+    try:
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("INSERT INTO conversations (session_id,user_query,assistant_response,llm_used,books_used) VALUES (%s,%s,%s,%s,%s)",
+                            (session_id, query, response, llm, books))
+            conn.commit()
+    except Exception as e:
+        print(f"Conversation store error: {e}")
+
+# ── Dense retrieval ────────────────────────────────────────────────────────────
+def retrieve_dense(query: str, top_k: int = DENSE_K):
+    try:
+        cols = [c.name for c in qdrant.get_collections().collections]
+        if COLLECTION not in cols:
+            return [], 0.0
+    except:
+        return [], 0.0
+
+    emb      = embedder.encode([query])[0].tolist()
+    response = qdrant.query_points(
+        collection_name=COLLECTION,
+        query=emb,
+        limit=top_k,
+        with_payload=True,
+        score_threshold=SCORE_THRESH,
+    )
+    pts = response.points
+    if not pts:
+        return [], 0.0
+
+    chunks = [
+        {"text":  p.payload["text"],
+         "book":  p.payload["book"],
+         "page":  p.payload.get("page", 1),
+         "score": p.score,
+         "qdrant_id": str(p.id)}
+        for p in pts
+    ]
+    return chunks, pts[0].score
+
+# ── BM25 retrieval ─────────────────────────────────────────────────────────────
+def retrieve_bm25(query: str, top_k: int = BM25_K):
+    results = bm25_index.search(query, top_k=top_k)
+    return [
+        {"text":  r["text"],
+         "book":  r["book_filename"],
+         "page":  r["page"],
+         "bm25_score": r["bm25_score"],
+         "qdrant_id": r.get("qdrant_id","")}
+        for r in results
+    ]
+
+# ── Reciprocal Rank Fusion ─────────────────────────────────────────────────────
+def rrf_fuse(dense_chunks, bm25_chunks, k: int = RRF_K):
+    """
+    Combine two ranked lists with RRF.
+    score(d) = Σ 1/(k + rank_i(d))
+
+    Returns unified list of chunks with rrf_score, deduped by qdrant_id.
+    """
+    scores  = {}  # qdrant_id → rrf_score
+    payload = {}  # qdrant_id → chunk dict
+
+    for rank, c in enumerate(dense_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        payload[cid] = {**c, "dense_rank": rank + 1}
+
+    for rank, c in enumerate(bm25_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        if cid not in payload:
+            payload[cid] = {**c}
+        else:
+            payload[cid]["bm25_rank"] = rank + 1
+        payload[cid]["rrf_score"] = scores[cid]
+
+    # Tag each chunk with which source retrieved it
+    fused = []
+    for cid, chunk in payload.items():
+        chunk["rrf_score"] = scores[cid]
+        has_dense = "dense_rank" in chunk
+        has_bm25  = "bm25_rank"  in chunk
+        chunk["source"] = (
+            "both"    if has_dense and has_bm25 else
+            "dense"   if has_dense else
+            "bm25"
+        )
+        fused.append(chunk)
+
+    fused.sort(key=lambda x: x["rrf_score"], reverse=True)
+    # Boost chunks found by both sources
+    both_hits = [c for c in fused if c["source"] == "both"]
+    print(f"  RRF: {len(dense_chunks)} dense + {len(bm25_chunks)} BM25 → {len(fused)} fused ({len(both_hits)} in both)")
+    return fused
+
+# ── CrossEncoder rerank ────────────────────────────────────────────────────────
+def rerank(query: str, chunks: list, keep: int = RERANK_INPUT):
+    if not chunks:
+        return chunks
+    pairs  = [(query, c["text"]) for c in chunks]
+    scores = reranker.predict(pairs)
+    for i, c in enumerate(chunks):
+        c["rerank_score"] = float(scores[i])
+    return sorted(chunks, key=lambda x: x["rerank_score"], reverse=True)[:keep]
+
+# ── MMR diversity ──────────────────────────────────────────────────────────────
+def mmr(query: str, chunks: list, keep: int = FINAL_K, lam: float = MMR_LAMBDA):
+    if len(chunks) <= keep:
+        return chunks
+    texts     = [c["text"] for c in chunks]
+    q_emb     = embedder.encode([query])[0]
+    c_embs    = embedder.encode(texts, show_progress_bar=False)
+    norm      = lambda v: v / (np.linalg.norm(v) or 1)
+    q_emb     = norm(q_emb)
+    c_embs    = [norm(e) for e in c_embs]
+    selected  = []
+    remaining = list(range(len(chunks)))
+    for _ in range(keep):
+        if not remaining: break
+        rel = np.array([c_embs[i].dot(q_emb) for i in remaining])
+        if not selected:
+            best = remaining[int(np.argmax(rel))]
+        else:
+            sel_embs = [c_embs[i] for i in selected]
+            red = np.array([max(c_embs[i].dot(s) for s in sel_embs) for i in remaining])
+            best = remaining[int(np.argmax(lam * rel - (1 - lam) * red))]
+        selected.append(best)
+        remaining.remove(best)
+    return [chunks[i] for i in selected]
+
+# ── Prompt ─────────────────────────────────────────────────────────────────────
+def build_prompt(query: str, chunks: list, memory: str):
+    source_blocks = []
+    for i, c in enumerate(chunks, 1):
+        book_short = c["book"].replace(".pdf","")
+        source_tag = f"{'🔗 ' if c.get('source')=='both' else ''}"
+        chunk_text = c['text'][:400] + ('...' if len(c['text']) > 400 else '')
+        source_blocks.append(
+            f"[{i}] «{book_short}» — صفحة {c['page']}:\n{chunk_text}"
+        )
+    context    = "\n\n".join(source_blocks)
+    mem_block  = f"\n\nمن محادثات سابقة:\n{memory}" if memory else ""
+
+    return f"""أنت باحث إسلامي متخصص في الفقه والحديث والعلوم الشرعية. أجب على الأسئلة اعتماداً على المصادر المقدمة فقط.
+{mem_block}
+
+المصادر:
+{context}
+
+السؤال: {query}
+
+تعليمات:
+١. حلّل المصادر وحدد الأجزاء المتعلقة بالسؤال مباشرةً.
+٢. أجب بشكل دقيق ومفصل، ووثّق كل معلومة بالمصدر [١][٢][٣].
+٣. إذا تعارضت المصادر، أشر إلى ذلك وقدّم الأقوى دليلاً.
+٤. إن لم تكفِ المصادر للإجابة الكاملة، صرّح بذلك بوضوح.
+٥. الإجابة بالعربية الفصحى.
+
+الإجابة:"""
+
+# ── LLM ────────────────────────────────────────────────────────────────────────
+
+def clean_answer(text):
+    import re as _r
+    text = _r.sub(u'[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF]', u'', text)
+    text = _r.sub(r'\*+', u'', text)
+    text = _r.sub(r' {2,}', u' ', text)
+    return text.strip()
+
+
+def call_ollama(prompt):
+    resp = http_requests.post(
+        f"{OLLAMA_URL}/api/generate",
+        json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":False,
+              "options":{"num_predict":450,"temperature":0.2,"top_p":0.9}},
+        timeout=180
+    )
+    return clean_answer(resp.json().get("response","").strip())
+
+
+def stream_ollama(prompt):
+    """Generator: yields token strings as Ollama produces them."""
+    import json as _json
+    with http_requests.post(
+        f"{OLLAMA_URL}/api/generate",
+        json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":True,
+              "options":{"num_predict":450,"temperature":0.2,"top_p":0.9}},
+        stream=True, timeout=180
+    ) as resp:
+        for line in resp.iter_lines():
+            if not line:
+                continue
+            try:
+                chunk = _json.loads(line)
+                token = chunk.get("response","")
+                if token:
+                    yield token
+                if chunk.get("done"):
+                    break
+            except Exception:
+                continue
+
+def call_claude(prompt, session_id="", query=""):
+    key = get_claude_key()
+    if not key:
+        return call_ollama(prompt)
+    import anthropic
+    c = anthropic.Anthropic(api_key=key)
+    msg = c.messages.create(
+        model="claude-sonnet-4-20250514", max_tokens=1024,
+        messages=[{"role":"user","content":prompt}]
+    )
+    log_claude_usage(session_id, query, msg.usage.input_tokens, msg.usage.output_tokens)
+    return clean_answer(msg.content[0].text.strip())
+
+# ── Main ────────────────────────────────────────────────────────────────────────
+def process_query(query: str, session_id: str = "default"):
+    # 1. Cache
+    cached = get_cache(query)
+    if cached:
+        return {"answer":cached,"source":"cache","chunks":[],"llm":"cache","confidence":1.0,"books_used":[]}
+
+    # 2. Dense + BM25 retrieval (parallel concept, sequential execution)
+    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+
+    # 3. RRF fusion
+    fused = rrf_fuse(dense_chunks, bm25_chunks)
+    if not fused:
+        return {"answer":"لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة.",
+                "source":"none","chunks":[],"llm":"none","confidence":0.0,"books_used":[]}
+
+    # 4. Cross-encoder rerank (top RERANK_INPUT from fused)
+    top_fused = fused[:RERANK_INPUT]
+    reranked  = rerank(query, top_fused, keep=RERANK_INPUT)
+
+    # 5. MMR diversity filter → final FINAL_K chunks
+    final_chunks = mmr(query, reranked, keep=FINAL_K)
+
+    # 6. Memory
+    memory = get_memory(session_id, query)
+
+    # 7. Prompt + LLM
+    prompt     = build_prompt(query, final_chunks, memory)
+    use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
+
+    if use_claude:
+        answer   = call_claude(prompt, session_id=session_id, query=query)
+        llm_used = "claude"
+    else:
+        answer   = call_ollama(prompt)
+        llm_used = f"ollama/{OLLAMA_MODEL}"
+
+    # 8. Store — memory is async (Mem0 calls Ollama internally, ~100s blocking)
+    books_used = list(set(c["book"] for c in final_chunks))
+    import threading as _t
+    _t.Thread(target=store_memory, args=(session_id, query, answer), daemon=True).start()
+    store_conversation(session_id, query, answer, llm_used, books_used)
+    set_cache(query, answer)
+
+    return {
+        "answer":       answer,
+        "source":       "hybrid",
+        "llm":          llm_used,
+        "confidence":   top_score,
+        "chunks":       final_chunks,
+        "books_used":   books_used,
+        "retrieval": {
+            "dense":  len(dense_chunks),
+            "bm25":   len(bm25_chunks),
+            "fused":  len(fused),
+            "both":   sum(1 for c in fused if c.get("source") == "both"),
+        }
+    }
+
+def stream_query(query: str, session_id: str = "default"):
+    """
+    Generator version of process_query.
+    Yields SSE-formatted lines:
+      data: {"t":"token","v":"..."}   — one per Ollama token
+      data: {"t":"meta","v":{...}}    — once at the end (llm, confidence, books_used)
+      data: [DONE]
+    """
+    import json as _j, re as _r
+
+    # 1. Cache hit → stream the cached answer char by char in one shot
+    cached = get_cache(query)
+    if cached:
+        yield f'data: {_j.dumps({"t":"token","v":cached})}\n\n'
+        yield f'data: {_j.dumps({"t":"meta","v":{"llm":"cache","confidence":1.0,"books_used":[]}})}\n\n'
+        yield "data: [DONE]\n\n"
+        return
+
+    # 2. Retrieval (same as process_query)
+    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+    fused                   = rrf_fuse(dense_chunks, bm25_chunks)
+
+    if not fused:
+        msg = "لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة."
+        yield f'data: {_j.dumps({"t":"token","v":msg})}\n\n'
+        yield f'data: {_j.dumps({"t":"meta","v":{"llm":"none","confidence":0.0,"books_used":[]}})}\n\n'
+        yield "data: [DONE]\n\n"
+        return
+
+    top_fused    = fused[:RERANK_INPUT]
+    reranked     = rerank(query, top_fused, keep=RERANK_INPUT)
+    final_chunks = mmr(query, reranked, keep=FINAL_K)
+    memory       = get_memory(session_id, query)
+    prompt       = build_prompt(query, final_chunks, memory)
+    books_used   = list(set(c["book"] for c in final_chunks))
+
+    # 3. Stream tokens
+    use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
+    full_answer = []
+
+    if use_claude:
+        # Claude doesn't easily stream here — fall back to single call
+        answer = call_claude(prompt, session_id=session_id, query=query)
+        yield f'data: {_j.dumps({"t":"token","v":answer})}\n\n'
+        full_answer = [answer]
+        llm_used = "claude"
+    else:
+        for token in stream_ollama(prompt):
+            if token:
+                full_answer.append(token)
+                yield f'data: {_j.dumps({"t":"token","v":token})}\n\n'
+        llm_used = f"ollama/{OLLAMA_MODEL}"
+
+    # Clean the complete answer (not individual tokens — strip() kills inter-word spaces)
+    answer = clean_answer("".join(full_answer))
+
+    # 4. Meta frame
+    yield f'data: {_j.dumps({"t":"meta","v":{"llm":llm_used,"confidence":top_score,"books_used":books_used}})}\n\n'
+    yield "data: [DONE]\n\n"
+
+    # 5. Store async (non-blocking)
+    import threading as _t
+    _t.Thread(target=store_memory,    args=(session_id, query, answer), daemon=True).start()
+    store_conversation(session_id, query, answer, llm_used, books_used)
+    set_cache(query, answer)
+
+
+def rebuild_bm25():
+    """Called by worker after new ingest."""
+    bm25_index.build(POSTGRES_URL)

brain.py.bak4

@@ -0,0 +1,562 @@
+"""
+brain.py — Hybrid RAG: Dense (semantic) + BM25 (keyword) + RRF fusion + CrossEncoder rerank
+Pipeline:
+  1. Dense retrieval  → top 15 from Qdrant (semantic similarity)
+  2. BM25 retrieval   → top 15 from in-memory index (keyword matching)
+  3. RRF fusion       → combine ranked lists into unified score
+  4. CrossEncoder     → rerank fused top 10 by (query, chunk) relevance
+  5. MMR              → diversity filter → final 5 chunks
+  6. LLM              → build prompt with numbered citations → call Ollama/Claude
+"""
+import os, hashlib, threading
+import requests as http_requests
+import psycopg2
+from psycopg2.extras import RealDictCursor
+import redis
+import numpy as np
+from qdrant_client import QdrantClient
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from rank_bm25 import BM25Okapi
+from mem0 import Memory
+
+# ── Config ─────────────────────────────────────────────────────────────────────
+QDRANT_URL    = os.getenv("QDRANT_URL",   "http://localhost:6333")
+REDIS_URL     = os.getenv("REDIS_URL",    "redis://localhost:6379")
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+OLLAMA_URL    = os.getenv("OLLAMA_URL",   "http://localhost:11434")
+COLLECTION    = "arabic_books"
+MODEL_NAME    = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
+OLLAMA_MODEL  = "qwen2:7b"
+CACHE_TTL     = 60 * 60 * 24   # 24h
+
+DENSE_K       = 15   # retrieve from Qdrant
+BM25_K        = 15   # retrieve from BM25
+RRF_K         = 60   # RRF constant (higher = less aggressive rank differences)
+RERANK_INPUT  = 10   # cross-encoder input size (top N from RRF)
+FINAL_K       = 4    # final chunks after MMR
+SCORE_THRESH  = 0.20 # discard dense results below this
+MMR_LAMBDA    = 0.6  # relevance vs diversity trade-off
+COMPLEX_WORDS = 30   # query word count → use Claude
+
+# ── Clients ────────────────────────────────────────────────────────────────────
+print("Loading embedding model...")
+embedder = SentenceTransformer(MODEL_NAME)
+
+print("Loading cross-encoder reranker...")
+reranker = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", max_length=512)
+
+_qdrant_key = os.getenv("QDRANT_API_KEY", "")
+qdrant       = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
+_redis_password = os.getenv("REDIS_PASSWORD", "")
+_redis_url_auth = REDIS_URL
+if _redis_password and "://:@" not in _redis_url_auth and "@" not in _redis_url_auth.split("://")[1]:
+    _redis_url_auth = REDIS_URL.replace("redis://", f"redis://:{_redis_password}@")
+redis_client = redis.from_url(_redis_url_auth, decode_responses=True)
+
+# ── BM25 Index (in-memory, rebuilt from PostgreSQL on startup) ──────────────────
+class BM25Index:
+    """Thread-safe BM25 index over all chunks. Rebuilt when new content is ingested."""
+
+    def __init__(self):
+        self._lock   = threading.RLock()
+        self._index  = None
+        self._chunks = []   # list of {text, book, page, qdrant_id}
+        self._corpus = []   # tokenized
+
+    def _tokenize(self, text: str):
+        """
+        Arabic-aware tokenization:
+        - Remove diacritics (tashkeel)
+        - Normalize alef variants
+        - Split on whitespace/punctuation
+        """
+        import re, unicodedata
+        # Remove Arabic diacritics (harakat)
+        text = re.sub(r'[\u064B-\u065F\u0670]', '', text)
+        # Normalize alef variants → plain alef
+        text = re.sub(r'[إأآ]', 'ا', text)
+        # Normalize teh marbuta
+        text = re.sub(r'ة', 'ه', text)
+        # Remove punctuation
+        text = re.sub(r'[^\w\s]', ' ', text)
+        tokens = text.lower().split()
+        return [t for t in tokens if len(t) > 1]
+
+    def build(self, db_url: str):
+        """Load all chunks from PostgreSQL and build BM25 index."""
+        try:
+            conn = psycopg2.connect(db_url)
+            cur  = conn.cursor(cursor_factory=RealDictCursor)
+            cur.execute("""
+                SELECT c.qdrant_id, c.book_filename, c.page, c.text, c.word_count
+                FROM chunks c
+                ORDER BY c.book_filename, c.page
+            """)
+            rows = cur.fetchall()
+            cur.close(); conn.close()
+        except Exception as e:
+            print(f"BM25 index build failed: {e}")
+            return 0
+
+        with self._lock:
+            self._chunks = [dict(r) for r in rows]
+            self._corpus = [self._tokenize(c['text']) for c in self._chunks]
+            if self._corpus:
+                self._index = BM25Okapi(self._corpus)
+            else:
+                self._index = None
+
+        print(f"BM25 index built: {len(self._chunks)} chunks")
+        return len(self._chunks)
+
+    def search(self, query: str, top_k: int = BM25_K):
+        """Return list of (chunk_dict, bm25_score) sorted by score desc."""
+        with self._lock:
+            if self._index is None or not self._chunks:
+                return []
+            tokens = self._tokenize(query)
+            if not tokens:
+                return []
+            scores = self._index.get_scores(tokens)
+            # Get top_k indices
+            top_idx = np.argsort(scores)[::-1][:top_k]
+            return [
+                {**self._chunks[i], "bm25_score": float(scores[i])}
+                for i in top_idx
+                if scores[i] > 0
+            ]
+
+    def add_chunks(self, chunks: list, db_url: str):
+        """Rebuild after new ingest."""
+        self.build(db_url)
+
+
+# Singleton index
+bm25_index = BM25Index()
+
+# ── Mem0 ───────────────────────────────────────────────────────────────────────
+print("Initialising Mem0...")
+# Use url= (not host+port) so Mem0's QdrantClient stays on HTTP even with api_key
+_mem0_qdrant_cfg = {
+    "url":                  QDRANT_URL,
+    "collection_name":      "mem0_memory",
+    "embedding_model_dims": 768,   # nomic-embed-text output dimension
+}
+if _qdrant_key:
+    _mem0_qdrant_cfg["api_key"] = _qdrant_key
+
+mem0 = Memory.from_config({
+    "llm":     {"provider":"ollama","config":{"model":OLLAMA_MODEL,"ollama_base_url":OLLAMA_URL}},
+    "embedder":{"provider":"ollama","config":{"model":"nomic-embed-text","ollama_base_url":OLLAMA_URL,"embedding_dims":768}},
+    "vector_store":{"provider":"qdrant","config": _mem0_qdrant_cfg},
+})
+print("Mem0 ready.")
+
+# Build BM25 index on startup
+print("Building BM25 index...")
+n = bm25_index.build(POSTGRES_URL)
+print(f"BM25 ready: {n} chunks indexed")
+if n == 0:
+    print("[WARNING] BM25 index empty — check POSTGRES_URL and chunks table")
+
+# ── DB helpers ─────────────────────────────────────────────────────────────────
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+def get_claude_key():
+    key = os.getenv("CLAUDE_API_KEY", "")
+    if key: return key
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT value FROM api_settings WHERE key='claude_api_key'")
+        row = cur.fetchone(); cur.close(); conn.close()
+        return row[0] if row and row[0] else ""
+    except: return ""
+
+def log_claude_usage(session_id, query, in_tok, out_tok):
+    cost = (in_tok * 0.000003) + (out_tok * 0.000015)
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("INSERT INTO claude_usage (session_id,query,input_tokens,output_tokens,cost_usd) VALUES (%s,%s,%s,%s,%s)",
+                    (session_id, query[:200], in_tok, out_tok, cost))
+        conn.commit(); cur.close(); conn.close()
+    except: pass
+
+# ── Cache ──────────────────────────────────────────────────────────────────────
+def cache_key(q):
+    return "rag:cache:" + hashlib.md5(q.strip().lower().encode()).hexdigest()
+
+def get_cache(q):
+    try: return redis_client.get(cache_key(q))
+    except: return None
+
+def set_cache(q, resp):
+    try: redis_client.setex(cache_key(q), CACHE_TTL, resp)
+    except Exception as e: print(f"Cache write skipped: {e}")
+
+# ── Memory ─────────────────────────────────────────────────────────────────────
+def get_memory(session_id, query):
+    try:
+        results  = mem0.search(query=query, filters={"user_id": session_id}, limit=3)
+        memories = results if isinstance(results, list) else results.get("results", [])
+        return "\n".join(m.get("memory", m.get("text","")) for m in memories if m)
+    except Exception as e:
+        print(f"Mem0 search error: {e}"); return ""
+
+def store_memory(session_id, query, response):
+    try:
+        mem0.add(
+            messages=[{"role":"user","content":query},{"role":"assistant","content":response}],
+            user_id=str(session_id)
+        )
+    except Exception as e:
+        print(f"Mem0 store error: {e}")
+
+def store_conversation(session_id, query, response, llm, books):
+    try:
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("INSERT INTO conversations (session_id,user_query,assistant_response,llm_used,books_used) VALUES (%s,%s,%s,%s,%s)",
+                            (session_id, query, response, llm, books))
+            conn.commit()
+    except Exception as e:
+        print(f"Conversation store error: {e}")
+
+# ── Dense retrieval ────────────────────────────────────────────────────────────
+def retrieve_dense(query: str, top_k: int = DENSE_K):
+    try:
+        cols = [c.name for c in qdrant.get_collections().collections]
+        if COLLECTION not in cols:
+            return [], 0.0
+    except:
+        return [], 0.0
+
+    emb      = embedder.encode([query])[0].tolist()
+    response = qdrant.query_points(
+        collection_name=COLLECTION,
+        query=emb,
+        limit=top_k,
+        with_payload=True,
+        score_threshold=SCORE_THRESH,
+    )
+    pts = response.points
+    if not pts:
+        return [], 0.0
+
+    chunks = [
+        {"text":  p.payload["text"],
+         "book":  p.payload["book"],
+         "page":  p.payload.get("page", 1),
+         "score": p.score,
+         "qdrant_id": str(p.id)}
+        for p in pts
+    ]
+    return chunks, pts[0].score
+
+# ── BM25 retrieval ─────────────────────────────────────────────────────────────
+def retrieve_bm25(query: str, top_k: int = BM25_K):
+    results = bm25_index.search(query, top_k=top_k)
+    return [
+        {"text":  r["text"],
+         "book":  r["book_filename"],
+         "page":  r["page"],
+         "bm25_score": r["bm25_score"],
+         "qdrant_id": r.get("qdrant_id","")}
+        for r in results
+    ]
+
+# ── Reciprocal Rank Fusion ─────────────────────────────────────────────────────
+def rrf_fuse(dense_chunks, bm25_chunks, k: int = RRF_K):
+    """
+    Combine two ranked lists with RRF.
+    score(d) = Σ 1/(k + rank_i(d))
+
+    Returns unified list of chunks with rrf_score, deduped by qdrant_id.
+    """
+    scores  = {}  # qdrant_id → rrf_score
+    payload = {}  # qdrant_id → chunk dict
+
+    for rank, c in enumerate(dense_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        payload[cid] = {**c, "dense_rank": rank + 1}
+
+    for rank, c in enumerate(bm25_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        if cid not in payload:
+            payload[cid] = {**c}
+        else:
+            payload[cid]["bm25_rank"] = rank + 1
+        payload[cid]["rrf_score"] = scores[cid]
+
+    # Tag each chunk with which source retrieved it
+    fused = []
+    for cid, chunk in payload.items():
+        chunk["rrf_score"] = scores[cid]
+        has_dense = "dense_rank" in chunk
+        has_bm25  = "bm25_rank"  in chunk
+        chunk["source"] = (
+            "both"    if has_dense and has_bm25 else
+            "dense"   if has_dense else
+            "bm25"
+        )
+        fused.append(chunk)
+
+    fused.sort(key=lambda x: x["rrf_score"], reverse=True)
+    # Boost chunks found by both sources
+    both_hits = [c for c in fused if c["source"] == "both"]
+    print(f"  RRF: {len(dense_chunks)} dense + {len(bm25_chunks)} BM25 → {len(fused)} fused ({len(both_hits)} in both)")
+    return fused
+
+# ── CrossEncoder rerank ────────────────────────────────────────────────────────
+def rerank(query: str, chunks: list, keep: int = RERANK_INPUT):
+    if not chunks:
+        return chunks
+    pairs  = [(query, c["text"]) for c in chunks]
+    scores = reranker.predict(pairs)
+    for i, c in enumerate(chunks):
+        c["rerank_score"] = float(scores[i])
+    return sorted(chunks, key=lambda x: x["rerank_score"], reverse=True)[:keep]
+
+# ── MMR diversity ──────────────────────────────────────────────────────────────
+def mmr(query: str, chunks: list, keep: int = FINAL_K, lam: float = MMR_LAMBDA):
+    if len(chunks) <= keep:
+        return chunks
+    texts     = [c["text"] for c in chunks]
+    q_emb     = embedder.encode([query])[0]
+    c_embs    = embedder.encode(texts, show_progress_bar=False)
+    norm      = lambda v: v / (np.linalg.norm(v) or 1)
+    q_emb     = norm(q_emb)
+    c_embs    = [norm(e) for e in c_embs]
+    selected  = []
+    remaining = list(range(len(chunks)))
+    for _ in range(keep):
+        if not remaining: break
+        rel = np.array([c_embs[i].dot(q_emb) for i in remaining])
+        if not selected:
+            best = remaining[int(np.argmax(rel))]
+        else:
+            sel_embs = [c_embs[i] for i in selected]
+            red = np.array([max(c_embs[i].dot(s) for s in sel_embs) for i in remaining])
+            best = remaining[int(np.argmax(lam * rel - (1 - lam) * red))]
+        selected.append(best)
+        remaining.remove(best)
+    return [chunks[i] for i in selected]
+
+# ── Prompt ─────────────────────────────────────────────────────────────────────
+def build_prompt(query: str, chunks: list, memory: str):
+    source_blocks = []
+    for i, c in enumerate(chunks, 1):
+        book_short = c["book"].replace(".pdf","")
+        source_tag = f"{'🔗 ' if c.get('source')=='both' else ''}"
+        chunk_text = c['text'][:400] + ('...' if len(c['text']) > 400 else '')
+        source_blocks.append(
+            f"[{i}] «{book_short}» — صفحة {c['page']}:\n{chunk_text}"
+        )
+    context    = "\n\n".join(source_blocks)
+    mem_block  = f"\n\nمن محادثات سابقة:\n{memory}" if memory else ""
+
+    return f"""أنت باحث إسلامي متخصص في الفقه والحديث والعلوم الشرعية. أجب على الأسئلة اعتماداً على المصادر المقدمة فقط.
+{mem_block}
+
+المصادر:
+{context}
+
+السؤال: {query}
+
+تعليمات:
+١. حلّل المصادر وحدد الأجزاء المتعلقة بالسؤال مباشرةً.
+٢. أجب بشكل دقيق ومفصل، ووثّق كل معلومة بالمصدر [١][٢][٣].
+٣. إذا تعارضت المصادر، أشر إلى ذلك وقدّم الأقوى دليلاً.
+٤. إن لم تكفِ المصادر للإجابة الكاملة، صرّح بذلك بوضوح.
+٥. الإجابة بالعربية الفصحى.
+
+الإجابة:"""
+
+# ── LLM ────────────────────────────────────────────────────────────────────────
+
+def clean_answer(text):
+    import re as _r
+    text = _r.sub(u'[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF]', u'', text)
+    text = _r.sub(r'\*+', u'', text)
+    text = _r.sub(r' {2,}', u' ', text)
+    return text.strip()
+
+
+def call_ollama(prompt):
+    resp = http_requests.post(
+        f"{OLLAMA_URL}/api/generate",
+        json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":False,
+              "options":{"num_predict":450,"temperature":0.2,"top_p":0.9}},
+        timeout=180
+    )
+    return clean_answer(resp.json().get("response","").strip())
+
+
+def stream_ollama(prompt):
+    """Generator: yields token strings as Ollama produces them."""
+    import json as _json
+    with http_requests.post(
+        f"{OLLAMA_URL}/api/generate",
+        json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":True,
+              "options":{"num_predict":450,"temperature":0.2,"top_p":0.9}},
+        stream=True, timeout=180
+    ) as resp:
+        for line in resp.iter_lines():
+            if not line:
+                continue
+            try:
+                chunk = _json.loads(line)
+                token = chunk.get("response","")
+                if token:
+                    yield token
+                if chunk.get("done"):
+                    break
+            except Exception:
+                continue
+
+def call_claude(prompt, session_id="", query=""):
+    key = get_claude_key()
+    if not key:
+        return call_ollama(prompt)
+    import anthropic
+    c = anthropic.Anthropic(api_key=key)
+    msg = c.messages.create(
+        model="claude-sonnet-4-20250514", max_tokens=1024,
+        messages=[{"role":"user","content":prompt}]
+    )
+    log_claude_usage(session_id, query, msg.usage.input_tokens, msg.usage.output_tokens)
+    return clean_answer(msg.content[0].text.strip())
+
+# ── Main ────────────────────────────────────────────────────────────────────────
+def process_query(query: str, session_id: str = "default"):
+    # 1. Cache
+    cached = get_cache(query)
+    if cached:
+        return {"answer":cached,"source":"cache","chunks":[],"llm":"cache","confidence":1.0,"books_used":[]}
+
+    # 2. Dense + BM25 retrieval (parallel concept, sequential execution)
+    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+
+    # 3. RRF fusion
+    fused = rrf_fuse(dense_chunks, bm25_chunks)
+    if not fused:
+        return {"answer":"لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة.",
+                "source":"none","chunks":[],"llm":"none","confidence":0.0,"books_used":[]}
+
+    # 4. Cross-encoder rerank (top RERANK_INPUT from fused)
+    top_fused = fused[:RERANK_INPUT]
+    reranked  = rerank(query, top_fused, keep=RERANK_INPUT)
+
+    # 5. MMR diversity filter → final FINAL_K chunks
+    final_chunks = mmr(query, reranked, keep=FINAL_K)
+
+    # 6. Memory
+    memory = get_memory(session_id, query)
+
+    # 7. Prompt + LLM
+    prompt     = build_prompt(query, final_chunks, memory)
+    use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
+
+    if use_claude:
+        answer   = call_claude(prompt, session_id=session_id, query=query)
+        llm_used = "claude"
+    else:
+        answer   = call_ollama(prompt)
+        llm_used = f"ollama/{OLLAMA_MODEL}"
+
+    # 8. Store — memory is async (Mem0 calls Ollama internally, ~100s blocking)
+    books_used = list(set(c["book"] for c in final_chunks))
+    import threading as _t
+    _t.Thread(target=store_memory, args=(session_id, query, answer), daemon=True).start()
+    store_conversation(session_id, query, answer, llm_used, books_used)
+    set_cache(query, answer)
+
+    return {
+        "answer":       answer,
+        "source":       "hybrid",
+        "llm":          llm_used,
+        "confidence":   top_score,
+        "chunks":       final_chunks,
+        "books_used":   books_used,
+        "retrieval": {
+            "dense":  len(dense_chunks),
+            "bm25":   len(bm25_chunks),
+            "fused":  len(fused),
+            "both":   sum(1 for c in fused if c.get("source") == "both"),
+        }
+    }
+
+def stream_query(query: str, session_id: str = "default"):
+    """
+    Generator version of process_query.
+    Yields SSE-formatted lines:
+      data: {"t":"token","v":"..."}   — one per Ollama token
+      data: {"t":"meta","v":{...}}    — once at the end (llm, confidence, books_used)
+      data: [DONE]
+    """
+    import json as _j, re as _r
+
+    # 1. Cache hit → stream the cached answer char by char in one shot
+    cached = get_cache(query)
+    if cached:
+        yield f'data: {_j.dumps({"t":"token","v":cached})}\n\n'
+        yield f'data: {_j.dumps({"t":"meta","v":{"llm":"cache","confidence":1.0,"books_used":[]}})}\n\n'
+        yield "data: [DONE]\n\n"
+        return
+
+    # 2. Retrieval (same as process_query)
+    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+    fused                   = rrf_fuse(dense_chunks, bm25_chunks)
+
+    if not fused:
+        msg = "لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة."
+        yield f'data: {_j.dumps({"t":"token","v":msg})}\n\n'
+        yield f'data: {_j.dumps({"t":"meta","v":{"llm":"none","confidence":0.0,"books_used":[]}})}\n\n'
+        yield "data: [DONE]\n\n"
+        return
+
+    top_fused    = fused[:RERANK_INPUT]
+    reranked     = rerank(query, top_fused, keep=RERANK_INPUT)
+    final_chunks = mmr(query, reranked, keep=FINAL_K)
+    memory       = get_memory(session_id, query)
+    prompt       = build_prompt(query, final_chunks, memory)
+    books_used   = list(set(c["book"] for c in final_chunks))
+
+    # 3. Stream tokens
+    use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
+    full_answer = []
+
+    if use_claude:
+        # Claude doesn't easily stream here — fall back to single call
+        answer = call_claude(prompt, session_id=session_id, query=query)
+        yield f'data: {_j.dumps({"t":"token","v":answer})}\n\n'
+        full_answer = [answer]
+        llm_used = "claude"
+    else:
+        for token in stream_ollama(prompt):
+            if token:
+                full_answer.append(token)
+                yield f'data: {_j.dumps({"t":"token","v":token})}\n\n'
+        llm_used = f"ollama/{OLLAMA_MODEL}"
+
+    # Clean the complete answer (not individual tokens — strip() kills inter-word spaces)
+    answer = clean_answer("".join(full_answer))
+
+    # 4. Meta frame
+    yield f'data: {_j.dumps({"t":"meta","v":{"llm":llm_used,"confidence":top_score,"books_used":books_used}})}\n\n'
+    yield "data: [DONE]\n\n"
+
+    # 5. Store async (non-blocking)
+    import threading as _t
+    _t.Thread(target=store_memory,    args=(session_id, query, answer), daemon=True).start()
+    store_conversation(session_id, query, answer, llm_used, books_used)
+    set_cache(query, answer)
+
+
+def rebuild_bm25():
+    """Called by worker after new ingest."""
+    bm25_index.build(POSTGRES_URL)

cover.py

@@ -0,0 +1,82 @@
+"""
+cover.py — Cover generation and R2 upload.
+Called from tasks.py (Step 2.5) for every new PDF.
+Also used by the repair script.
+"""
+import os, logging
+import psycopg2
+from psycopg2.extras import RealDictCursor
+
+log = logging.getLogger("rag.cover")
+COVER_DIR = "/mnt/rag-media/covers"
+
+
+def generate_and_upload_cover(book_id: int, pdf_path: str) -> str:
+    """
+    Generate JPEG cover from PDF page 1, upload to R2, update books.cover_url.
+    Returns the public cover URL (or empty string if R2 not configured).
+    Raises on failure so the caller can log/skip gracefully.
+    """
+    os.makedirs(COVER_DIR, exist_ok=True)
+    cover_path = os.path.join(COVER_DIR, f"{book_id}.jpg")
+
+    # ── Generate from PDF first page ─────────────────────────────────────
+    try:
+        import fitz
+        doc  = fitz.open(pdf_path)
+        page = doc[0]
+        mat  = fitz.Matrix(1.5, 1.5)
+        pix  = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
+        pix.save(cover_path)
+        doc.close()
+        log.info(f"[cover] generated {cover_path} ({os.path.getsize(cover_path)//1024}KB)")
+    except Exception as e:
+        raise RuntimeError(f"fitz cover generation failed for book {book_id}: {e}")
+
+    # ── Upload to R2 if configured ────────────────────────────────────────
+    r2_ep  = os.getenv("R2_ENDPOINT",    "").strip()
+    r2_key = os.getenv("R2_ACCESS_KEY",  "").strip()
+    r2_sec = os.getenv("R2_SECRET_KEY",  "").strip()
+    r2_bkt = os.getenv("R2_BUCKET",      "").strip()
+    r2_pub = os.getenv("R2_PUBLIC_URL",  "").rstrip("/")
+
+    cover_url = ""
+    if all([r2_ep, r2_key, r2_sec, r2_bkt]):
+        try:
+            import boto3
+            s3 = boto3.client(
+                "s3",
+                endpoint_url=r2_ep,
+                aws_access_key_id=r2_key,
+                aws_secret_access_key=r2_sec,
+            )
+            with open(cover_path, "rb") as fh:
+                s3.upload_fileobj(
+                    fh, r2_bkt, f"covers/{book_id}.jpg",
+                    ExtraArgs={
+                        "ContentType": "image/jpeg",
+                        "CacheControl": "public, max-age=31536000",
+                    },
+                )
+            cover_url = f"{r2_pub}/covers/{book_id}.jpg" if r2_pub else ""
+            log.info(f"[cover] uploaded to R2: {cover_url}")
+        except Exception as e:
+            log.warning(f"[cover] R2 upload failed for book {book_id}: {e}")
+            # Don't raise — local cover is still usable
+
+    # ── Update DB ─────────────────────────────────────────────────────────
+    try:
+        pg   = os.getenv("POSTGRES_URL", "")
+        conn = psycopg2.connect(pg)
+        cur  = conn.cursor()
+        cur.execute(
+            "UPDATE books SET cover_url=%s WHERE id=%s",
+            (cover_url or None, book_id),
+        )
+        conn.commit()
+        cur.close()
+        conn.close()
+    except Exception as e:
+        log.warning(f"[cover] DB update failed for book {book_id}: {e}")
+
+    return cover_url

credentials/gdrive-service-account.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "canzati-1571595913242",
+  "private_key_id": "2cbe97d3ded7ed17bd239ea347c71048592e8866",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDIwD3ZM8ghG0gl\n+pd69KWKaoG8IgsuKULM3JoyxltRhaH4tENNwKlcIZo3Yax9gizN2/BePs6WbvlJ\npt0XuGLXtiB+qwjgiAGHLh/rtCTHE3LAcX1xubWQQS9M/JHv0lPtOK3u+XNFKbZb\nSgW50s3kOp3dHD4kYdEhJRoREg6UaPbsuvdrviLEE/5a+RFODzgOAGgIX+R1rhoW\ng26Jth3Ho3PPlBc+x4KNM4Reo5b0SBWaZhuJJ/5+5NAVHbMfE3ccMTe+w6siqoWd\nHXsQF8QHiZEa7fIqLCM678qSQBDpkrjZg5Mm6MJLf4yim1PbzQYZVkDypF5T78q2\n1xllNbyrAgMBAAECggEAQZGUweQpWjLcelcice0FQrPt4uJlr3KXmGz3Sl1CR1qz\n4BUUpIkoZ+irw6n52laRxr0vAcFW3YiqkJVZQl2dnyIj8+tuL1QM17fHgkdo+cLO\n6K9Iz0PeWxsHVYhrLB8Q/XrVSKgeL7ljj+aZ/yssx34jwKvwWha+5SoGnQu+Sld3\n/e+SOFADWu+lEJEBgFGcNXG2n2chZ7U4fgI4Bn1s2gzepAuAW/SNhRiLvuf8uVnV\nx9PlCK5WCvbvUlFlhTgA9KXIMA2eRlABDDw5W716+gsfNGUDP6bmLq+aY9VfZnte\nG6DshxqnM3yh9cgPMWhLBi5YgifqkYKmuKlz7FqTjQKBgQDvKumUrvw+lsaapc4V\nlWVqEHxsVegNGKWGBl8TidnhZf9vGYlB7SCURv15OEm2MDCTCwosg83mF0+HuXsL\nkWwyhLCh7HWOuCqoiEt1Cr6ZUYhRQEyAYC1BZ6nTRkwDyxVBVNqHWfebTvdg+A0o\ngEWLHIpqF19B5txWEW/8hlT9bQKBgQDW4SzSDVse9vbQeKJCz5s5w1hjzbcFFKip\nEr8lvbmAUujtUDL6BG+zrsRBodAJZoUIry/1yHq68XP4zle8Yq617jrkmdo4F1p3\nwFEm//BEYDfIL8etbF/jUbfYB3tXPXfVNZVdY2KJfVODahfameLuDPF0rLTVm8gH\nr1hV8ylLdwKBgEk0vWKBXTxbMldLwRTiMbSRb0RJ10mK7b9kW93DKCnR6qm7C0Re\nrMY2JIfiLs3RWkzn6ClP/sobAIDuyTb1QEUGUHp/wkv5GzWS4mXk8ApF5o/YOpJg\n3flTHaxJrCLNGqhwrKMZaqzjMt0/j+eBu8ek6MkodkPZwj+l/r0w8PrRAoGAF11I\nmFZncAh+cdCMJCOaxz8cX03A9sZ2SFwHeL0WbIOXXCZ6RIEy/CCmqlRD4KHLGVSj\nLGBuJ+GVMJqPI6a/RHzw3mZ0OeEgLyE0Mb9miqjDp/q6BXthOcBq8wxtNuEyJHa9\nFPh3gGemKwXgLh7JdAToRD17NDbKGzvPqRjVhqUCgYEA5ccgXciM5qrlW2BAOHKI\nWdqWnUOe+i8ruTYleQAW3Hs8L5LEwy3FDralZZtDIrtJiXERmpnXkFQnA+uT4E3f\nx/t6F9mPek9punq0Zez+1AXT3jnJ2gnPzXC4udH7JTdUMm0fNI8u13vxFStwCM0C\nWNdogFbxbobbziAbbSFbMro=\n-----END PRIVATE KEY-----\n",
+  "client_email": "rag-drive@canzati-1571595913242.iam.gserviceaccount.com",
+  "client_id": "102740801235055915290",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/rag-drive%40canzati-1571595913242.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

docker-compose.yml

@@ -0,0 +1,30 @@
+services:
+  rag-api:
+    build: .
+    container_name: rag-api
+    restart: always
+    volumes:
+      - /opt/rag:/opt/rag
+      - /mnt/rag-media:/mnt/rag-media
+      - /root/.cache:/root/.cache
+    environment:
+      - QDRANT_URL=http://172.16.2.1:6333
+      - REDIS_URL=redis://172.16.2.1:6379
+      - POSTGRES_URL=postgresql://raguser:rag_secure_2025@172.16.2.1:5432/ragdb
+      - OLLAMA_URL=http://172.16.2.1:11434
+      - RAG_API_KEY=zRf77E_brSEnA2BUi40uxLxT4uofAk661WMZb-cdv4M
+      - ADMIN_KEY=afakadmin-Ihb7kflFA682Ax3EICnV2ymr4KuNZSo7
+      - JWT_SECRET=Xb8d_XzH9Kx47v3Y0KZVUPWMTdlhIsIXw6IZV93mUGiKIy6wLxE-DuK6ZJdShKHp
+    networks:
+      - coolify
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.rag-api.rule=Host(`rag.afaqzaid.com`)
+      - traefik.http.routers.rag-api.entrypoints=websecure
+      - traefik.http.routers.rag-api.tls=true
+      - traefik.http.routers.rag-api.tls.certresolver=letsencrypt
+      - traefik.http.services.rag-api.loadbalancer.server.port=8100
+
+networks:
+  coolify:
+    external: true

export.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+# AfakZaid RAG — Full Export Script
+# Run on SOURCE server: bash /opt/rag/export.sh
+# Output: /opt/rag-export.tar.gz (~500MB)
+
+set -e
+EXPORT_DIR="/tmp/rag-export"
+rm -rf $EXPORT_DIR && mkdir -p $EXPORT_DIR
+echo "=== AfakZaid RAG Export ==="
+
+# 1. Code
+echo "[1/6] Copying code..."
+cp -r /opt/rag $EXPORT_DIR/code
+rm -f $EXPORT_DIR/code/export.sh
+
+# 2. Docker image
+echo "[2/6] Exporting Docker image (may take 2-3 min)..."
+docker commit rag-api rag-api-img:export 2>/dev/null || true
+docker save rag-api-img:secure | gzip > $EXPORT_DIR/rag-image.tar.gz
+echo "  Image size: $(du -sh $EXPORT_DIR/rag-image.tar.gz | cut -f1)"
+
+# 3. PostgreSQL dump
+echo "[3/6] Dumping PostgreSQL..."
+docker exec rag-postgres pg_dump -U raguser ragdb > $EXPORT_DIR/ragdb.sql
+echo "  DB size: $(wc -l < $EXPORT_DIR/ragdb.sql) lines"
+
+# 4. Qdrant snapshot
+echo "[4/6] Exporting Qdrant vectors..."
+curl -s -X POST "http://localhost:6333/collections/rag_chunks/snapshots" | python3 -c "
+import sys, json, urllib.request
+d = json.load(sys.stdin)
+name = d['result']['name']
+print(f'  Snapshot: {name}')
+url = f'http://localhost:6333/collections/rag_chunks/snapshots/{name}'
+urllib.request.urlretrieve(url, '/tmp/rag-export/qdrant_chunks.snapshot')
+" 2>/dev/null || echo "  Qdrant: backing up data directory instead"
+# Also backup Qdrant data dir as fallback
+if [ ! -f "$EXPORT_DIR/qdrant_chunks.snapshot" ]; then
+  docker exec qdrant tar czf - /qdrant/storage 2>/dev/null > $EXPORT_DIR/qdrant_storage.tar.gz || true
+fi
+# mem0 collection
+curl -s -X POST "http://localhost:6333/collections/mem0_memory/snapshots" | python3 -c "
+import sys, json, urllib.request
+d = json.load(sys.stdin)
+name = d['result']['name']
+url = f'http://localhost:6333/collections/mem0_memory/snapshots/{name}'
+urllib.request.urlretrieve(url, '/tmp/rag-export/qdrant_mem0.snapshot')
+print(f'  Mem0 snapshot: done')
+" 2>/dev/null || true
+
+# 5. Media files
+echo "[5/6] Copying media files..."
+mkdir -p $EXPORT_DIR/media
+cp -r /mnt/rag-media/pdfs  $EXPORT_DIR/media/ 2>/dev/null || true
+cp -r /mnt/rag-media/covers $EXPORT_DIR/media/ 2>/dev/null || true
+echo "  PDFs: $(ls $EXPORT_DIR/media/pdfs/*.pdf 2>/dev/null | wc -l) files"
+echo "  Covers: $(ls $EXPORT_DIR/media/covers/*.jpg 2>/dev/null | wc -l) files"
+
+# 6. Secrets (without sensitive keys — you'll fill these in on new server)
+echo "[6/6] Exporting config template..."
+cat > $EXPORT_DIR/secrets.template << 'SECRETS'
+# Fill in these values on the new server
+# Generate new passwords — don't reuse old ones
+REDIS_PASSWORD=CHANGE_ME
+QDRANT_API_KEY=CHANGE_ME
+TYPESENSE_API_KEY=CHANGE_ME
+MINIO_ROOT_USER=CHANGE_ME
+MINIO_ROOT_PASSWORD=CHANGE_ME
+FLOWER_USER=afakadmin
+FLOWER_PASSWORD=CHANGE_ME
+CELERY_BROKER_URL=  # auto-generated by setup script
+# R2 — reuse your existing credentials
+R2_ENDPOINT=https://ba9387fe0eff62e35451939231a47baa.r2.cloudflarestorage.com
+R2_BUCKET=library
+R2_PUBLIC_URL=https://cdn.afaqzaid.com   # UPDATE to new CDN domain
+# API keys — generate new ones
+RAG_API_KEY=CHANGE_ME
+ADMIN_KEY=CHANGE_ME
+JWT_SECRET=CHANGE_ME
+SECRETS
+
+# Pack everything
+echo ""
+echo "Packing archive..."
+tar czf /opt/rag-export.tar.gz -C /tmp rag-export
+SIZE=$(du -sh /opt/rag-export.tar.gz | cut -f1)
+echo ""
+echo "=== Export complete ==="
+echo "File: /opt/rag-export.tar.gz ($SIZE)"
+echo ""
+echo "Transfer to new server with:"
+echo "  scp root@141.136.36.129:/opt/rag-export.tar.gz root@NEW_SERVER_IP:/opt/"

import.sh

@@ -0,0 +1,271 @@
+#!/bin/bash
+# AfakZaid RAG — Full Import / Setup Script
+# Run on NEW server after copying rag-export.tar.gz:
+#   bash /opt/rag-import.sh YOUR_DOMAIN.COM
+#
+# Prerequisites: Ubuntu 22/24, root access, ports 80+443 open
+
+set -e
+NEW_DOMAIN="${1:-rag.example.com}"
+echo "=== AfakZaid RAG Import ==="
+echo "Domain: $NEW_DOMAIN"
+echo ""
+
+# ── 0. Install Docker ─────────────────────────────────────────────────────────
+if ! command -v docker &>/dev/null; then
+  echo "[0] Installing Docker..."
+  curl -fsSL https://get.docker.com | sh
+  systemctl enable --now docker
+else
+  echo "[0] Docker already installed: $(docker --version)"
+fi
+
+# ── 1. Unpack ─────────────────────────────────────────────────────────────────
+echo "[1] Unpacking archive..."
+tar xzf /opt/rag-export.tar.gz -C /tmp
+EXPORT="/tmp/rag-export"
+
+# ── 2. Copy code ──────────────────────────────────────────────────────────────
+echo "[2] Installing code..."
+mkdir -p /opt/rag
+cp -r $EXPORT/code/* /opt/rag/
+chmod 700 /opt/rag
+
+# ── 3. Generate secrets ───────────────────────────────────────────────────────
+echo "[3] Generating secrets..."
+gen() { openssl rand -base64 32 | tr -d '/+=' | head -c ${1:-32}; }
+REDIS_PASS=$(gen 32)
+QDRANT_KEY=$(gen 32)
+TYPESENSE_KEY=$(gen 40)
+FLOWER_PASS=$(gen 20)
+RAG_API_KEY="rag-$(gen 36)"
+ADMIN_KEY="admin-$(gen 36)"
+JWT_SECRET=$(gen 64)
+CELERY_BROKER="redis://:${REDIS_PASS}@localhost:6379/1"
+
+cat > /opt/rag/.secrets << SECRETS
+REDIS_PASSWORD=${REDIS_PASS}
+QDRANT_API_KEY=${QDRANT_KEY}
+TYPESENSE_API_KEY=${TYPESENSE_KEY}
+FLOWER_USER=afakadmin
+FLOWER_PASSWORD=${FLOWER_PASS}
+CELERY_BROKER_URL=${CELERY_BROKER}
+RAG_API_KEY=${RAG_API_KEY}
+ADMIN_KEY=${ADMIN_KEY}
+JWT_SECRET=${JWT_SECRET}
+SECRETS
+
+# Append R2 credentials from template (user fills these in)
+grep "^R2_" $EXPORT/secrets.template >> /opt/rag/.secrets || true
+chmod 600 /opt/rag/.secrets
+echo "  Secrets written to /opt/rag/.secrets"
+echo "  ⚠️  Edit /opt/rag/.secrets and fill in R2_ACCESS_KEY and R2_SECRET_KEY"
+
+# ── 4. Create media directories ───────────────────────────────────────────────
+echo "[4] Setting up media directories..."
+mkdir -p /mnt/rag-media/pdfs /mnt/rag-media/covers
+cp -r $EXPORT/media/pdfs/*   /mnt/rag-media/pdfs/   2>/dev/null || true
+cp -r $EXPORT/media/covers/* /mnt/rag-media/covers/  2>/dev/null || true
+echo "  PDFs: $(ls /mnt/rag-media/pdfs/*.pdf 2>/dev/null | wc -l)"
+echo "  Covers: $(ls /mnt/rag-media/covers/*.jpg 2>/dev/null | wc -l)"
+
+# ── 5. Create Docker network ──────────────────────────────────────────────────
+echo "[5] Creating Docker network..."
+docker network create rag-net 2>/dev/null || echo "  Network exists"
+
+# ── 6. Start PostgreSQL ───────────────────────────────────────────────────────
+echo "[6] Starting PostgreSQL..."
+docker run -d --name rag-postgres --network rag-net \
+  -e POSTGRES_DB=ragdb \
+  -e POSTGRES_USER=raguser \
+  -e POSTGRES_PASSWORD=rag_secure_2025 \
+  -v rag-pgdata:/var/lib/postgresql/data \
+  --restart unless-stopped \
+  postgres:15-alpine
+
+sleep 8
+echo "  Restoring database..."
+cat $EXPORT/ragdb.sql | docker exec -i rag-postgres psql -U raguser -d ragdb
+echo "  Database restored"
+
+# ── 7. Start Redis ────────────────────────────────────────────────────────────
+echo "[7] Starting Redis..."
+docker run -d --name rag-redis --network rag-net \
+  --restart unless-stopped \
+  redis:7-alpine redis-server \
+  --requirepass "${REDIS_PASS}" \
+  --appendonly yes
+sleep 3
+
+# ── 8. Start Qdrant ───────────────────────────────────────────────────────────
+echo "[8] Starting Qdrant..."
+mkdir -p /opt/qdrant/storage
+docker run -d --name qdrant --network rag-net \
+  -e QDRANT__SERVICE__API_KEY="${QDRANT_KEY}" \
+  -v /opt/qdrant/storage:/qdrant/storage \
+  --restart unless-stopped \
+  qdrant/qdrant:latest
+sleep 8
+
+# Restore Qdrant snapshots
+if [ -f "$EXPORT/qdrant_chunks.snapshot" ]; then
+  echo "  Restoring rag_chunks vectors..."
+  curl -s -X PUT "http://localhost:6333/collections/rag_chunks/snapshots/upload?priority=snapshot" \
+    -H "api-key: ${QDRANT_KEY}" \
+    -H "Content-Type: multipart/form-data" \
+    -F "snapshot=@$EXPORT/qdrant_chunks.snapshot" | python3 -c "import sys,json; d=json.load(sys.stdin); print('  Chunks:', d.get('status','?'))"
+fi
+if [ -f "$EXPORT/qdrant_mem0.snapshot" ]; then
+  echo "  Restoring mem0 vectors..."
+  curl -s -X PUT "http://localhost:6333/collections/mem0_memory/snapshots/upload?priority=snapshot" \
+    -H "api-key: ${QDRANT_KEY}" \
+    -H "Content-Type: multipart/form-data" \
+    -F "snapshot=@$EXPORT/qdrant_mem0.snapshot" | python3 -c "import sys,json; d=json.load(sys.stdin); print('  Mem0:', d.get('status','?'))"
+fi
+
+# ── 9. Start Typesense ────────────────────────────────────────────────────────
+echo "[9] Starting Typesense..."
+mkdir -p /opt/typesense/data
+docker run -d --name typesense --network rag-net \
+  -e TYPESENSE_API_KEY="${TYPESENSE_KEY}" \
+  -e TYPESENSE_DATA_DIR=/data \
+  -v /opt/typesense/data:/data \
+  --restart unless-stopped \
+  typesense/typesense:0.25.2 \
+  --data-dir /data --api-key "${TYPESENSE_KEY}" --enable-cors
+sleep 5
+
+# ── 10. Install Ollama + models ───────────────────────────────────────────────
+echo "[10] Installing Ollama..."
+curl -fsSL https://ollama.ai/install.sh | sh
+systemctl enable --now ollama
+sleep 5
+echo "  Pulling models (this takes 10-20 min on first run)..."
+ollama pull nomic-embed-text &
+ollama pull qwen2:7b &
+wait
+echo "  Models ready"
+
+# ── 11. Load Docker image ─────────────────────────────────────────────────────
+echo "[11] Loading RAG API image..."
+docker load < $EXPORT/rag-image.tar.gz
+echo "  Image loaded"
+
+# ── 12. Get host IP for container-to-host communication ───────────────────────
+HOST_IP=$(docker network inspect rag-net --format='{{range .IPAM.Config}}{{.Gateway}}{{end}}' | head -1)
+echo "  Host IP for containers: $HOST_IP"
+
+# ── 13. Start RAG API ─────────────────────────────────────────────────────────
+echo "[13] Starting RAG API..."
+docker run -d --name rag-api --network rag-net \
+  -v /opt/rag:/opt/rag -v /mnt/rag-media:/mnt/rag-media \
+  -e POSTGRES_URL="postgresql://raguser:rag_secure_2025@rag-postgres:5432/ragdb" \
+  -e REDIS_URL="redis://rag-redis:6379" \
+  -e REDIS_PASSWORD="${REDIS_PASS}" \
+  -e QDRANT_URL="http://qdrant:6333" \
+  -e QDRANT_API_KEY="${QDRANT_KEY}" \
+  -e OLLAMA_URL="http://${HOST_IP}:11434" \
+  -e RAG_API_KEY="${RAG_API_KEY}" \
+  -e ADMIN_KEY="${ADMIN_KEY}" \
+  -e JWT_SECRET="${JWT_SECRET}" \
+  -e TYPESENSE_HOST="typesense" \
+  -e TYPESENSE_PORT="8108" \
+  -e TYPESENSE_API_KEY="${TYPESENSE_KEY}" \
+  -e CELERY_BROKER_URL="${CELERY_BROKER}" \
+  -e R2_ENDPOINT="$(grep ^R2_ENDPOINT /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_ACCESS_KEY="$(grep ^R2_ACCESS_KEY /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_SECRET_KEY="$(grep ^R2_SECRET_KEY /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_BUCKET="$(grep ^R2_BUCKET /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_PUBLIC_URL="$(grep ^R2_PUBLIC_URL /opt/rag/.secrets | cut -d= -f2)" \
+  --restart unless-stopped \
+  -p 8100:8100 \
+  rag-api-img:secure
+
+sleep 20
+
+# ── 14. Start Celery worker ───────────────────────────────────────────────────
+echo "[14] Starting Celery worker..."
+docker run -d --name rag-celery --network rag-net \
+  -v /opt/rag:/opt/rag -v /mnt/rag-media:/mnt/rag-media \
+  -e POSTGRES_URL="postgresql://raguser:rag_secure_2025@rag-postgres:5432/ragdb" \
+  -e REDIS_URL="redis://rag-redis:6379" \
+  -e REDIS_PASSWORD="${REDIS_PASS}" \
+  -e QDRANT_URL="http://qdrant:6333" \
+  -e QDRANT_API_KEY="${QDRANT_KEY}" \
+  -e OLLAMA_URL="http://${HOST_IP}:11434" \
+  -e TYPESENSE_HOST="typesense" \
+  -e TYPESENSE_PORT="8108" \
+  -e TYPESENSE_API_KEY="${TYPESENSE_KEY}" \
+  -e CELERY_BROKER_URL="${CELERY_BROKER}" \
+  -e R2_ENDPOINT="$(grep ^R2_ENDPOINT /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_ACCESS_KEY="$(grep ^R2_ACCESS_KEY /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_SECRET_KEY="$(grep ^R2_SECRET_KEY /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_BUCKET="$(grep ^R2_BUCKET /opt/rag/.secrets | cut -d= -f2)" \
+  -e R2_PUBLIC_URL="$(grep ^R2_PUBLIC_URL /opt/rag/.secrets | cut -d= -f2)" \
+  -w /opt/rag --restart unless-stopped \
+  rag-api-img:secure \
+  python3 -m celery -A tasks worker --loglevel=info --concurrency=2
+
+# ── 15. Setup Nginx reverse proxy ─────────────────────────────────────────────
+echo "[15] Setting up Nginx + SSL..."
+apt-get install -y nginx certbot python3-certbot-nginx -qq
+
+cat > /etc/nginx/sites-available/rag << NGINX
+server {
+    listen 80;
+    server_name ${NEW_DOMAIN};
+    location / {
+        proxy_pass http://localhost:8100;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto \$scheme;
+        client_max_body_size 100M;
+        proxy_read_timeout 300s;
+    }
+}
+NGINX
+
+ln -sf /etc/nginx/sites-available/rag /etc/nginx/sites-enabled/rag
+rm -f /etc/nginx/sites-enabled/default
+nginx -t && systemctl reload nginx
+
+# SSL
+certbot --nginx -d ${NEW_DOMAIN} --non-interactive --agree-tos -m admin@${NEW_DOMAIN} || \
+  echo "  ⚠️  SSL failed — make sure DNS points to this server first"
+
+# ── 16. Firewall ──────────────────────────────────────────────────────────────
+echo "[16] Configuring firewall..."
+ufw allow 22/tcp
+ufw allow 80/tcp
+ufw allow 443/tcp
+ufw --force enable
+
+# ── 17. Health check ──────────────────────────────────────────────────────────
+echo ""
+echo "=== Health Check ==="
+sleep 5
+curl -s http://localhost:8100/stats | python3 -c "
+import sys,json
+d=json.load(sys.stdin)
+print(f'  Books: {d.get(\"books\")}')
+print(f'  Vectors: {d.get(\"vectors\")}')
+print(f'  Conversations: {d.get(\"conversations\")}')
+" 2>/dev/null || echo "  API starting up..."
+
+echo ""
+echo "=== Import Complete ==="
+echo ""
+echo "Your RAG system is running at: https://${NEW_DOMAIN}"
+echo ""
+echo "IMPORTANT — Complete these steps:"
+echo "  1. Edit /opt/rag/.secrets and add R2_ACCESS_KEY + R2_SECRET_KEY"
+echo "  2. Update R2_PUBLIC_URL to your new CDN domain"
+echo "  3. Add the new domain to Cloudflare R2 custom domains if needed"
+echo "  4. Reset admin password: visit https://${NEW_DOMAIN}/admin"
+echo ""
+echo "  RAG_API_KEY : ${RAG_API_KEY}"
+echo "  ADMIN_KEY   : ${ADMIN_KEY}"
+echo "  Flower pass : ${FLOWER_PASS}"
+echo ""
+echo "Save these credentials securely."

ingest.py

@@ -9,14 +9,15 @@ Improvements over v1:
 """
 import os, sys, hashlib, uuid, re, shutil
 import fitz                                       # PyMuPDF
-from sentence_transformers import SentenceTransformer
+import requests as _requests
 from qdrant_client import QdrantClient
 from qdrant_client.models import Distance, VectorParams, PointStruct
 
 QDRANT_URL   = os.getenv("QDRANT_URL",   "http://localhost:6333")
 POSTGRES_URL = os.getenv("POSTGRES_URL", "")
 COLLECTION   = "arabic_books"
-MODEL_NAME   = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
+EMBED_MODEL  = "bge-m3"
+OLLAMA_URL   = os.getenv("OLLAMA_URL", "http://localhost:11434")
 
 TARGET_WORDS  = 150      # ideal chunk size
 MAX_WORDS     = 200      # hard cap before forced split
@@ -27,9 +28,8 @@ DELETE_AFTER_INGEST = os.getenv("DELETE_PDF_AFTER_INGEST", "false").lower() == "
 COVER_DIR = "/mnt/rag-media/covers"
 os.makedirs(COVER_DIR, exist_ok=True)
 
-print("Loading embedding model...")
+VECTOR_SIZE = 1024  # bge-m3 output dimension
-model       = SentenceTransformer(MODEL_NAME)
+print(f"Embedding model: {EMBED_MODEL} via Ollama ({VECTOR_SIZE} dims)")
-VECTOR_SIZE = model.get_embedding_dimension()
 _qdrant_key = os.getenv("QDRANT_API_KEY", "")
 client      = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
 
@@ -54,10 +54,14 @@ def get_file_hash(path):
     return h.hexdigest()
 
 def is_already_ingested(file_hash):
-    """Returns existing filename if hash matches, None otherwise."""
+    """Returns existing filename only if hash matches AND chunks exist."""
     try:
         conn = get_db(); cur = conn.cursor()
-        cur.execute("SELECT filename, title FROM books WHERE file_hash = %s", (file_hash,))
+        cur.execute("""
+            SELECT b.filename FROM books b
+            WHERE b.file_hash = %s
+            AND EXISTS (SELECT 1 FROM chunks c WHERE c.book_filename = b.filename)
+        """, (file_hash,))
         row = cur.fetchone(); cur.close(); conn.close()
         return row[0] if row else None
     except: return None
@@ -103,6 +107,20 @@ def extract_cover(pdf_path, book_id):
         print(f"  Cover extraction failed: {e}")
         return None
 
+
+def normalize_arabic(text: str) -> str:
+    """Normalize Arabic text — replace Persian/Urdu variants and fix common PDF extraction errors."""
+    import unicodedata
+    replacements = {
+        '\u06cc': 'ي', '\u0643': 'ك',
+        '\u06a9': 'ك', '\u0647': 'ه',
+        '\u06be': 'ه', '\u06c1': 'ه',
+    }
+    for src, dst in replacements.items():
+        text = text.replace(src, dst)
+    text = text.replace('هللا', 'الله')
+    return unicodedata.normalize('NFC', text)
+
 # ── Paragraph-aware chunking ──────────────────────────────────────────────────
 def split_sentences(text):
     """
@@ -197,6 +215,7 @@ def ingest_pdf(pdf_path):
         text = page.get_text()
         if not text.strip():
             continue
+        text = normalize_arabic(text)
         for chunk in chunk_text_smart(text):
             all_chunks.append({
                 "text": chunk,
@@ -227,14 +246,16 @@ def ingest_pdf(pdf_path):
     batch_size = 32
     for i in range(0, len(all_chunks), batch_size):
         batch      = all_chunks[i:i + batch_size]
-        embeddings = model.encode([c["text"] for c in batch], show_progress_bar=False)
+        batch = [{**c, "text": c["text"].replace("\x00", "")} for c in batch]
+        resp = _requests.post(f"{OLLAMA_URL}/api/embed", json={"model": EMBED_MODEL, "input": [c["text"] for c in batch]}, timeout=120)
+        embeddings = resp.json()["embeddings"]
         qdrant_points = []
         pg_rows       = []
         for j, c in enumerate(batch):
             vid = str(uuid.uuid4())
             qdrant_points.append(PointStruct(
                 id=vid,
-                vector=embeddings[j].tolist(),
+                vector=embeddings[j],
                 payload={"text":c["text"],"book":c["book"],"page":c["page"],"word_count":c["word_count"]}
             ))
             pg_rows.append((vid, book_id, c["book"], c["page"], c["text"], c["word_count"]))

meta_extract.py

@@ -11,11 +11,46 @@ OLLAMA_MODEL = "qwen2:7b"
 MEDIA_DIR    = "/mnt/rag-media/pdfs"
 
 
+def _normalize_arabic(text: str) -> str:
+    """Normalize Arabic/Persian character variants for consistent comparison."""
+    if not text:
+        return text
+    # Normalize Persian ی/ک to Arabic ي/ك
+    text = text.replace("\u06cc", "\u064a").replace("\u06a9", "\u0643")
+    # Normalize alef variants
+    text = text.replace("\u0622", "\u0627").replace("\u0623", "\u0627").replace("\u0625", "\u0627")
+    # Normalize teh marbuta
+    text = text.replace("\u0629", "\u0647")
+    import re as _re
+    return _re.sub(r"\s+", " ", text).strip()
+
+def _hint_from_filename(filename: str) -> dict:
+    """Extract author/title hints from common filename patterns like:
+    '1416-09-16 الشيخ نور الدين قره علي - قيام الليل (3).pdf'
+    'الشيخ جمال الدين سيروان - عدد الطلاق (65).pdf'
+    """
+    import re as _re
+    name = filename.replace(".pdf", "").strip()
+    # Remove leading date patterns
+    name = _re.sub(r"^\d{4}-\d{2}-\d{2}\s*", "", name)
+    name = _re.sub(r"^\d{4}-\d{1,2}-\d{1,2}\s*", "", name)
+    # Split on " - " separator
+    parts = [p.strip() for p in name.split(" - ") if p.strip()]
+    # Remove trailing numbers in parentheses
+    parts = [_re.sub(r"\s*\(\d+\)\s*$", "", p).strip() for p in parts]
+    if len(parts) >= 2:
+        return {"author_hint": parts[0], "title_hint": parts[1]}
+    elif len(parts) == 1:
+        return {"author_hint": "", "title_hint": parts[0]}
+    return {"author_hint": "", "title_hint": name}
+
 def extract_meta(pdf_path: str, filename: str) -> dict:
     """Extract title/author/category/description from PDF first 3 pages via Ollama."""
+    hints = _hint_from_filename(filename)
+
     fallback = {
-        "title":       filename.replace(".pdf", "").strip(),
+        "title":       hints["title_hint"] or filename.replace(".pdf", "").strip(),
-        "author":      "",
+        "author":      hints["author_hint"],
         "category":    "عام",
         "description": ""
     }
@@ -27,42 +62,163 @@ def extract_meta(pdf_path: str, filename: str) -> dict:
         doc.close()
     except Exception as e:
         print(f"[meta] fitz error: {e}")
-        text = filename.replace(".pdf", "")
+        text = ""
+
+    # Build a smarter prompt with filename hints
+    hint_line = ""
+    if hints["author_hint"]:
+        hint_line += f"المؤلف المحتمل من اسم الملف: {hints['author_hint']}\n"
+    if hints["title_hint"]:
+        hint_line += f"العنوان المحتمل من اسم الملف: {hints['title_hint']}\n"
 
     prompt = (
-        "استخرج من النص التالي معلومات الكتاب وأجب بـ JSON فقط بدون أي نص إضافي:\n"
+        "أنت مساعد متخصص في تحليل الكتب العربية الإسلامية.\n"
-        '{"title":"عنوان الكتاب","author":"اسم المؤلف",'
+        "مهمتك: استخرج معلومات الكتاب من النص والمعلومات المتاحة، وأجب بـ JSON فقط.\n"
-        '"category":"التصنيف مثل فقه أو حديث أو تفسير أو أدب","description":"وصف مختصر"}\n\n'
+        "قواعد مهمة:\n"
-        f"النص:\n{text}"
+        "- إذا لم تجد العنوان الحقيقي في النص، استخدم العنوان المحتمل من اسم الملف\n"
+        "- إذا لم تجد المؤلف في النص، استخدم المؤلف المحتمل من اسم الملف\n"
+        "- لا تكتب أبداً كلمة عنوان_الكتاب أو اسم_المؤلف كقيمة\n"
+        "- التصنيف: اختر كلمة واحدة فقط بالضبط من هذه القائمة ولا تكتب غيرها:\n"
+        "  فقه | حديث | تفسير | أدب | عقيدة | تاريخ | تربية وتعليم | طب ورقية | فكر وفلسفة | عبادة وصلاة | عام\n"
+        "- لا تكتب: تفسير القرآن، تفسير القرآن الكريم، أو أي تركيب — فقط الكلمة من القائمة\n\n"
+        f"{hint_line}"
+        f"النص:\n{text[:2000]}\n\n"
+        "أجب بهذا الشكل فقط:\n"
+        '{"title":"...","author":"...","category":"...","description":"..."}'
     )
 
     try:
         r = http_req.post(
             f"{OLLAMA_URL}/api/generate",
             json={"model": OLLAMA_MODEL, "prompt": prompt, "stream": False,
-                  "options": {"num_predict": 256, "temperature": 0.1}},
+                  "options": {"num_predict": 300, "temperature": 0.1}},
             timeout=90
         )
         raw = r.json().get("response", "")
-        m   = re.search(r'\{[^}]+\}', raw, re.DOTALL)
+        m   = re.search(r'\{[^{}]+\}', raw, re.DOTALL)
         if m:
             parsed = json.loads(m.group())
+            title  = (parsed.get("title") or "").strip()
+            author = (parsed.get("author") or "").strip()
+            cat    = (parsed.get("category") or "عام").strip() or "عام"
+            desc   = (parsed.get("description") or "").strip()
+
+            # Reject placeholder values
+            bad_titles  = {"عنوان الكتاب","عنوان_الكتاب","العنوان الحقيقي","...","title","عنوان",""}
+            bad_descs   = {"وصف مختصر","وصف","وصف_مختصر","description","...","وصف مختصر للكتاب",""}
+            bad_authors = {"اسم المؤلف","اسم_المؤلف","اسم المؤلف الحقيقي","...","author","مؤلف",""}
+            VALID_CATS  = {'فقه','حديث','تفسير','أدب','عقيدة','تاريخ','تربية وتعليم','طب ورقية','فكر وفلسفة','عبادة وصلاة','عام'}
+            _CAT_NORM   = {
+    'تفسير القرآن': 'تفسير', 'تفسير القرآن الكريم': 'تفسير',
+    'تفسير القرآن والسنة': 'تفسير', 'علوم القرآن': 'تفسير',
+    'علم الحديث': 'حديث', 'السنة النبوية': 'حديث', 'الحديث النبوي': 'حديث',
+    'أصول الفقه': 'فقه', 'الفقه الإسلامي': 'فقه', 'الفقه': 'فقه',
+    'العقيدة': 'عقيدة', 'العقيدة الإسلامية': 'عقيدة', 'التوحيد': 'عقيدة',
+    'الأدب': 'أدب', 'أدب إسلامي': 'أدب',
+    'التاريخ': 'تاريخ', 'التاريخ الإسلامي': 'تاريخ', 'السيرة النبوية': 'تاريخ',
+    'التربية': 'تربية وتعليم', 'التعليم': 'تربية وتعليم', 'التربية الإسلامية': 'تربية وتعليم',
+    'الطب': 'طب ورقية', 'الطب النبوي': 'طب ورقية', 'الرقية': 'طب ورقية', 'الرقية الشرعية': 'طب ورقية',
+    'الفكر': 'فكر وفلسفة', 'الفلسفة': 'فكر وفلسفة', 'الفكر الإسلامي': 'فكر وفلسفة',
+    'العبادة': 'عبادة وصلاة', 'الصلاة': 'عبادة وصلاة', 'العبادات': 'عبادة وصلاة',
+}
+            bad = bad_titles  # keep compat
+            import re as _re2
+            # Reject if contains non-Arabic/non-space characters (Chinese, English etc.)
+            def _is_clean_arabic(s):
+                # Allow full Arabic/Persian unicode block + common punctuation
+                return not _re2.search(r'[^\u0600-\u06ff\u0750-\u077f\s\-\.،,؟!:\(\)\d]', s or '')
+            if title in bad_titles or not _is_clean_arabic(title):
+                title = fallback["title"]
+            if author in bad_authors or not _is_clean_arabic(author):
+                author = fallback["author"]
+            cat = _CAT_NORM.get(cat, cat)
+            if cat not in VALID_CATS or not _is_clean_arabic(cat):
+                cat = "عام"
+            if desc in bad_descs or not _is_clean_arabic(desc):
+                desc = ""
+
+            # Extra check: if title still looks like a filename, use hint
+            import re as _re3
+            if _re3.match(r'^\d{4}-\d{2}-\d{2}', title) or _re3.search(r'\(\d+\)\s*$', title):
+                title = fallback["title"]
+
             return {
-                "title":       (parsed.get("title") or fallback["title"]).strip(),
+                "title":       _normalize_arabic(title),
-                "author":      (parsed.get("author") or "").strip(),
+                "author":      _normalize_arabic(author),
-                "category":    (parsed.get("category") or "عام").strip() or "عام",
+                "category":    cat,
-                "description": (parsed.get("description") or "").strip(),
+                "description": desc,
             }
     except Exception as e:
         print(f"[meta] Ollama error for {filename}: {e}")
 
-    return fallback
+    # Last resort: scan first 10 lines of PDF for sheikh/author pattern
+    author_from_pdf = _extract_author_from_pdf_lines(pdf_path)
+    if author_from_pdf:
+        fallback["author"] = author_from_pdf
 
+    return {**fallback,
+            "title": _normalize_arabic(fallback["title"]),
+            "author": _normalize_arabic(fallback["author"])}
+
+
+VALID_CATS = {'فقه','حديث','تفسير','أدب','عقيدة','تاريخ','تربية وتعليم','طب ورقية','فكر وفلسفة','عبادة وصلاة','عام'}
+CAT_NORM = {
+    'تفسير القرآن': 'تفسير', 'تفسير القرآن الكريم': 'تفسير',
+    'تفسير القرآن والسنة': 'تفسير', 'علوم القرآن': 'تفسير',
+    'علم الحديث': 'حديث', 'السنة النبوية': 'حديث', 'الحديث النبوي': 'حديث',
+    'أصول الفقه': 'فقه', 'الفقه الإسلامي': 'فقه', 'الفقه': 'فقه',
+    'العقيدة': 'عقيدة', 'العقيدة الإسلامية': 'عقيدة', 'التوحيد': 'عقيدة',
+    'الأدب': 'أدب', 'أدب إسلامي': 'أدب',
+    'التاريخ': 'تاريخ', 'التاريخ الإسلامي': 'تاريخ', 'السيرة النبوية': 'تاريخ',
+    'التربية': 'تربية وتعليم', 'التعليم': 'تربية وتعليم', 'التربية الإسلامية': 'تربية وتعليم',
+    'الطب': 'طب ورقية', 'الطب النبوي': 'طب ورقية', 'الرقية': 'طب ورقية', 'الرقية الشرعية': 'طب ورقية',
+    'الفكر': 'فكر وفلسفة', 'الفلسفة': 'فكر وفلسفة', 'الفكر الإسلامي': 'فكر وفلسفة',
+    'العبادة': 'عبادة وصلاة', 'الصلاة': 'عبادة وصلاة', 'العبادات': 'عبادة وصلاة',
+}
+
+def _sanitize_meta(meta: dict, filename: str = "") -> dict:
+    """Final gate: reject any non-Arabic or placeholder values before saving."""
+    import re as _re
+    def has_foreign(s):
+        return bool(_re.search(r'[^\u0600-\u06ff\s\-\.\,،؟!\d\(\)]', s or ''))
+    
+    hints = _hint_from_filename(filename) if filename else {}
+    a_hint = hints.get('author_hint','')
+    t_hint = hints.get('title_hint','')
+    
+    BAD_TITLES  = {'عنوان الكتاب','عنوان_الكتاب','العنوان الحقيقي','...',''}
+    BAD_AUTHORS = {'اسم المؤلف','اسم_المؤلف','اسم المؤلف الحقيقي','...',''}
+    BAD_DESCS   = {'وصف مختصر','وصف','وصف_مختصر','وصف مختصر للكتاب','...',''}
+    
+    title  = meta.get('title','')
+    author = meta.get('author','')
+    cat    = meta.get('category','عام')
+    desc   = meta.get('description','')
+    
+    if title in BAD_TITLES or has_foreign(title) or _re.match(r'^\d{4}-\d{2}-\d{2}', title) or (t_hint and title == filename.replace('.pdf','')):
+        title = _normalize_arabic(t_hint)
+        if not title:
+            # Extract from filename directly — never store raw filename with date/author
+            _fn = filename.replace('.pdf','').strip()
+            _fn = _re.sub(r'^\d{4}-\d{2}-\d{2}\s*', '', _fn)
+            _parts = [_re.sub(r'\s*\(\d+\)\s*$', '', p).strip() for p in _fn.split(' - ') if p.strip()]
+            title = _parts[-1] if len(_parts) > 1 else _parts[0]
+    if author in BAD_AUTHORS or has_foreign(author):
+        author = _normalize_arabic(a_hint)
+    cat = CAT_NORM.get(cat, cat)
+    if cat not in VALID_CATS or has_foreign(cat):
+        cat = 'عام'
+    if desc in BAD_DESCS or has_foreign(desc):
+        desc = ''
+    
+    return {'title': title, 'author': _normalize_arabic(author),
+            'category': cat, 'description': desc}
 
 def save_meta_to_db(book_filename: str, meta: dict, uploader_id: int = None,
                     pg_url: str = None):
     """Update books table with extracted metadata."""
     import psycopg2
+    meta = _sanitize_meta(meta, book_filename)
     if not pg_url:
         pg_url = os.getenv("POSTGRES_URL", "")
     try:

postgres_data/PG_VERSION

@@ -0,0 +1 @@
+16