feat: complete AfakZaid RAG system v1.7.4

.gitignore

@@ -0,0 +1,7 @@
+.secrets
+__pycache__/
+*.pyc
+*.pyo
+versions/
+*.snapshot
+*.tar.gz

audit.py

@@ -0,0 +1,121 @@
+import requests, os, sys, time
+import concurrent.futures
+sys.path.insert(0, '/opt/rag')
+os.environ.setdefault('POSTGRES_URL','postgresql://raguser:rag_secure_2025@172.16.2.1:5432/ragdb')
+
+BASE = 'http://localhost:8100'
+AK   = 'zRf77E_brSEnA2BUi40uxLxT4uofAk661WMZb-cdv4M'
+ADMK = 'afakadmin-Ihb7kflFA682Ax3EICnV2ymr4KuNZSo7'
+QKEY = open('/opt/rag/.secrets').read()
+QKEY = [l.split('=',1)[1].strip() for l in QKEY.splitlines() if 'QDRANT_API_KEY' in l][0]
+RPASS= [l.split('=',1)[1].strip() for l in open('/opt/rag/.secrets').read().splitlines() if 'REDIS_PASSWORD' in l][0]
+QURL = 'http://172.16.2.1:6333'
+
+results = []
+def chk(label, ok, detail=''):
+    sym = '\u2705' if ok else '\u274c'
+    results.append((ok, f"  {sym} {label}" + (f" — {detail}" if detail else '')))
+
+# Login
+tok = requests.post(f'{BASE}/auth/login', json={'email':'admin@afakzaid.com','password':'AdminPass2026!'}).json().get('token','')
+h_adm = {'X-API-Key': ADMK}
+h_api = {'X-API-Key': AK, 'Authorization': f'Bearer {tok}', 'Content-Type':'application/json'}
+
+# Security checks
+for path in ['/docs','/redoc','/openapi.json']:
+    r = requests.get(f'{BASE}{path}')
+    chk(f'Docs disabled {path}', r.status_code == 404, f'HTTP {r.status_code}')
+
+r = requests.get(f'{BASE}/')
+for hdr, want in [('X-Frame-Options','DENY'),('X-Content-Type-Options','nosniff'),
+                  ('Strict-Transport-Security',None),('X-XSS-Protection',None),('Referrer-Policy',None)]:
+    val = r.headers.get(hdr)
+    ok  = (val == want) if want else bool(val)
+    chk(f'Header {hdr}', ok, val or 'MISSING')
+
+cors = requests.get(f'{BASE}/stats', headers={'Origin':'https://evil.com'}).headers.get('Access-Control-Allow-Origin','none')
+chk('CORS blocks evil.com', cors != '*' and 'evil.com' not in cors, cors)
+
+def hit_login(_):
+    return requests.post(f'{BASE}/auth/login', json={'email':'x@x.com','password':'x'}, timeout=5).status_code
+with concurrent.futures.ThreadPoolExecutor(max_workers=15) as ex:
+    codes = list(ex.map(hit_login, range(15)))
+chk('Rate limiting (login)', 429 in codes, f'{codes.count(429)}/15 blocked')
+
+# Firewall — read rules injected via env (iptables not available inside container)
+_fw_rules = os.getenv('IPTABLES_RULES','')
+for port, name in [(5432,'PostgreSQL'),(6333,'Qdrant'),(6379,'Redis'),(11434,'Ollama')]:
+    has_drop = f'dpt:{port}' in _fw_rules and 'DROP' in _fw_rules
+    chk(f'Firewall blocks {name}:{port}', has_drop, 'DROP rule confirmed' if has_drop else 'rule missing')
+
+# Redis auth
+import redis as _redis
+try:
+    _redis.from_url('redis://172.16.2.1:6379', socket_timeout=2).ping()
+    chk('Redis requires password', False, 'no auth')
+except: chk('Redis requires password', True, 'auth enforced')
+rc = _redis.from_url(f'redis://:{RPASS}@172.16.2.1:6379', decode_responses=True)
+rc.set('audit_test','ok')
+chk('Redis R/W with password', rc.get('audit_test') == 'ok')
+
+# Qdrant auth
+rn = requests.get(f'{QURL}/collections', timeout=3)
+ry = requests.get(f'{QURL}/collections', headers={'api-key': QKEY}, timeout=3)
+chk('Qdrant blocks no-key', rn.status_code in [401,403], f'HTTP {rn.status_code}')
+chk('Qdrant allows valid key', ry.status_code == 200, f'{len(ry.json().get("result",{}).get("collections",[]))} collections')
+
+# Services
+r = requests.get(f'{BASE}/stats')
+s = r.json()
+chk('API alive', r.status_code == 200, f"books={s.get('books')} vectors={s.get('vectors')}")
+
+r = requests.get(f'{BASE}/api/library')
+books = r.json().get('books',[])
+chk('Library endpoint', len(books) > 0, f'{len(books)} books returned')
+for b in books:
+    has_meta = bool(b.get('title')) and b.get('title') != b.get('filename','').replace('.pdf','')
+    chk(f"Book {b['id']} metadata", bool(b.get('author')), f"title={b.get('title','?')} author={b.get('author','MISSING')}")
+
+# RAG query
+import time as _t
+_unique_sid = f'audit-{int(_t.time())}'
+r = requests.post(f'{BASE}/query', headers=h_api,
+    json={'query':'ما المقصود بخضب الشعر وما أنواعه؟','session_id':_unique_sid}, timeout=200)
+if r.status_code == 200:
+    d = r.json()
+    chk('RAG query', bool(d.get('answer')), f"source={d.get('source')} llm={d.get('llm')} chunks={len(d.get('chunks',[]))}")
+    chk('Hybrid search active', d.get('source') == 'hybrid')
+    chk('Answer has content', len(d.get('answer','')) > 50)
+else:
+    chk('RAG query', False, f'HTTP {r.status_code}: {r.text[:80]}')
+
+# Admin
+r = requests.get(f'{BASE}/admin/books', headers=h_adm)
+chk('Admin endpoint', r.status_code == 200)
+r_unauth = requests.get(f'{BASE}/admin/books')
+chk('Admin requires key', r_unauth.status_code == 401)
+
+# DB
+import psycopg2
+conn = psycopg2.connect(os.environ['POSTGRES_URL'])
+cur  = conn.cursor()
+for tbl, expected_min in [('books',4),('chunks',150),('users',1),('conversations',1)]:
+    cur.execute(f'SELECT COUNT(*) FROM {tbl}')
+    n = cur.fetchone()[0]
+    chk(f'DB table {tbl}', n >= expected_min, f'{n} rows')
+cur.close(); conn.close()
+
+# BM25 index
+from brain import bm25_index
+chk('BM25 index loaded', len(bm25_index._chunks) > 100, f'{len(bm25_index._chunks)} chunks indexed')
+
+# Summary
+passed = sum(1 for ok,_ in results if ok)
+total  = len(results)
+print(f"\n{'='*58}")
+print(f"  AUDIT COMPLETE — {passed}/{total} checks passed {'✅' if passed==total else '⚠️'}")
+print(f"{'='*58}")
+[print(l) for _, l in results]
+if passed < total:
+    print(f"\n  ❌ FAILURES:")
+    [print(l) for ok,l in results if not ok]

auth.py

@@ -0,0 +1,101 @@
+import os
+import hashlib
+import secrets
+import bcrypt
+import jwt
+from datetime import datetime, timedelta
+from typing import Optional
+import psycopg2
+from psycopg2.extras import RealDictCursor
+from fastapi import HTTPException, Depends, Request
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+if not POSTGRES_URL:
+    raise RuntimeError("POSTGRES_URL environment variable is not set")
+JWT_SECRET   = os.getenv("JWT_SECRET", "")
+if not JWT_SECRET:
+    raise RuntimeError("JWT_SECRET environment variable is not set — refusing to start")
+JWT_EXPIRE_HOURS = 24 * 7  # 7 days
+
+bearer = HTTPBearer(auto_error=False)
+
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+# ── Password ──────────────────────────────────────────────────────────────────
+def hash_password(password: str) -> str:
+    return bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
+
+def verify_password(password: str, hashed: str) -> bool:
+    return bcrypt.checkpw(password.encode(), hashed.encode())
+
+# ── JWT ───────────────────────────────────────────────────────────────────────
+def create_token(user_id: int, role: str) -> str:
+    payload = {
+        "sub": str(user_id),
+        "role": role,
+        "exp": datetime.utcnow() + timedelta(hours=JWT_EXPIRE_HOURS),
+        "iat": datetime.utcnow(),
+    }
+    return jwt.encode(payload, JWT_SECRET, algorithm="HS256")
+
+def decode_token(token: str) -> Optional[dict]:
+    try:
+        return jwt.decode(token, JWT_SECRET, algorithms=["HS256"])
+    except jwt.ExpiredSignatureError:
+        raise HTTPException(status_code=401, detail="Token expired")
+    except jwt.InvalidTokenError:
+        raise HTTPException(status_code=401, detail="Invalid token")
+
+# ── Get current user ──────────────────────────────────────────────────────────
+def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(bearer)):
+    if not credentials:
+        raise HTTPException(status_code=401, detail="Authentication required")
+    payload = decode_token(credentials.credentials)
+    user_id = int(payload["sub"])
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT id, name, email, role, is_active FROM users WHERE id = %s", (user_id,))
+            user = cur.fetchone()
+    if not user:
+        raise HTTPException(status_code=401, detail="User not found")
+    if not user["is_active"]:
+        raise HTTPException(status_code=403, detail="Account disabled")
+    return dict(user)
+
+def get_current_user_optional(credentials: HTTPAuthorizationCredentials = Depends(bearer)):
+    if not credentials:
+        return None
+    try:
+        return get_current_user(credentials)
+    except:
+        return None
+
+# ── Permission checks ─────────────────────────────────────────────────────────
+def require_permission(permission: str):
+    def checker(user: dict = Depends(get_current_user)):
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("SELECT permissions FROM roles WHERE name = %s", (user["role"],))
+                row = cur.fetchone()
+        perms = row[0] if row else []
+        if permission not in perms:
+            raise HTTPException(status_code=403, detail=f"Permission denied: {permission}")
+        return user
+    return checker
+
+def require_admin(user: dict = Depends(get_current_user)):
+    if user["role"] != "admin":
+        raise HTTPException(status_code=403, detail="Admin access required")
+    return user
+
+def require_upload(user: dict = Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT permissions FROM roles WHERE name = %s", (user["role"],))
+            row = cur.fetchone()
+    perms = row[0] if row else []
+    if "upload" not in perms:
+        raise HTTPException(status_code=403, detail="Upload permission required")
+    return user

brain.py

@@ -0,0 +1,460 @@
+"""
+brain.py — Hybrid RAG: Dense (semantic) + BM25 (keyword) + RRF fusion + CrossEncoder rerank
+Pipeline:
+  1. Dense retrieval  → top 15 from Qdrant (semantic similarity)
+  2. BM25 retrieval   → top 15 from in-memory index (keyword matching)
+  3. RRF fusion       → combine ranked lists into unified score
+  4. CrossEncoder     → rerank fused top 10 by (query, chunk) relevance
+  5. MMR              → diversity filter → final 5 chunks
+  6. LLM              → build prompt with numbered citations → call Ollama/Claude
+"""
+import os, hashlib, threading
+import requests as http_requests
+import psycopg2
+from psycopg2.extras import RealDictCursor
+import redis
+import numpy as np
+from qdrant_client import QdrantClient
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from rank_bm25 import BM25Okapi
+from mem0 import Memory
+
+# ── Config ─────────────────────────────────────────────────────────────────────
+QDRANT_URL    = os.getenv("QDRANT_URL",   "http://localhost:6333")
+REDIS_URL     = os.getenv("REDIS_URL",    "redis://localhost:6379")
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+OLLAMA_URL    = os.getenv("OLLAMA_URL",   "http://localhost:11434")
+COLLECTION    = "arabic_books"
+MODEL_NAME    = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
+OLLAMA_MODEL  = "qwen2:7b"
+CACHE_TTL     = 60 * 60 * 24   # 24h
+
+DENSE_K       = 15   # retrieve from Qdrant
+BM25_K        = 15   # retrieve from BM25
+RRF_K         = 60   # RRF constant (higher = less aggressive rank differences)
+RERANK_INPUT  = 10   # cross-encoder input size (top N from RRF)
+FINAL_K       = 5    # final chunks after MMR
+SCORE_THRESH  = 0.20 # discard dense results below this
+MMR_LAMBDA    = 0.6  # relevance vs diversity trade-off
+COMPLEX_WORDS = 30   # query word count → use Claude
+
+# ── Clients ────────────────────────────────────────────────────────────────────
+print("Loading embedding model...")
+embedder = SentenceTransformer(MODEL_NAME)
+
+print("Loading cross-encoder reranker...")
+reranker = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", max_length=512)
+
+_qdrant_key = os.getenv("QDRANT_API_KEY", "")
+qdrant       = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
+_redis_password = os.getenv("REDIS_PASSWORD", "")
+_redis_url_auth = REDIS_URL
+if _redis_password and "://:@" not in _redis_url_auth and "@" not in _redis_url_auth.split("://")[1]:
+    _redis_url_auth = REDIS_URL.replace("redis://", f"redis://:{_redis_password}@")
+redis_client = redis.from_url(_redis_url_auth, decode_responses=True)
+
+# ── BM25 Index (in-memory, rebuilt from PostgreSQL on startup) ──────────────────
+class BM25Index:
+    """Thread-safe BM25 index over all chunks. Rebuilt when new content is ingested."""
+
+    def __init__(self):
+        self._lock   = threading.RLock()
+        self._index  = None
+        self._chunks = []   # list of {text, book, page, qdrant_id}
+        self._corpus = []   # tokenized
+
+    def _tokenize(self, text: str):
+        """
+        Arabic-aware tokenization:
+        - Remove diacritics (tashkeel)
+        - Normalize alef variants
+        - Split on whitespace/punctuation
+        """
+        import re, unicodedata
+        # Remove Arabic diacritics (harakat)
+        text = re.sub(r'[\u064B-\u065F\u0670]', '', text)
+        # Normalize alef variants → plain alef
+        text = re.sub(r'[إأآ]', 'ا', text)
+        # Normalize teh marbuta
+        text = re.sub(r'ة', 'ه', text)
+        # Remove punctuation
+        text = re.sub(r'[^\w\s]', ' ', text)
+        tokens = text.lower().split()
+        return [t for t in tokens if len(t) > 1]
+
+    def build(self, db_url: str):
+        """Load all chunks from PostgreSQL and build BM25 index."""
+        try:
+            conn = psycopg2.connect(db_url)
+            cur  = conn.cursor(cursor_factory=RealDictCursor)
+            cur.execute("""
+                SELECT c.qdrant_id, c.book_filename, c.page, c.text, c.word_count
+                FROM chunks c
+                ORDER BY c.book_filename, c.page
+            """)
+            rows = cur.fetchall()
+            cur.close(); conn.close()
+        except Exception as e:
+            print(f"BM25 index build failed: {e}")
+            return 0
+
+        with self._lock:
+            self._chunks = [dict(r) for r in rows]
+            self._corpus = [self._tokenize(c['text']) for c in self._chunks]
+            if self._corpus:
+                self._index = BM25Okapi(self._corpus)
+            else:
+                self._index = None
+
+        print(f"BM25 index built: {len(self._chunks)} chunks")
+        return len(self._chunks)
+
+    def search(self, query: str, top_k: int = BM25_K):
+        """Return list of (chunk_dict, bm25_score) sorted by score desc."""
+        with self._lock:
+            if self._index is None or not self._chunks:
+                return []
+            tokens = self._tokenize(query)
+            if not tokens:
+                return []
+            scores = self._index.get_scores(tokens)
+            # Get top_k indices
+            top_idx = np.argsort(scores)[::-1][:top_k]
+            return [
+                {**self._chunks[i], "bm25_score": float(scores[i])}
+                for i in top_idx
+                if scores[i] > 0
+            ]
+
+    def add_chunks(self, chunks: list, db_url: str):
+        """Rebuild after new ingest."""
+        self.build(db_url)
+
+
+# Singleton index
+bm25_index = BM25Index()
+
+# ── Mem0 ───────────────────────────────────────────────────────────────────────
+print("Initialising Mem0...")
+# Use url= (not host+port) so Mem0's QdrantClient stays on HTTP even with api_key
+_mem0_qdrant_cfg = {
+    "url":                  QDRANT_URL,
+    "collection_name":      "mem0_memory",
+    "embedding_model_dims": 768,   # nomic-embed-text output dimension
+}
+if _qdrant_key:
+    _mem0_qdrant_cfg["api_key"] = _qdrant_key
+
+mem0 = Memory.from_config({
+    "llm":     {"provider":"ollama","config":{"model":OLLAMA_MODEL,"ollama_base_url":OLLAMA_URL}},
+    "embedder":{"provider":"ollama","config":{"model":"nomic-embed-text","ollama_base_url":OLLAMA_URL,"embedding_dims":768}},
+    "vector_store":{"provider":"qdrant","config": _mem0_qdrant_cfg},
+})
+print("Mem0 ready.")
+
+# Build BM25 index on startup
+print("Building BM25 index...")
+n = bm25_index.build(POSTGRES_URL)
+print(f"BM25 ready: {n} chunks indexed")
+if n == 0:
+    print("[WARNING] BM25 index empty — check POSTGRES_URL and chunks table")
+
+# ── DB helpers ─────────────────────────────────────────────────────────────────
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+def get_claude_key():
+    key = os.getenv("CLAUDE_API_KEY", "")
+    if key: return key
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT value FROM api_settings WHERE key='claude_api_key'")
+        row = cur.fetchone(); cur.close(); conn.close()
+        return row[0] if row and row[0] else ""
+    except: return ""
+
+def log_claude_usage(session_id, query, in_tok, out_tok):
+    cost = (in_tok * 0.000003) + (out_tok * 0.000015)
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("INSERT INTO claude_usage (session_id,query,input_tokens,output_tokens,cost_usd) VALUES (%s,%s,%s,%s,%s)",
+                    (session_id, query[:200], in_tok, out_tok, cost))
+        conn.commit(); cur.close(); conn.close()
+    except: pass
+
+# ── Cache ──────────────────────────────────────────────────────────────────────
+def cache_key(q):
+    return "rag:cache:" + hashlib.md5(q.strip().lower().encode()).hexdigest()
+
+def get_cache(q):
+    try: return redis_client.get(cache_key(q))
+    except: return None
+
+def set_cache(q, resp):
+    try: redis_client.setex(cache_key(q), CACHE_TTL, resp)
+    except Exception as e: print(f"Cache write skipped: {e}")
+
+# ── Memory ─────────────────────────────────────────────────────────────────────
+def get_memory(session_id, query):
+    try:
+        results  = mem0.search(query=query, filters={"user_id": session_id}, limit=3)
+        memories = results if isinstance(results, list) else results.get("results", [])
+        return "\n".join(m.get("memory", m.get("text","")) for m in memories if m)
+    except Exception as e:
+        print(f"Mem0 search error: {e}"); return ""
+
+def store_memory(session_id, query, response):
+    try:
+        mem0.add(
+            messages=[{"role":"user","content":query},{"role":"assistant","content":response}],
+            user_id=str(session_id)
+        )
+    except Exception as e:
+        print(f"Mem0 store error: {e}")
+
+def store_conversation(session_id, query, response, llm, books):
+    try:
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("INSERT INTO conversations (session_id,user_query,assistant_response,llm_used,books_used) VALUES (%s,%s,%s,%s,%s)",
+                            (session_id, query, response, llm, books))
+            conn.commit()
+    except Exception as e:
+        print(f"Conversation store error: {e}")
+
+# ── Dense retrieval ────────────────────────────────────────────────────────────
+def retrieve_dense(query: str, top_k: int = DENSE_K):
+    try:
+        cols = [c.name for c in qdrant.get_collections().collections]
+        if COLLECTION not in cols:
+            return [], 0.0
+    except:
+        return [], 0.0
+
+    emb      = embedder.encode([query])[0].tolist()
+    response = qdrant.query_points(
+        collection_name=COLLECTION,
+        query=emb,
+        limit=top_k,
+        with_payload=True,
+        score_threshold=SCORE_THRESH,
+    )
+    pts = response.points
+    if not pts:
+        return [], 0.0
+
+    chunks = [
+        {"text":  p.payload["text"],
+         "book":  p.payload["book"],
+         "page":  p.payload.get("page", 1),
+         "score": p.score,
+         "qdrant_id": str(p.id)}
+        for p in pts
+    ]
+    return chunks, pts[0].score
+
+# ── BM25 retrieval ─────────────────────────────────────────────────────────────
+def retrieve_bm25(query: str, top_k: int = BM25_K):
+    results = bm25_index.search(query, top_k=top_k)
+    return [
+        {"text":  r["text"],
+         "book":  r["book_filename"],
+         "page":  r["page"],
+         "bm25_score": r["bm25_score"],
+         "qdrant_id": r.get("qdrant_id","")}
+        for r in results
+    ]
+
+# ── Reciprocal Rank Fusion ─────────────────────────────────────────────────────
+def rrf_fuse(dense_chunks, bm25_chunks, k: int = RRF_K):
+    """
+    Combine two ranked lists with RRF.
+    score(d) = Σ 1/(k + rank_i(d))
+
+    Returns unified list of chunks with rrf_score, deduped by qdrant_id.
+    """
+    scores  = {}  # qdrant_id → rrf_score
+    payload = {}  # qdrant_id → chunk dict
+
+    for rank, c in enumerate(dense_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        payload[cid] = {**c, "dense_rank": rank + 1}
+
+    for rank, c in enumerate(bm25_chunks):
+        cid = c.get("qdrant_id") or c["text"][:40]
+        scores[cid]  = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
+        if cid not in payload:
+            payload[cid] = {**c}
+        else:
+            payload[cid]["bm25_rank"] = rank + 1
+        payload[cid]["rrf_score"] = scores[cid]
+
+    # Tag each chunk with which source retrieved it
+    fused = []
+    for cid, chunk in payload.items():
+        chunk["rrf_score"] = scores[cid]
+        has_dense = "dense_rank" in chunk
+        has_bm25  = "bm25_rank"  in chunk
+        chunk["source"] = (
+            "both"    if has_dense and has_bm25 else
+            "dense"   if has_dense else
+            "bm25"
+        )
+        fused.append(chunk)
+
+    fused.sort(key=lambda x: x["rrf_score"], reverse=True)
+    # Boost chunks found by both sources
+    both_hits = [c for c in fused if c["source"] == "both"]
+    print(f"  RRF: {len(dense_chunks)} dense + {len(bm25_chunks)} BM25 → {len(fused)} fused ({len(both_hits)} in both)")
+    return fused
+
+# ── CrossEncoder rerank ────────────────────────────────────────────────────────
+def rerank(query: str, chunks: list, keep: int = RERANK_INPUT):
+    if not chunks:
+        return chunks
+    pairs  = [(query, c["text"]) for c in chunks]
+    scores = reranker.predict(pairs)
+    for i, c in enumerate(chunks):
+        c["rerank_score"] = float(scores[i])
+    return sorted(chunks, key=lambda x: x["rerank_score"], reverse=True)[:keep]
+
+# ── MMR diversity ──────────────────────────────────────────────────────────────
+def mmr(query: str, chunks: list, keep: int = FINAL_K, lam: float = MMR_LAMBDA):
+    if len(chunks) <= keep:
+        return chunks
+    texts     = [c["text"] for c in chunks]
+    q_emb     = embedder.encode([query])[0]
+    c_embs    = embedder.encode(texts, show_progress_bar=False)
+    norm      = lambda v: v / (np.linalg.norm(v) or 1)
+    q_emb     = norm(q_emb)
+    c_embs    = [norm(e) for e in c_embs]
+    selected  = []
+    remaining = list(range(len(chunks)))
+    for _ in range(keep):
+        if not remaining: break
+        rel = np.array([c_embs[i].dot(q_emb) for i in remaining])
+        if not selected:
+            best = remaining[int(np.argmax(rel))]
+        else:
+            sel_embs = [c_embs[i] for i in selected]
+            red = np.array([max(c_embs[i].dot(s) for s in sel_embs) for i in remaining])
+            best = remaining[int(np.argmax(lam * rel - (1 - lam) * red))]
+        selected.append(best)
+        remaining.remove(best)
+    return [chunks[i] for i in selected]
+
+# ── Prompt ─────────────────────────────────────────────────────────────────────
+def build_prompt(query: str, chunks: list, memory: str):
+    source_blocks = []
+    for i, c in enumerate(chunks, 1):
+        book_short = c["book"].replace(".pdf","")
+        source_tag = f"{'🔗 ' if c.get('source')=='both' else ''}"
+        source_blocks.append(
+            f"[{i}] «{book_short}» — صفحة {c['page']}:\n{c['text']}"
+        )
+    context    = "\n\n".join(source_blocks)
+    mem_block  = f"\n\nمن محادثات سابقة:\n{memory}" if memory else ""
+
+    return f"""أنت باحث إسلامي متخصص في الفقه والحديث والعلوم الشرعية. أجب على الأسئلة اعتماداً على المصادر المقدمة فقط.
+{mem_block}
+
+المصادر:
+{context}
+
+السؤال: {query}
+
+تعليمات:
+١. حلّل المصادر وحدد الأجزاء المتعلقة بالسؤال مباشرةً.
+٢. أجب بشكل دقيق ومفصل، ووثّق كل معلومة بالمصدر [١][٢][٣].
+٣. إذا تعارضت المصادر، أشر إلى ذلك وقدّم الأقوى دليلاً.
+٤. إن لم تكفِ المصادر للإجابة الكاملة، صرّح بذلك بوضوح.
+٥. الإجابة بالعربية الفصحى.
+
+الإجابة:"""
+
+# ── LLM ────────────────────────────────────────────────────────────────────────
+def call_ollama(prompt):
+    resp = http_requests.post(
+        f"{OLLAMA_URL}/api/generate",
+        json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":False,
+              "options":{"num_predict":700,"temperature":0.2,"top_p":0.9}},
+        timeout=180
+    )
+    return resp.json().get("response","").strip()
+
+def call_claude(prompt, session_id="", query=""):
+    key = get_claude_key()
+    if not key:
+        return call_ollama(prompt)
+    import anthropic
+    c = anthropic.Anthropic(api_key=key)
+    msg = c.messages.create(
+        model="claude-sonnet-4-20250514", max_tokens=1024,
+        messages=[{"role":"user","content":prompt}]
+    )
+    log_claude_usage(session_id, query, msg.usage.input_tokens, msg.usage.output_tokens)
+    return msg.content[0].text.strip()
+
+# ── Main ────────────────────────────────────────────────────────────────────────
+def process_query(query: str, session_id: str = "default"):
+    # 1. Cache
+    cached = get_cache(query)
+    if cached:
+        return {"answer":cached,"source":"cache","chunks":[],"llm":"cache","confidence":1.0,"books_used":[]}
+
+    # 2. Dense + BM25 retrieval (parallel concept, sequential execution)
+    dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
+    bm25_chunks             = retrieve_bm25(query, top_k=BM25_K)
+
+    # 3. RRF fusion
+    fused = rrf_fuse(dense_chunks, bm25_chunks)
+    if not fused:
+        return {"answer":"لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة.",
+                "source":"none","chunks":[],"llm":"none","confidence":0.0,"books_used":[]}
+
+    # 4. Cross-encoder rerank (top RERANK_INPUT from fused)
+    top_fused = fused[:RERANK_INPUT]
+    reranked  = rerank(query, top_fused, keep=RERANK_INPUT)
+
+    # 5. MMR diversity filter → final FINAL_K chunks
+    final_chunks = mmr(query, reranked, keep=FINAL_K)
+
+    # 6. Memory
+    memory = get_memory(session_id, query)
+
+    # 7. Prompt + LLM
+    prompt     = build_prompt(query, final_chunks, memory)
+    use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
+
+    if use_claude:
+        answer   = call_claude(prompt, session_id=session_id, query=query)
+        llm_used = "claude"
+    else:
+        answer   = call_ollama(prompt)
+        llm_used = f"ollama/{OLLAMA_MODEL}"
+
+    # 8. Store
+    books_used = list(set(c["book"] for c in final_chunks))
+    store_memory(session_id, query, answer)
+    store_conversation(session_id, query, answer, llm_used, books_used)
+    set_cache(query, answer)
+
+    return {
+        "answer":       answer,
+        "source":       "hybrid",
+        "llm":          llm_used,
+        "confidence":   top_score,
+        "chunks":       final_chunks,
+        "books_used":   books_used,
+        "retrieval": {
+            "dense":  len(dense_chunks),
+            "bm25":   len(bm25_chunks),
+            "fused":  len(fused),
+            "both":   sum(1 for c in fused if c.get("source") == "both"),
+        }
+    }
+
+def rebuild_bm25():
+    """Called by worker after new ingest."""
+    bm25_index.build(POSTGRES_URL)

celery_app.py

@@ -0,0 +1,38 @@
+"""
+celery_app.py — Celery application instance
+Shared by worker tasks and Flower monitor
+"""
+import os
+from celery import Celery
+
+REDIS_URL      = os.getenv("REDIS_URL",      "redis://localhost:6379")
+REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
+CELERY_BROKER  = os.getenv("CELERY_BROKER_URL", "")
+
+# Build authenticated broker URL if not already set
+if not CELERY_BROKER:
+    if REDIS_PASSWORD and "@" not in REDIS_URL.split("://")[1]:
+        CELERY_BROKER = REDIS_URL.replace("redis://", f"redis://:{REDIS_PASSWORD}@") + "/1"
+    else:
+        CELERY_BROKER = REDIS_URL + "/1"
+
+# Result backend on DB 2 (separate from cache on DB 0 and broker on DB 1)
+RESULT_BACKEND = CELERY_BROKER.replace("/1", "/2")
+
+app = Celery(
+    "rag_worker",
+    broker=CELERY_BROKER,
+    backend=RESULT_BACKEND,
+)
+
+app.conf.update(
+    task_serializer        = "json",
+    result_serializer      = "json",
+    accept_content         = ["json"],
+    task_track_started     = True,
+    task_acks_late         = True,          # re-queue on worker crash
+    worker_prefetch_multiplier = 1,         # one task at a time (PDF heavy)
+    result_expires         = 60 * 60 * 24,  # 24h result retention
+    worker_send_task_events = True,         # needed for Flower
+    task_send_sent_event    = True,
+)

compress.py

@@ -0,0 +1,109 @@
+"""
+compress.py — PDF compression via Ghostscript
+Runs after file lands on disk, before ingest.
+Only replaces original if compressed version is smaller.
+Skips files under MIN_SIZE_KB (already small text PDFs gain nothing).
+"""
+import os, subprocess, logging, shutil
+
+log = logging.getLogger("rag.compress")
+
+# Only attempt compression on files larger than this
+MIN_SIZE_KB    = 200
+# Ghostscript quality preset
+# /screen  = 72 DPI  (smallest, low quality — avoid)
+# /ebook   = 150 DPI (good for text+images, our default)
+# /printer = 300 DPI (high quality, moderate compression)
+GS_QUALITY     = os.getenv("PDF_COMPRESS_QUALITY", "/ebook")
+GS_COMPAT      = "1.4"
+
+
+def _ghostscript_available() -> bool:
+    try:
+        r = subprocess.run(["gs", "--version"], capture_output=True, timeout=5)
+        return r.returncode == 0
+    except FileNotFoundError:
+        return False
+
+
+def compress_pdf(pdf_path: str) -> dict:
+    """
+    Compress a PDF in-place using Ghostscript.
+    Returns a dict with: original_kb, compressed_kb, saved_pct, action
+    action: 'compressed' | 'skipped_small' | 'skipped_larger' | 'error' | 'gs_missing'
+    """
+    if not os.path.exists(pdf_path):
+        return {"action": "error", "detail": "file not found"}
+
+    if not _ghostscript_available():
+        log.warning("Ghostscript not available — skipping compression")
+        return {"action": "gs_missing"}
+
+    original_size = os.path.getsize(pdf_path)
+    original_kb   = original_size // 1024
+
+    # Skip very small files — already optimised
+    if original_kb < MIN_SIZE_KB:
+        log.info(f"Skipping compression (small file {original_kb}KB): {os.path.basename(pdf_path)}")
+        return {"action": "skipped_small", "original_kb": original_kb}
+
+    tmp_path = pdf_path + ".compressed.tmp"
+    try:
+        result = subprocess.run([
+            "gs",
+            "-sDEVICE=pdfwrite",
+            f"-dCompatibilityLevel={GS_COMPAT}",
+            f"-dPDFSETTINGS={GS_QUALITY}",
+            "-dNOPAUSE",
+            "-dQUIET",
+            "-dBATCH",
+            "-dDetectDuplicateImages=true",
+            "-dCompressFonts=true",
+            f"-sOutputFile={tmp_path}",
+            pdf_path,
+        ], capture_output=True, text=True, timeout=300)
+
+        if result.returncode != 0:
+            log.error(f"Ghostscript error for {pdf_path}: {result.stderr[:200]}")
+            return {"action": "error", "detail": result.stderr[:200]}
+
+        if not os.path.exists(tmp_path) or os.path.getsize(tmp_path) == 0:
+            return {"action": "error", "detail": "output file empty"}
+
+        compressed_size = os.path.getsize(tmp_path)
+        compressed_kb   = compressed_size // 1024
+
+        if compressed_size >= original_size:
+            # Compressed is same or larger — keep original
+            os.remove(tmp_path)
+            saved_pct = ((original_size - compressed_size) * 100) // original_size
+            log.info(f"Compression skipped (no benefit {saved_pct}%): {os.path.basename(pdf_path)}")
+            return {
+                "action":        "skipped_larger",
+                "original_kb":   original_kb,
+                "compressed_kb": compressed_kb,
+                "saved_pct":     saved_pct,
+            }
+
+        # Compressed is smaller — replace original
+        shutil.move(tmp_path, pdf_path)
+        saved_pct = ((original_size - compressed_size) * 100) // original_size
+        log.info(f"Compressed {os.path.basename(pdf_path)}: {original_kb}KB → {compressed_kb}KB ({saved_pct}% saved)")
+        return {
+            "action":        "compressed",
+            "original_kb":   original_kb,
+            "compressed_kb": compressed_kb,
+            "saved_pct":     saved_pct,
+        }
+
+    except subprocess.TimeoutExpired:
+        log.error(f"Ghostscript timeout for {pdf_path}")
+        if os.path.exists(tmp_path):
+            os.remove(tmp_path)
+        return {"action": "error", "detail": "timeout after 300s"}
+
+    except Exception as e:
+        log.error(f"Compression error for {pdf_path}: {e}")
+        if os.path.exists(tmp_path):
+            os.remove(tmp_path)
+        return {"action": "error", "detail": str(e)}

ingest.py

@@ -0,0 +1,288 @@
+"""
+ingest.py — PDF ingestion with paragraph-aware chunking
+Improvements over v1:
+  - Paragraph-aware chunking (respects sentence/paragraph boundaries)
+  - Target chunk size: ~150 words, max 200
+  - Minimum chunk quality filter (skip noise, headers, single lines)
+  - Stores cover image (first page as JPEG)
+  - Deduplication by file hash
+"""
+import os, sys, hashlib, uuid, re, shutil
+import fitz                                       # PyMuPDF
+from sentence_transformers import SentenceTransformer
+from qdrant_client import QdrantClient
+from qdrant_client.models import Distance, VectorParams, PointStruct
+
+QDRANT_URL   = os.getenv("QDRANT_URL",   "http://localhost:6333")
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+COLLECTION   = "arabic_books"
+MODEL_NAME   = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
+
+TARGET_WORDS  = 150      # ideal chunk size
+MAX_WORDS     = 200      # hard cap before forced split
+OVERLAP_SENTS = 2        # sentence overlap between chunks
+MIN_CHUNK_LEN = 40       # skip chunks shorter than this (chars)
+
+DELETE_AFTER_INGEST = os.getenv("DELETE_PDF_AFTER_INGEST", "false").lower() == "true"
+COVER_DIR = "/mnt/rag-media/covers"
+os.makedirs(COVER_DIR, exist_ok=True)
+
+print("Loading embedding model...")
+model       = SentenceTransformer(MODEL_NAME)
+VECTOR_SIZE = model.get_embedding_dimension()
+_qdrant_key = os.getenv("QDRANT_API_KEY", "")
+client      = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
+
+collections = [c.name for c in client.get_collections().collections]
+if COLLECTION not in collections:
+    client.create_collection(
+        collection_name=COLLECTION,
+        vectors_config=VectorParams(size=VECTOR_SIZE, distance=Distance.COSINE)
+    )
+    print(f"Created collection: {COLLECTION}")
+
+# ── DB helpers ────────────────────────────────────────────────────────────────
+def get_db():
+    import psycopg2
+    return psycopg2.connect(POSTGRES_URL)
+
+def get_file_hash(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(8192), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+def is_already_ingested(file_hash):
+    """Returns existing filename if hash matches, None otherwise."""
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT filename, title FROM books WHERE file_hash = %s", (file_hash,))
+        row = cur.fetchone(); cur.close(); conn.close()
+        return row[0] if row else None
+    except: return None
+
+def get_existing_by_filename(book_name):
+    """Check if a book with this filename already exists."""
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT id, file_hash FROM books WHERE filename = %s", (book_name,))
+        row = cur.fetchone(); cur.close(); conn.close()
+        return row if row else None
+    except: return None
+
+def register_book(book_name, total_chunks, file_hash):
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute(
+            """INSERT INTO books (filename, total_chunks, file_hash)
+               VALUES (%s, %s, %s)
+               ON CONFLICT (filename) DO UPDATE
+               SET total_chunks = EXCLUDED.total_chunks, file_hash = EXCLUDED.file_hash""",
+            (book_name, total_chunks, file_hash)
+        )
+        conn.commit(); cur.close(); conn.close()
+        print(f"Registered: {book_name}")
+    except Exception as e:
+        print(f"DB registration skipped: {e}")
+
+# ── Cover extraction ──────────────────────────────────────────────────────────
+def extract_cover(pdf_path, book_id):
+    """Render first page of PDF as JPEG, save to COVER_DIR/{book_id}.jpg"""
+    try:
+        doc = fitz.open(pdf_path)
+        page = doc[0]
+        mat  = fitz.Matrix(1.5, 1.5)          # 1.5x zoom → ~120 DPI
+        pix  = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
+        cover_path = os.path.join(COVER_DIR, f"{book_id}.jpg")
+        pix.save(cover_path)
+        doc.close()
+        print(f"  Cover saved: {cover_path}")
+        return cover_path
+    except Exception as e:
+        print(f"  Cover extraction failed: {e}")
+        return None
+
+# ── Paragraph-aware chunking ──────────────────────────────────────────────────
+def split_sentences(text):
+    """
+    Split Arabic/mixed text into sentences.
+    Splits on: period, question mark, exclamation, Arabic full stop (؟، .)
+    """
+    pattern = r'(?<=[.!?؟])\s+|(?<=\n)\s*(?=\S)'
+    parts = re.split(pattern, text.strip())
+    # Also split on paragraph breaks (double newline)
+    sentences = []
+    for part in parts:
+        sub = [s.strip() for s in part.split('\n\n') if s.strip()]
+        sentences.extend(sub)
+    return [s for s in sentences if len(s) > 10]
+
+def chunk_text_smart(text):
+    """
+    Paragraph-aware chunking:
+    1. Split into sentences
+    2. Accumulate sentences until TARGET_WORDS reached
+    3. Add OVERLAP_SENTS from previous chunk for continuity
+    4. Hard-cap at MAX_WORDS — force split at word level if needed
+    """
+    sentences = split_sentences(text)
+    if not sentences:
+        return []
+
+    chunks = []
+    current_sents = []
+    current_words = 0
+
+    for sent in sentences:
+        sent_words = len(sent.split())
+        # If single sentence exceeds max, force-split it
+        if sent_words > MAX_WORDS:
+            # flush current first
+            if current_sents:
+                chunks.append(" ".join(current_sents))
+                current_sents = current_sents[-OVERLAP_SENTS:]
+                current_words = sum(len(s.split()) for s in current_sents)
+            # word-level split for the long sentence
+            words = sent.split()
+            for i in range(0, len(words), TARGET_WORDS - 10):
+                sub = " ".join(words[i:i + TARGET_WORDS])
+                if sub.strip():
+                    chunks.append(sub)
+            continue
+
+        if current_words + sent_words > MAX_WORDS and current_sents:
+            chunks.append(" ".join(current_sents))
+            # carry overlap
+            current_sents = current_sents[-OVERLAP_SENTS:]
+            current_words = sum(len(s.split()) for s in current_sents)
+
+        current_sents.append(sent)
+        current_words += sent_words
+
+    if current_sents:
+        chunks.append(" ".join(current_sents))
+
+    # Filter noise
+    return [c.strip() for c in chunks if len(c.strip()) >= MIN_CHUNK_LEN]
+
+# ── Main ingest ───────────────────────────────────────────────────────────────
+def ingest_pdf(pdf_path):
+    book_name = os.path.basename(pdf_path)
+
+    # Dedup check
+    file_hash = get_file_hash(pdf_path)
+    existing  = is_already_ingested(file_hash)
+    if existing:
+        print(f"SKIP (duplicate): {book_name}")
+        return
+
+    print(f"Processing: {book_name}")
+    doc = fitz.open(pdf_path)
+
+    # Extract cover first
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT id FROM books WHERE filename = %s", (book_name,))
+        row = cur.fetchone(); cur.close(); conn.close()
+        book_id = row[0] if row else book_name.replace('.pdf', '')
+    except:
+        book_id = book_name.replace('.pdf', '')
+
+    extract_cover(pdf_path, book_id)
+
+    # Extract text page by page
+    all_chunks = []
+    for page_num, page in enumerate(doc):
+        text = page.get_text()
+        if not text.strip():
+            continue
+        for chunk in chunk_text_smart(text):
+            all_chunks.append({
+                "text": chunk,
+                "book": book_name,
+                "page": page_num + 1,
+                "word_count": len(chunk.split())
+            })
+
+    doc.close()
+    print(f"  Chunks: {len(all_chunks)} | avg words: {sum(c['word_count'] for c in all_chunks)//max(len(all_chunks),1)}")
+
+    # Get book_id from DB
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("SELECT id FROM books WHERE filename = %s", (book_name,))
+        row = cur.fetchone(); book_id = row[0] if row else None
+        cur.close(); conn.close()
+    except: book_id = None
+
+    # Clear old chunks for this book
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("DELETE FROM chunks WHERE book_filename = %s", (book_name,))
+        conn.commit(); cur.close(); conn.close()
+    except: pass
+
+    # Embed + store in Qdrant AND PostgreSQL
+    batch_size = 32
+    for i in range(0, len(all_chunks), batch_size):
+        batch      = all_chunks[i:i + batch_size]
+        embeddings = model.encode([c["text"] for c in batch], show_progress_bar=False)
+        qdrant_points = []
+        pg_rows       = []
+        for j, c in enumerate(batch):
+            vid = str(uuid.uuid4())
+            qdrant_points.append(PointStruct(
+                id=vid,
+                vector=embeddings[j].tolist(),
+                payload={"text":c["text"],"book":c["book"],"page":c["page"],"word_count":c["word_count"]}
+            ))
+            pg_rows.append((vid, book_id, c["book"], c["page"], c["text"], c["word_count"]))
+        client.upsert(collection_name=COLLECTION, points=qdrant_points)
+        # Store in PostgreSQL for BM25 index
+        try:
+            conn = get_db(); cur = conn.cursor()
+            cur.executemany(
+                "INSERT INTO chunks (qdrant_id,book_id,book_filename,page,text,word_count) VALUES (%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING",
+                pg_rows
+            )
+            conn.commit(); cur.close(); conn.close()
+        except Exception as e:
+            print(f"  PG chunks write skipped: {e}")
+        print(f"  Stored {i+1}–{i+len(batch)}")
+
+    register_book(book_name, len(all_chunks), file_hash)
+
+    # Store PDF
+    pdf_dest = os.path.join("/mnt/rag-media/pdfs", book_name)
+    if pdf_path != pdf_dest:
+        shutil.copy2(pdf_path, pdf_dest)
+        if DELETE_AFTER_INGEST:
+            os.remove(pdf_path)
+
+    file_size = os.path.getsize(pdf_dest)
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute(
+            "UPDATE books SET pdf_path=%s, file_size=%s WHERE filename=%s",
+            (pdf_dest, file_size, book_name)
+        )
+        conn.commit(); cur.close(); conn.close()
+    except Exception as e:
+        print(f"PDF path update skipped: {e}")
+
+    print(f"Done: {book_name} — {len(all_chunks)} chunks stored")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python3 ingest.py <pdf_path_or_folder>")
+        sys.exit(1)
+    path = sys.argv[1]
+    if os.path.isdir(path):
+        for f in sorted(os.listdir(path)):
+            if f.endswith(".pdf"):
+                ingest_pdf(os.path.join(path, f))
+    else:
+        ingest_pdf(path)
+    info = client.get_collection(COLLECTION)
+    print(f"\nTotal vectors: {info.points_count}")

meta_extract.py

@@ -0,0 +1,142 @@
+"""
+meta_extract.py — Shared metadata extraction via Ollama
+Used by both single upload (routes.py) and bulk upload (worker.py)
+"""
+import os, re, json, threading
+import fitz
+import requests as http_req
+
+OLLAMA_URL   = os.getenv("OLLAMA_URL",   "http://localhost:11434")
+OLLAMA_MODEL = "qwen2:7b"
+MEDIA_DIR    = "/mnt/rag-media/pdfs"
+
+
+def extract_meta(pdf_path: str, filename: str) -> dict:
+    """Extract title/author/category/description from PDF first 3 pages via Ollama."""
+    fallback = {
+        "title":       filename.replace(".pdf", "").strip(),
+        "author":      "",
+        "category":    "عام",
+        "description": ""
+    }
+
+    # Read PDF text
+    try:
+        doc  = fitz.open(pdf_path)
+        text = "".join(doc[i].get_text() for i in range(min(3, len(doc))))[:3000]
+        doc.close()
+    except Exception as e:
+        print(f"[meta] fitz error: {e}")
+        text = filename.replace(".pdf", "")
+
+    prompt = (
+        "استخرج من النص التالي معلومات الكتاب وأجب بـ JSON فقط بدون أي نص إضافي:\n"
+        '{"title":"عنوان الكتاب","author":"اسم المؤلف",'
+        '"category":"التصنيف مثل فقه أو حديث أو تفسير أو أدب","description":"وصف مختصر"}\n\n'
+        f"النص:\n{text}"
+    )
+
+    try:
+        r = http_req.post(
+            f"{OLLAMA_URL}/api/generate",
+            json={"model": OLLAMA_MODEL, "prompt": prompt, "stream": False,
+                  "options": {"num_predict": 256, "temperature": 0.1}},
+            timeout=90
+        )
+        raw = r.json().get("response", "")
+        m   = re.search(r'\{[^}]+\}', raw, re.DOTALL)
+        if m:
+            parsed = json.loads(m.group())
+            return {
+                "title":       (parsed.get("title") or fallback["title"]).strip(),
+                "author":      (parsed.get("author") or "").strip(),
+                "category":    (parsed.get("category") or "عام").strip() or "عام",
+                "description": (parsed.get("description") or "").strip(),
+            }
+    except Exception as e:
+        print(f"[meta] Ollama error for {filename}: {e}")
+
+    return fallback
+
+
+def save_meta_to_db(book_filename: str, meta: dict, uploader_id: int = None,
+                    pg_url: str = None):
+    """Update books table with extracted metadata."""
+    import psycopg2
+    if not pg_url:
+        pg_url = os.getenv("POSTGRES_URL", "")
+    try:
+        conn = psycopg2.connect(pg_url)
+        cur  = conn.cursor()
+        if uploader_id:
+            cur.execute("""
+                UPDATE books SET title=%s, author=%s, category=%s,
+                    description=%s, uploader_id=%s
+                WHERE filename=%s
+            """, (meta["title"], meta["author"], meta["category"],
+                  meta["description"], uploader_id, book_filename))
+        else:
+            cur.execute("""
+                UPDATE books SET title=%s, author=%s, category=%s,
+                    description=%s
+                WHERE filename=%s
+            """, (meta["title"], meta["author"], meta["category"],
+                  meta["description"], book_filename))
+        conn.commit(); cur.close(); conn.close()
+        print(f"[meta] saved: {meta['title']} | {meta['author']} | {meta['category']}")
+    except Exception as e:
+        print(f"[meta] DB save failed: {e}")
+
+
+def extract_and_save_bg(book_filename: str, uploader_id: int = None,
+                        pg_url: str = None, _retry: int = 0):
+    """
+    Fire-and-forget: extract metadata and save in background thread.
+    If author is still empty after first extraction, retries once after 5 minutes.
+    Max retries: 2.
+    """
+    pdf_path = os.path.join(MEDIA_DIR, book_filename)
+    if not os.path.exists(pdf_path):
+        print(f"[meta] PDF not found: {pdf_path}")
+        return
+
+    MAX_RETRIES = 2
+    RETRY_DELAY = 300  # 5 minutes
+
+    def _run():
+        meta = extract_meta(pdf_path, book_filename)
+        save_meta_to_db(book_filename, meta, uploader_id=uploader_id, pg_url=pg_url)
+
+        # If author is still empty and we haven't exhausted retries, schedule retry
+        if not meta.get("author") and _retry < MAX_RETRIES:
+            attempt = _retry + 1
+            print(f"[meta] author empty — retry {attempt}/{MAX_RETRIES} in {RETRY_DELAY}s for {book_filename}")
+            timer = threading.Timer(
+                RETRY_DELAY,
+                extract_and_save_bg,
+                kwargs=dict(book_filename=book_filename, uploader_id=uploader_id,
+                            pg_url=pg_url, _retry=attempt)
+            )
+            timer.daemon = True
+            timer.start()
+        elif not meta.get("author"):
+            print(f"[meta] author still empty after {MAX_RETRIES} retries for {book_filename} — admin can set manually")
+
+    t = threading.Thread(target=_run, daemon=True)
+    t.start()
+    return t
+
+
+def extract_and_save_sync(book_filename: str, uploader_id: int = None, pg_url: str = None):
+    """Blocking version — for use in worker subprocess context."""
+    pdf_path = os.path.join(MEDIA_DIR, book_filename)
+    if not os.path.exists(pdf_path):
+        # Try as-is (full path)
+        if os.path.exists(book_filename):
+            pdf_path = book_filename
+        else:
+            print(f"[meta] PDF not found: {book_filename}")
+            return None
+    meta = extract_meta(pdf_path, book_filename)
+    save_meta_to_db(os.path.basename(book_filename), meta, uploader_id=uploader_id, pg_url=pg_url)
+    return meta

routes.py

@@ -0,0 +1,657 @@
+from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Request
+from pydantic import BaseModel, EmailStr
+from typing import Optional, List
+from psycopg2.extras import RealDictCursor
+import os, shutil, subprocess, uuid, json
+
+from meta_extract import extract_and_save_bg as _meta_bg
+try:
+    from typesense_client import search_books as _ts_search, index_book as _ts_index_book, delete_book as _ts_delete_book
+    _TYPESENSE_OK = True
+except Exception:
+    _TYPESENSE_OK = False
+    def _ts_search(q, **kw): return []
+    def _ts_index_book(bid): pass
+    def _ts_delete_book(bid): pass
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+from fastapi import Request as _Request
+limiter = Limiter(key_func=get_remote_address)
+from auth import (
+    get_db, hash_password, verify_password, create_token,
+    get_current_user, get_current_user_optional,
+    require_admin, require_upload
+)
+
+router = APIRouter()
+
+# ── Duplicate detection ───────────────────────────────────────────────────────
+def _file_hash(data: bytes) -> str:
+    import hashlib
+    return hashlib.sha256(data).hexdigest()
+
+def _find_duplicate(file_hash: str, filename: str) -> dict | None:
+    """Return existing book record if hash OR filename already exists, else None."""
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT id, filename, title, author, category, uploaded_at
+                FROM books
+                WHERE file_hash = %s OR filename = %s
+                LIMIT 1
+            """, (file_hash, filename))
+            row = cur.fetchone()
+    return dict(row) if row else None
+BOOKS_DIR = "/mnt/rag-media/pdfs"
+API_KEY   = os.getenv("RAG_API_KEY", "")
+
+# ── Models ────────────────────────────────────────────────────────────────────
+class RegisterRequest(BaseModel):
+    name: str
+    email: str
+    password: str
+    phone: Optional[str] = None
+    country_code: Optional[str] = None
+    country_name: Optional[str] = None
+
+class LoginRequest(BaseModel):
+    email: str
+    password: str
+
+class UpdateRoleRequest(BaseModel):
+    role: str
+
+class BookMetaUpdate(BaseModel):
+    title: Optional[str] = None
+    author: Optional[str] = None
+    description: Optional[str] = None
+    category: Optional[str] = None
+    language: Optional[str] = None
+    is_public: Optional[bool] = None
+
+# ── Auth ──────────────────────────────────────────────────────────────────────
+@router.post("/auth/register")
+def register(req: RegisterRequest):
+    if len(req.password) < 6:
+        raise HTTPException(400, "Password must be at least 6 characters")
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT id FROM users WHERE email = %s", (req.email.lower(),))
+            if cur.fetchone():
+                raise HTTPException(400, "Email already registered")
+            cur.execute("""
+                INSERT INTO users (name, email, password_hash, phone, country_code, country_name, role)
+                VALUES (%s, %s, %s, %s, %s, %s, 'reader') RETURNING id, role
+            """, (req.name, req.email.lower(), hash_password(req.password),
+                  req.phone, req.country_code, req.country_name))
+            user = cur.fetchone()
+        conn.commit()
+    token = create_token(user[0], user[1])
+    return {"token": token, "user_id": user[0], "role": user[1], "name": req.name}
+
+@router.post("/auth/login")
+@limiter.limit("10/minute")
+def login(request: Request, req: LoginRequest):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT * FROM users WHERE email = %s AND is_active = TRUE", (req.email.lower(),))
+            user = cur.fetchone()
+    if not user or not verify_password(req.password, user["password_hash"]):
+        raise HTTPException(401, "Invalid email or password")
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("UPDATE users SET last_login = NOW() WHERE id = %s", (user["id"],))
+        conn.commit()
+    token = create_token(user["id"], user["role"])
+    # Track session in sessions table
+    try:
+        import hashlib, datetime
+        token_hash = hashlib.sha256(token.encode()).hexdigest()
+        expires_at = datetime.datetime.utcnow() + datetime.timedelta(days=7)
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("""INSERT INTO sessions (user_id, token_hash, token, expires_at)
+                    VALUES (%s, %s, %s, %s) ON CONFLICT (token_hash) DO NOTHING""",
+                    (user["id"], token_hash, token, expires_at))
+            conn.commit()
+    except: pass
+    return {
+        "token": token,
+        "user": {"id": user["id"], "name": user["name"], "email": user["email"], "role": user["role"]}
+    }
+
+@router.get("/auth/me")
+def me(user: dict = Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT id,name,email,phone,country_code,country_name,role,created_at,last_login FROM users WHERE id=%s", (user["id"],))
+            return dict(cur.fetchone())
+
+# ── Admin: User Management ────────────────────────────────────────────────────
+@router.get("/admin/users")
+def list_users(_=Depends(require_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT id,name,email,phone,country_name,role,is_active,created_at,last_login FROM users ORDER BY created_at DESC")
+            return {"users": [dict(r) for r in cur.fetchall()]}
+
+@router.patch("/admin/users/{user_id}/role")
+def update_role(user_id: int, req: UpdateRoleRequest, _=Depends(require_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT name FROM roles WHERE name = %s", (req.role,))
+            if not cur.fetchone():
+                raise HTTPException(400, f"Invalid role: {req.role}")
+            cur.execute("UPDATE users SET role = %s WHERE id = %s", (req.role, user_id))
+        conn.commit()
+    return {"status": "updated", "user_id": user_id, "role": req.role}
+
+@router.patch("/admin/users/{user_id}/status")
+def toggle_user(user_id: int, _=Depends(require_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("UPDATE users SET is_active = NOT is_active WHERE id = %s RETURNING is_active", (user_id,))
+            result = cur.fetchone()
+        conn.commit()
+    return {"status": "updated", "is_active": result[0]}
+
+@router.delete("/admin/users/{user_id}")
+def delete_user(user_id: int, _=Depends(require_admin)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("DELETE FROM users WHERE id = %s", (user_id,))
+        conn.commit()
+    return {"status": "deleted"}
+
+# ── Roles ─────────────────────────────────────────────────────────────────────
+@router.get("/admin/roles")
+def list_roles(_=Depends(require_admin)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT * FROM roles ORDER BY name")
+            return {"roles": [dict(r) for r in cur.fetchall()]}
+
+# ── Library: Public ───────────────────────────────────────────────────────────
+@router.get("/api/library")
+def get_library(
+    category: str = None,
+    author: str = None,
+    language: str = None,
+    search: str = None,
+    sort: str = "newest",
+    page: int = 1,
+    limit: int = 20,
+    user=Depends(get_current_user_optional)
+):
+    sort_map = {
+        "newest":     "b.uploaded_at DESC",
+        "oldest":     "b.uploaded_at ASC",
+        "title_az":   "b.title ASC",
+        "title_za":   "b.title DESC",
+        "chunks_desc":"b.total_chunks DESC",
+    }
+    order = sort_map.get(sort, "b.uploaded_at DESC")
+    where = ["is_public = TRUE"]
+    params = []
+    if category:
+        where.append("category = %s"); params.append(category)
+    if author:
+        where.append("author ILIKE %s"); params.append(f"%{author}%")
+    if language:
+        where.append("language = %s"); params.append(language)
+    if search:
+        where.append("(filename ILIKE %s OR title ILIKE %s OR author ILIKE %s)")
+        params.extend([f"%{search}%", f"%{search}%", f"%{search}%"])
+    offset = (page - 1) * limit
+    params_count = params.copy()
+    params.extend([limit, offset])
+    where_sql = " AND ".join(where)
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute(f"SELECT COUNT(*) FROM books WHERE {where_sql}", params_count)
+            total = cur.fetchone()["count"]
+            cur.execute(f"""
+                SELECT b.id, b.filename, b.title, b.author, b.category, b.language,
+                       b.description, b.cover_url, b.total_chunks, b.uploaded_at,
+                       u.name as uploader_name
+                FROM books b
+                LEFT JOIN users u ON b.uploader_id = u.id
+                WHERE {where_sql}
+                ORDER BY {order}
+                LIMIT %s OFFSET %s
+            """, params)
+            books = [dict(r) for r in cur.fetchall()]
+    return {"books": books, "total": total, "page": page, "pages": -(-total // limit)}
+
+@router.get("/api/library/categories")
+def get_categories():
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT DISTINCT category, COUNT(*) as cnt FROM books WHERE is_public=TRUE AND category IS NOT NULL GROUP BY category ORDER BY cnt DESC")
+            return {"categories": [{"name": r[0], "count": r[1]} for r in cur.fetchall()]}
+
+@router.get("/api/library/{book_id}")
+def get_book(book_id: int):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT b.*, u.name as uploader_name
+                FROM books b LEFT JOIN users u ON b.uploader_id = u.id
+                WHERE b.id = %s AND b.is_public = TRUE
+            """, (book_id,))
+            book = cur.fetchone()
+    if not book:
+        raise HTTPException(404, "Book not found")
+    return dict(book)
+
+# ── Library: Upload (librarian/admin only) ────────────────────────────────────
+@router.post("/api/library/upload")
+async def library_upload(
+    file: UploadFile = File(...),
+    title: str = None,
+    author: str = None,
+    category: str = "",
+    language: str = "ar",
+    description: str = None,
+    user: dict = Depends(require_upload)
+):
+    safe_name = _safe_filename(file.filename)
+    if not safe_name.lower().endswith(".pdf"):
+        raise HTTPException(400, "Only PDF files allowed")
+
+    # File size limit: 50MB
+    file_bytes = await file.read()
+    if len(file_bytes) > 50 * 1024 * 1024:
+        raise HTTPException(413, "File too large — maximum 50MB")
+
+    fhash = _file_hash(file_bytes)
+    # Use sanitized name
+    file_filename_orig = file.filename
+    file.filename = safe_name
+
+    # Duplicate check BEFORE saving to disk
+    existing = _find_duplicate(fhash, file.filename)
+    if existing:
+        raise HTTPException(409, {
+            "error":    "duplicate",
+            "message":  "هذا الكتاب موجود مسبقاً في المكتبة",
+            "book": {
+                "id":       existing["id"],
+                "title":    existing.get("title") or existing["filename"],
+                "author":   existing.get("author") or "",
+                "category": existing.get("category") or "",
+                "uploaded_at": str(existing.get("uploaded_at") or ""),
+            }
+        })
+
+    # Not a duplicate — save to disk
+    dest = os.path.join(BOOKS_DIR, file.filename)
+    with open(dest, "wb") as f:
+        f.write(file_bytes)
+
+    # Create a single-file job and dispatch to Celery
+    # (compress → ingest → meta → R2 → Typesense all run async in worker)
+    import uuid as _uuid
+    job_id = str(_uuid.uuid4())
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("""INSERT INTO bulk_jobs (id, uploader_id, status, total)
+                VALUES (%s, %s, 'pending', 1)""", (job_id, user["id"]))
+            cur.execute("""INSERT INTO bulk_job_files (job_id, filename, status)
+                VALUES (%s, %s, 'queued')""", (job_id, file.filename))
+            cur.execute("""INSERT INTO books (filename, title, author, category, language,
+                    description, uploader_id, file_hash)
+                VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
+                ON CONFLICT (filename) DO UPDATE
+                SET title=EXCLUDED.title, author=EXCLUDED.author,
+                    category=EXCLUDED.category, uploader_id=EXCLUDED.uploader_id,
+                    file_hash=EXCLUDED.file_hash""",
+                (file.filename, title or file.filename.replace(".pdf",""),
+                 author or "", category or "", language, description,
+                 user["id"], fhash))
+        conn.commit()
+
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT id FROM bulk_job_files WHERE job_id=%s LIMIT 1", (job_id,))
+            file_row = cur.fetchone()
+
+    try:
+        from tasks import process_bulk_file
+        process_bulk_file.delay(job_id, file_row["id"], file.filename, user["id"])
+    except Exception as _ce:
+        with get_db() as conn:
+            with conn.cursor() as cur:
+                cur.execute("UPDATE bulk_jobs SET status='error' WHERE id=%s", (job_id,))
+            conn.commit()
+        raise HTTPException(503, f"Task queue unavailable: {_ce}")
+
+    return {
+        "status":     "queued",
+        "file":       file.filename,
+        "job_id":     job_id,
+        "status_url": f"/api/library/bulk-status/{job_id}",
+    }
+
+@router.patch("/api/library/{book_id}")
+def update_book(book_id: int, data: BookMetaUpdate, user: dict = Depends(require_upload)):
+    ALLOWED_FIELDS = {"title", "author", "description", "category", "language", "is_public"}
+    updates = {k: v for k, v in data.dict().items() if v is not None and k in ALLOWED_FIELDS}
+    if not updates:
+        raise HTTPException(400, "No fields to update")
+    set_sql = ", ".join([f"{k}=%s" for k in updates])  # keys from allowlist only
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(f"UPDATE books SET {set_sql} WHERE id=%s", list(updates.values()) + [book_id])
+        conn.commit()
+    # Sync updated metadata to Typesense
+    try:
+        _ts_index_book(book_id)
+    except Exception as _e:
+        print(f"[update_book] Typesense sync failed: {_e}")
+    return {"status": "updated"}
+
+# ── Favorites ─────────────────────────────────────────────────────────────────
+@router.post("/api/library/{book_id}/favorite")
+def toggle_favorite(book_id: int, user: dict = Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT id FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
+            exists = cur.fetchone()
+            if exists:
+                cur.execute("DELETE FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
+                action = "removed"
+            else:
+                cur.execute("INSERT INTO favorites (user_id, book_id) VALUES (%s,%s)", (user["id"], book_id))
+                action = "added"
+        conn.commit()
+    return {"status": action}
+
+@router.get("/api/library/favorites")
+def get_favorites(user: dict = Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT b.id, b.filename, b.title, b.author, b.category, b.total_chunks, b.uploaded_at
+                FROM favorites f JOIN books b ON b.id=f.book_id
+                WHERE f.user_id=%s ORDER BY f.created_at DESC
+            """, (user["id"],))
+            return {"favorites": [dict(r) for r in cur.fetchall()]}
+
+@router.get("/api/library/my-favorites")
+def my_favorite_ids(user: dict = Depends(get_current_user)):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT book_id FROM favorites WHERE user_id=%s", (user["id"],))
+            return {"ids": [r[0] for r in cur.fetchall()]}
+
+# ── Download & Read ───────────────────────────────────────────────────────────
+@router.get("/api/library/{book_id}/download")
+def download_book(book_id: int):
+    import os
+    from urllib.parse import quote
+    from fastapi.responses import StreamingResponse, RedirectResponse
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
+            book = cur.fetchone()
+    if not book:
+        raise HTTPException(404, "Book not found")
+
+    # R2 configured → redirect, zero VPS bandwidth used
+    r2_public = os.getenv("R2_PUBLIC_URL", "").rstrip("/")
+    if r2_public:
+        return RedirectResponse(
+            url=f"{r2_public}/pdfs/{book['filename']}",
+            status_code=302
+        )
+
+    # Fallback: stream from VPS disk
+    path = book.get("pdf_path") or os.path.join(BOOKS_DIR, book["filename"])
+    if not os.path.exists(path):
+        raise HTTPException(404, "PDF file not available. Please re-upload this book.")
+    safe_name = quote(book["filename"], safe="")
+    def stream():
+        with open(path, "rb") as f: yield from f
+    return StreamingResponse(stream(), media_type="application/pdf", headers={
+        "Content-Disposition": f"attachment; filename*=UTF-8''{safe_name}",
+        "Content-Length": str(os.path.getsize(path)), "Cache-Control": "private, max-age=3600"
+    })
+
+@router.get("/api/library/{book_id}/read")
+def read_book(book_id: int):
+    import os
+    from urllib.parse import quote
+    from fastapi.responses import StreamingResponse, RedirectResponse
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
+            book = cur.fetchone()
+    if not book:
+        raise HTTPException(404, "Book not found")
+
+    # R2 configured → redirect to R2 (reader iframe follows the redirect automatically)
+    r2_public = os.getenv("R2_PUBLIC_URL", "").rstrip("/")
+    if r2_public:
+        return RedirectResponse(
+            url=f"{r2_public}/pdfs/{book['filename']}",
+            status_code=302
+        )
+
+    # Fallback: stream from VPS disk
+    path = book.get("pdf_path") or os.path.join(BOOKS_DIR, book["filename"])
+    if not os.path.exists(path):
+        raise HTTPException(404, "PDF file not available. Please re-upload this book.")
+    safe_name = quote(book["filename"], safe="")
+    def stream():
+        with open(path, "rb") as f: yield from f
+    return StreamingResponse(stream(), media_type="application/pdf", headers={
+        "Content-Disposition": f"inline; filename*=UTF-8''{safe_name}",
+        "Content-Length": str(os.path.getsize(path)), "Cache-Control": "private, max-age=3600"
+    })
+
+
+# ── Bulk Upload ───────────────────────────────────────────────────────────────
+
+@router.post("/api/library/bulk-upload")
+async def bulk_upload(
+    files: List[UploadFile] = File(...),
+    user: dict = Depends(require_upload)
+):
+    """Accept multiple PDFs, save to disk, queue for background processing."""
+    if not files:
+        raise HTTPException(400, "No files provided")
+
+    job_id = str(uuid.uuid4())
+    saved = []
+    skipped = []
+
+    for file in files:
+        safe_name = _safe_filename(file.filename)
+        if not safe_name.lower().endswith(".pdf"):
+            skipped.append({"filename": file.filename, "reason": "ليس ملف PDF"})
+            continue
+
+        try:
+            file_bytes = await file.read()
+        except Exception as e:
+            skipped.append({"filename": file.filename, "reason": f"خطأ في القراءة: {e}"})
+            continue
+
+        # File size limit: 50MB per file
+        if len(file_bytes) > 50 * 1024 * 1024:
+            skipped.append({"filename": file.filename, "reason": "الملف أكبر من 50MB"})
+            continue
+
+        file.filename = safe_name
+
+        fhash    = _file_hash(file_bytes)
+        existing = _find_duplicate(fhash, file.filename)
+        if existing:
+            orig_title = existing.get("title") or existing["filename"]
+            skipped.append({
+                "filename": file.filename,
+                "reason":   "مكرر",
+                "duplicate_of": orig_title,
+                "existing_id":  existing["id"],
+            })
+            continue
+
+        dest = os.path.join(BOOKS_DIR, file.filename)
+        try:
+            with open(dest, "wb") as f:
+                f.write(file_bytes)
+            saved.append(file.filename)
+        except Exception as e:
+            skipped.append({"filename": file.filename, "reason": str(e)})
+
+    if not saved:
+        # All files were either invalid or duplicates — return details
+        raise HTTPException(400, {
+            "error":   "all_duplicates",
+            "message": "جميع الملفات موجودة مسبقاً في المكتبة",
+            "skipped": skipped,
+        })
+
+    # Create job + file records in DB (include skipped duplicates for visibility)
+    total_for_job = len(saved) + len([s for s in skipped if s.get("reason") == "مكرر"])
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute("""INSERT INTO bulk_jobs (id, uploader_id, status, total)
+                VALUES (%s, %s, 'pending', %s)""",
+                (job_id, user["id"], len(saved)))
+            for filename in saved:
+                cur.execute("""INSERT INTO bulk_job_files (job_id, filename, status)
+                    VALUES (%s, %s, 'queued')""", (job_id, filename))
+            # Add skipped duplicates as 'skipped' status so UI can show them
+            for skip in skipped:
+                if skip.get("reason") == "مكرر":
+                    cur.execute("""INSERT INTO bulk_job_files (job_id, filename, status, error)
+                        VALUES (%s, %s, 'skipped', %s)""",
+                        (job_id, skip["filename"],
+                         f"مكرر: {skip.get('duplicate_of', '')}"))
+        conn.commit()
+
+    # Dispatch each saved file as a Celery task
+    if saved:
+        try:
+            from tasks import process_bulk_file
+            _db2 = get_db()
+            with _db2.cursor(cursor_factory=RealDictCursor) as _cur2:
+                _cur2.execute(
+                    "SELECT id, filename FROM bulk_job_files WHERE job_id=%s AND status='queued'",
+                    (job_id,)
+                )
+                _qfiles = _cur2.fetchall()
+            _db2.close()
+            for _qf in _qfiles:
+                process_bulk_file.delay(job_id, _qf["id"], _qf["filename"], user["id"])
+        except Exception as _ce:
+            print(f"Celery dispatch failed, using threading worker: {_ce}")
+
+    return {
+        "job_id":   job_id,
+        "queued":   len(saved),
+        "skipped":  skipped,
+        "status_url": f"/api/library/bulk-status/{job_id}"
+    }
+
+
+@router.get("/api/library/bulk-status/{job_id}")
+def bulk_status(job_id: str, user: dict = Depends(require_upload)):
+    """Poll job progress."""
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT * FROM bulk_jobs WHERE id=%s", (job_id,))
+            job = cur.fetchone()
+            if not job:
+                raise HTTPException(404, "Job not found")
+            cur.execute("""SELECT id, filename, status, meta, error, book_id, updated_at
+                FROM bulk_job_files WHERE job_id=%s ORDER BY id""", (job_id,))
+            files = cur.fetchall()
+    return {
+        "job_id":  job_id,
+        "status":  job["status"],
+        "total":   job["total"],
+        "done":    job["done"],
+        "failed":  job["failed"],
+        "files":   [dict(f) for f in files]
+    }
+
+
+@router.patch("/api/library/bulk-file/{file_id}")
+def update_bulk_file_meta(file_id: int, data: BookMetaUpdate, user: dict = Depends(require_upload)):
+    """Allow user to edit extracted metadata before it's finalised."""
+    ALLOWED_FIELDS = {"title", "author", "description", "category", "language"}
+    updates = {k: v for k, v in data.dict().items() if v is not None and k in ALLOWED_FIELDS}
+    if not updates:
+        raise HTTPException(400, "No fields to update")
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT book_id FROM bulk_job_files WHERE id=%s", (file_id,))
+            row = cur.fetchone()
+            if not row:
+                raise HTTPException(404, "File record not found")
+            if row["book_id"]:
+                # Update the actual books table — keys from allowlist only
+                set_clause = ", ".join(f"{k}=%s" for k in updates)
+                cur.execute(f"UPDATE books SET {set_clause} WHERE id=%s",
+                    (*updates.values(), row["book_id"]))
+            # Update the meta snapshot in bulk_job_files too
+            cur.execute("UPDATE bulk_job_files SET meta=%s WHERE id=%s",
+                (json.dumps(updates), file_id))
+        conn.commit()
+    return {"status": "updated"}
+
+@router.get("/library/{book_id}/cover")
+def serve_cover(book_id: int):
+    """Serve book cover image. Generates on-demand if missing."""
+    import os
+    from fastapi.responses import FileResponse, Response
+    from psycopg2.extras import RealDictCursor
+
+    COVER_DIR = "/mnt/rag-media/covers"
+    cover_path = os.path.join(COVER_DIR, f"{book_id}.jpg")
+
+    # R2 configured → redirect to R2 cover
+    r2_public = os.getenv("R2_PUBLIC_URL", "").rstrip("/")
+    if r2_public:
+        from fastapi.responses import RedirectResponse
+        return RedirectResponse(
+            url=f"{r2_public}/covers/{book_id}.jpg",
+            status_code=302,
+            headers={"Cache-Control": "public, max-age=86400"}
+        )
+
+    # Already generated locally
+    if os.path.exists(cover_path):
+        return FileResponse(cover_path, media_type="image/jpeg",
+                           headers={"Cache-Control": "public, max-age=86400"})
+
+    # Generate on-demand from PDF
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT pdf_path, filename FROM books WHERE id = %s", (book_id,))
+            book = cur.fetchone()
+
+    if not book or not book.get("pdf_path"):
+        raise HTTPException(404, "No PDF found for this book")
+
+    pdf_path = book["pdf_path"]
+    if not os.path.exists(pdf_path):
+        raise HTTPException(404, "PDF file not found on disk")
+
+    try:
+        import fitz
+        doc  = fitz.open(pdf_path)
+        page = doc[0]
+        mat  = fitz.Matrix(1.5, 1.5)
+        pix  = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
+        os.makedirs(COVER_DIR, exist_ok=True)
+        pix.save(cover_path)
+        doc.close()
+        return FileResponse(cover_path, media_type="image/jpeg",
+                           headers={"Cache-Control": "public, max-age=86400"})
+    except Exception as e:
+        raise HTTPException(500, f"Cover generation failed: {e}")
+

static/index.html

@@ -0,0 +1,548 @@
+<!DOCTYPE html>
+<html class="dark" lang="ar" dir="rtl">
+<head>
+<meta charset="UTF-8"/>
+<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+<title>AfakZaid RAG</title>
+<script src="https://cdn.tailwindcss.com?plugins=forms"></script>
+<link href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700;800&display=swap" rel="stylesheet"/>
+<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:wght,FILL@100..700,0..1&display=swap" rel="stylesheet"/>
+<script>tailwind.config={darkMode:"class",theme:{extend:{colors:{"primary":"#4edea3","surface":"#0c1324","surface-low":"#151b2d","surface-container":"#191f31","surface-high":"#23293c","on-surface":"#dce1fb","on-surface-variant":"#bbcabf","outline-variant":"#3c4a42","secondary":"#95d3ba","tertiary":"#45dfa4"},fontFamily:{sans:["Manrope","sans-serif"]}}}}</script>
+<style>
+/* ── Emil design vars ─────────────────────────────────────────────────────── */
+:root {
+  --ease-out:    cubic-bezier(0.23, 1, 0.32, 1);
+  --ease-in-out: cubic-bezier(0.77, 0, 0.175, 1);
+  --ease-drawer: cubic-bezier(0.32, 0.72, 0, 1);
+}
+
+*{box-sizing:border-box;margin:0;padding:0}
+body{background:radial-gradient(circle at 50% -20%,#151b2d 0%,#0c1324 100%);font-family:'Manrope',sans-serif;color:#dce1fb;min-height:100vh;overflow-x:hidden}
+.glass{background:rgba(21,27,45,0.5);backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border:1px solid rgba(78,222,163,0.12)}
+
+/* Exact properties — never transition:all */
+.glass-hover {
+  transition:background 200ms ease, border-color 200ms ease;
+}
+@media (hover: hover) and (pointer: fine) {
+  .glass-hover:hover{background:rgba(78,222,163,0.06);border-color:rgba(78,222,163,0.25)}
+}
+
+.glow{box-shadow:0 0 20px rgba(78,222,163,0.15)}
+.glow-sm{box-shadow:0 0 10px rgba(78,222,163,0.1)}
+.msym{font-family:'Material Symbols Outlined';font-variation-settings:'FILL' 0,'wght' 400,'GRAD' 0,'opsz' 24;vertical-align:middle;line-height:1}
+
+/* Upload zone — specific props, no transform on touch */
+.upload-zone{
+  border:2px dashed rgba(78,222,163,0.2);
+  transition:border-color 200ms ease, background 200ms ease;
+}
+.upload-zone.dragover{border-color:rgba(78,222,163,0.6);background:rgba(78,222,163,0.05)}
+@media (hover: hover) and (pointer: fine) {
+  .upload-zone:hover{border-color:rgba(78,222,163,0.4);background:rgba(78,222,163,0.03)}
+}
+
+.scrollbar::-webkit-scrollbar{width:4px}
+.scrollbar::-webkit-scrollbar-thumb{background:rgba(78,222,163,0.2);border-radius:4px}
+
+/* ── Entry keyframes ──────────────────────────────────────────────────────── */
+/* Displacement: 8px max — subtle, professional */
+@keyframes fadeUp {
+  from { opacity:0; transform:translateY(8px); }
+  to   { opacity:1; transform:translateY(0); }
+}
+@keyframes fadeIn {
+  from { opacity:0; }
+  to   { opacity:1; }
+}
+@keyframes slideDown {
+  from { opacity:0; transform:translateY(-8px); }
+  to   { opacity:1; transform:translateY(0); }
+}
+/* Panels: 16px — noticeable but not theatrical */
+@keyframes slideRight {
+  from { opacity:0; transform:translateX(16px); }
+  to   { opacity:1; transform:translateX(0); }
+}
+@keyframes slideLeft {
+  from { opacity:0; transform:translateX(-16px); }
+  to   { opacity:1; transform:translateX(0); }
+}
+/* Scale from 0.95 not 0.88 — nothing appears from nothing */
+@keyframes scalePop {
+  0%   { opacity:0; transform:scale(0.95); }
+  100% { opacity:1; transform:scale(1); }
+}
+/* Answer box entry */
+@keyframes fadeUpFast {
+  from { opacity:0; transform:translateY(6px); }
+  to   { opacity:1; transform:translateY(0); }
+}
+
+/* ── Ambient orb — slow linear drift, off main thread ────────────────────── */
+@keyframes orbDrift {
+  0%,100% { transform:translate(0,0) scale(1); }
+  50%     { transform:translate(-8px,10px) scale(1.05); }
+}
+@keyframes orbDrift2 {
+  0%,100% { transform:translate(0,0) scale(1); }
+  50%     { transform:translate(10px,-8px) scale(0.97); }
+}
+
+/* ── Scan line on brand icon — runs once on load, then stops ─────────────── */
+@keyframes scanLine {
+  0%   { top:-100%; opacity:.5; }
+  100% { top:200%; opacity:0; }
+}
+
+/* ── Scroll reveal ────────────────────────────────────────────────────────── */
+.reveal {
+  opacity:0;
+  transform:translateY(8px);
+  /* ease-out: entering element, starts fast */
+  transition:opacity 400ms var(--ease-out), transform 400ms var(--ease-out);
+}
+.reveal.visible { opacity:1; transform:translateY(0); }
+
+/* ── Entry sequencing ────────────────────────────────────────────────────── */
+/* Header: instant, just a subtle drop */
+header { animation:slideDown 350ms var(--ease-out) both; }
+
+/* Stats: stagger 50ms — Emil says 30-80ms */
+.stat-card { animation:scalePop 400ms var(--ease-out) both; }
+.stat-card:nth-child(1) { animation-delay: 80ms; }
+.stat-card:nth-child(2) { animation-delay:130ms; }
+.stat-card:nth-child(3) { animation-delay:180ms; }
+.stat-card:nth-child(4) { animation-delay:230ms; }
+
+/* Panels enter slightly later */
+.query-card       { animation:slideLeft  450ms var(--ease-out) 300ms both; }
+.upload-card-wrap { animation:slideRight 450ms var(--ease-out) 340ms both; }
+.books-card       { animation:fadeUp     400ms var(--ease-out) 440ms both; }
+
+/* ── Stat card — hover only on pointer devices ───────────────────────────── */
+.stat-card {
+  cursor:default;
+  will-change:transform;
+}
+.stat-num {
+  font-variant-numeric:tabular-nums;
+  transition:color 200ms ease;
+}
+@media (hover: hover) and (pointer: fine) {
+  .stat-card {
+    transition:transform 200ms var(--ease-out), box-shadow 200ms ease;
+  }
+  .stat-card:hover {
+    transform:translateY(-3px) scale(1.02);
+    box-shadow:0 8px 24px rgba(78,222,163,0.14);
+  }
+  .stat-card:hover .stat-num { color:#fff; }
+}
+/* Active state — works on all devices */
+.stat-card:active { transform:scale(0.97); }
+
+/* ── Book cards ──────────────────────────────────────────────────────────── */
+.book-card {
+  will-change:transform;
+}
+@media (hover: hover) and (pointer: fine) {
+  .book-card {
+    transition:transform 200ms var(--ease-out), box-shadow 200ms ease, border-color 200ms ease;
+  }
+  .book-card:hover {
+    transform:translateY(-2px);
+    box-shadow:0 6px 20px rgba(78,222,163,0.12);
+    border-color:rgba(78,222,163,0.28) !important;
+  }
+}
+.book-card:active { transform:scale(0.98); }
+
+/* ── Query button ────────────────────────────────────────────────────────── */
+#query-btn {
+  /* No constant animation — button is used frequently */
+  transition:transform 150ms var(--ease-out), box-shadow 200ms ease, filter 200ms ease;
+  position:relative; overflow:hidden;
+}
+/* Glow only on focus/hover — not constantly */
+@media (hover: hover) and (pointer: fine) {
+  #query-btn:hover {
+    transform:scale(1.015);
+    box-shadow:0 0 28px rgba(78,222,163,0.45);
+    filter:brightness(1.08);
+  }
+}
+#query-btn:focus-visible {
+  box-shadow:0 0 0 3px rgba(78,222,163,0.4);
+  outline:none;
+}
+/* Active: fast press feedback — Emil says 100-160ms */
+#query-btn:active { transform:scale(0.97); transition-duration:100ms; }
+
+/* Shimmer on hover */
+#query-btn::after {
+  content:'';
+  position:absolute; inset:0;
+  background:linear-gradient(90deg, transparent 0%, rgba(255,255,255,.1) 50%, transparent 100%);
+  transform:translateX(-100%);
+  transition:transform 380ms var(--ease-in-out);
+  pointer-events:none;
+}
+@media (hover: hover) and (pointer: fine) {
+  #query-btn:hover::after { transform:translateX(100%); }
+}
+
+/* ── Brand icon scan line — fires once on load ───────────────────────────── */
+.brand-icon-wrap { position:relative; overflow:hidden; }
+.brand-icon-wrap::after {
+  content:'';
+  position:absolute; left:0; right:0; height:30%;
+  background:linear-gradient(to bottom, transparent, rgba(78,222,163,.25), transparent);
+  animation:scanLine 1.8s var(--ease-in-out) 0.6s 1 forwards;
+  opacity:0;
+}
+
+/* ── Answer reveal ───────────────────────────────────────────────────────── */
+#answer-wrap { animation:fadeUpFast 280ms var(--ease-out) both; }
+
+/* ── Textarea ────────────────────────────────────────────────────────────── */
+#query-input {
+  transition:border-color 200ms ease, box-shadow 200ms ease;
+}
+#query-input:focus { box-shadow:0 0 0 2px rgba(78,222,163,.12); }
+
+/* ── Orbs: slow linear, off main thread ─────────────────────────────────── */
+.orb-1 { animation:orbDrift  18s linear infinite; }
+.orb-2 { animation:orbDrift2 24s linear infinite 3s; }
+
+/* ── prefers-reduced-motion — keep opacity fades, remove transforms ──────── */
+@media (prefers-reduced-motion: reduce) {
+  *,*::before,*::after {
+    animation-duration:.01ms !important;
+    animation-iteration-count:1 !important;
+    transition-duration:.01ms !important;
+  }
+  .reveal { opacity:1; transform:none; }
+}
+</style>
+<script src="/static/analytics.js" defer></script>
+</head>
+<body>
+
+<!-- Ambient orbs -->
+<div class="fixed top-0 right-0 -z-10 w-96 h-96 bg-primary/5 rounded-full blur-[120px] pointer-events-none orb-1"></div>
+<div class="fixed bottom-0 left-0 -z-10 w-72 h-72 bg-tertiary/5 rounded-full blur-[100px] pointer-events-none orb-2"></div>
+<div class="fixed top-1/2 left-1/2 -z-10 w-64 h-64 bg-secondary/3 rounded-full blur-[140px] pointer-events-none" style="transform:translate(-50%,-50%);animation:orbFloat 16s ease-in-out infinite 4s"></div>
+
+<!-- Header -->
+<header class="sticky top-0 z-20 glass border-b border-primary/20">
+  <div class="max-w-6xl mx-auto px-4 lg:px-8 py-3 flex items-center justify-between gap-3">
+    <div class="flex items-center gap-3 shrink-0">
+      <div class="w-8 h-8 bg-primary/15 rounded-xl flex items-center justify-center shrink-0 brand-icon-wrap">
+        <span class="msym text-primary text-base">auto_awesome</span>
+      </div>
+      <div class="min-w-0 hidden sm:block">
+        <h1 class="text-sm font-bold text-primary leading-tight truncate" id="brand-name">AfakZaid RAG</h1>
+        <p class="text-[10px] text-on-surface-variant/60 uppercase tracking-widest" id="brand-sub">Scholarly Intelligence</p>
+      </div>
+      <div class="h-5 w-px bg-outline-variant/40 hidden sm:block"></div>
+      <a href="/library" class="glass glass-hover flex items-center gap-1 px-2.5 py-2 rounded-xl text-xs text-on-surface-variant hover:text-primary transition-colors">
+        <span class="msym text-base">library_books</span>
+        <span class="hidden sm:inline">المكتبة</span>
+      </a>
+    </div>
+    <nav class="flex items-center gap-2 shrink-0">
+      <div id="nav-user" class="shrink-0"></div>
+    </nav>
+  </div>
+</header>
+
+<main class="max-w-6xl mx-auto px-4 lg:px-8 py-8">
+
+  <!-- Stats -->
+  <div class="grid grid-cols-2 lg:grid-cols-4 gap-4 mb-8">
+    <div class="glass glow rounded-2xl p-4 text-center stat-card">
+      <div class="text-2xl font-bold text-primary stat-num" id="s-books">0</div>
+      <div class="text-xs text-on-surface-variant mt-1">كتاب</div>
+    </div>
+    <div class="glass glow rounded-2xl p-4 text-center stat-card">
+      <div class="text-2xl font-bold text-primary stat-num" id="s-vectors">0</div>
+      <div class="text-xs text-on-surface-variant mt-1">متجه</div>
+    </div>
+    <div class="glass glow rounded-2xl p-4 text-center stat-card">
+      <div class="text-2xl font-bold text-secondary stat-num" id="s-convos">0</div>
+      <div class="text-xs text-on-surface-variant mt-1">محادثة</div>
+    </div>
+    <div class="glass glow rounded-2xl p-4 text-center stat-card">
+      <div class="text-2xl font-bold text-tertiary stat-num" id="s-memory">0</div>
+      <div class="text-xs text-on-surface-variant mt-1">ذاكرة</div>
+    </div>
+  </div>
+
+  <div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-6">
+
+    <!-- Query -->
+    <div class="glass rounded-3xl p-6 query-card">
+      <h2 class="text-primary text-xs font-bold uppercase tracking-widest mb-4 flex items-center gap-2">
+        <span class="msym text-base">search</span> اسأل الكتب
+      </h2>
+      <textarea id="query-input" rows="3" dir="rtl"
+        class="w-full bg-surface-high border border-outline-variant rounded-2xl px-4 py-3 text-on-surface text-sm resize-none outline-none focus:border-primary transition-colors mb-3"
+        placeholder="اكتب سؤالك هنا..."></textarea>
+      <button onclick="doQuery()" id="query-btn"
+        class="w-full bg-primary text-surface font-bold py-3 rounded-xl flex items-center justify-center gap-2">
+        <span class="msym text-lg">search</span> بحث
+      </button>
+      <div id="answer-wrap" class="hidden mt-4">
+        <div id="answer-box" class="bg-surface-high rounded-2xl p-4 text-sm leading-7 text-on-surface whitespace-pre-wrap max-h-64 overflow-y-auto scrollbar"></div>
+        <div id="answer-meta" class="flex flex-wrap gap-2 mt-3"></div>
+      </div>
+    </div>
+
+    <!-- Upload -->
+    <div class="upload-card-wrap">
+      <div class="glass rounded-3xl p-6 h-full" id="upload-card">
+        <h2 class="text-primary text-xs font-bold uppercase tracking-widest mb-4 flex items-center gap-2">
+          <span class="msym text-base">upload_file</span> رفع كتاب PDF
+        </h2>
+        <div class="upload-zone rounded-2xl p-8 text-center cursor-pointer relative mb-4" id="drop-zone">
+          <input type="file" accept=".pdf" multiple class="absolute inset-0 opacity-0 cursor-pointer w-full h-full" onchange="handleFiles(this.files)"/>
+          <div class="w-14 h-14 bg-primary/10 rounded-full flex items-center justify-center mx-auto mb-3 glow" style="transition:transform .3s">
+            <span class="msym text-primary text-3xl">cloud_upload</span>
+          </div>
+          <p class="text-on-surface-variant text-sm">اسحب ملفات PDF وأفلتها هنا</p>
+          <p class="text-primary text-xs font-bold mt-1">أو انقر للاختيار</p>
+        </div>
+        <div class="flex gap-2">
+          <input type="text" id="url-input" dir="ltr" placeholder="https://example.com/book.pdf"
+            class="flex-1 bg-surface-high border border-outline-variant rounded-xl px-3 py-2 text-xs text-on-surface outline-none focus:border-primary transition-colors"/>
+          <button onclick="ingestURL()" class="bg-primary/10 border border-primary/30 text-primary px-3 py-2 rounded-xl text-xs font-bold hover:bg-primary/20 transition-colors flex items-center gap-1">
+            <span class="msym text-sm">add</span>
+          </button>
+        </div>
+        <div id="upload-progress" class="hidden mt-3">
+          <div class="bg-surface-high rounded-full h-1.5 overflow-hidden">
+            <div id="progress-fill" class="h-1.5 bg-primary rounded-full transition-all duration-500" style="width:0%"></div>
+          </div>
+          <p id="progress-text" class="text-xs text-on-surface-variant mt-2"></p>
+        </div>
+        <pre id="log-box" class="hidden mt-2 text-xs text-on-surface-variant bg-surface-high rounded-xl p-3 max-h-24 overflow-y-auto scrollbar font-mono"></pre>
+      </div>
+    </div>
+
+  </div>
+
+  <!-- Books -->
+  <div class="glass rounded-3xl p-6 books-card reveal" id="books-section">
+    <div class="flex items-center justify-between mb-5">
+      <h2 class="text-primary text-xs font-bold uppercase tracking-widest flex items-center gap-2">
+        <span class="msym text-base">library_books</span> الكتب المُدرجة
+      </h2>
+      <a href="/library" class="text-xs text-on-surface-variant hover:text-primary transition-colors flex items-center gap-1">
+        عرض الكل <span class="msym text-sm">arrow_back</span>
+      </a>
+    </div>
+    <div id="books-list" class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-3"></div>
+  </div>
+
+</main>
+
+<script>
+const API_KEY = 'zRf77E_brSEnA2BUi40uxLxT4uofAk661WMZb-cdv4M';
+let userToken = localStorage.getItem('rag_token');
+let currentUser = null;
+
+// ── Branding ──────────────────────────────────────────────────────────────────
+fetch('/branding').then(r=>r.json()).then(b=>{
+  if(b.site_name){document.title=b.site_name;document.getElementById('brand-name').textContent=b.site_name;}
+  if(b.site_subtitle) document.getElementById('brand-sub').textContent=b.site_subtitle;
+}).catch(()=>{});
+
+// ── Counter animation ─────────────────────────────────────────────────────────
+function animateCount(el, target, duration=900) {
+  const start = performance.now();
+  const from = parseInt(el.textContent.replace(/,/g,'')) || 0;
+  const isLarge = target > 999;
+  function tick(now) {
+    const elapsed = now - start;
+    const progress = Math.min(elapsed / duration, 1);
+    // ease out cubic
+    const eased = 1 - Math.pow(1 - progress, 3);
+    const current = Math.round(from + (target - from) * eased);
+    el.textContent = isLarge ? current.toLocaleString() : current;
+    if (progress < 1) requestAnimationFrame(tick);
+    else el.textContent = isLarge ? target.toLocaleString() : target;
+  }
+  requestAnimationFrame(tick);
+}
+
+// ── Scroll reveal ─────────────────────────────────────────────────────────────
+const observer = new IntersectionObserver(entries => {
+  entries.forEach(e => { if(e.isIntersecting) { e.target.classList.add('visible'); observer.unobserve(e.target); } });
+}, { threshold: 0.12 });
+document.querySelectorAll('.reveal').forEach(el => observer.observe(el));
+
+// ── Auth ──────────────────────────────────────────────────────────────────────
+async function initAuth() {
+  const nav = document.getElementById('nav-user');
+  if (!userToken) {
+    nav.innerHTML = `<a href="/login" class="bg-primary text-surface font-bold px-3 py-2 rounded-xl text-xs hover:shadow-[0_0_15px_rgba(78,222,163,0.4)] transition-all flex items-center gap-1.5 whitespace-nowrap" style="animation:scalePop .4s cubic-bezier(.22,1,.36,1) .8s both"><span class="msym text-base">login</span><span>دخول</span></a>`;
+    document.getElementById('upload-card').classList.add('hidden');
+    return;
+  }
+  try {
+    const r = await fetch('/auth/me', {headers:{'Authorization':'Bearer '+userToken}});
+    if (!r.ok) { localStorage.removeItem('rag_token'); initAuth(); return; }
+    currentUser = await r.json();
+    const perms = currentUser.permissions || [];
+    if (!perms.includes('upload')) document.getElementById('upload-card').classList.add('hidden');
+    nav.innerHTML = `
+      <div class="flex items-center gap-2" style="animation:fadeIn .4s both .6s">
+        <div class="text-right hidden sm:block">
+          <p class="text-xs font-bold">${currentUser.name}</p>
+          <p class="text-[10px] text-primary">${currentUser.role}</p>
+        </div>
+        ${perms.includes('admin') ? '<a href="/admin" class="glass glass-hover p-2 rounded-xl transition-colors"><span class="msym text-base text-on-surface-variant">admin_panel_settings</span></a>' : ''}
+        <button onclick="logout()" class="glass glass-hover p-2 rounded-xl transition-colors"><span class="msym text-base text-on-surface-variant">logout</span></button>
+      </div>`;
+  } catch(e){}
+}
+
+function logout() {
+  if(userToken) fetch('/auth/logout',{method:'POST',headers:{'Authorization':'Bearer '+userToken}});
+  localStorage.removeItem('rag_token');
+  window.location.reload();
+}
+
+// ── Stats ─────────────────────────────────────────────────────────────────────
+async function loadStats() {
+  try {
+    const d = await (await fetch('/stats')).json();
+    animateCount(document.getElementById('s-books'),   d.books||0);
+    animateCount(document.getElementById('s-vectors'), d.vectors||0);
+    animateCount(document.getElementById('s-convos'),  d.conversations||0);
+    animateCount(document.getElementById('s-memory'),  d.memories||0);
+  } catch(e){}
+}
+
+// ── Books ─────────────────────────────────────────────────────────────────────
+async function loadBooks() {
+  try {
+    const d = await (await fetch('/api/library?limit=6')).json();
+    const list = document.getElementById('books-list');
+    if (!d.books||!d.books.length) {
+      list.innerHTML='<p class="text-on-surface-variant text-sm">لا توجد كتب حتى الآن</p>'; return;
+    }
+    list.innerHTML = d.books.map((b,i)=>{
+      const name = b.title || b.filename.replace(/\.pdf$/i,'').replace(/^\d{4}-\d{2}-\d{2}\s*/,'');
+      return `<div class="glass glass-hover rounded-2xl p-4 flex items-center gap-3 cursor-pointer book-card"
+        style="animation:fadeUp .5s cubic-bezier(.22,1,.36,1) ${.85+i*.08}s both"
+        onclick="window.location='/library'">
+        <div class="w-9 h-9 bg-primary/10 rounded-xl flex items-center justify-center shrink-0" style="transition:transform .25s">
+          <span class="msym text-primary">menu_book</span>
+        </div>
+        <div class="min-w-0">
+          <p class="text-sm font-medium truncate">${name}</p>
+          ${b.author?`<p class="text-xs text-on-surface-variant">${b.author}</p>`:''}
+        </div>
+      </div>`;
+    }).join('');
+  } catch(e){}
+}
+
+// ── Query ─────────────────────────────────────────────────────────────────────
+async function doQuery() {
+  const q = document.getElementById('query-input').value.trim();
+  if(!q) return;
+  const btn = document.getElementById('query-btn');
+  btn.disabled = true;
+  btn.style.animation = 'none';
+  btn.innerHTML = '<span class="msym text-lg" style="animation:spin .8s linear infinite">autorenew</span> جاري البحث...';
+  const wrap = document.getElementById('answer-wrap');
+  const box  = document.getElementById('answer-box');
+  const meta = document.getElementById('answer-meta');
+  wrap.classList.remove('hidden');
+  box.textContent = '...'; meta.innerHTML = '';
+  try {
+    const headers = {'Content-Type':'application/json','X-API-Key':API_KEY};
+    if(userToken) headers['Authorization'] = 'Bearer '+userToken;
+    const r = await fetch('/query',{method:'POST',headers,body:JSON.stringify({query:q,session_id:'web-'+(currentUser?.id||'guest')})});
+    const d = await r.json();
+    box.textContent = d.answer;
+    meta.innerHTML = [
+      `<span class="glass px-3 py-1 rounded-full text-xs text-primary font-bold">${d.llm}</span>`,
+      `<span class="glass px-3 py-1 rounded-full text-xs text-secondary">${((d.confidence||0)*100).toFixed(0)}% ثقة</span>`,
+      ...(d.books_used||[]).map(b=>`<span class="glass px-3 py-1 rounded-full text-xs text-on-surface-variant">${b}</span>`)
+    ].join('');
+  } catch(e){ box.textContent='خطأ: '+e.message; }
+  btn.disabled = false;
+  btn.style.animation = '';
+  btn.innerHTML = '<span class="msym text-lg">search</span> بحث';
+}
+
+// ── Upload ────────────────────────────────────────────────────────────────────
+async function handleFiles(files){for(const f of files)await uploadFile(f);loadStats();loadBooks();}
+
+async function uploadFile(file){
+  const prog=document.getElementById('upload-progress'),fill=document.getElementById('progress-fill');
+  const txt=document.getElementById('progress-text'),log=document.getElementById('log-box');
+  prog.classList.remove('hidden');log.classList.remove('hidden');
+  txt.textContent='رفع: '+file.name;fill.style.width='30%';
+  const fd=new FormData();fd.append('file',file);
+  try{
+    fill.style.width='60%';
+    const headers={'X-API-Key':API_KEY};
+    if(userToken) headers['Authorization']='Bearer '+userToken;
+    const r=await fetch('/ingest',{method:'POST',headers,body:fd});
+    fill.style.width='100%';
+    const d=await r.json();
+    txt.textContent='✅ تم: '+file.name;log.textContent=d.log||'Done';
+  }catch(e){txt.textContent='❌ خطأ: '+e.message;}
+}
+
+async function ingestURL(){
+  const url=document.getElementById('url-input').value.trim();if(!url)return;
+  const prog=document.getElementById('upload-progress'),fill=document.getElementById('progress-fill');
+  const txt=document.getElementById('progress-text'),log=document.getElementById('log-box');
+  prog.classList.remove('hidden');log.classList.remove('hidden');
+  txt.textContent='جلب الملف...';fill.style.width='40%';
+  try{
+    fill.style.width='70%';
+    const headers={'Content-Type':'application/json','X-API-Key':API_KEY};
+    if(userToken) headers['Authorization']='Bearer '+userToken;
+    const r=await fetch('/ingest-url',{method:'POST',headers,body:JSON.stringify({url})});
+    fill.style.width='100%';
+    const d=await r.json();
+    txt.textContent='✅ تم: '+d.file;log.textContent=d.log||'Done';
+    document.getElementById('url-input').value='';
+    loadStats();loadBooks();
+  }catch(e){txt.textContent='❌ خطأ: '+e.message;}
+}
+
+// ── Upload zone drag ─────────────────────────────────────────────────────────
+const zone = document.getElementById('drop-zone');
+const cloudIcon = zone.querySelector('.w-14');
+zone.addEventListener('dragover',e=>{e.preventDefault();zone.classList.add('dragover');cloudIcon.style.transform='scale(1.15)';});
+zone.addEventListener('dragleave',()=>{zone.classList.remove('dragover');cloudIcon.style.transform='';});
+zone.addEventListener('drop',e=>{e.preventDefault();zone.classList.remove('dragover');cloudIcon.style.transform='';handleFiles(e.dataTransfer.files);});
+
+// ── Keyboard shortcut ────────────────────────────────────────────────────────
+document.getElementById('query-input').addEventListener('keydown',e=>{if(e.ctrlKey&&e.key==='Enter')doQuery();});
+
+// ── URL pre-fill ─────────────────────────────────────────────────────────────
+const urlQ = new URLSearchParams(location.search).get('q');
+if(urlQ){document.getElementById('query-input').value=decodeURIComponent(urlQ);setTimeout(doQuery,600);}
+
+// ── Spin keyframe for loading ────────────────────────────────────────────────
+const spinStyle = document.createElement('style');
+spinStyle.textContent = '@keyframes spin{to{transform:rotate(360deg)}}';
+document.head.appendChild(spinStyle);
+
+// ── Init ──────────────────────────────────────────────────────────────────────
+initAuth();
+loadStats();
+loadBooks();
+setInterval(loadStats, 30000);
+</script>
+</body>
+</html>

static/login.html

@@ -0,0 +1,501 @@
+<!DOCTYPE html>
+<html class="dark" lang="en">
+<head>
+<meta charset="UTF-8"/>
+<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+<title>AfakZaid RAG — Sign In</title>
+<script src="https://cdn.tailwindcss.com?plugins=forms"></script>
+<link href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700;800&display=swap" rel="stylesheet"/>
+<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:wght,FILL@100..700,0..1&display=swap" rel="stylesheet"/>
+<script>tailwind.config={darkMode:"class",theme:{extend:{colors:{"primary":"#4edea3","surface":"#0c1324","surface-low":"#151b2d","surface-container":"#191f31","surface-high":"#23293c","on-surface":"#dce1fb","on-surface-variant":"#bbcabf","outline-variant":"#3c4a42","secondary":"#95d3ba","error":"#f87171"},fontFamily:{sans:["Manrope","sans-serif"]}}}}</script>
+<style>
+*{box-sizing:border-box}
+body{background:radial-gradient(circle at 50% -20%,#151b2d 0%,#0c1324 100%);font-family:'Manrope',sans-serif;color:#dce1fb;min-height:100vh;display:flex;align-items:center;justify-content:center;padding:20px}
+.glass{background:rgba(21,27,45,0.6);backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border:1px solid rgba(78,222,163,0.15)}
+.glow{box-shadow:0 0 30px rgba(78,222,163,0.15)}
+.msym{font-family:'Material Symbols Outlined';font-variation-settings:'FILL' 0,'wght' 400,'GRAD' 0,'opsz' 24;vertical-align:middle;line-height:1}
+:root { --ease-out: cubic-bezier(0.23,1,0.32,1); }
+
+.inp{width:100%;background:#23293c;border:1px solid #3c4a42;border-radius:12px;padding:12px 16px;color:#dce1fb;font-size:14px;outline:none;transition:border-color 180ms ease, box-shadow 180ms ease;font-family:'Manrope',sans-serif}
+.inp:focus{border-color:#4edea3;box-shadow:0 0 0 2px rgba(78,222,163,0.1)}
+.inp.inp-error{border-color:#f87171;box-shadow:0 0 0 2px rgba(248,113,113,0.1)}
+.inp-icon{position:relative}
+.inp-icon .msym{position:absolute;left:12px;top:50%;transform:translateY(-50%);color:#bbcabf;font-size:18px;pointer-events:none}
+.inp-icon input{padding-left:40px}
+
+/* Primary button — specific transitions, no transition:all */
+.btn-primary{width:100%;background:#4edea3;color:#003824;font-weight:700;border:none;border-radius:12px;padding:14px;cursor:pointer;font-size:15px;font-family:'Manrope',sans-serif;position:relative;overflow:hidden;transition:box-shadow 200ms ease, transform 150ms var(--ease-out)}
+.btn-primary:disabled{opacity:0.5;cursor:not-allowed}
+.btn-primary:active:not(:disabled){transform:scale(0.97);transition-duration:100ms}
+@media (hover:hover) and (pointer:fine){.btn-primary:hover:not(:disabled){box-shadow:0 0 24px rgba(78,222,163,0.4)}}
+
+/* Shimmer on btn hover */
+.btn-primary::after{content:'';position:absolute;inset:0;background:linear-gradient(90deg,transparent 0%,rgba(255,255,255,0.12) 50%,transparent 100%);transform:translateX(-100%);transition:transform 380ms cubic-bezier(0.77,0,0.175,1);pointer-events:none}
+@media (hover:hover) and (pointer:fine){.btn-primary:hover::after{transform:translateX(100%)}}
+
+/* Tabs — specific transitions */
+.tab{padding:10px 24px;border-radius:10px;font-size:14px;font-weight:600;cursor:pointer;border:none;background:transparent;color:#bbcabf;font-family:'Manrope',sans-serif;transition:background 150ms var(--ease-out),color 150ms var(--ease-out)}
+.tab.active{background:rgba(78,222,163,0.1);color:#4edea3}
+.tab:active{transform:scale(0.97)}
+
+/* Field validation */
+.field-error{font-size:12px;color:#f87171;margin-top:4px;display:none}
+.field-error.visible{display:block}
+
+/* Country dropdown */
+#country-dropdown{position:absolute;top:calc(100% + 4px);left:0;right:0;background:#191f31;border:1px solid rgba(78,222,163,0.2);border-radius:12px;z-index:50;max-height:200px;overflow-y:auto;box-shadow:0 8px 32px rgba(0,0,0,0.4)}
+#country-dropdown::-webkit-scrollbar{width:4px}
+#country-dropdown::-webkit-scrollbar-thumb{background:rgba(78,222,163,0.2);border-radius:4px}
+.country-item{display:flex;align-items:center;gap:10px;padding:10px 14px;cursor:pointer;font-size:13px;transition:background 120ms ease}
+@media (hover:hover) and (pointer:fine){.country-item:hover{background:rgba(78,222,163,0.08)}}
+
+/* Form entry */
+@keyframes fadeUp{from{opacity:0;transform:translateY(8px)}to{opacity:1;transform:translateY(0)}}
+.auth-card{animation:fadeUp 400ms var(--ease-out) both}
+
+/* Reduced motion */
+@media (prefers-reduced-motion:reduce){*,*::before,*::after{animation-duration:.01ms !important;transition-duration:.01ms !important}}
+
+
+/* ── Mobile-visible improvements ─────────────────────────────────────────── */
+
+/* Larger touch targets — 44px minimum per Apple HIG */
+button, a, .filter-chip, .book-card, .stat-card, .nav-btn {
+  -webkit-tap-highlight-color: transparent;
+}
+
+/* Tap feedback — visible on ALL devices including mobile */
+.book-card:active,
+.stat-card:active,
+.filter-chip:active {
+  transform: scale(0.97) !important;
+  transition: transform 100ms ease !important;
+}
+
+/* Buttons get a visible press state on tap */
+button:active:not(:disabled) {
+  filter: brightness(0.9);
+  transform: scale(0.97);
+  transition: transform 100ms ease, filter 100ms ease !important;
+}
+
+/* Restore .glow — ensure it stacks correctly */
+.glow {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    0 0 20px rgba(78,222,163,0.15) !important;
+}
+.glow-sm {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.04),
+    0 0 10px rgba(78,222,163,0.1) !important;
+}
+
+/* Mobile: book cards get a more visible pressed state */
+@media (max-width: 768px) {
+  .book-card {
+    transition: transform 150ms ease, opacity 150ms ease !important;
+  }
+  .book-card:active {
+    transform: scale(0.97) !important;
+    opacity: 0.85 !important;
+  }
+  
+  /* Ensure skeleton is visible on mobile */
+  .skeleton {
+    background: linear-gradient(90deg, #1a2035 25%, #2a3348 50%, #1a2035 75%) !important;
+    background-size: 800px 100% !important;
+    animation: shimmer 1.2s ease-in-out infinite !important;
+  }
+  
+  /* Better spacing on small screens */
+  .filter-chip {
+    padding: 8px 16px !important;
+    font-size: 13px !important;
+  }
+}
+/* ── design-taste-frontend: Liquid Glass + Spotlight ─────────────────────── */
+
+/* True liquid glass — inner refraction border */
+/* NOTE: does NOT define outer shadow — .glow still wins */
+.glass {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    inset 0 -1px 0 rgba(0,0,0,0.2);
+}
+/* Combined: liquid glass + green glow */
+.glass.glow {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    inset 0 -1px 0 rgba(0,0,0,0.2),
+    0 0 20px rgba(78,222,163,0.18),
+    0 4px 24px rgba(0,0,0,0.15);
+}
+.glass.glow-sm {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    inset 0 -1px 0 rgba(0,0,0,0.2),
+    0 0 10px rgba(78,222,163,0.12);
+}
+/* .glow overrides naturally on top — no !important needed */
+
+/* Spotlight border card — JS-driven cursor tracking */
+.spotlight-card {
+  position: relative;
+  overflow: hidden;
+}
+.spotlight-card::before {
+  content: '';
+  position: absolute;
+  inset: 0;
+  background: radial-gradient(
+    400px circle at var(--mx, 50%) var(--my, 50%),
+    rgba(78,222,163,0.06),
+    transparent 60%
+  );
+  opacity: 0;
+  transition: opacity 300ms ease;
+  pointer-events: none;
+  border-radius: inherit;
+  z-index: 0;
+}
+@media (hover: hover) and (pointer: fine) {
+  .spotlight-card:hover::before { opacity: 1; }
+}
+
+/* Ensure content above spotlight overlay */
+.spotlight-card > * { position: relative; z-index: 1; }
+
+/* Tactile button press — design-taste rule */
+button:not(:disabled):active,
+a[role=button]:active {
+  transform: translateY(-1px) scale(0.98) !important;
+  transition-duration: 80ms !important;
+}
+
+/* Hardware acceleration on animated elements */
+.stat-card, .book-card, .query-card, .books-card, .upload-card-wrap {
+  transform: translateZ(0);
+  backface-visibility: hidden;
+}
+
+/* Directional hover-aware CTA — shimmer from left on hover */
+.cta-primary {
+  position: relative; overflow: hidden;
+}
+.cta-primary::before {
+  content: '';
+  position: absolute;
+  top: 0; left: -100%; bottom: 0; width: 50%;
+  background: linear-gradient(90deg, transparent, rgba(255,255,255,0.1), transparent);
+  transition: left 400ms cubic-bezier(0.77,0,0.175,1);
+  pointer-events: none;
+}
+@media (hover: hover) and (pointer: fine) {
+  .cta-primary:hover::before { left: 200%; }
+}
+
+/* ── Mobile improvements (actually visible on mobile) ────────────────────── */
+
+/* Larger touch targets — 44px minimum per Apple HIG */
+@media (max-width: 768px) {
+  /* Bigger tap areas on buttons */
+  button, a { min-height: 44px; }
+  .filter-chip { padding: 8px 16px; font-size: 13px; }
+
+  /* Book cards: more breathing room on mobile */
+  .book-card { padding: 14px !important; }
+
+  /* Stats cards: larger numbers on mobile */
+  .stat-card .stat-num, #s-books, #s-vectors, #s-convos, #s-memory {
+    font-size: 1.75rem;
+  }
+
+  /* Header: tighter on mobile */
+  header { padding-top: 10px; padding-bottom: 10px; }
+
+  /* Search input: larger font so iOS doesn't zoom */
+  #search-input, .inp, textarea, input[type="text"], input[type="email"], input[type="password"] {
+    font-size: 16px !important; /* prevents iOS zoom on focus */
+  }
+
+  /* Bottom action bar spacing */
+  main { padding-bottom: env(safe-area-inset-bottom, 16px); }
+}
+
+/* Reader: full height on mobile */
+@media (max-width: 768px) {
+  #pdf-frame { position: absolute; inset: 0; }
+  .reader-header { padding: 8px 12px; }
+}
+</style>
+  <script src="/static/analytics.js" defer></script>
+</head>
+<body>
+
+<div class="w-full max-w-md">
+
+  <!-- Logo -->
+  <div class="text-center mb-8">
+    <div class="w-16 h-16 bg-primary/10 rounded-2xl flex items-center justify-center mx-auto mb-4 glow">
+      <span class="msym text-primary text-3xl">auto_awesome</span>
+    </div>
+    <h1 id="brand-name" class="text-2xl font-bold text-primary">AfakZaid RAG</h1>
+    <p id="brand-sub" class="text-on-surface-variant text-sm mt-1">Scholarly Intelligence</p>
+  </div>
+
+  <!-- Card -->
+  <div class="glass glow rounded-3xl p-8 auth-card">
+
+    <!-- Tabs -->
+    <div class="flex gap-2 bg-surface-container rounded-xl p-1 mb-8">
+      <button class="tab active flex-1" id="tab-login" onclick="switchTab('login')">Sign In</button>
+      <button class="tab flex-1" id="tab-register" onclick="switchTab('register')">Create Account</button>
+    </div>
+
+    <!-- LOGIN -->
+    <form id="form-login" onsubmit="doLogin(event)">
+      <div class="space-y-4 mb-6">
+        <div class="inp-icon">
+          <span class="msym">mail</span>
+          <input class="inp" type="email" id="login-email" placeholder="Email address" required/>
+        </div>
+        <div class="inp-icon">
+          <span class="msym">lock</span>
+          <input class="inp" type="password" id="login-pass" placeholder="Password" required/>
+        </div>
+      </div>
+      <p id="login-err" class="text-error text-xs mb-4 hidden"></p>
+      <button class="btn-primary" type="submit" id="login-btn">
+        <span class="msym text-lg align-middle mr-1">login</span> Sign In
+      </button>
+    </form>
+
+    <!-- REGISTER -->
+    <form id="form-register" class="hidden" onsubmit="doRegister(event)">
+      <div class="space-y-4 mb-6">
+        <div class="inp-icon">
+          <span class="msym">person</span>
+          <input class="inp" type="text" id="reg-name" placeholder="Full name" required/>
+        </div>
+        <div class="inp-icon">
+          <span class="msym">mail</span>
+          <input class="inp" type="email" id="reg-email" placeholder="Email address" required/>
+        </div>
+        <div class="inp-icon">
+          <span class="msym">lock</span>
+          <input class="inp" type="password" id="reg-pass" placeholder="Password (min 6 chars)" required minlength="6"/>
+        </div>
+
+        <!-- Phone + Country -->
+        <div class="grid grid-cols-2 gap-3">
+          <!-- Country picker -->
+          <div class="relative" id="country-wrap">
+            <div class="inp flex items-center gap-2 cursor-pointer select-none" onclick="toggleCountry()" id="country-display">
+              <span class="msym text-on-surface-variant text-base">public</span>
+              <span id="country-flag" class="text-lg hidden"></span>
+              <span id="country-label" class="text-on-surface-variant text-sm truncate flex-1">Country</span>
+              <span class="msym text-on-surface-variant text-base">expand_more</span>
+            </div>
+            <div id="country-dropdown" class="hidden">
+              <div class="p-2">
+                <input id="country-search" class="inp text-xs py-2" placeholder="Search country..." oninput="filterCountries(this.value)"/>
+              </div>
+              <div id="country-list"></div>
+            </div>
+            <input type="hidden" id="reg-country-code"/>
+            <input type="hidden" id="reg-country-name"/>
+          </div>
+          <!-- Phone -->
+          <div class="inp-icon">
+            <span class="msym">phone</span>
+            <input class="inp" type="tel" id="reg-phone" placeholder="Phone"/>
+          </div>
+        </div>
+
+        <!-- GPS button -->
+        <button type="button" onclick="detectLocation()" id="gps-btn"
+          class="w-full flex items-center justify-center gap-2 glass border border-primary/20 text-on-surface-variant hover:text-primary hover:border-primary/50 rounded-xl py-2.5 text-sm transition-all">
+          <span class="msym text-base">my_location</span>
+          <span id="gps-text">Detect my location</span>
+        </button>
+
+      </div>
+      <p id="reg-err" class="text-error text-xs mb-4 hidden"></p>
+      <button class="btn-primary" type="submit" id="reg-btn">
+        <span class="msym text-lg align-middle mr-1">person_add</span> Create Account
+      </button>
+    </form>
+
+  </div>
+
+  <p class="text-center text-on-surface-variant text-xs mt-6">
+    <a href="/" class="hover:text-primary transition-colors">← Back to library</a>
+  </p>
+</div>
+
+<!-- Background -->
+<div class="fixed top-0 right-0 -z-10 w-96 h-96 bg-primary/5 rounded-full blur-[120px] pointer-events-none"></div>
+<div class="fixed bottom-0 left-0 -z-10 w-72 h-72 bg-secondary/5 rounded-full blur-[100px] pointer-events-none"></div>
+
+<script>
+// ── Branding ───────────────────────────────────────────────────────────────
+fetch('/branding').then(r=>r.json()).then(b=>{
+  if(b.site_name){document.title=b.site_name+' — Sign In';document.getElementById('brand-name').textContent=b.site_name;}
+  if(b.site_subtitle) document.getElementById('brand-sub').textContent=b.site_subtitle;
+}).catch(()=>{});
+
+// Redirect if already logged in
+const token = localStorage.getItem('rag_token');
+if(token) {
+  fetch('/auth/me',{headers:{'Authorization':token}}).then(r=>{
+    if(r.ok) window.location.href='/';
+  }).catch(()=>{});
+}
+
+// ── Tabs ───────────────────────────────────────────────────────────────────
+function switchTab(tab) {
+  document.getElementById('form-login').classList.toggle('hidden', tab!=='login');
+  document.getElementById('form-register').classList.toggle('hidden', tab!=='register');
+  document.getElementById('tab-login').classList.toggle('active', tab==='login');
+  document.getElementById('tab-register').classList.toggle('active', tab==='register');
+}
+
+// ── Auth ───────────────────────────────────────────────────────────────────
+async function doLogin(e) {
+  e.preventDefault();
+  const btn=document.getElementById('login-btn');
+  if(btn.disabled) return;
+  const err=document.getElementById('login-err');
+  btn.disabled=true; btn.textContent='Signing in...'; err.classList.add('hidden');
+  try {
+    const r = await fetch('/auth/login',{method:'POST',headers:{'Content-Type':'application/json'},
+      body:JSON.stringify({email:document.getElementById('login-email').value,password:document.getElementById('login-pass').value})});
+    const d = await r.json();
+    if(!r.ok){err.textContent=d.detail||'Login failed';err.classList.remove('hidden');return;}
+    localStorage.setItem('rag_token',d.token);
+    localStorage.setItem('rag_user',JSON.stringify(d.user));
+    window.location.href = d.user.permissions?.admin ? '/admin' : '/';
+  } catch(ex){err.textContent='Connection error';err.classList.remove('hidden');}
+  finally{btn.disabled=false;btn.innerHTML='<span class="msym text-lg align-middle mr-1">login</span> Sign In';}
+}
+
+async function doRegister(e) {
+  e.preventDefault();
+  const btn=document.getElementById('reg-btn');
+  if(btn.disabled) return;
+  const err=document.getElementById('reg-err');
+  btn.disabled=true;btn.textContent='Creating account...';err.classList.add('hidden');
+  try {
+    const r = await fetch('/auth/register',{method:'POST',headers:{'Content-Type':'application/json'},
+      body:JSON.stringify({
+        name:document.getElementById('reg-name').value,
+        email:document.getElementById('reg-email').value,
+        password:document.getElementById('reg-pass').value,
+        phone:document.getElementById('reg-phone').value,
+        country_code:document.getElementById('reg-country-code').value,
+        country_name:document.getElementById('reg-country-name').value,
+      })});
+    const d = await r.json();
+    if(!r.ok){err.textContent=d.detail||'Registration failed';err.classList.remove('hidden');return;}
+    localStorage.setItem('rag_token',d.token);
+    window.location.href='/';
+  } catch(ex){err.textContent='Connection error';err.classList.remove('hidden');}
+  finally{btn.disabled=false;btn.innerHTML='<span class="msym text-lg align-middle mr-1">person_add</span> Create Account';}
+}
+
+// ── Country Picker ─────────────────────────────────────────────────────────
+let allCountries = [];
+let selectedCountry = null;
+
+async function loadCountries() {
+  try {
+    const r = await fetch('https://restcountries.com/v3.1/all?fields=name,cca2,idd,flag');
+    const data = await r.json();
+    allCountries = data.sort((a,b)=>a.name.common.localeCompare(b.name.common));
+    renderCountries(allCountries);
+  } catch(e) {
+    // fallback minimal list
+    allCountries = [
+      {name:{common:'Saudi Arabia'},cca2:'SA',flag:'🇸🇦'},
+      {name:{common:'UAE'},cca2:'AE',flag:'🇦🇪'},
+      {name:{common:'Kuwait'},cca2:'KW',flag:'🇰🇼'},
+      {name:{common:'Qatar'},cca2:'QA',flag:'🇶🇦'},
+      {name:{common:'Bahrain'},cca2:'BH',flag:'🇧🇭'},
+      {name:{common:'Egypt'},cca2:'EG',flag:'🇪🇬'},
+      {name:{common:'Jordan'},cca2:'JO',flag:'🇯🇴'},
+      {name:{common:'United States'},cca2:'US',flag:'🇺🇸'},
+      {name:{common:'United Kingdom'},cca2:'GB',flag:'🇬🇧'},
+    ];
+    renderCountries(allCountries);
+  }
+}
+
+function renderCountries(list) {
+  document.getElementById('country-list').innerHTML = list.slice(0,100).map(c=>`
+    <div class="country-item" onclick="selectCountry('${c.cca2}','${c.name.common.replace(/'/g,"\\'")}','${c.flag||''}')">
+      <span class="text-lg">${c.flag||'🌍'}</span>
+      <span class="text-sm">${c.name.common}</span>
+      <span class="text-xs text-on-surface-variant ml-auto">${c.cca2}</span>
+    </div>`).join('');
+}
+
+function filterCountries(q) {
+  const filtered = allCountries.filter(c=>c.name.common.toLowerCase().includes(q.toLowerCase())||c.cca2.toLowerCase().includes(q.toLowerCase()));
+  renderCountries(filtered);
+}
+
+function selectCountry(code, name, flag) {
+  document.getElementById('reg-country-code').value = code;
+  document.getElementById('reg-country-name').value = name;
+  document.getElementById('country-flag').textContent = flag;
+  document.getElementById('country-flag').classList.remove('hidden');
+  document.getElementById('country-label').textContent = name;
+  document.getElementById('country-label').style.color = '#dce1fb';
+  closeCountry();
+}
+
+function toggleCountry() {
+  const dd = document.getElementById('country-dropdown');
+  dd.classList.toggle('hidden');
+  if(!dd.classList.contains('hidden')) {
+    document.getElementById('country-search').focus();
+    loadCountries();
+  }
+}
+
+function closeCountry() {
+  document.getElementById('country-dropdown').classList.add('hidden');
+}
+
+document.addEventListener('click', e=>{
+  if(!document.getElementById('country-wrap').contains(e.target)) closeCountry();
+});
+
+// ── GPS Detection ──────────────────────────────────────────────────────────
+async function detectLocation() {
+  const btn = document.getElementById('gps-btn');
+  const txt = document.getElementById('gps-text');
+  txt.textContent = 'Detecting...';
+  btn.disabled = true;
+  if (!navigator.geolocation) { txt.textContent='GPS not supported'; btn.disabled=false; return; }
+  navigator.geolocation.getCurrentPosition(async pos => {
+    try {
+      const {latitude: lat, longitude: lon} = pos.coords;
+      const r = await fetch(`https://nominatim.openstreetmap.org/reverse?format=json&lat=${lat}&lon=${lon}`);
+      const d = await r.json();
+      const code = d.address?.country_code?.toUpperCase()||'';
+      const name = d.address?.country||'';
+      // Find flag
+      await loadCountries();
+      const country = allCountries.find(c=>c.cca2===code);
+      const flag = country?.flag||'🌍';
+      selectCountry(code, name, flag);
+      txt.textContent = '✓ ' + name;
+    } catch(e){ txt.textContent='Could not detect'; }
+    btn.disabled=false;
+  }, ()=>{ txt.textContent='Location denied'; btn.disabled=false; });
+}
+</script>
+</body>
+</html>

static/reader.html

@@ -0,0 +1,434 @@
+<!DOCTYPE html>
+<html class="dark" lang="ar" dir="rtl">
+<head>
+<meta charset="UTF-8"/>
+<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+<title>قارئ الكتب — AfakZaid</title>
+<script src="https://cdn.tailwindcss.com"></script>
+<link href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700&display=swap" rel="stylesheet"/>
+<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:wght,FILL@400,0&display=swap" rel="stylesheet"/>
+<script>tailwind.config={darkMode:"class",theme:{extend:{colors:{"primary":"#4edea3","surface":"#0c1324","surface-high":"#23293c","on-surface":"#dce1fb","on-surface-variant":"#bbcabf","outline-variant":"#3c4a42"},fontFamily:{sans:["Manrope","sans-serif"]}}}}</script>
+<style>
+:root { --ease-out: cubic-bezier(0.23,1,0.32,1); }
+*{box-sizing:border-box;margin:0;padding:0}
+html,body{height:100%;overflow:hidden}
+body{background:#0c1324;font-family:'Manrope',sans-serif;color:#dce1fb;display:flex;flex-direction:column}
+.glass{background:rgba(21,27,45,0.6);backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border:1px solid rgba(78,222,163,0.12);box-shadow:inset 0 1px 0 rgba(255,255,255,0.04)}
+.msym{font-family:'Material Symbols Outlined';font-variation-settings:'FILL' 0,'wght' 400,'GRAD' 0,'opsz' 24;vertical-align:middle;line-height:1}
+
+/* Header buttons — specific transitions only */
+.hdr-btn {
+  transition: background 150ms var(--ease-out), border-color 150ms var(--ease-out), color 150ms var(--ease-out);
+  border: 1px solid transparent;
+}
+@media (hover: hover) and (pointer: fine) {
+  .hdr-btn:hover { background: rgba(78,222,163,0.08); border-color: rgba(78,222,163,0.2); }
+}
+.hdr-btn:active { transform: scale(0.95); transition-duration: 100ms; }
+
+/* AI btn glow */
+.ai-btn {
+  background: rgba(78,222,163,0.12);
+  border: 1px solid rgba(78,222,163,0.25);
+  transition: background 150ms var(--ease-out), box-shadow 200ms ease;
+}
+@media (hover: hover) and (pointer: fine) {
+  .ai-btn:hover { background: rgba(78,222,163,0.2); box-shadow: 0 0 20px rgba(78,222,163,0.2); }
+}
+.ai-btn:active { transform: scale(0.96); transition-duration: 100ms; }
+
+/* Fav btn — love animation */
+@keyframes heartBeat {
+  0%  { transform: scale(1); }
+  30% { transform: scale(1.4); }
+  60% { transform: scale(0.9); }
+  100%{ transform: scale(1); }
+}
+#fav-btn { transition: color 200ms ease, transform 200ms var(--ease-out); }
+#fav-btn.faved { animation: heartBeat 400ms var(--ease-out); color: #f87171; }
+#fav-btn:active { transform: scale(0.9); }
+
+/* PDF frame */
+#pdf-frame { flex:1; width:100%; height:100%; border:none; background:#1a1a2e; display:block; }
+
+/* Loading state */
+#loading {
+  position:absolute; inset:0;
+  display:flex; align-items:center; justify-content:center; flex-direction:column;
+  background:#0c1324; gap:16px; z-index:5;
+  animation: fadeIn 200ms ease both;
+}
+@keyframes fadeIn { from{opacity:0} to{opacity:1} }
+
+/* Loading spinner */
+@keyframes spin { to { transform: rotate(360deg); } }
+.loading-ring {
+  width: 40px; height: 40px;
+  border: 2px solid rgba(78,222,163,0.15);
+  border-top-color: #4edea3;
+  border-radius: 50%;
+  animation: spin 800ms linear infinite;
+}
+
+/* AI panel slide */
+#ai-panel {
+  transition: max-height 300ms var(--ease-out), opacity 250ms ease;
+  overflow: hidden;
+}
+#ai-panel.collapsed { max-height: 0; opacity: 0; }
+#ai-panel.open { max-height: 400px; opacity: 1; }
+
+/* Ask input */
+#ask-input {
+  transition: border-color 180ms ease, box-shadow 180ms ease;
+}
+#ask-input:focus { border-color: #4edea3; box-shadow: 0 0 0 2px rgba(78,222,163,0.1); outline: none; }
+
+/* prefers-reduced-motion */
+@media (prefers-reduced-motion: reduce) {
+  *,*::before,*::after { animation-duration:.01ms !important; transition-duration:.01ms !important; }
+}
+
+
+/* ── Mobile-visible improvements ─────────────────────────────────────────── */
+
+/* Larger touch targets — 44px minimum per Apple HIG */
+button, a, .filter-chip, .book-card, .stat-card, .nav-btn {
+  -webkit-tap-highlight-color: transparent;
+}
+
+/* Tap feedback — visible on ALL devices including mobile */
+.book-card:active,
+.stat-card:active,
+.filter-chip:active {
+  transform: scale(0.97) !important;
+  transition: transform 100ms ease !important;
+}
+
+/* Buttons get a visible press state on tap */
+button:active:not(:disabled) {
+  filter: brightness(0.9);
+  transform: scale(0.97);
+  transition: transform 100ms ease, filter 100ms ease !important;
+}
+
+/* Restore .glow — ensure it stacks correctly */
+.glow {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    0 0 20px rgba(78,222,163,0.15) !important;
+}
+.glow-sm {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.04),
+    0 0 10px rgba(78,222,163,0.1) !important;
+}
+
+/* Mobile: book cards get a more visible pressed state */
+@media (max-width: 768px) {
+  .book-card {
+    transition: transform 150ms ease, opacity 150ms ease !important;
+  }
+  .book-card:active {
+    transform: scale(0.97) !important;
+    opacity: 0.85 !important;
+  }
+  
+  /* Ensure skeleton is visible on mobile */
+  .skeleton {
+    background: linear-gradient(90deg, #1a2035 25%, #2a3348 50%, #1a2035 75%) !important;
+    background-size: 800px 100% !important;
+    animation: shimmer 1.2s ease-in-out infinite !important;
+  }
+  
+  /* Better spacing on small screens */
+  .filter-chip {
+    padding: 8px 16px !important;
+    font-size: 13px !important;
+  }
+}
+/* ── design-taste-frontend: Liquid Glass + Spotlight ─────────────────────── */
+
+/* True liquid glass — inner refraction border */
+/* NOTE: does NOT define outer shadow — .glow still wins */
+.glass {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    inset 0 -1px 0 rgba(0,0,0,0.2);
+}
+/* Combined: liquid glass + green glow */
+.glass.glow {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    inset 0 -1px 0 rgba(0,0,0,0.2),
+    0 0 20px rgba(78,222,163,0.18),
+    0 4px 24px rgba(0,0,0,0.15);
+}
+.glass.glow-sm {
+  box-shadow:
+    inset 0 1px 0 rgba(255,255,255,0.06),
+    inset 0 -1px 0 rgba(0,0,0,0.2),
+    0 0 10px rgba(78,222,163,0.12);
+}
+/* .glow overrides naturally on top — no !important needed */
+
+/* Spotlight border card — JS-driven cursor tracking */
+.spotlight-card {
+  position: relative;
+  overflow: hidden;
+}
+.spotlight-card::before {
+  content: '';
+  position: absolute;
+  inset: 0;
+  background: radial-gradient(
+    400px circle at var(--mx, 50%) var(--my, 50%),
+    rgba(78,222,163,0.06),
+    transparent 60%
+  );
+  opacity: 0;
+  transition: opacity 300ms ease;
+  pointer-events: none;
+  border-radius: inherit;
+  z-index: 0;
+}
+@media (hover: hover) and (pointer: fine) {
+  .spotlight-card:hover::before { opacity: 1; }
+}
+
+/* Ensure content above spotlight overlay */
+.spotlight-card > * { position: relative; z-index: 1; }
+
+/* Tactile button press — design-taste rule */
+button:not(:disabled):active,
+a[role=button]:active {
+  transform: translateY(-1px) scale(0.98) !important;
+  transition-duration: 80ms !important;
+}
+
+/* Hardware acceleration on animated elements */
+.stat-card, .book-card, .query-card, .books-card, .upload-card-wrap {
+  transform: translateZ(0);
+  backface-visibility: hidden;
+}
+
+/* Directional hover-aware CTA — shimmer from left on hover */
+.cta-primary {
+  position: relative; overflow: hidden;
+}
+.cta-primary::before {
+  content: '';
+  position: absolute;
+  top: 0; left: -100%; bottom: 0; width: 50%;
+  background: linear-gradient(90deg, transparent, rgba(255,255,255,0.1), transparent);
+  transition: left 400ms cubic-bezier(0.77,0,0.175,1);
+  pointer-events: none;
+}
+@media (hover: hover) and (pointer: fine) {
+  .cta-primary:hover::before { left: 200%; }
+}
+
+/* ── Mobile improvements (actually visible on mobile) ────────────────────── */
+
+/* Larger touch targets — 44px minimum per Apple HIG */
+@media (max-width: 768px) {
+  /* Bigger tap areas on buttons */
+  button, a { min-height: 44px; }
+  .filter-chip { padding: 8px 16px; font-size: 13px; }
+
+  /* Book cards: more breathing room on mobile */
+  .book-card { padding: 14px !important; }
+
+  /* Stats cards: larger numbers on mobile */
+  .stat-card .stat-num, #s-books, #s-vectors, #s-convos, #s-memory {
+    font-size: 1.75rem;
+  }
+
+  /* Header: tighter on mobile */
+  header { padding-top: 10px; padding-bottom: 10px; }
+
+  /* Search input: larger font so iOS doesn't zoom */
+  #search-input, .inp, textarea, input[type="text"], input[type="email"], input[type="password"] {
+    font-size: 16px !important; /* prevents iOS zoom on focus */
+  }
+
+  /* Bottom action bar spacing */
+  main { padding-bottom: env(safe-area-inset-bottom, 16px); }
+}
+
+/* Reader: full height on mobile */
+@media (max-width: 768px) {
+  #pdf-frame { position: absolute; inset: 0; }
+  .reader-header { padding: 8px 12px; }
+}
+</style>
+  <script src="/static/analytics.js" defer></script>
+</head>
+<body>
+<!-- Top bar -->
+<header class="glass border-b border-primary/20 px-4 py-3 flex items-center gap-3 shrink-0">
+  <a href="/library" class="text-on-surface-variant hover:text-primary transition-colors">
+    <span class="msym text-2xl">arrow_back</span>
+  </a>
+  <div class="flex-1 min-w-0">
+    <h1 id="book-title" class="text-sm font-bold truncate text-primary">جاري التحميل...</h1>
+    <p id="book-author" class="text-xs text-on-surface-variant truncate"></p>
+  </div>
+  <div class="flex items-center gap-2">
+    <button id="fav-btn" onclick="toggleFavorite()" class="glass flex items-center gap-1.5 px-3 py-2 rounded-xl text-xs transition-all hover:border-primary/40">
+      <span class="msym text-lg" id="fav-icon">favorite_border</span>
+      <span id="fav-text" class="hidden sm:inline">إضافة</span>
+    </button>
+    <a id="download-btn" href="#" class="glass flex items-center gap-1.5 px-3 py-2 rounded-xl text-xs text-on-surface-variant hover:text-primary transition-all" download>
+      <span class="msym text-lg">download</span>
+      <span class="hidden sm:inline">تحميل</span>
+    </a>
+    <button onclick="askAboutBook()" class="bg-primary text-surface font-bold px-3 py-2 rounded-xl text-xs flex items-center gap-1.5 hover:shadow-[0_0_15px_rgba(78,222,163,0.4)] transition-all">
+      <span class="msym text-lg">auto_awesome</span>
+      <span class="hidden sm:inline">اسأل</span>
+    </button>
+  </div>
+</header>
+
+<!-- PDF Viewer -->
+<div class="flex-1 relative overflow-hidden" style="height:100%;min-height:0">
+  <div id="loading">
+    <div class="spinner mb-4"></div>
+    <p class="text-on-surface-variant text-sm">جاري تحميل الكتاب...</p>
+  </div>
+  <iframe id="pdf-frame" title="PDF Reader"></iframe>
+</div>
+
+<!-- Ask modal -->
+<div id="ask-modal" class="hidden fixed inset-0 z-50 flex items-end justify-center p-4 bg-black/60">
+  <div class="glass rounded-3xl p-6 w-full max-w-lg">
+    <div class="flex items-center justify-between mb-4">
+      <h3 class="font-bold flex items-center gap-2"><span class="msym text-primary">auto_awesome</span>اسأل عن هذا الكتاب</h3>
+      <button onclick="document.getElementById('ask-modal').classList.add('hidden')" class="text-on-surface-variant hover:text-primary"><span class="msym text-xl">close</span></button>
+    </div>
+    <textarea id="ask-input" rows="3" dir="rtl" placeholder="اكتب سؤالك..."
+      class="w-full bg-surface-high border border-outline-variant rounded-xl px-4 py-3 text-sm text-on-surface outline-none focus:border-primary transition-colors resize-none mb-3"></textarea>
+    <button onclick="doAsk()" class="w-full bg-primary text-surface font-bold py-3 rounded-xl text-sm hover:shadow-[0_0_20px_rgba(78,222,163,0.4)] transition-all">
+      <span class="msym text-lg align-middle mr-1">search</span> بحث
+    </button>
+    <div id="ask-answer" class="hidden mt-4 bg-surface-high rounded-xl p-4 text-sm leading-7 max-h-48 overflow-y-auto text-on-surface"></div>
+  </div>
+</div>
+
+<script>
+const params = new URLSearchParams(location.search);
+const bookId = params.get('id');
+const bookTitle = decodeURIComponent(params.get('title') || '');
+const bookAuthor = decodeURIComponent(params.get('author') || '');
+const userToken = localStorage.getItem('rag_token');
+const API_KEY = 'zRf77E_brSEnA2BUi40uxLxT4uofAk661WMZb-cdv4M';
+let isFavorite = false;
+
+// Set title
+document.getElementById('book-title').textContent = bookTitle || 'كتاب';
+document.getElementById('book-author').textContent = bookAuthor;
+document.title = (bookTitle || 'كتاب') + ' — AfakZaid';
+
+// Load PDF — public, no auth required
+if (bookId) {
+  const pdfUrl      = `/api/library/${bookId}/pdf`;
+  const titleEl = document.getElementById('book-title-loading');
+  if (titleEl) titleEl.textContent = bookTitle || '';
+
+  // Reading progress — restore last page
+  const savedPage = localStorage.getItem(`progress_${bookId}`);
+  if (savedPage && parseInt(savedPage) > 1) {
+    const resume = confirm(`استئناف القراءة من صفحة ${savedPage}؟`);
+    if (resume) {
+      window.history.replaceState({}, '', `?id=${bookId}&title=${encodeURIComponent(bookTitle||'')}&page=${savedPage}`);
+    }
+  }
+  const downloadUrl = `/api/library/${bookId}/download`;
+  const frame = document.getElementById('pdf-frame');
+  const isMobile = /Android|iPhone|iPad|iPod/i.test(navigator.userAgent);
+
+  document.getElementById('download-btn').href = downloadUrl;
+  document.getElementById('download-btn').removeAttribute('onclick');
+
+  function showPdfError() {
+    document.getElementById('loading').style.display = 'flex';
+    document.getElementById('loading').innerHTML = `
+      <div class="text-center px-8">
+        <span class="msym text-6xl text-on-surface-variant/30 block mb-4">picture_as_pdf</span>
+        <p class="text-on-surface-variant font-semibold mb-2">PDF غير متاح حالياً</p>
+        <p class="text-xs text-on-surface-variant/60 mb-6">يجب إعادة رفع الكتاب لتفعيل القراءة</p>
+        <a href="${downloadUrl}" class="bg-primary/10 border border-primary/20 text-primary px-6 py-3 rounded-xl text-sm inline-block">
+          تحميل الكتاب
+        </a>
+      </div>`;
+  }
+
+  if (isMobile) {
+    // Mobile browsers can't render PDFs in iframes — use Google Docs viewer
+    const fullUrl = window.location.origin + pdfUrl;
+    frame.style.cssText = 'position:absolute;inset:0;width:100%;height:100%;border:none';
+    frame.src = `https://docs.google.com/viewer?url=${encodeURIComponent(fullUrl)}&embedded=true`;
+    document.getElementById('loading').style.display = 'none';
+  } else {
+    // Desktop: direct iframe src
+    frame.style.cssText = 'position:absolute;inset:0;width:100%;height:100%;border:none';
+    frame.src = pdfUrl;
+    setTimeout(() => document.getElementById('loading').style.display = 'none', 1000);
+  }
+
+  if (userToken) checkFavorite();
+}
+
+async function checkFavorite() {
+  try {
+    const r = await fetch(`/api/favorites/check/${bookId}`, {headers:{'Authorization':'Bearer '+userToken}});
+    const d = await r.json();
+    isFavorite = d.is_favorite;
+    updateFavBtn();
+  } catch(e) {}
+}
+
+function updateFavBtn() {
+  document.getElementById('fav-icon').textContent = isFavorite ? 'favorite' : 'favorite_border';
+  document.getElementById('fav-icon').style.color = isFavorite ? '#f87171' : '';
+  document.getElementById('fav-text').textContent = isFavorite ? 'مفضلة' : 'إضافة';
+}
+
+async function toggleFavorite() {
+  if (!userToken) { window.location.href='/login'; return; }
+  try {
+    const method = isFavorite ? 'DELETE' : 'POST';
+    await fetch(`/api/favorites/${bookId}`, {method, headers:{'Authorization':'Bearer '+userToken}});
+    isFavorite = !isFavorite;
+    updateFavBtn();
+  } catch(e) {}
+}
+
+function askAboutBook() {
+  document.getElementById('ask-modal').classList.remove('hidden');
+  document.getElementById('ask-input').focus();
+}
+
+async function doAsk() {
+  const q = document.getElementById('ask-input').value.trim();
+  if (!q) return;
+  const ansEl = document.getElementById('ask-answer');
+  ansEl.classList.remove('hidden');
+  ansEl.textContent = '...';
+  try {
+    const r = await fetch('/query', {
+      method:'POST',
+      headers:{'Content-Type':'application/json','X-API-Key':API_KEY},
+      body: JSON.stringify({query: q + ' في كتاب: ' + bookTitle, session_id: 'reader-'+(bookId||'guest')})
+    });
+    const d = await r.json();
+    ansEl.textContent = d.answer;
+  } catch(e) { ansEl.textContent = 'خطأ: ' + e.message; }
+}
+
+document.getElementById('ask-input').addEventListener('keydown', e => {
+  if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); doAsk(); }
+});
+</script>
+</body>
+</html>

storage.py

@@ -0,0 +1,207 @@
+"""
+storage.py — Unified file storage via MinIO (primary) with optional R2 replication
+Architecture:
+  - Local disk: /mnt/rag-media/ (always kept as hot cache)
+  - MinIO:       primary object store on VPS (rag-media, rag-covers buckets)
+  - Cloudflare R2: replication target (configured via environment variables)
+
+All upload operations write to disk AND MinIO.
+R2 replication is handled by MinIO bucket replication (configured separately).
+For serving: use MinIO presigned URL or R2 public URL if CDN is enabled.
+"""
+import os, logging, boto3
+from botocore.exceptions import ClientError
+
+log = logging.getLogger("rag.storage")
+
+# ── MinIO config ──────────────────────────────────────────────────────────────
+MINIO_ENDPOINT  = os.getenv("MINIO_ENDPOINT",   "http://minio:9000")
+# Use app credentials if set, fall back to root credentials
+MINIO_ACCESS    = os.getenv("MINIO_ACCESS_KEY", "") or os.getenv("MINIO_ROOT_USER", "")
+MINIO_SECRET    = os.getenv("MINIO_SECRET_KEY", "") or os.getenv("MINIO_ROOT_PASSWORD", "")
+BUCKET_MEDIA    = "rag-media"
+BUCKET_COVERS   = "rag-covers"
+
+# ── Cloudflare R2 config (optional) ──────────────────────────────────────────
+R2_ENDPOINT     = os.getenv("R2_ENDPOINT",       "")   # https://<account>.r2.cloudflarestorage.com
+R2_ACCESS_KEY   = os.getenv("R2_ACCESS_KEY",     "")
+R2_SECRET_KEY   = os.getenv("R2_SECRET_KEY",     "")
+R2_BUCKET       = os.getenv("R2_BUCKET",         "rag-media")
+R2_PUBLIC_URL   = os.getenv("R2_PUBLIC_URL",     "")   # optional CDN URL
+
+_minio_client = None
+_r2_client    = None
+
+
+def _get_minio():
+    global _minio_client
+    if _minio_client is None:
+        if not MINIO_ACCESS:
+            raise RuntimeError("MINIO_ACCESS_KEY not set")
+        _minio_client = boto3.client(
+            "s3",
+            endpoint_url         = MINIO_ENDPOINT,
+            aws_access_key_id    = MINIO_ACCESS,
+            aws_secret_access_key = MINIO_SECRET,
+            region_name          = "us-east-1",
+        )
+    return _minio_client
+
+
+def _get_r2():
+    global _r2_client
+    if _r2_client is None and R2_ENDPOINT and R2_ACCESS_KEY:
+        _r2_client = boto3.client(
+            "s3",
+            endpoint_url          = R2_ENDPOINT,
+            aws_access_key_id     = R2_ACCESS_KEY,
+            aws_secret_access_key = R2_SECRET_KEY,
+            region_name           = "auto",
+        )
+    return _r2_client
+
+
+def _upload(bucket, key, file_path, content_type="application/octet-stream"):
+    """Upload to R2 only (MinIO disabled)."""
+    r2 = _get_r2()
+    if r2:
+        try:
+            r2.upload_file(file_path, R2_BUCKET, key,
+                           ExtraArgs={"ContentType": content_type})
+            log.info(f"R2 upload: r2://{R2_BUCKET}/{key}")
+        except Exception as e:
+            log.warning(f"R2 upload failed (non-fatal): {e}")
+
+
+def upload_pdf(file_path: str, filename: str, book_id: int = None):
+    """
+    Upload PDF to R2 (if configured).
+    If book_id provided, updates books.pdf_url in DB with R2 public URL.
+    """
+    if not R2_ENDPOINT or not R2_ACCESS_KEY:
+        log.info(f"R2 not configured — skipping upload for {filename}")
+        return
+
+    try:
+        r2 = _get_r2()
+        if r2:
+            r2.upload_file(file_path, R2_BUCKET, f"pdfs/{filename}",
+                           ExtraArgs={"ContentType": "application/pdf"})
+            log.info(f"R2 upload: r2://{R2_BUCKET}/pdfs/{filename}")
+
+            # Store R2 URL in DB if book_id known
+            if book_id and R2_PUBLIC_URL:
+                try:
+                    import psycopg2
+                    pdf_url = f"{R2_PUBLIC_URL.rstrip('/')}/pdfs/{filename}"
+                    conn = psycopg2.connect(os.getenv("POSTGRES_URL", ""))
+                    cur  = conn.cursor()
+                    cur.execute("UPDATE books SET pdf_url=%s WHERE id=%s", (pdf_url, book_id))
+                    conn.commit(); cur.close(); conn.close()
+                    log.info(f"DB pdf_url updated: {pdf_url}")
+                except Exception as e:
+                    log.warning(f"DB pdf_url update failed: {e}")
+    except Exception as e:
+        log.error(f"R2 PDF upload failed: {e}")
+
+
+def upload_cover(file_path: str, filename: str, book_id: int = None):
+    """
+    Upload cover image to R2 (if configured).
+    If book_id provided, updates books.cover_url in DB with R2 public URL.
+    """
+    if not R2_ENDPOINT or not R2_ACCESS_KEY:
+        return
+
+    try:
+        r2 = _get_r2()
+        if r2:
+            r2.upload_file(file_path, R2_BUCKET, f"covers/{filename}",
+                           ExtraArgs={"ContentType": "image/jpeg"})
+            log.info(f"R2 upload: r2://{R2_BUCKET}/covers/{filename}")
+
+            if book_id and R2_PUBLIC_URL:
+                try:
+                    import psycopg2
+                    cover_url = f"{R2_PUBLIC_URL.rstrip('/')}/covers/{filename}"
+                    conn = psycopg2.connect(os.getenv("POSTGRES_URL", ""))
+                    cur  = conn.cursor()
+                    cur.execute("UPDATE books SET cover_url=%s WHERE id=%s", (cover_url, book_id))
+                    conn.commit(); cur.close(); conn.close()
+                    log.info(f"DB cover_url updated: {cover_url}")
+                except Exception as e:
+                    log.warning(f"DB cover_url update failed: {e}")
+    except Exception as e:
+        log.error(f"R2 cover upload failed: {e}")
+
+
+def get_pdf_url(filename: str, expires: int = 3600) -> str:
+    """
+    Get a presigned URL for PDF download.
+    Returns R2 public URL if configured, otherwise MinIO presigned URL.
+    """
+    if R2_PUBLIC_URL:
+        return f"{R2_PUBLIC_URL}/pdfs/{filename}"
+    try:
+        s3  = _get_minio()
+        url = s3.generate_presigned_url(
+            "get_object",
+            Params={"Bucket": BUCKET_MEDIA, "Key": f"pdfs/{filename}"},
+            ExpiresIn=expires,
+        )
+        return url
+    except Exception as e:
+        log.error(f"get_pdf_url error: {e}")
+        return ""
+
+
+def get_cover_url(book_id: int, expires: int = 86400) -> str:
+    """Get a presigned URL for book cover image."""
+    if R2_PUBLIC_URL:
+        return f"{R2_PUBLIC_URL}/covers/{book_id}.jpg"
+    try:
+        s3  = _get_minio()
+        url = s3.generate_presigned_url(
+            "get_object",
+            Params={"Bucket": BUCKET_COVERS, "Key": f"{book_id}.jpg"},
+            ExpiresIn=expires,
+        )
+        return url
+    except Exception as e:
+        log.error(f"get_cover_url error: {e}")
+        return ""
+
+
+def file_exists_in_minio(bucket: str, key: str) -> bool:
+    try:
+        _get_minio().head_object(Bucket=bucket, Key=key)
+        return True
+    except ClientError:
+        return False
+
+
+def migrate_existing_files():
+    """One-time migration: upload all existing local PDFs and covers to MinIO."""
+    import os
+    pdf_dir   = "/mnt/rag-media/pdfs"
+    cover_dir = "/mnt/rag-media/covers"
+    migrated  = 0
+
+    if os.path.isdir(pdf_dir):
+        for fn in os.listdir(pdf_dir):
+            if fn.endswith(".pdf"):
+                path = os.path.join(pdf_dir, fn)
+                if not file_exists_in_minio(BUCKET_MEDIA, f"pdfs/{fn}"):
+                    upload_pdf(path, fn)
+                    migrated += 1
+
+    if os.path.isdir(cover_dir):
+        for fn in os.listdir(cover_dir):
+            if fn.endswith(".jpg"):
+                path = os.path.join(cover_dir, fn)
+                if not file_exists_in_minio(BUCKET_COVERS, fn):
+                    upload_cover(path, fn)
+                    migrated += 1
+
+    log.info(f"Migration complete: {migrated} files uploaded to MinIO")
+    return migrated

tasks.py

@@ -0,0 +1,169 @@
+"""
+tasks.py — Celery tasks for bulk PDF ingestion
+Replaces the threading-based worker.py background thread.
+Each bulk job file becomes a separate Celery task.
+"""
+import os, sys, logging
+import psycopg2
+from psycopg2.extras import RealDictCursor
+
+sys.path.insert(0, "/opt/rag")
+from celery_app import app
+
+log = logging.getLogger("rag.tasks")
+logging.basicConfig(level=logging.INFO)
+
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+BOOKS_DIR    = "/mnt/rag-media/pdfs"
+MEDIA_DIR    = "/mnt/rag-media/pdfs"
+
+
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+
+def set_file_status(file_id, status, error=None, meta=None, book_id=None):
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("""UPDATE bulk_job_files
+            SET status=%s, error=%s, meta=%s, book_id=%s
+            WHERE id=%s""",
+            (status, error,
+             __import__("json").dumps(meta) if meta else None,
+             book_id, file_id))
+        conn.commit(); cur.close(); conn.close()
+    except Exception as e:
+        log.error(f"set_file_status error: {e}")
+
+
+def update_job_counters(job_id):
+    try:
+        conn = get_db(); cur = conn.cursor()
+        cur.execute("""
+            SELECT
+                COUNT(*) FILTER (WHERE status='done')     as done,
+                COUNT(*) FILTER (WHERE status='error')    as failed,
+                COUNT(*) FILTER (WHERE status='skipped')  as skipped,
+                COUNT(*)                                   as total
+            FROM bulk_job_files WHERE job_id=%s""", (job_id,))
+        r = cur.fetchone()
+        done, failed, skipped, total = r
+        finished = (done + failed + skipped) >= total
+        job_status = "done" if finished else "processing"
+        cur.execute("""UPDATE bulk_jobs SET status=%s, done=%s, failed=%s
+            WHERE id=%s""", (job_status, done, failed, job_id))
+        conn.commit(); cur.close(); conn.close()
+    except Exception as e:
+        log.error(f"update_job_counters error: {e}")
+
+
+@app.task(bind=True, name="tasks.process_bulk_file",
+          max_retries=2, default_retry_delay=300)
+def process_bulk_file(self, job_id: str, file_id: int,
+                      filename: str, uploader_id: int,
+                      meta_override: dict = None):
+    """
+    Process a single PDF file.
+    Used by both single uploads (1 file) and bulk uploads (N files).
+    meta_override: optional {title, author, category} from single upload form.
+    For single uploads, metadata hints are stored as JSON in bulk_job_files.error.
+    """
+    # Read metadata hints from bulk_job_files if meta_override not passed
+    if meta_override is None:
+        try:
+            conn = get_db(); cur = conn.cursor()
+            cur.execute("SELECT error FROM bulk_job_files WHERE id=%s", (file_id,))
+            row = cur.fetchone(); cur.close(); conn.close()
+            if row and row[0] and row[0].startswith("{"):
+                import json
+                hints = json.loads(row[0])
+                if any(hints.values()):
+                    meta_override = hints
+                # Clear the hint so real errors can be stored
+                conn = get_db(); cur = conn.cursor()
+                cur.execute("UPDATE bulk_job_files SET error=NULL WHERE id=%s", (file_id,))
+                conn.commit(); cur.close(); conn.close()
+        except Exception as _e:
+            log.warning(f"[task] meta hint read failed: {_e}")
+    """
+    Process a single PDF file from a bulk upload job.
+    Retries up to 2 times with 5-minute delay on failure.
+    """
+    log.info(f"[task] start: {filename} (job={job_id})")
+    set_file_status(file_id, "processing")
+
+    pdf_src = os.path.join(BOOKS_DIR, filename)
+    if not os.path.exists(pdf_src):
+        set_file_status(file_id, "error", error="PDF not found on disk")
+        update_job_counters(job_id)
+        return
+
+    # ── Step 1: Compress (before ingest, only replaces if smaller) ─────────
+    set_file_status(file_id, "compressing")
+    try:
+        from compress import compress_pdf
+        c_result = compress_pdf(pdf_src)
+        log.info(f"[task] compress {filename}: {c_result}")
+    except Exception as e:
+        log.warning(f"[task] compression skipped: {e}")
+
+    # ── Step 2: Ingest ──────────────────────────────────────────────────────
+    set_file_status(file_id, "ingesting")
+    try:
+        import ingest as _ingest
+        _ingest.ingest_pdf(pdf_src)
+    except Exception as e:
+        log.error(f"[task] ingest failed: {e}")
+        try:
+            raise self.retry(exc=e)
+        except self.MaxRetriesExceededError:
+            set_file_status(file_id, "error", error=f"Ingest failed: {e}")
+            update_job_counters(job_id)
+            return
+
+    # ── Step 3: Extract metadata ─────────────────────────────────────────────
+    set_file_status(file_id, "extracting")
+    pdf_path = os.path.join(MEDIA_DIR, filename)
+    if not os.path.exists(pdf_path):
+        pdf_path = pdf_src
+
+    from meta_extract import extract_meta, save_meta_to_db
+    meta = extract_meta(pdf_path, filename)
+
+    # ── Step 4: Save metadata ────────────────────────────────────────────────
+    try:
+        save_meta_to_db(filename, meta, uploader_id=uploader_id)
+        conn = get_db()
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT id FROM books WHERE filename=%s", (filename,))
+            book = cur.fetchone()
+            book_id = book["id"] if book else None
+        conn.close()
+        # Apply form overrides from single upload (title/author/category)
+        if meta_override:
+            for field in ("title", "author", "category", "description"):
+                if meta_override.get(field):
+                    meta[field] = meta_override[field]
+            save_meta_to_db(filename, meta, uploader_id=uploader_id)
+
+        set_file_status(file_id, "done", meta=meta, book_id=book_id)
+    except Exception as e:
+        set_file_status(file_id, "error", error=str(e), meta=meta)
+
+    # ── Step 5: Rebuild BM25 ─────────────────────────────────────────────────
+    try:
+        from brain import rebuild_bm25
+        rebuild_bm25()
+    except Exception as e:
+        log.warning(f"[task] BM25 rebuild skipped: {e}")
+
+    # ── Step 5: Index in Typesense ────────────────────────────────────────────
+    try:
+        from typesense_client import index_book
+        if book_id:
+            index_book(book_id)
+    except Exception as e:
+        log.warning(f"[task] Typesense index skipped: {e}")
+
+    update_job_counters(job_id)
+    log.info(f"[task] done: {filename} — {meta.get('title')}")

typesense_client.py

@@ -0,0 +1,174 @@
+"""
+typesense_client.py — Typesense Arabic full-text search integration
+Collection: books — indexed fields: title, author, category, description, filename
+Search augments the library endpoint with typo-tolerance and relevance ranking.
+"""
+import os, logging, psycopg2
+from psycopg2.extras import RealDictCursor
+
+log = logging.getLogger("rag.typesense")
+
+TYPESENSE_HOST   = os.getenv("TYPESENSE_HOST",    "typesense")
+TYPESENSE_PORT   = int(os.getenv("TYPESENSE_PORT", "8108"))
+TYPESENSE_KEY    = os.getenv("TYPESENSE_API_KEY",  "")
+POSTGRES_URL     = os.getenv("POSTGRES_URL",       "")
+COLLECTION       = "books"
+
+_client = None
+
+
+def get_client():
+    global _client
+    if _client is None:
+        import typesense
+        _client = typesense.Client({
+            "nodes": [{"host": TYPESENSE_HOST, "port": TYPESENSE_PORT, "protocol": "http"}],
+            "api_key": TYPESENSE_KEY,
+            "connection_timeout_seconds": 5,
+        })
+    return _client
+
+
+def ensure_collection():
+    """Create the books collection if it doesn't exist."""
+    client = get_client()
+    schema = {
+        "name": COLLECTION,
+        "fields": [
+            {"name": "id",          "type": "string"},
+            {"name": "title",       "type": "string"},
+            {"name": "author",      "type": "string", "optional": True},
+            {"name": "category",    "type": "string", "optional": True, "facet": True},
+            {"name": "language",    "type": "string", "optional": True, "facet": True},
+            {"name": "description", "type": "string", "optional": True},
+            {"name": "filename",    "type": "string"},
+            {"name": "total_chunks","type": "int32",  "default": 0},
+            {"name": "uploaded_at", "type": "int64",  "default": 0},
+        ],
+        "default_sorting_field": "uploaded_at",
+        "token_separators": ["!", ".", "(", ")", "-"],
+    }
+    try:
+        existing = [c["name"] for c in client.collections.retrieve()]
+        if COLLECTION not in existing:
+            client.collections.create(schema)
+            log.info(f"Typesense collection '{COLLECTION}' created")
+        return True
+    except Exception as e:
+        log.error(f"ensure_collection error: {e}")
+        return False
+
+
+def index_book(book_id: int):
+    """Index or update a single book in Typesense."""
+    if not TYPESENSE_KEY:
+        return
+    try:
+        conn = psycopg2.connect(POSTGRES_URL)
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT id, title, author, category, language, description,
+                       filename, total_chunks, uploaded_at
+                FROM books WHERE id = %s
+            """, (book_id,))
+            book = cur.fetchone()
+        conn.close()
+        if not book:
+            return
+
+        client = get_client()
+        ensure_collection()
+
+        doc = {
+            "id":           str(book["id"]),
+            "title":        book["title"] or book["filename"].replace(".pdf", ""),
+            "author":       book["author"] or "",
+            "category":     book["category"] or "عام",
+            "language":     book["language"] or "ar",
+            "description":  book["description"] or "",
+            "filename":     book["filename"],
+            "total_chunks": book["total_chunks"] or 0,
+            "uploaded_at":  int(book["uploaded_at"].timestamp()) if book["uploaded_at"] else 0,
+        }
+        client.collections[COLLECTION].documents.upsert(doc)
+        log.info(f"Typesense indexed: {doc['title']} (id={book_id})")
+    except Exception as e:
+        log.error(f"index_book error (book_id={book_id}): {e}")
+
+
+def index_all_books():
+    """Bulk index all books. Called on startup or manual refresh."""
+    if not TYPESENSE_KEY:
+        log.warning("TYPESENSE_API_KEY not set — skipping index")
+        return 0
+    try:
+        conn = psycopg2.connect(POSTGRES_URL)
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT id, title, author, category, language, description,
+                       filename, total_chunks, uploaded_at
+                FROM books ORDER BY id
+            """)
+            books = cur.fetchall()
+        conn.close()
+
+        client = get_client()
+        ensure_collection()
+
+        docs = []
+        for b in books:
+            docs.append({
+                "id":           str(b["id"]),
+                "title":        b["title"] or b["filename"].replace(".pdf", ""),
+                "author":       b["author"] or "",
+                "category":     b["category"] or "عام",
+                "language":     b["language"] or "ar",
+                "description":  b["description"] or "",
+                "filename":     b["filename"],
+                "total_chunks": b["total_chunks"] or 0,
+                "uploaded_at":  int(b["uploaded_at"].timestamp()) if b["uploaded_at"] else 0,
+            })
+
+        if docs:
+            client.collections[COLLECTION].documents.import_(docs, {"action": "upsert"})
+            log.info(f"Typesense bulk indexed: {len(docs)} books")
+        return len(docs)
+    except Exception as e:
+        log.error(f"index_all_books error: {e}")
+        return 0
+
+
+def search_books(query: str, category: str = "", limit: int = 20) -> list:
+    """
+    Full-text search with Typesense.
+    Returns list of book dicts sorted by relevance.
+    Falls back to empty list on error (caller falls back to DB).
+    """
+    if not TYPESENSE_KEY or not query.strip():
+        return []
+    try:
+        client  = get_client()
+        params  = {
+            "q":                  query,
+            "query_by":           "title,author,category,description",
+            "query_by_weights":   "4,3,2,1",
+            "per_page":           limit,
+            "typo_tokens_threshold": 1,
+            "num_typos":          2,
+        }
+        if category:
+            params["filter_by"] = f"category:={category}"
+
+        result = client.collections[COLLECTION].documents.search(params)
+        return [hit["document"] for hit in result.get("hits", [])]
+    except Exception as e:
+        log.error(f"search_books error: {e}")
+        return []
+
+
+def delete_book(book_id: int):
+    """Remove a book from Typesense index."""
+    try:
+        get_client().collections[COLLECTION].documents[str(book_id)].delete()
+    except Exception as e:
+        log.warning(f"Typesense delete skipped: {e}")

worker.py

@@ -0,0 +1,206 @@
+"""
+Bulk Upload Worker — AfakZaid RAG
+Runs as a background thread inside FastAPI.
+Picks queued bulk jobs from DB, processes each PDF:
+  1. Ingest (embed into Qdrant)
+  2. Extract metadata via Ollama
+  3. Update DB record
+"""
+import os, subprocess, threading, time, json, re, logging
+import psycopg2
+from psycopg2.extras import RealDictCursor
+import requests as http_req
+
+log = logging.getLogger("worker")
+logging.basicConfig(level=logging.INFO)
+
+POSTGRES_URL = os.getenv("POSTGRES_URL", "")
+OLLAMA_URL   = os.getenv("OLLAMA_URL", "http://localhost:11434")
+OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2:7b")
+BOOKS_DIR    = "/opt/rag/books"
+MEDIA_DIR    = "/mnt/rag-media/pdfs"
+
+_stop_event = threading.Event()
+
+
+def get_db():
+    return psycopg2.connect(POSTGRES_URL)
+
+
+def set_file_status(file_id: int, status: str, error: str = None, meta: dict = None, book_id: int = None):
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            if meta and book_id:
+                cur.execute("""UPDATE bulk_job_files
+                    SET status=%s, error=%s, meta=%s, book_id=%s, updated_at=NOW()
+                    WHERE id=%s""", (status, error, json.dumps(meta), book_id, file_id))
+            elif meta:
+                cur.execute("""UPDATE bulk_job_files
+                    SET status=%s, error=%s, meta=%s, updated_at=NOW()
+                    WHERE id=%s""", (status, error, json.dumps(meta), file_id))
+            else:
+                cur.execute("""UPDATE bulk_job_files
+                    SET status=%s, error=%s, updated_at=NOW()
+                    WHERE id=%s""", (status, error, file_id))
+        conn.commit()
+
+
+def update_job_counters(job_id: str):
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""SELECT
+                COUNT(*) FILTER (WHERE status='done')   as done,
+                COUNT(*) FILTER (WHERE status='error')  as failed,
+                COUNT(*) as total
+                FROM bulk_job_files WHERE job_id=%s""", (job_id,))
+            r = cur.fetchone()
+            done, failed, total = r['done'], r['failed'], r['total']
+            finished = (done + failed) == total
+            job_status = 'done' if finished else 'processing'
+            cur.execute("""UPDATE bulk_jobs
+                SET done=%s, failed=%s, status=%s, updated_at=NOW()
+                WHERE id=%s""", (done, failed, job_status, job_id))
+        conn.commit()
+
+
+def extract_meta_ollama(pdf_path: str, filename: str) -> dict:
+    """Extract title, author, category, description from first 3 pages via Ollama."""
+    try:
+        import fitz
+        doc = fitz.open(pdf_path)
+        text = "".join(doc[i].get_text() for i in range(min(3, len(doc))))[:3000]
+        doc.close()
+    except Exception as e:
+        log.warning(f"fitz error for {filename}: {e}")
+        text = filename.replace('.pdf', '')
+
+    prompt = f"""استخرج من النص التالي معلومات الكتاب وأجب بـ JSON فقط بدون أي نص إضافي:
+{{"title":"عنوان الكتاب","author":"اسم المؤلف","category":"التصنيف مثل فقه أو حديث أو تفسير أو أدب","description":"وصف مختصر جملة أو جملتان"}}
+
+النص:
+{text}"""
+
+    try:
+        r = http_req.post(f"{OLLAMA_URL}/api/generate",
+            json={"model": OLLAMA_MODEL, "prompt": prompt, "stream": False,
+                  "options": {"num_predict": 256, "temperature": 0.2}},
+            timeout=90)
+        response_text = r.json().get("response", "")
+        m = re.search(r'\{[^}]+\}', response_text, re.DOTALL)
+        if m:
+            meta = json.loads(m.group())
+            # Ensure all keys present
+            return {
+                "title":       meta.get("title", filename.replace('.pdf','')).strip() or filename.replace('.pdf',''),
+                "author":      meta.get("author", "").strip(),
+                "category":    meta.get("category", "عام").strip() or "عام",
+                "description": meta.get("description", "").strip(),
+            }
+    except Exception as e:
+        log.warning(f"Ollama meta extraction failed for {filename}: {e}")
+
+    return {"title": filename.replace('.pdf',''), "author": "", "category": "عام", "description": ""}
+
+
+def process_file(file_id: int, job_id: str, filename: str, uploader_id: int):
+    """Full pipeline for one PDF file."""
+    pdf_src = os.path.join(BOOKS_DIR, filename)
+
+    # ── Step 1: Ingest (embed into Qdrant) ────────────────────────────────
+    set_file_status(file_id, 'ingesting')
+    try:
+        result = subprocess.run(
+            ["python3", "/opt/rag/ingest.py", pdf_src],
+            capture_output=True, text=True, timeout=300, env={**os.environ}
+        )
+        if result.returncode != 0 and "already ingested" not in result.stdout:
+            log.warning(f"ingest warning for {filename}: {result.stderr[-200:]}")
+    except subprocess.TimeoutExpired:
+        set_file_status(file_id, 'error', error='Ingest timed out')
+        update_job_counters(job_id)
+        return
+    except Exception as e:
+        set_file_status(file_id, 'error', error=str(e))
+        update_job_counters(job_id)
+        return
+
+    # ── Step 2: Extract metadata via Ollama ───────────────────────────────
+    set_file_status(file_id, 'extracting')
+    pdf_path = os.path.join(MEDIA_DIR, filename)
+    if not os.path.exists(pdf_path):
+        pdf_path = pdf_src
+    from meta_extract import extract_meta, save_meta_to_db
+    meta = extract_meta(pdf_path, filename)
+
+    # ── Step 3: Update book record in DB ──────────────────────────────────
+    try:
+        save_meta_to_db(filename, meta, uploader_id=uploader_id)
+        with get_db() as conn:
+            with conn.cursor(cursor_factory=RealDictCursor) as cur:
+                cur.execute("SELECT id FROM books WHERE filename=%s", (filename,))
+                book = cur.fetchone()
+                book_id = book['id'] if book else None
+        set_file_status(file_id, 'done', meta=meta, book_id=book_id)
+    except Exception as e:
+        set_file_status(file_id, 'error', error=str(e), meta=meta)
+
+    update_job_counters(job_id)
+    log.info(f"[worker] done: {filename} → {meta['title']}")
+    # Rebuild BM25 index
+    try:
+        from brain import rebuild_bm25; rebuild_bm25()
+    except Exception as e:
+        log.warning(f'[worker] BM25 rebuild skipped: {e}')
+
+
+def process_job(job_id: str):
+    """Process all pending files in a job sequentially."""
+    log.info(f"[worker] starting job {job_id}")
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("UPDATE bulk_jobs SET status='processing', updated_at=NOW() WHERE id=%s", (job_id,))
+            cur.execute("SELECT id, filename, job_id FROM bulk_job_files WHERE job_id=%s AND status='queued' ORDER BY id", (job_id,))
+            files = cur.fetchall()
+            cur.execute("SELECT uploader_id FROM bulk_jobs WHERE id=%s", (job_id,))
+            job = cur.fetchone()
+        conn.commit()
+
+    uploader_id = job['uploader_id'] if job else None
+    for f in files:
+        process_file(f['id'], f['job_id'], f['filename'], uploader_id)
+
+    log.info(f"[worker] job {job_id} complete")
+
+
+def worker_loop():
+    """Main worker loop — polls DB for pending jobs."""
+    log.info("[worker] started")
+    while not _stop_event.is_set():
+        try:
+            with get_db() as conn:
+                with conn.cursor(cursor_factory=RealDictCursor) as cur:
+                    cur.execute("""SELECT id FROM bulk_jobs
+                        WHERE status='pending' ORDER BY created_at LIMIT 1""")
+                    job = cur.fetchone()
+            if job:
+                process_job(job['id'])
+            else:
+                time.sleep(3)
+        except Exception as e:
+            log.error(f"[worker] loop error: {e}")
+            time.sleep(5)
+
+
+_worker_thread: threading.Thread = None
+
+
+def start_worker():
+    global _worker_thread
+    _stop_event.clear()
+    _worker_thread = threading.Thread(target=worker_loop, daemon=True, name="bulk-worker")
+    _worker_thread.start()
+    log.info("[worker] thread launched")
+
+
+def stop_worker():
+    _stop_event.set()