feat: complete AfakZaid RAG system v1.7.4
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
.secrets
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
versions/
|
||||
*.snapshot
|
||||
*.tar.gz
|
||||
@@ -0,0 +1,121 @@
|
||||
import requests, os, sys, time
|
||||
import concurrent.futures
|
||||
sys.path.insert(0, '/opt/rag')
|
||||
os.environ.setdefault('POSTGRES_URL','postgresql://raguser:rag_secure_2025@172.16.2.1:5432/ragdb')
|
||||
|
||||
BASE = 'http://localhost:8100'
|
||||
AK = 'zRf77E_brSEnA2BUi40uxLxT4uofAk661WMZb-cdv4M'
|
||||
ADMK = 'afakadmin-Ihb7kflFA682Ax3EICnV2ymr4KuNZSo7'
|
||||
QKEY = open('/opt/rag/.secrets').read()
|
||||
QKEY = [l.split('=',1)[1].strip() for l in QKEY.splitlines() if 'QDRANT_API_KEY' in l][0]
|
||||
RPASS= [l.split('=',1)[1].strip() for l in open('/opt/rag/.secrets').read().splitlines() if 'REDIS_PASSWORD' in l][0]
|
||||
QURL = 'http://172.16.2.1:6333'
|
||||
|
||||
results = []
|
||||
def chk(label, ok, detail=''):
|
||||
sym = '\u2705' if ok else '\u274c'
|
||||
results.append((ok, f" {sym} {label}" + (f" — {detail}" if detail else '')))
|
||||
|
||||
# Login
|
||||
tok = requests.post(f'{BASE}/auth/login', json={'email':'admin@afakzaid.com','password':'AdminPass2026!'}).json().get('token','')
|
||||
h_adm = {'X-API-Key': ADMK}
|
||||
h_api = {'X-API-Key': AK, 'Authorization': f'Bearer {tok}', 'Content-Type':'application/json'}
|
||||
|
||||
# Security checks
|
||||
for path in ['/docs','/redoc','/openapi.json']:
|
||||
r = requests.get(f'{BASE}{path}')
|
||||
chk(f'Docs disabled {path}', r.status_code == 404, f'HTTP {r.status_code}')
|
||||
|
||||
r = requests.get(f'{BASE}/')
|
||||
for hdr, want in [('X-Frame-Options','DENY'),('X-Content-Type-Options','nosniff'),
|
||||
('Strict-Transport-Security',None),('X-XSS-Protection',None),('Referrer-Policy',None)]:
|
||||
val = r.headers.get(hdr)
|
||||
ok = (val == want) if want else bool(val)
|
||||
chk(f'Header {hdr}', ok, val or 'MISSING')
|
||||
|
||||
cors = requests.get(f'{BASE}/stats', headers={'Origin':'https://evil.com'}).headers.get('Access-Control-Allow-Origin','none')
|
||||
chk('CORS blocks evil.com', cors != '*' and 'evil.com' not in cors, cors)
|
||||
|
||||
def hit_login(_):
|
||||
return requests.post(f'{BASE}/auth/login', json={'email':'x@x.com','password':'x'}, timeout=5).status_code
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=15) as ex:
|
||||
codes = list(ex.map(hit_login, range(15)))
|
||||
chk('Rate limiting (login)', 429 in codes, f'{codes.count(429)}/15 blocked')
|
||||
|
||||
# Firewall — read rules injected via env (iptables not available inside container)
|
||||
_fw_rules = os.getenv('IPTABLES_RULES','')
|
||||
for port, name in [(5432,'PostgreSQL'),(6333,'Qdrant'),(6379,'Redis'),(11434,'Ollama')]:
|
||||
has_drop = f'dpt:{port}' in _fw_rules and 'DROP' in _fw_rules
|
||||
chk(f'Firewall blocks {name}:{port}', has_drop, 'DROP rule confirmed' if has_drop else 'rule missing')
|
||||
|
||||
# Redis auth
|
||||
import redis as _redis
|
||||
try:
|
||||
_redis.from_url('redis://172.16.2.1:6379', socket_timeout=2).ping()
|
||||
chk('Redis requires password', False, 'no auth')
|
||||
except: chk('Redis requires password', True, 'auth enforced')
|
||||
rc = _redis.from_url(f'redis://:{RPASS}@172.16.2.1:6379', decode_responses=True)
|
||||
rc.set('audit_test','ok')
|
||||
chk('Redis R/W with password', rc.get('audit_test') == 'ok')
|
||||
|
||||
# Qdrant auth
|
||||
rn = requests.get(f'{QURL}/collections', timeout=3)
|
||||
ry = requests.get(f'{QURL}/collections', headers={'api-key': QKEY}, timeout=3)
|
||||
chk('Qdrant blocks no-key', rn.status_code in [401,403], f'HTTP {rn.status_code}')
|
||||
chk('Qdrant allows valid key', ry.status_code == 200, f'{len(ry.json().get("result",{}).get("collections",[]))} collections')
|
||||
|
||||
# Services
|
||||
r = requests.get(f'{BASE}/stats')
|
||||
s = r.json()
|
||||
chk('API alive', r.status_code == 200, f"books={s.get('books')} vectors={s.get('vectors')}")
|
||||
|
||||
r = requests.get(f'{BASE}/api/library')
|
||||
books = r.json().get('books',[])
|
||||
chk('Library endpoint', len(books) > 0, f'{len(books)} books returned')
|
||||
for b in books:
|
||||
has_meta = bool(b.get('title')) and b.get('title') != b.get('filename','').replace('.pdf','')
|
||||
chk(f"Book {b['id']} metadata", bool(b.get('author')), f"title={b.get('title','?')} author={b.get('author','MISSING')}")
|
||||
|
||||
# RAG query
|
||||
import time as _t
|
||||
_unique_sid = f'audit-{int(_t.time())}'
|
||||
r = requests.post(f'{BASE}/query', headers=h_api,
|
||||
json={'query':'ما المقصود بخضب الشعر وما أنواعه؟','session_id':_unique_sid}, timeout=200)
|
||||
if r.status_code == 200:
|
||||
d = r.json()
|
||||
chk('RAG query', bool(d.get('answer')), f"source={d.get('source')} llm={d.get('llm')} chunks={len(d.get('chunks',[]))}")
|
||||
chk('Hybrid search active', d.get('source') == 'hybrid')
|
||||
chk('Answer has content', len(d.get('answer','')) > 50)
|
||||
else:
|
||||
chk('RAG query', False, f'HTTP {r.status_code}: {r.text[:80]}')
|
||||
|
||||
# Admin
|
||||
r = requests.get(f'{BASE}/admin/books', headers=h_adm)
|
||||
chk('Admin endpoint', r.status_code == 200)
|
||||
r_unauth = requests.get(f'{BASE}/admin/books')
|
||||
chk('Admin requires key', r_unauth.status_code == 401)
|
||||
|
||||
# DB
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(os.environ['POSTGRES_URL'])
|
||||
cur = conn.cursor()
|
||||
for tbl, expected_min in [('books',4),('chunks',150),('users',1),('conversations',1)]:
|
||||
cur.execute(f'SELECT COUNT(*) FROM {tbl}')
|
||||
n = cur.fetchone()[0]
|
||||
chk(f'DB table {tbl}', n >= expected_min, f'{n} rows')
|
||||
cur.close(); conn.close()
|
||||
|
||||
# BM25 index
|
||||
from brain import bm25_index
|
||||
chk('BM25 index loaded', len(bm25_index._chunks) > 100, f'{len(bm25_index._chunks)} chunks indexed')
|
||||
|
||||
# Summary
|
||||
passed = sum(1 for ok,_ in results if ok)
|
||||
total = len(results)
|
||||
print(f"\n{'='*58}")
|
||||
print(f" AUDIT COMPLETE — {passed}/{total} checks passed {'✅' if passed==total else '⚠️'}")
|
||||
print(f"{'='*58}")
|
||||
[print(l) for _, l in results]
|
||||
if passed < total:
|
||||
print(f"\n ❌ FAILURES:")
|
||||
[print(l) for ok,l in results if not ok]
|
||||
@@ -0,0 +1,101 @@
|
||||
import os
|
||||
import hashlib
|
||||
import secrets
|
||||
import bcrypt
|
||||
import jwt
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
from fastapi import HTTPException, Depends, Request
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
|
||||
POSTGRES_URL = os.getenv("POSTGRES_URL", "")
|
||||
if not POSTGRES_URL:
|
||||
raise RuntimeError("POSTGRES_URL environment variable is not set")
|
||||
JWT_SECRET = os.getenv("JWT_SECRET", "")
|
||||
if not JWT_SECRET:
|
||||
raise RuntimeError("JWT_SECRET environment variable is not set — refusing to start")
|
||||
JWT_EXPIRE_HOURS = 24 * 7 # 7 days
|
||||
|
||||
bearer = HTTPBearer(auto_error=False)
|
||||
|
||||
def get_db():
|
||||
return psycopg2.connect(POSTGRES_URL)
|
||||
|
||||
# ── Password ──────────────────────────────────────────────────────────────────
|
||||
def hash_password(password: str) -> str:
|
||||
return bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode()
|
||||
|
||||
def verify_password(password: str, hashed: str) -> bool:
|
||||
return bcrypt.checkpw(password.encode(), hashed.encode())
|
||||
|
||||
# ── JWT ───────────────────────────────────────────────────────────────────────
|
||||
def create_token(user_id: int, role: str) -> str:
|
||||
payload = {
|
||||
"sub": str(user_id),
|
||||
"role": role,
|
||||
"exp": datetime.utcnow() + timedelta(hours=JWT_EXPIRE_HOURS),
|
||||
"iat": datetime.utcnow(),
|
||||
}
|
||||
return jwt.encode(payload, JWT_SECRET, algorithm="HS256")
|
||||
|
||||
def decode_token(token: str) -> Optional[dict]:
|
||||
try:
|
||||
return jwt.decode(token, JWT_SECRET, algorithms=["HS256"])
|
||||
except jwt.ExpiredSignatureError:
|
||||
raise HTTPException(status_code=401, detail="Token expired")
|
||||
except jwt.InvalidTokenError:
|
||||
raise HTTPException(status_code=401, detail="Invalid token")
|
||||
|
||||
# ── Get current user ──────────────────────────────────────────────────────────
|
||||
def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(bearer)):
|
||||
if not credentials:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
payload = decode_token(credentials.credentials)
|
||||
user_id = int(payload["sub"])
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT id, name, email, role, is_active FROM users WHERE id = %s", (user_id,))
|
||||
user = cur.fetchone()
|
||||
if not user:
|
||||
raise HTTPException(status_code=401, detail="User not found")
|
||||
if not user["is_active"]:
|
||||
raise HTTPException(status_code=403, detail="Account disabled")
|
||||
return dict(user)
|
||||
|
||||
def get_current_user_optional(credentials: HTTPAuthorizationCredentials = Depends(bearer)):
|
||||
if not credentials:
|
||||
return None
|
||||
try:
|
||||
return get_current_user(credentials)
|
||||
except:
|
||||
return None
|
||||
|
||||
# ── Permission checks ─────────────────────────────────────────────────────────
|
||||
def require_permission(permission: str):
|
||||
def checker(user: dict = Depends(get_current_user)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT permissions FROM roles WHERE name = %s", (user["role"],))
|
||||
row = cur.fetchone()
|
||||
perms = row[0] if row else []
|
||||
if permission not in perms:
|
||||
raise HTTPException(status_code=403, detail=f"Permission denied: {permission}")
|
||||
return user
|
||||
return checker
|
||||
|
||||
def require_admin(user: dict = Depends(get_current_user)):
|
||||
if user["role"] != "admin":
|
||||
raise HTTPException(status_code=403, detail="Admin access required")
|
||||
return user
|
||||
|
||||
def require_upload(user: dict = Depends(get_current_user)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT permissions FROM roles WHERE name = %s", (user["role"],))
|
||||
row = cur.fetchone()
|
||||
perms = row[0] if row else []
|
||||
if "upload" not in perms:
|
||||
raise HTTPException(status_code=403, detail="Upload permission required")
|
||||
return user
|
||||
@@ -0,0 +1,460 @@
|
||||
"""
|
||||
brain.py — Hybrid RAG: Dense (semantic) + BM25 (keyword) + RRF fusion + CrossEncoder rerank
|
||||
Pipeline:
|
||||
1. Dense retrieval → top 15 from Qdrant (semantic similarity)
|
||||
2. BM25 retrieval → top 15 from in-memory index (keyword matching)
|
||||
3. RRF fusion → combine ranked lists into unified score
|
||||
4. CrossEncoder → rerank fused top 10 by (query, chunk) relevance
|
||||
5. MMR → diversity filter → final 5 chunks
|
||||
6. LLM → build prompt with numbered citations → call Ollama/Claude
|
||||
"""
|
||||
import os, hashlib, threading
|
||||
import requests as http_requests
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
import redis
|
||||
import numpy as np
|
||||
from qdrant_client import QdrantClient
|
||||
from sentence_transformers import SentenceTransformer, CrossEncoder
|
||||
from rank_bm25 import BM25Okapi
|
||||
from mem0 import Memory
|
||||
|
||||
# ── Config ─────────────────────────────────────────────────────────────────────
|
||||
QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
|
||||
POSTGRES_URL = os.getenv("POSTGRES_URL", "")
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
||||
COLLECTION = "arabic_books"
|
||||
MODEL_NAME = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
|
||||
OLLAMA_MODEL = "qwen2:7b"
|
||||
CACHE_TTL = 60 * 60 * 24 # 24h
|
||||
|
||||
DENSE_K = 15 # retrieve from Qdrant
|
||||
BM25_K = 15 # retrieve from BM25
|
||||
RRF_K = 60 # RRF constant (higher = less aggressive rank differences)
|
||||
RERANK_INPUT = 10 # cross-encoder input size (top N from RRF)
|
||||
FINAL_K = 5 # final chunks after MMR
|
||||
SCORE_THRESH = 0.20 # discard dense results below this
|
||||
MMR_LAMBDA = 0.6 # relevance vs diversity trade-off
|
||||
COMPLEX_WORDS = 30 # query word count → use Claude
|
||||
|
||||
# ── Clients ────────────────────────────────────────────────────────────────────
|
||||
print("Loading embedding model...")
|
||||
embedder = SentenceTransformer(MODEL_NAME)
|
||||
|
||||
print("Loading cross-encoder reranker...")
|
||||
reranker = CrossEncoder("cross-encoder/mmarco-mMiniLMv2-L12-H384-v1", max_length=512)
|
||||
|
||||
_qdrant_key = os.getenv("QDRANT_API_KEY", "")
|
||||
qdrant = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
|
||||
_redis_password = os.getenv("REDIS_PASSWORD", "")
|
||||
_redis_url_auth = REDIS_URL
|
||||
if _redis_password and "://:@" not in _redis_url_auth and "@" not in _redis_url_auth.split("://")[1]:
|
||||
_redis_url_auth = REDIS_URL.replace("redis://", f"redis://:{_redis_password}@")
|
||||
redis_client = redis.from_url(_redis_url_auth, decode_responses=True)
|
||||
|
||||
# ── BM25 Index (in-memory, rebuilt from PostgreSQL on startup) ──────────────────
|
||||
class BM25Index:
|
||||
"""Thread-safe BM25 index over all chunks. Rebuilt when new content is ingested."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.RLock()
|
||||
self._index = None
|
||||
self._chunks = [] # list of {text, book, page, qdrant_id}
|
||||
self._corpus = [] # tokenized
|
||||
|
||||
def _tokenize(self, text: str):
|
||||
"""
|
||||
Arabic-aware tokenization:
|
||||
- Remove diacritics (tashkeel)
|
||||
- Normalize alef variants
|
||||
- Split on whitespace/punctuation
|
||||
"""
|
||||
import re, unicodedata
|
||||
# Remove Arabic diacritics (harakat)
|
||||
text = re.sub(r'[\u064B-\u065F\u0670]', '', text)
|
||||
# Normalize alef variants → plain alef
|
||||
text = re.sub(r'[إأآ]', 'ا', text)
|
||||
# Normalize teh marbuta
|
||||
text = re.sub(r'ة', 'ه', text)
|
||||
# Remove punctuation
|
||||
text = re.sub(r'[^\w\s]', ' ', text)
|
||||
tokens = text.lower().split()
|
||||
return [t for t in tokens if len(t) > 1]
|
||||
|
||||
def build(self, db_url: str):
|
||||
"""Load all chunks from PostgreSQL and build BM25 index."""
|
||||
try:
|
||||
conn = psycopg2.connect(db_url)
|
||||
cur = conn.cursor(cursor_factory=RealDictCursor)
|
||||
cur.execute("""
|
||||
SELECT c.qdrant_id, c.book_filename, c.page, c.text, c.word_count
|
||||
FROM chunks c
|
||||
ORDER BY c.book_filename, c.page
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
cur.close(); conn.close()
|
||||
except Exception as e:
|
||||
print(f"BM25 index build failed: {e}")
|
||||
return 0
|
||||
|
||||
with self._lock:
|
||||
self._chunks = [dict(r) for r in rows]
|
||||
self._corpus = [self._tokenize(c['text']) for c in self._chunks]
|
||||
if self._corpus:
|
||||
self._index = BM25Okapi(self._corpus)
|
||||
else:
|
||||
self._index = None
|
||||
|
||||
print(f"BM25 index built: {len(self._chunks)} chunks")
|
||||
return len(self._chunks)
|
||||
|
||||
def search(self, query: str, top_k: int = BM25_K):
|
||||
"""Return list of (chunk_dict, bm25_score) sorted by score desc."""
|
||||
with self._lock:
|
||||
if self._index is None or not self._chunks:
|
||||
return []
|
||||
tokens = self._tokenize(query)
|
||||
if not tokens:
|
||||
return []
|
||||
scores = self._index.get_scores(tokens)
|
||||
# Get top_k indices
|
||||
top_idx = np.argsort(scores)[::-1][:top_k]
|
||||
return [
|
||||
{**self._chunks[i], "bm25_score": float(scores[i])}
|
||||
for i in top_idx
|
||||
if scores[i] > 0
|
||||
]
|
||||
|
||||
def add_chunks(self, chunks: list, db_url: str):
|
||||
"""Rebuild after new ingest."""
|
||||
self.build(db_url)
|
||||
|
||||
|
||||
# Singleton index
|
||||
bm25_index = BM25Index()
|
||||
|
||||
# ── Mem0 ───────────────────────────────────────────────────────────────────────
|
||||
print("Initialising Mem0...")
|
||||
# Use url= (not host+port) so Mem0's QdrantClient stays on HTTP even with api_key
|
||||
_mem0_qdrant_cfg = {
|
||||
"url": QDRANT_URL,
|
||||
"collection_name": "mem0_memory",
|
||||
"embedding_model_dims": 768, # nomic-embed-text output dimension
|
||||
}
|
||||
if _qdrant_key:
|
||||
_mem0_qdrant_cfg["api_key"] = _qdrant_key
|
||||
|
||||
mem0 = Memory.from_config({
|
||||
"llm": {"provider":"ollama","config":{"model":OLLAMA_MODEL,"ollama_base_url":OLLAMA_URL}},
|
||||
"embedder":{"provider":"ollama","config":{"model":"nomic-embed-text","ollama_base_url":OLLAMA_URL,"embedding_dims":768}},
|
||||
"vector_store":{"provider":"qdrant","config": _mem0_qdrant_cfg},
|
||||
})
|
||||
print("Mem0 ready.")
|
||||
|
||||
# Build BM25 index on startup
|
||||
print("Building BM25 index...")
|
||||
n = bm25_index.build(POSTGRES_URL)
|
||||
print(f"BM25 ready: {n} chunks indexed")
|
||||
if n == 0:
|
||||
print("[WARNING] BM25 index empty — check POSTGRES_URL and chunks table")
|
||||
|
||||
# ── DB helpers ─────────────────────────────────────────────────────────────────
|
||||
def get_db():
|
||||
return psycopg2.connect(POSTGRES_URL)
|
||||
|
||||
def get_claude_key():
|
||||
key = os.getenv("CLAUDE_API_KEY", "")
|
||||
if key: return key
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("SELECT value FROM api_settings WHERE key='claude_api_key'")
|
||||
row = cur.fetchone(); cur.close(); conn.close()
|
||||
return row[0] if row and row[0] else ""
|
||||
except: return ""
|
||||
|
||||
def log_claude_usage(session_id, query, in_tok, out_tok):
|
||||
cost = (in_tok * 0.000003) + (out_tok * 0.000015)
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("INSERT INTO claude_usage (session_id,query,input_tokens,output_tokens,cost_usd) VALUES (%s,%s,%s,%s,%s)",
|
||||
(session_id, query[:200], in_tok, out_tok, cost))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except: pass
|
||||
|
||||
# ── Cache ──────────────────────────────────────────────────────────────────────
|
||||
def cache_key(q):
|
||||
return "rag:cache:" + hashlib.md5(q.strip().lower().encode()).hexdigest()
|
||||
|
||||
def get_cache(q):
|
||||
try: return redis_client.get(cache_key(q))
|
||||
except: return None
|
||||
|
||||
def set_cache(q, resp):
|
||||
try: redis_client.setex(cache_key(q), CACHE_TTL, resp)
|
||||
except Exception as e: print(f"Cache write skipped: {e}")
|
||||
|
||||
# ── Memory ─────────────────────────────────────────────────────────────────────
|
||||
def get_memory(session_id, query):
|
||||
try:
|
||||
results = mem0.search(query=query, filters={"user_id": session_id}, limit=3)
|
||||
memories = results if isinstance(results, list) else results.get("results", [])
|
||||
return "\n".join(m.get("memory", m.get("text","")) for m in memories if m)
|
||||
except Exception as e:
|
||||
print(f"Mem0 search error: {e}"); return ""
|
||||
|
||||
def store_memory(session_id, query, response):
|
||||
try:
|
||||
mem0.add(
|
||||
messages=[{"role":"user","content":query},{"role":"assistant","content":response}],
|
||||
user_id=str(session_id)
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Mem0 store error: {e}")
|
||||
|
||||
def store_conversation(session_id, query, response, llm, books):
|
||||
try:
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("INSERT INTO conversations (session_id,user_query,assistant_response,llm_used,books_used) VALUES (%s,%s,%s,%s,%s)",
|
||||
(session_id, query, response, llm, books))
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
print(f"Conversation store error: {e}")
|
||||
|
||||
# ── Dense retrieval ────────────────────────────────────────────────────────────
|
||||
def retrieve_dense(query: str, top_k: int = DENSE_K):
|
||||
try:
|
||||
cols = [c.name for c in qdrant.get_collections().collections]
|
||||
if COLLECTION not in cols:
|
||||
return [], 0.0
|
||||
except:
|
||||
return [], 0.0
|
||||
|
||||
emb = embedder.encode([query])[0].tolist()
|
||||
response = qdrant.query_points(
|
||||
collection_name=COLLECTION,
|
||||
query=emb,
|
||||
limit=top_k,
|
||||
with_payload=True,
|
||||
score_threshold=SCORE_THRESH,
|
||||
)
|
||||
pts = response.points
|
||||
if not pts:
|
||||
return [], 0.0
|
||||
|
||||
chunks = [
|
||||
{"text": p.payload["text"],
|
||||
"book": p.payload["book"],
|
||||
"page": p.payload.get("page", 1),
|
||||
"score": p.score,
|
||||
"qdrant_id": str(p.id)}
|
||||
for p in pts
|
||||
]
|
||||
return chunks, pts[0].score
|
||||
|
||||
# ── BM25 retrieval ─────────────────────────────────────────────────────────────
|
||||
def retrieve_bm25(query: str, top_k: int = BM25_K):
|
||||
results = bm25_index.search(query, top_k=top_k)
|
||||
return [
|
||||
{"text": r["text"],
|
||||
"book": r["book_filename"],
|
||||
"page": r["page"],
|
||||
"bm25_score": r["bm25_score"],
|
||||
"qdrant_id": r.get("qdrant_id","")}
|
||||
for r in results
|
||||
]
|
||||
|
||||
# ── Reciprocal Rank Fusion ─────────────────────────────────────────────────────
|
||||
def rrf_fuse(dense_chunks, bm25_chunks, k: int = RRF_K):
|
||||
"""
|
||||
Combine two ranked lists with RRF.
|
||||
score(d) = Σ 1/(k + rank_i(d))
|
||||
|
||||
Returns unified list of chunks with rrf_score, deduped by qdrant_id.
|
||||
"""
|
||||
scores = {} # qdrant_id → rrf_score
|
||||
payload = {} # qdrant_id → chunk dict
|
||||
|
||||
for rank, c in enumerate(dense_chunks):
|
||||
cid = c.get("qdrant_id") or c["text"][:40]
|
||||
scores[cid] = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
|
||||
payload[cid] = {**c, "dense_rank": rank + 1}
|
||||
|
||||
for rank, c in enumerate(bm25_chunks):
|
||||
cid = c.get("qdrant_id") or c["text"][:40]
|
||||
scores[cid] = scores.get(cid, 0.0) + 1.0 / (k + rank + 1)
|
||||
if cid not in payload:
|
||||
payload[cid] = {**c}
|
||||
else:
|
||||
payload[cid]["bm25_rank"] = rank + 1
|
||||
payload[cid]["rrf_score"] = scores[cid]
|
||||
|
||||
# Tag each chunk with which source retrieved it
|
||||
fused = []
|
||||
for cid, chunk in payload.items():
|
||||
chunk["rrf_score"] = scores[cid]
|
||||
has_dense = "dense_rank" in chunk
|
||||
has_bm25 = "bm25_rank" in chunk
|
||||
chunk["source"] = (
|
||||
"both" if has_dense and has_bm25 else
|
||||
"dense" if has_dense else
|
||||
"bm25"
|
||||
)
|
||||
fused.append(chunk)
|
||||
|
||||
fused.sort(key=lambda x: x["rrf_score"], reverse=True)
|
||||
# Boost chunks found by both sources
|
||||
both_hits = [c for c in fused if c["source"] == "both"]
|
||||
print(f" RRF: {len(dense_chunks)} dense + {len(bm25_chunks)} BM25 → {len(fused)} fused ({len(both_hits)} in both)")
|
||||
return fused
|
||||
|
||||
# ── CrossEncoder rerank ────────────────────────────────────────────────────────
|
||||
def rerank(query: str, chunks: list, keep: int = RERANK_INPUT):
|
||||
if not chunks:
|
||||
return chunks
|
||||
pairs = [(query, c["text"]) for c in chunks]
|
||||
scores = reranker.predict(pairs)
|
||||
for i, c in enumerate(chunks):
|
||||
c["rerank_score"] = float(scores[i])
|
||||
return sorted(chunks, key=lambda x: x["rerank_score"], reverse=True)[:keep]
|
||||
|
||||
# ── MMR diversity ──────────────────────────────────────────────────────────────
|
||||
def mmr(query: str, chunks: list, keep: int = FINAL_K, lam: float = MMR_LAMBDA):
|
||||
if len(chunks) <= keep:
|
||||
return chunks
|
||||
texts = [c["text"] for c in chunks]
|
||||
q_emb = embedder.encode([query])[0]
|
||||
c_embs = embedder.encode(texts, show_progress_bar=False)
|
||||
norm = lambda v: v / (np.linalg.norm(v) or 1)
|
||||
q_emb = norm(q_emb)
|
||||
c_embs = [norm(e) for e in c_embs]
|
||||
selected = []
|
||||
remaining = list(range(len(chunks)))
|
||||
for _ in range(keep):
|
||||
if not remaining: break
|
||||
rel = np.array([c_embs[i].dot(q_emb) for i in remaining])
|
||||
if not selected:
|
||||
best = remaining[int(np.argmax(rel))]
|
||||
else:
|
||||
sel_embs = [c_embs[i] for i in selected]
|
||||
red = np.array([max(c_embs[i].dot(s) for s in sel_embs) for i in remaining])
|
||||
best = remaining[int(np.argmax(lam * rel - (1 - lam) * red))]
|
||||
selected.append(best)
|
||||
remaining.remove(best)
|
||||
return [chunks[i] for i in selected]
|
||||
|
||||
# ── Prompt ─────────────────────────────────────────────────────────────────────
|
||||
def build_prompt(query: str, chunks: list, memory: str):
|
||||
source_blocks = []
|
||||
for i, c in enumerate(chunks, 1):
|
||||
book_short = c["book"].replace(".pdf","")
|
||||
source_tag = f"{'🔗 ' if c.get('source')=='both' else ''}"
|
||||
source_blocks.append(
|
||||
f"[{i}] «{book_short}» — صفحة {c['page']}:\n{c['text']}"
|
||||
)
|
||||
context = "\n\n".join(source_blocks)
|
||||
mem_block = f"\n\nمن محادثات سابقة:\n{memory}" if memory else ""
|
||||
|
||||
return f"""أنت باحث إسلامي متخصص في الفقه والحديث والعلوم الشرعية. أجب على الأسئلة اعتماداً على المصادر المقدمة فقط.
|
||||
{mem_block}
|
||||
|
||||
المصادر:
|
||||
{context}
|
||||
|
||||
السؤال: {query}
|
||||
|
||||
تعليمات:
|
||||
١. حلّل المصادر وحدد الأجزاء المتعلقة بالسؤال مباشرةً.
|
||||
٢. أجب بشكل دقيق ومفصل، ووثّق كل معلومة بالمصدر [١][٢][٣].
|
||||
٣. إذا تعارضت المصادر، أشر إلى ذلك وقدّم الأقوى دليلاً.
|
||||
٤. إن لم تكفِ المصادر للإجابة الكاملة، صرّح بذلك بوضوح.
|
||||
٥. الإجابة بالعربية الفصحى.
|
||||
|
||||
الإجابة:"""
|
||||
|
||||
# ── LLM ────────────────────────────────────────────────────────────────────────
|
||||
def call_ollama(prompt):
|
||||
resp = http_requests.post(
|
||||
f"{OLLAMA_URL}/api/generate",
|
||||
json={"model":OLLAMA_MODEL,"prompt":prompt,"stream":False,
|
||||
"options":{"num_predict":700,"temperature":0.2,"top_p":0.9}},
|
||||
timeout=180
|
||||
)
|
||||
return resp.json().get("response","").strip()
|
||||
|
||||
def call_claude(prompt, session_id="", query=""):
|
||||
key = get_claude_key()
|
||||
if not key:
|
||||
return call_ollama(prompt)
|
||||
import anthropic
|
||||
c = anthropic.Anthropic(api_key=key)
|
||||
msg = c.messages.create(
|
||||
model="claude-sonnet-4-20250514", max_tokens=1024,
|
||||
messages=[{"role":"user","content":prompt}]
|
||||
)
|
||||
log_claude_usage(session_id, query, msg.usage.input_tokens, msg.usage.output_tokens)
|
||||
return msg.content[0].text.strip()
|
||||
|
||||
# ── Main ────────────────────────────────────────────────────────────────────────
|
||||
def process_query(query: str, session_id: str = "default"):
|
||||
# 1. Cache
|
||||
cached = get_cache(query)
|
||||
if cached:
|
||||
return {"answer":cached,"source":"cache","chunks":[],"llm":"cache","confidence":1.0,"books_used":[]}
|
||||
|
||||
# 2. Dense + BM25 retrieval (parallel concept, sequential execution)
|
||||
dense_chunks, top_score = retrieve_dense(query, top_k=DENSE_K)
|
||||
bm25_chunks = retrieve_bm25(query, top_k=BM25_K)
|
||||
|
||||
# 3. RRF fusion
|
||||
fused = rrf_fuse(dense_chunks, bm25_chunks)
|
||||
if not fused:
|
||||
return {"answer":"لم أجد معلومات كافية للإجابة على هذا السؤال في الكتب المتاحة.",
|
||||
"source":"none","chunks":[],"llm":"none","confidence":0.0,"books_used":[]}
|
||||
|
||||
# 4. Cross-encoder rerank (top RERANK_INPUT from fused)
|
||||
top_fused = fused[:RERANK_INPUT]
|
||||
reranked = rerank(query, top_fused, keep=RERANK_INPUT)
|
||||
|
||||
# 5. MMR diversity filter → final FINAL_K chunks
|
||||
final_chunks = mmr(query, reranked, keep=FINAL_K)
|
||||
|
||||
# 6. Memory
|
||||
memory = get_memory(session_id, query)
|
||||
|
||||
# 7. Prompt + LLM
|
||||
prompt = build_prompt(query, final_chunks, memory)
|
||||
use_claude = (len(query.split()) > COMPLEX_WORDS or top_score < 0.35) and bool(get_claude_key())
|
||||
|
||||
if use_claude:
|
||||
answer = call_claude(prompt, session_id=session_id, query=query)
|
||||
llm_used = "claude"
|
||||
else:
|
||||
answer = call_ollama(prompt)
|
||||
llm_used = f"ollama/{OLLAMA_MODEL}"
|
||||
|
||||
# 8. Store
|
||||
books_used = list(set(c["book"] for c in final_chunks))
|
||||
store_memory(session_id, query, answer)
|
||||
store_conversation(session_id, query, answer, llm_used, books_used)
|
||||
set_cache(query, answer)
|
||||
|
||||
return {
|
||||
"answer": answer,
|
||||
"source": "hybrid",
|
||||
"llm": llm_used,
|
||||
"confidence": top_score,
|
||||
"chunks": final_chunks,
|
||||
"books_used": books_used,
|
||||
"retrieval": {
|
||||
"dense": len(dense_chunks),
|
||||
"bm25": len(bm25_chunks),
|
||||
"fused": len(fused),
|
||||
"both": sum(1 for c in fused if c.get("source") == "both"),
|
||||
}
|
||||
}
|
||||
|
||||
def rebuild_bm25():
|
||||
"""Called by worker after new ingest."""
|
||||
bm25_index.build(POSTGRES_URL)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""
|
||||
celery_app.py — Celery application instance
|
||||
Shared by worker tasks and Flower monitor
|
||||
"""
|
||||
import os
|
||||
from celery import Celery
|
||||
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
|
||||
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
|
||||
CELERY_BROKER = os.getenv("CELERY_BROKER_URL", "")
|
||||
|
||||
# Build authenticated broker URL if not already set
|
||||
if not CELERY_BROKER:
|
||||
if REDIS_PASSWORD and "@" not in REDIS_URL.split("://")[1]:
|
||||
CELERY_BROKER = REDIS_URL.replace("redis://", f"redis://:{REDIS_PASSWORD}@") + "/1"
|
||||
else:
|
||||
CELERY_BROKER = REDIS_URL + "/1"
|
||||
|
||||
# Result backend on DB 2 (separate from cache on DB 0 and broker on DB 1)
|
||||
RESULT_BACKEND = CELERY_BROKER.replace("/1", "/2")
|
||||
|
||||
app = Celery(
|
||||
"rag_worker",
|
||||
broker=CELERY_BROKER,
|
||||
backend=RESULT_BACKEND,
|
||||
)
|
||||
|
||||
app.conf.update(
|
||||
task_serializer = "json",
|
||||
result_serializer = "json",
|
||||
accept_content = ["json"],
|
||||
task_track_started = True,
|
||||
task_acks_late = True, # re-queue on worker crash
|
||||
worker_prefetch_multiplier = 1, # one task at a time (PDF heavy)
|
||||
result_expires = 60 * 60 * 24, # 24h result retention
|
||||
worker_send_task_events = True, # needed for Flower
|
||||
task_send_sent_event = True,
|
||||
)
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
compress.py — PDF compression via Ghostscript
|
||||
Runs after file lands on disk, before ingest.
|
||||
Only replaces original if compressed version is smaller.
|
||||
Skips files under MIN_SIZE_KB (already small text PDFs gain nothing).
|
||||
"""
|
||||
import os, subprocess, logging, shutil
|
||||
|
||||
log = logging.getLogger("rag.compress")
|
||||
|
||||
# Only attempt compression on files larger than this
|
||||
MIN_SIZE_KB = 200
|
||||
# Ghostscript quality preset
|
||||
# /screen = 72 DPI (smallest, low quality — avoid)
|
||||
# /ebook = 150 DPI (good for text+images, our default)
|
||||
# /printer = 300 DPI (high quality, moderate compression)
|
||||
GS_QUALITY = os.getenv("PDF_COMPRESS_QUALITY", "/ebook")
|
||||
GS_COMPAT = "1.4"
|
||||
|
||||
|
||||
def _ghostscript_available() -> bool:
|
||||
try:
|
||||
r = subprocess.run(["gs", "--version"], capture_output=True, timeout=5)
|
||||
return r.returncode == 0
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def compress_pdf(pdf_path: str) -> dict:
|
||||
"""
|
||||
Compress a PDF in-place using Ghostscript.
|
||||
Returns a dict with: original_kb, compressed_kb, saved_pct, action
|
||||
action: 'compressed' | 'skipped_small' | 'skipped_larger' | 'error' | 'gs_missing'
|
||||
"""
|
||||
if not os.path.exists(pdf_path):
|
||||
return {"action": "error", "detail": "file not found"}
|
||||
|
||||
if not _ghostscript_available():
|
||||
log.warning("Ghostscript not available — skipping compression")
|
||||
return {"action": "gs_missing"}
|
||||
|
||||
original_size = os.path.getsize(pdf_path)
|
||||
original_kb = original_size // 1024
|
||||
|
||||
# Skip very small files — already optimised
|
||||
if original_kb < MIN_SIZE_KB:
|
||||
log.info(f"Skipping compression (small file {original_kb}KB): {os.path.basename(pdf_path)}")
|
||||
return {"action": "skipped_small", "original_kb": original_kb}
|
||||
|
||||
tmp_path = pdf_path + ".compressed.tmp"
|
||||
try:
|
||||
result = subprocess.run([
|
||||
"gs",
|
||||
"-sDEVICE=pdfwrite",
|
||||
f"-dCompatibilityLevel={GS_COMPAT}",
|
||||
f"-dPDFSETTINGS={GS_QUALITY}",
|
||||
"-dNOPAUSE",
|
||||
"-dQUIET",
|
||||
"-dBATCH",
|
||||
"-dDetectDuplicateImages=true",
|
||||
"-dCompressFonts=true",
|
||||
f"-sOutputFile={tmp_path}",
|
||||
pdf_path,
|
||||
], capture_output=True, text=True, timeout=300)
|
||||
|
||||
if result.returncode != 0:
|
||||
log.error(f"Ghostscript error for {pdf_path}: {result.stderr[:200]}")
|
||||
return {"action": "error", "detail": result.stderr[:200]}
|
||||
|
||||
if not os.path.exists(tmp_path) or os.path.getsize(tmp_path) == 0:
|
||||
return {"action": "error", "detail": "output file empty"}
|
||||
|
||||
compressed_size = os.path.getsize(tmp_path)
|
||||
compressed_kb = compressed_size // 1024
|
||||
|
||||
if compressed_size >= original_size:
|
||||
# Compressed is same or larger — keep original
|
||||
os.remove(tmp_path)
|
||||
saved_pct = ((original_size - compressed_size) * 100) // original_size
|
||||
log.info(f"Compression skipped (no benefit {saved_pct}%): {os.path.basename(pdf_path)}")
|
||||
return {
|
||||
"action": "skipped_larger",
|
||||
"original_kb": original_kb,
|
||||
"compressed_kb": compressed_kb,
|
||||
"saved_pct": saved_pct,
|
||||
}
|
||||
|
||||
# Compressed is smaller — replace original
|
||||
shutil.move(tmp_path, pdf_path)
|
||||
saved_pct = ((original_size - compressed_size) * 100) // original_size
|
||||
log.info(f"Compressed {os.path.basename(pdf_path)}: {original_kb}KB → {compressed_kb}KB ({saved_pct}% saved)")
|
||||
return {
|
||||
"action": "compressed",
|
||||
"original_kb": original_kb,
|
||||
"compressed_kb": compressed_kb,
|
||||
"saved_pct": saved_pct,
|
||||
}
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error(f"Ghostscript timeout for {pdf_path}")
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
return {"action": "error", "detail": "timeout after 300s"}
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Compression error for {pdf_path}: {e}")
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
return {"action": "error", "detail": str(e)}
|
||||
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
ingest.py — PDF ingestion with paragraph-aware chunking
|
||||
Improvements over v1:
|
||||
- Paragraph-aware chunking (respects sentence/paragraph boundaries)
|
||||
- Target chunk size: ~150 words, max 200
|
||||
- Minimum chunk quality filter (skip noise, headers, single lines)
|
||||
- Stores cover image (first page as JPEG)
|
||||
- Deduplication by file hash
|
||||
"""
|
||||
import os, sys, hashlib, uuid, re, shutil
|
||||
import fitz # PyMuPDF
|
||||
from sentence_transformers import SentenceTransformer
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import Distance, VectorParams, PointStruct
|
||||
|
||||
QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
|
||||
POSTGRES_URL = os.getenv("POSTGRES_URL", "")
|
||||
COLLECTION = "arabic_books"
|
||||
MODEL_NAME = "CAMeL-Lab/bert-base-arabic-camelbert-mix"
|
||||
|
||||
TARGET_WORDS = 150 # ideal chunk size
|
||||
MAX_WORDS = 200 # hard cap before forced split
|
||||
OVERLAP_SENTS = 2 # sentence overlap between chunks
|
||||
MIN_CHUNK_LEN = 40 # skip chunks shorter than this (chars)
|
||||
|
||||
DELETE_AFTER_INGEST = os.getenv("DELETE_PDF_AFTER_INGEST", "false").lower() == "true"
|
||||
COVER_DIR = "/mnt/rag-media/covers"
|
||||
os.makedirs(COVER_DIR, exist_ok=True)
|
||||
|
||||
print("Loading embedding model...")
|
||||
model = SentenceTransformer(MODEL_NAME)
|
||||
VECTOR_SIZE = model.get_embedding_dimension()
|
||||
_qdrant_key = os.getenv("QDRANT_API_KEY", "")
|
||||
client = QdrantClient(url=QDRANT_URL, api_key=_qdrant_key if _qdrant_key else None)
|
||||
|
||||
collections = [c.name for c in client.get_collections().collections]
|
||||
if COLLECTION not in collections:
|
||||
client.create_collection(
|
||||
collection_name=COLLECTION,
|
||||
vectors_config=VectorParams(size=VECTOR_SIZE, distance=Distance.COSINE)
|
||||
)
|
||||
print(f"Created collection: {COLLECTION}")
|
||||
|
||||
# ── DB helpers ────────────────────────────────────────────────────────────────
|
||||
def get_db():
|
||||
import psycopg2
|
||||
return psycopg2.connect(POSTGRES_URL)
|
||||
|
||||
def get_file_hash(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
def is_already_ingested(file_hash):
|
||||
"""Returns existing filename if hash matches, None otherwise."""
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("SELECT filename, title FROM books WHERE file_hash = %s", (file_hash,))
|
||||
row = cur.fetchone(); cur.close(); conn.close()
|
||||
return row[0] if row else None
|
||||
except: return None
|
||||
|
||||
def get_existing_by_filename(book_name):
|
||||
"""Check if a book with this filename already exists."""
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("SELECT id, file_hash FROM books WHERE filename = %s", (book_name,))
|
||||
row = cur.fetchone(); cur.close(); conn.close()
|
||||
return row if row else None
|
||||
except: return None
|
||||
|
||||
def register_book(book_name, total_chunks, file_hash):
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""INSERT INTO books (filename, total_chunks, file_hash)
|
||||
VALUES (%s, %s, %s)
|
||||
ON CONFLICT (filename) DO UPDATE
|
||||
SET total_chunks = EXCLUDED.total_chunks, file_hash = EXCLUDED.file_hash""",
|
||||
(book_name, total_chunks, file_hash)
|
||||
)
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
print(f"Registered: {book_name}")
|
||||
except Exception as e:
|
||||
print(f"DB registration skipped: {e}")
|
||||
|
||||
# ── Cover extraction ──────────────────────────────────────────────────────────
|
||||
def extract_cover(pdf_path, book_id):
|
||||
"""Render first page of PDF as JPEG, save to COVER_DIR/{book_id}.jpg"""
|
||||
try:
|
||||
doc = fitz.open(pdf_path)
|
||||
page = doc[0]
|
||||
mat = fitz.Matrix(1.5, 1.5) # 1.5x zoom → ~120 DPI
|
||||
pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
|
||||
cover_path = os.path.join(COVER_DIR, f"{book_id}.jpg")
|
||||
pix.save(cover_path)
|
||||
doc.close()
|
||||
print(f" Cover saved: {cover_path}")
|
||||
return cover_path
|
||||
except Exception as e:
|
||||
print(f" Cover extraction failed: {e}")
|
||||
return None
|
||||
|
||||
# ── Paragraph-aware chunking ──────────────────────────────────────────────────
|
||||
def split_sentences(text):
|
||||
"""
|
||||
Split Arabic/mixed text into sentences.
|
||||
Splits on: period, question mark, exclamation, Arabic full stop (؟، .)
|
||||
"""
|
||||
pattern = r'(?<=[.!?؟])\s+|(?<=\n)\s*(?=\S)'
|
||||
parts = re.split(pattern, text.strip())
|
||||
# Also split on paragraph breaks (double newline)
|
||||
sentences = []
|
||||
for part in parts:
|
||||
sub = [s.strip() for s in part.split('\n\n') if s.strip()]
|
||||
sentences.extend(sub)
|
||||
return [s for s in sentences if len(s) > 10]
|
||||
|
||||
def chunk_text_smart(text):
|
||||
"""
|
||||
Paragraph-aware chunking:
|
||||
1. Split into sentences
|
||||
2. Accumulate sentences until TARGET_WORDS reached
|
||||
3. Add OVERLAP_SENTS from previous chunk for continuity
|
||||
4. Hard-cap at MAX_WORDS — force split at word level if needed
|
||||
"""
|
||||
sentences = split_sentences(text)
|
||||
if not sentences:
|
||||
return []
|
||||
|
||||
chunks = []
|
||||
current_sents = []
|
||||
current_words = 0
|
||||
|
||||
for sent in sentences:
|
||||
sent_words = len(sent.split())
|
||||
# If single sentence exceeds max, force-split it
|
||||
if sent_words > MAX_WORDS:
|
||||
# flush current first
|
||||
if current_sents:
|
||||
chunks.append(" ".join(current_sents))
|
||||
current_sents = current_sents[-OVERLAP_SENTS:]
|
||||
current_words = sum(len(s.split()) for s in current_sents)
|
||||
# word-level split for the long sentence
|
||||
words = sent.split()
|
||||
for i in range(0, len(words), TARGET_WORDS - 10):
|
||||
sub = " ".join(words[i:i + TARGET_WORDS])
|
||||
if sub.strip():
|
||||
chunks.append(sub)
|
||||
continue
|
||||
|
||||
if current_words + sent_words > MAX_WORDS and current_sents:
|
||||
chunks.append(" ".join(current_sents))
|
||||
# carry overlap
|
||||
current_sents = current_sents[-OVERLAP_SENTS:]
|
||||
current_words = sum(len(s.split()) for s in current_sents)
|
||||
|
||||
current_sents.append(sent)
|
||||
current_words += sent_words
|
||||
|
||||
if current_sents:
|
||||
chunks.append(" ".join(current_sents))
|
||||
|
||||
# Filter noise
|
||||
return [c.strip() for c in chunks if len(c.strip()) >= MIN_CHUNK_LEN]
|
||||
|
||||
# ── Main ingest ───────────────────────────────────────────────────────────────
|
||||
def ingest_pdf(pdf_path):
|
||||
book_name = os.path.basename(pdf_path)
|
||||
|
||||
# Dedup check
|
||||
file_hash = get_file_hash(pdf_path)
|
||||
existing = is_already_ingested(file_hash)
|
||||
if existing:
|
||||
print(f"SKIP (duplicate): {book_name}")
|
||||
return
|
||||
|
||||
print(f"Processing: {book_name}")
|
||||
doc = fitz.open(pdf_path)
|
||||
|
||||
# Extract cover first
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("SELECT id FROM books WHERE filename = %s", (book_name,))
|
||||
row = cur.fetchone(); cur.close(); conn.close()
|
||||
book_id = row[0] if row else book_name.replace('.pdf', '')
|
||||
except:
|
||||
book_id = book_name.replace('.pdf', '')
|
||||
|
||||
extract_cover(pdf_path, book_id)
|
||||
|
||||
# Extract text page by page
|
||||
all_chunks = []
|
||||
for page_num, page in enumerate(doc):
|
||||
text = page.get_text()
|
||||
if not text.strip():
|
||||
continue
|
||||
for chunk in chunk_text_smart(text):
|
||||
all_chunks.append({
|
||||
"text": chunk,
|
||||
"book": book_name,
|
||||
"page": page_num + 1,
|
||||
"word_count": len(chunk.split())
|
||||
})
|
||||
|
||||
doc.close()
|
||||
print(f" Chunks: {len(all_chunks)} | avg words: {sum(c['word_count'] for c in all_chunks)//max(len(all_chunks),1)}")
|
||||
|
||||
# Get book_id from DB
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("SELECT id FROM books WHERE filename = %s", (book_name,))
|
||||
row = cur.fetchone(); book_id = row[0] if row else None
|
||||
cur.close(); conn.close()
|
||||
except: book_id = None
|
||||
|
||||
# Clear old chunks for this book
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("DELETE FROM chunks WHERE book_filename = %s", (book_name,))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except: pass
|
||||
|
||||
# Embed + store in Qdrant AND PostgreSQL
|
||||
batch_size = 32
|
||||
for i in range(0, len(all_chunks), batch_size):
|
||||
batch = all_chunks[i:i + batch_size]
|
||||
embeddings = model.encode([c["text"] for c in batch], show_progress_bar=False)
|
||||
qdrant_points = []
|
||||
pg_rows = []
|
||||
for j, c in enumerate(batch):
|
||||
vid = str(uuid.uuid4())
|
||||
qdrant_points.append(PointStruct(
|
||||
id=vid,
|
||||
vector=embeddings[j].tolist(),
|
||||
payload={"text":c["text"],"book":c["book"],"page":c["page"],"word_count":c["word_count"]}
|
||||
))
|
||||
pg_rows.append((vid, book_id, c["book"], c["page"], c["text"], c["word_count"]))
|
||||
client.upsert(collection_name=COLLECTION, points=qdrant_points)
|
||||
# Store in PostgreSQL for BM25 index
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.executemany(
|
||||
"INSERT INTO chunks (qdrant_id,book_id,book_filename,page,text,word_count) VALUES (%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING",
|
||||
pg_rows
|
||||
)
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except Exception as e:
|
||||
print(f" PG chunks write skipped: {e}")
|
||||
print(f" Stored {i+1}–{i+len(batch)}")
|
||||
|
||||
register_book(book_name, len(all_chunks), file_hash)
|
||||
|
||||
# Store PDF
|
||||
pdf_dest = os.path.join("/mnt/rag-media/pdfs", book_name)
|
||||
if pdf_path != pdf_dest:
|
||||
shutil.copy2(pdf_path, pdf_dest)
|
||||
if DELETE_AFTER_INGEST:
|
||||
os.remove(pdf_path)
|
||||
|
||||
file_size = os.path.getsize(pdf_dest)
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute(
|
||||
"UPDATE books SET pdf_path=%s, file_size=%s WHERE filename=%s",
|
||||
(pdf_dest, file_size, book_name)
|
||||
)
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except Exception as e:
|
||||
print(f"PDF path update skipped: {e}")
|
||||
|
||||
print(f"Done: {book_name} — {len(all_chunks)} chunks stored")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 ingest.py <pdf_path_or_folder>")
|
||||
sys.exit(1)
|
||||
path = sys.argv[1]
|
||||
if os.path.isdir(path):
|
||||
for f in sorted(os.listdir(path)):
|
||||
if f.endswith(".pdf"):
|
||||
ingest_pdf(os.path.join(path, f))
|
||||
else:
|
||||
ingest_pdf(path)
|
||||
info = client.get_collection(COLLECTION)
|
||||
print(f"\nTotal vectors: {info.points_count}")
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
"""
|
||||
meta_extract.py — Shared metadata extraction via Ollama
|
||||
Used by both single upload (routes.py) and bulk upload (worker.py)
|
||||
"""
|
||||
import os, re, json, threading
|
||||
import fitz
|
||||
import requests as http_req
|
||||
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
||||
OLLAMA_MODEL = "qwen2:7b"
|
||||
MEDIA_DIR = "/mnt/rag-media/pdfs"
|
||||
|
||||
|
||||
def extract_meta(pdf_path: str, filename: str) -> dict:
|
||||
"""Extract title/author/category/description from PDF first 3 pages via Ollama."""
|
||||
fallback = {
|
||||
"title": filename.replace(".pdf", "").strip(),
|
||||
"author": "",
|
||||
"category": "عام",
|
||||
"description": ""
|
||||
}
|
||||
|
||||
# Read PDF text
|
||||
try:
|
||||
doc = fitz.open(pdf_path)
|
||||
text = "".join(doc[i].get_text() for i in range(min(3, len(doc))))[:3000]
|
||||
doc.close()
|
||||
except Exception as e:
|
||||
print(f"[meta] fitz error: {e}")
|
||||
text = filename.replace(".pdf", "")
|
||||
|
||||
prompt = (
|
||||
"استخرج من النص التالي معلومات الكتاب وأجب بـ JSON فقط بدون أي نص إضافي:\n"
|
||||
'{"title":"عنوان الكتاب","author":"اسم المؤلف",'
|
||||
'"category":"التصنيف مثل فقه أو حديث أو تفسير أو أدب","description":"وصف مختصر"}\n\n'
|
||||
f"النص:\n{text}"
|
||||
)
|
||||
|
||||
try:
|
||||
r = http_req.post(
|
||||
f"{OLLAMA_URL}/api/generate",
|
||||
json={"model": OLLAMA_MODEL, "prompt": prompt, "stream": False,
|
||||
"options": {"num_predict": 256, "temperature": 0.1}},
|
||||
timeout=90
|
||||
)
|
||||
raw = r.json().get("response", "")
|
||||
m = re.search(r'\{[^}]+\}', raw, re.DOTALL)
|
||||
if m:
|
||||
parsed = json.loads(m.group())
|
||||
return {
|
||||
"title": (parsed.get("title") or fallback["title"]).strip(),
|
||||
"author": (parsed.get("author") or "").strip(),
|
||||
"category": (parsed.get("category") or "عام").strip() or "عام",
|
||||
"description": (parsed.get("description") or "").strip(),
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"[meta] Ollama error for {filename}: {e}")
|
||||
|
||||
return fallback
|
||||
|
||||
|
||||
def save_meta_to_db(book_filename: str, meta: dict, uploader_id: int = None,
|
||||
pg_url: str = None):
|
||||
"""Update books table with extracted metadata."""
|
||||
import psycopg2
|
||||
if not pg_url:
|
||||
pg_url = os.getenv("POSTGRES_URL", "")
|
||||
try:
|
||||
conn = psycopg2.connect(pg_url)
|
||||
cur = conn.cursor()
|
||||
if uploader_id:
|
||||
cur.execute("""
|
||||
UPDATE books SET title=%s, author=%s, category=%s,
|
||||
description=%s, uploader_id=%s
|
||||
WHERE filename=%s
|
||||
""", (meta["title"], meta["author"], meta["category"],
|
||||
meta["description"], uploader_id, book_filename))
|
||||
else:
|
||||
cur.execute("""
|
||||
UPDATE books SET title=%s, author=%s, category=%s,
|
||||
description=%s
|
||||
WHERE filename=%s
|
||||
""", (meta["title"], meta["author"], meta["category"],
|
||||
meta["description"], book_filename))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
print(f"[meta] saved: {meta['title']} | {meta['author']} | {meta['category']}")
|
||||
except Exception as e:
|
||||
print(f"[meta] DB save failed: {e}")
|
||||
|
||||
|
||||
def extract_and_save_bg(book_filename: str, uploader_id: int = None,
|
||||
pg_url: str = None, _retry: int = 0):
|
||||
"""
|
||||
Fire-and-forget: extract metadata and save in background thread.
|
||||
If author is still empty after first extraction, retries once after 5 minutes.
|
||||
Max retries: 2.
|
||||
"""
|
||||
pdf_path = os.path.join(MEDIA_DIR, book_filename)
|
||||
if not os.path.exists(pdf_path):
|
||||
print(f"[meta] PDF not found: {pdf_path}")
|
||||
return
|
||||
|
||||
MAX_RETRIES = 2
|
||||
RETRY_DELAY = 300 # 5 minutes
|
||||
|
||||
def _run():
|
||||
meta = extract_meta(pdf_path, book_filename)
|
||||
save_meta_to_db(book_filename, meta, uploader_id=uploader_id, pg_url=pg_url)
|
||||
|
||||
# If author is still empty and we haven't exhausted retries, schedule retry
|
||||
if not meta.get("author") and _retry < MAX_RETRIES:
|
||||
attempt = _retry + 1
|
||||
print(f"[meta] author empty — retry {attempt}/{MAX_RETRIES} in {RETRY_DELAY}s for {book_filename}")
|
||||
timer = threading.Timer(
|
||||
RETRY_DELAY,
|
||||
extract_and_save_bg,
|
||||
kwargs=dict(book_filename=book_filename, uploader_id=uploader_id,
|
||||
pg_url=pg_url, _retry=attempt)
|
||||
)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
elif not meta.get("author"):
|
||||
print(f"[meta] author still empty after {MAX_RETRIES} retries for {book_filename} — admin can set manually")
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True)
|
||||
t.start()
|
||||
return t
|
||||
|
||||
|
||||
def extract_and_save_sync(book_filename: str, uploader_id: int = None, pg_url: str = None):
|
||||
"""Blocking version — for use in worker subprocess context."""
|
||||
pdf_path = os.path.join(MEDIA_DIR, book_filename)
|
||||
if not os.path.exists(pdf_path):
|
||||
# Try as-is (full path)
|
||||
if os.path.exists(book_filename):
|
||||
pdf_path = book_filename
|
||||
else:
|
||||
print(f"[meta] PDF not found: {book_filename}")
|
||||
return None
|
||||
meta = extract_meta(pdf_path, book_filename)
|
||||
save_meta_to_db(os.path.basename(book_filename), meta, uploader_id=uploader_id, pg_url=pg_url)
|
||||
return meta
|
||||
@@ -0,0 +1,657 @@
|
||||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Request
|
||||
from pydantic import BaseModel, EmailStr
|
||||
from typing import Optional, List
|
||||
from psycopg2.extras import RealDictCursor
|
||||
import os, shutil, subprocess, uuid, json
|
||||
|
||||
from meta_extract import extract_and_save_bg as _meta_bg
|
||||
try:
|
||||
from typesense_client import search_books as _ts_search, index_book as _ts_index_book, delete_book as _ts_delete_book
|
||||
_TYPESENSE_OK = True
|
||||
except Exception:
|
||||
_TYPESENSE_OK = False
|
||||
def _ts_search(q, **kw): return []
|
||||
def _ts_index_book(bid): pass
|
||||
def _ts_delete_book(bid): pass
|
||||
from slowapi import Limiter
|
||||
from slowapi.util import get_remote_address
|
||||
from fastapi import Request as _Request
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
from auth import (
|
||||
get_db, hash_password, verify_password, create_token,
|
||||
get_current_user, get_current_user_optional,
|
||||
require_admin, require_upload
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ── Duplicate detection ───────────────────────────────────────────────────────
|
||||
def _file_hash(data: bytes) -> str:
|
||||
import hashlib
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
def _find_duplicate(file_hash: str, filename: str) -> dict | None:
|
||||
"""Return existing book record if hash OR filename already exists, else None."""
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT id, filename, title, author, category, uploaded_at
|
||||
FROM books
|
||||
WHERE file_hash = %s OR filename = %s
|
||||
LIMIT 1
|
||||
""", (file_hash, filename))
|
||||
row = cur.fetchone()
|
||||
return dict(row) if row else None
|
||||
BOOKS_DIR = "/mnt/rag-media/pdfs"
|
||||
API_KEY = os.getenv("RAG_API_KEY", "")
|
||||
|
||||
# ── Models ────────────────────────────────────────────────────────────────────
|
||||
class RegisterRequest(BaseModel):
|
||||
name: str
|
||||
email: str
|
||||
password: str
|
||||
phone: Optional[str] = None
|
||||
country_code: Optional[str] = None
|
||||
country_name: Optional[str] = None
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
|
||||
class UpdateRoleRequest(BaseModel):
|
||||
role: str
|
||||
|
||||
class BookMetaUpdate(BaseModel):
|
||||
title: Optional[str] = None
|
||||
author: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
category: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
is_public: Optional[bool] = None
|
||||
|
||||
# ── Auth ──────────────────────────────────────────────────────────────────────
|
||||
@router.post("/auth/register")
|
||||
def register(req: RegisterRequest):
|
||||
if len(req.password) < 6:
|
||||
raise HTTPException(400, "Password must be at least 6 characters")
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM users WHERE email = %s", (req.email.lower(),))
|
||||
if cur.fetchone():
|
||||
raise HTTPException(400, "Email already registered")
|
||||
cur.execute("""
|
||||
INSERT INTO users (name, email, password_hash, phone, country_code, country_name, role)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, 'reader') RETURNING id, role
|
||||
""", (req.name, req.email.lower(), hash_password(req.password),
|
||||
req.phone, req.country_code, req.country_name))
|
||||
user = cur.fetchone()
|
||||
conn.commit()
|
||||
token = create_token(user[0], user[1])
|
||||
return {"token": token, "user_id": user[0], "role": user[1], "name": req.name}
|
||||
|
||||
@router.post("/auth/login")
|
||||
@limiter.limit("10/minute")
|
||||
def login(request: Request, req: LoginRequest):
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT * FROM users WHERE email = %s AND is_active = TRUE", (req.email.lower(),))
|
||||
user = cur.fetchone()
|
||||
if not user or not verify_password(req.password, user["password_hash"]):
|
||||
raise HTTPException(401, "Invalid email or password")
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("UPDATE users SET last_login = NOW() WHERE id = %s", (user["id"],))
|
||||
conn.commit()
|
||||
token = create_token(user["id"], user["role"])
|
||||
# Track session in sessions table
|
||||
try:
|
||||
import hashlib, datetime
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires_at = datetime.datetime.utcnow() + datetime.timedelta(days=7)
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO sessions (user_id, token_hash, token, expires_at)
|
||||
VALUES (%s, %s, %s, %s) ON CONFLICT (token_hash) DO NOTHING""",
|
||||
(user["id"], token_hash, token, expires_at))
|
||||
conn.commit()
|
||||
except: pass
|
||||
return {
|
||||
"token": token,
|
||||
"user": {"id": user["id"], "name": user["name"], "email": user["email"], "role": user["role"]}
|
||||
}
|
||||
|
||||
@router.get("/auth/me")
|
||||
def me(user: dict = Depends(get_current_user)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT id,name,email,phone,country_code,country_name,role,created_at,last_login FROM users WHERE id=%s", (user["id"],))
|
||||
return dict(cur.fetchone())
|
||||
|
||||
# ── Admin: User Management ────────────────────────────────────────────────────
|
||||
@router.get("/admin/users")
|
||||
def list_users(_=Depends(require_admin)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT id,name,email,phone,country_name,role,is_active,created_at,last_login FROM users ORDER BY created_at DESC")
|
||||
return {"users": [dict(r) for r in cur.fetchall()]}
|
||||
|
||||
@router.patch("/admin/users/{user_id}/role")
|
||||
def update_role(user_id: int, req: UpdateRoleRequest, _=Depends(require_admin)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT name FROM roles WHERE name = %s", (req.role,))
|
||||
if not cur.fetchone():
|
||||
raise HTTPException(400, f"Invalid role: {req.role}")
|
||||
cur.execute("UPDATE users SET role = %s WHERE id = %s", (req.role, user_id))
|
||||
conn.commit()
|
||||
return {"status": "updated", "user_id": user_id, "role": req.role}
|
||||
|
||||
@router.patch("/admin/users/{user_id}/status")
|
||||
def toggle_user(user_id: int, _=Depends(require_admin)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("UPDATE users SET is_active = NOT is_active WHERE id = %s RETURNING is_active", (user_id,))
|
||||
result = cur.fetchone()
|
||||
conn.commit()
|
||||
return {"status": "updated", "is_active": result[0]}
|
||||
|
||||
@router.delete("/admin/users/{user_id}")
|
||||
def delete_user(user_id: int, _=Depends(require_admin)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("DELETE FROM users WHERE id = %s", (user_id,))
|
||||
conn.commit()
|
||||
return {"status": "deleted"}
|
||||
|
||||
# ── Roles ─────────────────────────────────────────────────────────────────────
|
||||
@router.get("/admin/roles")
|
||||
def list_roles(_=Depends(require_admin)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT * FROM roles ORDER BY name")
|
||||
return {"roles": [dict(r) for r in cur.fetchall()]}
|
||||
|
||||
# ── Library: Public ───────────────────────────────────────────────────────────
|
||||
@router.get("/api/library")
|
||||
def get_library(
|
||||
category: str = None,
|
||||
author: str = None,
|
||||
language: str = None,
|
||||
search: str = None,
|
||||
sort: str = "newest",
|
||||
page: int = 1,
|
||||
limit: int = 20,
|
||||
user=Depends(get_current_user_optional)
|
||||
):
|
||||
sort_map = {
|
||||
"newest": "b.uploaded_at DESC",
|
||||
"oldest": "b.uploaded_at ASC",
|
||||
"title_az": "b.title ASC",
|
||||
"title_za": "b.title DESC",
|
||||
"chunks_desc":"b.total_chunks DESC",
|
||||
}
|
||||
order = sort_map.get(sort, "b.uploaded_at DESC")
|
||||
where = ["is_public = TRUE"]
|
||||
params = []
|
||||
if category:
|
||||
where.append("category = %s"); params.append(category)
|
||||
if author:
|
||||
where.append("author ILIKE %s"); params.append(f"%{author}%")
|
||||
if language:
|
||||
where.append("language = %s"); params.append(language)
|
||||
if search:
|
||||
where.append("(filename ILIKE %s OR title ILIKE %s OR author ILIKE %s)")
|
||||
params.extend([f"%{search}%", f"%{search}%", f"%{search}%"])
|
||||
offset = (page - 1) * limit
|
||||
params_count = params.copy()
|
||||
params.extend([limit, offset])
|
||||
where_sql = " AND ".join(where)
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute(f"SELECT COUNT(*) FROM books WHERE {where_sql}", params_count)
|
||||
total = cur.fetchone()["count"]
|
||||
cur.execute(f"""
|
||||
SELECT b.id, b.filename, b.title, b.author, b.category, b.language,
|
||||
b.description, b.cover_url, b.total_chunks, b.uploaded_at,
|
||||
u.name as uploader_name
|
||||
FROM books b
|
||||
LEFT JOIN users u ON b.uploader_id = u.id
|
||||
WHERE {where_sql}
|
||||
ORDER BY {order}
|
||||
LIMIT %s OFFSET %s
|
||||
""", params)
|
||||
books = [dict(r) for r in cur.fetchall()]
|
||||
return {"books": books, "total": total, "page": page, "pages": -(-total // limit)}
|
||||
|
||||
@router.get("/api/library/categories")
|
||||
def get_categories():
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT DISTINCT category, COUNT(*) as cnt FROM books WHERE is_public=TRUE AND category IS NOT NULL GROUP BY category ORDER BY cnt DESC")
|
||||
return {"categories": [{"name": r[0], "count": r[1]} for r in cur.fetchall()]}
|
||||
|
||||
@router.get("/api/library/{book_id}")
|
||||
def get_book(book_id: int):
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT b.*, u.name as uploader_name
|
||||
FROM books b LEFT JOIN users u ON b.uploader_id = u.id
|
||||
WHERE b.id = %s AND b.is_public = TRUE
|
||||
""", (book_id,))
|
||||
book = cur.fetchone()
|
||||
if not book:
|
||||
raise HTTPException(404, "Book not found")
|
||||
return dict(book)
|
||||
|
||||
# ── Library: Upload (librarian/admin only) ────────────────────────────────────
|
||||
@router.post("/api/library/upload")
|
||||
async def library_upload(
|
||||
file: UploadFile = File(...),
|
||||
title: str = None,
|
||||
author: str = None,
|
||||
category: str = "",
|
||||
language: str = "ar",
|
||||
description: str = None,
|
||||
user: dict = Depends(require_upload)
|
||||
):
|
||||
safe_name = _safe_filename(file.filename)
|
||||
if not safe_name.lower().endswith(".pdf"):
|
||||
raise HTTPException(400, "Only PDF files allowed")
|
||||
|
||||
# File size limit: 50MB
|
||||
file_bytes = await file.read()
|
||||
if len(file_bytes) > 50 * 1024 * 1024:
|
||||
raise HTTPException(413, "File too large — maximum 50MB")
|
||||
|
||||
fhash = _file_hash(file_bytes)
|
||||
# Use sanitized name
|
||||
file_filename_orig = file.filename
|
||||
file.filename = safe_name
|
||||
|
||||
# Duplicate check BEFORE saving to disk
|
||||
existing = _find_duplicate(fhash, file.filename)
|
||||
if existing:
|
||||
raise HTTPException(409, {
|
||||
"error": "duplicate",
|
||||
"message": "هذا الكتاب موجود مسبقاً في المكتبة",
|
||||
"book": {
|
||||
"id": existing["id"],
|
||||
"title": existing.get("title") or existing["filename"],
|
||||
"author": existing.get("author") or "",
|
||||
"category": existing.get("category") or "",
|
||||
"uploaded_at": str(existing.get("uploaded_at") or ""),
|
||||
}
|
||||
})
|
||||
|
||||
# Not a duplicate — save to disk
|
||||
dest = os.path.join(BOOKS_DIR, file.filename)
|
||||
with open(dest, "wb") as f:
|
||||
f.write(file_bytes)
|
||||
|
||||
# Create a single-file job and dispatch to Celery
|
||||
# (compress → ingest → meta → R2 → Typesense all run async in worker)
|
||||
import uuid as _uuid
|
||||
job_id = str(_uuid.uuid4())
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO bulk_jobs (id, uploader_id, status, total)
|
||||
VALUES (%s, %s, 'pending', 1)""", (job_id, user["id"]))
|
||||
cur.execute("""INSERT INTO bulk_job_files (job_id, filename, status)
|
||||
VALUES (%s, %s, 'queued')""", (job_id, file.filename))
|
||||
cur.execute("""INSERT INTO books (filename, title, author, category, language,
|
||||
description, uploader_id, file_hash)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)
|
||||
ON CONFLICT (filename) DO UPDATE
|
||||
SET title=EXCLUDED.title, author=EXCLUDED.author,
|
||||
category=EXCLUDED.category, uploader_id=EXCLUDED.uploader_id,
|
||||
file_hash=EXCLUDED.file_hash""",
|
||||
(file.filename, title or file.filename.replace(".pdf",""),
|
||||
author or "", category or "", language, description,
|
||||
user["id"], fhash))
|
||||
conn.commit()
|
||||
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT id FROM bulk_job_files WHERE job_id=%s LIMIT 1", (job_id,))
|
||||
file_row = cur.fetchone()
|
||||
|
||||
try:
|
||||
from tasks import process_bulk_file
|
||||
process_bulk_file.delay(job_id, file_row["id"], file.filename, user["id"])
|
||||
except Exception as _ce:
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("UPDATE bulk_jobs SET status='error' WHERE id=%s", (job_id,))
|
||||
conn.commit()
|
||||
raise HTTPException(503, f"Task queue unavailable: {_ce}")
|
||||
|
||||
return {
|
||||
"status": "queued",
|
||||
"file": file.filename,
|
||||
"job_id": job_id,
|
||||
"status_url": f"/api/library/bulk-status/{job_id}",
|
||||
}
|
||||
|
||||
@router.patch("/api/library/{book_id}")
|
||||
def update_book(book_id: int, data: BookMetaUpdate, user: dict = Depends(require_upload)):
|
||||
ALLOWED_FIELDS = {"title", "author", "description", "category", "language", "is_public"}
|
||||
updates = {k: v for k, v in data.dict().items() if v is not None and k in ALLOWED_FIELDS}
|
||||
if not updates:
|
||||
raise HTTPException(400, "No fields to update")
|
||||
set_sql = ", ".join([f"{k}=%s" for k in updates]) # keys from allowlist only
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"UPDATE books SET {set_sql} WHERE id=%s", list(updates.values()) + [book_id])
|
||||
conn.commit()
|
||||
# Sync updated metadata to Typesense
|
||||
try:
|
||||
_ts_index_book(book_id)
|
||||
except Exception as _e:
|
||||
print(f"[update_book] Typesense sync failed: {_e}")
|
||||
return {"status": "updated"}
|
||||
|
||||
# ── Favorites ─────────────────────────────────────────────────────────────────
|
||||
@router.post("/api/library/{book_id}/favorite")
|
||||
def toggle_favorite(book_id: int, user: dict = Depends(get_current_user)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
|
||||
exists = cur.fetchone()
|
||||
if exists:
|
||||
cur.execute("DELETE FROM favorites WHERE user_id=%s AND book_id=%s", (user["id"], book_id))
|
||||
action = "removed"
|
||||
else:
|
||||
cur.execute("INSERT INTO favorites (user_id, book_id) VALUES (%s,%s)", (user["id"], book_id))
|
||||
action = "added"
|
||||
conn.commit()
|
||||
return {"status": action}
|
||||
|
||||
@router.get("/api/library/favorites")
|
||||
def get_favorites(user: dict = Depends(get_current_user)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT b.id, b.filename, b.title, b.author, b.category, b.total_chunks, b.uploaded_at
|
||||
FROM favorites f JOIN books b ON b.id=f.book_id
|
||||
WHERE f.user_id=%s ORDER BY f.created_at DESC
|
||||
""", (user["id"],))
|
||||
return {"favorites": [dict(r) for r in cur.fetchall()]}
|
||||
|
||||
@router.get("/api/library/my-favorites")
|
||||
def my_favorite_ids(user: dict = Depends(get_current_user)):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT book_id FROM favorites WHERE user_id=%s", (user["id"],))
|
||||
return {"ids": [r[0] for r in cur.fetchall()]}
|
||||
|
||||
# ── Download & Read ───────────────────────────────────────────────────────────
|
||||
@router.get("/api/library/{book_id}/download")
|
||||
def download_book(book_id: int):
|
||||
import os
|
||||
from urllib.parse import quote
|
||||
from fastapi.responses import StreamingResponse, RedirectResponse
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
|
||||
book = cur.fetchone()
|
||||
if not book:
|
||||
raise HTTPException(404, "Book not found")
|
||||
|
||||
# R2 configured → redirect, zero VPS bandwidth used
|
||||
r2_public = os.getenv("R2_PUBLIC_URL", "").rstrip("/")
|
||||
if r2_public:
|
||||
return RedirectResponse(
|
||||
url=f"{r2_public}/pdfs/{book['filename']}",
|
||||
status_code=302
|
||||
)
|
||||
|
||||
# Fallback: stream from VPS disk
|
||||
path = book.get("pdf_path") or os.path.join(BOOKS_DIR, book["filename"])
|
||||
if not os.path.exists(path):
|
||||
raise HTTPException(404, "PDF file not available. Please re-upload this book.")
|
||||
safe_name = quote(book["filename"], safe="")
|
||||
def stream():
|
||||
with open(path, "rb") as f: yield from f
|
||||
return StreamingResponse(stream(), media_type="application/pdf", headers={
|
||||
"Content-Disposition": f"attachment; filename*=UTF-8''{safe_name}",
|
||||
"Content-Length": str(os.path.getsize(path)), "Cache-Control": "private, max-age=3600"
|
||||
})
|
||||
|
||||
@router.get("/api/library/{book_id}/read")
|
||||
def read_book(book_id: int):
|
||||
import os
|
||||
from urllib.parse import quote
|
||||
from fastapi.responses import StreamingResponse, RedirectResponse
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT filename, pdf_path FROM books WHERE id=%s AND is_public=TRUE", (book_id,))
|
||||
book = cur.fetchone()
|
||||
if not book:
|
||||
raise HTTPException(404, "Book not found")
|
||||
|
||||
# R2 configured → redirect to R2 (reader iframe follows the redirect automatically)
|
||||
r2_public = os.getenv("R2_PUBLIC_URL", "").rstrip("/")
|
||||
if r2_public:
|
||||
return RedirectResponse(
|
||||
url=f"{r2_public}/pdfs/{book['filename']}",
|
||||
status_code=302
|
||||
)
|
||||
|
||||
# Fallback: stream from VPS disk
|
||||
path = book.get("pdf_path") or os.path.join(BOOKS_DIR, book["filename"])
|
||||
if not os.path.exists(path):
|
||||
raise HTTPException(404, "PDF file not available. Please re-upload this book.")
|
||||
safe_name = quote(book["filename"], safe="")
|
||||
def stream():
|
||||
with open(path, "rb") as f: yield from f
|
||||
return StreamingResponse(stream(), media_type="application/pdf", headers={
|
||||
"Content-Disposition": f"inline; filename*=UTF-8''{safe_name}",
|
||||
"Content-Length": str(os.path.getsize(path)), "Cache-Control": "private, max-age=3600"
|
||||
})
|
||||
|
||||
|
||||
# ── Bulk Upload ───────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/api/library/bulk-upload")
|
||||
async def bulk_upload(
|
||||
files: List[UploadFile] = File(...),
|
||||
user: dict = Depends(require_upload)
|
||||
):
|
||||
"""Accept multiple PDFs, save to disk, queue for background processing."""
|
||||
if not files:
|
||||
raise HTTPException(400, "No files provided")
|
||||
|
||||
job_id = str(uuid.uuid4())
|
||||
saved = []
|
||||
skipped = []
|
||||
|
||||
for file in files:
|
||||
safe_name = _safe_filename(file.filename)
|
||||
if not safe_name.lower().endswith(".pdf"):
|
||||
skipped.append({"filename": file.filename, "reason": "ليس ملف PDF"})
|
||||
continue
|
||||
|
||||
try:
|
||||
file_bytes = await file.read()
|
||||
except Exception as e:
|
||||
skipped.append({"filename": file.filename, "reason": f"خطأ في القراءة: {e}"})
|
||||
continue
|
||||
|
||||
# File size limit: 50MB per file
|
||||
if len(file_bytes) > 50 * 1024 * 1024:
|
||||
skipped.append({"filename": file.filename, "reason": "الملف أكبر من 50MB"})
|
||||
continue
|
||||
|
||||
file.filename = safe_name
|
||||
|
||||
fhash = _file_hash(file_bytes)
|
||||
existing = _find_duplicate(fhash, file.filename)
|
||||
if existing:
|
||||
orig_title = existing.get("title") or existing["filename"]
|
||||
skipped.append({
|
||||
"filename": file.filename,
|
||||
"reason": "مكرر",
|
||||
"duplicate_of": orig_title,
|
||||
"existing_id": existing["id"],
|
||||
})
|
||||
continue
|
||||
|
||||
dest = os.path.join(BOOKS_DIR, file.filename)
|
||||
try:
|
||||
with open(dest, "wb") as f:
|
||||
f.write(file_bytes)
|
||||
saved.append(file.filename)
|
||||
except Exception as e:
|
||||
skipped.append({"filename": file.filename, "reason": str(e)})
|
||||
|
||||
if not saved:
|
||||
# All files were either invalid or duplicates — return details
|
||||
raise HTTPException(400, {
|
||||
"error": "all_duplicates",
|
||||
"message": "جميع الملفات موجودة مسبقاً في المكتبة",
|
||||
"skipped": skipped,
|
||||
})
|
||||
|
||||
# Create job + file records in DB (include skipped duplicates for visibility)
|
||||
total_for_job = len(saved) + len([s for s in skipped if s.get("reason") == "مكرر"])
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""INSERT INTO bulk_jobs (id, uploader_id, status, total)
|
||||
VALUES (%s, %s, 'pending', %s)""",
|
||||
(job_id, user["id"], len(saved)))
|
||||
for filename in saved:
|
||||
cur.execute("""INSERT INTO bulk_job_files (job_id, filename, status)
|
||||
VALUES (%s, %s, 'queued')""", (job_id, filename))
|
||||
# Add skipped duplicates as 'skipped' status so UI can show them
|
||||
for skip in skipped:
|
||||
if skip.get("reason") == "مكرر":
|
||||
cur.execute("""INSERT INTO bulk_job_files (job_id, filename, status, error)
|
||||
VALUES (%s, %s, 'skipped', %s)""",
|
||||
(job_id, skip["filename"],
|
||||
f"مكرر: {skip.get('duplicate_of', '')}"))
|
||||
conn.commit()
|
||||
|
||||
# Dispatch each saved file as a Celery task
|
||||
if saved:
|
||||
try:
|
||||
from tasks import process_bulk_file
|
||||
_db2 = get_db()
|
||||
with _db2.cursor(cursor_factory=RealDictCursor) as _cur2:
|
||||
_cur2.execute(
|
||||
"SELECT id, filename FROM bulk_job_files WHERE job_id=%s AND status='queued'",
|
||||
(job_id,)
|
||||
)
|
||||
_qfiles = _cur2.fetchall()
|
||||
_db2.close()
|
||||
for _qf in _qfiles:
|
||||
process_bulk_file.delay(job_id, _qf["id"], _qf["filename"], user["id"])
|
||||
except Exception as _ce:
|
||||
print(f"Celery dispatch failed, using threading worker: {_ce}")
|
||||
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"queued": len(saved),
|
||||
"skipped": skipped,
|
||||
"status_url": f"/api/library/bulk-status/{job_id}"
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/library/bulk-status/{job_id}")
|
||||
def bulk_status(job_id: str, user: dict = Depends(require_upload)):
|
||||
"""Poll job progress."""
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT * FROM bulk_jobs WHERE id=%s", (job_id,))
|
||||
job = cur.fetchone()
|
||||
if not job:
|
||||
raise HTTPException(404, "Job not found")
|
||||
cur.execute("""SELECT id, filename, status, meta, error, book_id, updated_at
|
||||
FROM bulk_job_files WHERE job_id=%s ORDER BY id""", (job_id,))
|
||||
files = cur.fetchall()
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"status": job["status"],
|
||||
"total": job["total"],
|
||||
"done": job["done"],
|
||||
"failed": job["failed"],
|
||||
"files": [dict(f) for f in files]
|
||||
}
|
||||
|
||||
|
||||
@router.patch("/api/library/bulk-file/{file_id}")
|
||||
def update_bulk_file_meta(file_id: int, data: BookMetaUpdate, user: dict = Depends(require_upload)):
|
||||
"""Allow user to edit extracted metadata before it's finalised."""
|
||||
ALLOWED_FIELDS = {"title", "author", "description", "category", "language"}
|
||||
updates = {k: v for k, v in data.dict().items() if v is not None and k in ALLOWED_FIELDS}
|
||||
if not updates:
|
||||
raise HTTPException(400, "No fields to update")
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT book_id FROM bulk_job_files WHERE id=%s", (file_id,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
raise HTTPException(404, "File record not found")
|
||||
if row["book_id"]:
|
||||
# Update the actual books table — keys from allowlist only
|
||||
set_clause = ", ".join(f"{k}=%s" for k in updates)
|
||||
cur.execute(f"UPDATE books SET {set_clause} WHERE id=%s",
|
||||
(*updates.values(), row["book_id"]))
|
||||
# Update the meta snapshot in bulk_job_files too
|
||||
cur.execute("UPDATE bulk_job_files SET meta=%s WHERE id=%s",
|
||||
(json.dumps(updates), file_id))
|
||||
conn.commit()
|
||||
return {"status": "updated"}
|
||||
|
||||
@router.get("/library/{book_id}/cover")
|
||||
def serve_cover(book_id: int):
|
||||
"""Serve book cover image. Generates on-demand if missing."""
|
||||
import os
|
||||
from fastapi.responses import FileResponse, Response
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
COVER_DIR = "/mnt/rag-media/covers"
|
||||
cover_path = os.path.join(COVER_DIR, f"{book_id}.jpg")
|
||||
|
||||
# R2 configured → redirect to R2 cover
|
||||
r2_public = os.getenv("R2_PUBLIC_URL", "").rstrip("/")
|
||||
if r2_public:
|
||||
from fastapi.responses import RedirectResponse
|
||||
return RedirectResponse(
|
||||
url=f"{r2_public}/covers/{book_id}.jpg",
|
||||
status_code=302,
|
||||
headers={"Cache-Control": "public, max-age=86400"}
|
||||
)
|
||||
|
||||
# Already generated locally
|
||||
if os.path.exists(cover_path):
|
||||
return FileResponse(cover_path, media_type="image/jpeg",
|
||||
headers={"Cache-Control": "public, max-age=86400"})
|
||||
|
||||
# Generate on-demand from PDF
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT pdf_path, filename FROM books WHERE id = %s", (book_id,))
|
||||
book = cur.fetchone()
|
||||
|
||||
if not book or not book.get("pdf_path"):
|
||||
raise HTTPException(404, "No PDF found for this book")
|
||||
|
||||
pdf_path = book["pdf_path"]
|
||||
if not os.path.exists(pdf_path):
|
||||
raise HTTPException(404, "PDF file not found on disk")
|
||||
|
||||
try:
|
||||
import fitz
|
||||
doc = fitz.open(pdf_path)
|
||||
page = doc[0]
|
||||
mat = fitz.Matrix(1.5, 1.5)
|
||||
pix = page.get_pixmap(matrix=mat, colorspace=fitz.csRGB)
|
||||
os.makedirs(COVER_DIR, exist_ok=True)
|
||||
pix.save(cover_path)
|
||||
doc.close()
|
||||
return FileResponse(cover_path, media_type="image/jpeg",
|
||||
headers={"Cache-Control": "public, max-age=86400"})
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Cover generation failed: {e}")
|
||||
|
||||
+1921
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,548 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="dark" lang="ar" dir="rtl">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<title>AfakZaid RAG</title>
|
||||
<script src="https://cdn.tailwindcss.com?plugins=forms"></script>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700;800&display=swap" rel="stylesheet"/>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:wght,FILL@100..700,0..1&display=swap" rel="stylesheet"/>
|
||||
<script>tailwind.config={darkMode:"class",theme:{extend:{colors:{"primary":"#4edea3","surface":"#0c1324","surface-low":"#151b2d","surface-container":"#191f31","surface-high":"#23293c","on-surface":"#dce1fb","on-surface-variant":"#bbcabf","outline-variant":"#3c4a42","secondary":"#95d3ba","tertiary":"#45dfa4"},fontFamily:{sans:["Manrope","sans-serif"]}}}}</script>
|
||||
<style>
|
||||
/* ── Emil design vars ─────────────────────────────────────────────────────── */
|
||||
:root {
|
||||
--ease-out: cubic-bezier(0.23, 1, 0.32, 1);
|
||||
--ease-in-out: cubic-bezier(0.77, 0, 0.175, 1);
|
||||
--ease-drawer: cubic-bezier(0.32, 0.72, 0, 1);
|
||||
}
|
||||
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{background:radial-gradient(circle at 50% -20%,#151b2d 0%,#0c1324 100%);font-family:'Manrope',sans-serif;color:#dce1fb;min-height:100vh;overflow-x:hidden}
|
||||
.glass{background:rgba(21,27,45,0.5);backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border:1px solid rgba(78,222,163,0.12)}
|
||||
|
||||
/* Exact properties — never transition:all */
|
||||
.glass-hover {
|
||||
transition:background 200ms ease, border-color 200ms ease;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.glass-hover:hover{background:rgba(78,222,163,0.06);border-color:rgba(78,222,163,0.25)}
|
||||
}
|
||||
|
||||
.glow{box-shadow:0 0 20px rgba(78,222,163,0.15)}
|
||||
.glow-sm{box-shadow:0 0 10px rgba(78,222,163,0.1)}
|
||||
.msym{font-family:'Material Symbols Outlined';font-variation-settings:'FILL' 0,'wght' 400,'GRAD' 0,'opsz' 24;vertical-align:middle;line-height:1}
|
||||
|
||||
/* Upload zone — specific props, no transform on touch */
|
||||
.upload-zone{
|
||||
border:2px dashed rgba(78,222,163,0.2);
|
||||
transition:border-color 200ms ease, background 200ms ease;
|
||||
}
|
||||
.upload-zone.dragover{border-color:rgba(78,222,163,0.6);background:rgba(78,222,163,0.05)}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.upload-zone:hover{border-color:rgba(78,222,163,0.4);background:rgba(78,222,163,0.03)}
|
||||
}
|
||||
|
||||
.scrollbar::-webkit-scrollbar{width:4px}
|
||||
.scrollbar::-webkit-scrollbar-thumb{background:rgba(78,222,163,0.2);border-radius:4px}
|
||||
|
||||
/* ── Entry keyframes ──────────────────────────────────────────────────────── */
|
||||
/* Displacement: 8px max — subtle, professional */
|
||||
@keyframes fadeUp {
|
||||
from { opacity:0; transform:translateY(8px); }
|
||||
to { opacity:1; transform:translateY(0); }
|
||||
}
|
||||
@keyframes fadeIn {
|
||||
from { opacity:0; }
|
||||
to { opacity:1; }
|
||||
}
|
||||
@keyframes slideDown {
|
||||
from { opacity:0; transform:translateY(-8px); }
|
||||
to { opacity:1; transform:translateY(0); }
|
||||
}
|
||||
/* Panels: 16px — noticeable but not theatrical */
|
||||
@keyframes slideRight {
|
||||
from { opacity:0; transform:translateX(16px); }
|
||||
to { opacity:1; transform:translateX(0); }
|
||||
}
|
||||
@keyframes slideLeft {
|
||||
from { opacity:0; transform:translateX(-16px); }
|
||||
to { opacity:1; transform:translateX(0); }
|
||||
}
|
||||
/* Scale from 0.95 not 0.88 — nothing appears from nothing */
|
||||
@keyframes scalePop {
|
||||
0% { opacity:0; transform:scale(0.95); }
|
||||
100% { opacity:1; transform:scale(1); }
|
||||
}
|
||||
/* Answer box entry */
|
||||
@keyframes fadeUpFast {
|
||||
from { opacity:0; transform:translateY(6px); }
|
||||
to { opacity:1; transform:translateY(0); }
|
||||
}
|
||||
|
||||
/* ── Ambient orb — slow linear drift, off main thread ────────────────────── */
|
||||
@keyframes orbDrift {
|
||||
0%,100% { transform:translate(0,0) scale(1); }
|
||||
50% { transform:translate(-8px,10px) scale(1.05); }
|
||||
}
|
||||
@keyframes orbDrift2 {
|
||||
0%,100% { transform:translate(0,0) scale(1); }
|
||||
50% { transform:translate(10px,-8px) scale(0.97); }
|
||||
}
|
||||
|
||||
/* ── Scan line on brand icon — runs once on load, then stops ─────────────── */
|
||||
@keyframes scanLine {
|
||||
0% { top:-100%; opacity:.5; }
|
||||
100% { top:200%; opacity:0; }
|
||||
}
|
||||
|
||||
/* ── Scroll reveal ────────────────────────────────────────────────────────── */
|
||||
.reveal {
|
||||
opacity:0;
|
||||
transform:translateY(8px);
|
||||
/* ease-out: entering element, starts fast */
|
||||
transition:opacity 400ms var(--ease-out), transform 400ms var(--ease-out);
|
||||
}
|
||||
.reveal.visible { opacity:1; transform:translateY(0); }
|
||||
|
||||
/* ── Entry sequencing ────────────────────────────────────────────────────── */
|
||||
/* Header: instant, just a subtle drop */
|
||||
header { animation:slideDown 350ms var(--ease-out) both; }
|
||||
|
||||
/* Stats: stagger 50ms — Emil says 30-80ms */
|
||||
.stat-card { animation:scalePop 400ms var(--ease-out) both; }
|
||||
.stat-card:nth-child(1) { animation-delay: 80ms; }
|
||||
.stat-card:nth-child(2) { animation-delay:130ms; }
|
||||
.stat-card:nth-child(3) { animation-delay:180ms; }
|
||||
.stat-card:nth-child(4) { animation-delay:230ms; }
|
||||
|
||||
/* Panels enter slightly later */
|
||||
.query-card { animation:slideLeft 450ms var(--ease-out) 300ms both; }
|
||||
.upload-card-wrap { animation:slideRight 450ms var(--ease-out) 340ms both; }
|
||||
.books-card { animation:fadeUp 400ms var(--ease-out) 440ms both; }
|
||||
|
||||
/* ── Stat card — hover only on pointer devices ───────────────────────────── */
|
||||
.stat-card {
|
||||
cursor:default;
|
||||
will-change:transform;
|
||||
}
|
||||
.stat-num {
|
||||
font-variant-numeric:tabular-nums;
|
||||
transition:color 200ms ease;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.stat-card {
|
||||
transition:transform 200ms var(--ease-out), box-shadow 200ms ease;
|
||||
}
|
||||
.stat-card:hover {
|
||||
transform:translateY(-3px) scale(1.02);
|
||||
box-shadow:0 8px 24px rgba(78,222,163,0.14);
|
||||
}
|
||||
.stat-card:hover .stat-num { color:#fff; }
|
||||
}
|
||||
/* Active state — works on all devices */
|
||||
.stat-card:active { transform:scale(0.97); }
|
||||
|
||||
/* ── Book cards ──────────────────────────────────────────────────────────── */
|
||||
.book-card {
|
||||
will-change:transform;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.book-card {
|
||||
transition:transform 200ms var(--ease-out), box-shadow 200ms ease, border-color 200ms ease;
|
||||
}
|
||||
.book-card:hover {
|
||||
transform:translateY(-2px);
|
||||
box-shadow:0 6px 20px rgba(78,222,163,0.12);
|
||||
border-color:rgba(78,222,163,0.28) !important;
|
||||
}
|
||||
}
|
||||
.book-card:active { transform:scale(0.98); }
|
||||
|
||||
/* ── Query button ────────────────────────────────────────────────────────── */
|
||||
#query-btn {
|
||||
/* No constant animation — button is used frequently */
|
||||
transition:transform 150ms var(--ease-out), box-shadow 200ms ease, filter 200ms ease;
|
||||
position:relative; overflow:hidden;
|
||||
}
|
||||
/* Glow only on focus/hover — not constantly */
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
#query-btn:hover {
|
||||
transform:scale(1.015);
|
||||
box-shadow:0 0 28px rgba(78,222,163,0.45);
|
||||
filter:brightness(1.08);
|
||||
}
|
||||
}
|
||||
#query-btn:focus-visible {
|
||||
box-shadow:0 0 0 3px rgba(78,222,163,0.4);
|
||||
outline:none;
|
||||
}
|
||||
/* Active: fast press feedback — Emil says 100-160ms */
|
||||
#query-btn:active { transform:scale(0.97); transition-duration:100ms; }
|
||||
|
||||
/* Shimmer on hover */
|
||||
#query-btn::after {
|
||||
content:'';
|
||||
position:absolute; inset:0;
|
||||
background:linear-gradient(90deg, transparent 0%, rgba(255,255,255,.1) 50%, transparent 100%);
|
||||
transform:translateX(-100%);
|
||||
transition:transform 380ms var(--ease-in-out);
|
||||
pointer-events:none;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
#query-btn:hover::after { transform:translateX(100%); }
|
||||
}
|
||||
|
||||
/* ── Brand icon scan line — fires once on load ───────────────────────────── */
|
||||
.brand-icon-wrap { position:relative; overflow:hidden; }
|
||||
.brand-icon-wrap::after {
|
||||
content:'';
|
||||
position:absolute; left:0; right:0; height:30%;
|
||||
background:linear-gradient(to bottom, transparent, rgba(78,222,163,.25), transparent);
|
||||
animation:scanLine 1.8s var(--ease-in-out) 0.6s 1 forwards;
|
||||
opacity:0;
|
||||
}
|
||||
|
||||
/* ── Answer reveal ───────────────────────────────────────────────────────── */
|
||||
#answer-wrap { animation:fadeUpFast 280ms var(--ease-out) both; }
|
||||
|
||||
/* ── Textarea ────────────────────────────────────────────────────────────── */
|
||||
#query-input {
|
||||
transition:border-color 200ms ease, box-shadow 200ms ease;
|
||||
}
|
||||
#query-input:focus { box-shadow:0 0 0 2px rgba(78,222,163,.12); }
|
||||
|
||||
/* ── Orbs: slow linear, off main thread ─────────────────────────────────── */
|
||||
.orb-1 { animation:orbDrift 18s linear infinite; }
|
||||
.orb-2 { animation:orbDrift2 24s linear infinite 3s; }
|
||||
|
||||
/* ── prefers-reduced-motion — keep opacity fades, remove transforms ──────── */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*,*::before,*::after {
|
||||
animation-duration:.01ms !important;
|
||||
animation-iteration-count:1 !important;
|
||||
transition-duration:.01ms !important;
|
||||
}
|
||||
.reveal { opacity:1; transform:none; }
|
||||
}
|
||||
</style>
|
||||
<script src="/static/analytics.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- Ambient orbs -->
|
||||
<div class="fixed top-0 right-0 -z-10 w-96 h-96 bg-primary/5 rounded-full blur-[120px] pointer-events-none orb-1"></div>
|
||||
<div class="fixed bottom-0 left-0 -z-10 w-72 h-72 bg-tertiary/5 rounded-full blur-[100px] pointer-events-none orb-2"></div>
|
||||
<div class="fixed top-1/2 left-1/2 -z-10 w-64 h-64 bg-secondary/3 rounded-full blur-[140px] pointer-events-none" style="transform:translate(-50%,-50%);animation:orbFloat 16s ease-in-out infinite 4s"></div>
|
||||
|
||||
<!-- Header -->
|
||||
<header class="sticky top-0 z-20 glass border-b border-primary/20">
|
||||
<div class="max-w-6xl mx-auto px-4 lg:px-8 py-3 flex items-center justify-between gap-3">
|
||||
<div class="flex items-center gap-3 shrink-0">
|
||||
<div class="w-8 h-8 bg-primary/15 rounded-xl flex items-center justify-center shrink-0 brand-icon-wrap">
|
||||
<span class="msym text-primary text-base">auto_awesome</span>
|
||||
</div>
|
||||
<div class="min-w-0 hidden sm:block">
|
||||
<h1 class="text-sm font-bold text-primary leading-tight truncate" id="brand-name">AfakZaid RAG</h1>
|
||||
<p class="text-[10px] text-on-surface-variant/60 uppercase tracking-widest" id="brand-sub">Scholarly Intelligence</p>
|
||||
</div>
|
||||
<div class="h-5 w-px bg-outline-variant/40 hidden sm:block"></div>
|
||||
<a href="/library" class="glass glass-hover flex items-center gap-1 px-2.5 py-2 rounded-xl text-xs text-on-surface-variant hover:text-primary transition-colors">
|
||||
<span class="msym text-base">library_books</span>
|
||||
<span class="hidden sm:inline">المكتبة</span>
|
||||
</a>
|
||||
</div>
|
||||
<nav class="flex items-center gap-2 shrink-0">
|
||||
<div id="nav-user" class="shrink-0"></div>
|
||||
</nav>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main class="max-w-6xl mx-auto px-4 lg:px-8 py-8">
|
||||
|
||||
<!-- Stats -->
|
||||
<div class="grid grid-cols-2 lg:grid-cols-4 gap-4 mb-8">
|
||||
<div class="glass glow rounded-2xl p-4 text-center stat-card">
|
||||
<div class="text-2xl font-bold text-primary stat-num" id="s-books">0</div>
|
||||
<div class="text-xs text-on-surface-variant mt-1">كتاب</div>
|
||||
</div>
|
||||
<div class="glass glow rounded-2xl p-4 text-center stat-card">
|
||||
<div class="text-2xl font-bold text-primary stat-num" id="s-vectors">0</div>
|
||||
<div class="text-xs text-on-surface-variant mt-1">متجه</div>
|
||||
</div>
|
||||
<div class="glass glow rounded-2xl p-4 text-center stat-card">
|
||||
<div class="text-2xl font-bold text-secondary stat-num" id="s-convos">0</div>
|
||||
<div class="text-xs text-on-surface-variant mt-1">محادثة</div>
|
||||
</div>
|
||||
<div class="glass glow rounded-2xl p-4 text-center stat-card">
|
||||
<div class="text-2xl font-bold text-tertiary stat-num" id="s-memory">0</div>
|
||||
<div class="text-xs text-on-surface-variant mt-1">ذاكرة</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-6">
|
||||
|
||||
<!-- Query -->
|
||||
<div class="glass rounded-3xl p-6 query-card">
|
||||
<h2 class="text-primary text-xs font-bold uppercase tracking-widest mb-4 flex items-center gap-2">
|
||||
<span class="msym text-base">search</span> اسأل الكتب
|
||||
</h2>
|
||||
<textarea id="query-input" rows="3" dir="rtl"
|
||||
class="w-full bg-surface-high border border-outline-variant rounded-2xl px-4 py-3 text-on-surface text-sm resize-none outline-none focus:border-primary transition-colors mb-3"
|
||||
placeholder="اكتب سؤالك هنا..."></textarea>
|
||||
<button onclick="doQuery()" id="query-btn"
|
||||
class="w-full bg-primary text-surface font-bold py-3 rounded-xl flex items-center justify-center gap-2">
|
||||
<span class="msym text-lg">search</span> بحث
|
||||
</button>
|
||||
<div id="answer-wrap" class="hidden mt-4">
|
||||
<div id="answer-box" class="bg-surface-high rounded-2xl p-4 text-sm leading-7 text-on-surface whitespace-pre-wrap max-h-64 overflow-y-auto scrollbar"></div>
|
||||
<div id="answer-meta" class="flex flex-wrap gap-2 mt-3"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Upload -->
|
||||
<div class="upload-card-wrap">
|
||||
<div class="glass rounded-3xl p-6 h-full" id="upload-card">
|
||||
<h2 class="text-primary text-xs font-bold uppercase tracking-widest mb-4 flex items-center gap-2">
|
||||
<span class="msym text-base">upload_file</span> رفع كتاب PDF
|
||||
</h2>
|
||||
<div class="upload-zone rounded-2xl p-8 text-center cursor-pointer relative mb-4" id="drop-zone">
|
||||
<input type="file" accept=".pdf" multiple class="absolute inset-0 opacity-0 cursor-pointer w-full h-full" onchange="handleFiles(this.files)"/>
|
||||
<div class="w-14 h-14 bg-primary/10 rounded-full flex items-center justify-center mx-auto mb-3 glow" style="transition:transform .3s">
|
||||
<span class="msym text-primary text-3xl">cloud_upload</span>
|
||||
</div>
|
||||
<p class="text-on-surface-variant text-sm">اسحب ملفات PDF وأفلتها هنا</p>
|
||||
<p class="text-primary text-xs font-bold mt-1">أو انقر للاختيار</p>
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<input type="text" id="url-input" dir="ltr" placeholder="https://example.com/book.pdf"
|
||||
class="flex-1 bg-surface-high border border-outline-variant rounded-xl px-3 py-2 text-xs text-on-surface outline-none focus:border-primary transition-colors"/>
|
||||
<button onclick="ingestURL()" class="bg-primary/10 border border-primary/30 text-primary px-3 py-2 rounded-xl text-xs font-bold hover:bg-primary/20 transition-colors flex items-center gap-1">
|
||||
<span class="msym text-sm">add</span>
|
||||
</button>
|
||||
</div>
|
||||
<div id="upload-progress" class="hidden mt-3">
|
||||
<div class="bg-surface-high rounded-full h-1.5 overflow-hidden">
|
||||
<div id="progress-fill" class="h-1.5 bg-primary rounded-full transition-all duration-500" style="width:0%"></div>
|
||||
</div>
|
||||
<p id="progress-text" class="text-xs text-on-surface-variant mt-2"></p>
|
||||
</div>
|
||||
<pre id="log-box" class="hidden mt-2 text-xs text-on-surface-variant bg-surface-high rounded-xl p-3 max-h-24 overflow-y-auto scrollbar font-mono"></pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<!-- Books -->
|
||||
<div class="glass rounded-3xl p-6 books-card reveal" id="books-section">
|
||||
<div class="flex items-center justify-between mb-5">
|
||||
<h2 class="text-primary text-xs font-bold uppercase tracking-widest flex items-center gap-2">
|
||||
<span class="msym text-base">library_books</span> الكتب المُدرجة
|
||||
</h2>
|
||||
<a href="/library" class="text-xs text-on-surface-variant hover:text-primary transition-colors flex items-center gap-1">
|
||||
عرض الكل <span class="msym text-sm">arrow_back</span>
|
||||
</a>
|
||||
</div>
|
||||
<div id="books-list" class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-3"></div>
|
||||
</div>
|
||||
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const API_KEY = 'zRf77E_brSEnA2BUi40uxLxT4uofAk661WMZb-cdv4M';
|
||||
let userToken = localStorage.getItem('rag_token');
|
||||
let currentUser = null;
|
||||
|
||||
// ── Branding ──────────────────────────────────────────────────────────────────
|
||||
fetch('/branding').then(r=>r.json()).then(b=>{
|
||||
if(b.site_name){document.title=b.site_name;document.getElementById('brand-name').textContent=b.site_name;}
|
||||
if(b.site_subtitle) document.getElementById('brand-sub').textContent=b.site_subtitle;
|
||||
}).catch(()=>{});
|
||||
|
||||
// ── Counter animation ─────────────────────────────────────────────────────────
|
||||
function animateCount(el, target, duration=900) {
|
||||
const start = performance.now();
|
||||
const from = parseInt(el.textContent.replace(/,/g,'')) || 0;
|
||||
const isLarge = target > 999;
|
||||
function tick(now) {
|
||||
const elapsed = now - start;
|
||||
const progress = Math.min(elapsed / duration, 1);
|
||||
// ease out cubic
|
||||
const eased = 1 - Math.pow(1 - progress, 3);
|
||||
const current = Math.round(from + (target - from) * eased);
|
||||
el.textContent = isLarge ? current.toLocaleString() : current;
|
||||
if (progress < 1) requestAnimationFrame(tick);
|
||||
else el.textContent = isLarge ? target.toLocaleString() : target;
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
// ── Scroll reveal ─────────────────────────────────────────────────────────────
|
||||
const observer = new IntersectionObserver(entries => {
|
||||
entries.forEach(e => { if(e.isIntersecting) { e.target.classList.add('visible'); observer.unobserve(e.target); } });
|
||||
}, { threshold: 0.12 });
|
||||
document.querySelectorAll('.reveal').forEach(el => observer.observe(el));
|
||||
|
||||
// ── Auth ──────────────────────────────────────────────────────────────────────
|
||||
async function initAuth() {
|
||||
const nav = document.getElementById('nav-user');
|
||||
if (!userToken) {
|
||||
nav.innerHTML = `<a href="/login" class="bg-primary text-surface font-bold px-3 py-2 rounded-xl text-xs hover:shadow-[0_0_15px_rgba(78,222,163,0.4)] transition-all flex items-center gap-1.5 whitespace-nowrap" style="animation:scalePop .4s cubic-bezier(.22,1,.36,1) .8s both"><span class="msym text-base">login</span><span>دخول</span></a>`;
|
||||
document.getElementById('upload-card').classList.add('hidden');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const r = await fetch('/auth/me', {headers:{'Authorization':'Bearer '+userToken}});
|
||||
if (!r.ok) { localStorage.removeItem('rag_token'); initAuth(); return; }
|
||||
currentUser = await r.json();
|
||||
const perms = currentUser.permissions || [];
|
||||
if (!perms.includes('upload')) document.getElementById('upload-card').classList.add('hidden');
|
||||
nav.innerHTML = `
|
||||
<div class="flex items-center gap-2" style="animation:fadeIn .4s both .6s">
|
||||
<div class="text-right hidden sm:block">
|
||||
<p class="text-xs font-bold">${currentUser.name}</p>
|
||||
<p class="text-[10px] text-primary">${currentUser.role}</p>
|
||||
</div>
|
||||
${perms.includes('admin') ? '<a href="/admin" class="glass glass-hover p-2 rounded-xl transition-colors"><span class="msym text-base text-on-surface-variant">admin_panel_settings</span></a>' : ''}
|
||||
<button onclick="logout()" class="glass glass-hover p-2 rounded-xl transition-colors"><span class="msym text-base text-on-surface-variant">logout</span></button>
|
||||
</div>`;
|
||||
} catch(e){}
|
||||
}
|
||||
|
||||
function logout() {
|
||||
if(userToken) fetch('/auth/logout',{method:'POST',headers:{'Authorization':'Bearer '+userToken}});
|
||||
localStorage.removeItem('rag_token');
|
||||
window.location.reload();
|
||||
}
|
||||
|
||||
// ── Stats ─────────────────────────────────────────────────────────────────────
|
||||
async function loadStats() {
|
||||
try {
|
||||
const d = await (await fetch('/stats')).json();
|
||||
animateCount(document.getElementById('s-books'), d.books||0);
|
||||
animateCount(document.getElementById('s-vectors'), d.vectors||0);
|
||||
animateCount(document.getElementById('s-convos'), d.conversations||0);
|
||||
animateCount(document.getElementById('s-memory'), d.memories||0);
|
||||
} catch(e){}
|
||||
}
|
||||
|
||||
// ── Books ─────────────────────────────────────────────────────────────────────
|
||||
async function loadBooks() {
|
||||
try {
|
||||
const d = await (await fetch('/api/library?limit=6')).json();
|
||||
const list = document.getElementById('books-list');
|
||||
if (!d.books||!d.books.length) {
|
||||
list.innerHTML='<p class="text-on-surface-variant text-sm">لا توجد كتب حتى الآن</p>'; return;
|
||||
}
|
||||
list.innerHTML = d.books.map((b,i)=>{
|
||||
const name = b.title || b.filename.replace(/\.pdf$/i,'').replace(/^\d{4}-\d{2}-\d{2}\s*/,'');
|
||||
return `<div class="glass glass-hover rounded-2xl p-4 flex items-center gap-3 cursor-pointer book-card"
|
||||
style="animation:fadeUp .5s cubic-bezier(.22,1,.36,1) ${.85+i*.08}s both"
|
||||
onclick="window.location='/library'">
|
||||
<div class="w-9 h-9 bg-primary/10 rounded-xl flex items-center justify-center shrink-0" style="transition:transform .25s">
|
||||
<span class="msym text-primary">menu_book</span>
|
||||
</div>
|
||||
<div class="min-w-0">
|
||||
<p class="text-sm font-medium truncate">${name}</p>
|
||||
${b.author?`<p class="text-xs text-on-surface-variant">${b.author}</p>`:''}
|
||||
</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
} catch(e){}
|
||||
}
|
||||
|
||||
// ── Query ─────────────────────────────────────────────────────────────────────
|
||||
async function doQuery() {
|
||||
const q = document.getElementById('query-input').value.trim();
|
||||
if(!q) return;
|
||||
const btn = document.getElementById('query-btn');
|
||||
btn.disabled = true;
|
||||
btn.style.animation = 'none';
|
||||
btn.innerHTML = '<span class="msym text-lg" style="animation:spin .8s linear infinite">autorenew</span> جاري البحث...';
|
||||
const wrap = document.getElementById('answer-wrap');
|
||||
const box = document.getElementById('answer-box');
|
||||
const meta = document.getElementById('answer-meta');
|
||||
wrap.classList.remove('hidden');
|
||||
box.textContent = '...'; meta.innerHTML = '';
|
||||
try {
|
||||
const headers = {'Content-Type':'application/json','X-API-Key':API_KEY};
|
||||
if(userToken) headers['Authorization'] = 'Bearer '+userToken;
|
||||
const r = await fetch('/query',{method:'POST',headers,body:JSON.stringify({query:q,session_id:'web-'+(currentUser?.id||'guest')})});
|
||||
const d = await r.json();
|
||||
box.textContent = d.answer;
|
||||
meta.innerHTML = [
|
||||
`<span class="glass px-3 py-1 rounded-full text-xs text-primary font-bold">${d.llm}</span>`,
|
||||
`<span class="glass px-3 py-1 rounded-full text-xs text-secondary">${((d.confidence||0)*100).toFixed(0)}% ثقة</span>`,
|
||||
...(d.books_used||[]).map(b=>`<span class="glass px-3 py-1 rounded-full text-xs text-on-surface-variant">${b}</span>`)
|
||||
].join('');
|
||||
} catch(e){ box.textContent='خطأ: '+e.message; }
|
||||
btn.disabled = false;
|
||||
btn.style.animation = '';
|
||||
btn.innerHTML = '<span class="msym text-lg">search</span> بحث';
|
||||
}
|
||||
|
||||
// ── Upload ────────────────────────────────────────────────────────────────────
|
||||
async function handleFiles(files){for(const f of files)await uploadFile(f);loadStats();loadBooks();}
|
||||
|
||||
async function uploadFile(file){
|
||||
const prog=document.getElementById('upload-progress'),fill=document.getElementById('progress-fill');
|
||||
const txt=document.getElementById('progress-text'),log=document.getElementById('log-box');
|
||||
prog.classList.remove('hidden');log.classList.remove('hidden');
|
||||
txt.textContent='رفع: '+file.name;fill.style.width='30%';
|
||||
const fd=new FormData();fd.append('file',file);
|
||||
try{
|
||||
fill.style.width='60%';
|
||||
const headers={'X-API-Key':API_KEY};
|
||||
if(userToken) headers['Authorization']='Bearer '+userToken;
|
||||
const r=await fetch('/ingest',{method:'POST',headers,body:fd});
|
||||
fill.style.width='100%';
|
||||
const d=await r.json();
|
||||
txt.textContent='✅ تم: '+file.name;log.textContent=d.log||'Done';
|
||||
}catch(e){txt.textContent='❌ خطأ: '+e.message;}
|
||||
}
|
||||
|
||||
async function ingestURL(){
|
||||
const url=document.getElementById('url-input').value.trim();if(!url)return;
|
||||
const prog=document.getElementById('upload-progress'),fill=document.getElementById('progress-fill');
|
||||
const txt=document.getElementById('progress-text'),log=document.getElementById('log-box');
|
||||
prog.classList.remove('hidden');log.classList.remove('hidden');
|
||||
txt.textContent='جلب الملف...';fill.style.width='40%';
|
||||
try{
|
||||
fill.style.width='70%';
|
||||
const headers={'Content-Type':'application/json','X-API-Key':API_KEY};
|
||||
if(userToken) headers['Authorization']='Bearer '+userToken;
|
||||
const r=await fetch('/ingest-url',{method:'POST',headers,body:JSON.stringify({url})});
|
||||
fill.style.width='100%';
|
||||
const d=await r.json();
|
||||
txt.textContent='✅ تم: '+d.file;log.textContent=d.log||'Done';
|
||||
document.getElementById('url-input').value='';
|
||||
loadStats();loadBooks();
|
||||
}catch(e){txt.textContent='❌ خطأ: '+e.message;}
|
||||
}
|
||||
|
||||
// ── Upload zone drag ─────────────────────────────────────────────────────────
|
||||
const zone = document.getElementById('drop-zone');
|
||||
const cloudIcon = zone.querySelector('.w-14');
|
||||
zone.addEventListener('dragover',e=>{e.preventDefault();zone.classList.add('dragover');cloudIcon.style.transform='scale(1.15)';});
|
||||
zone.addEventListener('dragleave',()=>{zone.classList.remove('dragover');cloudIcon.style.transform='';});
|
||||
zone.addEventListener('drop',e=>{e.preventDefault();zone.classList.remove('dragover');cloudIcon.style.transform='';handleFiles(e.dataTransfer.files);});
|
||||
|
||||
// ── Keyboard shortcut ────────────────────────────────────────────────────────
|
||||
document.getElementById('query-input').addEventListener('keydown',e=>{if(e.ctrlKey&&e.key==='Enter')doQuery();});
|
||||
|
||||
// ── URL pre-fill ─────────────────────────────────────────────────────────────
|
||||
const urlQ = new URLSearchParams(location.search).get('q');
|
||||
if(urlQ){document.getElementById('query-input').value=decodeURIComponent(urlQ);setTimeout(doQuery,600);}
|
||||
|
||||
// ── Spin keyframe for loading ────────────────────────────────────────────────
|
||||
const spinStyle = document.createElement('style');
|
||||
spinStyle.textContent = '@keyframes spin{to{transform:rotate(360deg)}}';
|
||||
document.head.appendChild(spinStyle);
|
||||
|
||||
// ── Init ──────────────────────────────────────────────────────────────────────
|
||||
initAuth();
|
||||
loadStats();
|
||||
loadBooks();
|
||||
setInterval(loadStats, 30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+1249
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,501 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="dark" lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<title>AfakZaid RAG — Sign In</title>
|
||||
<script src="https://cdn.tailwindcss.com?plugins=forms"></script>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700;800&display=swap" rel="stylesheet"/>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:wght,FILL@100..700,0..1&display=swap" rel="stylesheet"/>
|
||||
<script>tailwind.config={darkMode:"class",theme:{extend:{colors:{"primary":"#4edea3","surface":"#0c1324","surface-low":"#151b2d","surface-container":"#191f31","surface-high":"#23293c","on-surface":"#dce1fb","on-surface-variant":"#bbcabf","outline-variant":"#3c4a42","secondary":"#95d3ba","error":"#f87171"},fontFamily:{sans:["Manrope","sans-serif"]}}}}</script>
|
||||
<style>
|
||||
*{box-sizing:border-box}
|
||||
body{background:radial-gradient(circle at 50% -20%,#151b2d 0%,#0c1324 100%);font-family:'Manrope',sans-serif;color:#dce1fb;min-height:100vh;display:flex;align-items:center;justify-content:center;padding:20px}
|
||||
.glass{background:rgba(21,27,45,0.6);backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border:1px solid rgba(78,222,163,0.15)}
|
||||
.glow{box-shadow:0 0 30px rgba(78,222,163,0.15)}
|
||||
.msym{font-family:'Material Symbols Outlined';font-variation-settings:'FILL' 0,'wght' 400,'GRAD' 0,'opsz' 24;vertical-align:middle;line-height:1}
|
||||
:root { --ease-out: cubic-bezier(0.23,1,0.32,1); }
|
||||
|
||||
.inp{width:100%;background:#23293c;border:1px solid #3c4a42;border-radius:12px;padding:12px 16px;color:#dce1fb;font-size:14px;outline:none;transition:border-color 180ms ease, box-shadow 180ms ease;font-family:'Manrope',sans-serif}
|
||||
.inp:focus{border-color:#4edea3;box-shadow:0 0 0 2px rgba(78,222,163,0.1)}
|
||||
.inp.inp-error{border-color:#f87171;box-shadow:0 0 0 2px rgba(248,113,113,0.1)}
|
||||
.inp-icon{position:relative}
|
||||
.inp-icon .msym{position:absolute;left:12px;top:50%;transform:translateY(-50%);color:#bbcabf;font-size:18px;pointer-events:none}
|
||||
.inp-icon input{padding-left:40px}
|
||||
|
||||
/* Primary button — specific transitions, no transition:all */
|
||||
.btn-primary{width:100%;background:#4edea3;color:#003824;font-weight:700;border:none;border-radius:12px;padding:14px;cursor:pointer;font-size:15px;font-family:'Manrope',sans-serif;position:relative;overflow:hidden;transition:box-shadow 200ms ease, transform 150ms var(--ease-out)}
|
||||
.btn-primary:disabled{opacity:0.5;cursor:not-allowed}
|
||||
.btn-primary:active:not(:disabled){transform:scale(0.97);transition-duration:100ms}
|
||||
@media (hover:hover) and (pointer:fine){.btn-primary:hover:not(:disabled){box-shadow:0 0 24px rgba(78,222,163,0.4)}}
|
||||
|
||||
/* Shimmer on btn hover */
|
||||
.btn-primary::after{content:'';position:absolute;inset:0;background:linear-gradient(90deg,transparent 0%,rgba(255,255,255,0.12) 50%,transparent 100%);transform:translateX(-100%);transition:transform 380ms cubic-bezier(0.77,0,0.175,1);pointer-events:none}
|
||||
@media (hover:hover) and (pointer:fine){.btn-primary:hover::after{transform:translateX(100%)}}
|
||||
|
||||
/* Tabs — specific transitions */
|
||||
.tab{padding:10px 24px;border-radius:10px;font-size:14px;font-weight:600;cursor:pointer;border:none;background:transparent;color:#bbcabf;font-family:'Manrope',sans-serif;transition:background 150ms var(--ease-out),color 150ms var(--ease-out)}
|
||||
.tab.active{background:rgba(78,222,163,0.1);color:#4edea3}
|
||||
.tab:active{transform:scale(0.97)}
|
||||
|
||||
/* Field validation */
|
||||
.field-error{font-size:12px;color:#f87171;margin-top:4px;display:none}
|
||||
.field-error.visible{display:block}
|
||||
|
||||
/* Country dropdown */
|
||||
#country-dropdown{position:absolute;top:calc(100% + 4px);left:0;right:0;background:#191f31;border:1px solid rgba(78,222,163,0.2);border-radius:12px;z-index:50;max-height:200px;overflow-y:auto;box-shadow:0 8px 32px rgba(0,0,0,0.4)}
|
||||
#country-dropdown::-webkit-scrollbar{width:4px}
|
||||
#country-dropdown::-webkit-scrollbar-thumb{background:rgba(78,222,163,0.2);border-radius:4px}
|
||||
.country-item{display:flex;align-items:center;gap:10px;padding:10px 14px;cursor:pointer;font-size:13px;transition:background 120ms ease}
|
||||
@media (hover:hover) and (pointer:fine){.country-item:hover{background:rgba(78,222,163,0.08)}}
|
||||
|
||||
/* Form entry */
|
||||
@keyframes fadeUp{from{opacity:0;transform:translateY(8px)}to{opacity:1;transform:translateY(0)}}
|
||||
.auth-card{animation:fadeUp 400ms var(--ease-out) both}
|
||||
|
||||
/* Reduced motion */
|
||||
@media (prefers-reduced-motion:reduce){*,*::before,*::after{animation-duration:.01ms !important;transition-duration:.01ms !important}}
|
||||
|
||||
|
||||
/* ── Mobile-visible improvements ─────────────────────────────────────────── */
|
||||
|
||||
/* Larger touch targets — 44px minimum per Apple HIG */
|
||||
button, a, .filter-chip, .book-card, .stat-card, .nav-btn {
|
||||
-webkit-tap-highlight-color: transparent;
|
||||
}
|
||||
|
||||
/* Tap feedback — visible on ALL devices including mobile */
|
||||
.book-card:active,
|
||||
.stat-card:active,
|
||||
.filter-chip:active {
|
||||
transform: scale(0.97) !important;
|
||||
transition: transform 100ms ease !important;
|
||||
}
|
||||
|
||||
/* Buttons get a visible press state on tap */
|
||||
button:active:not(:disabled) {
|
||||
filter: brightness(0.9);
|
||||
transform: scale(0.97);
|
||||
transition: transform 100ms ease, filter 100ms ease !important;
|
||||
}
|
||||
|
||||
/* Restore .glow — ensure it stacks correctly */
|
||||
.glow {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
0 0 20px rgba(78,222,163,0.15) !important;
|
||||
}
|
||||
.glow-sm {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.04),
|
||||
0 0 10px rgba(78,222,163,0.1) !important;
|
||||
}
|
||||
|
||||
/* Mobile: book cards get a more visible pressed state */
|
||||
@media (max-width: 768px) {
|
||||
.book-card {
|
||||
transition: transform 150ms ease, opacity 150ms ease !important;
|
||||
}
|
||||
.book-card:active {
|
||||
transform: scale(0.97) !important;
|
||||
opacity: 0.85 !important;
|
||||
}
|
||||
|
||||
/* Ensure skeleton is visible on mobile */
|
||||
.skeleton {
|
||||
background: linear-gradient(90deg, #1a2035 25%, #2a3348 50%, #1a2035 75%) !important;
|
||||
background-size: 800px 100% !important;
|
||||
animation: shimmer 1.2s ease-in-out infinite !important;
|
||||
}
|
||||
|
||||
/* Better spacing on small screens */
|
||||
.filter-chip {
|
||||
padding: 8px 16px !important;
|
||||
font-size: 13px !important;
|
||||
}
|
||||
}
|
||||
/* ── design-taste-frontend: Liquid Glass + Spotlight ─────────────────────── */
|
||||
|
||||
/* True liquid glass — inner refraction border */
|
||||
/* NOTE: does NOT define outer shadow — .glow still wins */
|
||||
.glass {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
inset 0 -1px 0 rgba(0,0,0,0.2);
|
||||
}
|
||||
/* Combined: liquid glass + green glow */
|
||||
.glass.glow {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
inset 0 -1px 0 rgba(0,0,0,0.2),
|
||||
0 0 20px rgba(78,222,163,0.18),
|
||||
0 4px 24px rgba(0,0,0,0.15);
|
||||
}
|
||||
.glass.glow-sm {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
inset 0 -1px 0 rgba(0,0,0,0.2),
|
||||
0 0 10px rgba(78,222,163,0.12);
|
||||
}
|
||||
/* .glow overrides naturally on top — no !important needed */
|
||||
|
||||
/* Spotlight border card — JS-driven cursor tracking */
|
||||
.spotlight-card {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
.spotlight-card::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: radial-gradient(
|
||||
400px circle at var(--mx, 50%) var(--my, 50%),
|
||||
rgba(78,222,163,0.06),
|
||||
transparent 60%
|
||||
);
|
||||
opacity: 0;
|
||||
transition: opacity 300ms ease;
|
||||
pointer-events: none;
|
||||
border-radius: inherit;
|
||||
z-index: 0;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.spotlight-card:hover::before { opacity: 1; }
|
||||
}
|
||||
|
||||
/* Ensure content above spotlight overlay */
|
||||
.spotlight-card > * { position: relative; z-index: 1; }
|
||||
|
||||
/* Tactile button press — design-taste rule */
|
||||
button:not(:disabled):active,
|
||||
a[role=button]:active {
|
||||
transform: translateY(-1px) scale(0.98) !important;
|
||||
transition-duration: 80ms !important;
|
||||
}
|
||||
|
||||
/* Hardware acceleration on animated elements */
|
||||
.stat-card, .book-card, .query-card, .books-card, .upload-card-wrap {
|
||||
transform: translateZ(0);
|
||||
backface-visibility: hidden;
|
||||
}
|
||||
|
||||
/* Directional hover-aware CTA — shimmer from left on hover */
|
||||
.cta-primary {
|
||||
position: relative; overflow: hidden;
|
||||
}
|
||||
.cta-primary::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0; left: -100%; bottom: 0; width: 50%;
|
||||
background: linear-gradient(90deg, transparent, rgba(255,255,255,0.1), transparent);
|
||||
transition: left 400ms cubic-bezier(0.77,0,0.175,1);
|
||||
pointer-events: none;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.cta-primary:hover::before { left: 200%; }
|
||||
}
|
||||
|
||||
/* ── Mobile improvements (actually visible on mobile) ────────────────────── */
|
||||
|
||||
/* Larger touch targets — 44px minimum per Apple HIG */
|
||||
@media (max-width: 768px) {
|
||||
/* Bigger tap areas on buttons */
|
||||
button, a { min-height: 44px; }
|
||||
.filter-chip { padding: 8px 16px; font-size: 13px; }
|
||||
|
||||
/* Book cards: more breathing room on mobile */
|
||||
.book-card { padding: 14px !important; }
|
||||
|
||||
/* Stats cards: larger numbers on mobile */
|
||||
.stat-card .stat-num, #s-books, #s-vectors, #s-convos, #s-memory {
|
||||
font-size: 1.75rem;
|
||||
}
|
||||
|
||||
/* Header: tighter on mobile */
|
||||
header { padding-top: 10px; padding-bottom: 10px; }
|
||||
|
||||
/* Search input: larger font so iOS doesn't zoom */
|
||||
#search-input, .inp, textarea, input[type="text"], input[type="email"], input[type="password"] {
|
||||
font-size: 16px !important; /* prevents iOS zoom on focus */
|
||||
}
|
||||
|
||||
/* Bottom action bar spacing */
|
||||
main { padding-bottom: env(safe-area-inset-bottom, 16px); }
|
||||
}
|
||||
|
||||
/* Reader: full height on mobile */
|
||||
@media (max-width: 768px) {
|
||||
#pdf-frame { position: absolute; inset: 0; }
|
||||
.reader-header { padding: 8px 12px; }
|
||||
}
|
||||
</style>
|
||||
<script src="/static/analytics.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="w-full max-w-md">
|
||||
|
||||
<!-- Logo -->
|
||||
<div class="text-center mb-8">
|
||||
<div class="w-16 h-16 bg-primary/10 rounded-2xl flex items-center justify-center mx-auto mb-4 glow">
|
||||
<span class="msym text-primary text-3xl">auto_awesome</span>
|
||||
</div>
|
||||
<h1 id="brand-name" class="text-2xl font-bold text-primary">AfakZaid RAG</h1>
|
||||
<p id="brand-sub" class="text-on-surface-variant text-sm mt-1">Scholarly Intelligence</p>
|
||||
</div>
|
||||
|
||||
<!-- Card -->
|
||||
<div class="glass glow rounded-3xl p-8 auth-card">
|
||||
|
||||
<!-- Tabs -->
|
||||
<div class="flex gap-2 bg-surface-container rounded-xl p-1 mb-8">
|
||||
<button class="tab active flex-1" id="tab-login" onclick="switchTab('login')">Sign In</button>
|
||||
<button class="tab flex-1" id="tab-register" onclick="switchTab('register')">Create Account</button>
|
||||
</div>
|
||||
|
||||
<!-- LOGIN -->
|
||||
<form id="form-login" onsubmit="doLogin(event)">
|
||||
<div class="space-y-4 mb-6">
|
||||
<div class="inp-icon">
|
||||
<span class="msym">mail</span>
|
||||
<input class="inp" type="email" id="login-email" placeholder="Email address" required/>
|
||||
</div>
|
||||
<div class="inp-icon">
|
||||
<span class="msym">lock</span>
|
||||
<input class="inp" type="password" id="login-pass" placeholder="Password" required/>
|
||||
</div>
|
||||
</div>
|
||||
<p id="login-err" class="text-error text-xs mb-4 hidden"></p>
|
||||
<button class="btn-primary" type="submit" id="login-btn">
|
||||
<span class="msym text-lg align-middle mr-1">login</span> Sign In
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<!-- REGISTER -->
|
||||
<form id="form-register" class="hidden" onsubmit="doRegister(event)">
|
||||
<div class="space-y-4 mb-6">
|
||||
<div class="inp-icon">
|
||||
<span class="msym">person</span>
|
||||
<input class="inp" type="text" id="reg-name" placeholder="Full name" required/>
|
||||
</div>
|
||||
<div class="inp-icon">
|
||||
<span class="msym">mail</span>
|
||||
<input class="inp" type="email" id="reg-email" placeholder="Email address" required/>
|
||||
</div>
|
||||
<div class="inp-icon">
|
||||
<span class="msym">lock</span>
|
||||
<input class="inp" type="password" id="reg-pass" placeholder="Password (min 6 chars)" required minlength="6"/>
|
||||
</div>
|
||||
|
||||
<!-- Phone + Country -->
|
||||
<div class="grid grid-cols-2 gap-3">
|
||||
<!-- Country picker -->
|
||||
<div class="relative" id="country-wrap">
|
||||
<div class="inp flex items-center gap-2 cursor-pointer select-none" onclick="toggleCountry()" id="country-display">
|
||||
<span class="msym text-on-surface-variant text-base">public</span>
|
||||
<span id="country-flag" class="text-lg hidden"></span>
|
||||
<span id="country-label" class="text-on-surface-variant text-sm truncate flex-1">Country</span>
|
||||
<span class="msym text-on-surface-variant text-base">expand_more</span>
|
||||
</div>
|
||||
<div id="country-dropdown" class="hidden">
|
||||
<div class="p-2">
|
||||
<input id="country-search" class="inp text-xs py-2" placeholder="Search country..." oninput="filterCountries(this.value)"/>
|
||||
</div>
|
||||
<div id="country-list"></div>
|
||||
</div>
|
||||
<input type="hidden" id="reg-country-code"/>
|
||||
<input type="hidden" id="reg-country-name"/>
|
||||
</div>
|
||||
<!-- Phone -->
|
||||
<div class="inp-icon">
|
||||
<span class="msym">phone</span>
|
||||
<input class="inp" type="tel" id="reg-phone" placeholder="Phone"/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- GPS button -->
|
||||
<button type="button" onclick="detectLocation()" id="gps-btn"
|
||||
class="w-full flex items-center justify-center gap-2 glass border border-primary/20 text-on-surface-variant hover:text-primary hover:border-primary/50 rounded-xl py-2.5 text-sm transition-all">
|
||||
<span class="msym text-base">my_location</span>
|
||||
<span id="gps-text">Detect my location</span>
|
||||
</button>
|
||||
|
||||
</div>
|
||||
<p id="reg-err" class="text-error text-xs mb-4 hidden"></p>
|
||||
<button class="btn-primary" type="submit" id="reg-btn">
|
||||
<span class="msym text-lg align-middle mr-1">person_add</span> Create Account
|
||||
</button>
|
||||
</form>
|
||||
|
||||
</div>
|
||||
|
||||
<p class="text-center text-on-surface-variant text-xs mt-6">
|
||||
<a href="/" class="hover:text-primary transition-colors">← Back to library</a>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- Background -->
|
||||
<div class="fixed top-0 right-0 -z-10 w-96 h-96 bg-primary/5 rounded-full blur-[120px] pointer-events-none"></div>
|
||||
<div class="fixed bottom-0 left-0 -z-10 w-72 h-72 bg-secondary/5 rounded-full blur-[100px] pointer-events-none"></div>
|
||||
|
||||
<script>
|
||||
// ── Branding ───────────────────────────────────────────────────────────────
|
||||
fetch('/branding').then(r=>r.json()).then(b=>{
|
||||
if(b.site_name){document.title=b.site_name+' — Sign In';document.getElementById('brand-name').textContent=b.site_name;}
|
||||
if(b.site_subtitle) document.getElementById('brand-sub').textContent=b.site_subtitle;
|
||||
}).catch(()=>{});
|
||||
|
||||
// Redirect if already logged in
|
||||
const token = localStorage.getItem('rag_token');
|
||||
if(token) {
|
||||
fetch('/auth/me',{headers:{'Authorization':token}}).then(r=>{
|
||||
if(r.ok) window.location.href='/';
|
||||
}).catch(()=>{});
|
||||
}
|
||||
|
||||
// ── Tabs ───────────────────────────────────────────────────────────────────
|
||||
function switchTab(tab) {
|
||||
document.getElementById('form-login').classList.toggle('hidden', tab!=='login');
|
||||
document.getElementById('form-register').classList.toggle('hidden', tab!=='register');
|
||||
document.getElementById('tab-login').classList.toggle('active', tab==='login');
|
||||
document.getElementById('tab-register').classList.toggle('active', tab==='register');
|
||||
}
|
||||
|
||||
// ── Auth ───────────────────────────────────────────────────────────────────
|
||||
async function doLogin(e) {
|
||||
e.preventDefault();
|
||||
const btn=document.getElementById('login-btn');
|
||||
if(btn.disabled) return;
|
||||
const err=document.getElementById('login-err');
|
||||
btn.disabled=true; btn.textContent='Signing in...'; err.classList.add('hidden');
|
||||
try {
|
||||
const r = await fetch('/auth/login',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({email:document.getElementById('login-email').value,password:document.getElementById('login-pass').value})});
|
||||
const d = await r.json();
|
||||
if(!r.ok){err.textContent=d.detail||'Login failed';err.classList.remove('hidden');return;}
|
||||
localStorage.setItem('rag_token',d.token);
|
||||
localStorage.setItem('rag_user',JSON.stringify(d.user));
|
||||
window.location.href = d.user.permissions?.admin ? '/admin' : '/';
|
||||
} catch(ex){err.textContent='Connection error';err.classList.remove('hidden');}
|
||||
finally{btn.disabled=false;btn.innerHTML='<span class="msym text-lg align-middle mr-1">login</span> Sign In';}
|
||||
}
|
||||
|
||||
async function doRegister(e) {
|
||||
e.preventDefault();
|
||||
const btn=document.getElementById('reg-btn');
|
||||
if(btn.disabled) return;
|
||||
const err=document.getElementById('reg-err');
|
||||
btn.disabled=true;btn.textContent='Creating account...';err.classList.add('hidden');
|
||||
try {
|
||||
const r = await fetch('/auth/register',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({
|
||||
name:document.getElementById('reg-name').value,
|
||||
email:document.getElementById('reg-email').value,
|
||||
password:document.getElementById('reg-pass').value,
|
||||
phone:document.getElementById('reg-phone').value,
|
||||
country_code:document.getElementById('reg-country-code').value,
|
||||
country_name:document.getElementById('reg-country-name').value,
|
||||
})});
|
||||
const d = await r.json();
|
||||
if(!r.ok){err.textContent=d.detail||'Registration failed';err.classList.remove('hidden');return;}
|
||||
localStorage.setItem('rag_token',d.token);
|
||||
window.location.href='/';
|
||||
} catch(ex){err.textContent='Connection error';err.classList.remove('hidden');}
|
||||
finally{btn.disabled=false;btn.innerHTML='<span class="msym text-lg align-middle mr-1">person_add</span> Create Account';}
|
||||
}
|
||||
|
||||
// ── Country Picker ─────────────────────────────────────────────────────────
|
||||
let allCountries = [];
|
||||
let selectedCountry = null;
|
||||
|
||||
async function loadCountries() {
|
||||
try {
|
||||
const r = await fetch('https://restcountries.com/v3.1/all?fields=name,cca2,idd,flag');
|
||||
const data = await r.json();
|
||||
allCountries = data.sort((a,b)=>a.name.common.localeCompare(b.name.common));
|
||||
renderCountries(allCountries);
|
||||
} catch(e) {
|
||||
// fallback minimal list
|
||||
allCountries = [
|
||||
{name:{common:'Saudi Arabia'},cca2:'SA',flag:'🇸🇦'},
|
||||
{name:{common:'UAE'},cca2:'AE',flag:'🇦🇪'},
|
||||
{name:{common:'Kuwait'},cca2:'KW',flag:'🇰🇼'},
|
||||
{name:{common:'Qatar'},cca2:'QA',flag:'🇶🇦'},
|
||||
{name:{common:'Bahrain'},cca2:'BH',flag:'🇧🇭'},
|
||||
{name:{common:'Egypt'},cca2:'EG',flag:'🇪🇬'},
|
||||
{name:{common:'Jordan'},cca2:'JO',flag:'🇯🇴'},
|
||||
{name:{common:'United States'},cca2:'US',flag:'🇺🇸'},
|
||||
{name:{common:'United Kingdom'},cca2:'GB',flag:'🇬🇧'},
|
||||
];
|
||||
renderCountries(allCountries);
|
||||
}
|
||||
}
|
||||
|
||||
function renderCountries(list) {
|
||||
document.getElementById('country-list').innerHTML = list.slice(0,100).map(c=>`
|
||||
<div class="country-item" onclick="selectCountry('${c.cca2}','${c.name.common.replace(/'/g,"\\'")}','${c.flag||''}')">
|
||||
<span class="text-lg">${c.flag||'🌍'}</span>
|
||||
<span class="text-sm">${c.name.common}</span>
|
||||
<span class="text-xs text-on-surface-variant ml-auto">${c.cca2}</span>
|
||||
</div>`).join('');
|
||||
}
|
||||
|
||||
function filterCountries(q) {
|
||||
const filtered = allCountries.filter(c=>c.name.common.toLowerCase().includes(q.toLowerCase())||c.cca2.toLowerCase().includes(q.toLowerCase()));
|
||||
renderCountries(filtered);
|
||||
}
|
||||
|
||||
function selectCountry(code, name, flag) {
|
||||
document.getElementById('reg-country-code').value = code;
|
||||
document.getElementById('reg-country-name').value = name;
|
||||
document.getElementById('country-flag').textContent = flag;
|
||||
document.getElementById('country-flag').classList.remove('hidden');
|
||||
document.getElementById('country-label').textContent = name;
|
||||
document.getElementById('country-label').style.color = '#dce1fb';
|
||||
closeCountry();
|
||||
}
|
||||
|
||||
function toggleCountry() {
|
||||
const dd = document.getElementById('country-dropdown');
|
||||
dd.classList.toggle('hidden');
|
||||
if(!dd.classList.contains('hidden')) {
|
||||
document.getElementById('country-search').focus();
|
||||
loadCountries();
|
||||
}
|
||||
}
|
||||
|
||||
function closeCountry() {
|
||||
document.getElementById('country-dropdown').classList.add('hidden');
|
||||
}
|
||||
|
||||
document.addEventListener('click', e=>{
|
||||
if(!document.getElementById('country-wrap').contains(e.target)) closeCountry();
|
||||
});
|
||||
|
||||
// ── GPS Detection ──────────────────────────────────────────────────────────
|
||||
async function detectLocation() {
|
||||
const btn = document.getElementById('gps-btn');
|
||||
const txt = document.getElementById('gps-text');
|
||||
txt.textContent = 'Detecting...';
|
||||
btn.disabled = true;
|
||||
if (!navigator.geolocation) { txt.textContent='GPS not supported'; btn.disabled=false; return; }
|
||||
navigator.geolocation.getCurrentPosition(async pos => {
|
||||
try {
|
||||
const {latitude: lat, longitude: lon} = pos.coords;
|
||||
const r = await fetch(`https://nominatim.openstreetmap.org/reverse?format=json&lat=${lat}&lon=${lon}`);
|
||||
const d = await r.json();
|
||||
const code = d.address?.country_code?.toUpperCase()||'';
|
||||
const name = d.address?.country||'';
|
||||
// Find flag
|
||||
await loadCountries();
|
||||
const country = allCountries.find(c=>c.cca2===code);
|
||||
const flag = country?.flag||'🌍';
|
||||
selectCountry(code, name, flag);
|
||||
txt.textContent = '✓ ' + name;
|
||||
} catch(e){ txt.textContent='Could not detect'; }
|
||||
btn.disabled=false;
|
||||
}, ()=>{ txt.textContent='Location denied'; btn.disabled=false; });
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,434 @@
|
||||
<!DOCTYPE html>
|
||||
<html class="dark" lang="ar" dir="rtl">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<title>قارئ الكتب — AfakZaid</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;600;700&display=swap" rel="stylesheet"/>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:wght,FILL@400,0&display=swap" rel="stylesheet"/>
|
||||
<script>tailwind.config={darkMode:"class",theme:{extend:{colors:{"primary":"#4edea3","surface":"#0c1324","surface-high":"#23293c","on-surface":"#dce1fb","on-surface-variant":"#bbcabf","outline-variant":"#3c4a42"},fontFamily:{sans:["Manrope","sans-serif"]}}}}</script>
|
||||
<style>
|
||||
:root { --ease-out: cubic-bezier(0.23,1,0.32,1); }
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
html,body{height:100%;overflow:hidden}
|
||||
body{background:#0c1324;font-family:'Manrope',sans-serif;color:#dce1fb;display:flex;flex-direction:column}
|
||||
.glass{background:rgba(21,27,45,0.6);backdrop-filter:blur(20px);-webkit-backdrop-filter:blur(20px);border:1px solid rgba(78,222,163,0.12);box-shadow:inset 0 1px 0 rgba(255,255,255,0.04)}
|
||||
.msym{font-family:'Material Symbols Outlined';font-variation-settings:'FILL' 0,'wght' 400,'GRAD' 0,'opsz' 24;vertical-align:middle;line-height:1}
|
||||
|
||||
/* Header buttons — specific transitions only */
|
||||
.hdr-btn {
|
||||
transition: background 150ms var(--ease-out), border-color 150ms var(--ease-out), color 150ms var(--ease-out);
|
||||
border: 1px solid transparent;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.hdr-btn:hover { background: rgba(78,222,163,0.08); border-color: rgba(78,222,163,0.2); }
|
||||
}
|
||||
.hdr-btn:active { transform: scale(0.95); transition-duration: 100ms; }
|
||||
|
||||
/* AI btn glow */
|
||||
.ai-btn {
|
||||
background: rgba(78,222,163,0.12);
|
||||
border: 1px solid rgba(78,222,163,0.25);
|
||||
transition: background 150ms var(--ease-out), box-shadow 200ms ease;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.ai-btn:hover { background: rgba(78,222,163,0.2); box-shadow: 0 0 20px rgba(78,222,163,0.2); }
|
||||
}
|
||||
.ai-btn:active { transform: scale(0.96); transition-duration: 100ms; }
|
||||
|
||||
/* Fav btn — love animation */
|
||||
@keyframes heartBeat {
|
||||
0% { transform: scale(1); }
|
||||
30% { transform: scale(1.4); }
|
||||
60% { transform: scale(0.9); }
|
||||
100%{ transform: scale(1); }
|
||||
}
|
||||
#fav-btn { transition: color 200ms ease, transform 200ms var(--ease-out); }
|
||||
#fav-btn.faved { animation: heartBeat 400ms var(--ease-out); color: #f87171; }
|
||||
#fav-btn:active { transform: scale(0.9); }
|
||||
|
||||
/* PDF frame */
|
||||
#pdf-frame { flex:1; width:100%; height:100%; border:none; background:#1a1a2e; display:block; }
|
||||
|
||||
/* Loading state */
|
||||
#loading {
|
||||
position:absolute; inset:0;
|
||||
display:flex; align-items:center; justify-content:center; flex-direction:column;
|
||||
background:#0c1324; gap:16px; z-index:5;
|
||||
animation: fadeIn 200ms ease both;
|
||||
}
|
||||
@keyframes fadeIn { from{opacity:0} to{opacity:1} }
|
||||
|
||||
/* Loading spinner */
|
||||
@keyframes spin { to { transform: rotate(360deg); } }
|
||||
.loading-ring {
|
||||
width: 40px; height: 40px;
|
||||
border: 2px solid rgba(78,222,163,0.15);
|
||||
border-top-color: #4edea3;
|
||||
border-radius: 50%;
|
||||
animation: spin 800ms linear infinite;
|
||||
}
|
||||
|
||||
/* AI panel slide */
|
||||
#ai-panel {
|
||||
transition: max-height 300ms var(--ease-out), opacity 250ms ease;
|
||||
overflow: hidden;
|
||||
}
|
||||
#ai-panel.collapsed { max-height: 0; opacity: 0; }
|
||||
#ai-panel.open { max-height: 400px; opacity: 1; }
|
||||
|
||||
/* Ask input */
|
||||
#ask-input {
|
||||
transition: border-color 180ms ease, box-shadow 180ms ease;
|
||||
}
|
||||
#ask-input:focus { border-color: #4edea3; box-shadow: 0 0 0 2px rgba(78,222,163,0.1); outline: none; }
|
||||
|
||||
/* prefers-reduced-motion */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*,*::before,*::after { animation-duration:.01ms !important; transition-duration:.01ms !important; }
|
||||
}
|
||||
|
||||
|
||||
/* ── Mobile-visible improvements ─────────────────────────────────────────── */
|
||||
|
||||
/* Larger touch targets — 44px minimum per Apple HIG */
|
||||
button, a, .filter-chip, .book-card, .stat-card, .nav-btn {
|
||||
-webkit-tap-highlight-color: transparent;
|
||||
}
|
||||
|
||||
/* Tap feedback — visible on ALL devices including mobile */
|
||||
.book-card:active,
|
||||
.stat-card:active,
|
||||
.filter-chip:active {
|
||||
transform: scale(0.97) !important;
|
||||
transition: transform 100ms ease !important;
|
||||
}
|
||||
|
||||
/* Buttons get a visible press state on tap */
|
||||
button:active:not(:disabled) {
|
||||
filter: brightness(0.9);
|
||||
transform: scale(0.97);
|
||||
transition: transform 100ms ease, filter 100ms ease !important;
|
||||
}
|
||||
|
||||
/* Restore .glow — ensure it stacks correctly */
|
||||
.glow {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
0 0 20px rgba(78,222,163,0.15) !important;
|
||||
}
|
||||
.glow-sm {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.04),
|
||||
0 0 10px rgba(78,222,163,0.1) !important;
|
||||
}
|
||||
|
||||
/* Mobile: book cards get a more visible pressed state */
|
||||
@media (max-width: 768px) {
|
||||
.book-card {
|
||||
transition: transform 150ms ease, opacity 150ms ease !important;
|
||||
}
|
||||
.book-card:active {
|
||||
transform: scale(0.97) !important;
|
||||
opacity: 0.85 !important;
|
||||
}
|
||||
|
||||
/* Ensure skeleton is visible on mobile */
|
||||
.skeleton {
|
||||
background: linear-gradient(90deg, #1a2035 25%, #2a3348 50%, #1a2035 75%) !important;
|
||||
background-size: 800px 100% !important;
|
||||
animation: shimmer 1.2s ease-in-out infinite !important;
|
||||
}
|
||||
|
||||
/* Better spacing on small screens */
|
||||
.filter-chip {
|
||||
padding: 8px 16px !important;
|
||||
font-size: 13px !important;
|
||||
}
|
||||
}
|
||||
/* ── design-taste-frontend: Liquid Glass + Spotlight ─────────────────────── */
|
||||
|
||||
/* True liquid glass — inner refraction border */
|
||||
/* NOTE: does NOT define outer shadow — .glow still wins */
|
||||
.glass {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
inset 0 -1px 0 rgba(0,0,0,0.2);
|
||||
}
|
||||
/* Combined: liquid glass + green glow */
|
||||
.glass.glow {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
inset 0 -1px 0 rgba(0,0,0,0.2),
|
||||
0 0 20px rgba(78,222,163,0.18),
|
||||
0 4px 24px rgba(0,0,0,0.15);
|
||||
}
|
||||
.glass.glow-sm {
|
||||
box-shadow:
|
||||
inset 0 1px 0 rgba(255,255,255,0.06),
|
||||
inset 0 -1px 0 rgba(0,0,0,0.2),
|
||||
0 0 10px rgba(78,222,163,0.12);
|
||||
}
|
||||
/* .glow overrides naturally on top — no !important needed */
|
||||
|
||||
/* Spotlight border card — JS-driven cursor tracking */
|
||||
.spotlight-card {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
.spotlight-card::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: radial-gradient(
|
||||
400px circle at var(--mx, 50%) var(--my, 50%),
|
||||
rgba(78,222,163,0.06),
|
||||
transparent 60%
|
||||
);
|
||||
opacity: 0;
|
||||
transition: opacity 300ms ease;
|
||||
pointer-events: none;
|
||||
border-radius: inherit;
|
||||
z-index: 0;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.spotlight-card:hover::before { opacity: 1; }
|
||||
}
|
||||
|
||||
/* Ensure content above spotlight overlay */
|
||||
.spotlight-card > * { position: relative; z-index: 1; }
|
||||
|
||||
/* Tactile button press — design-taste rule */
|
||||
button:not(:disabled):active,
|
||||
a[role=button]:active {
|
||||
transform: translateY(-1px) scale(0.98) !important;
|
||||
transition-duration: 80ms !important;
|
||||
}
|
||||
|
||||
/* Hardware acceleration on animated elements */
|
||||
.stat-card, .book-card, .query-card, .books-card, .upload-card-wrap {
|
||||
transform: translateZ(0);
|
||||
backface-visibility: hidden;
|
||||
}
|
||||
|
||||
/* Directional hover-aware CTA — shimmer from left on hover */
|
||||
.cta-primary {
|
||||
position: relative; overflow: hidden;
|
||||
}
|
||||
.cta-primary::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0; left: -100%; bottom: 0; width: 50%;
|
||||
background: linear-gradient(90deg, transparent, rgba(255,255,255,0.1), transparent);
|
||||
transition: left 400ms cubic-bezier(0.77,0,0.175,1);
|
||||
pointer-events: none;
|
||||
}
|
||||
@media (hover: hover) and (pointer: fine) {
|
||||
.cta-primary:hover::before { left: 200%; }
|
||||
}
|
||||
|
||||
/* ── Mobile improvements (actually visible on mobile) ────────────────────── */
|
||||
|
||||
/* Larger touch targets — 44px minimum per Apple HIG */
|
||||
@media (max-width: 768px) {
|
||||
/* Bigger tap areas on buttons */
|
||||
button, a { min-height: 44px; }
|
||||
.filter-chip { padding: 8px 16px; font-size: 13px; }
|
||||
|
||||
/* Book cards: more breathing room on mobile */
|
||||
.book-card { padding: 14px !important; }
|
||||
|
||||
/* Stats cards: larger numbers on mobile */
|
||||
.stat-card .stat-num, #s-books, #s-vectors, #s-convos, #s-memory {
|
||||
font-size: 1.75rem;
|
||||
}
|
||||
|
||||
/* Header: tighter on mobile */
|
||||
header { padding-top: 10px; padding-bottom: 10px; }
|
||||
|
||||
/* Search input: larger font so iOS doesn't zoom */
|
||||
#search-input, .inp, textarea, input[type="text"], input[type="email"], input[type="password"] {
|
||||
font-size: 16px !important; /* prevents iOS zoom on focus */
|
||||
}
|
||||
|
||||
/* Bottom action bar spacing */
|
||||
main { padding-bottom: env(safe-area-inset-bottom, 16px); }
|
||||
}
|
||||
|
||||
/* Reader: full height on mobile */
|
||||
@media (max-width: 768px) {
|
||||
#pdf-frame { position: absolute; inset: 0; }
|
||||
.reader-header { padding: 8px 12px; }
|
||||
}
|
||||
</style>
|
||||
<script src="/static/analytics.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<!-- Top bar -->
|
||||
<header class="glass border-b border-primary/20 px-4 py-3 flex items-center gap-3 shrink-0">
|
||||
<a href="/library" class="text-on-surface-variant hover:text-primary transition-colors">
|
||||
<span class="msym text-2xl">arrow_back</span>
|
||||
</a>
|
||||
<div class="flex-1 min-w-0">
|
||||
<h1 id="book-title" class="text-sm font-bold truncate text-primary">جاري التحميل...</h1>
|
||||
<p id="book-author" class="text-xs text-on-surface-variant truncate"></p>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<button id="fav-btn" onclick="toggleFavorite()" class="glass flex items-center gap-1.5 px-3 py-2 rounded-xl text-xs transition-all hover:border-primary/40">
|
||||
<span class="msym text-lg" id="fav-icon">favorite_border</span>
|
||||
<span id="fav-text" class="hidden sm:inline">إضافة</span>
|
||||
</button>
|
||||
<a id="download-btn" href="#" class="glass flex items-center gap-1.5 px-3 py-2 rounded-xl text-xs text-on-surface-variant hover:text-primary transition-all" download>
|
||||
<span class="msym text-lg">download</span>
|
||||
<span class="hidden sm:inline">تحميل</span>
|
||||
</a>
|
||||
<button onclick="askAboutBook()" class="bg-primary text-surface font-bold px-3 py-2 rounded-xl text-xs flex items-center gap-1.5 hover:shadow-[0_0_15px_rgba(78,222,163,0.4)] transition-all">
|
||||
<span class="msym text-lg">auto_awesome</span>
|
||||
<span class="hidden sm:inline">اسأل</span>
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- PDF Viewer -->
|
||||
<div class="flex-1 relative overflow-hidden" style="height:100%;min-height:0">
|
||||
<div id="loading">
|
||||
<div class="spinner mb-4"></div>
|
||||
<p class="text-on-surface-variant text-sm">جاري تحميل الكتاب...</p>
|
||||
</div>
|
||||
<iframe id="pdf-frame" title="PDF Reader"></iframe>
|
||||
</div>
|
||||
|
||||
<!-- Ask modal -->
|
||||
<div id="ask-modal" class="hidden fixed inset-0 z-50 flex items-end justify-center p-4 bg-black/60">
|
||||
<div class="glass rounded-3xl p-6 w-full max-w-lg">
|
||||
<div class="flex items-center justify-between mb-4">
|
||||
<h3 class="font-bold flex items-center gap-2"><span class="msym text-primary">auto_awesome</span>اسأل عن هذا الكتاب</h3>
|
||||
<button onclick="document.getElementById('ask-modal').classList.add('hidden')" class="text-on-surface-variant hover:text-primary"><span class="msym text-xl">close</span></button>
|
||||
</div>
|
||||
<textarea id="ask-input" rows="3" dir="rtl" placeholder="اكتب سؤالك..."
|
||||
class="w-full bg-surface-high border border-outline-variant rounded-xl px-4 py-3 text-sm text-on-surface outline-none focus:border-primary transition-colors resize-none mb-3"></textarea>
|
||||
<button onclick="doAsk()" class="w-full bg-primary text-surface font-bold py-3 rounded-xl text-sm hover:shadow-[0_0_20px_rgba(78,222,163,0.4)] transition-all">
|
||||
<span class="msym text-lg align-middle mr-1">search</span> بحث
|
||||
</button>
|
||||
<div id="ask-answer" class="hidden mt-4 bg-surface-high rounded-xl p-4 text-sm leading-7 max-h-48 overflow-y-auto text-on-surface"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const params = new URLSearchParams(location.search);
|
||||
const bookId = params.get('id');
|
||||
const bookTitle = decodeURIComponent(params.get('title') || '');
|
||||
const bookAuthor = decodeURIComponent(params.get('author') || '');
|
||||
const userToken = localStorage.getItem('rag_token');
|
||||
const API_KEY = 'zRf77E_brSEnA2BUi40uxLxT4uofAk661WMZb-cdv4M';
|
||||
let isFavorite = false;
|
||||
|
||||
// Set title
|
||||
document.getElementById('book-title').textContent = bookTitle || 'كتاب';
|
||||
document.getElementById('book-author').textContent = bookAuthor;
|
||||
document.title = (bookTitle || 'كتاب') + ' — AfakZaid';
|
||||
|
||||
// Load PDF — public, no auth required
|
||||
if (bookId) {
|
||||
const pdfUrl = `/api/library/${bookId}/pdf`;
|
||||
const titleEl = document.getElementById('book-title-loading');
|
||||
if (titleEl) titleEl.textContent = bookTitle || '';
|
||||
|
||||
// Reading progress — restore last page
|
||||
const savedPage = localStorage.getItem(`progress_${bookId}`);
|
||||
if (savedPage && parseInt(savedPage) > 1) {
|
||||
const resume = confirm(`استئناف القراءة من صفحة ${savedPage}؟`);
|
||||
if (resume) {
|
||||
window.history.replaceState({}, '', `?id=${bookId}&title=${encodeURIComponent(bookTitle||'')}&page=${savedPage}`);
|
||||
}
|
||||
}
|
||||
const downloadUrl = `/api/library/${bookId}/download`;
|
||||
const frame = document.getElementById('pdf-frame');
|
||||
const isMobile = /Android|iPhone|iPad|iPod/i.test(navigator.userAgent);
|
||||
|
||||
document.getElementById('download-btn').href = downloadUrl;
|
||||
document.getElementById('download-btn').removeAttribute('onclick');
|
||||
|
||||
function showPdfError() {
|
||||
document.getElementById('loading').style.display = 'flex';
|
||||
document.getElementById('loading').innerHTML = `
|
||||
<div class="text-center px-8">
|
||||
<span class="msym text-6xl text-on-surface-variant/30 block mb-4">picture_as_pdf</span>
|
||||
<p class="text-on-surface-variant font-semibold mb-2">PDF غير متاح حالياً</p>
|
||||
<p class="text-xs text-on-surface-variant/60 mb-6">يجب إعادة رفع الكتاب لتفعيل القراءة</p>
|
||||
<a href="${downloadUrl}" class="bg-primary/10 border border-primary/20 text-primary px-6 py-3 rounded-xl text-sm inline-block">
|
||||
تحميل الكتاب
|
||||
</a>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
if (isMobile) {
|
||||
// Mobile browsers can't render PDFs in iframes — use Google Docs viewer
|
||||
const fullUrl = window.location.origin + pdfUrl;
|
||||
frame.style.cssText = 'position:absolute;inset:0;width:100%;height:100%;border:none';
|
||||
frame.src = `https://docs.google.com/viewer?url=${encodeURIComponent(fullUrl)}&embedded=true`;
|
||||
document.getElementById('loading').style.display = 'none';
|
||||
} else {
|
||||
// Desktop: direct iframe src
|
||||
frame.style.cssText = 'position:absolute;inset:0;width:100%;height:100%;border:none';
|
||||
frame.src = pdfUrl;
|
||||
setTimeout(() => document.getElementById('loading').style.display = 'none', 1000);
|
||||
}
|
||||
|
||||
if (userToken) checkFavorite();
|
||||
}
|
||||
|
||||
async function checkFavorite() {
|
||||
try {
|
||||
const r = await fetch(`/api/favorites/check/${bookId}`, {headers:{'Authorization':'Bearer '+userToken}});
|
||||
const d = await r.json();
|
||||
isFavorite = d.is_favorite;
|
||||
updateFavBtn();
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
function updateFavBtn() {
|
||||
document.getElementById('fav-icon').textContent = isFavorite ? 'favorite' : 'favorite_border';
|
||||
document.getElementById('fav-icon').style.color = isFavorite ? '#f87171' : '';
|
||||
document.getElementById('fav-text').textContent = isFavorite ? 'مفضلة' : 'إضافة';
|
||||
}
|
||||
|
||||
async function toggleFavorite() {
|
||||
if (!userToken) { window.location.href='/login'; return; }
|
||||
try {
|
||||
const method = isFavorite ? 'DELETE' : 'POST';
|
||||
await fetch(`/api/favorites/${bookId}`, {method, headers:{'Authorization':'Bearer '+userToken}});
|
||||
isFavorite = !isFavorite;
|
||||
updateFavBtn();
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
function askAboutBook() {
|
||||
document.getElementById('ask-modal').classList.remove('hidden');
|
||||
document.getElementById('ask-input').focus();
|
||||
}
|
||||
|
||||
async function doAsk() {
|
||||
const q = document.getElementById('ask-input').value.trim();
|
||||
if (!q) return;
|
||||
const ansEl = document.getElementById('ask-answer');
|
||||
ansEl.classList.remove('hidden');
|
||||
ansEl.textContent = '...';
|
||||
try {
|
||||
const r = await fetch('/query', {
|
||||
method:'POST',
|
||||
headers:{'Content-Type':'application/json','X-API-Key':API_KEY},
|
||||
body: JSON.stringify({query: q + ' في كتاب: ' + bookTitle, session_id: 'reader-'+(bookId||'guest')})
|
||||
});
|
||||
const d = await r.json();
|
||||
ansEl.textContent = d.answer;
|
||||
} catch(e) { ansEl.textContent = 'خطأ: ' + e.message; }
|
||||
}
|
||||
|
||||
document.getElementById('ask-input').addEventListener('keydown', e => {
|
||||
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); doAsk(); }
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
storage.py — Unified file storage via MinIO (primary) with optional R2 replication
|
||||
Architecture:
|
||||
- Local disk: /mnt/rag-media/ (always kept as hot cache)
|
||||
- MinIO: primary object store on VPS (rag-media, rag-covers buckets)
|
||||
- Cloudflare R2: replication target (configured via environment variables)
|
||||
|
||||
All upload operations write to disk AND MinIO.
|
||||
R2 replication is handled by MinIO bucket replication (configured separately).
|
||||
For serving: use MinIO presigned URL or R2 public URL if CDN is enabled.
|
||||
"""
|
||||
import os, logging, boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
log = logging.getLogger("rag.storage")
|
||||
|
||||
# ── MinIO config ──────────────────────────────────────────────────────────────
|
||||
MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "http://minio:9000")
|
||||
# Use app credentials if set, fall back to root credentials
|
||||
MINIO_ACCESS = os.getenv("MINIO_ACCESS_KEY", "") or os.getenv("MINIO_ROOT_USER", "")
|
||||
MINIO_SECRET = os.getenv("MINIO_SECRET_KEY", "") or os.getenv("MINIO_ROOT_PASSWORD", "")
|
||||
BUCKET_MEDIA = "rag-media"
|
||||
BUCKET_COVERS = "rag-covers"
|
||||
|
||||
# ── Cloudflare R2 config (optional) ──────────────────────────────────────────
|
||||
R2_ENDPOINT = os.getenv("R2_ENDPOINT", "") # https://<account>.r2.cloudflarestorage.com
|
||||
R2_ACCESS_KEY = os.getenv("R2_ACCESS_KEY", "")
|
||||
R2_SECRET_KEY = os.getenv("R2_SECRET_KEY", "")
|
||||
R2_BUCKET = os.getenv("R2_BUCKET", "rag-media")
|
||||
R2_PUBLIC_URL = os.getenv("R2_PUBLIC_URL", "") # optional CDN URL
|
||||
|
||||
_minio_client = None
|
||||
_r2_client = None
|
||||
|
||||
|
||||
def _get_minio():
|
||||
global _minio_client
|
||||
if _minio_client is None:
|
||||
if not MINIO_ACCESS:
|
||||
raise RuntimeError("MINIO_ACCESS_KEY not set")
|
||||
_minio_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url = MINIO_ENDPOINT,
|
||||
aws_access_key_id = MINIO_ACCESS,
|
||||
aws_secret_access_key = MINIO_SECRET,
|
||||
region_name = "us-east-1",
|
||||
)
|
||||
return _minio_client
|
||||
|
||||
|
||||
def _get_r2():
|
||||
global _r2_client
|
||||
if _r2_client is None and R2_ENDPOINT and R2_ACCESS_KEY:
|
||||
_r2_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url = R2_ENDPOINT,
|
||||
aws_access_key_id = R2_ACCESS_KEY,
|
||||
aws_secret_access_key = R2_SECRET_KEY,
|
||||
region_name = "auto",
|
||||
)
|
||||
return _r2_client
|
||||
|
||||
|
||||
def _upload(bucket, key, file_path, content_type="application/octet-stream"):
|
||||
"""Upload to R2 only (MinIO disabled)."""
|
||||
r2 = _get_r2()
|
||||
if r2:
|
||||
try:
|
||||
r2.upload_file(file_path, R2_BUCKET, key,
|
||||
ExtraArgs={"ContentType": content_type})
|
||||
log.info(f"R2 upload: r2://{R2_BUCKET}/{key}")
|
||||
except Exception as e:
|
||||
log.warning(f"R2 upload failed (non-fatal): {e}")
|
||||
|
||||
|
||||
def upload_pdf(file_path: str, filename: str, book_id: int = None):
|
||||
"""
|
||||
Upload PDF to R2 (if configured).
|
||||
If book_id provided, updates books.pdf_url in DB with R2 public URL.
|
||||
"""
|
||||
if not R2_ENDPOINT or not R2_ACCESS_KEY:
|
||||
log.info(f"R2 not configured — skipping upload for {filename}")
|
||||
return
|
||||
|
||||
try:
|
||||
r2 = _get_r2()
|
||||
if r2:
|
||||
r2.upload_file(file_path, R2_BUCKET, f"pdfs/{filename}",
|
||||
ExtraArgs={"ContentType": "application/pdf"})
|
||||
log.info(f"R2 upload: r2://{R2_BUCKET}/pdfs/{filename}")
|
||||
|
||||
# Store R2 URL in DB if book_id known
|
||||
if book_id and R2_PUBLIC_URL:
|
||||
try:
|
||||
import psycopg2
|
||||
pdf_url = f"{R2_PUBLIC_URL.rstrip('/')}/pdfs/{filename}"
|
||||
conn = psycopg2.connect(os.getenv("POSTGRES_URL", ""))
|
||||
cur = conn.cursor()
|
||||
cur.execute("UPDATE books SET pdf_url=%s WHERE id=%s", (pdf_url, book_id))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
log.info(f"DB pdf_url updated: {pdf_url}")
|
||||
except Exception as e:
|
||||
log.warning(f"DB pdf_url update failed: {e}")
|
||||
except Exception as e:
|
||||
log.error(f"R2 PDF upload failed: {e}")
|
||||
|
||||
|
||||
def upload_cover(file_path: str, filename: str, book_id: int = None):
|
||||
"""
|
||||
Upload cover image to R2 (if configured).
|
||||
If book_id provided, updates books.cover_url in DB with R2 public URL.
|
||||
"""
|
||||
if not R2_ENDPOINT or not R2_ACCESS_KEY:
|
||||
return
|
||||
|
||||
try:
|
||||
r2 = _get_r2()
|
||||
if r2:
|
||||
r2.upload_file(file_path, R2_BUCKET, f"covers/{filename}",
|
||||
ExtraArgs={"ContentType": "image/jpeg"})
|
||||
log.info(f"R2 upload: r2://{R2_BUCKET}/covers/{filename}")
|
||||
|
||||
if book_id and R2_PUBLIC_URL:
|
||||
try:
|
||||
import psycopg2
|
||||
cover_url = f"{R2_PUBLIC_URL.rstrip('/')}/covers/{filename}"
|
||||
conn = psycopg2.connect(os.getenv("POSTGRES_URL", ""))
|
||||
cur = conn.cursor()
|
||||
cur.execute("UPDATE books SET cover_url=%s WHERE id=%s", (cover_url, book_id))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
log.info(f"DB cover_url updated: {cover_url}")
|
||||
except Exception as e:
|
||||
log.warning(f"DB cover_url update failed: {e}")
|
||||
except Exception as e:
|
||||
log.error(f"R2 cover upload failed: {e}")
|
||||
|
||||
|
||||
def get_pdf_url(filename: str, expires: int = 3600) -> str:
|
||||
"""
|
||||
Get a presigned URL for PDF download.
|
||||
Returns R2 public URL if configured, otherwise MinIO presigned URL.
|
||||
"""
|
||||
if R2_PUBLIC_URL:
|
||||
return f"{R2_PUBLIC_URL}/pdfs/{filename}"
|
||||
try:
|
||||
s3 = _get_minio()
|
||||
url = s3.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": BUCKET_MEDIA, "Key": f"pdfs/{filename}"},
|
||||
ExpiresIn=expires,
|
||||
)
|
||||
return url
|
||||
except Exception as e:
|
||||
log.error(f"get_pdf_url error: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def get_cover_url(book_id: int, expires: int = 86400) -> str:
|
||||
"""Get a presigned URL for book cover image."""
|
||||
if R2_PUBLIC_URL:
|
||||
return f"{R2_PUBLIC_URL}/covers/{book_id}.jpg"
|
||||
try:
|
||||
s3 = _get_minio()
|
||||
url = s3.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": BUCKET_COVERS, "Key": f"{book_id}.jpg"},
|
||||
ExpiresIn=expires,
|
||||
)
|
||||
return url
|
||||
except Exception as e:
|
||||
log.error(f"get_cover_url error: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def file_exists_in_minio(bucket: str, key: str) -> bool:
|
||||
try:
|
||||
_get_minio().head_object(Bucket=bucket, Key=key)
|
||||
return True
|
||||
except ClientError:
|
||||
return False
|
||||
|
||||
|
||||
def migrate_existing_files():
|
||||
"""One-time migration: upload all existing local PDFs and covers to MinIO."""
|
||||
import os
|
||||
pdf_dir = "/mnt/rag-media/pdfs"
|
||||
cover_dir = "/mnt/rag-media/covers"
|
||||
migrated = 0
|
||||
|
||||
if os.path.isdir(pdf_dir):
|
||||
for fn in os.listdir(pdf_dir):
|
||||
if fn.endswith(".pdf"):
|
||||
path = os.path.join(pdf_dir, fn)
|
||||
if not file_exists_in_minio(BUCKET_MEDIA, f"pdfs/{fn}"):
|
||||
upload_pdf(path, fn)
|
||||
migrated += 1
|
||||
|
||||
if os.path.isdir(cover_dir):
|
||||
for fn in os.listdir(cover_dir):
|
||||
if fn.endswith(".jpg"):
|
||||
path = os.path.join(cover_dir, fn)
|
||||
if not file_exists_in_minio(BUCKET_COVERS, fn):
|
||||
upload_cover(path, fn)
|
||||
migrated += 1
|
||||
|
||||
log.info(f"Migration complete: {migrated} files uploaded to MinIO")
|
||||
return migrated
|
||||
@@ -0,0 +1,169 @@
|
||||
"""
|
||||
tasks.py — Celery tasks for bulk PDF ingestion
|
||||
Replaces the threading-based worker.py background thread.
|
||||
Each bulk job file becomes a separate Celery task.
|
||||
"""
|
||||
import os, sys, logging
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
sys.path.insert(0, "/opt/rag")
|
||||
from celery_app import app
|
||||
|
||||
log = logging.getLogger("rag.tasks")
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
POSTGRES_URL = os.getenv("POSTGRES_URL", "")
|
||||
BOOKS_DIR = "/mnt/rag-media/pdfs"
|
||||
MEDIA_DIR = "/mnt/rag-media/pdfs"
|
||||
|
||||
|
||||
def get_db():
|
||||
return psycopg2.connect(POSTGRES_URL)
|
||||
|
||||
|
||||
def set_file_status(file_id, status, error=None, meta=None, book_id=None):
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("""UPDATE bulk_job_files
|
||||
SET status=%s, error=%s, meta=%s, book_id=%s
|
||||
WHERE id=%s""",
|
||||
(status, error,
|
||||
__import__("json").dumps(meta) if meta else None,
|
||||
book_id, file_id))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except Exception as e:
|
||||
log.error(f"set_file_status error: {e}")
|
||||
|
||||
|
||||
def update_job_counters(job_id):
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE status='done') as done,
|
||||
COUNT(*) FILTER (WHERE status='error') as failed,
|
||||
COUNT(*) FILTER (WHERE status='skipped') as skipped,
|
||||
COUNT(*) as total
|
||||
FROM bulk_job_files WHERE job_id=%s""", (job_id,))
|
||||
r = cur.fetchone()
|
||||
done, failed, skipped, total = r
|
||||
finished = (done + failed + skipped) >= total
|
||||
job_status = "done" if finished else "processing"
|
||||
cur.execute("""UPDATE bulk_jobs SET status=%s, done=%s, failed=%s
|
||||
WHERE id=%s""", (job_status, done, failed, job_id))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except Exception as e:
|
||||
log.error(f"update_job_counters error: {e}")
|
||||
|
||||
|
||||
@app.task(bind=True, name="tasks.process_bulk_file",
|
||||
max_retries=2, default_retry_delay=300)
|
||||
def process_bulk_file(self, job_id: str, file_id: int,
|
||||
filename: str, uploader_id: int,
|
||||
meta_override: dict = None):
|
||||
"""
|
||||
Process a single PDF file.
|
||||
Used by both single uploads (1 file) and bulk uploads (N files).
|
||||
meta_override: optional {title, author, category} from single upload form.
|
||||
For single uploads, metadata hints are stored as JSON in bulk_job_files.error.
|
||||
"""
|
||||
# Read metadata hints from bulk_job_files if meta_override not passed
|
||||
if meta_override is None:
|
||||
try:
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("SELECT error FROM bulk_job_files WHERE id=%s", (file_id,))
|
||||
row = cur.fetchone(); cur.close(); conn.close()
|
||||
if row and row[0] and row[0].startswith("{"):
|
||||
import json
|
||||
hints = json.loads(row[0])
|
||||
if any(hints.values()):
|
||||
meta_override = hints
|
||||
# Clear the hint so real errors can be stored
|
||||
conn = get_db(); cur = conn.cursor()
|
||||
cur.execute("UPDATE bulk_job_files SET error=NULL WHERE id=%s", (file_id,))
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except Exception as _e:
|
||||
log.warning(f"[task] meta hint read failed: {_e}")
|
||||
"""
|
||||
Process a single PDF file from a bulk upload job.
|
||||
Retries up to 2 times with 5-minute delay on failure.
|
||||
"""
|
||||
log.info(f"[task] start: {filename} (job={job_id})")
|
||||
set_file_status(file_id, "processing")
|
||||
|
||||
pdf_src = os.path.join(BOOKS_DIR, filename)
|
||||
if not os.path.exists(pdf_src):
|
||||
set_file_status(file_id, "error", error="PDF not found on disk")
|
||||
update_job_counters(job_id)
|
||||
return
|
||||
|
||||
# ── Step 1: Compress (before ingest, only replaces if smaller) ─────────
|
||||
set_file_status(file_id, "compressing")
|
||||
try:
|
||||
from compress import compress_pdf
|
||||
c_result = compress_pdf(pdf_src)
|
||||
log.info(f"[task] compress {filename}: {c_result}")
|
||||
except Exception as e:
|
||||
log.warning(f"[task] compression skipped: {e}")
|
||||
|
||||
# ── Step 2: Ingest ──────────────────────────────────────────────────────
|
||||
set_file_status(file_id, "ingesting")
|
||||
try:
|
||||
import ingest as _ingest
|
||||
_ingest.ingest_pdf(pdf_src)
|
||||
except Exception as e:
|
||||
log.error(f"[task] ingest failed: {e}")
|
||||
try:
|
||||
raise self.retry(exc=e)
|
||||
except self.MaxRetriesExceededError:
|
||||
set_file_status(file_id, "error", error=f"Ingest failed: {e}")
|
||||
update_job_counters(job_id)
|
||||
return
|
||||
|
||||
# ── Step 3: Extract metadata ─────────────────────────────────────────────
|
||||
set_file_status(file_id, "extracting")
|
||||
pdf_path = os.path.join(MEDIA_DIR, filename)
|
||||
if not os.path.exists(pdf_path):
|
||||
pdf_path = pdf_src
|
||||
|
||||
from meta_extract import extract_meta, save_meta_to_db
|
||||
meta = extract_meta(pdf_path, filename)
|
||||
|
||||
# ── Step 4: Save metadata ────────────────────────────────────────────────
|
||||
try:
|
||||
save_meta_to_db(filename, meta, uploader_id=uploader_id)
|
||||
conn = get_db()
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT id FROM books WHERE filename=%s", (filename,))
|
||||
book = cur.fetchone()
|
||||
book_id = book["id"] if book else None
|
||||
conn.close()
|
||||
# Apply form overrides from single upload (title/author/category)
|
||||
if meta_override:
|
||||
for field in ("title", "author", "category", "description"):
|
||||
if meta_override.get(field):
|
||||
meta[field] = meta_override[field]
|
||||
save_meta_to_db(filename, meta, uploader_id=uploader_id)
|
||||
|
||||
set_file_status(file_id, "done", meta=meta, book_id=book_id)
|
||||
except Exception as e:
|
||||
set_file_status(file_id, "error", error=str(e), meta=meta)
|
||||
|
||||
# ── Step 5: Rebuild BM25 ─────────────────────────────────────────────────
|
||||
try:
|
||||
from brain import rebuild_bm25
|
||||
rebuild_bm25()
|
||||
except Exception as e:
|
||||
log.warning(f"[task] BM25 rebuild skipped: {e}")
|
||||
|
||||
# ── Step 5: Index in Typesense ────────────────────────────────────────────
|
||||
try:
|
||||
from typesense_client import index_book
|
||||
if book_id:
|
||||
index_book(book_id)
|
||||
except Exception as e:
|
||||
log.warning(f"[task] Typesense index skipped: {e}")
|
||||
|
||||
update_job_counters(job_id)
|
||||
log.info(f"[task] done: {filename} — {meta.get('title')}")
|
||||
@@ -0,0 +1,174 @@
|
||||
"""
|
||||
typesense_client.py — Typesense Arabic full-text search integration
|
||||
Collection: books — indexed fields: title, author, category, description, filename
|
||||
Search augments the library endpoint with typo-tolerance and relevance ranking.
|
||||
"""
|
||||
import os, logging, psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
|
||||
log = logging.getLogger("rag.typesense")
|
||||
|
||||
TYPESENSE_HOST = os.getenv("TYPESENSE_HOST", "typesense")
|
||||
TYPESENSE_PORT = int(os.getenv("TYPESENSE_PORT", "8108"))
|
||||
TYPESENSE_KEY = os.getenv("TYPESENSE_API_KEY", "")
|
||||
POSTGRES_URL = os.getenv("POSTGRES_URL", "")
|
||||
COLLECTION = "books"
|
||||
|
||||
_client = None
|
||||
|
||||
|
||||
def get_client():
|
||||
global _client
|
||||
if _client is None:
|
||||
import typesense
|
||||
_client = typesense.Client({
|
||||
"nodes": [{"host": TYPESENSE_HOST, "port": TYPESENSE_PORT, "protocol": "http"}],
|
||||
"api_key": TYPESENSE_KEY,
|
||||
"connection_timeout_seconds": 5,
|
||||
})
|
||||
return _client
|
||||
|
||||
|
||||
def ensure_collection():
|
||||
"""Create the books collection if it doesn't exist."""
|
||||
client = get_client()
|
||||
schema = {
|
||||
"name": COLLECTION,
|
||||
"fields": [
|
||||
{"name": "id", "type": "string"},
|
||||
{"name": "title", "type": "string"},
|
||||
{"name": "author", "type": "string", "optional": True},
|
||||
{"name": "category", "type": "string", "optional": True, "facet": True},
|
||||
{"name": "language", "type": "string", "optional": True, "facet": True},
|
||||
{"name": "description", "type": "string", "optional": True},
|
||||
{"name": "filename", "type": "string"},
|
||||
{"name": "total_chunks","type": "int32", "default": 0},
|
||||
{"name": "uploaded_at", "type": "int64", "default": 0},
|
||||
],
|
||||
"default_sorting_field": "uploaded_at",
|
||||
"token_separators": ["!", ".", "(", ")", "-"],
|
||||
}
|
||||
try:
|
||||
existing = [c["name"] for c in client.collections.retrieve()]
|
||||
if COLLECTION not in existing:
|
||||
client.collections.create(schema)
|
||||
log.info(f"Typesense collection '{COLLECTION}' created")
|
||||
return True
|
||||
except Exception as e:
|
||||
log.error(f"ensure_collection error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def index_book(book_id: int):
|
||||
"""Index or update a single book in Typesense."""
|
||||
if not TYPESENSE_KEY:
|
||||
return
|
||||
try:
|
||||
conn = psycopg2.connect(POSTGRES_URL)
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT id, title, author, category, language, description,
|
||||
filename, total_chunks, uploaded_at
|
||||
FROM books WHERE id = %s
|
||||
""", (book_id,))
|
||||
book = cur.fetchone()
|
||||
conn.close()
|
||||
if not book:
|
||||
return
|
||||
|
||||
client = get_client()
|
||||
ensure_collection()
|
||||
|
||||
doc = {
|
||||
"id": str(book["id"]),
|
||||
"title": book["title"] or book["filename"].replace(".pdf", ""),
|
||||
"author": book["author"] or "",
|
||||
"category": book["category"] or "عام",
|
||||
"language": book["language"] or "ar",
|
||||
"description": book["description"] or "",
|
||||
"filename": book["filename"],
|
||||
"total_chunks": book["total_chunks"] or 0,
|
||||
"uploaded_at": int(book["uploaded_at"].timestamp()) if book["uploaded_at"] else 0,
|
||||
}
|
||||
client.collections[COLLECTION].documents.upsert(doc)
|
||||
log.info(f"Typesense indexed: {doc['title']} (id={book_id})")
|
||||
except Exception as e:
|
||||
log.error(f"index_book error (book_id={book_id}): {e}")
|
||||
|
||||
|
||||
def index_all_books():
|
||||
"""Bulk index all books. Called on startup or manual refresh."""
|
||||
if not TYPESENSE_KEY:
|
||||
log.warning("TYPESENSE_API_KEY not set — skipping index")
|
||||
return 0
|
||||
try:
|
||||
conn = psycopg2.connect(POSTGRES_URL)
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("""
|
||||
SELECT id, title, author, category, language, description,
|
||||
filename, total_chunks, uploaded_at
|
||||
FROM books ORDER BY id
|
||||
""")
|
||||
books = cur.fetchall()
|
||||
conn.close()
|
||||
|
||||
client = get_client()
|
||||
ensure_collection()
|
||||
|
||||
docs = []
|
||||
for b in books:
|
||||
docs.append({
|
||||
"id": str(b["id"]),
|
||||
"title": b["title"] or b["filename"].replace(".pdf", ""),
|
||||
"author": b["author"] or "",
|
||||
"category": b["category"] or "عام",
|
||||
"language": b["language"] or "ar",
|
||||
"description": b["description"] or "",
|
||||
"filename": b["filename"],
|
||||
"total_chunks": b["total_chunks"] or 0,
|
||||
"uploaded_at": int(b["uploaded_at"].timestamp()) if b["uploaded_at"] else 0,
|
||||
})
|
||||
|
||||
if docs:
|
||||
client.collections[COLLECTION].documents.import_(docs, {"action": "upsert"})
|
||||
log.info(f"Typesense bulk indexed: {len(docs)} books")
|
||||
return len(docs)
|
||||
except Exception as e:
|
||||
log.error(f"index_all_books error: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
def search_books(query: str, category: str = "", limit: int = 20) -> list:
|
||||
"""
|
||||
Full-text search with Typesense.
|
||||
Returns list of book dicts sorted by relevance.
|
||||
Falls back to empty list on error (caller falls back to DB).
|
||||
"""
|
||||
if not TYPESENSE_KEY or not query.strip():
|
||||
return []
|
||||
try:
|
||||
client = get_client()
|
||||
params = {
|
||||
"q": query,
|
||||
"query_by": "title,author,category,description",
|
||||
"query_by_weights": "4,3,2,1",
|
||||
"per_page": limit,
|
||||
"typo_tokens_threshold": 1,
|
||||
"num_typos": 2,
|
||||
}
|
||||
if category:
|
||||
params["filter_by"] = f"category:={category}"
|
||||
|
||||
result = client.collections[COLLECTION].documents.search(params)
|
||||
return [hit["document"] for hit in result.get("hits", [])]
|
||||
except Exception as e:
|
||||
log.error(f"search_books error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def delete_book(book_id: int):
|
||||
"""Remove a book from Typesense index."""
|
||||
try:
|
||||
get_client().collections[COLLECTION].documents[str(book_id)].delete()
|
||||
except Exception as e:
|
||||
log.warning(f"Typesense delete skipped: {e}")
|
||||
@@ -0,0 +1,206 @@
|
||||
"""
|
||||
Bulk Upload Worker — AfakZaid RAG
|
||||
Runs as a background thread inside FastAPI.
|
||||
Picks queued bulk jobs from DB, processes each PDF:
|
||||
1. Ingest (embed into Qdrant)
|
||||
2. Extract metadata via Ollama
|
||||
3. Update DB record
|
||||
"""
|
||||
import os, subprocess, threading, time, json, re, logging
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
import requests as http_req
|
||||
|
||||
log = logging.getLogger("worker")
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
POSTGRES_URL = os.getenv("POSTGRES_URL", "")
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
||||
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2:7b")
|
||||
BOOKS_DIR = "/opt/rag/books"
|
||||
MEDIA_DIR = "/mnt/rag-media/pdfs"
|
||||
|
||||
_stop_event = threading.Event()
|
||||
|
||||
|
||||
def get_db():
|
||||
return psycopg2.connect(POSTGRES_URL)
|
||||
|
||||
|
||||
def set_file_status(file_id: int, status: str, error: str = None, meta: dict = None, book_id: int = None):
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
if meta and book_id:
|
||||
cur.execute("""UPDATE bulk_job_files
|
||||
SET status=%s, error=%s, meta=%s, book_id=%s, updated_at=NOW()
|
||||
WHERE id=%s""", (status, error, json.dumps(meta), book_id, file_id))
|
||||
elif meta:
|
||||
cur.execute("""UPDATE bulk_job_files
|
||||
SET status=%s, error=%s, meta=%s, updated_at=NOW()
|
||||
WHERE id=%s""", (status, error, json.dumps(meta), file_id))
|
||||
else:
|
||||
cur.execute("""UPDATE bulk_job_files
|
||||
SET status=%s, error=%s, updated_at=NOW()
|
||||
WHERE id=%s""", (status, error, file_id))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def update_job_counters(job_id: str):
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("""SELECT
|
||||
COUNT(*) FILTER (WHERE status='done') as done,
|
||||
COUNT(*) FILTER (WHERE status='error') as failed,
|
||||
COUNT(*) as total
|
||||
FROM bulk_job_files WHERE job_id=%s""", (job_id,))
|
||||
r = cur.fetchone()
|
||||
done, failed, total = r['done'], r['failed'], r['total']
|
||||
finished = (done + failed) == total
|
||||
job_status = 'done' if finished else 'processing'
|
||||
cur.execute("""UPDATE bulk_jobs
|
||||
SET done=%s, failed=%s, status=%s, updated_at=NOW()
|
||||
WHERE id=%s""", (done, failed, job_status, job_id))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def extract_meta_ollama(pdf_path: str, filename: str) -> dict:
|
||||
"""Extract title, author, category, description from first 3 pages via Ollama."""
|
||||
try:
|
||||
import fitz
|
||||
doc = fitz.open(pdf_path)
|
||||
text = "".join(doc[i].get_text() for i in range(min(3, len(doc))))[:3000]
|
||||
doc.close()
|
||||
except Exception as e:
|
||||
log.warning(f"fitz error for {filename}: {e}")
|
||||
text = filename.replace('.pdf', '')
|
||||
|
||||
prompt = f"""استخرج من النص التالي معلومات الكتاب وأجب بـ JSON فقط بدون أي نص إضافي:
|
||||
{{"title":"عنوان الكتاب","author":"اسم المؤلف","category":"التصنيف مثل فقه أو حديث أو تفسير أو أدب","description":"وصف مختصر جملة أو جملتان"}}
|
||||
|
||||
النص:
|
||||
{text}"""
|
||||
|
||||
try:
|
||||
r = http_req.post(f"{OLLAMA_URL}/api/generate",
|
||||
json={"model": OLLAMA_MODEL, "prompt": prompt, "stream": False,
|
||||
"options": {"num_predict": 256, "temperature": 0.2}},
|
||||
timeout=90)
|
||||
response_text = r.json().get("response", "")
|
||||
m = re.search(r'\{[^}]+\}', response_text, re.DOTALL)
|
||||
if m:
|
||||
meta = json.loads(m.group())
|
||||
# Ensure all keys present
|
||||
return {
|
||||
"title": meta.get("title", filename.replace('.pdf','')).strip() or filename.replace('.pdf',''),
|
||||
"author": meta.get("author", "").strip(),
|
||||
"category": meta.get("category", "عام").strip() or "عام",
|
||||
"description": meta.get("description", "").strip(),
|
||||
}
|
||||
except Exception as e:
|
||||
log.warning(f"Ollama meta extraction failed for {filename}: {e}")
|
||||
|
||||
return {"title": filename.replace('.pdf',''), "author": "", "category": "عام", "description": ""}
|
||||
|
||||
|
||||
def process_file(file_id: int, job_id: str, filename: str, uploader_id: int):
|
||||
"""Full pipeline for one PDF file."""
|
||||
pdf_src = os.path.join(BOOKS_DIR, filename)
|
||||
|
||||
# ── Step 1: Ingest (embed into Qdrant) ────────────────────────────────
|
||||
set_file_status(file_id, 'ingesting')
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["python3", "/opt/rag/ingest.py", pdf_src],
|
||||
capture_output=True, text=True, timeout=300, env={**os.environ}
|
||||
)
|
||||
if result.returncode != 0 and "already ingested" not in result.stdout:
|
||||
log.warning(f"ingest warning for {filename}: {result.stderr[-200:]}")
|
||||
except subprocess.TimeoutExpired:
|
||||
set_file_status(file_id, 'error', error='Ingest timed out')
|
||||
update_job_counters(job_id)
|
||||
return
|
||||
except Exception as e:
|
||||
set_file_status(file_id, 'error', error=str(e))
|
||||
update_job_counters(job_id)
|
||||
return
|
||||
|
||||
# ── Step 2: Extract metadata via Ollama ───────────────────────────────
|
||||
set_file_status(file_id, 'extracting')
|
||||
pdf_path = os.path.join(MEDIA_DIR, filename)
|
||||
if not os.path.exists(pdf_path):
|
||||
pdf_path = pdf_src
|
||||
from meta_extract import extract_meta, save_meta_to_db
|
||||
meta = extract_meta(pdf_path, filename)
|
||||
|
||||
# ── Step 3: Update book record in DB ──────────────────────────────────
|
||||
try:
|
||||
save_meta_to_db(filename, meta, uploader_id=uploader_id)
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("SELECT id FROM books WHERE filename=%s", (filename,))
|
||||
book = cur.fetchone()
|
||||
book_id = book['id'] if book else None
|
||||
set_file_status(file_id, 'done', meta=meta, book_id=book_id)
|
||||
except Exception as e:
|
||||
set_file_status(file_id, 'error', error=str(e), meta=meta)
|
||||
|
||||
update_job_counters(job_id)
|
||||
log.info(f"[worker] done: {filename} → {meta['title']}")
|
||||
# Rebuild BM25 index
|
||||
try:
|
||||
from brain import rebuild_bm25; rebuild_bm25()
|
||||
except Exception as e:
|
||||
log.warning(f'[worker] BM25 rebuild skipped: {e}')
|
||||
|
||||
|
||||
def process_job(job_id: str):
|
||||
"""Process all pending files in a job sequentially."""
|
||||
log.info(f"[worker] starting job {job_id}")
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("UPDATE bulk_jobs SET status='processing', updated_at=NOW() WHERE id=%s", (job_id,))
|
||||
cur.execute("SELECT id, filename, job_id FROM bulk_job_files WHERE job_id=%s AND status='queued' ORDER BY id", (job_id,))
|
||||
files = cur.fetchall()
|
||||
cur.execute("SELECT uploader_id FROM bulk_jobs WHERE id=%s", (job_id,))
|
||||
job = cur.fetchone()
|
||||
conn.commit()
|
||||
|
||||
uploader_id = job['uploader_id'] if job else None
|
||||
for f in files:
|
||||
process_file(f['id'], f['job_id'], f['filename'], uploader_id)
|
||||
|
||||
log.info(f"[worker] job {job_id} complete")
|
||||
|
||||
|
||||
def worker_loop():
|
||||
"""Main worker loop — polls DB for pending jobs."""
|
||||
log.info("[worker] started")
|
||||
while not _stop_event.is_set():
|
||||
try:
|
||||
with get_db() as conn:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||||
cur.execute("""SELECT id FROM bulk_jobs
|
||||
WHERE status='pending' ORDER BY created_at LIMIT 1""")
|
||||
job = cur.fetchone()
|
||||
if job:
|
||||
process_job(job['id'])
|
||||
else:
|
||||
time.sleep(3)
|
||||
except Exception as e:
|
||||
log.error(f"[worker] loop error: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
_worker_thread: threading.Thread = None
|
||||
|
||||
|
||||
def start_worker():
|
||||
global _worker_thread
|
||||
_stop_event.clear()
|
||||
_worker_thread = threading.Thread(target=worker_loop, daemon=True, name="bulk-worker")
|
||||
_worker_thread.start()
|
||||
log.info("[worker] thread launched")
|
||||
|
||||
|
||||
def stop_worker():
|
||||
_stop_event.set()
|
||||
Reference in New Issue
Block a user