v1.8.2: admin overview — library health, rebuild BM25, clear cache, reingest missing endpoints

api.py

@@ -490,6 +490,97 @@ def admin_overview(_=Depends(require_admin)):
         }
     }
 
+
+# ── Library Health ─────────────────────────────────────────────────────────────
+@app.get("/admin/library-health")
+def admin_library_health(_=Depends(require_admin)):
+    from qdrant_client import QdrantClient as _QC
+    import os as _os
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("SELECT COUNT(*) AS total FROM books")
+            total_books = cur.fetchone()["total"]
+            cur.execute("""
+                SELECT b.id, b.filename, b.title
+                FROM books b
+                WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.book_filename = b.filename)
+                ORDER BY b.filename
+            """)
+            missing = cur.fetchall()
+            try:
+                cur.execute("""
+                    SELECT filename, status, error_message, created_at
+                    FROM ingest_jobs WHERE status='failed'
+                    ORDER BY created_at DESC LIMIT 10
+                """)
+                failed = cur.fetchall()
+                cur.execute("SELECT COUNT(*) AS c FROM ingest_jobs WHERE status='pending' OR status='running'")
+                pending_jobs = cur.fetchone()["c"]
+            except Exception:
+                conn.rollback()
+                failed = []
+                pending_jobs = 0
+            cur.execute("SELECT COUNT(*) AS c FROM chunks")
+            total_chunks = cur.fetchone()["c"]
+
+    try:
+        qc = _QC(url=_os.getenv("QDRANT_URL","http://qdrant:6333"),
+                 api_key=_os.getenv("QDRANT_API_KEY",""))
+        info = qc.get_collection("arabic_books")
+        vector_count = info.vectors_count or info.points_count or 0
+    except Exception:
+        vector_count = 0
+
+    return {
+        "total_books":    total_books,
+        "indexed_books":  total_books - len(missing),
+        "missing_chunks": len(missing),
+        "missing_books":  [dict(r) for r in missing],
+        "failed_files":   [dict(r) for r in failed],
+        "pending_jobs":   pending_jobs,
+        "total_chunks":   total_chunks,
+        "vector_count":   vector_count,
+    }
+
+# ── Rebuild BM25 ───────────────────────────────────────────────────────────────
+@app.post("/admin/rebuild-bm25")
+def admin_rebuild_bm25(_=Depends(require_admin)):
+    from brain import rebuild_bm25
+    rebuild_bm25()
+    return {"status": "ok", "message": "BM25 index rebuilt"}
+
+# ── Clear Cache ────────────────────────────────────────────────────────────────
+@app.post("/admin/clear-cache")
+def admin_clear_cache(_=Depends(require_admin)):
+    import redis as _redis, os as _os
+    url  = _os.getenv("REDIS_URL", "redis://rag-redis:6379")
+    pwd  = _os.getenv("REDIS_PASSWORD", "")
+    r    = _redis.from_url(url + (f"?password={pwd}" if pwd else ""), decode_responses=True)
+    keys = r.keys("rag:cache:*")
+    deleted = len(keys)
+    if keys: r.delete(*keys)
+    return {"status": "ok", "deleted_keys": deleted}
+
+# ── Re-ingest Missing ──────────────────────────────────────────────────────────
+@app.post("/admin/reingest-missing")
+def admin_reingest_missing(_=Depends(require_admin)):
+    from tasks import ingest_pdf_task
+    import os as _os
+    with get_db() as conn:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute("""
+                SELECT filename FROM books b
+                WHERE NOT EXISTS (SELECT 1 FROM chunks c WHERE c.book_filename = b.filename)
+            """)
+            rows = cur.fetchall()
+    queued = 0
+    for r in rows:
+        path = f"/mnt/rag-media/pdfs/{r['filename']}"
+        if _os.path.exists(path):
+            ingest_pdf_task.delay(path)
+            queued += 1
+    return {"status": "ok", "queued": queued}
+
 @app.get("/admin/books")
 def admin_books(_=Depends(require_admin)):
     with get_db() as conn:

redis_data/appendonlydir/appendonly.aof.2.incr.aof

@@ -51786,3 +51786,223 @@ $3
 del
 $13
 unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+628356265ac811f199e682dacbd0314b
+$4
+PXAT
+$13
+1779995511793
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+9e205f585ac811f199e682dacbd0314b
+$4
+PXAT
+$13
+1779995611807
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+d9bd229e5ac811f199e682dacbd0314b
+$4
+PXAT
+$13
+1779995711820
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+155a061e5ac911f199e682dacbd0314b
+$4
+PXAT
+$13
+1779995811834
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+50f698cc5ac911f199e682dacbd0314b
+$4
+PXAT
+$13
+1779995911846
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+8c9320265ac911f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996011857
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+c82fbed25ac911f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996111869
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+03cc2ce65aca11f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996211880
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+3f688ed45aca11f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996311890
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+7b047f3e5aca11f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996411898
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+b6a0cc8c5aca11f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996511908
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+f23dee5a5aca11f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996611923
+*2
+$3
+del
+$13
+unacked_mutex
+*5
+$3
+SET
+$13
+unacked_mutex
+$32
+2ddb13a25acb11f199e682dacbd0314b
+$4
+PXAT
+$13
+1779996711939
+*2
+$3
+del
+$13
+unacked_mutex
+*2
+$6
+SELECT
+$1
+0
+*3
+$3
+DEL
+$42
+rag:cache:29818f3e30b650fe3989c671faac13a7
+$42
+rag:cache:0af1b0865c46edc9187b4091cda597a9

static/admin.html

@@ -370,6 +370,30 @@ a[role=button]:active {
         </button>
       </div>
       <div id="stats-grid" class="grid grid-cols-2 lg:grid-cols-3 xl:grid-cols-6 gap-4 mb-6"></div>
+      <!-- ── Library Health ─────────────────────────────────────────────── -->
+      <div class="glass rounded-2xl p-6 spotlight-card mb-6">
+        <div class="flex items-center justify-between mb-4">
+          <h3 class="text-primary text-xs font-bold uppercase tracking-widest flex items-center gap-2">
+            <span class="msym text-base">library_books</span> Library Health
+          </h3>
+          <div class="flex gap-2">
+            <button onclick="rebuildBM25()" id="btn-bm25" class="glass glass-hover px-3 py-1.5 rounded-xl text-xs font-bold flex items-center gap-1 text-secondary">
+              <span class="msym text-sm">refresh</span> Rebuild BM25
+            </button>
+            <button onclick="clearCache()" id="btn-cache" class="glass glass-hover px-3 py-1.5 rounded-xl text-xs font-bold flex items-center gap-1 text-yellow-400">
+              <span class="msym text-sm">delete_sweep</span> Clear Cache
+            </button>
+            <button onclick="reingestMissing()" id="btn-reingest" class="glass glass-hover px-3 py-1.5 rounded-xl text-xs font-bold flex items-center gap-1 text-red-400 hidden">
+              <span class="msym text-sm">sync</span> Re-ingest Missing
+            </button>
+          </div>
+        </div>
+        <div id="library-health-content">
+          <div class="text-on-surface-variant text-sm">Loading...</div>
+        </div>
+      </div>
+
+
       <div class="grid grid-cols-1 lg:grid-cols-2 gap-6 mb-6">
         <div class="glass rounded-2xl p-6 spotlight-card">
           <h3 class="text-primary text-xs font-bold uppercase tracking-widest mb-4 flex items-center gap-2"><span class="msym text-base">bar_chart</span>Daily Queries</h3>
@@ -949,7 +973,128 @@ function badge(llm) {
 }
 
 // ── Page Loaders ──────────────────────────────────────────────────────────────
+async function loadLibraryHealth() {
+  try {
+    const d = await api('/admin/library-health');
+    const el = document.getElementById('library-health-content');
+    const btnReingest = document.getElementById('btn-reingest');
+
+    // Show/hide reingest button
+    if (d.missing_chunks > 0) btnReingest.classList.remove('hidden');
+    else btnReingest.classList.add('hidden');
+
+    // Health summary pills
+    const health = d.missing_chunks === 0 ? 'text-emerald-400' : 'text-red-400';
+    const healthIcon = d.missing_chunks === 0 ? 'check_circle' : 'warning';
+
+    let html = `<div class="grid grid-cols-2 lg:grid-cols-4 gap-4 mb-4">
+      <div class="bg-surface-high rounded-xl p-3 text-center">
+        <div class="text-xl font-bold text-primary">${d.total_books}</div>
+        <div class="text-xs text-on-surface-variant mt-1">Total Books</div>
+      </div>
+      <div class="bg-surface-high rounded-xl p-3 text-center">
+        <div class="text-xl font-bold text-emerald-400">${d.indexed_books}</div>
+        <div class="text-xs text-on-surface-variant mt-1">Indexed</div>
+      </div>
+      <div class="bg-surface-high rounded-xl p-3 text-center">
+        <div class="text-xl font-bold ${health}">${d.missing_chunks}</div>
+        <div class="text-xs text-on-surface-variant mt-1">Missing Chunks</div>
+      </div>
+      <div class="bg-surface-high rounded-xl p-3 text-center">
+        <div class="text-xl font-bold text-blue-400">${d.vectors?.toLocaleString()}</div>
+        <div class="text-xs text-on-surface-variant mt-1">Vectors</div>
+      </div>
+    </div>`;
+
+    // Books with no chunks
+    if (d.no_chunks?.length > 0) {
+      html += `<div class="mb-4">
+        <div class="text-xs font-bold text-red-400 uppercase tracking-widest mb-2 flex items-center gap-1">
+          <span class="msym text-sm">warning</span> Books with no chunks (${d.no_chunks.length})
+        </div>
+        <div class="space-y-1 max-h-40 overflow-y-auto">
+          ${d.no_chunks.map(b => `<div class="bg-red-900/20 border border-red-900/30 rounded-lg px-3 py-2 text-xs text-on-surface-variant">${b.title || b.filename}</div>`).join('')}
+        </div>
+      </div>`;
+    }
+
+    // Failed files
+    if (d.failed_files?.length > 0) {
+      html += `<div class="mb-4">
+        <div class="text-xs font-bold text-orange-400 uppercase tracking-widest mb-2 flex items-center gap-1">
+          <span class="msym text-sm">error</span> Failed Files (${d.failed_files.length})
+        </div>
+        <div class="space-y-1 max-h-40 overflow-y-auto">
+          ${d.failed_files.map(f => `<div class="bg-orange-900/20 border border-orange-900/30 rounded-lg px-3 py-2 text-xs">
+            <span class="text-on-surface-variant">${f.filename}</span>
+            <span class="text-red-400 ml-2">${f.error || 'Unknown error'}</span>
+          </div>`).join('')}
+        </div>
+      </div>`;
+    }
+
+    // Pending jobs
+    if (d.pending_jobs?.length > 0) {
+      html += `<div>
+        <div class="text-xs font-bold text-yellow-400 uppercase tracking-widest mb-2 flex items-center gap-1">
+          <span class="msym text-sm">pending</span> Active Jobs (${d.pending_jobs.length})
+        </div>
+        <div class="space-y-1">
+          ${d.pending_jobs.map(j => `<div class="bg-yellow-900/20 border border-yellow-900/30 rounded-lg px-3 py-2 text-xs flex justify-between">
+            <span class="text-on-surface-variant">${j.status} — ${j.done}/${j.total} done</span>
+            <span class="text-red-400">${j.errors > 0 ? j.errors + ' errors' : ''}</span>
+          </div>`).join('')}
+        </div>
+      </div>`;
+    }
+
+    if (d.missing_chunks === 0 && d.failed_files?.length === 0 && d.pending_jobs?.length === 0) {
+      html += `<div class="flex items-center gap-2 text-emerald-400 text-sm">
+        <span class="msym">check_circle</span> All ${d.total_books} books are fully indexed
+      </div>`;
+    }
+
+    el.innerHTML = html;
+  } catch(e) {
+    document.getElementById('library-health-content').innerHTML = `<div class="text-red-400 text-sm">Error loading health: ${e.message}</div>`;
+  }
+}
+
+async function rebuildBM25() {
+  const btn = document.getElementById('btn-bm25');
+  btn.disabled = true; btn.innerHTML = '<span class="msym text-sm animate-spin">refresh</span> Rebuilding...';
+  try {
+    const r = await api('/admin/rebuild-bm25', {method:'POST'});
+    toast(`BM25 rebuilt — ${r.chunks_indexed} chunks indexed`, 'success');
+  } catch(e) { toast('BM25 rebuild failed: ' + e.message, 'error'); }
+  btn.disabled = false; btn.innerHTML = '<span class="msym text-sm">refresh</span> Rebuild BM25';
+}
+
+async function clearCache() {
+  const btn = document.getElementById('btn-cache');
+  btn.disabled = true; btn.innerHTML = '<span class="msym text-sm">delete_sweep</span> Clearing...';
+  try {
+    const r = await api('/admin/clear-cache', {method:'POST'});
+    toast(`Cache cleared — ${r.cleared} entries removed`, 'success');
+    loadOverview();
+  } catch(e) { toast('Cache clear failed: ' + e.message, 'error'); }
+  btn.disabled = false; btn.innerHTML = '<span class="msym text-sm">delete_sweep</span> Clear Cache';
+}
+
+async function reingestMissing() {
+  const btn = document.getElementById('btn-reingest');
+  btn.disabled = true; btn.innerHTML = '<span class="msym text-sm">sync</span> Queuing...';
+  try {
+    const r = await api('/admin/reingest-missing', {method:'POST'});
+    toast(`${r.queued} books queued for re-ingestion`, 'success');
+    setTimeout(loadLibraryHealth, 2000);
+  } catch(e) { toast('Re-ingest failed: ' + e.message, 'error'); }
+  btn.disabled = false; btn.innerHTML = '<span class="msym text-sm">sync</span> Re-ingest Missing';
+}
+
+
 async function loadOverview() {
+  loadLibraryHealth();
   try {
     const d = await api('/admin/overview');
     document.getElementById('stats-grid').innerHTML = [