Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index/index.faiss filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,81 @@
 ---
-title: Fecb Rag
-emoji: 🏃
-colorFrom: blue
-colorTo: pink
+title: FECB RAG Search
+emoji: 📚
+colorFrom: green
+colorTo: blue
 sdk: gradio
-sdk_version: 6.14.0
-python_version: '3.13'
+sdk_version: "6.14.0"
 app_file: app.py
 pinned: false
+license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# FECB RAG Search App
+
+A RAG (Retrieval-Augmented Generation) application that lets you search and query a collection of PDF documents using semantic AI search, powered by Claude (Anthropic).
+
+## Project Structure
+
+```
+FECB/
+├── app.py           # Gradio web interface
+├── ingest.py        # PDF ingestion & FAISS index builder
+├── requirements.txt
+├── pdfs/            # Drop your PDF files here
+├── faiss_index/     # Generated by ingest.py (do not edit)
+└── metadata.json    # Generated by ingest.py
+```
+
+## Setup
+
+### 1. Install dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+### 2. Add your PDFs
+
+Copy your PDF files into the `pdfs/` folder (subdirectories are supported).
+
+### 3. Build the vector index
+
+```bash
+python ingest.py
+```
+
+Options:
+```
+--pdf-dir       Path to PDF folder (default: pdfs)
+--index-dir     Where to save the FAISS index (default: faiss_index)
+--chunk-size    Characters per chunk (default: 800)
+--chunk-overlap Overlap between chunks (default: 100)
+```
+
+### 4. Set your Anthropic API key
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+### 5. Run the app
+
+```bash
+python app.py
+```
+
+Open http://localhost:7860 in your browser.
+
+## Configuration
+
+All settings can be overridden with environment variables:
+
+| Variable           | Default                  | Description                      |
+|--------------------|--------------------------|----------------------------------|
+| `ANTHROPIC_API_KEY`| —                        | **Required.** Your Anthropic key |
+| `CLAUDE_MODEL`     | `claude-sonnet-4-6`      | Claude model to use              |
+| `EMBED_MODEL`      | `BAAI/bge-small-en-v1.5` | HuggingFace embedding model      |
+| `TOP_K`            | `5`                      | Number of documents to retrieve  |
+| `INDEX_DIR`        | `faiss_index`            | FAISS index directory            |
+| `META_FILE`        | `metadata.json`          | Metadata file path               |
+| `PDF_DIR`          | `pdfs`                   | PDF source directory (ingest)    |

app.py

@@ -0,0 +1,222 @@
+"""
+app.py — FECB RAG Search Application
+
+Loads a pre-built FAISS index (produced by ingest.py) and provides a
+Gradio interface for semantic search and AI-assisted Q&A over your PDF documents.
+
+Environment variables:
+    ANTHROPIC_API_KEY  — Anthropic API key (required)
+    CLAUDE_MODEL       — Claude model ID (default: claude-sonnet-4-6)
+    EMBED_MODEL        — Embedding model (default: BAAI/bge-small-en-v1.5)
+    TOP_K              — Max documents to retrieve (default: 5)
+    INDEX_DIR          — Path to FAISS index (default: faiss_index)
+    META_FILE          — Path to metadata JSON (default: metadata.json)
+
+Run:
+    python app.py
+"""
+
+import json
+import os
+import re
+from pathlib import Path
+
+import anthropic
+import gradio as gr
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+
+# ── Config ────────────────────────────────────────────────────────────────────
+EMBED_MODEL = os.getenv("EMBED_MODEL", "BAAI/bge-small-en-v1.5")
+CLAUDE_MODEL = os.getenv("CLAUDE_MODEL", "claude-sonnet-4-6")
+API_KEY     = os.getenv("ANTHROPIC_API_KEY")
+TOP_K       = int(os.getenv("TOP_K", "5"))
+INDEX_DIR   = Path(os.getenv("INDEX_DIR", "faiss_index"))
+META_FILE   = Path(os.getenv("META_FILE", "metadata.json"))
+
+SYSTEM_PROMPT = (
+    "You are a knowledgeable research assistant. You help users find relevant "
+    "information from a document collection and synthesize key findings. "
+    "When answering, cite the specific document(s) by their bracketed number [N]. "
+    "Be concise and precise. If the context doesn't contain enough information "
+    "to answer fully, say so clearly."
+)
+
+# ── Load resources ────────────────────────────────────────────────────────────
+print(f"Loading embedding model: {EMBED_MODEL}")
+_embeddings = HuggingFaceEmbeddings(
+    model_name=EMBED_MODEL,
+    model_kwargs={"device": "cpu"},
+    encode_kwargs={"normalize_embeddings": True},
+)
+
+if not INDEX_DIR.exists():
+    raise FileNotFoundError(
+        f"FAISS index not found at '{INDEX_DIR}'. "
+        "Run 'python ingest.py' first to build the index from your PDFs."
+    )
+
+print(f"Loading FAISS index from: {INDEX_DIR}")
+_vectorstore = FAISS.load_local(
+    str(INDEX_DIR), _embeddings, allow_dangerous_deserialization=True
+)
+
+_metadata: dict[str, dict] = {}
+if META_FILE.exists():
+    print(f"Loading metadata from: {META_FILE}")
+    with open(META_FILE, encoding="utf-8") as f:
+        for record in json.load(f):
+            _metadata[record["doc_id"]] = record
+    print(f"  Loaded metadata for {len(_metadata)} documents")
+else:
+    print(f"  [WARN] {META_FILE} not found — document names will be inferred from IDs")
+
+if not API_KEY:
+    raise EnvironmentError("ANTHROPIC_API_KEY is not set. Export it before running.")
+
+_client = anthropic.Anthropic(api_key=API_KEY)
+print(f"Claude model: {CLAUDE_MODEL}")
+print("Ready.\n")
+
+
+# ── RAG helpers ───────────────────────────────────────────────────────────────
+
+def retrieve(query: str, n_docs: int) -> list[tuple]:
+    """Return up to n_docs unique (doc, score) pairs, deduplicated by doc_id."""
+    raw = _vectorstore.similarity_search_with_score(query, k=n_docs * 4)
+    seen: dict[str, tuple] = {}
+    for doc, score in raw:
+        doc_id = doc.metadata.get("doc_id", "")
+        if doc_id not in seen:
+            seen[doc_id] = (doc, score)
+        if len(seen) >= n_docs:
+            break
+    return sorted(seen.values(), key=lambda x: x[1])
+
+
+def build_context(hits: list[tuple]) -> str:
+    parts = []
+    for i, (doc, _) in enumerate(hits, 1):
+        doc_id   = doc.metadata.get("doc_id", f"doc_{i}")
+        filename = doc.metadata.get("filename", f"{doc_id}.pdf")
+        excerpt  = doc.page_content.replace("\n", " ").strip()[:600]
+        parts.append(f"[{i}] {filename} (ID: {doc_id})\n{excerpt}")
+    return "\n\n---\n\n".join(parts)
+
+
+def ask_claude(query: str, context: str) -> str:
+    user_content = (
+        f"Using the document excerpts below, answer the following question. "
+        f"Cite documents by their bracketed number.\n\n"
+        f"Question: {query}\n\nContext:\n{context}"
+    )
+    try:
+        message = _client.messages.create(
+            model=CLAUDE_MODEL,
+            max_tokens=800,
+            system=SYSTEM_PROMPT,
+            messages=[{"role": "user", "content": user_content}],
+        )
+        return message.content[0].text.strip()
+    except anthropic.APIError as exc:
+        return (
+            f"Could not reach Claude ({exc}).\n\n"
+            "Check that **ANTHROPIC_API_KEY** is set and valid."
+        )
+
+
+def cosine_to_pct(score: float) -> str:
+    """Convert FAISS L2 distance (normalised embeddings) to 0–100% relevance."""
+    pct = (1.0 - min(max(score, 0.0), 2.0) / 2.0) * 100
+    return f"{pct:.1f}%"
+
+
+# ── Main search function ──────────────────────────────────────────────────────
+
+def rag_search(query: str, n_docs: int) -> tuple[str, str]:
+    query = query.strip()
+    if not query:
+        return "Please enter a question or keyword.", ""
+
+    hits = retrieve(query, n_docs)
+    if not hits:
+        return "No relevant documents found. Try different keywords.", ""
+
+    context = build_context(hits)
+    answer  = ask_claude(query, context)
+
+    cards = []
+    for i, (doc, score) in enumerate(hits, 1):
+        doc_id   = doc.metadata.get("doc_id", f"doc_{i}")
+        filename = doc.metadata.get("filename", f"{doc_id}.pdf")
+        rel      = cosine_to_pct(score)
+        snippet  = doc.page_content.replace("\n", " ").strip()[:350]
+
+        cards.append(
+            f"### [{i}] {filename}\n"
+            f"**Relevance:** {rel}  \n"
+            f"**ID:** {doc_id}  \n"
+            f"> {snippet}…"
+        )
+
+    return answer, "\n\n---\n\n".join(cards)
+
+
+# ── Gradio UI ─────────────────────────────────────────────────────────────────
+
+with gr.Blocks(title="FECB Document Search") as demo:
+
+    gr.Markdown(
+        """
+# FECB Document Search — AI-Powered RAG
+
+Search your document collection using semantic AI search.
+Ask a question or enter keywords; the app retrieves the most relevant
+documents and generates a synthesised answer with citations.
+
+> **Powered by** `BAAI/bge-small-en-v1.5` embeddings · Claude via Anthropic API
+        """
+    )
+
+    with gr.Row():
+        with gr.Column(scale=5):
+            query_box = gr.Textbox(
+                label="Question or keywords",
+                placeholder="e.g. 'What are the main findings about X?'",
+                lines=2,
+                elem_id="query-box",
+            )
+        with gr.Column(scale=1, min_width=160):
+            n_slider = gr.Slider(
+                minimum=3, maximum=10, value=TOP_K, step=1,
+                label="Documents to return",
+            )
+
+    search_btn = gr.Button("Search", variant="primary", size="lg")
+
+    gr.Markdown("---")
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            gr.Markdown("### AI Answer")
+            answer_md = gr.Markdown(value="*Results will appear here after searching.*")
+
+        with gr.Column(scale=3):
+            gr.Markdown("### Relevant Documents")
+            papers_md = gr.Markdown(value="")
+
+    search_btn.click(rag_search, [query_box, n_slider], [answer_md, papers_md])
+    query_box.submit(rag_search, [query_box, n_slider], [answer_md, papers_md])
+
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        theme=gr.themes.Soft(primary_hue="blue", font=gr.themes.GoogleFont("Inter")),
+        css="""
+            .gradio-container { max-width: 1100px; margin: auto; }
+            #query-box textarea { font-size: 16px; }
+        """,
+    )

faiss_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f844d7a2bbd7893e5ff263ff16e66485d29279d6917e9e0a67af82ffce17eed3
+size 457773

faiss_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94c160696b5d439ab03fdbb7872d37a1ed3a7a3ba53ad0a64ba582c6a3091a56
+size 251399

ingest.py

@@ -0,0 +1,103 @@
+"""
+ingest.py — Build a FAISS vector index from a folder of PDFs.
+
+Usage:
+    python ingest.py [--pdf-dir pdfs] [--index-dir faiss_index] [--chunk-size 800] [--chunk-overlap 100]
+
+Environment variables:
+    EMBED_MODEL   — Embedding model (default: BAAI/bge-small-en-v1.5)
+    PDF_DIR       — Folder containing PDF files (default: pdfs)
+    INDEX_DIR     — Where to save the FAISS index (default: faiss_index)
+"""
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+import fitz  # PyMuPDF
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+from tqdm import tqdm
+
+EMBED_MODEL = os.getenv("EMBED_MODEL", "BAAI/bge-small-en-v1.5")
+PDF_DIR     = Path(os.getenv("PDF_DIR", "pdfs"))
+INDEX_DIR   = Path(os.getenv("INDEX_DIR", "faiss_index"))
+META_FILE   = Path("metadata.json")
+
+
+def extract_text(pdf_path: Path) -> str:
+    doc = fitz.open(str(pdf_path))
+    return "\n".join(page.get_text() for page in doc)
+
+
+def load_pdfs(pdf_dir: Path) -> list[Document]:
+    pdfs = sorted(pdf_dir.glob("**/*.pdf"))
+    if not pdfs:
+        raise FileNotFoundError(f"No PDF files found in {pdf_dir}")
+    print(f"Found {len(pdfs)} PDF(s) in {pdf_dir}")
+
+    docs = []
+    metadata_records = []
+    for pdf_path in tqdm(pdfs, desc="Reading PDFs"):
+        text = extract_text(pdf_path)
+        if not text.strip():
+            print(f"  [WARN] No text extracted from {pdf_path.name}, skipping")
+            continue
+        doc_id = pdf_path.stem
+        docs.append(Document(
+            page_content=text,
+            metadata={"doc_id": doc_id, "filename": pdf_path.name, "source": str(pdf_path)},
+        ))
+        metadata_records.append({"doc_id": doc_id, "filename": pdf_path.name})
+
+    with open(META_FILE, "w", encoding="utf-8") as f:
+        json.dump(metadata_records, f, ensure_ascii=False, indent=2)
+    print(f"Saved metadata for {len(metadata_records)} documents → {META_FILE}")
+    return docs
+
+
+def chunk_documents(docs: list[Document], chunk_size: int, chunk_overlap: int) -> list[Document]:
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separators=["\n\n", "\n", ". ", " ", ""],
+    )
+    chunks = splitter.split_documents(docs)
+    print(f"Split into {len(chunks)} chunks (chunk_size={chunk_size}, overlap={chunk_overlap})")
+    return chunks
+
+
+def build_index(chunks: list[Document], embeddings: HuggingFaceEmbeddings, index_dir: Path) -> None:
+    print(f"Building FAISS index with {EMBED_MODEL}...")
+    vectorstore = FAISS.from_documents(chunks, embeddings)
+    index_dir.mkdir(parents=True, exist_ok=True)
+    vectorstore.save_local(str(index_dir))
+    print(f"FAISS index saved → {index_dir}/")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Ingest PDFs into a FAISS vector index")
+    parser.add_argument("--pdf-dir",      type=Path, default=PDF_DIR)
+    parser.add_argument("--index-dir",    type=Path, default=INDEX_DIR)
+    parser.add_argument("--chunk-size",   type=int,  default=800)
+    parser.add_argument("--chunk-overlap",type=int,  default=100)
+    args = parser.parse_args()
+
+    print(f"Loading embedding model: {EMBED_MODEL}")
+    embeddings = HuggingFaceEmbeddings(
+        model_name=EMBED_MODEL,
+        model_kwargs={"device": "cpu"},
+        encode_kwargs={"normalize_embeddings": True},
+    )
+
+    docs   = load_pdfs(args.pdf_dir)
+    chunks = chunk_documents(docs, args.chunk_size, args.chunk_overlap)
+    build_index(chunks, embeddings, args.index_dir)
+    print("\nDone! You can now run: python app.py")
+
+
+if __name__ == "__main__":
+    main()

metadata.json

@@ -0,0 +1,14 @@
+[
+  {
+    "doc_id": "Bene et al 2026 UPF consumption in Vietnam",
+    "filename": "Bene et al 2026 UPF consumption in Vietnam.pdf"
+  },
+  {
+    "doc_id": "YoshiokaVargas_etal_2026",
+    "filename": "YoshiokaVargas_etal_2026.pdf"
+  },
+  {
+    "doc_id": "agriculture-16-01013-v2",
+    "filename": "agriculture-16-01013-v2.pdf"
+  }
+]

requirements.txt

@@ -0,0 +1,17 @@
+# Core RAG dependencies
+pymupdf>=1.24.0
+langchain>=0.2.0
+langchain-community>=0.2.0
+langchain-huggingface>=0.0.3
+langchain-text-splitters>=0.2.0
+faiss-cpu>=1.8.0
+sentence-transformers>=3.0.0
+
+# LLM
+anthropic>=0.30.0
+
+# UI
+gradio>=6.9.0
+
+# Utilities
+tqdm>=4.66.0