Add final files and prepare for deployment

# .gitignore

@@ -0,0 +1,17 @@
+# .gitignore
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.DS_Store
+*.log
+*.tmp
+*.swp
+
+# Optional: If dataset is too large or shouldn't be committed
+# combined_emails_with_natural_pii.csv
+# api_output_results.jsonl
+
+# Optional: If model is very large and managed by Git LFS or stored elsewhere
+# saved_models/

README.md

@@ -1,103 +1,120 @@
 # Email Classification and PII Masking API
 
-This project implements an API to classify support emails into predefined categories while masking Personally Identifiable Information (PII) before processing.
+This project implements an API service for classifying support emails into predefined categories while masking Personally Identifiable Information (PII) before classification.
+
+## Objective
+
+To build an email classification system for a support team that:
+1.  Masks PII (Full Name, Email, Phone, DOB, Aadhar, Card Numbers, CVV, Expiry) using Regex and SpaCy NER (without LLMs).
+2.  Classifies emails into categories (e.g., Billing Issues, Technical Support) using a trained machine learning model.
+3.  Exposes this functionality via a FastAPI endpoint.
 
 ## Project Structure
 
 ```
-.
-├── api.py             # FastAPI application logic
-├── models.py          # Model loading, prediction (and optional training) logic
-├── utils.py           # PII masking and text utility functions
-├── requirements.txt   # Python dependencies
-├── saved_models/      # Directory for trained models (classifier, vectorizer)
-├── data/              # Directory for datasets (add your data here)
-├── .env               # (Optional) For environment variables
-├── .gitignore         # Git ignore file
-└── README.md          # This file
+/workspaces/internship1/
+├── saved_models/
+│   └── email_classifier_pipeline.pkl   # Saved classification model pipeline
+├── api.py                # FastAPI application logic and endpoints
+├── app.py                # Script to run the Uvicorn server
+├── models.py             # Model loading, prediction functions
+├── utils.py              # PII masking logic, text cleaning
+├── train.py              # Script to train the classification model
+├── requirements.txt      # Python package dependencies
+├── combined_emails_with_natural_pii.csv # Dataset used for training (ensure this is present)
+├── README.md             # This file
+└── .gitignore            # (Optional but recommended: add *.pyc, __pycache__, .venv, saved_models/*, *.csv)
 ```
 
 ## Setup
 
-1.  **Clone the repository:**
+1.  **Clone the repository (if applicable):**
     ```bash
     git clone <your-repo-url>
-    cd <your-repo-name>
+    cd internship1
     ```
-
-2.  **Create and activate a virtual environment:** (Recommended)
+2.  **Create and activate a virtual environment:**
     ```bash
-    python -m venv venv
-    source venv/bin/activate  # On Windows use `venv\Scripts\activate`
+    python -m venv .venv
+    source .venv/bin/activate
+    # On Windows use: .venv\Scripts\activate
     ```
-
 3.  **Install dependencies:**
     ```bash
     pip install -r requirements.txt
     ```
-
-4.  **Download NLP models:**
+4.  **Download SpaCy Model (if not done automatically by `utils.py`):**
     ```bash
     python -m spacy download en_core_web_sm
-    # Run inside python interpreter if needed:
-    # import nltk; nltk.download('punkt'); nltk.download('stopwords')
     ```
+5.  **Train the Model (if `saved_models/email_classifier_pipeline.pkl` is not present):**
+    *   Ensure the dataset (`combined_emails_with_natural_pii.csv`) is in the root directory.
+    *   Run the training script:
+        ```bash
+        python train.py
+        ```
 
-5.  **(Crucial Step) Train the Model:**
-    *   You need to train the classification model first. Add your training data to the `data/` directory.
-    *   Adapt and run the training logic (e.g., the `train_and_save_model` function in `models.py`, potentially moving it to a separate `train.py` script). This will create the necessary files in `saved_models/`.
-    *   Example (modify as needed): `python models.py` (if you add training execution there) or `python train.py`
-
-## Running the API Locally
+## Running the API
 
-Once the models are trained and saved:
-
-```bash
-uvicorn api:app --reload --host 0.0.0.0 --port 8000
-```
-
-The API will be available at `http://localhost:8000` (or the port forwarded by your Codespace).
-
-## API Usage
+1.  **Start the FastAPI server:**
+    ```bash
+    python app.py
+    # OR directly using uvicorn:
+    # uvicorn api:app --host 0.0.0.0 --port 8000 --reload
+    ```
+    The API will be available at `http://127.0.0.1:8000` (or the port specified).
 
-Send a POST request to the `/classify/` endpoint with a JSON body containing the email text:
+## Using the API
 
-**Endpoint:** `POST /classify/`
+Send a POST request to the `/classify/` endpoint with a JSON body containing the email text.
 
-**Request Body:**
+**Example using `curl`:**
 
-```json
-{
-  "email_body": "Your email content here. My name is John Smith and my number is 555-123-4567."
-}
+```bash
+curl -X POST "http://127.0.0.1:8000/classify/" \
+     -H "Content-Type: application/json" \
+     -d '{
+           "email_body": "Hello, my name is Jane Doe and my card number is 4111-1111-1111-1111. My email is jane.doe@example.com. Please help with billing."
+         }'
 ```
 
-**Example Response:**
+**Expected Response Structure:**
 
 ```json
 {
-  "input_email_body": "Your email content here. My name is John Smith and my number is 555-123-4567.",
+  "input_email_body": "Hello, my name is Jane Doe and my card number is 4111-1111-1111-1111. My email is jane.doe@example.com. Please help with billing.",
   "list_of_masked_entities": [
     {
-      "position": [34, 44],
+      "position": [18, 26],
       "classification": "full_name",
-      "entity": "John Smith"
+      "entity": "Jane Doe"
     },
     {
-      "position": [61, 73],
-      "classification": "phone_number",
-      "entity": "555-123-4567"
+      "position": [49, 68],
+      "classification": "credit_debit_no",
+      "entity": "4111-1111-1111-1111"
+    },
+    {
+      "position": [83, 103],
+      "classification": "email",
+      "entity": "jane.doe@example.com"
     }
   ],
-  "masked_email": "Your email content here. My name is [full_name] and my number is [phone_number].",
-  "category_of_the_email": "Predicted Category" // e.g., "Account Management"
+  "masked_email": "Hello, my name is [full_name] and my card number is [credit_debit_no]. My email is [email]. Please help with billing.",
+  "category_of_the_email": "Billing Issues" // Example category
 }
 ```
 
-## Deployment
+## Model Details
 
-(Add instructions for deploying to Hugging Face Spaces later)
+*   **PII Masking:** SpaCy (`en_core_web_sm` for PERSON) and custom Regex patterns.
+*   **Classification Model:** TF-IDF Vectorizer + Multinomial Naive Bayes.
+*   **Training Accuracy:** Approx. 69% (as per last training run).
 
-## Report
+## Deployment
 
-(Link to or include your report here)
+(Add link to your Hugging Face Space deployment here once completed)
+
+```
+HF Space: [Link]
+```

api.py

@@ -1,20 +1,26 @@
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel, Field
-from typing import List, Dict, Tuple # Added Tuple
-import os # To potentially load environment variables if needed
+from typing import List, Dict, Tuple, Any, Optional
+import os  # To potentially load environment variables if needed
 
-# Import functions from our modules
-from utils import mask_pii
-from models import load_model_and_vectorizer, predict_category
+# Make sure these imports work and don't cause errors themselves
+try:
+    from utils import mask_pii
+    from models import load_model_pipeline, predict_category, Pipeline  # Ensure Pipeline is imported if type hint used
+except ImportError as e:
+    print(f"Error importing from utils or models in api.py: {e}")
+    # Optionally raise the error to make it obvious during startup
+    # raise e
+except Exception as e:
+    print(f"Unexpected error during imports in api.py: {e}")
+    # raise e
 
-# --- API Initialization ---
-app = FastAPI(
-    title="Email Classification API",
-    description="Classifies support emails and masks PII.",
-    version="1.0.0"
-)
+# --- FastAPI App ---
+# >>>>> THIS LINE IS CRUCIAL <<<<<
+app = FastAPI()
+# >>>>> MUST BE NAMED 'app' <<<<<
 
-# --- Data Models for Request and Response ---
+# --- Pydantic Models ---
 class EmailInput(BaseModel):
     email_body: str = Field(..., example="Hello, my name is Jane Doe and my email is jane.doe@example.com. I have a billing question.")
 
@@ -29,19 +35,23 @@ class ClassificationOutput(BaseModel):
     masked_email: str
     category_of_the_email: str
 
-# --- Load Models on Startup ---
-# This ensures the model is loaded once when the API starts, not per request
-model, vectorizer = load_model_and_vectorizer()
+# --- Load Model at Startup ---
+# Ensure load_model_pipeline() exists and works
+model_pipeline: Optional[Pipeline] = load_model_pipeline()
+
+# --- API Endpoints ---
+@app.get("/")
+async def read_root():
+    return {"message": "Email Classification API is running. Use the /classify/ endpoint."}
 
-# --- API Endpoint ---
 @app.post("/classify/", response_model=ClassificationOutput)
 async def classify_email(email_input: EmailInput):
     """
     Accepts an email body, masks PII, classifies the email,
     and returns the results in the specified JSON format.
     """
-    if not model or not vectorizer:
-        raise HTTPException(status_code=503, detail="Model or Vectorizer not available. Please train/load them first.")
+    if not model_pipeline:
+        raise HTTPException(status_code=503, detail="Model pipeline not available. Please train/load it first.")
 
     input_email = email_input.email_body
 
@@ -53,7 +63,7 @@ async def classify_email(email_input: EmailInput):
         validated_entities = [MaskedEntity(**entity) for entity in entities]
 
         # 2. Classify the masked email
-        predicted_class = predict_category(masked_email_body, model, vectorizer)
+        predicted_class = predict_category(masked_email_body, model_pipeline)
 
         # 3. Construct the response
         response = ClassificationOutput(
@@ -70,11 +80,6 @@ async def classify_email(email_input: EmailInput):
         # Optionally include more detail in the error response during development
         raise HTTPException(status_code=500, detail=f"Internal Server Error: {str(e)}")
 
-# --- Root Endpoint (Optional - for basic check) ---
-@app.get("/")
-async def root():
-    return {"message": "Email Classification API is running. Use the /classify/ endpoint."}
-
 # --- Running the API (for local development) ---
 # You'll typically run this using uvicorn from the terminal:
 # uvicorn api:app --reload

app.py

@@ -0,0 +1,27 @@
+# filepath: /workspaces/internship1/app.py
+import uvicorn
+import os
+
+# Import the FastAPI app instance from api.py
+# Ensure the FastAPI instance in api.py is named 'app'
+try:
+    from api import app
+except ImportError:
+    print("Error: Could not import 'app' from api.py.")
+    print("Make sure api.py exists and contains a FastAPI instance named 'app'.")
+    app = None
+except Exception as e:
+    print(f"An unexpected error occurred during import: {e}")
+    app = None
+
+
+if __name__ == "__main__":
+    if app:
+        # Get port from environment variable PORT, default to 8000
+        # Hugging Face Spaces and other platforms often set the PORT variable
+        port = int(os.environ.get("PORT", 8000))
+        # Use host="0.0.0.0" to make it accessible externally (in Codespaces/Docker/HF)
+        print(f"Starting Uvicorn server on host 0.0.0.0, port {port}")
+        uvicorn.run(app, host="0.0.0.0", port=port)
+    else:
+        print("Could not start server because the FastAPI app instance was not loaded.")

generate_output.py

@@ -0,0 +1,100 @@
+import pandas as pd
+import requests
+import json
+from pathlib import Path
+import time
+
+# --- Configuration ---
+DATASET_PATH = Path("combined_emails_with_natural_pii.csv") # Path to your input CSV
+OUTPUT_PATH = Path("api_output_results.jsonl") # Where to save the results (JSON Lines format)
+API_ENDPOINT = "http://127.0.0.1:8000/classify/" # Your running API endpoint
+
+# Adjust column names if different
+email_body_column = 'email'
+# category_column = 'type' # Original category column (optional, for reference)
+
+# --- Main Function ---
+def process_emails_via_api(data_path: Path, output_path: Path, api_url: str):
+    """
+    Reads emails from a CSV, sends them to the classification API,
+    and saves the JSON responses to a file.
+    """
+    if not data_path.exists():
+        print(f"Error: Input dataset not found at {data_path}")
+        return
+
+    print(f"Loading dataset from {data_path}...")
+    try:
+        # Skip bad lines just in case, consistent with training
+        df = pd.read_csv(data_path, on_bad_lines='skip')
+    except Exception as e:
+        print(f"Error loading CSV: {e}")
+        return
+
+    if email_body_column not in df.columns:
+        print(f"Error: Email body column '{email_body_column}' not found.")
+        return
+
+    # Handle potential missing email bodies
+    df.dropna(subset=[email_body_column], inplace=True)
+    if df.empty:
+        print("Error: No valid email bodies found after handling missing values.")
+        return
+
+    results = []
+    total_emails = len(df)
+    print(f"Processing {total_emails} emails via API: {api_url}")
+
+    # Open output file in write mode (clears existing content)
+    with open(output_path, 'w') as f_out:
+        for index, row in df.iterrows():
+            email_text = str(row[email_body_column]) # Ensure it's a string
+            payload = {"email_body": email_text}
+
+            try:
+                response = requests.post(api_url, json=payload, timeout=30) # Added timeout
+                response.raise_for_status() # Raise an exception for bad status codes (4xx or 5xx)
+
+                api_result = response.json()
+
+                # Write result as a JSON line
+                f_out.write(json.dumps(api_result) + '\n')
+
+                if (index + 1) % 50 == 0: # Print progress every 50 emails
+                    print(f"Processed {index + 1}/{total_emails} emails...")
+
+            except requests.exceptions.RequestException as e:
+                print(f"\nError processing email index {index}: {e}")
+                # Optionally write error info to the file or a separate log
+                error_info = {
+                    "error": str(e),
+                    "input_email_body": email_text,
+                    "index": index
+                }
+                f_out.write(json.dumps(error_info) + '\n')
+                # Optional: add a small delay if API is overloaded
+                # time.sleep(0.5)
+            except json.JSONDecodeError as e:
+                 print(f"\nError decoding JSON response for email index {index}: {e}")
+                 print(f"Response status code: {response.status_code}")
+                 print(f"Response text: {response.text[:500]}...") # Print beginning of text
+                 error_info = {
+                    "error": f"JSONDecodeError: {e}",
+                    "response_text": response.text,
+                    "input_email_body": email_text,
+                    "index": index
+                 }
+                 f_out.write(json.dumps(error_info) + '\n')
+
+
+    print(f"\nProcessing complete. Results saved to {output_path}")
+
+
+# --- Script Execution ---
+if __name__ == "__main__":
+    # Make sure the API is running before executing this script!
+    print("--- Starting API Output Generation ---")
+    print("Ensure the FastAPI server (python app.py or uvicorn) is running in another terminal.")
+    input("Press Enter to continue once the API is running...")
+    process_emails_via_api(DATASET_PATH, OUTPUT_PATH, API_ENDPOINT)
+    print("--- Finished API Output Generation ---")

models.py

@@ -10,7 +10,6 @@ import re
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 from utils import clean_text_for_classification, mask_pii
-from models import MODEL_PATH, load_model_pipeline, predict_category
 
 # --- Constants ---
 MODEL_DIR = Path("saved_models")
@@ -37,7 +36,7 @@ class ClassificationOutput(BaseModel):
 
 # --- Load Model at Startup ---
 # Load the model pipeline once when the application starts
-model_pipeline: Optional[Pipeline] = load_model_pipeline()
+model_pipeline: Optional[Pipeline] = None
 
 # --- Model Loading ---
 def load_model_pipeline() -> Optional[Pipeline]:

requirements.txt

@@ -1,19 +1,19 @@
-# Core libraries
-pandas
-numpy
-scikit-learn
-python-dotenv
+# --- Core ---
+fastapi==0.111.0
+uvicorn[standard]==0.29.0
+pandas==2.2.2
+numpy==1.26.4
+scikit-learn==1.4.2
+joblib==1.4.2
 
-# NLP libraries for PII masking (non-LLM)
-spacy>=3.0.0,<4.0.0
-nltk
+# --- PII Masking ---
+spacy==3.7.4
+# The following model is downloaded by spacy CLI, but good to list
+# en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
 
-# API framework
-fastapi>=0.70.0,<1.0.0
-uvicorn[standard]>=0.15.0,<1.0.0
-pydantic>=1.8.0,<2.0.0 # For data validation in FastAPI
+# --- Other potential dependencies (add if you used them) ---
+# nltk==3.8.1 # If you used NLTK for preprocessing
+# python-dotenv==1.0.1 # If loading environment variables
 
-# Add libraries for your chosen classification model later
-# e.g., transformers, torch, tensorflow
-# joblib or pickle for saving models if not using framework methods
-joblib
+# --- Development/Testing (Optional) ---
+# requests==2.31.0 # For testing the API or running generate_output.py