Create app.py

app.py

@@ -0,0 +1,128 @@
+import gradio as gr
+import torch
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from sklearn.metrics import accuracy_score, f1_score
+import numpy as np
+
+# --- CONFIGURATION ---
+# REPLACE THIS WITH YOUR UPLOADED MODEL NAME!
+MODEL_REPO = "angelperedo01/proj2" 
+DATASET_NAME = "nvidia/Aegis-AI-Content-Safety-Dataset-2.0"
+MAX_SAMPLES = 200  # Limit samples for the demo so it doesn't take hours
+
+def get_text_and_label(example):
+    """
+    Your custom logic to parse the NVIDIA dataset labels.
+    """
+    text = example.get('prompt', '')
+    label = None
+    
+    # Try 'prompt_label' first
+    if 'prompt_label' in example:
+        raw_label = example['prompt_label']
+        if isinstance(raw_label, str):
+            raw_lower = raw_label.lower()
+            if any(x in raw_lower for x in ['unsafe', 'harmful', 'toxic', 'attack']):
+                label = 1
+            elif any(x in raw_lower for x in ['safe', 'benign']):
+                label = 0
+            else:
+                try: label = int(raw_label)
+                except: label = 1 if 'unsafe' in raw_lower else 0
+        else:
+            label = int(raw_label)
+    
+    # Default to Safe (0) if we really can't find it
+    if label is None: label = 0
+    return text, label
+
+def run_live_evaluation(progress=gr.Progress()):
+    # 1. Load Model & Data
+    yield "Loading Model from Hub...", "-", "-", []
+    
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(MODEL_REPO)
+        model = AutoModelForSequenceClassification.from_pretrained(MODEL_REPO)
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model.to(device)
+        model.eval()
+    except Exception as e:
+        yield f"Error loading model: {str(e)}", "Error", "Error", []
+        return
+
+    # Load Dataset (Streaming or small slice for speed)
+    yield "Loading NVIDIA Dataset...", "-", "-", []
+    try:
+        # Try test split, fallback to train
+        ds = load_dataset(DATASET_NAME, split="test")
+    except:
+        ds = load_dataset(DATASET_NAME, split="train")
+    
+    # Shuffle and select subset for the demo
+    ds = ds.shuffle(seed=42).select(range(MAX_SAMPLES))
+    
+    true_labels = []
+    predictions = []
+    logs = [] # To store misclassifications
+    
+    # 2. The Evaluation Loop
+    for i, item in enumerate(progress.tqdm(ds, desc="Evaluating...")):
+        text, true_label = get_text_and_label(item)
+        true_labels.append(true_label)
+        
+        # Tokenize
+        inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=256).to(device)
+        
+        # Predict
+        with torch.no_grad():
+            logits = model(**inputs).logits
+            pred = torch.argmax(logits, dim=-1).item()
+            predictions.append(pred)
+        
+        # Log Errors (If prediction is wrong)
+        if pred != true_label:
+            status = "🔴 MISS" 
+            logs.insert(0, [status, text[:80] + "...", "Safe" if true_label==0 else "Unsafe", "Safe" if pred==0 else "Unsafe"])
+        else:
+            # Optional: Log successes too if you want, but it clutters the view
+            pass
+
+        # Update UI every 5 steps
+        if i % 5 == 0 or i == len(ds)-1:
+            acc = accuracy_score(true_labels, predictions)
+            f1 = f1_score(true_labels, predictions, zero_division=0)
+            
+            status_msg = f"Processed {i+1}/{MAX_SAMPLES}"
+            yield status_msg, f"{acc:.2%}", f"{f1:.2f}", logs[:10] # Show last 10 errors
+
+# --- UI LAYOUT ---
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown(f"## 🛡️ Live Safety Model Evaluation")
+    gr.Markdown(f"Running `{MODEL_REPO}` on `{DATASET_NAME}` (Live Inference)")
+    
+    with gr.Row():
+        start_btn = gr.Button("▶️ Start Live Test", variant="primary")
+    
+    with gr.Row():
+        with gr.Column():
+            status_box = gr.Label(value="Ready", label="Status")
+        with gr.Column():
+            acc_box = gr.Label(value="-", label="Current Accuracy")
+        with gr.Column():
+            f1_box = gr.Label(value="-", label="Current F1 Score")
+
+    gr.Markdown("### 🚨 Recent Misclassifications (Live Feed)")
+    log_table = gr.Dataframe(
+        headers=["Status", "Text Snippet", "True Label", "Predicted"], 
+        datatype=["str", "str", "str", "str"],
+        row_count=10
+    )
+
+    start_btn.click(
+        fn=run_live_evaluation,
+        inputs=None,
+        outputs=[status_box, acc_box, f1_box, log_table]
+    )
+
+demo.queue().launch()