Hallucinations

!pip install numpy==2.2.6 \
             torch==2.6.0 \
             torchvision==0.21.0 \
             nltk==3.9.4 \
             spacy==3.8.14 \
             matplotlib==3.10.9 \
             tqdm==4.67.3 \
             transformers==5.9.0 \
             datasets==4.8.5 \
             scikit-learn==1.7.2 
!python -m spacy download en_core_web_sm

caption = "The cats are running quickly."

{
    "tokens": ["The", "cats", "are", "running", "quickly", "."],
    "pos": ["DET", "NOUN", "AUX", "VERB", "ADV", "PUNCT"],
    "tag": ["DT", "NNS", "VBP", "VBG", "RB", "."],
    "is_stop": [True, False, True, False, False, False],
    "lemma": ["the", "cat", "be", "run", "quickly", "."]
}

import numpy as np
import torch
import nltk
import spacy
import pandas as pd
import matplotlib.pyplot as plt

from tqdm import tqdm
from transformers import CLIPProcessor, CLIPModel
from datasets import load_dataset

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline

# 1. NLTK Setup
print("Downloading NLTK resources (punkt)...")
nltk.download("punkt", quiet=True)
nltk.download("punkt_tab", quiet=True)

# 2. spaCy Setup
print("Loading spaCy (en_core_web_sm)...")
nlp = spacy.load("en_core_web_sm")
def get_linguistic_features(caption: str) -> dict:
    """
    Helper function that provides only raw linguistic data.
    """
    doc = nlp(caption)
    return {
        "tokens": [t.text for t in doc],
        "pos": [t.pos_ for t in doc],
        "tag": [t.tag_ for t in doc],
        "is_stop": [t.is_stop for t in doc],
        "lemma": [t.lemma_ for t in doc]
    }

# 3. Loading the data
print("Loading the training and validation data...")
labeled_data = load_dataset("delyanboychev/hallucinations_problem_data")
print(f"Successfully loaded splits: {list(labeled_data.keys())}")

# 4. CLIP Setup
print("Initialising the CLIP model...")
if torch.cuda.is_available():
    device = "cuda"
elif torch.backends.mps.is_available():
    device = "mps"
else:
    device = "cpu"

print(f"Device used: {device.upper()}")

model_id = "openai/clip-vit-base-patch32"
processor = CLIPProcessor.from_pretrained(model_id)
model = CLIPModel.from_pretrained(model_id).to(device)
model.eval()

print("All resources and data were loaded successfully!")

# Helper function provided by the senior researchers :)
def get_pos_tags(text):
    """
    Returns a structure suitable for visualisation (text)
    and for mathematical processing (dict).
    """
    features = get_linguistic_features(text)

    readable_tags = [f"{token} ({pos})" for token, pos in zip(features["tokens"], features["pos"])]
    return readable_tags

# Dictionary to make the classes easier to read
label_map = {
    0: "0: Original (Correct)",
    1: "1: Substituted object (Hallucination)",
    2: "2: Shuffled words (Incoherent)"
}

# We visualise exactly 3 examples
for i in range(3):
    sample = labeled_data["train"][i]
    image = sample["image"]
    caption = sample["caption"]
    label = sample["label"]

    print("\n" + "="*60)
    print(f"Example {i+1}")
    print(f"Label: {label_map.get(label, 'Unknown')}")
    print(f"Text (Caption): {caption}")
    print("-" * 60)

    # We call the helper function
    pos_tags = get_pos_tags(caption)
    print("Parts of speech (POS Tags):")
    print(" | ".join(pos_tags))
    print("="*60)

    # We show the image itself
    plt.figure(figsize=(4, 4))
    plt.imshow(image)
    plt.title(f"Label: {label}")
    plt.axis("off")
    plt.show()

# !!! YOU MAY IMPORT SPECIFIC MODULES AND KEEP GLOBAL VARIABLES, BUT ONLY IN THIS CELL !!!

def extract_features(dataset_split, batch_size=64, is_train=False):
    """
    Extracts features in batches.
    Returns ONLY the matrix X (with features, max 64 columns).
    is_train can help you if you need to keep some state extracted from
    the training set that you can use at test time.
    """

    # =========================================================
    # THIS IS WHERE YOU MUST WRITE YOUR SOLUTION
    # =========================================================
    # Basic functions are given to help you with the solution!

    def get_simple_nltk_feature(text: str) -> float:
        """Example: Lexical diversity using NLTK."""
        tokens = nltk.word_tokenize(text.lower())
        return len(set(tokens)) / len(tokens) if tokens else 0.0

    def get_noun_density(ling_data: dict) -> float:
        """Example: Noun density, using ling_data."""
        nouns = [1 for pos in ling_data["pos"] if pos == "NOUN"]
        return sum(nouns) / len(ling_data["tokens"]) if ling_data["tokens"] else 0.0

    # =========================================================

    X = []

    for i in tqdm(range(0, len(dataset_split), batch_size), desc="Extracting features"):
        batch = dataset_split[i : i + batch_size]
        images = batch["image"]
        captions = batch["caption"]

        # --- Step 1: Extracting linguistic primitives ---
        # We use the mandatory helper function
        ling_results = [get_linguistic_features(cap) for cap in captions]

        # --- Step 2: Computing the CLIP features ---
        inputs = processor(
            text=captions, images=images, return_tensors="pt", padding=True
        ).to(device)

        with torch.no_grad():
            outputs = model(**inputs)

            # Normalising the embeddings
            image_embeds = outputs.image_embeds / outputs.image_embeds.norm(p=2, dim=-1, keepdim=True)
            text_embeds = outputs.text_embeds / outputs.text_embeds.norm(p=2, dim=-1, keepdim=True)

            # Batched Cosine Similarity
            cosine_sims = (image_embeds * text_embeds).sum(dim=-1).cpu().numpy()

        # --- Step 3: Assembling the final vector ---
        for idx, sim in enumerate(cosine_sims):

            # ATTENTION: Add your new features to this list!
            # Maximum limit: 16 elements.

            # We take the ready-made data for the current example
            ling_data = ling_results[idx]

            # We compute the features
            nltk_feat = get_simple_nltk_feature(captions[idx])
            noun_feat = get_noun_density(ling_data)

            feature_vector = [sim, nltk_feat, noun_feat]

            X.append(feature_vector)

    return np.array(X)

# ==========================================
# ATTENTION: IT IS STRICTLY FORBIDDEN TO CHANGE THIS BLOCK!
# ==========================================
BATCH_SIZE = 64

print("--- Processing the Train Split ---")
# We take the labels for training
y_train = np.array(labeled_data["train"]["label"])

safe_train_data = labeled_data["train"].remove_columns("label")
safe_val_data = labeled_data["val"].remove_columns("label")


X_train = extract_features(safe_train_data, batch_size=BATCH_SIZE, is_train=True)


print("\n--- Processing the Val Split ---")
X_val = extract_features(safe_val_data, batch_size=BATCH_SIZE, is_train=False)


assert X_train.shape[1] <= 16, f"Error: You are using {X_train.shape[1]} features, and the limit is 16!"
assert X_val.shape[1] <= 16, f"Error: The validation set has {X_val.shape[1]} features!"

print("\nTraining the fixed Gradient Boosting Classifier...")

# We use a pipeline with StandardScaler for numerical stability
clf = make_pipeline(
    StandardScaler(),
    GradientBoostingClassifier(
        n_estimators=400,
        max_depth=5,
        learning_rate=0.05,
        subsample=0.8,
        max_features="sqrt",
        min_samples_leaf=5,
        random_state=42,
    ),
)

# We train the model
clf.fit(X_train, y_train)

# We predict on the validation set
print("\nGenerating predictions...")
y_val_pred = clf.predict(X_val)

# Creating a file for submission to Kaggle
print("Creating submission.csv...")

submission_df = pd.DataFrame({
    "id": labeled_data["val"]["id"],
    "label": y_val_pred
})

submission_df.to_csv("submission.csv", index=False)
print("\n" + "=" * 50)
print("DONE! The file 'submission.csv' was saved successfully.")
print("You can download it and submit it to Kaggle to see your score!")
print("=" * 50)