Select Page

Diversity & Novelty Code 

import os
import pandas as pd
import numpy as np
import glob
import os
from tqdm import tqdm
from collections import Counter, defaultdict
from scipy.stats import pearsonr, shapiro

# === CONFIGURATION ===
folder_paths = [
    "/Users/samshaw/Library/Mobile Documents/com~apple~CloudDocs/university /masters/LIS/Capstone/final assessment/clarifai_object_counts",
    "/Users/samshaw/Library/Mobile Documents/com~apple~CloudDocs/university /masters/LIS/Capstone/final assessment/2nd_clarifai_object_counts"
]
confidence_thresh = 0.87

# === 1. MERGE ALL FILES ===
csv_files = []
for path in folder_paths:
    csv_files.extend(glob.glob(os.path.join(path, "*.csv")))

all_rows = []
for file in tqdm(csv_files, desc="📂 Merging CSVs"):
    df = pd.read_csv(file)
    df['source_file'] = os.path.basename(file)
    all_rows.append(df)
df = pd.concat(all_rows, ignore_index=True)

# Rename category for clarity
df.rename(columns={'category': 'clarifai_category'}, inplace=True)

# === 2. WIDE → LONG FORMAT, FILTERED BY CONFIDENCE ===
meta_cols = ['image', 'clarifai_category', 'source_file']
object_cols = [c for c in df.columns if c not in meta_cols]
# Force all object columns to numeric, replacing errors/blanks with 0.0
for col in object_cols:
    df[col] = pd.to_numeric(df[col], errors='coerce').fillna(0.0)
object_cols = [c for c in df.columns if c not in meta_cols]

long_rows = []

for idx, row in tqdm(df.iterrows(), total=df.shape[0], desc="🔄 Reshaping to long"):
    for obj in object_cols:
        conf = row[obj]
        if conf > confidence_thresh:
            long_rows.append({
                'clarifai_category': row['clarifai_category'],
                'image': row['image'],
                'object_label': obj,
                'confidence': conf,
                'source_file': row['source_file']
            })
long_df = pd.DataFrame(long_rows)
# Save long format for audit
long_df.to_csv("clarifai_long_format.csv", index=False)

# =========== DIVERSITY & NOVELTY ANALYSIS ===========

def shannon_diversity(object_counts):
    total = sum(object_counts)
    if total == 0:
        return 0
    proportions = [count / total for count in object_counts]
    return -sum(p * np.log2(p) if p > 0 else 0 for p in proportions)

results = []
for cat, group in long_df.groupby('clarifai_category'):
    object_counts = Counter(group['object_label'])
    richness = len(object_counts)
    total_objs = sum(object_counts.values())
    H = shannon_diversity(object_counts.values())
    avg_objs_per_image = group.groupby('image')['object_label'].count().mean()
    results.append({
        'category': cat,
        'shannon_diversity': H,
        'unique_objects': richness,
        'total_objects': total_objs,
        'object_richness_ratio': richness / total_objs if total_objs else 0,
        'avg_objs_per_image': avg_objs_per_image,
        'category_size': group['image'].nunique()
    })
cat_df = pd.DataFrame(results)

# --- Coefficient of Variation
cv = cat_df['shannon_diversity'].std() / cat_df['shannon_diversity'].mean()

# --- Normality Test
stat, p_shapiro = shapiro(cat_df['shannon_diversity'])

# --- IQR Outlier Detection
Q1 = cat_df['shannon_diversity'].quantile(0.25)
Q3 = cat_df['shannon_diversity'].quantile(0.75)
IQR = Q3 - Q1
outlier_mask = ~cat_df['shannon_diversity'].between(Q1 - 1.5 * IQR, Q3 + 1.5 * IQR)
outliers = cat_df[outlier_mask]

# --- Correlations
def safe_corr(x, y):
    try:
        r, p = pearsonr(x, y)
        return r, p
    except Exception as e:
        return np.nan, np.nan

r1, p1 = safe_corr(cat_df['shannon_diversity'], cat_df['unique_objects'])
r2, p2 = safe_corr(cat_df['shannon_diversity'], cat_df['object_richness_ratio'])
r3, p3 = safe_corr(cat_df['shannon_diversity'], cat_df['avg_objs_per_image'])
r4, p4 = safe_corr(cat_df['category_size'], cat_df['shannon_diversity'])

# --- Object Novelty Analysis
novelty_results = []
for cat, group in long_df.groupby('clarifai_category'):
    image_objs = group.groupby('image')['object_label'].apply(list)
    seen = set()
    total_new = 0
    total_imgs = len(image_objs)
    for objs in image_objs:
        new_objs = set(objs) - seen
        total_new += len(new_objs)
        seen.update(objs)
    novelty_rate = total_new / total_imgs if total_imgs else 0
    novelty_results.append({'category': cat, 'novelty_rate': novelty_rate, 'category_size': total_imgs})
novelty_df = pd.DataFrame(novelty_results)
# Correlate novelty with category size
r5, p5 = safe_corr(novelty_df['novelty_rate'], novelty_df['category_size'])

# --- OUTPUTS ---
print("\n--- Shannon Diversity Index per Category ---")
print(cat_df[['category', 'shannon_diversity', 'unique_objects', 'object_richness_ratio', 'avg_objs_per_image', 'category_size']])

print("\nCoefficient of Variation (Shannon Diversity):", round(cv, 3))
print("Normality test (Shapiro-Wilk): p =", round(p_shapiro, 3), "| normal" if p_shapiro > 0.05 else "| not normal")
print("Outlier categories (IQR):", outliers['category'].tolist())

print("\n--- Correlations ---")
print("Shannon diversity vs Unique objects: r =", round(r1, 3), "p =", round(p1, 3))
print("Shannon diversity vs Object richness ratio: r =", round(r2, 3), "p =", round(p2, 3))
print("Shannon diversity vs Avg objects per image: r =", round(r3, 3), "p =", round(p3, 3))
print("Category size vs Shannon diversity: r =", round(r4, 3), "p =", round(p4, 3))

print("\n--- Object Novelty ---")
print(novelty_df)
print("Novelty rate vs Category size: r =", round(r5, 3), "p =", round(p5, 3))

# Optional: Save results
cat_df.to_csv("category_shannon_diversity.csv", index=False)
novelty_df.to_csv("category_object_novelty.csv", index=False)

←→