import os
import pandas as pd
import numpy as np
import glob
import os
from tqdm import tqdm
from collections import Counter, defaultdict
from scipy.stats import pearsonr, shapiro
# === CONFIGURATION ===
folder_paths = [
"/Users/samshaw/Library/Mobile Documents/com~apple~CloudDocs/university /masters/LIS/Capstone/final assessment/clarifai_object_counts",
"/Users/samshaw/Library/Mobile Documents/com~apple~CloudDocs/university /masters/LIS/Capstone/final assessment/2nd_clarifai_object_counts"
]
confidence_thresh = 0.87
# === 1. MERGE ALL FILES ===
csv_files = []
for path in folder_paths:
csv_files.extend(glob.glob(os.path.join(path, "*.csv")))
all_rows = []
for file in tqdm(csv_files, desc="📂 Merging CSVs"):
df = pd.read_csv(file)
df['source_file'] = os.path.basename(file)
all_rows.append(df)
df = pd.concat(all_rows, ignore_index=True)
# Rename category for clarity
df.rename(columns={'category': 'clarifai_category'}, inplace=True)
# === 2. WIDE → LONG FORMAT, FILTERED BY CONFIDENCE ===
meta_cols = ['image', 'clarifai_category', 'source_file']
object_cols = [c for c in df.columns if c not in meta_cols]
# Force all object columns to numeric, replacing errors/blanks with 0.0
for col in object_cols:
df[col] = pd.to_numeric(df[col], errors='coerce').fillna(0.0)
object_cols = [c for c in df.columns if c not in meta_cols]
long_rows = []
for idx, row in tqdm(df.iterrows(), total=df.shape[0], desc="🔄 Reshaping to long"):
for obj in object_cols:
conf = row[obj]
if conf > confidence_thresh:
long_rows.append({
'clarifai_category': row['clarifai_category'],
'image': row['image'],
'object_label': obj,
'confidence': conf,
'source_file': row['source_file']
})
long_df = pd.DataFrame(long_rows)
# Save long format for audit
long_df.to_csv("clarifai_long_format.csv", index=False)
# =========== DIVERSITY & NOVELTY ANALYSIS ===========
def shannon_diversity(object_counts):
total = sum(object_counts)
if total == 0:
return 0
proportions = [count / total for count in object_counts]
return -sum(p * np.log2(p) if p > 0 else 0 for p in proportions)
results = []
for cat, group in long_df.groupby('clarifai_category'):
object_counts = Counter(group['object_label'])
richness = len(object_counts)
total_objs = sum(object_counts.values())
H = shannon_diversity(object_counts.values())
avg_objs_per_image = group.groupby('image')['object_label'].count().mean()
results.append({
'category': cat,
'shannon_diversity': H,
'unique_objects': richness,
'total_objects': total_objs,
'object_richness_ratio': richness / total_objs if total_objs else 0,
'avg_objs_per_image': avg_objs_per_image,
'category_size': group['image'].nunique()
})
cat_df = pd.DataFrame(results)
# --- Coefficient of Variation
cv = cat_df['shannon_diversity'].std() / cat_df['shannon_diversity'].mean()
# --- Normality Test
stat, p_shapiro = shapiro(cat_df['shannon_diversity'])
# --- IQR Outlier Detection
Q1 = cat_df['shannon_diversity'].quantile(0.25)
Q3 = cat_df['shannon_diversity'].quantile(0.75)
IQR = Q3 - Q1
outlier_mask = ~cat_df['shannon_diversity'].between(Q1 - 1.5 * IQR, Q3 + 1.5 * IQR)
outliers = cat_df[outlier_mask]
# --- Correlations
def safe_corr(x, y):
try:
r, p = pearsonr(x, y)
return r, p
except Exception as e:
return np.nan, np.nan
r1, p1 = safe_corr(cat_df['shannon_diversity'], cat_df['unique_objects'])
r2, p2 = safe_corr(cat_df['shannon_diversity'], cat_df['object_richness_ratio'])
r3, p3 = safe_corr(cat_df['shannon_diversity'], cat_df['avg_objs_per_image'])
r4, p4 = safe_corr(cat_df['category_size'], cat_df['shannon_diversity'])
# --- Object Novelty Analysis
novelty_results = []
for cat, group in long_df.groupby('clarifai_category'):
image_objs = group.groupby('image')['object_label'].apply(list)
seen = set()
total_new = 0
total_imgs = len(image_objs)
for objs in image_objs:
new_objs = set(objs) - seen
total_new += len(new_objs)
seen.update(objs)
novelty_rate = total_new / total_imgs if total_imgs else 0
novelty_results.append({'category': cat, 'novelty_rate': novelty_rate, 'category_size': total_imgs})
novelty_df = pd.DataFrame(novelty_results)
# Correlate novelty with category size
r5, p5 = safe_corr(novelty_df['novelty_rate'], novelty_df['category_size'])
# --- OUTPUTS ---
print("\n--- Shannon Diversity Index per Category ---")
print(cat_df[['category', 'shannon_diversity', 'unique_objects', 'object_richness_ratio', 'avg_objs_per_image', 'category_size']])
print("\nCoefficient of Variation (Shannon Diversity):", round(cv, 3))
print("Normality test (Shapiro-Wilk): p =", round(p_shapiro, 3), "| normal" if p_shapiro > 0.05 else "| not normal")
print("Outlier categories (IQR):", outliers['category'].tolist())
print("\n--- Correlations ---")
print("Shannon diversity vs Unique objects: r =", round(r1, 3), "p =", round(p1, 3))
print("Shannon diversity vs Object richness ratio: r =", round(r2, 3), "p =", round(p2, 3))
print("Shannon diversity vs Avg objects per image: r =", round(r3, 3), "p =", round(p3, 3))
print("Category size vs Shannon diversity: r =", round(r4, 3), "p =", round(p4, 3))
print("\n--- Object Novelty ---")
print(novelty_df)
print("Novelty rate vs Category size: r =", round(r5, 3), "p =", round(p5, 3))
# Optional: Save results
cat_df.to_csv("category_shannon_diversity.csv", index=False)
novelty_df.to_csv("category_object_novelty.csv", index=False)