Select Page

Summary of Dataset Bias analysis with quantitative Validation Code

import os
import pandas as pd
import numpy as np
from scipy.stats import chisquare

# Load the data
objects = pd.read_csv('clarifai_long_format.csv')

# 1. Images per category
images_per_cat = objects.groupby('clarifai_category')['image'].nunique()
category_names = images_per_cat.index.tolist()
category_sizes = images_per_cat.values
num_categories = len(category_sizes)
total_images = category_sizes.sum()

print(f"Images per category:\n{images_per_cat}\n")

# 2. Category imbalance ratio
imbalance_ratio = category_sizes.max() / category_sizes.min()
print(f"Category imbalance ratio: {imbalance_ratio:.2f}:1 (range: {category_sizes.min()}–{category_sizes.max()})")

# 3. Chi-square test for evenness
expected = np.ones(num_categories) * (total_images / num_categories)
chi2_stat, p_value = chisquare(f_obs=category_sizes, f_exp=expected)
print(f"Chi-square statistic: {chi2_stat:.1f}, p = {p_value:.3g}")

# 4. Cohen's w
cohens_w = np.sqrt(chi2_stat / total_images)
print(f"Cohen's w: {cohens_w:.3f}")

# 5. Power variation
power_variation = 100 * (category_sizes.max() - category_sizes.min()) / category_sizes.max()
print(f"Statistical power variation: {power_variation:.1f}% difference between strongest and weakest categories")

# 6. Object-level impact: frequency of "people" (adjust as needed for your objects/categories)
people_freq = objects[objects['object_label'] == 'people'].groupby('clarifai_category')['object_label'].count()
nature_cats = [cat for cat in category_names if "nature" in cat.lower()]  # Adjust if your category names differ
tech_cats = [cat for cat in category_names if "tech" in cat.lower() or "digital" in cat.lower() or "social" in cat.lower()]

people_tech = people_freq[tech_cats].sum() if tech_cats else np.nan
people_nature = people_freq[nature_cats].sum() if nature_cats else np.nan
expected_people = people_freq.sum() / num_categories

print(f"People in tech categories: {people_tech/expected_people:.2f}× expected frequency" if not np.isnan(people_tech) else "No tech category found.")
print(f"People in nature categories: {people_nature/expected_people:.2f}× expected frequency" if not np.isnan(people_nature) else "No nature category found.")

# 7. Double-weighted analysis (first: by category size, second: by total objects per category)
total_objects_per_cat = objects.groupby('clarifai_category')['object_label'].count().values
first_weight = 1 / category_sizes
second_weight = 1 / total_objects_per_cat
double_weight = first_weight * second_weight
norm_double_weight = double_weight / double_weight.sum()
print(f"Double-weighted contributions (%):\n{(norm_double_weight * 100).round(3)}")

# 8. Inverse size weighting for fair contribution
weights = 1 / category_sizes
norm_weights = weights / weights.sum()
print(f"Weighted contributions (%):\n{(norm_weights * 100).round(3)}")
print(f"Variance of weighted contributions: {np.var(norm_weights):.8f}")
if np.allclose(norm_weights * 100, 5.0, atol=0.05):
    print("All categories contribute exactly 5.0% to weighted analyses (perfect balance).")
else:
    print("Category contributions are not perfectly balanced.")

# 9. Count of representational bias instances (categories outside expected +/- X%)
tolerance = 0.10  # 10% tolerance
bias_instances = ((category_sizes < (expected * (1 - tolerance))) | (category_sizes > (expected * (1 + tolerance)))).sum()
print(f"Quantified inequality: {bias_instances} instances of representational bias across the dataset.")

# --- Optional: print more detailed object/category stats ---
print(objects.groupby(['clarifai_category', 'object_label']).size().unstack(fill_value=0))

# Collect results as strings
results = [
    f"Images per category:\n{images_per_cat.to_string()}\n",
    f"Category imbalance ratio: {imbalance_ratio:.2f}:1 (range: {category_sizes.min()}–{category_sizes.max()})",
    f"Chi-square statistic: {chi2_stat:.1f}, p = {p_value:.3g}",
    f"Cohen's w: {cohens_w:.3f}",
    f"Statistical power variation: {power_variation:.1f}%",
    (f"People in tech categories: {people_tech/expected_people:.2f}× expected frequency"
        if not np.isnan(people_tech) else "No tech category found."),
    (f"People in nature categories: {people_nature/expected_people:.2f}× expected frequency"
        if not np.isnan(people_nature) else "No nature category found."),
    f"Weighted contributions (%):\n{(norm_weights * 100).round(3)}",
    f"Variance of weighted contributions: {np.var(norm_weights):.8f}",
    "All categories contribute exactly 5.0% to weighted analyses (perfect balance)." if np.allclose(norm_weights * 100, 5.0, atol=0.05)
        else "Category contributions are not perfectly balanced.",
    f"Double-weighted contributions (%):\n{(norm_double_weight * 100).round(3)}",
    f"Quantified inequality: {bias_instances} instances of representational bias across the dataset."
]

# Write to txt file
with open("dataset_bias_metrics.txt", "w") as f:
    for line in results:
        f.write(str(line) + "\n")
print("Results saved to dataset_bias_metrics.txt")


←→