import os
#!/usr/bin/env python3
"""
H4 — Category gaps in L5–L6 share, single-file runner
Uses: Maslow
Research question: Do education images have lower L5–L6 share than play, nature, and creativity images?
Statistical question: Are group medians/means different with education lower than the reference set?
- Kruskal–Wallis global test across groups
- One-sided post-hoc contrasts (education < each reference) via Mann–Whitney U
Follow-up: Estimated difference and its CI after multiple-testing correction (Bonferroni for familywise 95%)
Inputs:
- combined_with_maslow_columns_2.0.csv (unweighted presence per image; wide)
- maslow_weighted_per_object.csv (weighted per object; long or already grouped by image)
Outputs (saved next to inputs by default):
- h4_unweighted_pairwise.csv
- h4_weighted_pairwise.csv
- h4_unweighted_sizes.csv
- h4_weighted_sizes.csv
Example:
python h4_hypothesis_runner.py \
--unweighted /path/to/combined_with_maslow_columns_2.0.csv \
--weighted /path/to/maslow_weighted_per_object.csv \
--education education \
--references play nature creativity \
--alpha 0.05 \
--n-boot 5000 \
--seed 42
"""
from __future__ import annotations
import argparse
from pathlib import Path
from dataclasses import dataclass
from typing import Tuple, Optional, Dict
import numpy as np
import pandas as pd
from scipy import stats
from statsmodels.stats.multitest import multipletests
MASLOW_COLUMNS = [
"Physiological Needs",
"Safety Needs",
"Love and Belonging",
"Esteem Needs",
"Self-Actualization",
"Self-Transcendence",
]
@dataclass
class H4Config:
education_label: str = "education"
reference_labels: Tuple[str, str, str] = ("play", "nature", "creativity")
alpha: float = 0.05
n_boot: int = 5000
seed: Optional[int] = 42
def _presence_from_text(cell) -> int:
if pd.isna(cell):
return 0
s = str(cell).strip()
return 1 if len(s) > 0 else 0
def compute_unweighted_l56_share(df: pd.DataFrame) -> pd.DataFrame:
required_cols = {"image", "category", *MASLOW_COLUMNS}
missing = required_cols - set(df.columns)
if missing:
raise ValueError(f"Missing required columns: {missing}")
maslow_presence = df[MASLOW_COLUMNS].applymap(_presence_from_text).astype(int)
total_present = maslow_presence.sum(axis=1)
l56 = maslow_presence[["Self-Actualization", "Self-Transcendence"]].sum(axis=1)
share = np.where(total_present > 0, l56 / total_present, 0.0)
out = pd.DataFrame({
"image": df["image"],
"category": df["category"].str.lower(),
"l56_share": share.astype(float),
"n_levels_present": total_present.astype(int),
})
return out
def compute_weighted_l56_share(df_weighted: pd.DataFrame) -> pd.DataFrame:
required_cols = {"image","category", *MASLOW_COLUMNS}
missing = required_cols - set(df_weighted.columns)
if missing:
raise ValueError(f"Missing required columns: {missing}")
grouped = df_weighted.groupby(["image","category"], as_index=False)[list(MASLOW_COLUMNS)].sum()
total = grouped[MASLOW_COLUMNS].sum(axis=1)
l56 = grouped[["Self-Actualization","Self-Transcendence"]].sum(axis=1)
share = np.where(total > 0, l56 / total, 0.0)
out = pd.DataFrame({
"image": grouped["image"],
"category": grouped["category"].str.lower(),
"l56_share": share.astype(float),
"total_weight": total.astype(float),
})
return out
def _kruskal_wallis(across_groups: Dict[str, np.ndarray]):
arrays = [np.asarray(v, dtype=float) for v in across_groups.values()]
H, p = stats.kruskal(*arrays, nan_policy="omit")
return float(H), float(p)
def _one_sided_mwu_less(x: np.ndarray, y: np.ndarray) -> float:
res = stats.mannwhitneyu(x, y, alternative="less")
return float(res.pvalue)
def _bootstrap_diff_ci(x, y, func, alpha: float, n_boot: int, seed: Optional[int] = None):
rng = np.random.default_rng(seed)
x = np.asarray(x, dtype=float)
y = np.asarray(y, dtype=float)
point = float(func(x) - func(y))
n_x, n_y = len(x), len(y)
boots = np.empty(n_boot, dtype=float)
for i in range(n_boot):
bx = rng.choice(x, size=n_x, replace=True)
by = rng.choice(y, size=n_y, replace=True)
boots[i] = func(bx) - func(by)
lo = float(np.percentile(boots, 100 * (alpha/2)))
hi = float(np.percentile(boots, 100 * (1 - alpha/2)))
return point, (lo, hi)
def h4_analysis(df_shares: pd.DataFrame, config: H4Config, variant_label: str):
needed = {"image","category","l56_share"}
if missing := (needed - set(df_shares.columns)):
raise ValueError(f"df_shares missing columns: {missing}")
groups = [config.education_label, *config.reference_labels]
sub = df_shares[df_shares["category"].isin(groups)].copy()
sub["category"] = sub["category"].str.lower()
arrays = {g: sub.loc[sub["category"] == g, "l56_share"].dropna().to_numpy() for g in groups}
H, p_kw = _kruskal_wallis(arrays)
pvals = []
pairs = []
for ref in config.reference_labels:
p = _one_sided_mwu_less(arrays[config.education_label], arrays[ref])
pvals.append(p)
pairs.append((config.education_label, ref))
reject, pvals_corr, _, _ = multipletests(pvals, alpha=config.alpha, method="holm")
m = len(pairs)
ci_alpha = config.alpha / m
rng_seed_seq = np.random.SeedSequence(config.seed)
rows = []
for i, (edu, ref) in enumerate(pairs):
seed_i = rng_seed_seq.spawn(1)[0].entropy
med_diff, med_ci = _bootstrap_diff_ci(arrays[edu], arrays[ref], np.median, ci_alpha, config.n_boot, int(seed_i))
seed_j = rng_seed_seq.spawn(1)[0].entropy
mean_diff, mean_ci = _bootstrap_diff_ci(arrays[edu], arrays[ref], np.mean, ci_alpha, config.n_boot, int(seed_j))
rows.append({
"contrast": f"{edu} vs {ref}",
"variant": variant_label,
"p_mwu_one_sided": pvals[i],
"p_mwu_holm": pvals_corr[i],
"reject_holm": bool(reject[i]),
"diff_median": med_diff,
"ci_median_lo": med_ci[0],
"ci_median_hi": med_ci[1],
"diff_mean": mean_diff,
"ci_mean_lo": mean_ci[0],
"ci_mean_hi": mean_ci[1],
"ci_familywise_alpha": ci_alpha,
"n_education": len(arrays[edu]),
"n_reference": len(arrays[ref]),
})
pairwise_df = pd.DataFrame(rows)
sizes = pd.DataFrame({
"variant": variant_label,
"category": list(arrays.keys()),
"n": [len(v) for v in arrays.values()],
"median_share": [np.median(v) if len(v)>0 else np.nan for v in arrays.values()],
"mean_share": [np.mean(v) if len(v)>0 else np.nan for v in arrays.values()],
})
return {"kw": {"H": H, "pvalue": p_kw, "k": len(groups), "variant": variant_label},
"pairwise": pairwise_df, "sizes": sizes}
def main():
p = argparse.ArgumentParser(description="H4 — Category gaps in L5–L6 share")
p.add_argument("--unweighted", type=str, default=None, help="Path to combined_with_maslow_columns_2.0.csv")
p.add_argument("--weighted", type=str, default=None, help="Path to maslow_weighted_per_object.csv")
p.add_argument("--education", type=str, default="education")
p.add_argument("--references", nargs="+", default=["play","nature","creativity"])
p.add_argument("--alpha", type=float, default=0.05)
p.add_argument("--n-boot", type=int, default=5000)
p.add_argument("--seed", type=int, default=42)
p.add_argument("--outdir", type=str, default=None, help="Directory to save outputs. Defaults to the dir of the input.")
args = p.parse_args()
cfg = H4Config(
education_label=args.education.lower(),
reference_labels=tuple([r.lower() for r in args.references]),
alpha=args.alpha,
n_boot=args.n_boot,
seed=args.seed,
)
if not args.unweighted and not args.weighted:
p.error("Please provide at least one of --unweighted or --weighted")
outdir = None
if args.outdir:
outdir = Path(args.outdir)
outdir.mkdir(parents=True, exist_ok=True)
if args.unweighted:
df_u = pd.read_csv(args.unweighted)
shares_u = compute_unweighted_l56_share(df_u)
res_u = h4_analysis(shares_u, cfg, "unweighted")
print("\nUnweighted: Kruskal–Wallis")
print(res_u["kw"])
print("\nUnweighted: Pairwise contrasts (education < reference)")
print(res_u["pairwise"].to_string(index=False))
print("\nUnweighted: Group sizes and summaries")
print(res_u["sizes"].to_string(index=False))
save_dir = outdir or Path(args.unweighted).resolve().parent
res_u["pairwise"].to_csv(save_dir / "h4_unweighted_pairwise.csv", index=False)
res_u["sizes"].to_csv(save_dir / "h4_unweighted_sizes.csv", index=False)
if args.weighted:
df_w = pd.read_csv(args.weighted)
shares_w = compute_weighted_l56_share(df_w)
res_w = h4_analysis(shares_w, cfg, "weighted")
print("\nWeighted: Kruskal–Wallis")
print(res_w["kw"])
print("\nWeighted: Pairwise contrasts (education < reference)")
print(res_w["pairwise"].to_string(index=False))
print("\nWeighted: Group sizes and summaries")
print(res_w["sizes"].to_string(index=False))
save_dir = outdir or Path(args.weighted).resolve().parent
res_w["pairwise"].to_csv(save_dir / "h4_weighted_pairwise.csv", index=False)
res_w["sizes"].to_csv(save_dir / "h4_weighted_sizes.csv", index=False)
if __name__ == "__main__":
main()