import os
from pathlib import Path
import json, math
from typing import Iterable, List, Optional, Tuple, Dict
import numpy as np
import pandas as pd
import statsmodels.formula.api as smf
from scipy.stats import spearmanr
DEFAULTS = {
"raw_maslow": "combined_with_maslow_columns_2.0.csv",
"raw_da": "all_object_da_categorie.csv",
"weighted_maslow": "maslow_weighted_per_object.csv",
"weighted_da": "objDA_weighted_per_object.csv",
"out_json": "h3_results.json",
}
ID_COL = "image"
CATEGORY_COL = "category"
MASLOW_COLS = [
"Physiological Needs","Safety Needs","Love and Belonging",
"Esteem Needs","Self-Actualization","Self-Transcendence",
]
DA_COLS = [
"Humans","Animals & Non-Human Organisms","Plants & Botanical Elements",
"Natural Landscapes & Phenomena","Built Structures & Indoor/Urban Spaces",
"Vehicles & Transportation","Food, Beverages & Tableware",
"Manual Tools & Functional Objects","Technology & Digital Media",
"Art, Sports & Recreational Objects",
]
def _ensure_cols(df: pd.DataFrame, cols: List[str], label: str):
missing = [c for c in cols if c not in df.columns]
if missing: raise ValueError(f"Missing required columns in {label}: {missing}")
if ID_COL not in df.columns or CATEGORY_COL not in df.columns:
raise ValueError(f"Missing '{ID_COL}' or '{CATEGORY_COL}' in {label}")
def _norm_category(df: pd.DataFrame):
df[CATEGORY_COL] = df[CATEGORY_COL].astype(str).str.strip().str.lower()
def _shannon_entropy(counts: Iterable[float]) -> float:
arr = np.asarray(list(counts), dtype=float); total = arr.sum()
if total <= 0: return float("nan")
p = arr/total; p = p[p>0]
return float(-(p*np.log(p)).sum())
def _l13_entropy(l1,l2,l3)->float:
c = np.array([l1,l2,l3],float); s=c.sum()
if s<=0: return float("nan")
p=c/s; p=p[p>0]
return float(-(p*np.log(p)).sum())
def _l13_maxshare(l1,l2,l3)->float:
c = np.array([l1,l2,l3],float); s=c.sum()
if s<=0: return float("nan")
return float((c/s).max())
def _raw_presence(df: pd.DataFrame, cols: List[str]) -> pd.DataFrame:
w = df.copy()
def nonempty_to_one(x):
if x is None: return 0.0
if isinstance(x,float) and math.isnan(x): return 0.0
return 1.0 if str(x).strip() else 0.0
for c in cols: w[c] = w[c].apply(nonempty_to_one).astype(float)
g = w.groupby([ID_COL, CATEGORY_COL], as_index=False).agg({c:"sum" for c in cols})
for c in cols: g[c] = (g[c] > 0).astype(float) # binary per (image,category)
return g
def _weighted_sum(df: pd.DataFrame, cols: List[str]) -> pd.DataFrame:
w = df.copy()
for c in cols: w[c] = pd.to_numeric(w[c], errors="coerce").fillna(0.0)
return w.groupby([ID_COL, CATEGORY_COL], as_index=False).agg({c:"sum" for c in cols})
def _eng_maslow(img: pd.DataFrame) -> pd.DataFrame:
df = img.copy()
df["lower_sum"] = df["Physiological Needs"]+df["Safety Needs"]+df["Love and Belonging"]
df["upper_sum"] = df["Self-Actualization"]+df["Self-Transcendence"]
df["total_sum"] = df[MASLOW_COLS].sum(axis=1)
df["L56_share"] = np.where(df["total_sum"]>0,(df["upper_sum"]/df["total_sum"]).astype(float), np.nan)
df["L13_entropy"] = df.apply(lambda r: _l13_entropy(r["Physiological Needs"],r["Safety Needs"],r["Love and Belonging"]), axis=1)
df["L13_maxshare"] = df.apply(lambda r: _l13_maxshare(r["Physiological Needs"],r["Safety Needs"],r["Love and Belonging"]), axis=1)
df = df[df["lower_sum"]>0].copy()
return df[[ID_COL, CATEGORY_COL, "L56_share","L13_entropy","L13_maxshare","total_sum"]]
def _eng_da(img: pd.DataFrame) -> pd.DataFrame:
df = img.copy()
df["da_sum"] = df[DA_COLS].sum(axis=1)
df = df[df["da_sum"]>0].copy()
df["DA_entropy"] = df[DA_COLS].apply(_shannon_entropy, axis=1)
return df[[ID_COL, CATEGORY_COL, "DA_entropy","da_sum"]]
def _merge_on_image_category(m_df: pd.DataFrame, d_df: pd.DataFrame) -> pd.DataFrame:
inter = set(zip(m_df[ID_COL].astype(str), m_df[CATEGORY_COL].astype(str))) & \
set(zip(d_df[ID_COL].astype(str), d_df[CATEGORY_COL].astype(str)))
if not inter: return m_df.iloc[0:0][[ID_COL, CATEGORY_COL]].copy()
m_mask = [(str(i),str(c)) in inter for i,c in zip(m_df[ID_COL], m_df[CATEGORY_COL])]
d_mask = [(str(i),str(c)) in inter for i,c in zip(d_df[ID_COL], d_df[CATEGORY_COL])]
m2 = m_df[m_mask].copy(); d2 = d_df[d_mask].copy()
return pd.merge(m2, d2, on=[ID_COL, CATEGORY_COL], how="inner")
def _h3a(df: pd.DataFrame) -> Dict[str,float]:
X = df.dropna(subset=["L56_share","L13_entropy","L13_maxshare"]).copy()
m1 = smf.ols(f"L13_entropy ~ L56_share + total_sum + C({CATEGORY_COL})", data=X).fit(cov_type="HC3")
m2 = smf.ols(f"L13_maxshare ~ L56_share + total_sum + C({CATEGORY_COL})", data=X).fit(cov_type="HC3")
return {
"entropy_coef_l56": float(m1.params.get("L56_share", np.nan)),
"entropy_p_l56": float(m1.pvalues.get("L56_share", np.nan)),
"maxshare_coef_l56": float(m2.params.get("L56_share", np.nan)),
"maxshare_p_l56": float(m2.pvalues.get("L56_share", np.nan)),
"n_used": int(len(X)),
}
def _h3b(df: pd.DataFrame) -> Dict[str,object]:
X = df.dropna(subset=["L56_share","DA_entropy"]).copy()
rho,p = spearmanr(X["L56_share"], X["DA_entropy"])
reg = smf.ols(f"DA_entropy ~ L56_share + total_sum + C({CATEGORY_COL})", data=X).fit(cov_type="HC3")
X["L56_share_sq"] = X["L56_share"]**2
regq = smf.ols(f"DA_entropy ~ L56_share + L56_share_sq + total_sum + C({CATEGORY_COL})", data=X).fit(cov_type="HC3")
rows=[]
for cat,g in X.groupby(CATEGORY_COL):
if len(g)>=8:
try:
m = smf.ols("DA_entropy ~ L56_share + total_sum", data=g).fit(cov_type="HC3")
rows.append({"category":cat, "slope_l56": float(m.params.get("L56_share", np.nan)), "n": int(len(g))})
except Exception:
rows.append({"category":cat, "slope_l56": float("nan"), "n": int(len(g))})
return {
"spearman_rho": float(rho) if rho is not None else float("nan"),
"spearman_p": float(p) if p is not None else float("nan"),
"linear_coef_l56": float(reg.params.get("L56_share", np.nan)),
"linear_p_l56": float(reg.pvalues.get("L56_share", np.nan)),
"quadratic_p_l56_sq": float(regq.pvalues.get("L56_share_sq", np.nan)),
"by_category": rows,
"n_used": int(len(X)),
}
def _process_pair(name:str, m_path:Optional[Path], d_path:Optional[Path], raw_mode:bool)->Tuple[Optional[dict],dict]:
if m_path is None or d_path is None:
return None, {"reason": "missing files"}
m = pd.read_csv(m_path); d = pd.read_csv(d_path)
for df in (m,d):
_ensure_cols(df, [ID_COL, CATEGORY_COL], f"{name} base")
_norm_category(df)
# Aggregate to (image,category)
if raw_mode:
_ensure_cols(m, MASLOW_COLS, f"{name} maslow"); _ensure_cols(d, DA_COLS, f"{name} da")
m_img = _raw_presence(m, MASLOW_COLS); d_img = _raw_presence(d, DA_COLS)
else:
_ensure_cols(m, MASLOW_COLS, f"{name} maslow"); _ensure_cols(d, DA_COLS, f"{name} da")
m_img = _weighted_sum(m, MASLOW_COLS); d_img = _weighted_sum(d, DA_COLS)
# Engineer features
m_eng = _eng_maslow(m_img)
d_eng = _eng_da(d_img)
# H3a on Maslow-only (NO DA intersection)
h3a = _h3a(m_eng)
# H3b on intersection (image+category)
tidy = _merge_on_image_category(m_eng, d_eng)
h3b = None
if len(tidy) > 0:
h3b = _h3b(tidy)
# Summaries
maslow_pairs = len(set(zip(m_img[ID_COL].astype(str), m_img[CATEGORY_COL].astype(str))))
da_pairs = len(set(zip(d_img[ID_COL].astype(str), d_img[CATEGORY_COL].astype(str))))
inter_pairs = len(set(zip(tidy[ID_COL].astype(str), tidy[CATEGORY_COL].astype(str)))) if len(tidy)>0 else 0
out = {
"h3a": h3a,
"h3b": h3b,
"n_pairs_maslow": int(maslow_pairs),
"n_pairs_da": int(da_pairs),
"n_pairs_intersection": int(inter_pairs),
}
return out, {
"rows_maslow_csv": int(len(m)),
"rows_da_csv": int(len(d)),
"pairs_maslow": int(maslow_pairs),
"pairs_da": int(da_pairs),
"pairs_intersection": int(inter_pairs),
"h3a_n_used": int(h3a.get("n_used", 0)),
"h3b_n_used": int(h3b.get("n_used", 0)) if h3b else 0,
}
def main():
cwd = Path(".")
files = {k: cwd / v for k,v in DEFAULTS.items() if k!="out_json"}
found = {k:p for k,p in files.items() if p.exists()}
results = {"drop_summary": {}}
raw_out, raw_drop = _process_pair("raw", found.get("raw_maslow"), found.get("raw_da"), raw_mode=True)
results["drop_summary"]["raw"] = raw_drop or {}
if raw_out is not None: results["raw"] = raw_out
w_out, w_drop = _process_pair("weighted", found.get("weighted_maslow"), found.get("weighted_da"), raw_mode=False)
results["drop_summary"]["weighted"] = w_drop or {}
if w_out is not None: results["weighted"] = w_out
with open(DEFAULTS["out_json"], "w") as f:
json.dump(results, f, indent=2)
print(f"Done. Wrote {DEFAULTS['out_json']}")
print(json.dumps(results["drop_summary"], indent=2))
if __name__ == "__main__":
main()