import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm
from statsmodels.api import OLS, add_constant
from statsmodels.stats.stattools import durbin_watson
from statsmodels.stats.diagnostic import het_breuschpagan
from statsmodels.stats.outliers_influence import OLSInfluence
from statsmodels.stats.anova import anova_lm
from statsmodels.formula.api import ols
from statsmodels.stats.multicomp import pairwise_tukeyhsd
from scipy.stats import jarque_bera, ttest_ind, f
from sklearn.model_selection import KFold, LeaveOneOut
from sklearn.utils import resample
from sklearn.linear_model import LinearRegression
from statsmodels.nonparametric.smoothers_lowess import lowess
# --- Load data ---
diversity = pd.read_csv('category_shannon_diversity.csv')
# --- Flags ---
abstract_categories = {
'danger','Connection','freedom','social_media_addiction',
'standardisation','artificial','instinct','disconnection',
'conformity','sadness'
}
diversity['abstract'] = diversity['category'].isin(abstract_categories).astype(int)
diversity['human_related'] = diversity['category'].str.contains(
r'human|Connection|play|happiness|Creativity|education|social_media',
case=False, regex=True
).astype(int)
# --- Model 1: OLS (Shannon ~ unique_objects) ---
X_single = add_constant(diversity['unique_objects'])
y = diversity['shannon_diversity']
model1 = OLS(y, X_single).fit()
residuals = model1.resid
fitted = model1.fittedvalues
influence = OLSInfluence(model1)
student_resid = influence.resid_studentized_internal
leverage = influence.hat_matrix_diag
sqrt_std_resid = np.sqrt(np.abs(student_resid))
# --- Assumption Tests ---
jb_stat, jb_p = jarque_bera(residuals)
print(f"Jarque-Bera statistic: {jb_stat:.3f}, p = {jb_p:.3f}")
dw = durbin_watson(residuals)
print(f"Durbin-Watson: {dw:.3f}")
cooks = influence.cooks_distance[0]
threshold = 4 / len(diversity)
print(f"Cook's D threshold: {threshold:.3f}")
print("Potential influential categories (Cook's D > threshold):")
print(diversity.loc[cooks > threshold, ['category', 'unique_objects', 'shannon_diversity']])
print(f"Maximum Cook's D: {cooks.max():.3f}")
bp = het_breuschpagan(residuals, X_single)
print(f"Breusch-Pagan LM={bp[0]:.2f}, p={bp[1]:.3f}")
# -------------------- DIAGNOSTIC PLOTS --------------------
# Figure 1 – Residuals vs Fitted Values (straight fit line)
plt.figure(figsize=(7, 6))
plt.scatter(residuals, fitted, color='cornflowerblue', alpha=0.8)
m, b = np.polyfit(residuals, fitted, 1)
plt.plot(residuals, m * residuals + b, color='red', lw=2)
plt.axhline(0, color='black', linestyle='--', lw=1)
plt.xlabel("Residuals")
plt.ylabel("Fitted Values")
plt.title("Figure 1: Residuals vs Fitted Values")
plt.tight_layout()
plt.savefig('fig1.png', dpi=300)
plt.close()
# Figure 2 – Q-Q Plot
fig, ax = plt.subplots(figsize=(7, 6))
sm.qqplot(residuals, line='45', ax=ax, marker='o', alpha=0.9)
ax.set_title("Figure 2: Q-Q Plot (Normality Check)")
plt.tight_layout()
plt.savefig('fig2.png', dpi=300)
plt.close()
# Figure 3 – Scale–Location Plot
plt.figure(figsize=(7, 6))
plt.scatter(fitted, sqrt_std_resid, color='mediumseagreen', alpha=0.8)
m, b = np.polyfit(fitted, sqrt_std_resid, 1)
plt.plot(fitted, m * fitted + b, color='red', lw=2)
plt.xlabel("Fitted Values")
plt.ylabel("√|Standardised Residuals|")
plt.title("Figure 3: Scale–Location Plot")
plt.tight_layout()
plt.savefig('fig3.png', dpi=300)
plt.close()
# Figure 4 – Residuals vs Leverage (with Cook’s D lines)
fig, ax = plt.subplots(figsize=(7, 6))
ax.scatter(student_resid, leverage, color='orange', alpha=0.85, edgecolor='k')
ax.axvline(0, color='grey', lw=1, ls='--')
ax.set_xlabel("Studentised Residuals")
ax.set_ylabel("Leverage")
ax.set_title("Figure 4: Residuals vs Leverage")
p = int(model1.df_model) + 1
xgrid = np.linspace(-3, 3, 300)
for d, ls in [(0.5, '--'), (1.0, ':')]:
y = d * p / (1 + xgrid**2)
y = np.clip(y, 0, 1)
ax.plot(xgrid, y, color='red', linestyle=ls, lw=1, label=f"Cook's D = {d}")
ax.set_ylim(0, max(leverage) * 1.2)
ax.legend(loc='upper right')
plt.tight_layout()
plt.savefig('fig4.png', dpi=300)
plt.close()
# -------------------- Multivariate Regression + Interaction --------------------
# Combine predictors and outcome into one frame
df_main = diversity[['unique_objects', 'object_richness_ratio', 'shannon_diversity']].dropna()
X_main = add_constant(df_main[['unique_objects', 'object_richness_ratio']])
y_main = df_main['shannon_diversity']
model2 = OLS(y_main, X_main).fit()
print("\nMultivariate Model (no interaction):")
print(model2.summary())
diversity['interaction'] = diversity['unique_objects'] * diversity['abstract']
df_int = diversity[['unique_objects', 'object_richness_ratio', 'interaction', 'shannon_diversity']].dropna()
X_int = add_constant(df_int[['unique_objects', 'object_richness_ratio', 'interaction']])
y_int = df_int['shannon_diversity']
model3 = OLS(y_int, X_int).fit()
print("\nMultivariate Model (with interaction):")
print(model3.summary())
# ΔR² comparison
rss2 = sum(model2.resid ** 2)
rss3 = sum(model3.resid ** 2)
df2 = model2.df_resid
df3 = model3.df_resid
F_val = ((rss2 - rss3) / (df2 - df3)) / (rss3 / df3)
p_val = 1 - f.cdf(F_val, df2 - df3, df3)
delta_r2 = model3.rsquared - model2.rsquared
print(f"\nΔR² = {delta_r2:.4f}, F = {F_val:.2f}, p = {p_val:.3f}")
# -------------------- Model Validation --------------------
X_sk = diversity[['unique_objects', 'object_richness_ratio']].values
y_sk = diversity['shannon_diversity'].values
kf = KFold(n_splits=5, shuffle=True, random_state=1)
cv_r2 = []
for train, test in kf.split(X_sk):
mdl = LinearRegression().fit(X_sk[train], y_sk[train])
cv_r2.append(mdl.score(X_sk[test], y_sk[test]))
print(f"5-fold CV: Mean R² = {np.mean(cv_r2):.3f}, SD = {np.std(cv_r2):.3f}")
loo = LeaveOneOut()
mae = []
max_err = 0
for train, test in loo.split(X_sk):
mdl = LinearRegression().fit(X_sk[train], y_sk[train])
pred = mdl.predict(X_sk[test])
err = abs(pred[0] - y_sk[test][0])
mae.append(err)
if err > max_err:
max_err = err
print(f"LOOCV: Mean absolute error = {np.mean(mae):.3f}, Max error = {max_err:.3f}")
boot_r2 = []
for _ in range(1000):
Xb, yb = resample(X_sk, y_sk)
mdl = LinearRegression().fit(Xb, yb)
boot_r2.append(mdl.score(X_sk, y_sk))
boot_ci = np.percentile(boot_r2, [2.5, 97.5])
print(f"Bootstrap R² = {np.mean(boot_r2):.3f} (95% CI [{boot_ci[0]:.3f}, {boot_ci[1]:.3f}])")
# -------------------- ANOVA & Contrasts --------------------
diversity['quartile'] = pd.qcut(diversity['shannon_diversity'], 4, labels=False)
print("\nANOVA: unique_objects by diversity quartile")
model_anova = ols('unique_objects ~ C(quartile)', data=diversity).fit()
print(anova_lm(model_anova, typ=2))
print("Tukey HSD (unique_objects):")
print(pairwise_tukeyhsd(diversity['unique_objects'], diversity['quartile']))
print("\nANOVA: object_richness_ratio by diversity quartile")
model_anova2 = ols('object_richness_ratio ~ C(quartile)', data=diversity).fit()
print(anova_lm(model_anova2, typ=2))
print("Tukey HSD (richness ratio):")
print(pairwise_tukeyhsd(diversity['object_richness_ratio'], diversity['quartile']))
# -------------------- Planned Contrasts --------------------
t1, p1 = ttest_ind(
diversity[diversity['abstract'] == 1]['shannon_diversity'],
diversity[diversity['abstract'] == 0]['shannon_diversity'],
equal_var=False
)
print(f"\nAbstract vs Concrete: t = {t1:.2f}, p = {p1:.3f}")
t2, p2 = ttest_ind(
diversity[diversity['human_related'] == 1]['shannon_diversity'],
diversity[diversity['human_related'] == 0]['shannon_diversity'],
equal_var=False
)
print(f"Human vs Non-Human: t = {t2:.2f}, p = {p2:.3f}")
print("\nScript complete. All assumptions, diagnostics, validations, and models complete.")