Select Page

Correlation Analysis and Hypothesis Testing Code 

import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm
from statsmodels.api import OLS, add_constant
from statsmodels.stats.stattools import durbin_watson
from statsmodels.stats.diagnostic import het_breuschpagan
from statsmodels.stats.outliers_influence import OLSInfluence
from statsmodels.stats.anova import anova_lm
from statsmodels.formula.api import ols
from statsmodels.stats.multicomp import pairwise_tukeyhsd
from scipy.stats import jarque_bera, ttest_ind, f
from sklearn.model_selection import KFold, LeaveOneOut
from sklearn.utils import resample
from sklearn.linear_model import LinearRegression
from statsmodels.nonparametric.smoothers_lowess import lowess

# --- Load data ---
diversity = pd.read_csv('category_shannon_diversity.csv')

# --- Flags ---
abstract_categories = {
    'danger','Connection','freedom','social_media_addiction',
    'standardisation','artificial','instinct','disconnection',
    'conformity','sadness'
}
diversity['abstract'] = diversity['category'].isin(abstract_categories).astype(int)
diversity['human_related'] = diversity['category'].str.contains(
    r'human|Connection|play|happiness|Creativity|education|social_media',
    case=False, regex=True
).astype(int)

# --- Model 1: OLS (Shannon ~ unique_objects) ---
X_single = add_constant(diversity['unique_objects'])
y = diversity['shannon_diversity']
model1 = OLS(y, X_single).fit()
residuals = model1.resid
fitted = model1.fittedvalues
influence = OLSInfluence(model1)
student_resid = influence.resid_studentized_internal
leverage = influence.hat_matrix_diag
sqrt_std_resid = np.sqrt(np.abs(student_resid))

# --- Assumption Tests ---
jb_stat, jb_p = jarque_bera(residuals)
print(f"Jarque-Bera statistic: {jb_stat:.3f}, p = {jb_p:.3f}")
dw = durbin_watson(residuals)
print(f"Durbin-Watson: {dw:.3f}")
cooks = influence.cooks_distance[0]
threshold = 4 / len(diversity)
print(f"Cook's D threshold: {threshold:.3f}")
print("Potential influential categories (Cook's D > threshold):")
print(diversity.loc[cooks > threshold, ['category', 'unique_objects', 'shannon_diversity']])
print(f"Maximum Cook's D: {cooks.max():.3f}")
bp = het_breuschpagan(residuals, X_single)
print(f"Breusch-Pagan LM={bp[0]:.2f}, p={bp[1]:.3f}")

# -------------------- DIAGNOSTIC PLOTS --------------------

# Figure 1 – Residuals vs Fitted Values (straight fit line)
plt.figure(figsize=(7, 6))
plt.scatter(residuals, fitted, color='cornflowerblue', alpha=0.8)
m, b = np.polyfit(residuals, fitted, 1)
plt.plot(residuals, m * residuals + b, color='red', lw=2)
plt.axhline(0, color='black', linestyle='--', lw=1)
plt.xlabel("Residuals")
plt.ylabel("Fitted Values")
plt.title("Figure 1: Residuals vs Fitted Values")
plt.tight_layout()
plt.savefig('fig1.png', dpi=300)
plt.close()

# Figure 2 – Q-Q Plot
fig, ax = plt.subplots(figsize=(7, 6))
sm.qqplot(residuals, line='45', ax=ax, marker='o', alpha=0.9)
ax.set_title("Figure 2: Q-Q Plot (Normality Check)")
plt.tight_layout()
plt.savefig('fig2.png', dpi=300)
plt.close()

# Figure 3 – Scale–Location Plot
plt.figure(figsize=(7, 6))
plt.scatter(fitted, sqrt_std_resid, color='mediumseagreen', alpha=0.8)
m, b = np.polyfit(fitted, sqrt_std_resid, 1)
plt.plot(fitted, m * fitted + b, color='red', lw=2)
plt.xlabel("Fitted Values")
plt.ylabel("√|Standardised Residuals|")
plt.title("Figure 3: Scale–Location Plot")
plt.tight_layout()
plt.savefig('fig3.png', dpi=300)
plt.close()

# Figure 4 – Residuals vs Leverage (with Cook’s D lines)
fig, ax = plt.subplots(figsize=(7, 6))
ax.scatter(student_resid, leverage, color='orange', alpha=0.85, edgecolor='k')
ax.axvline(0, color='grey', lw=1, ls='--')
ax.set_xlabel("Studentised Residuals")
ax.set_ylabel("Leverage")
ax.set_title("Figure 4: Residuals vs Leverage")
p = int(model1.df_model) + 1
xgrid = np.linspace(-3, 3, 300)
for d, ls in [(0.5, '--'), (1.0, ':')]:
    y = d * p / (1 + xgrid**2)
    y = np.clip(y, 0, 1)
    ax.plot(xgrid, y, color='red', linestyle=ls, lw=1, label=f"Cook's D = {d}")
ax.set_ylim(0, max(leverage) * 1.2)
ax.legend(loc='upper right')
plt.tight_layout()
plt.savefig('fig4.png', dpi=300)
plt.close()

# -------------------- Multivariate Regression + Interaction --------------------

# Combine predictors and outcome into one frame
df_main = diversity[['unique_objects', 'object_richness_ratio', 'shannon_diversity']].dropna()

X_main = add_constant(df_main[['unique_objects', 'object_richness_ratio']])
y_main = df_main['shannon_diversity']

model2 = OLS(y_main, X_main).fit()

print("\nMultivariate Model (no interaction):")
print(model2.summary())

diversity['interaction'] = diversity['unique_objects'] * diversity['abstract']
df_int = diversity[['unique_objects', 'object_richness_ratio', 'interaction', 'shannon_diversity']].dropna()
X_int = add_constant(df_int[['unique_objects', 'object_richness_ratio', 'interaction']])
y_int = df_int['shannon_diversity']
model3 = OLS(y_int, X_int).fit()

print("\nMultivariate Model (with interaction):")
print(model3.summary())

# ΔR² comparison
rss2 = sum(model2.resid ** 2)
rss3 = sum(model3.resid ** 2)
df2 = model2.df_resid
df3 = model3.df_resid
F_val = ((rss2 - rss3) / (df2 - df3)) / (rss3 / df3)
p_val = 1 - f.cdf(F_val, df2 - df3, df3)
delta_r2 = model3.rsquared - model2.rsquared
print(f"\nΔR² = {delta_r2:.4f}, F = {F_val:.2f}, p = {p_val:.3f}")

# -------------------- Model Validation --------------------
X_sk = diversity[['unique_objects', 'object_richness_ratio']].values
y_sk = diversity['shannon_diversity'].values


kf = KFold(n_splits=5, shuffle=True, random_state=1)
cv_r2 = []
for train, test in kf.split(X_sk):
    mdl = LinearRegression().fit(X_sk[train], y_sk[train])
    cv_r2.append(mdl.score(X_sk[test], y_sk[test]))
print(f"5-fold CV: Mean R² = {np.mean(cv_r2):.3f}, SD = {np.std(cv_r2):.3f}")

loo = LeaveOneOut()
mae = []
max_err = 0
for train, test in loo.split(X_sk):
    mdl = LinearRegression().fit(X_sk[train], y_sk[train])
    pred = mdl.predict(X_sk[test])
    err = abs(pred[0] - y_sk[test][0])
    mae.append(err)
    if err > max_err:
        max_err = err
print(f"LOOCV: Mean absolute error = {np.mean(mae):.3f}, Max error = {max_err:.3f}")

boot_r2 = []
for _ in range(1000):
    Xb, yb = resample(X_sk, y_sk)
    mdl = LinearRegression().fit(Xb, yb)
    boot_r2.append(mdl.score(X_sk, y_sk))
boot_ci = np.percentile(boot_r2, [2.5, 97.5])
print(f"Bootstrap R² = {np.mean(boot_r2):.3f} (95% CI [{boot_ci[0]:.3f}, {boot_ci[1]:.3f}])")

# -------------------- ANOVA & Contrasts --------------------

diversity['quartile'] = pd.qcut(diversity['shannon_diversity'], 4, labels=False)

print("\nANOVA: unique_objects by diversity quartile")
model_anova = ols('unique_objects ~ C(quartile)', data=diversity).fit()
print(anova_lm(model_anova, typ=2))
print("Tukey HSD (unique_objects):")
print(pairwise_tukeyhsd(diversity['unique_objects'], diversity['quartile']))

print("\nANOVA: object_richness_ratio by diversity quartile")
model_anova2 = ols('object_richness_ratio ~ C(quartile)', data=diversity).fit()
print(anova_lm(model_anova2, typ=2))
print("Tukey HSD (richness ratio):")
print(pairwise_tukeyhsd(diversity['object_richness_ratio'], diversity['quartile']))

# -------------------- Planned Contrasts --------------------
t1, p1 = ttest_ind(
    diversity[diversity['abstract'] == 1]['shannon_diversity'],
    diversity[diversity['abstract'] == 0]['shannon_diversity'],
    equal_var=False
)
print(f"\nAbstract vs Concrete: t = {t1:.2f}, p = {p1:.3f}")

t2, p2 = ttest_ind(
    diversity[diversity['human_related'] == 1]['shannon_diversity'],
    diversity[diversity['human_related'] == 0]['shannon_diversity'],
    equal_var=False
)
print(f"Human vs Non-Human: t = {t2:.2f}, p = {p2:.3f}")

print("\nScript complete. All assumptions, diagnostics, validations, and models complete.")

←→