分布分析の完全ガイド — データの分布を正しく理解し、前処理に活かす

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

fig, axes = plt.subplots(2, 3, figsize=(15, 10))
numeric_cols = df.select_dtypes(include=["number"]).columns[:6]

for ax, col in zip(axes.flat, numeric_cols):
    data = df[col].dropna()

    # ヒストグラム + KDE
    sns.histplot(data, kde=True, ax=ax, stat="density", alpha=0.6)

    # 平均と中央値のライン
    ax.axvline(data.mean(), color="red", linestyle="--",
              label=f"平均: {data.mean():.2f}")
    ax.axvline(data.median(), color="green", linestyle="-.",
              label=f"中央値: {data.median():.2f}")

    ax.set_title(f"{col} (歪度: {data.skew():.2f})")
    ax.legend(fontsize=8)

plt.suptitle("各カラムのヒストグラム + KDE", fontsize=14)
plt.tight_layout()
plt.savefig("histogram_kde.png", dpi=150)
plt.show()

from scipy import stats
import matplotlib.pyplot as plt

fig, axes = plt.subplots(2, 3, figsize=(15, 10))
numeric_cols = df.select_dtypes(include=["number"]).columns[:6]

for ax, col in zip(axes.flat, numeric_cols):
    data = df[col].dropna()

    # Q-Qプロット
    stats.probplot(data, dist="norm", plot=ax)
    ax.set_title(f"Q-Q Plot: {col}")
    ax.get_lines()[0].set_markerfacecolor("steelblue")
    ax.get_lines()[0].set_markersize(3)
    ax.get_lines()[1].set_color("red")

plt.suptitle("正規Q-Qプロット", fontsize=14)
plt.tight_layout()
plt.savefig("qq_plots.png", dpi=150)
plt.show()

from scipy import stats
import pandas as pd

def normality_test_suite(data: pd.Series) -> pd.DataFrame:
    """4つの正規性検定を一括実行する。"""
    clean = data.dropna()
    results = []

    # 1. Shapiro-Wilk
    if len(clean) <= 5000:
        stat, pval = stats.shapiro(clean)
        results.append({"検定名": "Shapiro-Wilk", "統計量": stat, "p値": pval})

    # 2. Anderson-Darling
    ad_result = stats.anderson(clean, dist="norm")
    results.append({
        "検定名": "Anderson-Darling",
        "統計量": ad_result.statistic,
        "p値": None,  # 臨界値で判断
    })

    # 3. Kolmogorov-Smirnov
    stat, pval = stats.kstest(clean, "norm",
                              args=(clean.mean(), clean.std()))
    results.append({"検定名": "Kolmogorov-Smirnov", "統計量": stat, "p値": pval})

    # 4. D'Agostino-Pearson
    if len(clean) >= 20:
        stat, pval = stats.normaltest(clean)
        results.append({"検定名": "D'Agostino-Pearson", "統計量": stat, "p値": pval})

    return pd.DataFrame(results)

for col in df.select_dtypes(include=["number"]).columns:
    print(f"\n=== {col} ===")
    result = normality_test_suite(df[col])
    print(result.to_string(index=False))

from scipy import stats
import numpy as np
import pandas as pd

def fit_distributions(data: pd.Series) -> pd.DataFrame:
    """複数の理論分布をフィッティングし、AIC/BICで比較する。"""
    clean = data.dropna().values
    n = len(clean)
    candidates = {
        "正規分布": stats.norm,
        "対数正規分布": stats.lognorm,
        "指数分布": stats.expon,
        "ガンマ分布": stats.gamma,
        "ベータ分布": stats.beta,
    }

    results = []
    for name, dist in candidates.items():
        try:
            # 対数正規・ガンマ・ベータは正の値が必要
            if name in ["対数正規分布", "ガンマ分布", "ベータ分布"]:
                if clean.min() <= 0:
                    continue
            if name == "ベータ分布":
                if clean.max() >= 1 or clean.min() <= 0:
                    continue

            params = dist.fit(clean)
            k = len(params)  # パラメータ数
            log_lik = np.sum(dist.logpdf(clean, *params))
            aic = 2 * k - 2 * log_lik
            bic = k * np.log(n) - 2 * log_lik

            # KS検定で適合度を評価
            ks_stat, ks_pval = stats.kstest(clean, dist.cdf, args=params)

            results.append({
                "分布": name,
                "AIC": round(aic, 2),
                "BIC": round(bic, 2),
                "KS統計量": round(ks_stat, 4),
                "KS p値": round(ks_pval, 4),
            })
        except Exception:
            continue

    return pd.DataFrame(results).sort_values("AIC")

for col in df.select_dtypes(include=["number"]).columns[:3]:
    print(f"\n=== {col} の分布フィッティング ===")
    result = fit_distributions(df[col])
    print(result.to_string(index=False))

from scipy import stats
import numpy as np
import pandas as pd

# 右に歪んだ分布に対する変換の比較
col = "price"  # 例: 右に歪んだカラム
data = df[col].dropna()

# 1. 対数変換
log_data = np.log1p(data)  # log(x + 1) で 0 を含むデータに対応

# 2. Box-Cox 変換（正の値のみ）
if data.min() > 0:
    boxcox_data, lambda_opt = stats.boxcox(data)
    print(f"Box-Cox 最適 λ: {lambda_opt:.4f}")

# 変換前後の歪度を比較
print(f"\n歪度の変化:")
print(f"  変換前:     {data.skew():.4f}")
print(f"  対数変換後: {log_data.skew():.4f}")
if data.min() > 0:
    print(f"  Box-Cox後:  {pd.Series(boxcox_data).skew():.4f}")

# 変換後の正規性検定
print(f"\nShapiro-Wilk 検定 (p値):")
print(f"  変換前:     {stats.shapiro(data[:5000])[1]:.6f}")
print(f"  対数変換後: {stats.shapiro(log_data[:5000])[1]:.6f}")
if data.min() > 0:
    print(f"  Box-Cox後:  {stats.shapiro(boxcox_data[:5000])[1]:.6f}")