数据分析技术训练营

import seaborn as sns
import matplotlib.pyplot as plt

sns.set_theme(style='darkgrid')
sns.set_palette('husl')

# 可用主题: darkgrid, whitegrid, dark, white, ticks
# 常用调色板: deep, muted, bright, pastel, dark, colorblind

import numpy as np

np.random.seed(42)
data = np.random.randn(200)

# 分布图
sns.histplot(data, bins=25, kde=True, color='steelblue')
plt.title('数据分布（含核密度曲线）')
plt.show()

# 箱线图
sns.boxplot(data=data, orient='h', width=0.3)
plt.title('箱线图')
plt.show()

import pandas as pd

df = pd.DataFrame({
    '年龄': np.random.randint(20, 60, 100),
    '工资': np.random.randint(3000, 25000, 100),
    '经验年数': np.random.randint(1, 30, 100),
    '绩效分': np.random.uniform(60, 100, 100)
})

# 相关系数热力图
corr = df.corr()
plt.figure(figsize=(8, 6))
sns.heatmap(corr, annot=True, cmap='coolwarm', fmt='.2f',
            linewidths=0.5, square=True)
plt.title('特征相关系数热力图')
plt.show()

# 创建分类数据
tips = pd.DataFrame({
    '性别': np.random.choice(['男', '女'], 100),
    '星期': np.random.choice(['周一','周二','周三','周四','周五'], 100),
    '消费额': np.random.randint(50, 500, 100),
    '小费': np.random.randint(5, 80, 100)
})

# 分类计数
sns.countplot(data=tips, x='星期', hue='性别')
plt.title('各星期顾客性别分布')
plt.show()

# 分组柱状图
sns.barplot(data=tips, x='性别', y='消费额', hue='星期')
plt.title('性别与星期的消费额对比')
plt.show()

import pandas as pd
import numpy as np

df = pd.read_csv('data.csv')
print(df.info())
print(df.describe().round(2))
print("缺失值:\n", df.isnull().sum())

# 分类变量
print(df['类别'].value_counts())
print(df['类别'].value_counts(normalize=True).round(3))  # 占比

# 数值变量
print(f"偏度: {df['价格'].skew():.3f}")  # 0=对称, >0右偏, <0左偏
print(f"峰度: {df['价格'].kurtosis():.3f}")  # 峰度 >0 尖峰, <0 平峰

# 相关系数
corr = df.select_dtypes(include=[np.number]).corr()
print(corr['目标列'].sort_values(ascending=False))

# 交叉表
ct = pd.crosstab(df['地区'], df['产品类型'], margins=True, normalize='index')
print(ct.round(3))

# 缺失值热力图（列维度）
missing_pct = (df.isnull().sum() / len(df)).sort_values(ascending=False)
print("缺失率>10%的列:")
print(missing_pct[missing_pct > 0.1])

# IQR 异常值检测
def find_outliers(df, col):
    Q1, Q3 = df[col].quantile([0.25, 0.75])
    IQR = Q3 - Q1
    lower = Q1 - 1.5 * IQR
    upper = Q3 + 1.5 * IQR
    outliers = df[(df[col] < lower) | (df[col] > upper)]
    return len(outliers), lower, upper

import numpy as np

data = np.array([12, 15, 14, 18, 20, 25, 16, 13, 22, 200])
print(f"均值: {data.mean():.2f}")
print(f"中位数: {np.median(data):.2f}")
print(f"标准差: {data.std():.2f}")
print(f"变异系数(CV): {data.std()/data.mean():.3f}")

# 验证中心极限定理
np.random.seed(42)
population = np.random.exponential(scale=3, size=100000)  # 指数分布（严重偏斜）

sample_means = []
for _ in range(1000):
    sample = np.random.choice(population, size=30)
    sample_means.append(sample.mean())

sample_means = np.array(sample_means)
print(f"样本均值分布: 均值={sample_means.mean():.3f}, 标准差={sample_means.std():.3f}")
print("原始总体均值:", population.mean())

from scipy import stats

# 两组样本
group_a = np.array([85, 92, 78, 88, 95, 83, 90, 87])
group_b = np.array([72, 68, 75, 80, 71, 65, 78, 74])

t_stat, p_value = stats.ttest_ind(group_a, group_b)
print(f"t 统计量: {t_stat:.4f}")
print(f"p 值: {p_value:.6f}")
print(f"A组均值: {group_a.mean():.2f}, B组均值: {group_b.mean():.2f}")
if p_value < 0.05:
    print("结论: 两组有显著差异 (p<0.05)")
else:
    print("结论: 无统计学显著差异")

# A/B 测试模拟
np.random.seed(42)
n_A, n_B = 500, 500
conv_A = np.random.binomial(1, 0.12, n_A)  # 转化率 12%
conv_B = np.random.binomial(1, 0.15, n_B)  # 转化率 15%

conv_rate_A = conv_A.mean()
conv_rate_B = conv_B.mean()
lift = (conv_rate_B - conv_rate_A) / conv_rate_A

print(f"A 组转化率: {conv_rate_A:.2%}")
print(f"B 组转化率: {conv_rate_B:.2%}")
print(f"相对提升: {lift:.2%}")
print(f"样本量: 每组 {n_A} 人")

import numpy as np

# 创建特征矩阵 (n=100, 3个特征)
np.random.seed(42)
X = np.random.randn(100, 3)
true_w = np.array([3.5, -2.0, 1.8])
true_b = 5.0
y = X @ true_w + true_b + np.random.randn(100) * 0.5

# 正规方程
X_b = np.c_[np.ones((100, 1)), X]  # 加偏置列
w_opt = np.linalg.inv(X_b.T @ X_b) @ X_b.T @ y
print("真实系数:", np.concatenate([[true_b], true_w]))
print("估计系数:", w_opt.round(4))

def poly_features(x, degree):
    return np.column_stack([x**i for i in range(degree+1)])

np.random.seed(42)
x = np.random.uniform(-3, 3, 30)
y_true = np.sin(x) + np.random.normal(0, 0.2, 30)

# 不同多项式次数
for deg in [1, 3, 15]:
    X_poly = poly_features(x, deg)
    w = np.linalg.inv(X_poly.T @ X_poly) @ X_poly.T @ y_true
    y_pred = X_poly @ w
    mse = ((y_true - y_pred)**2).mean()
    print(f"degree={deg:2d}, MSE={mse:.6f}, 参数数量={deg+1}")

def train_test_split(X, y, test_size=0.2, seed=42):
    np.random.seed(seed)
    n = len(X)
    idx = np.random.permutation(n)
    split = int(n * (1 - test_size))
    train_idx, test_idx = idx[:split], idx[split:]
    return X[train_idx], X[test_idx], y[train_idx], y[test_idx]

X_train, X_test, y_train, y_test = train_test_split(X, y)
print(f"训练集: {len(X_train)} 样本, 测试集: {len(X_test)} 样本")

# 岭回归（L2正则化）
def ridge_regression(X, y, lambda_=1.0):
    n_features = X.shape[1]
    # (XᵀX + λI)⁻¹Xᵀy
    w = np.linalg.inv(X.T @ X + lambda_ * np.eye(n_features)) @ X.T @ y
    return w

w_ridge = ridge_regression(X_poly, y_true, lambda_=10)
y_ridge = X_poly @ w_ridge
print(f"岭回归 MSE: {((y_true-y_ridge)**2).mean():.6f}")
print(f"系数均值大小: {np.abs(w_ridge).mean():.4f}")

import numpy as np
# 混淆矩阵元素
y_true = np.array([1,0,1,1,0,1,0,0])
y_pred = np.array([1,0,1,0,0,1,1,0])
TP = ((y_pred==1) & (y_true==1)).sum()
FP = ((y_pred==1) & (y_true==0)).sum()
FN = ((y_pred==0) & (y_true==1)).sum()
TN = ((y_pred==0) & (y_true==0)).sum()
print(f"TP={TP} FP={FP} FN={FN} TN={TN}")
precision = TP/(TP+FP) if TP+FP else 0
recall = TP/(TP+FN) if TP+FN else 0
F1 = 2*precision*recall/(precision+recall) if precision+recall else 0
print(f"精确率={precision:.2%} 召回率={recall:.2%} F1={F1:.2%}")

import numpy as np
def sigmoid(z): return 1/(1+np.exp(-z))
np.random.seed(42)
X = np.random.randn(20, 2)
# 模拟逻辑回归预测
z = 1.5*X[:,0] - 0.8*X[:,1] + 0.2
probs = sigmoid(z)
preds = (probs >= 0.5).astype(int)
print("预测概率前5个:", probs[:5].round(3))
print("预测类别前5个:", preds[:5])

import numpy as np
# 模拟集成投票：3个弱分类器
np.random.seed(42)
n_samples = 10
true_labels = np.array([0,1,0,1,0,1,1,0,1,0])
# 3个分类器各自预测
clf1 = np.array([0,1,0,1,0,1,1,0,0,0])  # 90%准确
clf2 = np.array([0,0,0,1,0,1,1,1,1,0])  # 80%准确
clf3 = np.array([1,1,0,1,0,1,0,0,1,0])  # 70%准确
# 多数投票
all_votes = np.vstack([clf1, clf2, clf3])
ensemble_pred = np.array([np.bincount(all_votes[:, i]).argmax() for i in range(n_samples)])
acc_ensemble = (ensemble_pred == true_labels).mean()
print(f"集成投票准确率: {acc_ensemble:.2%}")
print(f"单个最高准确率: {(clf1==true_labels).mean():.2%}")

import numpy as np
# 构造含趋势和季节性的时间序列
np.random.seed(42)
t = np.arange(60)
trend = 0.1 * t                 # 上升趋势
seasonal = 5 * np.sin(2*np.pi*t/12)  # 12期季节性
noise = np.random.normal(0, 1, 60)  # 随机噪声
ts = trend + seasonal + noise
print("时间序列前12个值:", ts[:12].round(2))
print(f"趋势分量范围: [{trend.min():.2f}, {trend.max():.2f}]")

import numpy as np
np.random.seed(42)
sales = np.cumsum(np.random.randn(24)*5) + 100
# 3期移动平均
window = 3
weights = np.ones(window) / window
ma_3 = np.convolve(sales, weights, mode='valid')
# 简单指数平滑
alpha = 0.3
ewma = np.zeros_like(sales)
ewma[0] = sales[0]
for i in range(1, len(sales)):
    ewma[i] = alpha*sales[i] + (1-alpha)*ewma[i-1]
print("原始vs平滑（后10个）:")
for i in range(-10, 0):
    print(f"  {i+25:2d}: {sales[i]:8.2f} | 平滑:{ewma[i]:.2f}")

import numpy as np
np.random.seed(42)
ts = np.cumsum(np.random.randn(100))  # 非平稳（随机游走）
# 一阶差分使其平稳
ts_diff = np.diff(ts)
# 自相关计算（滞后1）
lag = 1
y = ts_diff[lag:]
y_lag = ts_diff[:-lag]
corr = np.corrcoef(y, y_lag)[0, 1]
print(f"差分后自相关（滞后1）: {corr:.4f}")
print(f"差分后均值: {ts_diff.mean():.4f}（接近0表示平稳）")

import numpy as np
text = "数据科学是一门非常有趣和有用的学科"
# 简单分词
words = text.split()
print("分词结果:", words)
# 去除停用词
stop_words = {'的', '和', '非常'}
filtered = [w for w in words if w not in stop_words]
print("去停用词后:", filtered)
print(f"词汇数量: {len(filtered)}")

import numpy as np
docs = ["我喜欢数据科学", "数据科学很有趣", "我喜欢编程"]
# 构建词袋
all_words = sorted(set(' '.join(docs).split()))
vocab = {w:i for i,w in enumerate(all_words)}
bow = np.zeros((len(docs), len(vocab)), dtype=int)
for i, doc in enumerate(docs):
    for w in doc.split():
        bow[i, vocab[w]] += 1
print("词袋矩阵:\n", bow)
print("词汇表:", list(vocab.keys()))

import numpy as np
# 简单情感词典分析
positive_words = {'好','喜欢','优秀','棒','有趣','有用'}
negative_words = {'差','讨厌','糟糕','无聊','没用'}
reviews = ["这个产品非常好 我很喜欢", "质量很差 非常糟糕", "很有趣也很有用 很棒"]
for i, review in enumerate(reviews):
    words = review.split()
    pos = sum(1 for w in words if w in positive_words)
    neg = sum(1 for w in words if w in negative_words)
    sentiment = "正向" if pos > neg else ("负向" if neg > pos else "中立")
    print(f"评论{i+1}: 正向词={pos} 负向词={neg} 情感={sentiment}")

import numpy as np
import pandas as pd
# 工作流概览
steps = ["问题定义", "数据收集", "数据清洗", "EDA探索",
         "特征工程", "建模", "评估", "结论"]
for i, step in enumerate(steps, 1):
    print(f"Step {i}: {step}")

import pandas as pd
import numpy as np
np.random.seed(42)
df = pd.DataFrame({'年龄': np.random.randint(18,65,100),
    '收入': np.random.normal(8000,3000,100).round(0),
    '消费金额': np.random.normal(500,200,100).round(0)})
df.loc[0:4, '收入'] = np.nan
print("缺失值:\n", df.isnull().sum())
df['收入'] = df['收入'].fillna(df['收入'].median())
print("描述统计:\n", df.describe())
print("相关系数:\n", df.corr().round(3))

import numpy as np
# 模拟训练/测试划分与评估
np.random.seed(42)
X = np.random.rand(100, 3)
y = 3*X[:,0] + 1.5*X[:,1] - 2*X[:,2] + np.random.normal(0, 0.2, 100)
split = 80
X_train, X_test = X[:split], X[split:]
y_train, y_test = y[:split], y[split:]
# 线性回归（正规方程）
X_train_b = np.c_[np.ones(split), X_train]
theta = np.linalg.inv(X_train_b.T @ X_train_b) @ X_train_b.T @ y_train
X_test_b = np.c_[np.ones(20), X_test]
y_pred = X_test_b @ theta
mse = ((y_test - y_pred)**2).mean()
r2 = 1 - ((y_test - y_pred)**2).sum() / ((y_test - y_test.mean())**2).sum()
print(f"MSE: {mse:.4f}, R²: {r2:.4f}")