Logistic Regression 是統(tǒng)計學(xué)習(xí)的經(jīng)典分類算法，是一種 對數(shù)線性模型。
1、數(shù)據(jù)集與特征空間
2、假設(shè)空間
Logistic Regression 的假設(shè)函數(shù)由 對數(shù)幾率（log odds） 假設(shè)推導(dǎo)而來。
圖示：Sigmoid函數(shù)及其導(dǎo)數(shù)
3、目標(biāo)函數(shù)及其推導(dǎo)
從極大似然估計的角度推出 Logistic Regression 的目標(biāo)函數(shù)。
4、優(yōu)化算法（梯度下降）
梯度向量
這里考慮兩種方式：一種是基本的梯度下降，一種是只針對梯度的單位向量進行下降剔除其模長
梯度下降由泰勒一階展開推導(dǎo)而來，同理，可以由二階展開推出牛頓法以及擬牛頓法，此處不予贅述
在實際應(yīng)用中，使用邏輯斯回歸建議進行如下處理：
1、特征標(biāo)準化（z-scale），使所有特征的尺度縮放到 [-1,+1] 之間，避免出現(xiàn)鋸齒效應(yīng)，影響收斂速度。
2、考慮加入動量（Momentum），抑制震蕩，加快收斂速度。
手寫算法并與 Sklearn 進行對比
# -*- coding: utf-8 -*-
import numpy as np
import pandas as pd
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
class LogitRegModel(object):
def __init__(self, max_iter=5000, eta=0.01, alpha=0.5, beta=0.9):
self.max_iter = max_iter
self.eta = eta
self.alpha = alpha
self.beta = beta
def z_scale(self, x_train):
'''z標(biāo)準化，在動用距離度量的算法中，必須先進行標(biāo)準化以消除數(shù)據(jù)量綱的影響'''
mu = np.mean(x_train, axis=0)
std = np.std(x_train, axis=0)
return mu, std
def data_transform(self, mu, std, x_train, x_test):
'''
數(shù)據(jù)變換
1、執(zhí)行標(biāo)準化操作
2、插入截距項
'''
x_train_scale = (x_train - mu) / std
x_test_scale = (x_test - mu) / std
intercept_train = np.ones(x_train_scale.shape[0]).reshape(-1, 1)
intercept_test = np.ones(x_test_scale.shape[0]).reshape(-1, 1)
x_train_scale = np.concatenate([intercept_train, x_train_scale], axis=1)
x_test_scale = np.concatenate([intercept_test, x_test_scale], axis=1)
return x_train_scale, x_test_scale
def get_loss(self, x_train_scale, y_train, w):
'''計算損失函數(shù)值'''
loss = np.mean(np.log(1.0 + np.exp(-x_train_scale.dot(w) * y_train)))
return loss
def get_derivative(self, x_train_scale, y_train, w, dv):
'''計算梯度(含動量, beta = 0 則為原始梯度下降)'''
fenzi = -y_train * x_train_scale
fenmu = 1.0 + np.exp(x_train_scale.dot(w) * y_train)
dw = np.mean(fenzi / fenmu, axis=0)
dw = dw.reshape(-1, 1)
dv = self.beta * dv + (1 - self.beta) * dw
return dv
def fit(self, x_train_scale, y_train):
'''模型訓(xùn)練'''
# 參數(shù)初始化
w = np.zeros(x_train_scale.shape[1]) + 0.001
w = w.reshape(-1, 1)
dv = np.zeros_like(w)
# 損失值保存列表
loss_res = []
# 迭代
for epoch in range(self.max_iter):
# 計算梯度
dv = self.get_derivative(x_train_scale, y_train, w, dv)
# 梯度下降
w = w - self.eta * dv
# 更新?lián)p失值
loss = self.get_loss(x_train_scale, y_train, w)
loss_res.append(loss)
return w, loss_res
def predict(self, x_test_scale, w):
'''模型預(yù)測'''
y_pred_probs = 1.0 / (1.0 + np.exp(-x_test_scale.dot(w)))
y_pred = np.where(y_pred_probs > self.alpha, 1, -1)
return y_pred_probs, y_pred
def get_score(self, y_true, y_pred):
'''模型評估'''
score = sum(y_true == y_pred) / len(y_true)
return score
if __name__ == "__main__":
# 構(gòu)造二分類數(shù)據(jù)集
N = 200; n = 4
x1 = np.random.uniform(low=1, high=5, size=[N, n]) + np.random.randn(N, n)
y1 = np.tile(-1, N)
x2 = np.random.uniform(low=5, high=10, size=[N, n]) + np.random.randn(N, n)
y2 = np.tile(1, N)
x = np.concatenate([x1, x2], axis=0)
y = np.concatenate([y1, y2]).reshape(-1, 1)
x, y = shuffle(x, y, random_state=0)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
# 手寫模型
model = LogitRegModel(max_iter=5000, eta=0.01, alpha=0.5, beta=0.9)
mu, std = model.z_scale(x_train)
x_train_scale, x_test_scale = model.data_transform(mu, std, x_train, x_test)
w, loss_res = model.fit(x_train_scale, y_train)
print(f"LogitRegModel 參數(shù)：\n{w}")
fig, ax = plt.subplots(figsize=(8, 4))
ax.plot(loss_res)
plt.xlabel("epoch")
plt.ylabel("loss")
plt.title("LogitRegModel Loss")
plt.show()
y_pred_probs, y_pred = model.predict(x_test_scale, w)
score = model.get_score(y_test, y_pred)
print(f"LogitRegModel 預(yù)測準確率：{score}")
# sklean
scale = StandardScaler(with_mean=True, with_std=True)
scale.fit(x_train)
x_train_scale = scale.transform(x_train)
x_test_scale = scale.transform(x_test)
clf = LogisticRegression(fit_intercept=True, solver="lbfgs", max_iter=5000, multi_class="ovr")
clf.fit(x_train_scale, y_train)
clf.coef_
clf.intercept_
y_pred = clf.predict(x_test_scale).reshape(-1, 1)
score = sum(y_test == y_pred) / len(y_test)
print(f"Sklearn 預(yù)測準確率：{score}")
LogitRegModel 參數(shù)：
[[0.18014117]
[1.63708775]
[1.64705508]
[1.5463744 ]
[1.61056801]]
LogitRegModel 預(yù)測準確率：[0.975]
array([[1.83258875, 1.93575921, 1.69735311, 1.87150825]])
array([0.39294385])
Sklearn 預(yù)測準確率：[0.975]
