线性回归2021-09-04

一元线性回归模型定义损失函数凹凸性证明参数求解求偏置b求参数w多元线性回归模型定义损失函数凹凸性判断求 $\hat{w}$

一元线性回归

模型定义

模型定义如下所示：

\begin{matrix} (1) & \begin{aligned} f (x) & = \hat{y} = w x + b \\ 其 中 ， w 和 b & 为 模 型 要 学 习 的 参 数 ， x 为 样 本 值 \end{aligned} \end{matrix}

根据最小二乘法，模型的损失函数为：

\begin{matrix} (2) & \begin{aligned} E (w, b) & = \sum_{i = 1}^{m} (y_{i} - {\hat{y}}_{i})^{2} \\ = \sum_{i = 1}^{m} {(y_{i} - (w x_{i} + b))}^{2} \\ 其 中 ， y_{i} 为 & 真 实 值 ， {\hat{y}}_{i} 为 预 测 值 ， m 为 样 本 个 数 \end{aligned} \end{matrix}

$w,b$ $\mathbf{E(w,b)}$ 的值最小

\begin{matrix} (3) & w, b = \underset{w, b}{\arg min} E (w, b) \end{matrix}

损失函数凹凸性证明

$\mathbf{E(w,b)}$ $w$ $b$ 的凸函数。

$A=f_{xx}^{''}(x,y)$
$\begin{matrix} (4) & \begin{aligned} \frac{\partial E (w, b)}{\partial w} & = \frac{\partial}{\partial w} [\sum_{i = 1}^{m} {(y_{i} - w x_{i} - b)}^{2}] \\ = \sum_{i = 1}^{m} \frac{\partial}{\partial w} {(y_{i} - w x_{i} - b)}^{2} \\ = \sum_{i = 1}^{m} 2 \cdot (y_{i} - w x_{i} - b) \cdot (- x_{i}) \\ = 2 (w \sum_{i = 1}^{m} x_{i}^{2} - \sum_{i = 1}^{m} (y_{i} - b) x_{i}) \end{aligned} \end{matrix}$ $\begin{matrix} (5) & \begin{aligned} \frac{\partial^{2} E (w, b)}{\partial w^{2}} & = \frac{\partial}{\partial w} (\frac{\partial E (w, b)}{\partial w}) \\ = \frac{\partial}{\partial w} [2 (w \sum_{i = 1}^{m} x_{i}^{2} - \sum_{i = 1}^{m} (y_{i} - b) x_{i})] \\ = \frac{\partial}{\partial w} [2 w \sum_{i = 1}^{m} x_{i}^{2}] \\ = 2 \sum_{i = 1}^{m} x_{i}^{2} \end{aligned} \end{matrix}$
$A = 2 \sum_{i=1}^{m} x_{i}^{2}$
$B=f_{x y}^{''}(x,y)$
$\begin{matrix} (6) & \begin{aligned} \frac{\partial^{2} E (w, b)}{\partial w \partial b} & = \frac{\partial}{\partial b} (\frac{\partial E (w, b)}{\partial w}) \\ = \frac{\partial}{\partial b} [2 (w \sum_{i = 1}^{m} x_{i}^{2} - \sum_{i = 1}^{m} (y_{i} - b) x_{i})] \\ = \frac{\partial}{\partial b} [- 2 \sum_{i = 1}^{m} (y_{i} - b) x_{i}] \\ = \frac{\partial}{\partial b} (- 2 \sum_{i = 1}^{m} y_{i} x_{i} + 2 \sum_{i = 1}^{m} b x_{i}) \\ = \frac{\partial}{\partial b} (2 \sum_{i = 1}^{m} b x_{i}) \\ = 2 \sum_{i = 1}^{m} x_{i} \end{aligned} \end{matrix}$
$B = 2 \sum_{i=1}^{m} x_{i}$
$C = f_{yy}^{''}(x,y)$
$\begin{matrix} (7) & \begin{aligned} \frac{\partial E (w, b)}{\partial b} & = \frac{\partial}{\partial b} [\sum_{i = 1}^{m} {(y_{i} - w x_{i} - b)}^{2}] \\ = \sum_{i = 1}^{m} \frac{\partial}{\partial b} {(y_{i} - w x_{i} - b)}^{2} \\ = \sum_{i = 1}^{m} 2 \cdot (y_{i} - w x_{i} - b) \cdot (- 1) \\ = 2 (m b - \sum_{i = 1}^{m} (y_{i} - w x_{i})) \end{aligned} \end{matrix}$ $\begin{matrix} (8) & \begin{aligned} \frac{\partial^{2} E_{(w, b)}}{\partial b^{2}} & = \frac{\partial}{\partial b} (\frac{\partial E_{(w, b)}}{\partial b}) \\ = \frac{\partial}{\partial b} [2 (m b - \sum_{i = 1}^{m} (y_{i} - w x_{i}))] \\ = \frac{\partial}{\partial b} (2 m b) \\ = 2 m \end{aligned} \end{matrix}$

$C = 2m$
$AC - B^2$

\begin{matrix} (9) & \begin{aligned} A C - B^{2} & = 4 m \sum_{i = 1}^{m} x_{i}^{2} - 4 m \cdot \bar{x} \cdot \sum_{i = 1}^{m} x_{i} \\ = 4 m (\sum_{i = 1}^{m} x_{i}^{2} - \sum_{i = 1}^{m} x_{i} \bar{x}) \\ = 4 m \sum_{i = 1}^{m} (x_{i}^{2} - x_{i} \bar{x}) \end{aligned} \end{matrix}

又因为：

\begin{matrix} (10) & \begin{array}{r} \sum_{i = 1}^{m} x_{i} \bar{x} = \bar{x} \sum_{i = 1}^{m} x_{i} = \bar{x} \cdot m \cdot \frac{1}{m} \cdot \sum_{i = 1}^{m} x_{i} = m {\bar{x}}^{2} = \sum_{i = 1}^{m} {\bar{x}}^{2} \end{array} \end{matrix}

所以：

\begin{matrix} (11) & \begin{aligned} A C - B^{2} & = 4 m \sum_{i = 1}^{m} (x_{i}^{2} - x_{i} \bar{x}) \\ = 4 m \sum_{i = 1}^{m} (x_{i}^{2} - x_{i} \bar{x} - x_{i} \bar{x} + x_{i} \bar{x}) \\ = 4 m \sum_{i = 1}^{m} (x_{i}^{2} - x_{i} \bar{x} - x_{i} \bar{x} + {\bar{x}}^{2}) \\ = 4 m \sum_{i = 1}^{m} {(x_{i} - \bar{x})}^{2} \\ \geq 0 \end{aligned} \end{matrix}

$\mathbf{E(w,b)}$ $w$ $b$ 的凸函数，问题得证。

参数求解

求偏置b

$\mathbf{E(w,b)}$ $b$ 求一阶偏导数：

\begin{matrix} (12) & \frac{\partial E (w, b)}{\partial b} = 2 (m b - \sum_{i = 1}^{m} (y_{i} - w x_{i})) \end{matrix}

$b$ ：

\begin{matrix} (13) & \begin{matrix} \frac{\partial E (w, b)}{\partial b} = 2 (m b - \sum_{i = 1}^{m} (y_{i} - w x_{i})) = 0 \\ m b - \sum_{i = 1}^{m} (y_{i} - w x_{i}) = 0 \\ b = \frac{1}{m} \sum_{i = 1}^{m} (y_{i} - w x_{i}) \\ b = \frac{1}{m} \sum_{i = 1}^{m} y_{i} - w \cdot \frac{1}{m} \sum_{i = 1}^{m} x_{i} = \bar{y} - w \bar{x} \end{matrix} \end{matrix}

求参数w

$w$ ：

\begin{matrix} (14) & \frac{\partial E (w, b)}{\partial w} = 2 (w \sum_{i = 1}^{m} x_{i}^{2} - \sum_{i = 1}^{m} (y_{i} - b) x_{i}) = 0 w \sum_{i = 1}^{m} x_{i}^{2} - \sum_{i = 1}^{m} (y_{i} - b) x_{i} = 0 w \sum_{i = 1}^{m} x_{i}^{2} = \sum_{i = 1}^{m} y_{i} x_{i} - \sum_{i = 1}^{m} b x_{i} \end{matrix}

$b=\bar{y}-w \bar{x}$ 带入得：

\begin{matrix} (15) & \begin{matrix} w \sum_{i = 1}^{m} x_{i}^{2} = \sum_{i = 1}^{m} y_{i} x_{i} - \sum_{i = 1}^{m} (\bar{y} - w \bar{x}) x_{i} \\ w \sum_{i = 1}^{m} x_{i}^{2} = \sum_{i = 1}^{m} y_{i} x_{i} - \bar{y} \sum_{i = 1}^{m} x_{i} + w \bar{x} \sum_{i = 1}^{m} x_{i} \\ w \sum_{i = 1}^{m} x_{i}^{2} - w \bar{x} \sum_{i = 1}^{m} x_{i} = \sum_{i = 1}^{m} y_{i} x_{i} - \bar{y} \sum_{i = 1}^{m} x_{i} \\ w (\sum_{i = 1}^{m} x_{i}^{2} - \bar{x} \sum_{i = 1}^{m} x_{i}) = \sum_{i = 1}^{m} y_{i} x_{i} - \bar{y} \sum_{i = 1}^{m} x_{i} \end{matrix} \end{matrix}

因为：

\begin{matrix} (16) & \bar{y} \sum_{i = 1}^{m} x_{i} = \frac{1}{m} \sum_{i = 1}^{m} y_{i} \sum_{i = 1}^{m} x_{i} = \bar{x} \sum_{i = 1}^{m} y_{i} \bar{x} \sum_{i = 1}^{m} x_{i} = \frac{1}{m} \sum_{i = 1}^{m} x_{i} \sum_{i = 1}^{m} x_{i} = \frac{1}{m} {(\sum_{i = 1}^{m} x_{i})}^{2} \end{matrix}

所以：

\begin{matrix} (17) & w = \frac{\sum_{i = 1}^{m} y_{i} x_{i} - \bar{x} \sum_{i = 1}^{m} y_{i}}{\sum_{i = 1}^{m} x_{i}^{2} - \frac{1}{m} {(\sum_{i = 1}^{m} x_{i})}^{2}} = \frac{\sum_{i = 1}^{m} y_{i} (x_{i} - \bar{x})}{\sum_{i = 1}^{m} x_{i}^{2} - \frac{1}{m} {(\sum_{i = 1}^{m} x_{i})}^{2}} \end{matrix}

$w$ 进行向量化处理。

$\frac{1}{m}\left(\sum_{i=1}^{m} x_{i}\right)^{2}=\bar{x} \sum_{i=1}^{m} x_{i}=\sum_{i=1}^{m} x_{i} \bar{x}$ 带入分母可得：

\begin{matrix} (18) & \begin{aligned} w & = \frac{\sum_{i = 1}^{m} y_{i} (x_{i} - \bar{x})}{\sum_{i = 1}^{m} x_{i}^{2} - \sum_{i = 1}^{m} x_{i} \bar{x}} \\ = \frac{\sum_{i = 1}^{m} (y_{i} x_{i} - y_{i} \bar{x})}{\sum_{i = 1}^{m} (x_{i = 1}^{2} x_{i} \bar{x})} \end{aligned} \end{matrix}

由于：

\begin{matrix} (19) & 由于 {\begin{cases} \sum_{i = 1}^{m} y_{i} \bar{x} = \bar{x} \sum_{i = 1}^{m} y_{i} = \frac{1}{m} \sum_{i = 1}^{m} x_{i} \sum_{i = 1}^{m} y_{i} = \sum_{i = 1}^{m} x_{i} \cdot \frac{1}{m} \cdot \sum_{i = 1}^{m} y_{i} = \sum_{i = 1}^{m} x_{i} \bar{y} \\ \sum_{i = 1}^{m} y_{i} \bar{x} = \bar{x} \sum_{i = 1}^{m} y_{i} = \bar{x} \cdot m \cdot \frac{1}{m} \cdot \sum_{i = 1}^{m} y_{i} = m \bar{x} \bar{y} = \sum_{i = 1}^{m} \bar{x} \bar{y} \\ \sum_{i = 1}^{m} x_{i} \bar{x} = \bar{x} \sum_{i = 1}^{m} x_{i} = \bar{x} \cdot m \cdot \frac{1}{m} \cdot \sum_{i = 1}^{m} x_{i} = m {\bar{x}}^{2} = \sum_{i = 1}^{m} {\bar{x}}^{2} \end{cases} \end{matrix}

带入可得：

\begin{matrix} (20) & \begin{aligned} w & = \frac{\sum_{i = 1}^{m} (y_{i} x_{i} - y_{i} \bar{x})}{\sum_{i = 1}^{m} (x_{i}^{2} - x_{i} \bar{x})} \\ = \frac{\sum_{i = 1}^{m} (y_{i} x_{i} - y_{i} \bar{x} - y_{i} \bar{x} + y_{i} \bar{x})}{\sum_{i = 1}^{m} (x_{i}^{2} - x_{i} \bar{x} - x_{i} \bar{x} + x_{i} \bar{x})} \\ = \frac{\sum_{i = 1}^{m} (y_{i} x_{i} - y_{i} \bar{x} - x_{i} \bar{y} + \bar{x} \bar{y})}{\sum_{i = 1}^{m} (x_{i}^{2} - x_{i} {\bar{x}}_{i} - x_{i} \bar{x} + {\bar{x}}^{2})} \\ = \frac{\sum_{i = 1}^{m} (x_{i} - \bar{x}) (y_{i} - \bar{y})}{\sum_{i = 1}^{m} {(x_{i} - \bar{x})}^{2}} \end{aligned} \end{matrix}

定义向量：

\begin{matrix} (21) & \begin{matrix} x = (x_{1}, x_{2}, \dots, x_{m})^{T} \\ y = (y_{1}, y_{2}, \dots, y_{m})^{T} \\ x_{d} = {(x_{1} - \bar{x}, x_{2} - \bar{x}, \dots, x_{m} - \bar{x})}^{T} \\ y_{d} = {(y_{1} - \bar{y}, y_{2} - \bar{y}, \dots, y_{m} - \bar{y})}^{T} \end{matrix} \end{matrix}

那么：

\begin{matrix} (22) & \begin{aligned} w & = \frac{\sum_{i = 1}^{m} (x_{i} - \bar{x}) (y_{i} - \bar{y})}{\sum_{i = 1}^{m} {(x_{i} - \bar{x})}^{2}} \\ = \frac{x_{d}^{T} y_{d}}{x_{d}^{T} x_{d}} \end{aligned} \end{matrix}

多元线性回归

模型定义

多元线性回归模型定义如下：

\begin{matrix} (23) & \begin{aligned} f (x_{i}) & = w_{1} x_{i 1} + w_{2} x_{i 2} + \dots + w_{d} x_{i d} + b \\ = w_{1} x_{i 1} + w_{2} x_{i 2} + \dots + w_{d} x_{i d} + w_{d + 1} \cdot 1 \\ = (w_{1} w_{2} \dots w_{d + 1}) \cdot (\begin{array}{c} x_{i 1} \\ x_{i 2} \\ \dots \\ x_{i d} \\ 1 \end{array}) \\ f (\hat{x_{i}}) & = {\hat{w}}^{T} \cdot \hat{x_{i}} \\ 其 中 ， & {\hat{w}}^{T} = (w_{1} w_{2} \dots w_{d + 1}) ， {\hat{x_{i}}}^{T} = (x_{i 1} x_{i 2} \dots x_{i d} 1) \end{aligned} \end{matrix}

$X$ 为：

\begin{matrix} (24) & X = {(\begin{array}{ccccc} x_{11} & x_{12} & \dots & x_{1 d} & 1 \\ x_{21} & x_{22} & \dots & x_{2 d} & 1 \\ ⋮ & ⋮ & ⋱ & ⋮ & ⋮ \\ x_{m 1} & x_{m 2} & \dots & x_{m d} & 1 \end{array})}_{m \times d + 1} = {(\begin{array}{cc} x_{1}^{T} & 1 \\ x_{2}^{T} & 1 \\ ⋮ & ⋮ \\ x_{m}^{T} & 1 \end{array})}_{m \times d + 1} = {(\begin{matrix} {\hat{x}}_{1}^{T} \\ {\hat{x}}_{2}^{T} \\ ⋮ \\ {\hat{x}}_{m}^{T} \end{matrix})}_{m \times d + 1} \end{matrix}

损失函数为：

\begin{matrix} (25) & \begin{aligned} E (\hat{w}) & = \sum_{i = 1}^{m} {(y_{i} - f ({\hat{x}}_{i}))}^{2} \\ = \sum_{i = 1}^{m} {(y_{i} - {\hat{w}}^{T} {\hat{x}}_{i})}^{2} \\ = (y_{1} - {\hat{w}}^{T} {\hat{x}}_{1} y_{2} - {\hat{w}}^{T} {\hat{x}}_{2} \dots y_{m} - {\hat{w}}^{T} {\hat{x}}_{m}) \cdot (\begin{array}{c} y_{1} - {\hat{w}}^{T} {\hat{x}}_{1} \\ y_{2} - {\hat{w}}^{T} {\hat{x}}_{2} \\ \dots \\ y_{m} - {\hat{w}}^{T} {\hat{x}}_{m} \end{array}) \\ = (y - X \cdot \hat{w})^{T} \cdot (y - X \cdot \hat{w}) \end{aligned} \end{matrix}

损失函数凹凸性判断

损失函数对变量求一阶偏导：

\begin{matrix} (26) & \begin{aligned} \frac{\partial E (\hat{w})}{\partial \hat{w}} & = \frac{\partial}{\partial \hat{w}} [(y - X \hat{w})^{T} (y - X \hat{w})] \\ = \frac{\partial}{\partial \hat{w}} [(y^{T} - {\hat{w}}^{T} X^{T}) (y - X \hat{w})] \\ = \frac{\partial}{\partial \hat{w}} [y^{T} y - y^{T} X \hat{w} - {\hat{w}}^{T} X^{T} y + {\hat{w}}^{T} X^{T} X \hat{w}] \\ = \frac{\partial}{\partial \hat{w}} [- y^{T} X \hat{w} - {\hat{w}}^{T} X^{T} y + {\hat{w}}^{T} X^{T} X \hat{w}] \\ = - \frac{\partial y^{T} X \hat{w}}{\partial \hat{w}} - \frac{\partial {\hat{w}}^{T} X^{T} y}{\partial \hat{w}} + \frac{\partial {\hat{w}}^{T} X^{T} X \hat{w}}{\partial \hat{w}} \\ = - X^{T} y - X^{T} y + (X^{T} X + X^{T} X) \hat{w} \\ = 2 X^{T} (X \hat{w} - y) \end{aligned} \end{matrix}

损失函数对变量求二阶偏导（Hessian矩阵）：

\begin{matrix} (27) & \begin{aligned} \frac{\partial^{2} E (\hat{w})}{\partial \hat{w} \partial {\hat{w}}^{T}} & = \frac{\partial}{\partial \hat{w}} (\frac{\partial E_{\hat{w}}}{\partial \hat{w}}) \\ = \frac{\partial}{\partial \hat{w}} [2 X^{T} (X \hat{w} - y)] \\ = \frac{\partial}{\partial \hat{w}} (2 X^{T} X \hat{w} - 2 X^{T} y) \\ = 2 X^{T} X \end{aligned} \end{matrix}

说明：该Hessian矩阵无法保证为正定矩阵，但是，此处直接假设该矩阵为正定矩阵，否则无法继续推导。

$\hat{w}$

一阶偏导数为：

\begin{matrix} (28) & \begin{array}{r} \frac{\partial E (\hat{w})}{\partial \hat{w}} = 2 X^{T} (X \hat{w} - y) \end{array} \end{matrix}

在Hessian矩阵为正定矩阵的假设下，令一阶偏导数为0，可得：

\begin{matrix} (29) & \frac{\partial E (\hat{w})}{\partial \hat{w}} = 2 X^{T} (X \hat{w} - y) = 0 2 X^{T} X \hat{w} - 2 X^{T} y = 0 2 X^{T} X \hat{w} = 2 X^{T} y \hat{w} = (X^{T} X)^{- 1} X^{T} y \end{matrix}

一元线性回归

模型定义

损失函数凹凸性证明

参数求解

求偏置b

求参数w

多元线性回归

模型定义

损失函数凹凸性判断

求w^\hat{w}

$\hat{w}$