Regression Is Not PCA

Math 123: Mathematical Aspects of Data Analysis — Fall 2026

Published

September 29, 2026

This page is meant to be read alongside Lecture 7. Both least squares and PCA “fit features to a point cloud”, so it is worth being precise about how they differ. We fit both to the same $20$ points in the plane and compare.

import numpy as np
import plotly.graph_objects as go
from plotly.subplots import make_subplots

rng = np.random.default_rng(123)
n = 20
x = rng.uniform(0, 10, n)
y = 1 + 0.6 * x + rng.normal(0, 1.5, n)

Two fits

Least squares treats $y$ as the response, with design matrix $\bfX=[\mathbf{1}\ \bfx]\in\Rbb^{n\times2}$, and chooses $\theta_0,\theta_1$ to minimize the sum of squared vertical distances $\sum_i(y_i-\theta_0-\theta_1x_i)^2$.

PCA treats the points $(x_i,y_i)$ as rows of a data matrix, centers it, and takes the line through the mean in the direction of the top right singular vector $\bfv_1$. By best-fitting subspaces come from the SVD, this line minimizes the sum of squared perpendicular distances from the points to a line through the mean, and the minimum is $\sigma_2^2$.

# Least squares: regress y on [1, x].
X = np.column_stack([np.ones(n), x])
(ls_int, ls_slope), *_ = np.linalg.lstsq(X, y, rcond=None)

# PCA: the top right singular vector of the centered data.
mean = np.array([x.mean(), y.mean()])
D = np.column_stack([x, y]) - mean
_, s, Vt = np.linalg.svd(D, full_matrices=False)
v1 = Vt[0]
pca_slope = v1[1] / v1[0]
pca_int = mean[1] - pca_slope * mean[0]

print(f"least squares: y = {ls_int:.3f} + {ls_slope:.3f} x")
print(f"PCA:           y = {pca_int:.3f} + {pca_slope:.3f} x")

least squares: y = 1.461 + 0.587 x
PCA:           y = 0.589 + 0.750 x

Figure 1 draws each line together with the distances it minimizes.

Plot both fits and their residuals

# The same muted palette as the other examples, which reads correctly against
# both the light and the dark version of this site.
INK = "#7c7c78"
GRID = "rgba(128, 128, 128, 0.28)"
BLUE, ORANGE, GREEN = "#2a78d6", "#eb6834", "#1baf7a"

xs = np.array([-0.5, 10.5])


def line(intercept, slope, name, color, dash="solid", showlegend=True):
    """The line y = intercept + slope * x across the plot."""
    return go.Scatter(x=xs, y=intercept + slope * xs, mode="lines", name=name,
                      showlegend=showlegend, hoverinfo="skip",
                      line=dict(color=color, width=3 if dash == "solid" else 2, dash=dash))


def segments(x0, y0, x1, y1, name):
    """Segments from each (x0, y0) to (x1, y1), as one trace broken by None."""
    sx = np.column_stack([x0, x1, np.full(n, np.nan)]).ravel()
    sy = np.column_stack([y0, y1, np.full(n, np.nan)]).ravel()
    return go.Scatter(x=sx, y=sy, mode="lines", name=name, hoverinfo="skip",
                      line=dict(color=ORANGE, width=1.5))


points = go.Scatter(x=x, y=y, mode="markers", name="data", showlegend=False,
                    marker=dict(color=INK, size=8, line=dict(color="white", width=1)))

# Foot of each perpendicular: project the centered point onto v_1.
feet = mean + np.outer(D @ v1, v1)

fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.08,
                    subplot_titles=["least squares: vertical residuals",
                                    "PCA: perpendicular residuals"])
fig.add_trace(segments(x, y, x, ls_int + ls_slope * x, "vertical residuals"), row=1, col=1)
fig.add_trace(line(pca_int, pca_slope, "the other method's line", INK, "dash"), row=1, col=1)
fig.add_trace(line(ls_int, ls_slope, "least squares line", BLUE), row=1, col=1)
fig.add_trace(points, row=1, col=1)
fig.add_trace(segments(x, y, feet[:, 0], feet[:, 1], "perpendicular residuals"), row=1, col=2)
fig.add_trace(line(ls_int, ls_slope, "least squares line", INK, "dash", showlegend=False),
              row=1, col=2)
fig.add_trace(line(pca_int, pca_slope, "PCA line", GREEN), row=1, col=2)
fig.add_trace(points, row=1, col=2)

fig.update_layout(
    template="simple_white",
    paper_bgcolor="rgba(0,0,0,0)",
    plot_bgcolor="rgba(0,0,0,0)",
    font=dict(color=INK, size=13),
    margin=dict(l=60, r=30, t=40, b=50),
    height=480,
    legend=dict(orientation="h", y=-0.15, x=0),
)
# Equal aspect ratio, so that perpendicular segments look perpendicular. Both panels
# get the same fixed ranges and shrink their domain (not their range) to keep the aspect.
ys = [min(y.min(), feet[:, 1].min()) - 0.5, max(y.max(), feet[:, 1].max()) + 0.5]
for col in (1, 2):
    fig.update_xaxes(range=xs, constrain="domain", title_text="x", gridcolor=GRID,
                     linecolor=GRID, zerolinecolor=GRID, row=1, col=col)
    fig.update_yaxes(range=ys, constrain="domain", scaleanchor="x" if col == 1 else "x2",
                     scaleratio=1, gridcolor=GRID, linecolor=GRID, zerolinecolor=GRID,
                     row=1, col=col)
fig.update_yaxes(title_text="y", row=1, col=1)
fig.show()

Figure 1: Least squares (left) minimizes the vertical distances from the points to the line, while PCA (right) minimizes the perpendicular distances. Each panel shows the other line dashed for comparison.

These are optimizing for fundamentally different things! They each win at their own game. Scoring both lines under both criteria:

def vertical_error(intercept, slope):
    return np.sum((y - intercept - slope * x) ** 2)


def perpendicular_error(intercept, slope):
    # The distance from (x, y) to the line y = a + b x is |y - a - b x| / sqrt(1 + b^2).
    return vertical_error(intercept, slope) / (1 + slope ** 2)


print(f"{'':15s}{'vertical':>12s}{'perpendicular':>16s}")
for name, (a, b) in {"least squares": (ls_int, ls_slope), "PCA": (pca_int, pca_slope)}.items():
    print(f"{name:15s}{vertical_error(a, b):12.3f}{perpendicular_error(a, b):16.3f}")
print(f"\nsigma_2^2 = {s[1] ** 2:.3f}")

                   vertical   perpendicular
least squares        54.697          40.676
PCA                  59.351          37.968

sigma_2^2 = 37.968

The PCA line’s perpendicular error is exactly $\sigma_2^2$, as heralded by Eckart Young.

To summarize:

Regression is supervised: one coordinate, $y$, is singled out as the thing to predict. It assumes the features are measured exactly and all the noise is in $y$, so it measures error vertically, in the $y$ direction only.
PCA is unsupervised: It measures error perpendicular to the subspace, treating all coordinates symmetrically.

In a nutshell, regression asks “how well can $\bfX$ explain $\bfy$?” while PCA asks “what is the most efficient way to represent $\bfX$?”

Tip

Now, technically, people do study total least squares, which is essentially the PCA version. We wont get into that this course but, yes, it is interesting!

--- title: "Regression Is Not PCA" subtitle: "Math 123: Mathematical Aspects of Data Analysis — Fall 2026" date: 2026-09-29 number-sections: false jupyter: python3 format: html: toc: true toc-depth: 2 code-line-numbers: true code-tools: true out-width: "100%" column: page execute: echo: true draft: false --- This page is meant to be read alongside [Lecture 7](../../lectures/lecture-07.qmd). Both [least squares](../../lectures/lecture-07.qmd#def-least-squares) and [PCA](../../lectures/lecture-06.qmd#def-pca) "fit features to a point cloud", so it is worth being precise about how they differ. We fit both to the same $20$ points in the plane and compare. ```{python} import numpy as np import plotly.graph_objects as go from plotly.subplots import make_subplots rng = np.random.default_rng(123) n = 20 x = rng.uniform(0, 10, n) y = 1 + 0.6 * x + rng.normal(0, 1.5, n) ``` ## Two fits Least squares treats $y$ as the response, with design matrix $\bfX=[\mathbf{1}\ \bfx]\in\Rbb^{n\times2}$, and chooses $\theta_0,\theta_1$ to minimize the sum of squared *vertical* distances $\sum_i(y_i-\theta_0-\theta_1x_i)^2$. PCA treats the points $(x_i,y_i)$ as rows of a data matrix, centers it, and takes the line through the mean in the direction of the top right singular vector $\bfv_1$. By [best-fitting subspaces come from the SVD](../../lectures/lecture-06.qmd#thm-best-subspace), this line minimizes the sum of squared *perpendicular* distances from the points to a line through the mean, and the minimum is $\sigma_2^2$. ```{python} # Least squares: regress y on [1, x]. X = np.column_stack([np.ones(n), x]) (ls_int, ls_slope), *_ = np.linalg.lstsq(X, y, rcond=None) # PCA: the top right singular vector of the centered data. mean = np.array([x.mean(), y.mean()]) D = np.column_stack([x, y]) - mean _, s, Vt = np.linalg.svd(D, full_matrices=False) v1 = Vt[0] pca_slope = v1[1] / v1[0] pca_int = mean[1] - pca_slope * mean[0] print(f"least squares: y = {ls_int:.3f} + {ls_slope:.3f} x") print(f"PCA: y = {pca_int:.3f} + {pca_slope:.3f} x") ``` @fig-residuals draws each line together with the distances it minimizes. ```{python} #| label: fig-residuals #| fig-cap: "Least squares (left) minimizes the vertical distances from the points to the line, while PCA (right) minimizes the perpendicular distances. Each panel shows the other line dashed for comparison." #| fig-alt: "Two scatter plots of the same data. Left: the least squares line with vertical residual segments from each point to the line. Right: the PCA line with perpendicular residual segments. Each panel also shows the other line as a dashed gray line; the PCA line is steeper." #| code-fold: true #| code-summary: "Plot both fits and their residuals" # The same muted palette as the other examples, which reads correctly against # both the light and the dark version of this site. INK = "#7c7c78" GRID = "rgba(128, 128, 128, 0.28)" BLUE, ORANGE, GREEN = "#2a78d6", "#eb6834", "#1baf7a" xs = np.array([-0.5, 10.5]) def line(intercept, slope, name, color, dash="solid", showlegend=True): """The line y = intercept + slope * x across the plot.""" return go.Scatter(x=xs, y=intercept + slope * xs, mode="lines", name=name, showlegend=showlegend, hoverinfo="skip", line=dict(color=color, width=3 if dash == "solid" else 2, dash=dash)) def segments(x0, y0, x1, y1, name): """Segments from each (x0, y0) to (x1, y1), as one trace broken by None.""" sx = np.column_stack([x0, x1, np.full(n, np.nan)]).ravel() sy = np.column_stack([y0, y1, np.full(n, np.nan)]).ravel() return go.Scatter(x=sx, y=sy, mode="lines", name=name, hoverinfo="skip", line=dict(color=ORANGE, width=1.5)) points = go.Scatter(x=x, y=y, mode="markers", name="data", showlegend=False, marker=dict(color=INK, size=8, line=dict(color="white", width=1))) # Foot of each perpendicular: project the centered point onto v_1. feet = mean + np.outer(D @ v1, v1) fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.08, subplot_titles=["least squares: vertical residuals", "PCA: perpendicular residuals"]) fig.add_trace(segments(x, y, x, ls_int + ls_slope * x, "vertical residuals"), row=1, col=1) fig.add_trace(line(pca_int, pca_slope, "the other method's line", INK, "dash"), row=1, col=1) fig.add_trace(line(ls_int, ls_slope, "least squares line", BLUE), row=1, col=1) fig.add_trace(points, row=1, col=1) fig.add_trace(segments(x, y, feet[:, 0], feet[:, 1], "perpendicular residuals"), row=1, col=2) fig.add_trace(line(ls_int, ls_slope, "least squares line", INK, "dash", showlegend=False), row=1, col=2) fig.add_trace(line(pca_int, pca_slope, "PCA line", GREEN), row=1, col=2) fig.add_trace(points, row=1, col=2) fig.update_layout( template="simple_white", paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", font=dict(color=INK, size=13), margin=dict(l=60, r=30, t=40, b=50), height=480, legend=dict(orientation="h", y=-0.15, x=0), ) # Equal aspect ratio, so that perpendicular segments look perpendicular. Both panels # get the same fixed ranges and shrink their domain (not their range) to keep the aspect. ys = [min(y.min(), feet[:, 1].min()) - 0.5, max(y.max(), feet[:, 1].max()) + 0.5] for col in (1, 2): fig.update_xaxes(range=xs, constrain="domain", title_text="x", gridcolor=GRID, linecolor=GRID, zerolinecolor=GRID, row=1, col=col) fig.update_yaxes(range=ys, constrain="domain", scaleanchor="x" if col == 1 else "x2", scaleratio=1, gridcolor=GRID, linecolor=GRID, zerolinecolor=GRID, row=1, col=col) fig.update_yaxes(title_text="y", row=1, col=1) fig.show() ``` These are optimizing for fundamentally different things! They each win at their own game. Scoring both lines under both criteria: ```{python} def vertical_error(intercept, slope): return np.sum((y - intercept - slope * x) ** 2) def perpendicular_error(intercept, slope): # The distance from (x, y) to the line y = a + b x is |y - a - b x| / sqrt(1 + b^2). return vertical_error(intercept, slope) / (1 + slope ** 2) print(f"{'':15s}{'vertical':>12s}{'perpendicular':>16s}") for name, (a, b) in {"least squares": (ls_int, ls_slope), "PCA": (pca_int, pca_slope)}.items(): print(f"{name:15s}{vertical_error(a, b):12.3f}{perpendicular_error(a, b):16.3f}") print(f"\nsigma_2^2 = {s[1] ** 2:.3f}") ``` The PCA line's perpendicular error is exactly $\sigma_2^2$, as heralded by Eckart Young. To summarize: - Regression is *supervised*: one coordinate, $y$, is singled out as the thing to predict. It assumes the features are measured exactly and all the noise is in $y$, so it measures error vertically, in the $y$ direction only. - PCA is *unsupervised*: It measures error perpendicular to the subspace, treating all coordinates symmetrically. In a nutshell, regression asks "how well can $\bfX$ explain $\bfy$?" while PCA asks "what is the most efficient way to represent $\bfX$?" ::: {.callout-tip} Now, technically, people do study *total least squares*, which is essentially the PCA version. We wont get into that this course but, yes, it is interesting! :::