#set document(title: "6.3 Least Squares: Beat the Normal Equations", author: "XYZ Homework")
#set page(width: 8.5in, height: auto, margin: 1in)
#import "@preview/cetz:0.5.2"
#set text(font: ("STIX Two Text", "Libertinus Serif", "New Computer Modern"), size: 10.5pt, lang: "en")
#show math.equation: set text(font: ("STIX Two Math", "New Computer Modern Math"))
#set par(justify: true, leading: 0.62em, spacing: 0.9em)
#set enum(spacing: 1.1em)   // room between list items so tall inline fractions don't collide
#set list(spacing: 1.1em)
#set table(stroke: 0.5pt + rgb("#c7ccd3"))

#let BLUE = rgb("#183B6F")     // brand navy — section bars + example/solution labels (white on navy 11.09:1)
#let ORANGE = rgb("#A94509")   // brand primary-700 — AA-safe deep orange for TEXT (5.93:1 on white; raw brand #F37021 is 2.94:1 and must never carry text)
#let RED = rgb("#DC2626")      // brand error-600
#let GREEN = rgb("#059669")    // brand success-600 (decoration only; small green text uses green-text #007942)

#show heading.where(level: 1): it => block(width: 100%, above: 0pt, below: 16pt,
  fill: gradient.linear(BLUE, rgb("#2C5AA0")), inset: (x: 14pt, y: 12pt), radius: 3pt,
  text(fill: white, weight: "bold", size: 19pt, it.body))
#show heading.where(level: 2): it => block(width: 100%, above: 18pt, below: 10pt,
  fill: BLUE, inset: (x: 10pt, y: 6pt), radius: 2pt,
  text(fill: white, weight: "bold", size: 12pt, it.body))
#show heading.where(level: 3): it => text(fill: ORANGE, weight: "bold", size: 12.5pt, it.body)
#show heading.where(level: 4): it => text(fill: BLUE, weight: "bold", size: 10.5pt, it.body)

#let examplebox(label, title, body) = block(width: 100%, breakable: true,
  fill: rgb("#EFF1F5"), stroke: 0.5pt + rgb("#CFDDF0"), radius: 4pt, inset: 10pt,
  above: 12pt, below: 12pt)[
  #block(below: 6pt)[#box(fill: BLUE, inset: (x: 6pt, y: 2pt), radius: 2pt,
    text(fill: white, weight: "bold", size: 8.5pt, label)) #h(0.4em) #strong[#title]]
  #body]

// rail = decorative left rule (raw brand token); labelcolor = AA-safe label text shade
#let notebox(label, rail, labelcolor, tint, body) = block(width: 100%, breakable: true, fill: tint,
    stroke: (left: 3pt + rail), inset: (left: 10pt, rest: 8pt), radius: (right: 4pt),
    above: 11pt, below: 11pt)[
    #text(fill: labelcolor, weight: "bold", size: 7.5pt, tracking: 0.5pt)[#upper(label)]
    #linebreak() #body]

#let solutionbox(body) = block(above: 4pt, below: 8pt)[
  #text(fill: BLUE, weight: "bold", size: 8.5pt)[Solution] #linebreak() #body]

#let figph(msg) = block(width: 100%, height: 60pt, fill: rgb("#f6f7f9"),
  stroke: (paint: rgb("#c7ccd3"), dash: "dashed"), radius: 4pt, inset: 10pt)[
  #align(center + horizon, text(fill: rgb("#889"), style: "italic", size: 9pt, msg))]

// Standardize inlined figure sizes: measure the natural CeTZ canvas, then scale to a
// consistent envelope (aspect-aware; see build_typst.py FIG_* constants). Unlike the
// print preamble, dimensions are FLOORED: in an editor a user can trim a figure to a
// degenerate 1-D shape (a bare line), and w/h or tw/w would then divide by zero.
#let _STD_W = 3.5
#let _WIDE_W = 5.6
#let _MAX_H = 3.4
#let _ASPECT_WIDE = 2.2
#let _UPSCALE_MAX = 1.15
#let stdfig(body) = context {
  let m = measure(body)
  let w = calc.max(m.width / 1in, 0.01)
  let h = calc.max(m.height / 1in, 0.01)
  let tw = if w / h > _ASPECT_WIDE { _WIDE_W } else { _STD_W }
  let s = calc.min(tw / w, _MAX_H / h, _UPSCALE_MAX)
  align(center, box(scale(x: s * 100%, y: s * 100%, reflow: true, body)))
}

#show figure: set block(breakable: false)
#set figure(gap: 8pt)
#show figure.caption: set text(size: 8.5pt, fill: rgb("#555"))


    
== 6.3#h(0.6em)Least Squares: Beat the Normal Equations


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
#emph[Objectives]


      - Interpret fitting a plane to data as minimizing the sum of squared vertical residuals.
- Derive the normal equations from perpendicularity and solve them for a real data set.
- Explain why the least-squares fit cannot be beaten: it is a projection, and projection is the definition of closest.
]


    Seven measurements: #math.equation(block: false, alt: "open parenthesis minus 2 , minus 2 , minus 2.4 close parenthesis")[$( − 2 , − 2 , − 2.4 )$], #math.equation(block: false, alt: "open parenthesis minus 2 , 1 , 0.5 close parenthesis")[$( − 2 , 1 , 0.5 )$], #math.equation(block: false, alt: "open parenthesis minus 1 , minus 1 , minus 0.9 close parenthesis")[$( − 1 , − 1 , − 0.9 )$], #math.equation(block: false, alt: "open parenthesis 0 , 0 , 0.6 close parenthesis")[$( 0 , 0 , 0.6 )$], #math.equation(block: false, alt: "open parenthesis 1 , minus 1 , minus 0.3 close parenthesis")[$( 1 , − 1 , − 0.3 )$], #math.equation(block: false, alt: "open parenthesis 1 , 2 , 2.7 close parenthesis")[$( 1 , 2 , 2.7 )$], #math.equation(block: false, alt: "open parenthesis 2 , 1 , 2.2 close parenthesis")[$( 2 , 1 , 2.2 )$]. One model: a plane #math.equation(block: false, alt: "z equals a x plus b y plus c")[$z = a x + b y + c$]. Each data point would like the coefficients to satisfy #math.equation(block: false, alt: "a x sub i plus b y sub i plus c equals z sub i")[$a x_(i) + b y_(i) + c = z_(i)$] — and read that way, each measurement is a #emph[linear equation in the unknowns #math.equation(block: false, alt: "open parenthesis a , b , c close parenthesis")[$( a , b , c )$]]: seven planes in coefficient space. Chapter 1 says three independent planes already pin down a point, so seven generically share nothing. The system is overdetermined and inconsistent; no plane passes through all seven points, and the interesting question is which plane misses them #emph[least].


    "Least" needs a score. The #strong[residual] of point #math.equation(block: false, alt: "i")[$i$] is the vertical gap #math.equation(block: false, alt: "r sub i equals z sub i minus open parenthesis a x sub i plus b y sub i plus c close parenthesis")[$r_(i) = z_(i) − ( a x_(i) + b y_(i) + c )$], and the score is the sum of squared residuals over all seven points — squared so that misses above and below both count against you, and big misses count hardest. Your job in the first figure: tune #math.equation(block: false, alt: "a")[$a$], #math.equation(block: false, alt: "b")[$b$], #math.equation(block: false, alt: "c")[$c$] by hand and drive the score as low as you can. Then find out whether you beat the machine. (You will not. The point is understanding why.)


    
      {"camera":{"fov":50,"position":\[7,5,4.5\],"projection":"perspective","target":\[0,0,0.3\]},"grid":{"axisColors":{"x":"\#ef4444","y":"\#22c55e","z":"\#3b82f6"},"divisions":10,"gridColor":"\#e5e7eb","showAxes":true,"showLabels":true,"size":10,"visible":true},"id":"1649948e-36b5-4829-814b-c7381b0991c9","metadata":{"alt\_text":"Seven small dark spheres scattered in 3D space around a tilted, semi-transparent plane through the origin. Two vertical dashed red segments connect two of the spheres to the plane, showing how far the plane misses them vertically. A second, better-fitting plane is included but hidden until the student reveals it.","created\_at":"2026-07-07T00:00:00.000Z","description":"Workbook scene: seven 3D data points near z = 0.5x + y + 0.4, a deliberately imperfect starting-guess plane z = 0.5x + 0.5y, a hidden least-squares solution plane z = 0.4042x + 0.9729y + 0.4006 (from the normal equations), and two dashed residual segments at (1,2) and (-2,-2).","tags":\["workbook","linear-algebra","least-squares","best-fit","residuals"\],"updated\_at":"2026-07-07T00:00:00.000Z"},"objects":\[{"color":"\#111827","id":"e57d1e5a-b4ed-4422-bc21-47caad7fc8d0","kind":"point","label":{"text":"(-2, -2, -2.4)","visible":false},"radius":0.1,"visible":true,"x":-2,"y":-2,"z":-2.4},{"color":"\#111827","id":"c623a5f6-f098-40a4-811e-8a2be74daaaf","kind":"point","label":{"text":"(-2, 1, 0.5)","visible":false},"radius":0.1,"visible":true,"x":-2,"y":1,"z":0.5},{"color":"\#111827","id":"3996075f-8c35-4062-831e-04067ccff0a2","kind":"point","label":{"text":"(-1, -1, -0.9)","visible":false},"radius":0.1,"visible":true,"x":-1,"y":-1,"z":-0.9},{"color":"\#111827","id":"0bf1a323-8411-4912-88b4-96655cd79ac6","kind":"point","label":{"text":"(0, 0, 0.6)","visible":false},"radius":0.1,"visible":true,"x":0,"y":0,"z":0.6},{"color":"\#111827","id":"d5708fee-2e8a-474d-a84d-6757e4756838","kind":"point","label":{"text":"(1, -1, -0.3)","visible":false},"radius":0.1,"visible":true,"x":1,"y":-1,"z":-0.3},{"color":"\#111827","id":"a9e77f12-bbee-486f-aa54-a0c48d3f1be4","kind":"point","label":{"text":"(1, 2, 2.7)","visible":false},"radius":0.1,"visible":true,"x":1,"y":2,"z":2.7},{"color":"\#111827","id":"f7898e1d-4b59-494d-85fb-22ccc4baf378","kind":"point","label":{"text":"(2, 1, 2.2)","visible":false},"radius":0.1,"visible":true,"x":2,"y":1,"z":2.2},{"expression":"0.5\*x + 0.5\*y","id":"f6eeba6c-ab61-4cde-a941-18658e0304be","kind":"explicit-surface","label":{"text":"your plane: z = 0.5x + 0.5y","visible":true},"resolution":32,"style":{"colormap":"coolwarm","doubleSided":true,"opacity":0.5,"wireframe":false,"wireframeColor":"\#000000"},"visible":true,"xDomain":\[-3,3\],"yDomain":\[-3,3\]},{"expression":"0.4042\*x + 0.9729\*y + 0.4006","id":"333d43ae-1963-46fc-9857-94087554bf3d","kind":"explicit-surface","label":{"text":"least-squares plane","visible":true},"resolution":32,"style":{"colormap":"plasma","doubleSided":true,"opacity":0.6,"wireframe":false,"wireframeColor":"\#000000"},"visible":false,"xDomain":\[-3,3\],"yDomain":\[-3,3\]},{"id":"4b4f82b2-51f6-43ea-9cfc-b70b8cd3331d","kind":"parametric-curve","label":{"text":"residual at (1, 2)","visible":true},"samples":50,"style":{"color":"\#ef4444","dash":\[4,4\],"lineWidth":2},"tDomain":\[0,1\],"visible":true,"xExpr":"1","yExpr":"2","zExpr":"2.7 - t\*(2.7 - 1.5)"},{"id":"6a44e69a-77be-44d4-838e-7c63cb051a36","kind":"parametric-curve","label":{"text":"residual at (-2, -2)","visible":true},"samples":50,"style":{"color":"\#ef4444","dash":\[4,4\],"lineWidth":2},"tDomain":\[0,1\],"visible":true,"xExpr":"-2","yExpr":"-2","zExpr":"-2.4 - t\*(-2.4 + 2)"}\],"title":"Beat the normal equations","version":1}
      Seven data points and the deliberately imperfect starting plane #math.equation(block: false, alt: "z equals 0.5 x plus 0.5 y")[$z = 0.5 x + 0.5 y$]. The dashed red segments at #math.equation(block: false, alt: "open parenthesis 1 , 2 close parenthesis")[$( 1 , 2 )$] and #math.equation(block: false, alt: "open parenthesis minus 2 , minus 2 close parenthesis")[$( − 2 , − 2 )$] are residuals — vertical gaps between point and plane. A least-squares plane is hidden in the scene; leave it hidden until you have a fit you are proud of.
    
    
=== Explore the figure


    + Orbit the data and read the two drawn residuals. Above #math.equation(block: false, alt: "open parenthesis 1 , 2 close parenthesis")[$( 1 , 2 )$] the plane sits at #math.equation(block: false, alt: "z equals 1.5")[$z = 1.5$] while the point is at #math.equation(block: false, alt: "2.7")[$2.7$] — a gap of #math.equation(block: false, alt: "1.2")[$1.2$] above. At #math.equation(block: false, alt: "open parenthesis minus 2 , minus 2 close parenthesis")[$( − 2 , − 2 )$] the plane is at #math.equation(block: false, alt: "minus 2")[$− 2$] and the point at #math.equation(block: false, alt: "minus 2.4")[$− 2.4$] — a gap of #math.equation(block: false, alt: "0.4")[$0.4$] below.
+ Judge all seven points: five sit above the starting plane, only two below. A plane that splits its misses is doing better than one that is systematically low — this one is too low, and too flat in the #math.equation(block: false, alt: "y")[$y$] direction. Its score is #math.equation(block: false, alt: "3.55")[$3.55$].
+ Hide the two objects labeled #emph[residual at #math.equation(block: false, alt: "open parenthesis 1 , 2 close parenthesis")[$( 1 , 2 )$]] and #emph[residual at #math.equation(block: false, alt: "open parenthesis minus 2 , minus 2 close parenthesis")[$( − 2 , − 2 )$]] — they are drawn for the starting plane only and will not follow your edits. Then edit the plane's expression: try #math.equation(block: false, alt: "0.5 x plus 0.9 y plus 0.3")[$0.5 x + 0.9 y + 0.3$] first, and watch the plane move as you retype. Better or worse? How can you tell from the picture alone?
+ Keep tuning — treat the three coefficients like three sliders and iterate until you cannot visibly improve. Hand fits usually land near #math.equation(block: false, alt: "a approximately equals 0.4")[$a ≈ 0.4$]–#math.equation(block: false, alt: "0.5")[$0.5$], #math.equation(block: false, alt: "b approximately equals 0.9")[$b ≈ 0.9$]–#math.equation(block: false, alt: "1.0")[$1.0$], #math.equation(block: false, alt: "c approximately equals 0.3")[$c ≈ 0.3$]–#math.equation(block: false, alt: "0.5")[$0.5$], scoring somewhere between #math.equation(block: false, alt: "0.1")[$0.1$] and #math.equation(block: false, alt: "0.3")[$0.3$]. Write your three numbers down.
+ Now reveal the hidden object labeled #emph[least-squares plane] and orbit. How close did you come — and can you find any single coefficient change that would visibly improve on it?


    
=== The normal equations


    Your tuning loop was a search of a subspace. Collect the seven heights a candidate plane produces into a vector in #math.equation(block: false, alt: "ℝ to the power 7")[$ℝ^(7)$]: it equals #math.equation(block: false, alt: "a")[$a$] times the column of #math.equation(block: false, alt: "x sub i")[$x_(i)$]'s, plus #math.equation(block: false, alt: "b")[$b$] times the column of #math.equation(block: false, alt: "y sub i")[$y_(i)$]'s, plus #math.equation(block: false, alt: "c")[$c$] times a column of ones. As #math.equation(block: false, alt: "open parenthesis a , b , c close parenthesis")[$( a , b , c )$] range over all values, these vectors sweep out the #emph[span of three columns] — a three-dimensional subspace of #math.equation(block: false, alt: "ℝ to the power 7")[$ℝ^(7)$] — and you were hunting the point of it closest to the data vector #math.equation(block: false, alt: "𝐳 equals open parenthesis z sub 1 , and so on , z sub 7 close parenthesis")[$𝐳 = ( z_(1) , … , z_(7) )$]. Section 6.1 answered that hunt in general: closest means the residual vector #math.equation(block: false, alt: "open parenthesis r sub 1 , and so on , r sub 7 close parenthesis")[$( r_(1) , … , r_(7) )$] is #strong[perpendicular to the subspace], i.e. to each of the three spanning columns. Writing out those three dot products — #math.equation(block: false, alt: "∑ x sub i r sub i equals 0")[$∑ x_(i) r_(i) = 0$], #math.equation(block: false, alt: "∑ y sub i r sub i equals 0")[$∑ y_(i) r_(i) = 0$], #math.equation(block: false, alt: "∑ r sub i equals 0")[$∑ r_(i) = 0$] — and substituting the definition of #math.equation(block: false, alt: "r sub i")[$r_(i)$] gives the #strong[normal equations]:


    #math.equation(block: true, alt: "15 a plus 6 b minus c equals 11.5 , 6 a plus 12 b equals 14.1 , minus a plus 7 c equals 2.4")[$15 a + 6 b − c = 11.5 , #h(2em) 6 a + 12 b = 14.1 , #h(2em) − a + 7 c = 2.4$]


    Three equations, three unknowns, one solution — the whole search compressed onto an index card.


    #examplebox("Example 1")[Solve the normal equations][
Solve the system above and compute the winning coefficients to four decimal places.


      The second equation gives #math.equation(block: false, alt: "b equals open parenthesis 14.1 minus 6 a close parenthesis / 12")[$b = ( 14.1 − 6 a ) / 12$] and the third gives #math.equation(block: false, alt: "c equals open parenthesis 2.4 plus a close parenthesis / 7")[$c = ( 2.4 + a ) / 7$]. Substituting both into the first: #math.equation(block: false, alt: "15 a plus open parenthesis 14.1 minus 6 a close parenthesis / 2 minus open parenthesis 2.4 plus a close parenthesis / 7 equals 11.5")[$15 a + ( 14.1 − 6 a ) / 2 − ( 2.4 + a ) / 7 = 11.5$]. Multiplying through by #math.equation(block: false, alt: "14")[$14$]: #math.equation(block: false, alt: "210 a plus 7 open parenthesis 14.1 minus 6 a close parenthesis minus 2 open parenthesis 2.4 plus a close parenthesis equals 161")[$210 a + 7 ( 14.1 − 6 a ) − 2 ( 2.4 + a ) = 161$], so #math.equation(block: false, alt: "210 a plus 98.7 minus 42 a minus 4.8 minus 2 a equals 161")[$210 a + 98.7 − 42 a − 4.8 − 2 a = 161$], giving #math.equation(block: false, alt: "166 a equals 67.1")[$166 a = 67.1$] and #math.equation(block: false, alt: "a equals 0.4042")[$a = 0.4042$]. Back-substituting, #math.equation(block: false, alt: "b equals 1.175 minus 0.5 open parenthesis 0.4042 close parenthesis equals 0.9729")[$b = 1.175 − 0.5 ( 0.4042 ) = 0.9729$] and #math.equation(block: false, alt: "c equals open parenthesis 2.4 plus 0.4042 close parenthesis / 7 equals 0.4006")[$c = ( 2.4 + 0.4042 ) / 7 = 0.4006$]. The machine's plane is #math.equation(block: false, alt: "z equals 0.4042 x plus 0.9729 y plus 0.4006")[$z = 0.4042 x + 0.9729 y + 0.4006$] — compare it with the hidden plane you revealed, and with your own hand fit.
]


    The winner scores #math.equation(block: false, alt: "0.0723")[$0.0723$] — nearly fifty times better than the starting plane's #math.equation(block: false, alt: "3.55")[$3.55$], and better than any hand fit. Its seven residuals are #math.equation(block: false, alt: "minus 0.046")[$− 0.046$], #math.equation(block: false, alt: "minus 0.065")[$− 0.065$], #math.equation(block: false, alt: "plus 0.077")[$+ 0.077$], #math.equation(block: false, alt: "plus 0.199")[$+ 0.199$], #math.equation(block: false, alt: "minus 0.132")[$− 0.132$], #math.equation(block: false, alt: "minus 0.051")[$− 0.051$], #math.equation(block: false, alt: "plus 0.018")[$+ 0.018$]: small everywhere, mixed in sign, and summing to zero — the visible face of the perpendicularity conditions, since #math.equation(block: false, alt: "∑ r sub i equals 0")[$∑ r_(i) = 0$] is literally one of them.


    
=== Feel the trade-off


    The second figure replays the contest with the coefficients on actual sliders and the two residual segments now #emph[live] — they stretch and shrink as you drag.


    
      {"camera":{"fov":50,"position":\[7,5,4.5\],"projection":"perspective","target":\[0,0,0.3\]},"grid":{"axisColors":{"x":"\#ef4444","y":"\#22c55e","z":"\#3b82f6"},"divisions":10,"gridColor":"\#e5e7eb","showAxes":true,"showLabels":true,"size":10,"visible":true},"id":"4f8e1c72-9b5d-4a36-8e04-1c6f3a9d5b28","metadata":{"alt\_text":"Seven small dark spheres scattered in 3D space around a tilted, semi-transparent plane. Three sliders named a, b, and c control the plane z = ax + by + c. Two vertical dashed red segments connect two of the spheres to the plane and stretch or shrink live as the sliders move. A second, better-fitting plane is included but hidden.","created\_at":"2026-07-26T00:00:00.000Z","description":"Embed variant of the workbook scene 'Beat the Normal Equations': seven 3D data points near z = 0.5x + y + 0.4, a hand-tunable plane z = ax + by + c on sliders a, b, c in \[-2,2\], live residual segments at (1,2) and (-2,-2), and a hidden least-squares solution plane z = 0.4042x + 0.9729y + 0.4006.","tags":\["embed","linear-algebra","least-squares","best-fit","slider"\],"updated\_at":"2026-07-26T00:00:00.000Z"},"objects":\[{"id":"1a2b3c4d-1111-4aaa-9001-000000000001","kind":"parameter","label":{"text":"a","visible":true},"max":2,"min":-2,"name":"a","step":0.05,"value":0.5,"visible":true},{"id":"1a2b3c4d-1111-4aaa-9002-000000000002","kind":"parameter","label":{"text":"b","visible":true},"max":2,"min":-2,"name":"b","step":0.05,"value":0.5,"visible":true},{"id":"1a2b3c4d-1111-4aaa-9003-000000000003","kind":"parameter","label":{"text":"c","visible":true},"max":2,"min":-2,"name":"c","step":0.05,"value":0,"visible":true},{"color":"\#111827","id":"e57d1e5a-b4ed-4422-bc21-47caad7fc8d0","kind":"point","label":{"text":"(-2, -2, -2.4)","visible":false},"radius":0.1,"visible":true,"x":-2,"y":-2,"z":-2.4},{"color":"\#111827","id":"c623a5f6-f098-40a4-811e-8a2be74daaaf","kind":"point","label":{"text":"(-2, 1, 0.5)","visible":false},"radius":0.1,"visible":true,"x":-2,"y":1,"z":0.5},{"color":"\#111827","id":"3996075f-8c35-4062-831e-04067ccff0a2","kind":"point","label":{"text":"(-1, -1, -0.9)","visible":false},"radius":0.1,"visible":true,"x":-1,"y":-1,"z":-0.9},{"color":"\#111827","id":"0bf1a323-8411-4912-88b4-96655cd79ac6","kind":"point","label":{"text":"(0, 0, 0.6)","visible":false},"radius":0.1,"visible":true,"x":0,"y":0,"z":0.6},{"color":"\#111827","id":"d5708fee-2e8a-474d-a84d-6757e4756838","kind":"point","label":{"text":"(1, -1, -0.3)","visible":false},"radius":0.1,"visible":true,"x":1,"y":-1,"z":-0.3},{"color":"\#111827","id":"a9e77f12-bbee-486f-aa54-a0c48d3f1be4","kind":"point","label":{"text":"(1, 2, 2.7)","visible":false},"radius":0.1,"visible":true,"x":1,"y":2,"z":2.7},{"color":"\#111827","id":"f7898e1d-4b59-494d-85fb-22ccc4baf378","kind":"point","label":{"text":"(2, 1, 2.2)","visible":false},"radius":0.1,"visible":true,"x":2,"y":1,"z":2.2},{"expression":"a\*x + b\*y + c","id":"f6eeba6c-ab61-4cde-a941-18658e0304be","kind":"explicit-surface","label":{"text":"your plane: z = ax + by + c","visible":true},"resolution":32,"style":{"colormap":"coolwarm","doubleSided":true,"opacity":0.5,"wireframe":false,"wireframeColor":"\#000000"},"visible":true,"xDomain":\[-3,3\],"yDomain":\[-3,3\]},{"expression":"0.4042\*x + 0.9729\*y + 0.4006","id":"333d43ae-1963-46fc-9857-94087554bf3d","kind":"explicit-surface","label":{"text":"least-squares plane","visible":true},"resolution":32,"style":{"colormap":"plasma","doubleSided":true,"opacity":0.6,"wireframe":false,"wireframeColor":"\#000000"},"visible":false,"xDomain":\[-3,3\],"yDomain":\[-3,3\]},{"id":"4b4f82b2-51f6-43ea-9cfc-b70b8cd3331d","kind":"parametric-curve","label":{"text":"residual at (1, 2)","visible":true},"samples":50,"style":{"color":"\#ef4444","dash":\[4,4\],"lineWidth":2},"tDomain":\[0,1\],"visible":true,"xExpr":"1","yExpr":"2","zExpr":"2.7 - t\*(2.7 - (a + 2\*b + c))"},{"id":"6a44e69a-77be-44d4-838e-7c63cb051a36","kind":"parametric-curve","label":{"text":"residual at (-2, -2)","visible":true},"samples":50,"style":{"color":"\#ef4444","dash":\[4,4\],"lineWidth":2},"tDomain":\[0,1\],"visible":true,"xExpr":"-2","yExpr":"-2","zExpr":"-2.4 - t\*(-2.4 - (-2\*a - 2\*b + c))"}\],"title":"Beat the normal equations","version":1}
      The same seven data points, with the plane #math.equation(block: false, alt: "z equals a x plus b y plus c")[$z = a x + b y + c$] driven by sliders #math.equation(block: false, alt: "a")[$a$], #math.equation(block: false, alt: "b")[$b$], #math.equation(block: false, alt: "c")[$c$] and the dashed residual segments at #math.equation(block: false, alt: "open parenthesis 1 , 2 close parenthesis")[$( 1 , 2 )$] and #math.equation(block: false, alt: "open parenthesis minus 2 , minus 2 close parenthesis")[$( − 2 , − 2 )$] updating live as you drag. The least-squares plane is hidden in this scene too.
    
    + From the starting setting #math.equation(block: false, alt: "open parenthesis 0.5 , 0.5 , 0 close parenthesis")[$( 0.5 , 0.5 , 0 )$], drag #math.equation(block: false, alt: "b")[$b$] upward and watch the residual at #math.equation(block: false, alt: "open parenthesis 1 , 2 close parenthesis")[$( 1 , 2 )$] shrink — then watch what your improvement costs at #math.equation(block: false, alt: "open parenthesis minus 2 , minus 2 close parenthesis")[$( − 2 , − 2 )$]. Every coefficient helps some points at other points' expense; that tension is what the score arbitrates.
+ Try to make both drawn residuals small #emph[at once], then check the rest of the data by orbiting. The best reachable setting on these sliders is about #math.equation(block: false, alt: "open parenthesis 0.40 , 0.95 , 0.40 close parenthesis")[$( 0.40 , 0.95 , 0.40 )$], scoring #math.equation(block: false, alt: "0.0800")[$0.0800$] — the true optimum lies #emph[between] slider clicks, and no setting the sliders can reach attains #math.equation(block: false, alt: "0.0723")[$0.0723$].
+ Reveal the hidden least-squares plane and set the sliders to hug it as closely as you can. At the optimum, no single-slider move improves the score: each slider direction has been made exactly perpendicular to the remaining error. That is the normal equations, felt by hand.


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
#emph[Vertical, not perpendicular]


      The residuals here are #emph[vertical] gaps, not perpendicular distances to the plane — unlike Section 6.1, where the drop ran along the normal. There is no contradiction: the projection in least squares happens in #math.equation(block: false, alt: "ℝ to the power 7")[$ℝ^(7)$], where the residual #emph[vector] #math.equation(block: false, alt: "open parenthesis r sub 1 , and so on , r sub 7 close parenthesis")[$( r_(1) , … , r_(7) )$] is genuinely perpendicular to the subspace of producible height-lists. The picture's planes live in #math.equation(block: false, alt: "ℝ cubed")[$ℝ^(3)$]; the right angle lives in #math.equation(block: false, alt: "ℝ to the power 7")[$ℝ^(7)$], where only the algebra can see it.
]


    #notebox("Note", rgb("#8a94a6"), rgb("#556666"), rgb("#f7f8fa"))[
Least squares is not a clever search that happens to win; it is Section 6.1's projection, run in a space too big to draw. The residual of the winning plane is perpendicular to everything the model can produce, so — by the same Pythagoras certificate as before — every other choice of coefficients scores strictly worse. You cannot beat the normal equations, and now you know it is geometry, not luck.
]