From a015e9054611cedfef55e6bc7c0b423b717ed5f9 Mon Sep 17 00:00:00 2001 From: "ml-systems-publisher[bot]" <304658946+ml-systems-publisher[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 10:23:14 +0000 Subject: [PATCH] Add post: Neural Networks From Zero: From a Single Number to a Billion Parameters --- .../.write-source.json | 3647 +++++++++++++++++ .../posts/neural-networks-from-zero/index.mdx | 415 ++ 2 files changed, 4062 insertions(+) create mode 100644 src/content/posts/neural-networks-from-zero/.write-source.json create mode 100644 src/content/posts/neural-networks-from-zero/index.mdx diff --git a/src/content/posts/neural-networks-from-zero/.write-source.json b/src/content/posts/neural-networks-from-zero/.write-source.json new file mode 100644 index 0000000..2086417 --- /dev/null +++ b/src/content/posts/neural-networks-from-zero/.write-source.json @@ -0,0 +1,3647 @@ +{ + "kind": "mlsys-write-source", + "version": 1, + "meta": { + "title": "Neural Networks From Zero: From a Single Number to a Billion Parameters", + "summary": "A neural network never sees a word, an image, or a sound — only a list of numbers. Starting from that one fact and a single neuron, this day-zero guide builds the whole machine: how any input becomes numbers, why weights, biases, and activations each exist, how neurons stack into layers and layers into a model, and how to compute a model's size and running cost by hand.", + "authors": [ + "dinesh" + ], + "writerName": "Author Name", + "topicId": "architecture", + "topicName": "Architecture", + "tags": [ + "fundamentals", + "perceptron", + "neural-networks", + "parameters", + "flops" + ], + "slug": "neural-networks-from-zero", + "coverFileName": "", + "ogCard": false, + "proposedTopic": "", + "newAuthor": null, + "date": "2026-07-12" + }, + "blocks": [ + { + "id": "paragraph-1", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Almost every explanation of neural networks starts one step too late. It shows you a diagram of circles connected by lines, tells you “these are neurons,” and moves on. But it never answers the question a genuinely curious person asks first:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-2", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "What are those circles even looking at?", + "styles": { + "bold": true + } + } + ], + "children": [] + }, + { + "id": "paragraph-3", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "When someone writes ", + "styles": {} + }, + { + "type": "text", + "text": "input = [0.2, 0.9, 0.1, 0.7]", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ", why are there ", + "styles": {} + }, + { + "type": "text", + "text": "four", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " numbers? Why numbers at all? What if my input is the word “cat”? A photo? A ten-second voice clip? An entire paragraph? How does ", + "styles": {} + }, + { + "type": "text", + "text": "any", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " of that turn into something a network can process?", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-4", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "This is day zero. If you don’t understand what an input is, what a single neuron does to it, and why each piece of a neuron exists, then everything downstream — attention, transformers, “7 billion parameters,” GPUs running out of memory — is memorization, not understanding. So we’re going to build the whole thing from the smallest possible unit, and by the end you’ll be able to look at any model and compute, by hand, both ", + "styles": {} + }, + { + "type": "text", + "text": "how big it is", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " and ", + "styles": {} + }, + { + "type": "text", + "text": "how much math it costs to run", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ".", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-5", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Let’s start with the single most important sentence in this entire field.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-6", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 1, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "A network only ever sees numbers", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-7", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Here it is:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "quote-8", + "type": "quote", + "props": { + "backgroundColor": "default", + "textColor": "default" + }, + "content": [ + { + "type": "text", + "text": "A neural network cannot see a word, a pixel, a sound, or a sentence. It can only see ", + "styles": {} + }, + { + "type": "text", + "text": "a list of numbers", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ". Everything else is a translation problem that happens ", + "styles": {} + }, + { + "type": "text", + "text": "before", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " the network.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-9", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "That list of numbers has a name: a ", + "styles": {} + }, + { + "type": "text", + "text": "vector", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ". When you saw ", + "styles": {} + }, + { + "type": "text", + "text": "[0.2, 0.9, 0.1, 0.7]", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ", that was a vector of length 4 — the network’s entire universe for that one input. The “4” isn’t magic; it’s just how many numbers this particular network was built to accept. A different network might take 784 numbers, or 4096. That fixed count is called the ", + "styles": {} + }, + { + "type": "text", + "text": "input dimension", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ".", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-10", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "So the real question isn’t “how does a network understand a cat?” It’s: ", + "styles": {} + }, + { + "type": "text", + "text": "how do we turn a cat — the word, or the photo — into a list of numbers?", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " That translation is where all the interesting representational choices live, and it’s completely separate from the network itself. Let’s do it for every kind of input.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-11", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 2, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "Turning anything into numbers", + "styles": {} + } + ], + "children": [] + }, + { + "id": "note-12", + "type": "note", + "props": {}, + "content": [ + { + "type": "text", + "text": "The rule to hold onto: the network is fixed and dumb — it eats a vector of numbers. The ", + "styles": {} + }, + { + "type": "text", + "text": "encoding", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " is where meaning gets packed in. Change the encoding and you change what the network can possibly learn, without touching the network at all.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-13", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "A single quantity — the easy case.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " Say you’re predicting whether it’ll rain from today’s temperature. Temperature is ", + "styles": {} + }, + { + "type": "text", + "text": "already", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " a number: ", + "styles": {} + }, + { + "type": "text", + "text": "28.4", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ". Your input vector is length 1: ", + "styles": {} + }, + { + "type": "text", + "text": "[28.4]", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ". Add humidity and pressure and you have ", + "styles": {} + }, + { + "type": "text", + "text": "[28.4, 0.66, 1013]", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " — length 3. Each slot in the vector is called a ", + "styles": {} + }, + { + "type": "text", + "text": "feature", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ". This is the whole reason the classic examples use 3 or 4 numbers: they’re toy problems with 3 or 4 features. Nothing deeper.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-14", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "A category — a color, a yes/no, a day of the week.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " These aren’t numbers, and you can’t just assign ", + "styles": {} + }, + { + "type": "text", + "text": "Monday = 1, Tuesday = 2", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ", because that would trick the network into thinking Wednesday is “twice” Monday, which is nonsense. Instead we use ", + "styles": {} + }, + { + "type": "text", + "text": "one-hot encoding", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ": one slot per possible value, all zeros except a single 1.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-15", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " Monday → [1, 0, 0, 0, 0, 0, 0]\n Wednesday → [0, 0, 1, 0, 0, 0, 0]\n Sunday → [0, 0, 0, 0, 0, 0, 1]", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-16", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Seven categories → a vector of length 7. No false ordering, no fake arithmetic. Just “which one is it.”", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-17", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "A word — where it gets interesting.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " A word like “cat” is a category too, but the vocabulary has ~50,000 words, so a one-hot vector would be 50,000 numbers that are almost entirely zeros. Wasteful, and worse, it says every word is equally unrelated to every other — “cat” is exactly as far from “dog” as it is from “helicopter.” That’s clearly wrong.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-18", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "So instead we ", + "styles": {} + }, + { + "type": "text", + "text": "learn", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " a compact vector for each word, called an ", + "styles": {} + }, + { + "type": "text", + "text": "embedding", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — typically a few hundred to a few thousand numbers:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-19", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " \"cat\" → [ 0.21, -0.44, 0.10, ..., 0.05] (say, 768 numbers)\n \"dog\" → [ 0.19, -0.41, 0.14, ..., 0.08] ← close to \"cat\"\n \"helicopter\" → [-0.88, 0.30, -0.52, ..., 0.77] ← far from both", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-20", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "The magic: these numbers are learned so that ", + "styles": {} + }, + { + "type": "text", + "text": "similar meanings end up close together", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " in the vector space. “cat” and “dog” land near each other; “helicopter” lands far away. The network never sees the letters c-a-t — it sees 768 numbers that encode what “cat” ", + "styles": {} + }, + { + "type": "text", + "text": "means", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ", in a geometry where distance is similarity. (How text gets split into these units in the first place — “tokenization” — and how embeddings are learned are their own deep dives. For today, just hold the shape: ", + "styles": {} + }, + { + "type": "text", + "text": "word → vector of numbers.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ")", + "styles": {} + } + ], + "children": [] + }, + { + "id": "svg-21", + "type": "svg", + "props": { + "code": "\n \n \n \n \n \n \n \n \n cat\n dog\n helicopter\n \n \n similar meaning → close\n unrelated → far\n \n", + "caption": "Word embeddings live in a space where distance means similarity: 'cat' and 'dog' land close together, 'helicopter' sits far away — and the network only ever sees the coordinates, never the letters." + }, + "children": [] + }, + { + "id": "paragraph-22", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "An image.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " A grayscale photo is a grid of brightness values, each from 0 (black) to 255 (white). A small 28×28 image is 784 of those numbers. Flatten the grid row by row into one long list and you have a vector of length 784. A color photo has three such grids (red, green, blue), so a 224×224 color image is ", + "styles": {} + }, + { + "type": "text", + "text": "224 × 224 × 3 = 150,528", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " numbers. A picture really is worth a (few hundred) thousand numbers.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "svg-23", + "type": "svg", + "props": { + "code": "\n \n \n \n \n \n \n \n 30230230230\n 23030230230\n 23023030230\n 23023023030\n \n \n →\n [30, 230, 230, 230, 230, 30, 230, 230, 230, 230, 30, 230, 230, 230, 230, 30]\n \n", + "caption": "A 4×4 grayscale patch is literally 16 numbers (0 = black, 255 = white). Read the grid row by row and you get the flat vector the network actually receives. This diagonal is a '16-pixel image'." + }, + "children": [] + }, + { + "id": "paragraph-24", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Audio (an mp3).", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " Sound is a wave — air pressure changing over time. To digitize it, we measure that pressure many times per second (44,100 times/second for CD quality). Ten seconds of audio is ", + "styles": {} + }, + { + "type": "text", + "text": "44,100 × 10 = 441,000", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " numbers, each a sample of the wave’s height at that instant. A vector, again — just a very long, time-ordered one.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "svg-25", + "type": "svg", + "props": { + "code": "\n \n \n \n \n \n \n \n \n \n +0.7+0.8−0.6\n \n time →\n", + "caption": "Sound is a wave; digitizing it means measuring the wave's height thousands of times per second. Each measurement (each dot) is one number. Ten seconds of CD-quality audio is 441,000 of them, in order." + }, + "children": [] + }, + { + "id": "paragraph-26", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "A paragraph.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " A paragraph is a ", + "styles": {} + }, + { + "type": "text", + "text": "sequence", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " of words, so it becomes a ", + "styles": {} + }, + { + "type": "text", + "text": "sequence of embedding vectors", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " — one vector per word (or token), in order. This is the one case where we don’t collapse to a single flat vector, because ", + "styles": {} + }, + { + "type": "text", + "text": "order matters", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " (“dog bites man” ≠ “man bites dog”). Handling sequences of vectors while respecting order is exactly the problem transformers were invented to solve — but that’s getting ahead of ourselves. The atom is still the same: ", + "styles": {} + }, + { + "type": "text", + "text": "each word is a vector of numbers.", + "styles": { + "bold": true + } + } + ], + "children": [] + }, + { + "id": "paragraph-27", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Here’s the whole picture in one table:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "table-28", + "type": "table", + "props": { + "textColor": "default" + }, + "content": { + "type": "tableContent", + "columnWidths": [ + null, + null, + null + ], + "rows": [ + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "Input", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "How it becomes numbers", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "Example length", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + }, + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "Temperature", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "it’s already a number", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "1", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + }, + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "Day of week", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "one-hot (one slot per category)", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "7", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + }, + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "A word", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "learned embedding vector", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "~768", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + }, + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "28×28 grayscale image", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "flatten the pixel grid", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "784", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + }, + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "224×224 color image", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "flatten 3 color grids", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "150,528", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + }, + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "10s of CD-quality audio", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "samples of the wave over time", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "441,000", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + }, + { + "cells": [ + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "A paragraph", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "a ", + "styles": {} + }, + { + "type": "text", + "text": "sequence", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " of word vectors", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + }, + { + "type": "tableCell", + "content": [ + { + "type": "text", + "text": "(n words) × 768", + "styles": {} + } + ], + "props": { + "colspan": 1, + "rowspan": 1, + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + } + } + ] + } + ] + }, + "children": [] + }, + { + "id": "paragraph-29", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Every single one is just numbers. ", + "styles": {} + }, + { + "type": "text", + "text": "That", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " is what the circles in the diagram are looking at. Now let’s build a circle.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-30", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 1, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "The neuron: the atom of the whole field", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-31", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Everything — every model with billions of parameters — is built from one tiny unit repeated a staggering number of times. Understand this one unit completely and the rest is assembly.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-32", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "The unit is loosely inspired by a biological neuron, and the analogy is genuinely useful. A brain cell receives signals from many others through its dendrites. Some of those connections are strong, some weak. The cell adds up all its incoming signals (weighted by connection strength), and ", + "styles": {} + }, + { + "type": "text", + "text": "if the total is strong enough, it fires.", + "styles": { + "italic": true + } + } + ], + "children": [] + }, + { + "id": "svg-33", + "type": "svg", + "props": { + "code": "\n \n \n \n \n \n \n \n \n \n dendrites\n axon\n \n \n soma\n biological neuron\n \n ≈\n \n \n \n \n \n \n \n \n \n \n \n \n \n Σ\n ＋ b\n f\n y\n artificial neuron\n \n", + "caption": "The artificial neuron mirrors the biological one, part for part: dendrites → inputs, synapse strengths → weights, the cell body's summation → Σ + bias, and 'firing' → the activation function f." + }, + "children": [] + }, + { + "id": "paragraph-34", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "An artificial neuron does exactly that, with three ingredients:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "svg-35", + "type": "svg", + "props": { + "code": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n x₁\n x₂\n x₃\n x₄\n Σ\n f\n y\n \n \n w₁\n w₂\n w₃\n w₄\n ＋ b\n \n", + "caption": "A single neuron: each input is scaled by its own weight, the results are summed, a bias is added, and the total is passed through an activation function to produce one output." + }, + "children": [] + }, + { + "id": "paragraph-36", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "In math, that entire picture is one short line:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-37", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " y = f( w₁·x₁ + w₂·x₂ + w₃·x₃ + w₄·x₄ + b )\n └─┬─┘ └──────────────┬───────────────┘\n activation weighted sum + bias", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-38", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Three ingredients: ", + "styles": {} + }, + { + "type": "text", + "text": "weights", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " (", + "styles": {} + }, + { + "type": "text", + "text": "w", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": "), a ", + "styles": {} + }, + { + "type": "text", + "text": "bias", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " (", + "styles": {} + }, + { + "type": "text", + "text": "b", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": "), and an ", + "styles": {} + }, + { + "type": "text", + "text": "activation function", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " (", + "styles": {} + }, + { + "type": "text", + "text": "f", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": "). Each of them exists for a specific reason, and the fastest way to understand ", + "styles": {} + }, + { + "type": "text", + "text": "why", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " is to ask what breaks if you remove it.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-39", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 2, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "Why weights? (importance knobs)", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-40", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "A weight is a ", + "styles": {} + }, + { + "type": "text", + "text": "dimmer switch on each input", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — a learned number that says “how much should ", + "styles": {} + }, + { + "type": "text", + "text": "this", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " feature matter?” A large positive weight means “this input strongly pushes the answer up.” A weight near zero means “ignore this input.” A negative weight means “this input pushes the answer ", + "styles": {} + }, + { + "type": "text", + "text": "down", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ".”", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-41", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Predicting rain: the humidity input probably deserves a big weight, the day-of-week input a tiny one. The network doesn’t know that in advance — it ", + "styles": {} + }, + { + "type": "text", + "text": "learns", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " the weights from data. Training a neural network is, almost entirely, the process of nudging billions of these dimmer switches until the outputs come out right.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-42", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Remove them", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — force every weight to 1 — and every input counts equally. The neuron loses all ability to say some features matter more than others. It can no longer learn. Weights are ", + "styles": {} + }, + { + "type": "text", + "text": "where the learning lives.", + "styles": { + "italic": true + } + } + ], + "children": [] + }, + { + "id": "heading-43", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 2, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "Why a bias? (the offset that frees the decision)", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-44", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "The bias is a single number added ", + "styles": {} + }, + { + "type": "text", + "text": "after", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " the weighted sum, and it’s the ingredient people find most mysterious — until you see the one-dimensional version.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-45", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Strip a neuron down to one input: ", + "styles": {} + }, + { + "type": "text", + "text": "y = w·x + b", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ". That’s the equation of a ", + "styles": {} + }, + { + "type": "text", + "text": "straight line", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ", exactly the ", + "styles": {} + }, + { + "type": "text", + "text": "y = mx + c", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " from school. The weight ", + "styles": {} + }, + { + "type": "text", + "text": "w", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " is the slope; the bias ", + "styles": {} + }, + { + "type": "text", + "text": "b", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " is the intercept — where the line crosses the axis.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-46", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Now remove the bias: ", + "styles": {} + }, + { + "type": "text", + "text": "y = w·x", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ". You can only make lines that ", + "styles": {} + }, + { + "type": "text", + "text": "pass through the origin (0,0)", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ". You can spin the line to any angle, but it’s forever nailed to that one point. If the pattern in your data lives anywhere else — “predict rain when temperature is above 30” — a line through the origin simply cannot express it. The bias is what lets the neuron ", + "styles": {} + }, + { + "type": "text", + "text": "slide its decision boundary anywhere", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ", instead of being pinned to zero.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "note-47", + "type": "note", + "props": {}, + "content": [ + { + "type": "text", + "text": "Analogy: the weights set the ", + "styles": {} + }, + { + "type": "text", + "text": "tilt", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " of the decision boundary; the bias sets its ", + "styles": {} + }, + { + "type": "text", + "text": "position", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ". Without a bias, every boundary in the whole network is forced through the origin — like trying to describe every line on a graph using only lines that touch (0,0). You lose almost everything.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-48", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 2, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "Why an activation function? (the one that makes deep learning possible)", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-49", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "This is the subtle one, and it’s the reason the whole field is called ", + "styles": {} + }, + { + "type": "text", + "text": "deep", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " learning. The activation ", + "styles": {} + }, + { + "type": "text", + "text": "f", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " is a function applied to the neuron’s output — historically an S-shaped squash, today usually ", + "styles": {} + }, + { + "type": "text", + "text": "ReLU", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ", which is almost comically simple: ", + "styles": {} + }, + { + "type": "text", + "text": "ReLU(z) = max(0, z)", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " — keep positives, zero out negatives.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-50", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Why bother? Because a weighted sum plus a bias is a ", + "styles": {} + }, + { + "type": "text", + "text": "linear", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " operation — it can only draw straight lines and flat planes. And here’s the killer fact: ", + "styles": {} + }, + { + "type": "text", + "text": "stacking linear operations gives you another linear operation.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " If layer 2 is just a weighted sum of layer 1, which is a weighted sum of the input, the whole thing collapses algebraically into a ", + "styles": {} + }, + { + "type": "text", + "text": "single", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " weighted sum. A hundred layers with no activation is mathematically identical to one layer. All that depth buys you ", + "styles": {} + }, + { + "type": "text", + "text": "nothing", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ".", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-51", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "The activation function inserts a ", + "styles": {} + }, + { + "type": "text", + "text": "bend", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — a non-linearity — between layers, and that bend is what stops the collapse. With bends between them, layers can ", + "styles": {} + }, + { + "type": "text", + "text": "compose", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ": bend, then bend again, then again. Now the network can carve curved, intricate boundaries and represent genuinely complex relationships. The classic proof-by-example is ", + "styles": {} + }, + { + "type": "text", + "text": "XOR", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " (“true when exactly one input is true”) — a pattern no single straight line can separate, and which becomes trivial the moment you have two neurons and a non-linearity to combine them.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "quote-52", + "type": "quote", + "props": { + "backgroundColor": "default", + "textColor": "default" + }, + "content": [ + { + "type": "text", + "text": "Weights and bias let a neuron draw ", + "styles": {} + }, + { + "type": "text", + "text": "one", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " straight boundary. The activation is what lets many neurons combine into something that isn’t straight at all. No activation, no depth, no deep learning.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-53", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "That’s the entire atom: ", + "styles": {} + }, + { + "type": "text", + "text": "weights", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " decide what matters, the ", + "styles": {} + }, + { + "type": "text", + "text": "bias", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " frees the boundary from the origin, the ", + "styles": {} + }, + { + "type": "text", + "text": "activation", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " bends the line so depth means something. Three ingredients, each earning its place.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-54", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 1, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "Why one neuron is never enough", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-55", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "A single neuron draws a single boundary — one line, one plane. Real problems need many. So we do the obvious thing: use lots of neurons. But ", + "styles": {} + }, + { + "type": "text", + "text": "how", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " we arrange them splits into two independent choices, and understanding the difference between them is understanding the shape of every model.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-56", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 2, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "Many neurons side by side = width (one layer)", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-57", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Put several neurons in a row, all looking at the ", + "styles": {} + }, + { + "type": "text", + "text": "same", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " input, each with its own weights and bias. Each learns to detect a ", + "styles": {} + }, + { + "type": "text", + "text": "different", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " feature of that input. Feed in an image and, side by side, one neuron might learn to spot a vertical edge, another a patch of red, another a curve. They all work in parallel on the same input, and together they produce a ", + "styles": {} + }, + { + "type": "text", + "text": "new", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " list of numbers — one per neuron — describing the input in terms of the features they detected.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-58", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "That row of neurons is a ", + "styles": {} + }, + { + "type": "text", + "text": "layer", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ", and the number of neurons in it is the layer’s ", + "styles": {} + }, + { + "type": "text", + "text": "width", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ". Width = how many different features you can detect ", + "styles": {} + }, + { + "type": "text", + "text": "at the same level of abstraction", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ", simultaneously.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-59", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 2, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "Layers behind layers = depth (a stack)", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-60", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Now the powerful move: take the list of numbers that first layer produced and feed it as input to ", + "styles": {} + }, + { + "type": "text", + "text": "another", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " layer. The second layer isn’t looking at raw pixels anymore — it’s looking at the ", + "styles": {} + }, + { + "type": "text", + "text": "features the first layer found", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": ", and it learns to combine them into bigger patterns. The number of layers stacked this way is the network’s ", + "styles": {} + }, + { + "type": "text", + "text": "depth", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ".", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-61", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "This is the famous hierarchy, and it’s genuinely how deep vision networks organize themselves:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-62", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " raw pixels\n │\n ▼\n Layer 1 → detects edges & simple colors (\"there's an edge here\")\n │\n ▼\n Layer 2 → combines edges into shapes (\"edges here form a curve\")\n │\n ▼\n Layer 3 → combines shapes into parts (\"curves here form an eye\")\n │\n ▼\n Layer 4 → combines parts into objects (\"eyes + nose + fur → cat\")", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-63", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Each layer builds on the one below, working at a higher level of abstraction. That’s why we go deep: ", + "styles": {} + }, + { + "type": "text", + "text": "depth is composition.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " Width gives you ", + "styles": {} + }, + { + "type": "text", + "text": "breadth", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " of features at one level; depth gives you ", + "styles": {} + }, + { + "type": "text", + "text": "hierarchy", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " across levels. They are different axes, and a good architecture needs both.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "svg-64", + "type": "svg", + "props": { + "code": "", + "caption": "A small multi-layer network. Left: the input vector (4 numbers). Middle: a hidden layer of neurons, each detecting a feature. Right: the output. Every line is a weight; every node adds a bias and an activation." + }, + "children": [] + }, + { + "id": "paragraph-65", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "This whole arrangement — an input, one or more hidden layers, an output — is the ", + "styles": {} + }, + { + "type": "text", + "text": "Multi-Layer Perceptron (MLP)", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ", the original neural network and still a building block inside every transformer today.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-66", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 2, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "So how many neurons and how many layers?", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-67", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "The honest answer: ", + "styles": {} + }, + { + "type": "text", + "text": "there is no formula, and anyone who gives you one is selling something.", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " Width and depth are the network’s ", + "styles": {} + }, + { + "type": "text", + "text": "capacity", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " — its room to learn patterns. Too little and it can’t capture the problem (underfitting); too much and it memorizes noise, wastes compute, and gets harder to train (overfitting). The right size depends on how complex the true pattern is, how much data you have, and how much compute you can afford.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-68", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "In practice it’s chosen empirically — start from sizes that worked on similar problems, then adjust. But the ", + "styles": {} + }, + { + "type": "text", + "text": "rules of thumb", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " are worth knowing: more ", + "styles": {} + }, + { + "type": "text", + "text": "depth", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " tends to help when the problem is compositional and hierarchical (vision, language); more ", + "styles": {} + }, + { + "type": "text", + "text": "width", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " helps when there are many independent features to track at once. Modern large models are both very deep ", + "styles": {} + }, + { + "type": "text", + "text": "and", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " very wide, which is precisely why they have so many parameters — which brings us to the payoff.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-69", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 1, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "From atoms to a model: counting the size", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-70", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Here’s where everything connects, and where “7 billion parameters” stops being a marketing number and becomes something you can compute yourself.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-71", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "First, collapse a whole layer into one clean object. A layer is just many neurons, each with a weight per input. Stack their weights and you get a ", + "styles": {} + }, + { + "type": "text", + "text": "matrix", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ". A layer that takes ", + "styles": {} + }, + { + "type": "text", + "text": "d_in", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " numbers in and produces ", + "styles": {} + }, + { + "type": "text", + "text": "d_out", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " numbers out has:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "bulletListItem-72", + "type": "bulletListItem", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "a ", + "styles": {} + }, + { + "type": "text", + "text": "weight matrix", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " of shape ", + "styles": {} + }, + { + "type": "text", + "text": "d_out × d_in", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " — that’s ", + "styles": {} + }, + { + "type": "text", + "text": "d_out × d_in", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " individual weights, one for every (output neuron, input) pair,", + "styles": {} + } + ], + "children": [] + }, + { + "id": "bulletListItem-73", + "type": "bulletListItem", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "a ", + "styles": {} + }, + { + "type": "text", + "text": "bias vector", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " of length ", + "styles": {} + }, + { + "type": "text", + "text": "d_out", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " — one bias per output neuron.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-74", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "And the entire layer’s computation — every neuron at once — is a single matrix multiply plus the biases, then the activation:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-75", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " y = f( W · x + b ) W is [d_out × d_in], x is [d_in], b is [d_out]", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-76", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "So counting a model’s size is now pure bookkeeping. ", + "styles": {} + }, + { + "type": "text", + "text": "A “parameter” is one single learned number", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — one entry in a weight matrix, or one bias. That’s all it is. “7 billion parameters” means 7 billion individual numbers that training tuned. To find the total, count the weights in every layer and add:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-77", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " parameters in one layer ≈ d_in × d_out (+ d_out biases, usually negligible)\n\n total parameters = sum over all layers", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-78", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Worked example — a tiny 3-layer MLP that takes a 784-pixel image and classifies it into 10 digits:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-79", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " Layer 1: 784 → 256 784 × 256 = 200,704 weights\n Layer 2: 256 → 256 256 × 256 = 65,536 weights\n Layer 3: 256 → 10 256 × 10 = 2,560 weights\n ────────────────────────────────────────────────────────\n total ≈ 268,800 parameters (plus 522 biases)", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-80", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "That’s it. You just counted a model’s size by hand. Real transformers use the same arithmetic, just with a specific set of matrices per layer. As a rule of thumb, one transformer layer with hidden size ", + "styles": {} + }, + { + "type": "text", + "text": "d", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " has roughly ", + "styles": {} + }, + { + "type": "text", + "text": "12 × d²", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " weights (four matrices for attention plus a wider feed-forward block). So a model with ", + "styles": {} + }, + { + "type": "text", + "text": "d = 8192", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " and 80 layers has about:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-81", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " 80 layers × 12 × 8192² ≈ 64 billion weights (+ embeddings → ~70B)", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-82", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "There’s your “70B model,” derived from nothing but “a parameter is one number in a matrix, and I can count matrices.”", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-83", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "And the size on disk or in memory", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " follows immediately, because each parameter is just a number stored in some precision. In FP16, one number takes 2 bytes:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-84", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " 70,000,000,000 params × 2 bytes = 140 GB", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-85", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "That 140 GB is why a 70B model doesn’t fit on a single 80 GB GPU, why quantization (storing each number in 1 byte instead of 2) is such a big deal, and why “how many GPUs do I need?” has a concrete answer. It all traces back to ", + "styles": {} + }, + { + "type": "text", + "text": "one number per parameter.", + "styles": { + "italic": true + } + } + ], + "children": [] + }, + { + "id": "heading-86", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 1, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "The cost of running it: FLOPs", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-87", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "One last piece, and it’s the ground truth for every performance conversation you’ll ever have. We can count the ", + "styles": {} + }, + { + "type": "text", + "text": "math", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " a model does just as precisely as we counted its size.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-88", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Look again at what one weight does inside ", + "styles": {} + }, + { + "type": "text", + "text": "W · x", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ": it gets ", + "styles": {} + }, + { + "type": "text", + "text": "multiplied", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " by its input, and that product gets ", + "styles": {} + }, + { + "type": "text", + "text": "added", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " into a running sum. One multiply, one add — two arithmetic operations. Engineers call a floating-point operation a ", + "styles": {} + }, + { + "type": "text", + "text": "FLOP", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": ", so:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "quote-89", + "type": "quote", + "props": { + "backgroundColor": "default", + "textColor": "default" + }, + "content": [ + { + "type": "text", + "text": "Every parameter costs ", + "styles": {} + }, + { + "type": "text", + "text": "2 FLOPs", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " each time one input passes through the network — one multiply and one add.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-90", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "For one input (in language models, one “token”) flowing through a model with ", + "styles": {} + }, + { + "type": "text", + "text": "N", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " parameters, the total work is therefore about:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "codeBlock-91", + "type": "codeBlock", + "props": { + "language": "text" + }, + "content": [ + { + "type": "text", + "text": " FLOPs per token ≈ 2 × N", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-92", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "For our 70B model: ", + "styles": {} + }, + { + "type": "text", + "text": "2 × 70,000,000,000 = 140 billion FLOPs", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " to process a single token. Not “a lot” — ", + "styles": {} + }, + { + "type": "text", + "text": "exactly", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " 140 billion, a number you derived from first principles.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-93", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "This single formula is the seed of an enormous amount of systems knowledge. It’s why processing a long prompt (many tokens at once, reusing the same weights) behaves completely differently from generating one token at a time (dragging all 140 GB of weights out of memory to do a comparatively tiny 140 GFLOPs of math). That contrast — compute-bound versus memory-bound — is the beating heart of why inference systems look the way they do. But that’s the next deep dive. For now, sit with what you can already do:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "heading-94", + "type": "heading", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left", + "level": 1, + "isToggleable": false + }, + "content": [ + { + "type": "text", + "text": "What you can now do", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-95", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Start to finish, with no hand-waving, you can now explain:", + "styles": {} + } + ], + "children": [] + }, + { + "id": "bulletListItem-96", + "type": "bulletListItem", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Input", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — a network only ever sees a vector of numbers, and ", + "styles": {} + }, + { + "type": "text", + "text": "every", + "styles": { + "italic": true + } + }, + { + "type": "text", + "text": " input type (number, category, word, image, audio, paragraph) is a translation into that vector. The encoding is where meaning gets packed in.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "bulletListItem-97", + "type": "bulletListItem", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "The neuron", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — ", + "styles": {} + }, + { + "type": "text", + "text": "y = f(w·x + b)", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": ": weights are learned importance knobs, the bias frees the decision boundary from the origin, and the activation’s bend is what makes depth meaningful.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "bulletListItem-98", + "type": "bulletListItem", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Width vs depth", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — many neurons side by side detect many features at one level; stacked layers compose features into a hierarchy. Both axes, for different reasons.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "bulletListItem-99", + "type": "bulletListItem", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Model size", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — a parameter is one number in a matrix; count the matrices and you have the parameter count; multiply by bytes-per-number and you have the memory.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "bulletListItem-100", + "type": "bulletListItem", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Model cost", + "styles": { + "bold": true + } + }, + { + "type": "text", + "text": " — two FLOPs per parameter per token; ", + "styles": {} + }, + { + "type": "text", + "text": "2N", + "styles": { + "code": true + } + }, + { + "type": "text", + "text": " for the whole forward pass.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-101", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "That is the atom, the molecule, the organism, its size, and its running cost — the day-zero ground truth. Every advanced topic — attention, transformers, KV caches, quantization, distributed training — is built on exactly these pieces. When they show up, they won’t be magic. They’ll be assembly.", + "styles": {} + } + ], + "children": [] + }, + { + "id": "paragraph-102", + "type": "separator", + "props": {}, + "children": [] + }, + { + "id": "44bdadb3-da54-4a7b-9d64-0799eadb9d0e", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [], + "children": [] + }, + { + "id": "c7f19f6d-69ac-401a-8c43-1f1116ab3e3a", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [ + { + "type": "text", + "text": "Next: why generating text one token at a time makes a GPU spend 99% of its time waiting on memory instead of computing — the compute-bound vs memory-bound story that shapes every serving system.", + "styles": { + "italic": true + } + } + ], + "children": [] + }, + { + "id": "9e403107-b71e-4bb2-86b2-573648d35a88", + "type": "paragraph", + "props": { + "backgroundColor": "default", + "textColor": "default", + "textAlignment": "left" + }, + "content": [], + "children": [] + } + ], + "tableVariants": {} +} diff --git a/src/content/posts/neural-networks-from-zero/index.mdx b/src/content/posts/neural-networks-from-zero/index.mdx new file mode 100644 index 0000000..b62d9c6 --- /dev/null +++ b/src/content/posts/neural-networks-from-zero/index.mdx @@ -0,0 +1,415 @@ +--- +title: 'Neural Networks From Zero: From a Single Number to a Billion Parameters' +summary: 'A neural network never sees a word, an image, or a sound — only a list of numbers. Starting from that one fact and a single neuron, this day-zero guide builds the whole machine: how any input becomes numbers, why weights, biases, and activations each exist, how neurons stack into layers and layers into a model, and how to compute a model''s size and running cost by hand.' +authors: ['dinesh'] +date: '2026-07-12' +updated: '2026-07-18' +readMin: 14 +topic: 'Architecture' +topicId: 'architecture' +tags: ['fundamentals', 'perceptron', 'neural-networks', 'parameters', 'flops'] +--- + +Almost every explanation of neural networks starts one step too late. It shows you a diagram of circles connected by lines, tells you “these are neurons,” and moves on. But it never answers the question a genuinely curious person asks first: + +**What are those circles even looking at?** + +When someone writes `input = [0.2, 0.9, 0.1, 0.7]`, why are there _four_ numbers? Why numbers at all? What if my input is the word “cat”? A photo? A ten-second voice clip? An entire paragraph? How does _any_ of that turn into something a network can process? + +This is day zero. If you don’t understand what an input is, what a single neuron does to it, and why each piece of a neuron exists, then everything downstream — attention, transformers, “7 billion parameters,” GPUs running out of memory — is memorization, not understanding. So we’re going to build the whole thing from the smallest possible unit, and by the end you’ll be able to look at any model and compute, by hand, both _how big it is_ and _how much math it costs to run_. + +Let’s start with the single most important sentence in this entire field. + +## A network only ever sees numbers + +Here it is: + +> A neural network cannot see a word, a pixel, a sound, or a sentence. It can only see **a list of numbers**. Everything else is a translation problem that happens _before_ the network. + +That list of numbers has a name: a **vector**. When you saw `[0.2, 0.9, 0.1, 0.7]`, that was a vector of length 4 — the network’s entire universe for that one input. The “4” isn’t magic; it’s just how many numbers this particular network was built to accept. A different network might take 784 numbers, or 4096. That fixed count is called the **input dimension**. + +So the real question isn’t “how does a network understand a cat?” It’s: **how do we turn a cat — the word, or the photo — into a list of numbers?** That translation is where all the interesting representational choices live, and it’s completely separate from the network itself. Let’s do it for every kind of input. + +### Turning anything into numbers + +The rule to hold onto: the network is fixed and dumb — it eats a vector of numbers. The _encoding_ is where meaning gets packed in. Change the encoding and you change what the network can possibly learn, without touching the network at all. + +**A single quantity — the easy case.** Say you’re predicting whether it’ll rain from today’s temperature. Temperature is _already_ a number: `28.4`. Your input vector is length 1: `[28.4]`. Add humidity and pressure and you have `[28.4, 0.66, 1013]` — length 3. Each slot in the vector is called a **feature**. This is the whole reason the classic examples use 3 or 4 numbers: they’re toy problems with 3 or 4 features. Nothing deeper. + +**A category — a color, a yes/no, a day of the week.** These aren’t numbers, and you can’t just assign `Monday = 1, Tuesday = 2`, because that would trick the network into thinking Wednesday is “twice” Monday, which is nonsense. Instead we use **one-hot encoding**: one slot per possible value, all zeros except a single 1. + +```text + Monday → [1, 0, 0, 0, 0, 0, 0] + Wednesday → [0, 0, 1, 0, 0, 0, 0] + Sunday → [0, 0, 0, 0, 0, 0, 1] +``` + +Seven categories → a vector of length 7. No false ordering, no fake arithmetic. Just “which one is it.” + +**A word — where it gets interesting.** A word like “cat” is a category too, but the vocabulary has \~50,000 words, so a one-hot vector would be 50,000 numbers that are almost entirely zeros. Wasteful, and worse, it says every word is equally unrelated to every other — “cat” is exactly as far from “dog” as it is from “helicopter.” That’s clearly wrong. + +So instead we _learn_ a compact vector for each word, called an **embedding** — typically a few hundred to a few thousand numbers: + +```text + "cat" → [ 0.21, -0.44, 0.10, ..., 0.05] (say, 768 numbers) + "dog" → [ 0.19, -0.41, 0.14, ..., 0.08] ← close to "cat" + "helicopter" → [-0.88, 0.30, -0.52, ..., 0.77] ← far from both +``` + +The magic: these numbers are learned so that _similar meanings end up close together_ in the vector space. “cat” and “dog” land near each other; “helicopter” lands far away. The network never sees the letters c-a-t — it sees 768 numbers that encode what “cat” _means_, in a geometry where distance is similarity. (How text gets split into these units in the first place — “tokenization” — and how embeddings are learned are their own deep dives. For today, just hold the shape: **word → vector of numbers.**) + +
+ + + + + + + + + + cat + dog + helicopter + + + similar meaning → close + unrelated → far + + +
+ +**An image.** A grayscale photo is a grid of brightness values, each from 0 (black) to 255 (white). A small 28×28 image is 784 of those numbers. Flatten the grid row by row into one long list and you have a vector of length 784. A color photo has three such grids (red, green, blue), so a 224×224 color image is `224 × 224 × 3 = 150,528` numbers. A picture really is worth a (few hundred) thousand numbers. + +
+ + + + + + + + + 30230230230 + 23030230230 + 23023030230 + 23023023030 + + + → + [30, 230, 230, 230, 230, 30, 230, 230, 230, 230, 30, 230, 230, 230, 230, 30] + + +
+ +**Audio (an mp3).** Sound is a wave — air pressure changing over time. To digitize it, we measure that pressure many times per second (44,100 times/second for CD quality). Ten seconds of audio is `44,100 × 10 = 441,000` numbers, each a sample of the wave’s height at that instant. A vector, again — just a very long, time-ordered one. + +
+ + + + + + + + + + + +0.7+0.8−0.6 + + time → + +
+ +**A paragraph.** A paragraph is a _sequence_ of words, so it becomes a _sequence of embedding vectors_ — one vector per word (or token), in order. This is the one case where we don’t collapse to a single flat vector, because _order matters_ (“dog bites man” ≠ “man bites dog”). Handling sequences of vectors while respecting order is exactly the problem transformers were invented to solve — but that’s getting ahead of ourselves. The atom is still the same: **each word is a vector of numbers.** + +Here’s the whole picture in one table: + +| Input | How it becomes numbers | Example length | +| --- | --- | --- | +| Temperature | it’s already a number | 1 | +| Day of week | one-hot (one slot per category) | 7 | +| A word | learned embedding vector | \~768 | +| 28×28 grayscale image | flatten the pixel grid | 784 | +| 224×224 color image | flatten 3 color grids | 150,528 | +| 10s of CD-quality audio | samples of the wave over time | 441,000 | +| A paragraph | a _sequence_ of word vectors | (n words) × 768 | + +Every single one is just numbers. **That** is what the circles in the diagram are looking at. Now let’s build a circle. + +## The neuron: the atom of the whole field + +Everything — every model with billions of parameters — is built from one tiny unit repeated a staggering number of times. Understand this one unit completely and the rest is assembly. + +The unit is loosely inspired by a biological neuron, and the analogy is genuinely useful. A brain cell receives signals from many others through its dendrites. Some of those connections are strong, some weak. The cell adds up all its incoming signals (weighted by connection strength), and _if the total is strong enough, it fires._ + +
+ + + + + + + + + + + dendrites + axon + + + soma + biological neuron + + ≈ + + + + + + + + + + + + + + Σ + ＋ b + f + y + artificial neuron + + +
+ +An artificial neuron does exactly that, with three ingredients: + +
+ + + + + + + + + + + + + + + + + + + x₁ + x₂ + x₃ + x₄ + Σ + f + y + + + w₁ + w₂ + w₃ + w₄ + ＋ b + + +
+ +In math, that entire picture is one short line: + +```text + y = f( w₁·x₁ + w₂·x₂ + w₃·x₃ + w₄·x₄ + b ) + └─┬─┘ └──────────────┬───────────────┘ + activation weighted sum + bias +``` + +Three ingredients: **weights** (`w`), a **bias** (`b`), and an **activation function** (`f`). Each of them exists for a specific reason, and the fastest way to understand _why_ is to ask what breaks if you remove it. + +### Why weights? (importance knobs) + +A weight is a **dimmer switch on each input** — a learned number that says “how much should _this_ feature matter?” A large positive weight means “this input strongly pushes the answer up.” A weight near zero means “ignore this input.” A negative weight means “this input pushes the answer _down_.” + +Predicting rain: the humidity input probably deserves a big weight, the day-of-week input a tiny one. The network doesn’t know that in advance — it _learns_ the weights from data. Training a neural network is, almost entirely, the process of nudging billions of these dimmer switches until the outputs come out right. + +**Remove them** — force every weight to 1 — and every input counts equally. The neuron loses all ability to say some features matter more than others. It can no longer learn. Weights are _where the learning lives._ + +### Why a bias? (the offset that frees the decision) + +The bias is a single number added _after_ the weighted sum, and it’s the ingredient people find most mysterious — until you see the one-dimensional version. + +Strip a neuron down to one input: `y = w·x + b`. That’s the equation of a **straight line**, exactly the `y = mx + c` from school. The weight `w` is the slope; the bias `b` is the intercept — where the line crosses the axis. + +Now remove the bias: `y = w·x`. You can only make lines that **pass through the origin (0,0)**. You can spin the line to any angle, but it’s forever nailed to that one point. If the pattern in your data lives anywhere else — “predict rain when temperature is above 30” — a line through the origin simply cannot express it. The bias is what lets the neuron **slide its decision boundary anywhere**, instead of being pinned to zero. + +Analogy: the weights set the _tilt_ of the decision boundary; the bias sets its _position_. Without a bias, every boundary in the whole network is forced through the origin — like trying to describe every line on a graph using only lines that touch (0,0). You lose almost everything. + +### Why an activation function? (the one that makes deep learning possible) + +This is the subtle one, and it’s the reason the whole field is called _deep_ learning. The activation `f` is a function applied to the neuron’s output — historically an S-shaped squash, today usually **ReLU**, which is almost comically simple: `ReLU(z) = max(0, z)` — keep positives, zero out negatives. + +Why bother? Because a weighted sum plus a bias is a **linear** operation — it can only draw straight lines and flat planes. And here’s the killer fact: **stacking linear operations gives you another linear operation.** If layer 2 is just a weighted sum of layer 1, which is a weighted sum of the input, the whole thing collapses algebraically into a _single_ weighted sum. A hundred layers with no activation is mathematically identical to one layer. All that depth buys you _nothing_. + +The activation function inserts a **bend** — a non-linearity — between layers, and that bend is what stops the collapse. With bends between them, layers can _compose_: bend, then bend again, then again. Now the network can carve curved, intricate boundaries and represent genuinely complex relationships. The classic proof-by-example is **XOR** (“true when exactly one input is true”) — a pattern no single straight line can separate, and which becomes trivial the moment you have two neurons and a non-linearity to combine them. + +> Weights and bias let a neuron draw _one_ straight boundary. The activation is what lets many neurons combine into something that isn’t straight at all. No activation, no depth, no deep learning. + +That’s the entire atom: **weights** decide what matters, the **bias** frees the boundary from the origin, the **activation** bends the line so depth means something. Three ingredients, each earning its place. + +## Why one neuron is never enough + +A single neuron draws a single boundary — one line, one plane. Real problems need many. So we do the obvious thing: use lots of neurons. But _how_ we arrange them splits into two independent choices, and understanding the difference between them is understanding the shape of every model. + +### Many neurons side by side = width (one layer) + +Put several neurons in a row, all looking at the _same_ input, each with its own weights and bias. Each learns to detect a _different_ feature of that input. Feed in an image and, side by side, one neuron might learn to spot a vertical edge, another a patch of red, another a curve. They all work in parallel on the same input, and together they produce a _new_ list of numbers — one per neuron — describing the input in terms of the features they detected. + +That row of neurons is a **layer**, and the number of neurons in it is the layer’s **width**. Width = how many different features you can detect _at the same level of abstraction_, simultaneously. + +### Layers behind layers = depth (a stack) + +Now the powerful move: take the list of numbers that first layer produced and feed it as input to _another_ layer. The second layer isn’t looking at raw pixels anymore — it’s looking at the _features the first layer found_, and it learns to combine them into bigger patterns. The number of layers stacked this way is the network’s **depth**. + +This is the famous hierarchy, and it’s genuinely how deep vision networks organize themselves: + +```text + raw pixels + │ + ▼ + Layer 1 → detects edges & simple colors ("there's an edge here") + │ + ▼ + Layer 2 → combines edges into shapes ("edges here form a curve") + │ + ▼ + Layer 3 → combines shapes into parts ("curves here form an eye") + │ + ▼ + Layer 4 → combines parts into objects ("eyes + nose + fur → cat") +``` + +Each layer builds on the one below, working at a higher level of abstraction. That’s why we go deep: **depth is composition.** Width gives you _breadth_ of features at one level; depth gives you _hierarchy_ across levels. They are different axes, and a good architecture needs both. + +
+ + + + + + + + + + + + + + + + + + + input + hidden layer (width 5) + output + + +
+ +This whole arrangement — an input, one or more hidden layers, an output — is the **Multi-Layer Perceptron (MLP)**, the original neural network and still a building block inside every transformer today. + +### So how many neurons and how many layers? + +The honest answer: **there is no formula, and anyone who gives you one is selling something.** Width and depth are the network’s _capacity_ — its room to learn patterns. Too little and it can’t capture the problem (underfitting); too much and it memorizes noise, wastes compute, and gets harder to train (overfitting). The right size depends on how complex the true pattern is, how much data you have, and how much compute you can afford. + +In practice it’s chosen empirically — start from sizes that worked on similar problems, then adjust. But the _rules of thumb_ are worth knowing: more **depth** tends to help when the problem is compositional and hierarchical (vision, language); more **width** helps when there are many independent features to track at once. Modern large models are both very deep _and_ very wide, which is precisely why they have so many parameters — which brings us to the payoff. + +## From atoms to a model: counting the size + +Here’s where everything connects, and where “7 billion parameters” stops being a marketing number and becomes something you can compute yourself. + +First, collapse a whole layer into one clean object. A layer is just many neurons, each with a weight per input. Stack their weights and you get a **matrix**. A layer that takes `d_in` numbers in and produces `d_out` numbers out has: + +- a **weight matrix** of shape `d_out × d_in` — that’s `d_out × d_in` individual weights, one for every (output neuron, input) pair, +- a **bias vector** of length `d_out` — one bias per output neuron. + +And the entire layer’s computation — every neuron at once — is a single matrix multiply plus the biases, then the activation: + +```text + y = f( W · x + b ) W is [d_out × d_in], x is [d_in], b is [d_out] +``` + +So counting a model’s size is now pure bookkeeping. **A “parameter” is one single learned number** — one entry in a weight matrix, or one bias. That’s all it is. “7 billion parameters” means 7 billion individual numbers that training tuned. To find the total, count the weights in every layer and add: + +```text + parameters in one layer ≈ d_in × d_out (+ d_out biases, usually negligible) + + total parameters = sum over all layers +``` + +Worked example — a tiny 3-layer MLP that takes a 784-pixel image and classifies it into 10 digits: + +```text + Layer 1: 784 → 256 784 × 256 = 200,704 weights + Layer 2: 256 → 256 256 × 256 = 65,536 weights + Layer 3: 256 → 10 256 × 10 = 2,560 weights + ──────────────────────────────────────────────────────── + total ≈ 268,800 parameters (plus 522 biases) +``` + +That’s it. You just counted a model’s size by hand. Real transformers use the same arithmetic, just with a specific set of matrices per layer. As a rule of thumb, one transformer layer with hidden size `d` has roughly `12 × d²` weights (four matrices for attention plus a wider feed-forward block). So a model with `d = 8192` and 80 layers has about: + +```text + 80 layers × 12 × 8192² ≈ 64 billion weights (+ embeddings → ~70B) +``` + +There’s your “70B model,” derived from nothing but “a parameter is one number in a matrix, and I can count matrices.” + +**And the size on disk or in memory** follows immediately, because each parameter is just a number stored in some precision. In FP16, one number takes 2 bytes: + +```text + 70,000,000,000 params × 2 bytes = 140 GB +``` + +That 140 GB is why a 70B model doesn’t fit on a single 80 GB GPU, why quantization (storing each number in 1 byte instead of 2) is such a big deal, and why “how many GPUs do I need?” has a concrete answer. It all traces back to _one number per parameter._ + +## The cost of running it: FLOPs + +One last piece, and it’s the ground truth for every performance conversation you’ll ever have. We can count the _math_ a model does just as precisely as we counted its size. + +Look again at what one weight does inside `W · x`: it gets **multiplied** by its input, and that product gets **added** into a running sum. One multiply, one add — two arithmetic operations. Engineers call a floating-point operation a **FLOP**, so: + +> Every parameter costs **2 FLOPs** each time one input passes through the network — one multiply and one add. + +For one input (in language models, one “token”) flowing through a model with `N` parameters, the total work is therefore about: + +```text + FLOPs per token ≈ 2 × N +``` + +For our 70B model: `2 × 70,000,000,000 = 140 billion FLOPs` to process a single token. Not “a lot” — _exactly_ 140 billion, a number you derived from first principles. + +This single formula is the seed of an enormous amount of systems knowledge. It’s why processing a long prompt (many tokens at once, reusing the same weights) behaves completely differently from generating one token at a time (dragging all 140 GB of weights out of memory to do a comparatively tiny 140 GFLOPs of math). That contrast — compute-bound versus memory-bound — is the beating heart of why inference systems look the way they do. But that’s the next deep dive. For now, sit with what you can already do: + +## What you can now do + +Start to finish, with no hand-waving, you can now explain: + +- **Input** — a network only ever sees a vector of numbers, and _every_ input type (number, category, word, image, audio, paragraph) is a translation into that vector. The encoding is where meaning gets packed in. +- **The neuron** — `y = f(w·x + b)`: weights are learned importance knobs, the bias frees the decision boundary from the origin, and the activation’s bend is what makes depth meaningful. +- **Width vs depth** — many neurons side by side detect many features at one level; stacked layers compose features into a hierarchy. Both axes, for different reasons. +- **Model size** — a parameter is one number in a matrix; count the matrices and you have the parameter count; multiply by bytes-per-number and you have the memory. +- **Model cost** — two FLOPs per parameter per token; `2N` for the whole forward pass. + +That is the atom, the molecule, the organism, its size, and its running cost — the day-zero ground truth. Every advanced topic — attention, transformers, KV caches, quantization, distributed training — is built on exactly these pieces. When they show up, they won’t be magic. They’ll be assembly. + +--- + +_Next: why generating text one token at a time makes a GPU spend 99% of its time waiting on memory instead of computing — the compute-bound vs memory-bound story that shapes every serving system._