From 626c5f20a6df98dea7d3072cdad862715adf5647 Mon Sep 17 00:00:00 2001 From: Francesco Muia Date: Tue, 21 Jul 2026 12:28:24 +0200 Subject: [PATCH 1/3] Unified deck: labs hook + short geo arc in one file (apps/unified_slides) - apps/unified_slides_src.html (canonical src) -> unified_slides.html via make html-unified; build_unified_slides.py merges labs pins + nb07 shards into one token/DATA pass; verify_unified_deck.py runs geo claims (subset) + labs claims + deck-wide sweeps (331 checks green) - Part 1 rewired closer->opener (forward teasers, no notebook refs); geo slides gain Part-1 callbacks; em-dashes now zero deck-wide; labs Close renamed "One breath" (the full geo deck owns data-t "Close" in claims) - NEW slide 2 "PyMC Labs: what we do"; session title with geo author descriptions and both emails for Francesco - emphasis + speech files now describe the unified ~55-min session - refactors: build_geo_slides exposes build_bundle/inject_data/inject_nums; verify_labs_deck takes --deck (html-geo-sh / html-labs regression-green) Co-Authored-By: Claude Opus 4.8 (1M context) --- causal-marketing-pymc/Makefile | 11 + .../apps/build_geo_slides.py | 77 +- .../apps/build_unified_slides.py | 97 + .../apps/geo_lift_lecture_emphasis.md | 291 ++ .../apps/geo_lift_lecture_speech.md | 694 +++++ .../apps/unified_slides.html | 2696 +++++++++++++++++ .../apps/unified_slides_src.html | 2684 ++++++++++++++++ .../apps/verify_labs_deck.py | 15 +- .../apps/verify_unified_deck.py | 36 + 9 files changed, 6565 insertions(+), 36 deletions(-) create mode 100644 causal-marketing-pymc/apps/build_unified_slides.py create mode 100644 causal-marketing-pymc/apps/geo_lift_lecture_emphasis.md create mode 100644 causal-marketing-pymc/apps/geo_lift_lecture_speech.md create mode 100644 causal-marketing-pymc/apps/unified_slides.html create mode 100644 causal-marketing-pymc/apps/unified_slides_src.html create mode 100644 causal-marketing-pymc/apps/verify_unified_deck.py diff --git a/causal-marketing-pymc/Makefile b/causal-marketing-pymc/Makefile index f448423..ced052c 100644 --- a/causal-marketing-pymc/Makefile +++ b/causal-marketing-pymc/Makefile @@ -80,6 +80,10 @@ html-iv-slides: ## build the Ch.13 interactive slide deck (apps/iv_slides.html) # /*__DATA__*/, inlines MathJax. Edit apps/iv_slides_src.html, then rebuild. $(CORE)/python apps/build_iv_slides.py +html-iv-slides-sh: ## build the management-first rework of the Ch.13 deck (apps/iv_slides_sh.html) + # Same pipeline as html-iv-slides, different template/output. Edit apps/iv_slides_sh_src.html. + $(CORE)/python apps/build_iv_slides_sh.py + html-geo: ## build the standalone Ch.9 slide deck (apps/geo_lift_slides.html) — one offline file # Assembles apps/geo_slides_src.html: shard-token substitution (numbers never retyped), # the nb07 lecture bundle for the live charts, book figures inlined as PNGs, MathJax vendored. @@ -98,6 +102,13 @@ html-geo-sh: ## build the SHORT Ch.9 deck (apps/geo_lift_sh.html) — classical $(CORE)/python apps/build_geo_slides.py --src apps/geo_slides_sh_src.html --out apps/geo_lift_sh.html $(CORE)/python apps/verify_geo_deck.py --deck apps/geo_lift_sh.html --allow-missing-slides +html-unified: ## build the UNIFIED session deck (apps/unified_slides.html) — labs hook + short geo arc + # Part 1 (real PyMC Labs engagements) in front of the short classical geo deck, one file. + # Tokens from BOTH sources ({{labs.*}} pins + {{nb07*.*}} shards), one merged DATA bundle. + # Edit apps/unified_slides_src.html, then rebuild. Narrative: apps/geo_lift_lecture_emphasis.md. + $(CORE)/python apps/build_unified_slides.py + $(CORE)/python apps/verify_unified_deck.py + site: ## export the WASM-safe apps to static, in-browser pages under site/ $(CORE)/marimo export html-wasm apps/experiment_design.py -o site/experiment_design --mode run $(CORE)/marimo export html-wasm apps/iv_lecture.py -o site/iv_lecture --mode run diff --git a/causal-marketing-pymc/apps/build_geo_slides.py b/causal-marketing-pymc/apps/build_geo_slides.py index a91d5b3..e7c6f02 100644 --- a/causal-marketing-pymc/apps/build_geo_slides.py +++ b/causal-marketing-pymc/apps/build_geo_slides.py @@ -126,24 +126,9 @@ def naive_grid() -> dict: "diff-in-differences", "synthetic control"]} -def main(src: Path = SRC, out: Path = OUT) -> None: - html = src.read_text() - tokens = load_tokens() - - # 1 · scalar tokens ------------------------------------------------------------------ - used, missing = set(), [] - def sub_token(m: re.Match) -> str: - key = m.group(1).strip() - if key not in tokens: - missing.append(key) - return m.group(0) - used.add(key) - return tokens[key] - html = re.sub(r"\{\{([a-z0-9_.]+)\}\}", sub_token, html) - if missing: - sys.exit("FAIL: unknown tokens (not in the shards): " + ", ".join(sorted(set(missing)))) - - # 2 · the data bundle ---------------------------------------------------------------- +def build_bundle() -> dict: + """Assemble the full DATA bundle (nb07 lecture bundle + shard scalars + aliases + + real-data bundle + extras + the naive grid). Shared with build_unified_slides.py.""" if not BUNDLE.exists(): sys.exit(f"FAIL: {BUNDLE} missing — re-execute nb07 (its lecture-bundle cell writes it).") bundle_meta = json.loads(BUNDLE.read_text()) @@ -195,26 +180,56 @@ def sub_token(m: re.Match) -> str: bundle_meta["naive_grid"] = naive_grid() print(f"naive grid: {len(bundle_meta['naive_grid']['s'])}×" f"{len(bundle_meta['naive_grid']['sd'])} cells from cmp.dgp.geo_panel") + return bundle_meta + +def inject_data(html: str, bundle_meta: dict) -> str: + """Replace the JSON object after the /*__DATA__*/ marker with the baked bundle.""" if "/*__DATA__*/" not in html: sys.exit("FAIL: template has no /*__DATA__*/ marker.") j = html.index("{", html.index("/*__DATA__*/")) _, end = json.JSONDecoder().raw_decode(html, j) - html = html[:j] + json.dumps(bundle_meta, separators=(",", ":")) + html[end:] + return html[:j] + json.dumps(bundle_meta, separators=(",", ":")) + html[end:] + - # 2b · the N map: formatted number strings the deck's JS writes into prose spans. - # The template's /*__NUMS__*/{...}/*__ENDNUMS__*/ map supplies the KEYS; every value is - # re-derived from the shards (nb07.), so a number can no more go stale here than in - # the {{token}} prose layer. A key missing from the shards is a build error. +def inject_nums(html: str, tokens: dict[str, str]) -> str: + """The N map: formatted number strings the deck's JS writes into prose spans. + The template's /*__NUMS__*/{...}/*__ENDNUMS__*/ map supplies the KEYS; every value is + re-derived from the shards (nb07.), so a number can no more go stale here than in + the {{token}} prose layer. A key missing from the shards is a build error.""" nm = re.search(r"/\*__NUMS__\*/(\{.*?\})/\*__ENDNUMS__\*/", html, re.S) - if nm: - n_keys = list(json.loads(nm.group(1))) - missing_n = [k for k in n_keys if f"nb07.{k}" not in tokens and f"nb07b.{k}" not in tokens] - if missing_n: - sys.exit("FAIL: N-map keys not in the shards: " + ", ".join(missing_n)) - nmap = {k: tokens.get(f"nb07.{k}", tokens.get(f"nb07b.{k}")) for k in n_keys} - html = html[:nm.start(1)] + json.dumps(nmap, separators=(",", ":")) + html[nm.end(1):] - print(f"N map: {len(nmap)} formatted numbers injected from shards") + if not nm: + return html + n_keys = list(json.loads(nm.group(1))) + missing_n = [k for k in n_keys if f"nb07.{k}" not in tokens and f"nb07b.{k}" not in tokens] + if missing_n: + sys.exit("FAIL: N-map keys not in the shards: " + ", ".join(missing_n)) + nmap = {k: tokens.get(f"nb07.{k}", tokens.get(f"nb07b.{k}")) for k in n_keys} + html = html[:nm.start(1)] + json.dumps(nmap, separators=(",", ":")) + html[nm.end(1):] + print(f"N map: {len(nmap)} formatted numbers injected from shards") + return html + + +def main(src: Path = SRC, out: Path = OUT) -> None: + html = src.read_text() + tokens = load_tokens() + + # 1 · scalar tokens ------------------------------------------------------------------ + used, missing = set(), [] + def sub_token(m: re.Match) -> str: + key = m.group(1).strip() + if key not in tokens: + missing.append(key) + return m.group(0) + used.add(key) + return tokens[key] + html = re.sub(r"\{\{([a-z0-9_.]+)\}\}", sub_token, html) + if missing: + sys.exit("FAIL: unknown tokens (not in the shards): " + ", ".join(sorted(set(missing)))) + + # 2 · the data bundle, 2b · the N map ----------------------------------------------- + html = inject_data(html, build_bundle()) + html = inject_nums(html, tokens) # 3 · book figures ------------------------------------------------------------------- fig_names = sorted(set(re.findall(r"", html))) diff --git a/causal-marketing-pymc/apps/build_unified_slides.py b/causal-marketing-pymc/apps/build_unified_slides.py new file mode 100644 index 0000000..d2c7789 --- /dev/null +++ b/causal-marketing-pymc/apps/build_unified_slides.py @@ -0,0 +1,97 @@ +"""Generate the UNIFIED SDA session deck: apps/unified_slides.html. + +Part 1 ("Causal Inference in the Wild": real PyMC Labs engagements, the business hook) +spliced in FRONT of the short classical geo deck (synthetic control, Acts I-IV), closing +with the labs synthesis slides. One file, one chrome (the geo deck's, a strict superset), +one DATA bundle. The narrative this deck implements: apps/geo_lift_lecture_emphasis.md; +the spoken script: apps/geo_lift_lecture_speech.md. + +Inputs, none hand-typed: + tokens : {{nb07*.*}} from the executed notebook shards (build_geo_slides.load_tokens) + PLUS {{labs.*}} from apps/labs_deck_data.json (blog-pinned facts with source + URLs; build_labs_slides.load_pins). The two prefixes are disjoint (asserted). + DATA : the geo bundle (build_geo_slides.build_bundle) merged with the labs chart keys + (the deterministic counterfactual schematic + the pinned ROAS pair). The key + sets are disjoint (asserted), so the labs figure code reads the same DATA + global as the geo figures. + SOURCES: the labs Sources backup table rows at . + MATHJAX: the vendored tex-svg build, inlined once at /*__MATHJAX__*/. + +Build: .venv/bin/python apps/build_unified_slides.py (or `make html-unified`) +Template: apps/unified_slides_src.html (canonical; edit it, then rebuild). +Verify: .venv/bin/python apps/verify_unified_deck.py (geo claims subset + labs claims) +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import build_geo_slides as bg +import build_labs_slides as bl + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent +SRC = HERE / "unified_slides_src.html" +OUT = HERE / "unified_slides.html" + + +def main() -> None: + if not SRC.exists(): + sys.exit(f"FAIL: {SRC} missing.") + html = SRC.read_text() + + # 1 · scalar/quote tokens from BOTH sources ------------------------------------------ + pins = bl.load_pins() + tokens = bg.load_tokens() + labs_tokens = {k: str(v["text"]) for k, v in pins.items()} + dup = set(tokens) & set(labs_tokens) + if dup: + sys.exit("FAIL: token collision between shards and labs pins: " + ", ".join(sorted(dup))) + tokens |= labs_tokens + + used, missing = set(), [] + + def sub_token(m: re.Match) -> str: + key = m.group(1).strip() + if key not in tokens: + missing.append(key) + return m.group(0) + used.add(key) + return tokens[key] + + html = re.sub(r"\{\{([a-z0-9_.]+)\}\}", sub_token, html) + if missing: + sys.exit("FAIL: unknown tokens (neither shards nor labs pins): " + + ", ".join(sorted(set(missing)))) + + # 2 · ONE data bundle (geo + labs chart keys), 2b · the N map ------------------------ + bundle = bg.build_bundle() + labs_bundle = bl.bake_bundle(pins) + overlap = set(bundle) & set(labs_bundle) + if overlap: + sys.exit("FAIL: DATA key overlap between geo and labs bundles: " + ", ".join(sorted(overlap))) + bundle.update(labs_bundle) + html = bg.inject_data(html, bundle) + html = bg.inject_nums(html, tokens) + + # 3 · the labs Sources backup slide -------------------------------------------------- + if "" not in html: + sys.exit("FAIL: template has no marker.") + html = html.replace("", bl.sources_rows(pins), 1) + + # 4 · MathJax ------------------------------------------------------------------------ + if "/*__MATHJAX__*/" not in html: + sys.exit("FAIL: template has no /*__MATHJAX__*/ marker.") + html = html.replace("/*__MATHJAX__*/", bg.mathjax_js(), 1) + + OUT.write_text(html) + size = OUT.stat().st_size / 1e6 + n_slides = html.count('
` blocks. It keeps its full short-deck arc and STOPS at the classical verdict +("measure first"). Each act now calls back to Part 1's cases (🔗). Slide numbers below are the unified deck.* + +### 9 · Part 2 divider / Title — "Synthetic Control" +- 🎯 Reset the frame: "One treated market, no experiment, €4M on the line: did the campaign work?" Keep the + hero figure. 💬 "Colgate at least had a clean before-and-after. Take even that away — one market, no + control, no experiment. This is the general case, and it's the most common one." + +## ACT I — The question (~9 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 10 | The boardroom question | 🎯 Three questions IN ORDER: attribution → significance → decision. The metro was CHOSEN, not randomized — that's why it's hard. 💬 "Most companies jump to question 3 with an unchecked answer to question 1." 🔗 "Colgate's launch was national; here it's one metro — same missing number." | 📐 "unbiased" fold-out (only if asked). | +| 11 | The data (live) | 🎯 The counterfactual: the one number in no file — what the metro would have sold anyway. Show live that the naive bump changes when you subtract the crowd. 🖼 Rooster-takes-credit-for-sunrise ("the campaign made sales rise"). | 📐 The three technical readings, folded. | +| 12 | Poll: the 12% bump ✋ | 🎯 Naive before/after isn't "roughly right + noise": the tide can HIDE a real effect (as here) or inflate a fake one. Run the poll — commitment makes the teaching moment. | — | +| 13 | Potential outcomes | 🎯 Causal effect = gap between two histories; only one is observed. The 2×2: 3 cells are data, 1 is missing, every method fills that cell. Per-week vs 20-week total are different targets; the €4M rides on the total. 🖼 Doctor Strange "14,000,605 futures, we observe one." | 📐 Potential-outcome notation box — fold. | +| 14 | The data-generating model | 🎯 ONE honest confession: data is simulated SO every method can be graded against a known truth (~€284k) — the same recover-the-truth contract as Colgate. | 📐 The whole equation. "Four ingredients incl. per-market sensitivity" and move. | +| 15 | Simulate the world (live) | 🎯 ONE slider (macro shock): turbulence destroys before/after. 💬 "Compared to what?" 🖼 optional: stormy-sea "a rising tide lifts all boats" visual. | ⏭ Other sliders; the full "what sliders teach" list. | + +## ACT II — The counterfactual (~11 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 16 | Abadie's idea: the synthetic twin | 🎯 THE core idea: no twin exists, so blend markets into one. Twin matches RESPONSES to shocks, not levels → survives storms. Anchors: Meta GeoLift, Google geo tools. 🖼 "assemble a twin from parts" cartoon, tasteful. | 📐 Factor-model equation; pedigree fold-out (one clause). | +| 17 | The estimator | 🎯 3 business facts: (1) fitted ONLY pre-launch = no peeking, (2) weights are a readable recipe (40%+32%+17%), (3) effect = subtraction. 🔗 "The honest, weights-on-the-table version of what Colgate's model did in time." | 📐 argmin / simplex formulas — folded on the slide. | +| 18 | Inside the convex hull (live) | 🎯 Coffee+milk can't make orange juice: a blend can't reach outside the donors' range → the method can REFUSE. 💬 SIGNATURE MANAGEMENT LINE: "Ask any vendor — when does your method refuse to answer? If 'never', walk away." 🖼 big red OFF switch / "computer says no". (This is where the short deck teaches the off-switch.) | 📐 "simplex", "hull" — use the demo, not the words. | +| 19 | What the constraint buys (live) | 🎯 Better fit to the past ≠ better model. Drop the constraint: OLS fits tighter, predicts worse, uses absurd weights (−80%). 💬 "Anyone selling a model on past fit is selling the wrong metric." 🖼 the overfitting meme (squiggly line through every point). | 📐 n_eff (inverse Herfindahl) block — fold. | +| 20 | Why not just forecast it? | 🎯 Forecasting ≠ counterfactual: "what comes next" vs "what would have happened in a world that never happened." You cannot cross-validate the counterfactual → pick defendable assumptions, not demo accuracy. Auditability is a business feature. 🖼 confidently-wrong-AI / "trust me bro" robot. | The sweep numbers (one clause: "clearly worse across 24 worlds"). | +| 21 | Every shortcut is an assumption | 🎯 KEY SLIDE of Act II. Every method = a hidden claim about the counterfactual. Live dials: DiD dies when markets react differently. 💬 "A wrong method does not look wrong, and more data does not fix it." | 📐 The formulas — point at the dials instead. | +| 22–23 | Before DiD: two one-difference estimators / When does DiD work? (algebra) | ⏭ Compress BOTH to: before/after fails when the world moves; controls fail on size; DiD needs parallel reactions and dies silently; SC chooses weights instead of hoping. 💬 KEEP: "More weeks of data buy precision around the same wrong number." | 📐 All derivations, Var(B), both fold-outs. | +| 24 | Identification: what must hold | 🎯 Due-diligence framing: 4 assumptions, 3 testable (PASS), 1 untestable (spillover) — flag it, it returns at SUTVA. 🔗 "The untestable one is the second-launch leak you named for Colgate." Managers get checklists and audits. | 📐 Gate arithmetic. | + +## ACT III — Is it real, how big? (~8 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 25 | Poll: is it real? ✋ | 🎯 Open with the absorbed line: with ONE treated metro a t-test is not weak, it is UNDEFINED. Then let THEM invent permutation inference: 29 fake "treatments" measure the method's luck. Rank 1/30 → p ≈ 3%. 💬 "You just reinvented a rigorous test using common sense." 🖼 "is this a pigeon? / is this a real effect?" | The word "exchangeable". | +| 26 | Placebo-in-space (live) | 🎯 The picture: green line clear of the grey luck-cloud. 💬 "Real ≠ profitable" — hold applause for Act IV. | Abadie hygiene rule (one clause for analysts). | +| 27 | Interval by test inversion (live) | 🎯 Only the OUTPUT and its role: [€195k, €335k], built with NO model assumptions → it is the REFEREE all later models must answer to. | 📐 The whole inversion mechanics (steps ①–④, quantile algebra) — fold. | +| 28 | Falsification 1: placebo-in-time | 🎯 Fake launch 10 weeks early → finds nothing → pass. 💬 "Any vendor claiming lift should show their method finds zero where nothing happened." | Broken-world checkbox unless time allows. | +| 29 | Falsification 2: spillover (SUTVA) | 🎯 MANAGEMENT SLIDE, don't rush: the untestable leak (ads reach the commuter belt = your controls). 🔗 "This is the second-launch failure you named for Colgate, in space." Two takeaways: bias only SHRINKS the number (conservative floor), and 💬 "Design beats analysis" — keep controls off the media footprint at TEST-DESIGN time. | 📐 φ sweep mechanics. | +| 30 | What the statistics says | 🎯 Bank the 3 numbers (p≈0.033 / €260k / [195, 335]). Then the trap: 💬 "260 for 75, 3.5×, roll it out — every number true, conclusion doesn't follow." Cliffhanger into money. | Truth-check detail (one clause: truth 284 is inside the interval). | + +## ACT IV — The decision in euros (~12 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 31 | Poll: the margin ✋ | 🎯 Profit = margin × lift − cost ≈ €16k net. Option A (sales − cost = 185) is THE boardroom classic: revenue treated as profit. 🖼 "revenue vs profit — corporate needs you to find the difference" (callback to Nürnberger). | — | +| 32 | The margin trap (live) | 🎯 CHEAPEST LESSON OF THE DAY: break-even ROAS = 1/margin, not 1. Slide the margin: software 1.1 vs grocery 5.0 — same campaign, opposite verdicts. 💬 "ROAS > 1 is a vanity bar." | 📐 iROAS notation. | +| 33 | The decision space (live) | 🎯 "Looks profitable on average" ≠ "provably profitable": profit range [68, 117] straddles the 75 price. Our campaign lives in the dangerous zone. 🖼 the statistician who drowned crossing a river "1m deep on average." | 📐 The price-sweep table. | +| 34 | What you say to the board (classical) | 🎯 This memo answers the €75k PILOT only: ① real (p=0.033) ② big enough on the point (€16k net) ③ NOT confident (profit [68,117] straddles 75). Verdict: renegotiate or buy certainty with a bigger test. 💬 "This says nothing yet about the €4M — it is not the pilot multiplied." The one question it CAN'T answer ("78% chance it pays") is the forward tease to the probability view we add later. | Table row-by-row. | +| 35 | What €4M actually buys | 🎯 THE FREQUENTIST VERDICT, no probability needed. Carry the INTERVAL not the point, be maximally generous (δ=1, perfect transport): even then the 90% profit band is [−€360k, +€2.25M] → STRADDLES ZERO. 💬 "At the most optimistic assumption we can write down, we cannot show the €4M pays." Below δ≈0.64 it provably loses. ✋ ENGAGEMENT: run naive +€860k and poll for approval BEFORE carrying the interval. | 📐 δ-decomposition (already folded in a `
` on the slide — leave it folded). | +| 36 | Measure it, don't guess it | 🎯 MOST PROTECTED SLIDE — never cut. The call: even the best case straddled zero → do NOT commit €4M on this evidence ("not yet, not on this"). WHY analysis can't fix it: the interval is wide because there is ONE treated market; only more treated markets narrow it. THE FIX: (1) 💬 "before you spend a million on evidence, spend an hour checking you haven't already bought it"; (2) MEASURE — 8 markets that look like the nation, national intensity, €1.07M, so measured lift IS national lift; (3) pre-commit the release rule or the test is theatre. 🔗 "This is HelloFresh's loop, bought once — the counterfactual, anchored by an experiment." ✋ "so what would you spend?" → honest "not €4M yet." | 📐 Sizing arithmetic, p-floor — folded. | + +### BRIDGE — Part 2 → FINALE (say this, ~15s) +💬 "Notice what that last instinct was: don't argue about the number, go get a cleaner one. Counterfactual, +then experiment, then price the decision. That is not a synthetic-control trick — it is the pattern under +every real engagement, including the three we opened with." + +--- +--- + +# FINALE — the pattern, and the close (~2 min) + +*The labs closing slides come HOME here — written as a synthesis, they belong at the end. This is the +bookend: we opened with three cases, we close on the one pattern under all of them. When Part 3 (IV) is +appended later, this finale slides to after IV; for the labs+geo build it closes the session.* + +### 37 · The tools are open-source — and they are the syllabus (moved from Part 1) +- 🎯 Provenance beat, ~45s: CausalPy (synthetic control, ITS, DiD, RDD) and pymc-marketing are the tools + behind everything you just saw; one client's budget-allocation approach came back as a pull request + (Bolt); the consultancy's own webinar agenda is literally this session's syllabus (incl. the IV close). +- Placement rationale: it lands better here than in the hook — after Part 2 the audience knows what the + tools actually DO. Cuttable first if time runs out. +- 🖼 Package logos (already in deck). No meme. + +### 38 · The pattern in every engagement +- 🎯 Collapse the whole session to three lines: the deliverable is a counterfactual; an experiment anchors + every observational model; uncertainty prices the decision (boards act on headroom and P(pays), not a point). +- 🖼 The Decision Lab gag table already in the labs deck: vanilla agent → "Confidently wrong."; PyMC Labs' + honest system → "No valid model found. Run a geo-holdout experiment." 💬 "Even the machine's best answer + was this lecture's advice: run the experiment." + +### 39 · One breath (close) +- 🎯 Deliver the one-breath summary and STOP. 💬 "The toolkit a Bayesian consultancy sells: counterfactuals, + anchored by experiments, decided with their uncertainty on the table." Do not add content after it. +- Say-hello / repo line stays. (When IV is added, hold this slide for the very end of the 1.5h.) + +### Backup · Sources +- Auto-generated pin table from labs_deck_data.json + geo claims. Backup only. + +--- +--- + +## The management take-homes (if you must cut 10 minutes, protect these) + +1. **The counterfactual** (slides 4, 11, 13): a campaign's value is a missing number — "compared to what?" is always the question. +2. **Every method is a hidden claim** (slide 21): a wrong claim doesn't look wrong, and more data doesn't fix it. +3. **Calibration is the product** (slides 6–7, 36): an experiment anchors every model; it is not a luxury. +4. **When does the method refuse?** (slide 18): a tool that never says "I can't answer" is the one to distrust. +5. **Design beats analysis** (slides 24, 29): the untestable assumption (spillover) is controlled when you DESIGN the test. +6. **The margin trap** (slides 31–32): break-even ROAS = 1/margin. The single cheapest, most reusable lesson. +7. **Significance ≠ worthiness** (slides 30, 34, 36): "real" and "worth it" are different questions; the expensive mistakes live between them. +8. **The interval is the recommendation** (slides 35–36): even the BEST case for the €4M straddles zero — the analysis itself says "measure first." A wide interval from one treated unit is fixed by more units, not more analysis. + +## If you are told "you have 45 minutes" on the day + +Cut in this order: finale slide 37 (provenance) → geo 22–23 (keep the one quoted line) → 19 → 14–15 (merge +to one sentence each) → 33 (fold its point into 34) → the second poll of whichever act is running long. +Never cut: 2–6 (who we are + the hook), 10–13, 16, 18, 21, 25–26, 29, 31–32, 34–36 (35–36 above all), 38–39. + +## Meme / visual budget (keep it to ~6–8 across the whole session — scarcity keeps them funny) + +Highest-value, lowest-risk spots: slide 3 ("this is fine" CMO, no holdout) · slide 4 (two-Spider-Men +counterfactual) · slide 6 (overfitting / target-around-arrow) · slide 13 (Doctor Strange one-future) · +slide 18 (big red OFF switch — the method refuses) · slide 32 (revenue vs profit "same picture") · slide 33 +(river 1m deep on average). Everything else stays a clean figure. All optional; the polls and live widgets +are the primary engagement, memes are seasoning. diff --git a/causal-marketing-pymc/apps/geo_lift_lecture_speech.md b/causal-marketing-pymc/apps/geo_lift_lecture_speech.md new file mode 100644 index 0000000..902110d --- /dev/null +++ b/causal-marketing-pymc/apps/geo_lift_lecture_speech.md @@ -0,0 +1,694 @@ +# Unified Lecture — Full Speech (Management-Class Version) + +The spoken script for `apps/unified_slides.html` (40 slides): Part 1 "Causal Inference in the +Wild" (real PyMC Labs engagements, the hook) → Part 2 "Synthetic Control" (the short classical +arc, Acts I–IV) → the finale (the pattern, one breath). Slide numbers match the deck's counter. +The per-slide priorities live in `apps/geo_lift_lecture_emphasis.md`; this file is what you say. + +**Audience adjustment.** Written for a general management class, not an analytics class. Every +slide is presented as a **business decision problem first**. The math on the slides is optional +depth: whenever a slide is math-heavy, the script gives a short business translation to say out +loud, plus one sentence to hand the math to the curious ("the formula is in the fold for those +who want it"). You never need to derive anything live. + +**Language.** Short sentences. Plain words. No idioms. Read it as written or in your own words. + +**Timing guide (~55 min).** Part 1 ~10 min (title+labs ~2, poll 2.5, Colgate 2, break-it 2, +calibrate 1.5, HelloFresh 0.75, Nürnberger 2) · Act I ~9 · Act II ~11 · Act III ~8 · +Act IV ~12 (the €4M pair, slides 35–36, is the climax: protect it) · Finale ~2.5. +A third part (instrumental variables) will follow in the 1.5h session; this script leaves the +close open for it. + +Slides marked **[FAST]** can be covered in 30–60 seconds for this audience. +Slides marked **[SKIPPABLE MATH]** can be summarized in one sentence; the fold stays closed. + +--- +--- + +# PART 1 — CAUSAL INFERENCE IN THE WILD + +## Slide 1 — Title + +> Good morning. Before any theory, I want to show you what this subject looks like when real +> companies pay for it. Three client stories, ten minutes. Then we take the hardest version of +> their problem and build the machinery ourselves, end to end, on a four-million-euro decision. +> +> Nothing today needs a statistics background. It needs the willingness to ask one question, +> stubbornly: compared to what? + +## Slide 2 — PyMC Labs: what we do + +> One minute on who is talking, because it explains why these stories exist. +> +> PyMC Labs is a Bayesian modeling consultancy. Two things at once: we build custom +> decision-making models for companies where off-the-shelf tools fall short, and we maintain +> the open-source libraries this field runs on — PyMC, PyMC-Marketing, CausalPy. +> +> The business model matters for how you read the cases. We sell senior modeling expertise, not +> software licenses. Engagements come in three shapes — you see them in the table: a fixed-scope +> project when a company has one high-value decision problem; an advisory retainer when a firm +> is building its own team; and training when an analytics team is standardizing on the tools. +> The libraries are free. They build trust and reach. The consulting monetizes the expertise +> behind them. Small senior teams, no junior pyramid, and the client keeps the model at the end — +> not a black box. +> +> The client list for today is on the slide: Colgate-Palmolive, HelloFresh, a German insurer +> called Nürnberger Versicherung. You meet three of them in the next ten minutes. [The fold has +> the full areas-of-interest list, for later.] + +## Slide 3 — Poll: you are the consultant + +> Now you are the consultant, and the phone rings. [Read the poll.] Colgate-Palmolive: +> "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it +> stealing share from competitors, or from our own brands?" Four instruments on the slide. +> Which do you reach for first? +> +> [Collect hands on each option. Then reveal.] +> +> C. And here is why the others fail. An A/B test needs something left to randomize — the +> launch already happened, everywhere. Difference-in-differences needs a region without the +> product — there is none. An instrument for shelf placement — the shelf changed in every store +> at once. When the world gives you no comparison group, one move remains: learn the world +> before the launch, project it forward, and read the gap. That projection is called a +> counterfactual, and it is the single product every case today has in common. +> +> If you were not sure which to pick — good. That gap is exactly what this session is for. + +## Slide 4 — Colgate: incremental, or cannibalistic? + +> Here is the brief in the client's own words: "estimate the counterfactual sales of all +> products, had the new product not been introduced." Read that twice. The client is asking to +> buy a number that exists in no file, no dashboard, no report anywhere on earth: what would +> have happened without the launch. +> +> Why does it matter? Two words on the slide. **Incremental** sales are won from competitors or +> category growth — those justify the launch. **Cannibalistic** sales are stolen from your own +> shelf — those just moved euros between your own products. The launch verdict is the split. +> +> And notice the professional discipline, because it returns all day: before trusting the model +> on real data, they graded it on simulated data where the answer was planted. Planted truth: +> fifty percent incremental. The model's interval: 49 to 59 percent. It recovers the truth, so +> it earns the right to be believed. Remember this contract — recover a known truth first. +> We will hold every method today to it. +> +> [If asked for the method's name: "a multivariate Bayesian interrupted time series, later a +> nested-logit choice model — the names matter less than the projection idea."] + +## Slide 5 — What would break it? (open floor + live widget) + +> Before you trust that number, earn your fee. You are Colgate's CMO. Name one real-world event +> that would make this estimate wrong. [Open floor, two minutes. Typical answers: a competitor +> launch, a price change, a pandemic, a second launch of our own.] +> +> [Reveal.] The model itself confessed one: **a second product launch** inside the estimation +> window. When that happened, the same machinery reported 64 to 76 percent against a planted +> truth of one hundred. Why? The counterfactual learned "normal growth" from a window that +> already contained another launch — it absorbed part of the very effect it was meant to isolate. +> +> [Hand a student the widget: tick "a second product launches nearby", slide the month.] Watch +> the readout bend: inside the window it under-credits; near the edge it over-credits. Same +> machinery, wrong world, wrong number — and no error message anywhere. +> +> Two things to keep. First: an honest consultancy prints this failure in the same post as the +> win. When you buy analytics, ask for the failure cases; if there are none, walk away. Second: +> hold onto the phrase "something leaked into my comparison". It comes back this afternoon with +> a formal name, and it is the one assumption no statistical test can rescue. + +## Slide 6 — Why calibrate? A model alone can rank channels backwards + +> Second case, and first a warm-up from the tools themselves. A marketing-mix model — MMM on +> the slide — explains total sales as the sum of each channel's contribution, fitted on the +> spend history you already have. Every large advertiser runs one. +> +> Here is the published tutorial demo. Fit on observational spend alone, the model reported +> channel one earning 93 and channel two earning 171 — sorry, the reverse: it ranked them +> **backwards**. The truth was the direct opposite of what the dashboard said. And nothing on +> the dashboard warns you. The fit was good. The intervals were tight. The ranking was wrong. +> +> [Tick the toggle: "add two lift tests per channel".] A lift test is a small experiment: nudge +> one channel's spend by a known amount, measure the sales it causes. Feed two of those per +> channel into the model, and the ranking flips to the truth. +> +> The lesson of the whole session in one line: **the experiment is the model's anchor.** A model +> without an anchor can be confidently wrong. Hold that thought; the entire second half is about +> how to build the anchor when you only get one. + +## Slide 7 — HelloFresh runs the loop, at industrial scale **[FAST]** + +> Does anyone run this loop for real? HelloFresh does, at industrial scale. An always-on +> marketing-mix model, calibrated by a continuous stream of field experiments — the panel's own +> agenda says "calibrated to ensure consistency with incrementality measurements" — and the +> calibration cut prediction variance by sixty percent. The picture is the whole slide: the +> model asks for ground truth, the experiment supplies it, the counterfactual reads it out. +> +> One sentence to remember for the final part of this session: the experiments a company +> already runs are its **instrument supply**. When you cannot randomize the thing you care +> about, a randomized nudge nearby can stand in for it. That is the doorway to instrumental +> variables, and we walk through it later today. + +## Slide 8 — Poll: price the engagement (Nürnberger) + +> Last case, and this one has a number with an invoice attached. Nürnberger Versicherung, a +> German insurer. Their steering metric was last-touch attribution — credit the click that came +> last. They replaced it with a funnel-aware causal model. [Read the poll.] Over six months of +> model-guided spend, what happened to their cost per lead? +> +> [Collect. Reveal.] +> +> Down more than 27 percent. In the client's words, "very, very good". The mechanism is the +> management lesson: under GDPR consent rules, customer journeys appeared artificially +> shortened, so last-touch systematically under-credited the upper funnel — video, display — +> and budget chased the wrong clicks. The causal model measured what the upper funnel actually +> causes downstream, and the client is scaling it into full production. +> +> And listen to the client's bar for belief, on the slide: "Trust is not created by R-squared +> values. It is created when business reality matches model expectations." No statistical +> pitch ever beats that sentence. Keep it. +> +> **[The bridge — say this over the slide before advancing.]** Three cases, one shape. A +> counterfactual — the world without the launch, the campaign, the exposure. An experiment that +> anchors the model. And a decision priced in euros, with its uncertainty on the table. Colgate +> built the counterfactual in *time*: before the launch versus after. The hardest and most +> common version of the problem is built in *space*: one market got the campaign, and you must +> decide about a nation. No experiment, real money. For the next forty minutes we build exactly +> that one, end to end, and every idea from these three cases will come back with its proper name. + +--- +--- + +# PART 2 — SYNTHETIC CONTROL + +## Slide 9 — Part 2 divider + +> Now the deep dive. I will not teach you statistics. I will teach you how to make one specific +> business decision: whether to spend four million euros on a national marketing campaign, when +> the only evidence you have is one regional pilot. The method is called synthetic control. It +> is the engine inside the geo-testing tools used by Meta and Google. But the method is not the +> point. The point is the decision. +> +> Colgate at least had a clean national before-and-after. Now take even that away. + +# ACT I — THE QUESTION + +## Slide 10 — The boardroom question + +> Here is the situation. You are on the board. A campaign ran in one metro region for twenty +> weeks. It cost 75 thousand euros. Sales went up. Marketing is happy, and now they ask for +> 4 million euros to take the campaign national. You must decide. +> +> What do you actually have? Thirty markets, sixty weeks of weekly sales data. One metro got +> the campaign. The other twenty-nine did nothing. And one detail that matters a lot: the metro +> was **chosen** by the marketing team. It was not picked at random. This is not an A/B test. +> +> To decide, you must answer three questions, in this order. +> One: how much of the sales increase did the campaign actually **cause**? That is attribution — +> the Colgate question again: what would have happened anyway? +> Two: is that increase more than luck? That is significance. +> Three: is the profit worth the price? That is the decision. +> +> Notice the order. Most companies jump straight to question three with a number from question +> one that was never checked. Today we do it properly, and each question needs a different tool. + +*[If asked about the fold on "unbiased": "The slide has a precise note on when this measurement +can be trusted. Short version: the method learns the metro's normal behavior from before the +launch, so it does not matter why the metro was chosen — unless the choice used secret knowledge +about the future. We test for that later."]* + +## Slide 11 — The data (live) + +> This is the raw data. Every line is one market's weekly sales. The white line is our metro. +> At week 40 the campaign starts. +> +> [Run the live demo: show the before/after means, then subtract the cross-market average.] +> +> The simplest possible analysis: compare average sales before the launch with after. You get a +> bump. But watch what happens when I subtract the average of all the other markets — the crowd. +> The bump changes size. That is the whole problem in one picture: part of what you see is the +> campaign, and part is the tide — the economy, the season, everything that moved every market +> at once. +> +> And here is the one number you do not have, and never will: what this metro **would have sold +> without the campaign**. In no file, no dashboard, no report. Everything in Part 2 is a way to +> estimate that one missing number. You already know its name: the counterfactual. + +## Slide 12 — Poll: the 12% bump + +> Before I show you any method, commit. [Read the poll.] Sales averaged a certain level before +> launch and a higher level after — a bump of about six percent. What is the campaign's true +> weekly effect? Around 2 thousand per week — most of the bump is just trend? Around 7 thousand +> — the simple comparison is roughly right? Or around 14 thousand — double the bump? +> +> [Collect hands. Reveal.] +> +> About double. The economy happened to **dip** right after the launch. The tide went out, and +> it hid half of the campaign's effect. This is the first management lesson of the day: a naive +> before/after comparison is not "roughly right with some noise". It is charged with everything +> else that happened in the world. This time the tide hid a real effect. Next time it will +> flatter a dead one, and you will pay for a campaign that did nothing. + +## Slide 13 — Potential outcomes + +> The one piece of theory you need today. It fits on this slide. +> +> Our metro has two possible histories for the campaign weeks. History one: sales **with** the +> campaign. History two: sales **without** it. The campaign's true effect is the gap between +> the two histories. That is all "causal effect" means. +> +> The problem: only one history ever happens. Look at the table: three of the four cells are +> real data. The fourth — the metro's sales without the campaign, after launch — is missing. +> Every method you have ever heard of, A/B tests included, is a different way to fill that one +> missing cell. +> +> One more distinction that matters for the money: the effect **per week** and the **total** +> over twenty weeks are different numbers with different uncertainties. The 4-million decision +> rides on the total. Keep that in mind. + +*[SKIPPABLE MATH — the notation fold: "precise definitions for those who like them; you will +not need them to follow."]* + +## Slide 14 — The data-generating model **[FAST / SKIPPABLE MATH]** + +> One honest confession: today's data is simulated. We built the world ourselves. Why? Because +> in a simulated world we **know** the true answer — we planted a 12 percent lift — so we can +> grade every method against the truth. It is exactly Colgate's discipline from this morning: +> recover a planted truth first, then trust the method. +> +> The formula says the world has four ingredients: each market has its own size, a shared trend, +> a shared season, a shared economic wave — and each market feels these shared forces with its +> **own sensitivity**. That last part makes the problem hard, and you will see why. +> +> What matters for today: the planted truth is about 284 thousand euros of extra sales over +> twenty weeks. Remember 284. Every method gets graded against it. + +## Slide 15 — Simulate the world (live) **[FAST]** + +> [Play with one slider only — the macro shock.] +> +> Ten seconds of demonstration. The dashed blue line is the counterfactual itself — the metro +> in the world without the campaign. In real life this line does not exist. Watch when I make +> the economy more turbulent: the naive before/after estimate becomes garbage, because the tide +> dominates. "Sales went up after we launched" is not evidence. The question is always: +> compared to what? + +# ACT II — THE COUNTERFACTUAL + +## Slide 16 — Abadie's idea: the synthetic twin + +> So how do we fill the missing cell? The idea is genuinely simple. +> +> If our metro had an identical twin — same size, same seasonality, same sensitivity to the +> economy — we could just watch the twin. The twin got no campaign. Its sales during those +> twenty weeks are our answer. +> +> No single market is a perfect twin. But a **blend** can be. Forty percent of market eight, +> plus 32 percent of market twenty, plus 17 percent of market three behaves almost exactly like +> our metro. That blend is the synthetic control — a manufactured twin. +> +> The key property, in business terms: the twin is built to **react to shocks** the same way +> our metro reacts. When the economy dips or the season turns — even in ways nobody predicted — +> the twin dips and turns with it. Whatever gap remains between metro and twin is the campaign. +> +> Strong pedigree: called the most important innovation in policy evaluation in fifteen years, +> and it is the engine inside Meta's GeoLift and Google's geo tools — the tools your media +> agencies already use. + +## Slide 17 — The estimator **[SKIPPABLE MATH]** + +> This slide is the recipe in formulas. Three business facts, then you may ignore the algebra. +> +> Fact one: the twin is fitted **only on data from before the launch**. The method never sees +> the campaign period while it learns, so it cannot cheat. Same discipline you demand from any +> forecast: no peeking. +> +> Fact two: the blend must be a real recipe — weights positive, summing to one hundred percent. +> You get statements like "40 percent market eight plus 32 percent market twenty". A manager +> can read that, question it, audit it. No market enters at minus 80 percent. It is the +> weights-on-the-table version of what Colgate's projection did in time. +> +> Fact three: after fitting, the effect is a subtraction — actual sales minus twin sales, +> summed over the campaign weeks. Nothing exotic. + +## Slide 18 — Inside the convex hull (live) **[FAST]** + +> [Drag the treated market in and out of the hull — thirty seconds.] +> +> One picture on why the "real recipe" rule matters. A blend of real markets can only reproduce +> a market that lives **inside** the range of its donors. You can mix coffee and milk; no mix of +> coffee and milk gives you orange juice. When the target sits outside that range, an +> unconstrained model still produces a number — by extrapolating. The constrained one **refuses**. +> +> And the refusal is a feature, not a bug. It is the method telling you: widen the donor pool, +> do not trust me here. So make this a procurement question. Ask any vendor: when does your +> method refuse to answer? If the answer is "never", walk away. + +## Slide 19 — What the constraint buys (live) **[FAST]** + +> Here we removed the recipe rule and let an ordinary regression do what it wants. Look: the +> unconstrained model fits the past **better** — and predicts the missing counterfactual +> **worse**. It used absurd weights: minus 80 percent of one market, 190 percent of another. +> It memorized noise. +> +> The management lesson: a tighter fit to history is not a better model, and is often a worse +> one. Anyone selling you a model on "look how well it fits the past" is selling the wrong +> metric. You saw the same inversion this morning — the MMM with the backwards ranking fitted +> its history beautifully. + +## Slide 20 — Why not just forecast it? + +> Somebody always asks: why this old-fashioned blend? Why not gradient boosting, Prophet, a +> neural network? We ran that race. The fancy forecaster fits the past better — again. Its +> campaign estimate lands in the same place as the simple twin. And across twenty-four fresh +> simulated worlds it reconstructs the missing counterfactual clearly worse on average, with +> weights no planner can audit. +> +> The deep reason is a business reason. A forecaster answers: "what comes next, in a world like +> the past?" Our question is: "what would **this** market have done in a world that never +> happened?" You cannot cross-validate that answer — the truth is never observed. So accuracy +> on the past cannot pick your tool. You must pick the tool whose **assumptions you can defend +> in the boardroom**. The simple auditable blend wins the causal job even when it loses the +> forecasting beauty contest. + +## Slide 21 — Every shortcut is an assumption (live) + +> The slide I most want you to remember from Act II. Four common ways to measure the same +> campaign: before/after; treated versus the average of other markets; difference-in- +> differences — the consultant's favorite; and synthetic control. +> +> Each is really a hidden **claim** about the missing counterfactual. Before/after claims the +> world would have stood still. Treated-versus-control claims other markets are just like +> yours. Difference-in-differences claims all markets ride the economy in parallel. Synthetic +> control claims a blend of markets can mimic yours. +> +> [Live demo: move the two dials.] +> +> Watch the errors as I change the world. When markets react to the economy differently — the +> normal case — before/after and diff-in-diff drift away from the truth. Synthetic control +> stays close, because it matched how the metro **responds**, not just its level. +> +> And the uncomfortable fact: none of these numbers arrives with a warning label. A wrong +> method does not look wrong. More data does not fix it — a wrong claim stays wrong at any +> sample size. The cure is a better claim, not a bigger spreadsheet. + +## Slides 22–23 — The one-difference estimators / DiD algebra **[SKIPPABLE MATH]** + +> These two slides do in algebra what the dials just showed in pictures: exactly when +> before/after, treated-versus-control, and difference-in-differences give the wrong answer. +> The executive summary is enough: +> +> Before/after fails whenever the world does not stand still — so, almost always. Comparing to +> other markets fails because markets differ in size. Difference-in-differences fixes size but +> still requires every market to ride the economy in parallel — and that assumption dies +> quietly, no alarm. The one bias diff-in-diff cannot remove is precisely the one synthetic +> control removes by construction, because it **chooses** the comparison weights instead of +> hoping equal weights work. +> +> One line worth repeating to any analyst who reports to you: more weeks of data buy precision +> around the **same wrong number**. If the method is biased, patience does not help. Ask which +> assumption the comparison rests on. No answer — the number is decoration. + +## Slide 24 — Identification: what must hold + +> Every method rests on assumptions. The professional standard: list them, test what can be +> tested. Read this as a due-diligence checklist. +> +> One: the twin must track the metro before launch. Testable — it passes, with a quantitative +> gate. Two: sales must not react **before** the campaign — no leaked launch, no stockpiling. +> Testable with a placebo you will see shortly — passes. Three: the campaign must not touch the +> comparison markets. Ads do not respect borders. This one is **untestable** with statistics — +> it is Colgate's second-launch leak, in space. Mark it; it returns in ten minutes. Four: the +> metro must be reconstructable from the donors, and no single market may hold the answer +> hostage — drop any donor, the answer barely moves. Passes. +> +> Three passes, one honest "cannot test". Remember which one. + +# ACT III — IS IT REAL, AND HOW BIG? + +## Slide 25 — Poll: is it real? + +> The twin says: about 260 thousand euros of extra sales over twenty weeks. New question: real, +> or a lucky metro? Normally you would run a significance test. Here you cannot: every standard +> test measures luck across **many treated units**, and we have one treated metro. You cannot +> measure variation across one thing. The standard machinery is not weak here — it does not +> exist. +> +> And you can invent the replacement yourselves. [Read the poll.] Take each of the 29 markets +> that ran **no campaign**. Pretend, one at a time, that it was the treated one. Run the whole +> twin machinery on it. It should show roughly zero — nothing happened there. Whatever "effect" +> it shows anyway is pure luck: the noise level of our method. +> +> Thirty markets, thirty measured "effects", twenty-nine of them known to be luck. Where does +> our metro's 260 thousand rank? +> +> [Collect. Reveal.] +> +> Rank one of thirty. The most extreme of all. And the rank **is** the test: if the campaign +> truly did nothing, our metro would be just another market, and the chance of ranking first by +> luck is one in thirty — about three percent. You have just reinvented a rigorous statistical +> test using nothing but common sense. + +## Slide 26 — Placebo-in-space (live) + +> The idea, drawn. The grey cloud: 29 fake "effects" — the luck of the method, measured +> directly. The green line: our metro at 260, standing clear outside the whole cloud. +> +> Conclusion: the effect is real, with about a three percent chance we are fooling ourselves. +> One caveat for your analysts: a comparison market that fits badly can fake a big effect, so +> bad-fit placebos are dropped before ranking — the standard hygiene rule is on the slide. +> +> But hold the applause. "Real" is not "profitable". That is question three, and we are not +> there yet. + +## Slide 27 — Interval by test inversion (live) **[SKIPPABLE MATH]** + +> One more product from the same placebo idea, then money. We have a point estimate, 260. A +> board should never accept a point without a range. The range: 195 to 335 thousand, with 90 +> percent confidence. +> +> How it is built, in one sentence: for every possible "true" effect, we ask whether that truth +> could plausibly have produced our 260 given the luck we measured — and the survivors form the +> range. The mechanics are on the slide for the curious; the demo lets you drag the hypothesis. +> +> Why care about this particular range: it was built with **no model of the noise** — no bell +> curve, no software assumptions, only the placebo logic you invented by hand. That makes it +> the referee for every fancier number that comes later. Any model that contradicts this range +> has explaining to do. + +## Slide 28 — Falsification 1: placebo-in-time **[FAST]** + +> Two stress tests before the money. First: pretend the launch happened ten weeks earlier, and +> re-run everything. A real method should find **nothing** there — nothing happened. It finds +> nothing: about 1.7 thousand, inside noise. Pass. +> +> What this rules out: leaked launches, stockpiling, sales reacting early. Make it a habit: any +> vendor claiming a lift should show you their method finds zero where nothing happened. + +## Slide 29 — Falsification 2: spillover (SUTVA) + +> The second stress test is the one I flagged as untestable — the disease you named for Colgate +> this morning, now in space. Our metro's TV and outdoor ads reach the commuter belt. Those +> neighboring markets are in our comparison pool. So the campaign quietly lifts the very +> markets we use as "untreated". The twin gets inflated, and the measured effect **shrinks**. +> +> We cannot test for this in the data — the leak starts exactly at launch, so every check stays +> green. What we can do is simulate it deliberately and measure the damage: with a realistic +> leak, our 260 becomes 236. Two takeaways. +> +> First: the bias only pushes the number **down**. Our estimate is a conservative floor — if the +> campaign clears its cost at 260, a leak cannot overturn the decision. +> Second, and more important: the fix is not statistical, it is managerial. When you **design** +> a geo test, keep the control markets off the campaign's media footprint. The best analysis +> cannot repair a badly designed test. Design beats analysis. Every time. + +## Slide 30 — What the statistics says + +> Act III complete. Three numbers to carry. +> Is it real? Yes — three percent chance of self-deception. +> How big? 260 thousand euros of incremental sales. +> Give or take? 195 to 335, at 90 percent confidence. +> +> And a satisfying check only a simulation allows: the planted truth, 284, sits inside the +> range. The machinery is honest. +> +> Now — the sentence that loses money in real boardrooms: "The campaign drove 260 thousand of +> sales for 75 thousand of spend. That is a 3.5x return. Roll it out." Every number in that +> sentence is true. The conclusion still does not follow. Why not? Act IV. + +# ACT IV — THE DECISION IN EUROS + +## Slide 31 — Poll: the margin + +> [Read the poll.] The campaign cost 75 thousand. It generated 260 thousand of incremental +> sales. Gross margin is 35 percent. What did the campaign **earn**? +> +> [Collect. Reveal.] +> +> About 16 thousand net. Margin times lift, minus cost: 35 percent of 260 is 91, minus 75 — +> about 16. If you answered "185 — sales minus cost", you made the boardroom classic: you +> treated revenue as profit. The company keeps only the margin on the extra sales; it pays the +> campaign invoice in full. Nürnberger's last-touch trap was a cousin of this one: the +> comfortable metric, wrong by construction. + +## Slide 32 — The margin trap (live) + +> The general rule — the cheapest lesson of the day. "Return on ad spend" compares incremental +> **sales** to cost, so its break-even is not 1. It is 1 divided by your gross margin. At our +> 35 percent margin: 2.86. You need 214 thousand of incremental sales just to get your 75 back. +> +> [Move the margin slider.] +> +> A software company at 90 percent margin breaks even at 1.1. A grocery chain at 20 percent +> needs 5.0. The **same campaign with the same measured lift** is a good buy for one business +> and a money-loser for the other. If your agency reports "ROAS above 1" as success, that is a +> vanity bar. The real bar is 1 over margin. Ours: 3.47 achieved against 2.86 required. It paid. + +## Slide 33 — The decision space (live) **[FAST]** + +> Now bring the uncertainty back. Expected profit says the campaign pays at any price below 91 +> thousand. But the 90-percent range on profit runs 68 to 117 — and our price, 75, sits +> **inside** it. Translation: probably paid, but a loss is still consistent with the evidence. +> [Slide the price control.] Between "looks profitable on average" and "provably profitable" +> there is a wide dangerous zone, and our campaign lives in it. + +## Slide 34 — What you say to the board (classical) + +> The memo a competent analyst delivers with classical tools only. +> Real? Yes. Big enough on the point estimate? Yes — about 16 thousand net. +> Confident? **No.** The profit range straddles the price. +> +> Verdict on the €75k pilot: probably a profitable buy, not a confident one. Renegotiate the +> price, or buy certainty with a bigger test. And say the second sentence out loud: this memo +> says **nothing yet about the €4M** — the rollout is not the pilot multiplied. +> +> One more thing, in the last box. There are questions this memo cannot answer, in principle: +> "what is the **probability** the campaign pays?", "what is the highest price this evidence +> would still justify?", "is a follow-up study worth its cost?". Classical statistics answers +> yes/no about where a range sits; it has no language for "78 percent chance". Managers +> allocate money in exactly that language. That language exists — it is the Bayesian layer of +> this same toolkit, and it is where this course goes next. Today we stop at the honest +> classical verdict, which is already worth real money — watch. + +## Slide 35 — What €4M actually buys + +> We still owe the board the real question: the four million. Here is the number in the +> marketing deck: 4 million is 53 pilot budgets; 53 times the pilot's lift, at our margin, +> nets about **plus 860 thousand**. Every step is correct. +> +> [Poll the room: who approves the €4M? Same device as slides 12 and 31 — let them commit.] +> +> Now do the one thing the marketing deck did not: carry the **interval**, not the point. And +> be maximally generous — assume the campaign works nationally exactly as well as in the metro. +> Perfect transport, every euro. Even then, the 90 percent profit interval on the 4 million +> runs from **minus 360 thousand to plus 2.25 million**. It straddles zero. At the most +> optimistic assumption we can write down, we cannot show the 4 million pays. +> +> And that assumption is a ceiling. Two forces only pull it down: 4 million over 30 markets is +> 133 thousand per market — nearly double the pilot's intensity, and advertising has +> diminishing returns; and one metro is not the country. [Drag δ down.] Below about 64 percent +> surviving, the rollout provably loses. +> +> That is the whole frequentist verdict, and it needs no probability model: at its best case +> the 4 million is not provably profitable, and it can provably lose. **Do not commit the 4 +> million on this evidence.** The next slide is what you do instead. + +## Slide 36 — Measure it, don't guess it + +> The answer is not "yes" and not "no" — it is "not on this evidence". And notice why the +> evidence cannot decide: the interval is wide because there is **one** treated market. You +> cannot think your way out of that. Re-running the model, adding weeks, arguing about +> assumptions — none of it narrows the interval. Only more treated markets do. +> +> [Ask: so what would you spend? Take answers. The honest one is "not four million yet" — that +> is the lesson, not a dodge.] +> +> Two moves. First, step zero — it gets a laugh because it is so obvious and so often skipped: +> before you spend a million on evidence, spend an hour checking you have not already bought +> it. A firm with a 4-million budget has prior pilots, a media-mix model, agency benchmarks. +> The answer may be in a drawer. +> +> Second, if it is not: measure it, do not model it. Run the campaign in **eight markets chosen +> to look like the nation**, at full national intensity — about 133 thousand each, 1.07 million +> total. Because the cells span the country, the measured lift **is** the national lift, with +> an interval tight enough to clear the 4 million or kill it. And recognize the shape: this is +> HelloFresh's loop from this morning, bought once — the model asked for ground truth, and we +> went and bought the experiment. +> +> The discipline that separates a test from theatre: write the release rule **before** the data +> lands. "Release the remaining 2.9 million only if the measured profit interval clears zero." +> One quarter later you have a national campaign at a size the evidence supports. +> +> [Delivery notes: "transportability" is classroom language — in a client room say "how much of +> this travels". The fold prices this information at about 320 thousand euros using the course's +> Bayesian machinery — far above the test's true cost — but the decision to test was already +> made here, classically.] + +--- +--- + +# FINALE — THE PATTERN + +## Slide 37 — The tools were the product too **[FAST]** + +> Almost done. Three footnotes that are really credentials. +> +> Everything you just used exists as open source, industrialized by the same consultancy: +> CausalPy carries synthetic control, interrupted time series, difference-in-differences and +> regression discontinuity — today's method and its whole quasi-experimental family. Its launch +> example is the sentence this session opened with: exposure you cannot randomize, causal +> impact you still need. PyMC-Marketing is the MMM library behind the calibration story — and +> one client's budget-allocation approach came back as a pull request, which is what a healthy +> open-source business looks like. And the consultancy's own webinar agenda — geo tests, MMM, +> synthetic control, diff-in-diff, instrumental variables — is literally this session's +> syllabus. You did not take a course *about* the industry today. You took the industry's course. + +## Slide 38 — The pattern in every engagement + +> Compress the whole session to three lines. +> The deliverable is a **counterfactual** — a world minus the launch, the campaign, the +> exposure — priced in euros. +> An **experiment anchors** every observational model. Calibration is the product, not a luxury. +> And **uncertainty prices the decision** — a board acts on the probability it pays and on the +> headroom, not on a point estimate. +> +> One last exhibit, because it is 2026: we ran a vanilla coding agent on deliberately +> adversarial marketing data. It fit a model and confidently recommended budget reallocations. +> Confidently wrong. The honest system explored eleven model specifications, none converged, +> and returned: "No valid model found. Run a geo-holdout experiment." Even the machines, at +> their best, arrive at Part 2's closing advice: run the experiment. + +## Slide 39 — One breath + +> One breath, and we stop. What a Bayesian consultancy actually sells: counterfactuals, +> calibrated by experiments, priced as probabilities. You have now watched real clients buy it, +> and you have built one yourself. +> +> Every number today is pinned to a public source — the list is on the backup slide. The +> notebooks behind this session are the course repository. Both of us consult for PyMC Labs: +> say hello. +> +> [When Part 3 (instrumental variables) runs in the same sitting, do NOT stop here: say "after +> the break, the last tool — what to do when even the geo test is impossible" and hold this +> slide's close for the very end of the 1.5 hours.] + +## Slide 40 — Sources (backup) + +> [Only if asked.] Every fact from Part 1, pinned to its public post with the exact quote and +> retrieval date; Part 2's numbers are baked from the executed course notebooks and verified +> against a claims registry at build time. Nothing on these slides was typed by hand. + +--- +--- + +# If you are told "you have 45 minutes" on the day + +Cut in this order: slide 37 (provenance) → 22–23 to their one quoted line → 19 → 14–15 to one +sentence each → 33 (fold its point into 34) → the second poll of whichever act runs long. +Never cut: 2–6 (who we are + the hook), 10–13, 16, 18, 21, 25–26, 29, 31–32, 34–36 (35–36 +above all), 38–39. + +# The engagement beats (do not silently skip) + +Polls: 3 (route the call) · 8 (price the engagement) · 12 (the 12% bump) · 25 (rank the metro) +· 31 (the margin) · 35 (approve the €4M — hands, before the interval). Open floor: 5 (what +would break it). Live widgets to hand to a student: 5 (second launch), 6 (lift-test toggle), +15 (macro shock), 21 (estimator dials), 27 (drag the hypothesis), 32 (margin slider), +35 (drag δ). The polls are the lecture's spine: commitment before revelation, every time. diff --git a/causal-marketing-pymc/apps/unified_slides.html b/causal-marketing-pymc/apps/unified_slides.html new file mode 100644 index 0000000..a512501 --- /dev/null +++ b/causal-marketing-pymc/apps/unified_slides.html @@ -0,0 +1,2696 @@ + + + + + + +Causal Inference in the Wild: from real engagements to a €4M decision + + + + + +
+ + + +
+
Causal Inference & XAI for Business · SDA Bocconi
+

Causal Inference in the Wild

+
Real PyMC Labs engagements, then the machinery behind them: one treated market, no experiment, €4M on the line.
+
A guest lecture for Prof. Michele Russo's course
+
+
+
Francesco Muia
+
PhD in Theoretical Physics, EMBA.
Consultant for PyMC Labs and Brown University.
+
francesco.muia@pymc-labs.com
+
francesco.muia@ai-and-analytics-solutions.com
+
+
+
Alexander Fengler
+
PhD in Computational Cognitive Science.
Postdoc at Brown University, consultant for PyMC Labs.
+
alexander.fengler@pymc-labs.com
+
+
+
+ PyMC + PyMC Labs +
+
+ + +
+
Opening · Who is talking
+

PyMC Labs: what we do

+
A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
+
+
    +
  • What we sell: senior modeling expertise, not seats or software licenses: revenue is services-led.
    • +
  • Open source is the top of the funnel: the libraries build trust and reach; the consulting monetizes the deep expertise behind them.
    • +
  • How we work: small teams of senior people who start from the client's actual decision, encode domain knowledge and uncertainty explicitly, and hand back models the client can own and extend, not black boxes.
    • +
+ + + + + + + +
EngagementWhat it isTypical client
Project consultingFixed-scope build of a custom model: a marketing-mix model, a demand forecaster, a pricing modelA specific, high-value decision problem
Advisory / retainerOngoing access to our modelers, guiding an in-house teamFirms building their own capability
Enablement & trainingWorkshops and embedded upskilling on Bayesian methods and our toolingAnalytics teams standardizing on PyMC
+
The clients in this session
+ Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
+
▸ Areas of interest, and where we work
+
    +
  • Areas: marketing-mix modeling and media measurement, causal inference, demand forecasting and pricing, experimentation and A/B testing at scale, applied Bayesian modeling.
    • +
  • Sectors: consumer/CPG, retail and e-commerce, tech, and finance.
    • +
+
+
+
+ + +
+
Case 1 · Colgate-Palmolive
+

You are the consultant

+
The counterfactuals this session builds are what clients buy. First test: route a real call, on instinct.
+
+
+
✋ Poll
+
Colgate-Palmolive calls: "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it stealing share from competitors, or from our own brands?" Which tool do you reach for first?
+
+ + + + +
+ +
C. A national launch leaves nothing to randomize and no market to difference against: A and B need a control group that does not exist, and D's instrument does not exist for a shelf that changed everywhere at once. What remains is the move this whole session builds, here run in time: fit the world before the launch, project it forward, read the gap. PyMC Labs sold exactly that projection; the next slide shows it.
+
+
+
+ +
+
Case 1 · Colgate-Palmolive
+

Colgate-Palmolive: incremental, or cannibalistic?

+
Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The launch verdict is the split.
+
+
+
+
The launch, and the world without it schematic
+
+
Illustrative shape of the engagement's counterfactual read, not client data: fit the pre-period, project it forward, price the gap.
+
+
+
+
The brief, in their words
+ "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
+
    +
  • The client: Colgate-Palmolive came to PyMC Labs in 2023, in a market estimated at $20.8 billion in 2023.
    • +
  • The method: a multivariate Bayesian interrupted time series: the pre-launch world projected forward, pointed at a product; later extended to a nested-logit choice model.
    • +
  • The grading: on simulated data the model recovers a planted 50% incrementality as a 94% interval of 49-59%: the recover-the-truth contract of this whole session, run commercially.
    • +
+
+
+
+ +
+
Case 1 · Colgate-Palmolive · open floor
+

What would break it?

+
+
+
🗣 Open floor · 2 minutes
+
You are Colgate's CMO. The incrementality estimate you just saw (the share of the new product's sales that are genuinely new, not cannibalized) decides the launch review. Name one real-world event that would make it wrong.
+ +
A second launch. When another product entered the estimation window, the same machinery reported 64-76% incrementality against a planted truth of 100%: the counterfactual absorbed part of the very effect it was meant to isolate. An honest consultancy publishes exactly this: the 64-76% miss is printed in the same post as the win. In Part 2 the same disease returns with its formal name, spillover; the defence is design, not statistics.
+
+
    +
  • What the model learns: everything before the launch defines "normal growth", and the projection (red) extrapolates that normal forward.
    • +
  • What a second launch does: inside the window it becomes part of "normal", so the projection rises too fast and under-credits the true lift; after the launch it inflates the observed line instead, and over-credits.
    • +
+
+
Break it yourself: slide a second launch into the window schematic
+
+
+ + + +
+
Same schematic world as the previous slide. The real case reported 64-76% against a truth of 100%.
+
+
+
+ + +
+
Case 2 · HelloFresh · the tool, and its failure mode
+

Why calibrate? A model alone can rank channels backwards

+
Before the HelloFresh story, the tool it relies on: a warm-up from PyMC Labs' published calibration tutorial.
+
+
    +
  • The tool: a marketing-mix model (MMM) explains total sales as the sum of per-channel contributions, fit on observational spend data: no experiment anywhere in it.
    • +
  • The grading: the tutorial plants a truth: return on ad spend (ROAS, sales per unit of spend) of 93.39 for channel x1 against 171.41 for x2, so x2 is nearly twice as effective.
    • +
  • The experiment: a lift test nudges one channel's spend by a known amount and measures the sales change it causes: a small randomized ground-truth reading for that channel.
    • +
  • The repair: two lift tests per channel, entered into the likelihood, recover both values: the experiment is the model's anchor, the theme of everything that follows.
    • +
+
+
One MMM on observational spend alone, one planted truth, one inversion baked from the tutorial
+
+
+
Left: the ranking the uncalibrated model reported. Right: the planted truth the experiments recover.
+
+
The inversion
+ Fit on observational data alone, the baseline model ranked x1 above x2: the direct opposite of the truth.
+
+
+ +
+
Case 2 · HelloFresh
+

HelloFresh runs the loop, at industrial scale

+
+
+
    +
  • The loop: MMM priors fed by field experiments such as lift or incrementality tests; a 60% cut in prediction variance.
    • +
  • On stage: the panel's own agenda: Bayesian MMM can be calibrated to ensure consistency with incrementality measurements.
    • +
  • The experiment supply: a pipeline handling thousands of concurrent tests: A/B, ABC, and ABCD campaigns run simultaneously, the overnight batch down from 5–6 hours to 5–6 minutes; the Criteo experiment from the session's IV close (13,979,592 users) sits in exactly this regime.
    • +
+
The supply chain, for the IV close
+ The experiments a company already runs are its instrument supply: a randomized encouragement is the instrument for the exposure you cannot randomize.
+
+
+
+
The loop, in one picture
+
+
The model runs always-on; the experiment disciplines it; the counterfactual reads the experiment out.
+
+
+
+
+ + +
+
Case 3 · Nürnberger Versicherung
+

Price the engagement

+
A German insurer, last-touch attribution, and a funnel-aware causal MMM, in production.
+
+
+
✋ Poll
+
Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over June through November of model-guided spend, cost per lead (CPL) moved by how much?
+
+ + + + +
+ +
C. "This year we were able to drive the CPL down by more than 27%, which is very, very good" (Philip Herp, Nürnberger Versicherung). The mechanism is the lesson: under GDPR, customer journeys appeared artificially shortened, so last-touch under-credited the upper funnel and budget followed attribution mechanics instead of incremental business impact. The funnel model measured what video spend causes downstream, and the client is scaling it into full production for 2026.
+
+
The client's bar for belief
+ "Trust is not created by R² values. It is created when business reality matches model expectations."
+
+
+ + +
+
+
+
Part 2 · The deep dive
+

Synthetic Control

+
One treated market, no experiment, €4M on the line: did the campaign work?
+
+
+ +
Where the lecture lands: a synthetic twin rebuilds the metro's counterfactual, and the gap beyond break-even is the campaign's profit.
+
+
+
+ + +
+
Act I · The question
+

The boardroom question

+
A campaign ran in one region. Sales rose. Marketing wants €4M to go national. You decide. The Colgate question again: what would have happened anyway, now with €4M riding on it.
+
+
+
+

What you have

+
    +
  • 30 markets, 60 weeks of weekly sales.
    • +
  • One metro, week 40: a €75k campaign for 20 weeks; the other 29 did nothing.
    • +
  • No randomisation: the metro was chosen, not drawn.
    • +
  • 35% gross margin on incremental sales.
    • +
+
+
+

What the data forces you to answer, in order

+
+
① How much did the campaign actually cause? attribution
+
② Is that more than luck? significance
+
③ Is the profit worth the price? the decision
+
+
+
+
The method this data calls for
+
    +
  • One treated unit, an observational time series, no A/B test possible.
    • +
  • This is synthetic control's home ground.
    • +
  • It is the engine under Meta's GeoLift and Google's geo methodology.
    • +
+
The €4M hinge: the transportability discount
+
    +
  • δ = the share of the pilot's per-euro lift that survives national rollout.
    • +
  • Measuring this one metro is unbiased as the fit uses its own pre-launch record.
    • +
  • The €4M call reduces to one question: is δ large enough for the national rollout to still clear its cost?
    • +
+
▸ What "unbiased" means here, precisely, and where it would break
+

The estimator subtracts a reconstructed counterfactual from the treated metro's post-launch sales, and bias is its expected gap from the true lift:

+
+ \[ \hat\tau \;=\; \bar Y_{1,\text{post}} - \hat Y_{1,\text{post}}(0), \qquad \text{bias} \;\equiv\; \mathbb{E}[\hat\tau] - \tau \] +
+
    +
  • The counterfactual is the synthetic twin, built to reproduce the metro's pre-launch record (its level and its factor loadings) week by week.
    • +
  • Unbiasedness rests on one assumption: that pre-launch match would have continued through the post window had the campaign never run.
    • +
  • This needs no story about why the metro was chosen. Whatever pre-launch trait drove the pick, the twin already carries it, because it was fitted to match it: selection on anything visible before launch cannot tilt the estimate.
    • +
  • Bias enters only through what the pre-launch record cannot see: if the launch were timed on private knowledge of a coming local boom, the twin could not anticipate it and the gap would credit the boom to the campaign. The later placebo-in-time test hunts exactly that.
    • +
+
+
+
+ + +
+
Act I · The question
+

The data you actually get

+
The simulated world.
+
+
+
+
    +
  • \(Y_{jt}\): weekly sales of market \(j\) in week \(t\) (€000): 30 markets (treated metro = market 1, donors \(j=2,\dots,30\)), 60 weeks (\(t=0,\dots,59\)) = 1,800 observed numbers.
    • +
  • The design: the treated metro ran a €75k campaign from week \(T_0=40\) for 20 weeks at a 35% margin; the other 29 ran nothing.
    • +
  • The choice: the firm's marketing team picked the metro and the week. Not us, and not a coin.
    • +
+
+
+
The one number you do not have
+ What the treated metro would have sold over those 20 weeks with no campaign. It is nowhere in the file, i.e. the counterfactual.
+
+
+
+
+
+
▶ LIVE 30 markets, one treated at week 40
+
+
+ + + +
+
+
+
+
    +
  • Before/after: post minus pre-launch mean, /week.
    • +
  • Subtract the crowd: remove the shared wave and the step is ≈ /week.
    • +
  • The noise floor: pre-launch sd /week around a /week base.
    • +
+
Which number decides?
+
    +
  • Is this all due to the campaign?
    • +
  • What's the right number to take a decision?
    • +
+
+
+
▸ Where each of these three readings comes back later
+
    +
  • Before/after is the estimator \(\hat\tau^{\text{BA}}\) in the zoo you will grade: it assumes \(Y_{1t}(0)\) simply holds at the pre-launch mean.
    • +
  • Subtracting the crowd strips out the shared wave \(\gamma_j^{\top} f_t\) of the next slide's model, which is exactly what treated-vs-control and difference-in-differences try to do.
    • +
  • The noise floor is that model's own \(\varepsilon_{jt}\) term, the wobble every estimator must out-shout with a single treated unit.
    • +
+
+
+
+ + +
+
Act I · The question
+

Commit to a number

+
+
+
✋ Poll
+
Before/after on the treated metro: sales averaged ·/week before launch and ·/week after, a bump of ·/week (about +6%). What is the campaign's true weekly effect?
+
+ + + +
+ +
C. The true effect is ·/week, a 12% lift on a ·/week baseline. On the previous slide you already saw the naive bump grow once the shared crowd was removed: the first sign it understates. Before/after books barely half the truth because the shared macro wave happened to dip after launch: the tide can inflate a campaign's credit, or, as here, hide it. A naive comparison is not "roughly right plus noise"; it is charged with everything else that changed in the world, in whichever direction that happened to be. Making the counterfactual visible is the whole method.
+
+
+
+ + +
+
Act I · The question
+

Causal inference is a missing-data problem

+
+
+

Same panel as the data slide, one new idea: the treated metro (market 1) has two week-\(t\) sales, only one of which ever happens. \(Y_{1t}(1)\) with the campaign, \(Y_{1t}(0)\) without. The causal effect is the gap between those two worlds:

+
+ \[ \tau_t \;=\; Y_{1t}(1)-Y_{1t}(0), \qquad \tau \;=\; \sum_{t\ge T_0}\tau_t \] +
+
    +
  • \(\tau_t\) vs \(\tau\): the weekly effect, and the 20-week total the business actually buys.
    • +
  • Consistency: what we record is the realised arm, \(Y_{1t}=Y_{1t}(1)\) after launch and \(Y_{1t}=Y_{1t}(0)\) before it, one world per week, never both.
    • +
  • The one missing cell: \(Y_{1t}(0)\) for \(t\ge T_0\) is never observed, the fundamental problem of causal inference, and every method fills exactly that cell.
    • +
+
Define the estimand first
+ Per-week effect and 20-week total are different estimands with different uncertainties, and the rollout decision turns on the total.
+
+
+ + + + + + + +
pre (t < 40)post (t ≥ 40)
treated · \(Y_{1t}(1)\)·observed ✓
treated · \(Y_{1t}(0)\)observed ✓missing: the counterfactual
donors · \(Y_{jt}(0)\)observed ✓observed ✓
+
Three of four cells are data; the donors' untreated post-period is the live information a synthetic \(Y_{1t}(0)\) is rebuilt from.
+
▸ Reading the notation (observed vs potential)
+
    +
  • \(Y_{jt}\) from the data slide is what you observe; \(Y_{1t}(1),\,Y_{1t}(0)\) are the treated metro's two potential outcomes.
    • +
  • Subscript says which market, argument says which world (1 = campaign, 0 = none): distinct axes that happen to share the digit 1.
    • +
  • \(w_j\): the donor weights the synthetic control will choose so \(\sum_{j\ge 2} w_j Y_{jt}\) rebuilds the missing \(Y_{1t}(0)\).
    • +
+

"What would this metro have sold anyway?" is an estimation target, not a rhetorical question.

+
+
+
+ + + + +
+
Act I · The question, and the world behind the data
+

The model that generated the data

+
We simulate the data: here the counterfactual is known, so every estimator can be graded.
+
+

Each market's no-campaign sales are its level \(\alpha_j\), its exposures \(\gamma_j\) to three shared factors \(f_t\), plus noise: the whole world in one line (all €000).

+
+ \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} + \;=\; \alpha_j \;+\; \gamma_{j1}\underbrace{\,0.4\,t\,}_{\text{trend}} \;+\; \gamma_{j2}\underbrace{\bigl[\,8\sin(2\pi t/26)+4\sin(2\pi t/13)\,\bigr]}_{\text{seasonality}} \;+\; \gamma_{j3}\underbrace{\textstyle\sum_{s=1}^{t}\eta_s}_{\text{macro walk }m_t} \;+\; \varepsilon_{jt} \] +
+
    +
  • The macro walk \(m_t=\sum_{s\le t}\eta_s\): a running sum of shocks that wanders like an economy, its variance \(t\,\sigma_\eta^2\) growing every week.
    • +
  • The exposures \(\gamma_j\sim\mathrm{U}(1{-}s,1{+}s)\): each market responds its own amount, and the spread \(s\) is what kills parallel trends.
    • +
+
▸ All five ingredients, one by one
+
    +
  • Trend \(0.4\,t\): steady growth of 0.4 (€000) per week. Deterministic.
    • +
  • Seasonality \(8\sin(2\pi t/26)+4\sin(2\pi t/13)\): a 26-week half-year cycle of amplitude 8 and a 13-week quarterly cycle of amplitude 4. Identical for every market.
    • +
  • The macro walk \(m_t\): one fresh shock \(\eta_s\sim\mathcal{N}(0,\sigma_\eta^2)\) per week, none ever forgotten. This is "the economy".
    • +
  • Exposures \(\gamma_j\): each market multiplies each ingredient by its own loading.
    • +
  • Noise \(\varepsilon_{jt}\sim\mathcal{N}(0,\sigma_\varepsilon^2)\): the market's own weekly wobble, \(\sigma_\varepsilon\approx 3\).
    • +
+
+
Which of these is ever observed?
+ Only \(Y_{jt}\) reaches is observed. The level, loadings, factors and noise (market size, seasonal sensitivity, the economy) are real but recorded nowhere. Every estimator that follows is a way to cope with that.
+
The planted truth, in business units
+ On top of the world above we switch on a 12% lift at launch week \(T_0=40\): ·/week on average, · over the 20 campaign weeks.
+
▸ The planted lift, written out
+
+ \[ \tau_t \;=\; 0.12\,\bigl(\alpha_1+\gamma_1^{\top}f_t\bigr)\,\mathbf{1}_{\{t\ge T_0\}} \] + 12% of the treated metro's own baseline \(\alpha_1+\gamma_1^{\top}f_t\), switched on from week \(T_0=40\). +
+
+
+
+ + +
+
Act I · The question, and the world behind the data
+

Simulate the world yourself

+
Take a look at the counterfactual.
+
+
+
▶ LIVE The equations of the previous slide. Grey: donor markets. White: the treated metro.
+
+
+ + + + + + + + +
+
Dashed blue: the treated market's \(Y_{1t}(0)\), the potential outcome itself (not an estimate), which in real data does not exist.
+
+
+
+
    +
  • True lift: the planted 12% rate averaged over the 20 post-launch weeks (its euro value varies week to week).
    • +
  • Before/after: the treated series' post-launch mean minus its pre-launch mean.
    • +
  • Its error: the % gap between them, because before/after charges the whole tide to the campaign.
    • +
+
+
+
What the sliders teach
+
    +
  • Loading spread \(s\to 0\): every market reacts alike, trends run parallel, DiD lives.
    • +
  • Macro shock \(\sigma_\eta\) up: the shared wave dominates, before/after becomes garbage.
    • +
  • Noise \(\sigma_\varepsilon\) up: the weekly wobble grows, and any one estimate gets harder to trust.
    • +
+
+
+
+
+ +
+
Act II · The counterfactual problem
+

Abadie's idea: if no twin exists, build one

+
Idea: manufacture a treated market's twin using a blend of donor markets.
+
+

Assume untreated sales follow the factor structure of the DGP slide, now taken as the assumption rather than the recipe: a level \(\alpha_j\), loadings \(\gamma_j\) on shared latent factors \(f_t\), plus noise (equation in the fold below).

+
    +
  • \(f_t\), the latent factors: the shared shocks (macro wave rising, season turning) that every market feels, but not equally.
    • +
  • \(\gamma_j\), the loadings: how strongly market \(j\) responds to each factor (\(\alpha_j\) is its level); in real data none of these are observed.
    • +
  • The method never names them: in the sim \(f_t\) is trend + season + macro walk, but on real data it works without knowing what the factors are.
    • +
+

Blend the donors so their level and loadings equal the treated metro's:

+
+ \[ \sum_j w_j\,(\alpha_j,\;\gamma_j) = (\alpha_1,\;\gamma_1) \;\Longrightarrow\; \sum_j w_j\,Y_{jt} \approx Y_{1t}(0)\ \ \text{for all } t \] +
+
    +
  • Same loadings, same response to every factor movement, including ones that have not happened yet: that forward guarantee is the whole trick.
    • +
+
Note
+ DiD matches a level while synthetic control matches the response to shocks, which is why it survives the non-parallel trends that break DiD.
+
▸ The factor equation, pedigree, and the papers' notation
+
+ \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} \] + the assumed structure the blend must match, level and loadings, week by week +
+ Abadie & Gardeazabal (2003); Abadie, Diamond & Hainmueller (2010): "arguably the most important innovation in the policy-evaluation literature in the last 15 years" (Athey & Imbens 2017). The literature writes the factors \(\lambda_t\) and the loadings \(\mu_j\), and adds a common shock \(\delta_t\) felt identically by every market; any cross-market comparison cancels \(\delta_t\), so nothing changes here. +
+
+
+ +
+
Act II · The counterfactual problem
+

The estimator, precisely

+
A constrained least square optimization.
+
+

Fit the weights on the pre-period only, constrained to the simplex:

+
+ \[ \hat w \;=\; \arg\min_{w\in\Delta}\; \sum_{t \lt T_0}\Bigl(Y_{1t}-\sum_j w_j Y_{jt}\Bigr)^{2}, \qquad \Delta=\Bigl\{w : w_j\ge 0,\ \textstyle\sum_j w_j=1\Bigr\} \] +
+
    +
  • Least squares on the 40 pre-launch weeks: the post-period is never shown to the optimiser, so the fit cannot learn the effect it will measure.
    • +
  • The simplex \(\Delta\): weights are non-negative and sum to one, so the synthetic is an interpolation, never "−80% of Milan + 190% of Rome": the weights-on-the-table version of what Colgate's projection did in time.
    • +
  • No likelihood, no standard error: constrained least squares (SLSQP) ships a point estimate only, and inference comes later, from placebos.
    • +
+
▸ Reading the effect off the gap
+
+ \[ \hat\tau_t \;=\; Y_{1t}-\sum_j \hat w_j Y_{jt}, \qquad\qquad \hat\tau \;=\; \sum_{t\ge T_0}\hat\tau_t \] + weekly gap between the treated metro and its synthetic twin, summed over the 20 post-launch weeks +
+
+
Why constrain at all?
+ The simplex buys three things ordinary regression cannot: interpretability, regularisation, and an off-switch.
+
    +
  • Interpretability: "40% dma_08 + 32% dma_20 + 17% dma_03" is a sentence a planner can act on.
    • +
  • Regularisation: most weights land on exactly zero.
    • +
  • An off-switch: a market no blend can match fails loudly instead of extrapolating.
    • +
+
+
+ + +
+
Act II · The counterfactual problem
+

What the simplex buys, geometrically: stay inside the hull

+
A simplex blend can only land inside the donors' convex hull while OLS can leave it.
+
+
+
+
+
▶ LIVE Drag the treated market in and out of the donors' hull
+
+
+ + +
+
+
+
+
    +
  • With \(w_j\ge 0\) and \(\sum_j w_j=1\), every point a synthetic market can reach lies inside the shaded convex hull.
    • +
  • Inside the hull: a real blend reproduces the treated market's fingerprint, so a clean pre-period fit keeps working out of sample.
    • +
  • Outside the hull: no non-negative blend reaches it; the simplex stops at the frontier, and only extrapolation (negative weights) closes the gap.
    • +
+
Why constrain the weights
+ The simplex forces the synthetic to be an interpolation of markets you actually observed, so a good fit is a real blend, not an extrapolation that happens to line up.
+
When the treated market sits outside
+ That is the signal to widen the donor pool, not to trust an OLS fit that only matches by leaving the hull. The refusal is a feature. Ask any vendor: when does your method refuse to answer? If never, walk away.
+
+
+
+
+ + + +
+
Act II · The counterfactual problem
+

What the constraint buys: drop it and see

+
What if we used a simple unconstrained OLS?
+
+
+
▶ LIVE Two synthetics against the truth only a simulation can draw
+
+
OLS hugs the pre-period tighter ( vs ) yet drifts further from the true \(Y_{1t}(0)\) after launch ( vs ): overfitting the simplex refuses.
+
+
+
+
The weights OLS chose
+
+
OLS puts negative weight on donors (down to ) and gross weight , where the simplex uses exactly 1.
+
+
+
+ \[ n_{\mathrm{eff}} \;=\; \frac{1}{\sum_j \hat w_j^{2}} \] + Effective number of donors (inverse Herfindahl): one donor \(\Rightarrow n_{\mathrm{eff}}=1\), all 29 equally \(\Rightarrow 29\). Ours: 3.3. +
+
What n_eff = 3.3 tells you
+
    +
  • The synthetic leans on about three donor markets, not a fuzzy mix of all 29.
    • +
  • Sparse weights keep the blend interpretable and stable out of sample: fewer donors carrying real weight means less room to overfit noise.
    • +
  • It is a health check: near 1 the twin is hostage to a single market, near 29 it is averaging everything into mush. 3.3 is concentrated but not fragile.
    • +
+
+
+
+
+ + +
+
Act II · The counterfactual problem, versus machine learning
+

"Why not gradient boosting / Prophet / an LSTM?"

+
Is this a forecasting exercise?
+
+
+
▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
+
+
Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
+
+
    +
  • In-sample the forecaster fits tighter: pre-launch error €k/week vs the simplex's €k/week. More flexibility always buys a closer fit to the past.
    • +
  • Yet their campaign estimates are a wash: the forecaster reads and the simplex for the 20-week effect, both a touch under the €284k the simulation planted (the truth we can see only because this is a simulation). On this one dataset you cannot say which is better.
    • +
  • The gap shows only in the sweep: across 24 fresh worlds the unconstrained fit reconstructs the missing \(Y_{1t}(0)\) about 1.6× worse out of sample on average, and its weights leave nothing to inspect.
    • +
+
+
+
Principle 1: a counterfactual, not a forecast
+ A forecaster asks what comes next in a world like the training data. The campaign asks what this one market would have done in a world that never happened: extrapolation to an unobserved regime, not prediction.
+
    +
  • Principle 2: better in-sample fit is the trap, not the goal. More flexibility means more room to chase noise and extrapolate off-support.
    • +
  • Principle 3: you cannot validate the answer. With one treated unit and one post-period there is no held-out counterfactual, so cross-validation scores prediction error, never the missing cell.
    • +
  • Principle 4: the simplex ships an inspectable claim. Non-negative weights summing to one are an auditable blend of real markets while a boosted tree is a black box.
    • +
+
The forecaster is better at prediction. The simplex is better at the causal job.
+
+
+
+ +
+
Act II · The counterfactual problem
+

Compare the estimators

+
Every "obvious" estimator is a claim about \(Y_{1t}(0)\).
+
+
+
+
Four "obvious" estimators, four hidden assumptions
+ Each is a different guess at the missing \(Y_{1t}(0)\), unbiased only in its own special world (equations in the fold below).
+
    +
  • Before/after: unbiased only in a flat world, where the shared factors have equal pre- and post-means.
    • +
  • Treated vs average control: unbiased only if controls match the treated in level and loadings; the level gap survives even at spread \(s\to0\).
    • +
  • Difference-in-differences: removes the level gap, unbiased iff trends are parallel (\(\gamma_1=\bar\gamma\)), which \(s\to0\) delivers.
    • +
  • Synthetic control: unbiased iff a fitted blend matches the treated's loadings (\(\sum_j w_j\gamma_j=\gamma_1\), the treated inside the hull).
    • +
+
+
+
▶ LIVE Four estimates, one truth: the world's dials exposed
+
+
+ + + + +
+
+
+
▸ The four estimators, written out with their bias conditions
+
+ \[ \hat\tau^{\text{BA}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{1,\text{pre}} \] + Before/after: unbiased only in a flat world, equal shared-factor means pre and post (the macro-shock dial drives its error). +
+
+ \[ \hat\tau^{\text{T-C}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{\text{ctrl},\text{post}} \] + Treated vs average control: unbiased only if controls match the treated in level and loadings; the level gap survives even at spread \(s\to 0\). +
+
+ \[ \hat\tau^{\text{DiD}} \;=\; \Delta_{\text{treated}} - \Delta_{\text{ctrl}}, \quad \Delta = \bar Y_{\text{post}} - \bar Y_{\text{pre}} \] + Difference-in-differences: removes the level gap; unbiased iff trends are parallel (equal loadings \(\gamma_1=\bar\gamma\)), which spread \(s\to 0\) delivers. +
+
+ \[ \hat\tau^{\text{SC}} \;=\; \bar Y_{1,\text{post}} - \textstyle\sum_j w_j\, \bar Y_{j,\text{post}} \] + Synthetic control (the one you fitted): unbiased iff a fitted blend matches the treated's loadings (\(\sum_j w_j\gamma_j=\gamma_1\), the treated inside the hull). +
+
+
▸ How to read the dials (red = error against the truth)
+
    +
  • Loading spread → 0: markets react identically, trends become parallel, DiD's error collapses to noise. Treated-minus-control keeps its error: the level gap never dies.
    • +
  • Macro shock up, with spread > 0: before/after's error explodes, DiD's grows with the walk, and synthetic control stays pinned near the truth: it matches loadings instead of hoping they are equal.
    • +
+
+
None of these numbers ships a warning label
+ Each error is a wrong claim about \(Y_{1t}(0)\), and a wrong claim never improves with more data: the cure is a better claim, not a bigger sample.
+
+
+ + +
+
Act II · The counterfactual problem
+

Interesting limits for the simpler estimators

+
Where before/after and treated/control estimators break.
+
+
+
+
Before/after · one unit, across time
+ The treated metro's post-average minus its pre-average.
+
    +
  • The level \(\alpha_1\) cancels (same market, subtracted from itself), but nothing subtracts the shared wave.
    • +
  • The full factor drift \(\Delta\bar f\) lands at the metro's own loadings \(\gamma_1\approx1\), not at a mismatch: the largest of the three biases, and it grows with the horizon through the trend term \(0.4\,t\).
    • +
+
+
+
Treated-vs-control · one period, across units
+ The treated metro minus the control average, both in the post window.
+
    +
  • Time-differencing is gone, so the level gap \(\alpha_1-\bar\alpha_C\) survives, and market sizes differ a lot: this term dominates.
    • +
  • The loading gap now multiplies the factor level \(\bar f_{\text{post}}\), not its drift: this is why raw sales are never compared across cities.
    • +
+
+
+
▸ The two bias decompositions, term by term
+
+ \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] + the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. +
+
+ \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] + no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. +
+
+
+
+ + +
+
Act II · The counterfactual problem
+

Interesting limits for DiD

+
Where the DiD estimator breaks.
+
+
The one question
+ DiD subtracts the control trend from the treated trend, so it only works if the two would have drifted together. When is that true, and what does it cost when it is not? Two dials answer it (loading spread \(s\), macro shock \(\sigma_\eta\)); the algebra is in the fold.
+
Synthetic control solves it
+ The one bias DiD cannot shed, \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\), comes entirely from its equal weights \(\bar\gamma_C\): it must hope the spread \(s\) is small. Synthetic control chooses the weights instead, so \(\sum_j w_j\gamma_j=\gamma_1\) and \(B\to0\) at any spread. The residual DiD lives with, the twin removes by construction.
+
▸ The DiD bias, decomposed (and why more data cannot shrink it)
+
+ \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] +
+
    +
  • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
    • +
  • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
    • +
+
+ \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] +
+
    +
  • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
    • +
  • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
    • +
+
+
▸ The two limits, dial by dial: send a knob to zero and read what dies
+
+
Limit 1 · loading spread \(s \to 0\): sufficient on its own
+
    +
  • The prefactor \(s^2/3\) kills the whole bracket at once.
    • +
  • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
    • +
  • DiD then recovers \(\bar\tau\) up to noise.
    • +
+
Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
+
    +
  • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
    • +
  • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
    • +
  • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
    • +
+
+
+
▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
+ Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. +
+
+
+ + +
+
Act II · The counterfactual problem
+

What must be true for the gap to be causal

+
The four assumptions that must hold.
+
+ + + + + + + + + + + + + + + + + + + + +
AssumptionWhat it demandsThe checkVerdict
① Pre-fit qualityThe blend tracks the treated metro before launch: pre-RMSE below a third of the weekly lift we must detect.The gate: 3.2 vs a bar of 4.4.PASS
② No anticipationSales must not react before the campaign (no stockpiling, no leaked launch).Placebo-in-time: back-date the launch 10 weeks, the fake "effect" is ≈ €1.7k, inside noise.PASS
③ No spillover (SUTVA)The campaign must not touch donor markets (no travel, no online leakage): Colgate's second-launch leak, in space.No statistical check exists. A design duty: keep test and control regions apart.UNTESTABLE
④ Hull & stable loadingsThe treated metro is a feasible mix of donors, and exposures stay put through the window.Pre-fit gate + leave-one-out: drop any donor, the estimate stays in €239–276k.PASS
+
The referee principle
+ Three of the four are testable and the verdict is PASS. The one that is not, no-spillover, is exactly why geo-test design matters more than any statistic computed afterwards.
+
+
+ + +
+
Act III · Is it real, and how big?
+

€260k. Real, or a lucky metro? Build the null yourself

+
+
+
✋ Poll
+
The usual t-test needs many treated units (its standard error is a spread across treated units); with one treated metro it is not weak, it is undefined. So build the test yourself: run the whole synthetic-control pipeline 29 more times, each time pretending one untreated donor was the treated market ("placebo"). That yields 30 estimated "effects", 29 of them in markets where no campaign ran. Where does the real treated metro's €260k rank among the 30?
+
+ + + +
+ +
A: rank 1 of 30. And the rank is the inference. If the campaign did nothing, the treated metro is just another market, exchangeable with its donors, so P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test, the standard error the t-test could not supply.
+
+
+
+ + +
+
Act III · Is it real, and how big?
+

Placebo-in-space: measure the luck directly

+
Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
+
+
+
+
+
▶ LIVE Refit the estimator on every donor as if it were treated
+
+
The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
+
+
+
+
The null hypothesis, stated
+ \(H_0\): the campaign did nothing, \(\tau_t=0\) for every \(t\ge T_0\). Then the treated metro is just one more untreated market, exchangeable with its 29 donors, so its gap is equally likely to hold any of the 30 ranks.
+
+ \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] +
+
+
+
What the p-value means (and what it is not)
+ The probability, if \(H_0\) were true, of a gap as extreme as our €260k where nothing happened: a rank, not a bell-curve tail (no normality, independence, or asymptotics assumed). Nothing beat us, so \(p\) hits its floor \(1/(J+1)=1/30\), set by the donor count, not the weeks of data.
+
▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
+
    +
  • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
    • +
  • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
    • +
+
Conclusion: the €260k lift is real, at \(p\approx0.033\)
+ Our metro ranks 1 of 30, clear of the whole cloud of luck, so we reject \(H_0\): a real campaign effect, not a lucky draw, and the rank holds (p = 0.033 every time) when shaky placebos are dropped at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": that verdict waits for Act IV.
+
+
+ + +
+
Act III · Is it real, and how big?
+

From a test to an interval: inversion

+
We measured one number, €260k. Which true lifts could plausibly have produced it? We test every candidate against the placebos and keep the survivors, with no bell curve anywhere.
+
+
+
+
+
▶ LIVE Drag \(H\), a guess at the truth. Green line = our data, fixed. Blue cloud = the guess, it slides.
+
+
+ + +
+
Axis: 20-week total gap (€000). A the estimator's error at truth 0 · B that same error slid to \(H\), with its middle 90% · C every \(H\) whose band still covers €260k, collected: the interval. Drag past an edge to reject.
+
+
+
+
The whole idea
+ Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
+
    +
  • ① Measure the method's error. The 29 donor markets ran no campaign, yet refitting the estimator on each still reports a small fake "effect". Together they make a cloud centred on zero, spread about ±€50k: how far off the method typically lands when the truth is really zero.
    • +
  • ② Guess a true lift \(H\). \(H\) is a candidate for the truth, not our estimate: the dial you turn. If the real lift were \(H\), the method would report \(H\) plus that same error cloud, the blue cloud in the figure slid onto \(H\).
    • +
  • ③ Keep or reject \(H\). Keep \(H\) if our €260k lands inside the middle 90% of its cloud; reject it if €260k falls out in a tail (too far from \(H\) to be believable).
    • +
  • ④ Sweep and collect. Repeat for every \(H\). The smallest and largest that survive are €195k and €335k, so the interval is [€195k, €335k], read straight off the placebos.
    • +
+
+
+
▸ The inversion, in one line of algebra
+
+ \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} + \qquad\Longleftrightarrow\qquad + \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] + \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. +
+
+
▸ Why this interval is the referee for the rest of the lecture
+ It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. +
+
+
+ + +
+
Act III · Is it real, and how big? Stress-test the estimate
+

Falsification 1: a launch that did not happen

+
The anticipation check.
+
+
How it is computed
+ Refit using only weeks 0-29, pretend the launch happened at week 30, and read the "effect" in weeks 30-39, where the truth is zero.
+
▶ LIVE The weekly gap (treated minus synthetic) around a back-dated launch
+
+
+ + +
+
Clean world: the gap hugs zero through the fake window (average ≈ €1.7k) and lifts only after the real launch. Tick the box for a leaked launch: the gap climbs before week 40 and the fake window lights up.
+
+
What passing rules out
+ Stockpiling, pre-announcement, a leaked launch: anything that makes sales react before the campaign, which a synthetic control that "detects" an effect before treatment would reveal.
+
+
+ + +
+
Act III · Is it real, and how big? Stress-test the estimate
+

Falsification 2: the assumption no placebo can see

+
The no-spillover check: the disease you named for Colgate, now in space.
+
+
The assumption (SUTVA)
+ Every method so far assumed the campaign moved only the treated metro. Advertising ignores borders, so we break that on purpose and measure the damage.
+
+
▶ LIVE Four worlds, rebuilt with a bigger and bigger leak
+
+
Each point is a complete re-simulation and refit, not a correction applied afterwards.
+
+
+
The concrete story
+ Your metro's TV and outdoor ads reach the commuter belt. Those neighbouring markets are in your donor pool. So the campaign you are trying to measure quietly raises the very series you are using as "untreated" comparison.
+
+ \(\varphi\) is the leaked share: to each of 5 neighbour donors we add a fraction \(\varphi\) of the treated metro's own weekly lift \(\Delta_t\), then refit. Lifting the donors lifts the synthetic, so the measured gap shrinks. +
+
+
+
    +
  • The sweep: \(\varphi\in\{0,0.1,0.25,0.5\}\), from a clean world to half the lift leaking. Each point a full re-simulation and refit.
    • +
  • The result: the reported total slides from €260k (clean) to €236k, and the bias is always toward zero.
    • +
  • Dangerous because invisible: the leak starts at launch, so the pre-fit gate stays green and every placebo still passes, leaving no fingerprint to catch.
    • +
  • What drives the decision: because the bias can only attenuate, the number is a conservative floor: if it already clears the cost, spillover cannot overturn the call. The defence is design, keep donors off the treated metro's media footprint.
    • +
+
+
+ + +
+
Act III · Is it real, and how big?
+

Statistics done. Three numbers.

+
Questions ① and ② from the boardroom slide are now answered.
+
+ + + + + + + +
QuestionAnswerTool that answered it
Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
+
Truth check (only a simulation allows it)
+ The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
+
The sentence that loses money
+ "The campaign drove €260k of sales for €75k, a 3.5× return. Roll it out." Every number is true and the conclusion still does not follow: question ③, is the profit worth the price and with what confidence, is not a statistics question.
+
+
+ + +
+
Act IV · The decision in euros
+

Now the money

+
+
+
✋ Poll
+
Cost €75k, incremental sales €260k, gross margin 35%. What did the campaign earn?
+
+ + + + +
+ +
C: about €16k net. Profit = margin × lift − cost = 0.35 × 260 − 75 ≈ +€16k. Option A is the boardroom classic: it treats revenue as profit. The campaign returned €3.47 per €1 spent against a break-even of 2.86 at this margin: it paid, in expectation, with room to spare. Whether that expectation carries enough certainty to act on is the question the rest of this act prices.
+
+
+
+ + +
+
Act IV · The decision in euros
+

The margin trap

+
Compare profit to cost, not revenue to cost.
+
+
+
+
\[ \text{profit} \;=\; m\,\tau \;-\; c \]
+
  • Revenue is not cash: the business keeps only the gross margin \(m=35\%\) of the lift \(\tau\) yet pays the cost \(c\) in full, so comparing \(\tau\) to \(c\) treats revenue as profit.
+
+
+
\[ \tau_{\mathrm{BE}} = \frac{c}{m} = \frac{75}{0.35} = \text{€}214\mathrm{k}, \qquad \mathrm{iROAS}_{\mathrm{BE}} = \frac{1}{m} = 2.86 \]
+
  • Break-even is €214k of sales, not €75k: our €260k clears it by €46k of revenue (≈ €16k of profit), an iROAS of 3.47 against the \(1/m=2.86\) bar.
    • +
  • Hold onto €214k: it returns as the hinge of the €4M decision.
+
+
+
+
▶ LIVE Where the €260k goes: slide the margin
+
+
+ + +
+
"iROAS > 1" is a vanity bar; the real bar is 1/margin. Software (90% margin) breaks even at 1.1; grocery (20%) needs 5.0. Same campaign, opposite verdicts.
+
+
+
+ + +
+
Act IV · The decision in euros
+

The same evidence at every price

+
When is the campaign profitable?
+
+
+
  • Line 1, the point estimate: expected net \(m\hat\tau-c\) crosses zero at \(c=0.35\times260=\text{€}91\mathrm{k}\); below that price it looks profitable on average.
+
  • Line 2, the uncertainty zone: the profit interval \(0.35\times[195,335]=[\text{€}68\mathrm{k},\text{€}117\mathrm{k}]\), the straddle zone the evidence cannot separate from break-even.
+
+
+
▶ LIVE Slide the campaign price and watch the verdict move
+
+
+ + +
+
Between the lines live the dangerous campaigns: "looks profitable" (left of €91k) but not provably so. Ours at €75k is one of them, a comfortable mean the interval refuses to bless.
+
+ + + +
PriceBreak-even liftNet at \(\hat\tau\)VerdictThe lesson
+
+
+ + +
+
Act IV · The decision in euros, delivered
+

The pilot verdict, delivered to the board

+
This slide answers the €75k pilot. The €4M is a separate question, next.
+
+
+

Three questions about the €75k pilot, all answered by the permutation machinery:

+
    +
  • ① Real? Yes. Rank 1 of 30, \(p=0.033\).
    • +
  • ② Big enough? Yes, on the point estimate. \(\hat\tau = \text{€}260\mathrm{k}\) against a break-even of €214k: €46k of revenue to spare, about €16k of net profit.
    • +
  • ③ Confident? No. The profit interval [€68k, €117k] still straddles the €75k price. The upside is real but a loss is still inside the evidence.
    • +
+
Pilot verdict, banked
+ At €75k, probably profitable but not a confident buy: renegotiate the price or buy certainty with a bigger test. This says nothing yet about the €4M, which is not the pilot multiplied.
+
+
+ + + + + + + + + + +
Board numberValueSource
Incremental sales, 20 wk€260ksynthetic-control gap
Real, or luck?p = 0.033placebo permutation, rank 1/30
90% interval on sales[€195k, €335k]test inversion
iROAS vs break-even3.47 vs 2.86P&L: break-even = 1/margin
Gross profit vs price€91k vs €75k0.35 × 260 − 75
Pilot recommendationNOT A CONFIDENT BUYinterval straddles the price
+
The one question this memo cannot answer
+ Everything above is a yes/no about where an interval sits: it cannot say "78% chance it pays", name the highest price this evidence would still buy, or whether a follow-up study is worth funding. And it is only about €75k: the €4M rollout is the next two slides.
+
+
+
+ + +
+
Act IV · The decision in euros, scaled up
+

The €4M is not provably profitable

+
Even at its best case, the profit interval straddles zero.
+
+
+
+
+
▶ LIVE Rollout profit at €4M, carrying the 90% interval, as a share \(\delta\) of the pilot survives
+
+
+ + +
+
The point says +€860k at \(\delta=1\), but the 90% band [−€360k, +€2.25M] straddles zero even there. Drag \(\delta\) down (the real world, where each euro does less) and it only sinks; below \(\delta=0.64\) it wholly loses.
+
+
+
+
    +
  • ① The marketing number is a point: 53 pilot-budgets × €260k × 35% margin − €4M ≈ +€860k. Every step correct.
    • +
  • ② Carry the interval, and be generous: assume the pilot transports perfectly (\(\delta=1\)). Even then the 90% profit interval is [−€360k, +€2.25M]: it straddles zero.
    • +
  • ③ And \(\delta=1\) is a ceiling: €4M is €133k per market (1.78× the pilot, so each euro does less) and one metro is not the nation, so real \(\delta<1\) and the band only sinks.
    • +
+
The frequentist verdict, no probability needed
+ At its most generous the €4M is not provably profitable, and below \(\delta\approx0.64\) it provably loses. Do not commit €4M on this evidence.
+
+
+
▸ Where 0.82 and 0.64 come from, and a defensible estimate of δ
+
+
+
The bar: required δ (arithmetic)
+
\[ \delta^{*} = \frac{c}{m\,\tau} = \frac{\tau_{\mathrm{BE}}}{\tau} = \frac{214}{\tau} \]reach \(R=C/c\), so the budget \(C\) cancels: the bar is set by cost, margin and lift only
+
    +
  • Across the lift interval [€195k, €335k], required δ ∈ [0.64, 1.10], central 0.82 (82%).
    • +
  • Below 0.64 even the top of the interval loses: that is the "provably loses" edge.
    • +
+
+
+
The estimate: plausible δ (one assumption)
+
\[ \delta \;\approx\; 1.78^{\,\beta-1} \]per-euro lift at 1.78× intensity, up a concave response curve of elasticity \(\beta\)
+
    +
  • \(\beta\) from the client's MMM, or "double the spend → ≈1.6× response" → \(\beta\approx0.68\) → δ ≈ 0.83.
    • +
  • \(\beta\) 0.6–0.8 gives [0.79, 0.89], central 0.84 (84%); being one metro only widens this; ours sits at the median (effect CV ≈ 0.14).
    • +
+
+
+
Why we measure, not estimate
+ The estimate 0.84 lands on the bar 0.82: profit at €4M runs −€1.11M / about €0 / +€1.57M across the pessimistic / central / optimistic corners. Defensible enough to refuse €4M, too close to the line to commit it: only a multi-market test separates the two.
+
+
+
+ + +
+
Act IV · The decision in euros, scaled up
+

Measure before you commit

+
One treated market made this interval; only more will shrink it.
+
+
+
+
The call
+ Even the best case straddled zero, so on this evidence do not commit €4M. The rollout is not dead: the evidence simply cannot size it.
+
    +
  • Why analysis cannot rescue it: the interval is wide because there is one treated market. Re-running the model does not narrow it; only more treated markets do.
    • +
  • Step zero, first: check the drawer: prior pilots, the media-mix model, agency benchmarks may already narrow the range for the cost of one meeting.
    • +
+
+
+
The fix: measure the national effect directly
+ Run the campaign in 8 markets chosen to look like the nation, at full intensity (~€133k each, €1.07M). Because the cells span the country, the measured lift is the national lift, with an interval tight enough to clear or kill the €4M. HelloFresh's calibration loop, bought once.
+
The rule, written before the data lands
+ Release the remaining €2.9M only if the measured 90% profit interval clears zero. Written first, or the test is theatre.
+
The recommendation
+ ① Check the drawer. ② Do not commit €4M: the best case is not provably profitable. ③ Run the €1.07M, 8-market test at national intensity, randomised. ④ Separately: renegotiate the pilot toward €67k.
+
+
+
▸ Sizing the test, and why 8 markets
+
\[ k \;\ge\; \Bigl[\tfrac{(z_{0.95}+z_{0.80})\,\mathrm{CV}}{\Delta}\Bigr]^{2} = \Bigl[\tfrac{2.49\times0.30}{0.36}\Bigr]^{2} \approx 4.3 \]
+
    +
  • Sized from the decision: distinguish δ = 1 from δ = 0.64 (a 36% shortfall) against between-market spread; ≈ 5 markets on power, 8 to span the mix, and CV 0.4 pushes power itself to ≈ 8.
    • +
  • Constraints: randomise in matched pairs (that alone frees the p-value from the single-metro floor, 0.033 → ≈ 0.004); test at deployment intensity; keep controls off the media footprint.
    • +
  • Worth it: the Bayesian layer of this course prices this information at ≈ €320k, far above the test's incremental cost, since the €1.07M is real advertising working while it measures.
    • +
+
+
+
+ + + +
+
Closing · Provenance
+

The tools were the product too

+
+
    +
  • CausalPy: synthetic control, interrupted time series, difference in differences, and regression discontinuity in one open-source package: the method you just learned and its quasi-experimental family, industrialized by PyMC Labs. The IV estimator that closes this session joined the package later.
    • +
  • Its launch example: individual exposure to a TV campaign cannot be randomised, yet its causal impact remains a core business need: the sentence this whole session opened with.
    • +
  • pymc-marketing: the MMM library behind Case 2's calibration story; one client's budget allocation approach to PyMC-Marketing came back as a pull request (Bolt).
    • +
  • Webinars and content: the consultancy's own webinar walks geo-experimentation, MMM, synthetic control, difference-in-differences, regression discontinuity and Instrumental Variables: this session's syllabus.
    • +
+
+ CausalPy + PyMC-Marketing +
+
+
+ +
+
Closing
+

The pattern in every engagement

+
+
    +
  • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
    • +
  • An experiment anchors every observational model: calibration is the product, not a luxury.
    • +
  • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
    • +
+ + + + + + +
Agent, on adversarial MMM dataResult
Vanilla coding agentFit a model, recommended budget reallocations. Confidently wrong.
PyMC Labs' Decision LabExplored 11 approaches, 0 converged. Returned: "No valid model found. Run a geo-holdout experiment."
+
Even the machines know the punchline
+ The honest system's best answer was Part 2's closing advice: run the experiment.
+
+
+ +
+
Closing
+

One breath

+
+
The pattern to take home
+ The toolkit a Bayesian consultancy sells: counterfactuals, calibrated by experiments, priced as probabilities.
+
    +
  • Read the cases: pymc-labs.com/blog-posts: every number in this deck is pinned to a post, listed on the next slide.
    • +
  • Say hello: both authors consult for PyMC Labs; the notebooks behind this session are the course repository.
    • +
+
+
+ + +
+
Backup
+ Backup · Sources +

Every number, pinned

+
Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
+
+ + + + + + + + + + + + + + + + + +
SourceFacts pinned
ailab.criteo.com · criteo-uplift-prediction-datasetcriteo_rows
pymc-labs.com · 2022-11-11-HelloFreshhf_panel_calibration
pymc-labs.com · 2023-06-20-juan-marketing-analyticswebinar_agenda
pymc-labs.com · bayes-is-slow-speeding-up-hellofreshs-bayesian-ab-tests-by-60xhf_batch, hf_test_types, hf_thousands
pymc-labs.com · bayesian-media-mix-modeling-for-marketing-optimizationhf_priors_experiments
pymc-labs.com · causal-sales-analytics-are-my-sales-incremental-or-cannibalisticcolgate_ci, colgate_ci_level, colgate_truth, colgate_year, fail_range, fail_truth, market
pymc-labs.com · causal-sales-analytics-discrete-choice-modelingcolgate_counterfactual_quote
pymc-labs.com · causalpy-a-new-package-for-bayesian-causal-inference-for-quasi-experimentscausalpy_methods, causalpy_tv
pymc-labs.com · funnel-aware-mmmcpl, cpl_window, gdpr_sentence, herp_attribution_quote, nurn_2026, trust_quote
pymc-labs.com · marketing-mix-modeling-a-complete-guidebolt_pr
pymc-labs.com · mmm_roas_liftlift_tests_n, roas_gap_words, roas_wrong_ranking, roas_x1, roas_x2
pymc-labs.com · open-sourcing-decision-lab-scaling-ai-judgment-data-sciencedl_explored, dl_vanilla, dl_verdict
pymc-labs.com · reducing-customer-acquisition-costs-how-we-helped-optimizing-hellofreshs-marketing-budgethf_var
+
+
+ +
+ + + +
+ Causal Marketing · SDA Bocconi · Causal Inference in the Wild + + + + 1 / 1 +
+
+

Contents

    + + + + diff --git a/causal-marketing-pymc/apps/unified_slides_src.html b/causal-marketing-pymc/apps/unified_slides_src.html new file mode 100644 index 0000000..cf58ee6 --- /dev/null +++ b/causal-marketing-pymc/apps/unified_slides_src.html @@ -0,0 +1,2684 @@ + + + + + + +Causal Inference in the Wild: from real engagements to a €4M decision + + + + + +
    + + + +
    +
    Causal Inference & XAI for Business · SDA Bocconi
    +

    Causal Inference in the Wild

    +
    Real PyMC Labs engagements, then the machinery behind them: one treated market, no experiment, €4M on the line.
    +
    A guest lecture for Prof. Michele Russo's course
    +
    +
    +
    Francesco Muia
    +
    PhD in Theoretical Physics, EMBA.
    Consultant for PyMC Labs and Brown University.
    +
    francesco.muia@pymc-labs.com
    +
    francesco.muia@ai-and-analytics-solutions.com
    +
    +
    +
    Alexander Fengler
    +
    PhD in Computational Cognitive Science.
    Postdoc at Brown University, consultant for PyMC Labs.
    +
    alexander.fengler@pymc-labs.com
    +
    +
    +
    + PyMC + PyMC Labs +
    +
    + + +
    +
    Opening · Who is talking
    +

    PyMC Labs: what we do

    +
    A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
    +
    +
      +
    • What we sell: senior modeling expertise, not seats or software licenses: revenue is services-led.
      • +
    • Open source is the top of the funnel: the libraries build trust and reach; the consulting monetizes the deep expertise behind them.
      • +
    • How we work: small teams of senior people who start from the client's actual decision, encode domain knowledge and uncertainty explicitly, and hand back models the client can own and extend, not black boxes.
      • +
    + + + + + + + +
    EngagementWhat it isTypical client
    Project consultingFixed-scope build of a custom model: a marketing-mix model, a demand forecaster, a pricing modelA specific, high-value decision problem
    Advisory / retainerOngoing access to our modelers, guiding an in-house teamFirms building their own capability
    Enablement & trainingWorkshops and embedded upskilling on Bayesian methods and our toolingAnalytics teams standardizing on PyMC
    +
    The clients in this session
    + Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
    +
    ▸ Areas of interest, and where we work
    +
      +
    • Areas: marketing-mix modeling and media measurement, causal inference, demand forecasting and pricing, experimentation and A/B testing at scale, applied Bayesian modeling.
      • +
    • Sectors: consumer/CPG, retail and e-commerce, tech, and finance.
      • +
    +
    +
    +
    + + +
    +
    Case 1 · Colgate-Palmolive
    +

    You are the consultant

    +
    The counterfactuals this session builds are what clients buy. First test: route a real call, on instinct.
    +
    +
    +
    ✋ Poll
    +
    Colgate-Palmolive calls: "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it stealing share from competitors, or from our own brands?" Which tool do you reach for first?
    +
    + + + + +
    + +
    C. A national launch leaves nothing to randomize and no market to difference against: A and B need a control group that does not exist, and D's instrument does not exist for a shelf that changed everywhere at once. What remains is the move this whole session builds, here run in time: fit the world before the launch, project it forward, read the gap. PyMC Labs sold exactly that projection; the next slide shows it.
    +
    +
    +
    + +
    +
    Case 1 · Colgate-Palmolive
    +

    Colgate-Palmolive: incremental, or cannibalistic?

    +
    Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The launch verdict is the split.
    +
    +
    +
    +
    The launch, and the world without it schematic
    +
    +
    Illustrative shape of the engagement's counterfactual read, not client data: fit the pre-period, project it forward, price the gap.
    +
    +
    +
    +
    The brief, in their words
    + "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
    +
      +
    • The client: Colgate-Palmolive came to PyMC Labs in {{labs.colgate_year}}, in a market estimated at {{labs.market}}.
      • +
    • The method: a multivariate Bayesian interrupted time series: the pre-launch world projected forward, pointed at a product; later extended to a nested-logit choice model.
      • +
    • The grading: on simulated data the model recovers a planted {{labs.colgate_truth}} incrementality as a {{labs.colgate_ci_level}} interval of {{labs.colgate_ci}}: the recover-the-truth contract of this whole session, run commercially.
      • +
    +
    +
    +
    + +
    +
    Case 1 · Colgate-Palmolive · open floor
    +

    What would break it?

    +
    +
    +
    🗣 Open floor · 2 minutes
    +
    You are Colgate's CMO. The incrementality estimate you just saw (the share of the new product's sales that are genuinely new, not cannibalized) decides the launch review. Name one real-world event that would make it wrong.
    + +
    A second launch. When another product entered the estimation window, the same machinery reported {{labs.fail_range}} incrementality against a planted truth of {{labs.fail_truth}}: the counterfactual absorbed part of the very effect it was meant to isolate. An honest consultancy publishes exactly this: the {{labs.fail_range}} miss is printed in the same post as the win. In Part 2 the same disease returns with its formal name, spillover; the defence is design, not statistics.
    +
    +
      +
    • What the model learns: everything before the launch defines "normal growth", and the projection (red) extrapolates that normal forward.
      • +
    • What a second launch does: inside the window it becomes part of "normal", so the projection rises too fast and under-credits the true lift; after the launch it inflates the observed line instead, and over-credits.
      • +
    +
    +
    Break it yourself: slide a second launch into the window schematic
    +
    +
    + + + +
    +
    Same schematic world as the previous slide. The real case reported {{labs.fail_range}} against a truth of {{labs.fail_truth}}.
    +
    +
    +
    + + +
    +
    Case 2 · HelloFresh · the tool, and its failure mode
    +

    Why calibrate? A model alone can rank channels backwards

    +
    Before the HelloFresh story, the tool it relies on: a warm-up from PyMC Labs' published calibration tutorial.
    +
    +
      +
    • The tool: a marketing-mix model (MMM) explains total sales as the sum of per-channel contributions, fit on observational spend data: no experiment anywhere in it.
      • +
    • The grading: the tutorial plants a truth: return on ad spend (ROAS, sales per unit of spend) of {{labs.roas_x1}} for channel x1 against {{labs.roas_x2}} for x2, so x2 is {{labs.roas_gap_words}}.
      • +
    • The experiment: a lift test nudges one channel's spend by a known amount and measures the sales change it causes: a small randomized ground-truth reading for that channel.
      • +
    • The repair: {{labs.lift_tests_n}} per channel, entered into the likelihood, recover both values: the experiment is the model's anchor, the theme of everything that follows.
      • +
    +
    +
    One MMM on observational spend alone, one planted truth, one inversion baked from the tutorial
    +
    +
    +
    Left: the ranking the uncalibrated model reported. Right: the planted truth the experiments recover.
    +
    +
    The inversion
    + Fit on observational data alone, the baseline model ranked x1 above x2: {{labs.roas_wrong_ranking}}.
    +
    +
    + +
    +
    Case 2 · HelloFresh
    +

    HelloFresh runs the loop, at industrial scale

    +
    +
    +
      +
    • The loop: MMM priors fed by field experiments such as {{labs.hf_priors_experiments}}; a {{labs.hf_var}} cut in prediction variance.
      • +
    • On stage: the panel's own agenda: Bayesian MMM can be {{labs.hf_panel_calibration}}.
      • +
    • The experiment supply: a pipeline handling {{labs.hf_thousands}}: {{labs.hf_test_types}} campaigns run simultaneously, the overnight batch down from {{labs.hf_batch}}; the Criteo experiment from the session's IV close ({{labs.criteo_rows}} users) sits in exactly this regime.
      • +
    +
    The supply chain, for the IV close
    + The experiments a company already runs are its instrument supply: a randomized encouragement is the instrument for the exposure you cannot randomize.
    +
    +
    +
    +
    The loop, in one picture
    +
    +
    The model runs always-on; the experiment disciplines it; the counterfactual reads the experiment out.
    +
    +
    +
    +
    + + +
    +
    Case 3 · Nürnberger Versicherung
    +

    Price the engagement

    +
    A German insurer, last-touch attribution, and a funnel-aware causal MMM, in production.
    +
    +
    +
    ✋ Poll
    +
    Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over {{labs.cpl_window}} of model-guided spend, cost per lead (CPL) moved by how much?
    +
    + + + + +
    + +
    C. "This year we were able to drive the CPL down by {{labs.cpl}}, which is very, very good" (Philip Herp, Nürnberger Versicherung). The mechanism is the lesson: under GDPR, {{labs.gdpr_sentence}}, so last-touch under-credited the upper funnel and budget followed {{labs.herp_attribution_quote}}. The funnel model measured what video spend causes downstream, and the client is scaling it into {{labs.nurn_2026}}.
    +
    +
    The client's bar for belief
    + "{{labs.trust_quote}}"
    +
    +
    + + +
    +
    +
    +
    Part 2 · The deep dive
    +

    Synthetic Control

    +
    One treated market, no experiment, €4M on the line: did the campaign work?
    +
    +
    + +
    Where the lecture lands: a synthetic twin rebuilds the metro's counterfactual, and the gap beyond break-even is the campaign's profit.
    +
    +
    +
    + + +
    +
    Act I · The question
    +

    The boardroom question

    +
    A campaign ran in one region. Sales rose. Marketing wants €4M to go national. You decide. The Colgate question again: what would have happened anyway, now with €4M riding on it.
    +
    +
    +
    +

    What you have

    +
      +
    • 30 markets, 60 weeks of weekly sales.
      • +
    • One metro, week 40: a €75k campaign for 20 weeks; the other 29 did nothing.
      • +
    • No randomisation: the metro was chosen, not drawn.
      • +
    • 35% gross margin on incremental sales.
      • +
    +
    +
    +

    What the data forces you to answer, in order

    +
    +
    ① How much did the campaign actually cause? attribution
    +
    ② Is that more than luck? significance
    +
    ③ Is the profit worth the price? the decision
    +
    +
    +
    +
    The method this data calls for
    +
      +
    • One treated unit, an observational time series, no A/B test possible.
      • +
    • This is synthetic control's home ground.
      • +
    • It is the engine under Meta's GeoLift and Google's geo methodology.
      • +
    +
    The €4M hinge: the transportability discount
    +
      +
    • δ = the share of the pilot's per-euro lift that survives national rollout.
      • +
    • Measuring this one metro is unbiased as the fit uses its own pre-launch record.
      • +
    • The €4M call reduces to one question: is δ large enough for the national rollout to still clear its cost?
      • +
    +
    ▸ What "unbiased" means here, precisely, and where it would break
    +

    The estimator subtracts a reconstructed counterfactual from the treated metro's post-launch sales, and bias is its expected gap from the true lift:

    +
    + \[ \hat\tau \;=\; \bar Y_{1,\text{post}} - \hat Y_{1,\text{post}}(0), \qquad \text{bias} \;\equiv\; \mathbb{E}[\hat\tau] - \tau \] +
    +
      +
    • The counterfactual is the synthetic twin, built to reproduce the metro's pre-launch record (its level and its factor loadings) week by week.
      • +
    • Unbiasedness rests on one assumption: that pre-launch match would have continued through the post window had the campaign never run.
      • +
    • This needs no story about why the metro was chosen. Whatever pre-launch trait drove the pick, the twin already carries it, because it was fitted to match it: selection on anything visible before launch cannot tilt the estimate.
      • +
    • Bias enters only through what the pre-launch record cannot see: if the launch were timed on private knowledge of a coming local boom, the twin could not anticipate it and the gap would credit the boom to the campaign. The later placebo-in-time test hunts exactly that.
      • +
    +
    +
    +
    + + +
    +
    Act I · The question
    +

    The data you actually get

    +
    The simulated world.
    +
    +
    +
    +
      +
    • \(Y_{jt}\): weekly sales of market \(j\) in week \(t\) (€000): 30 markets (treated metro = market 1, donors \(j=2,\dots,30\)), 60 weeks (\(t=0,\dots,59\)) = 1,800 observed numbers.
      • +
    • The design: the treated metro ran a €75k campaign from week \(T_0=40\) for 20 weeks at a 35% margin; the other 29 ran nothing.
      • +
    • The choice: the firm's marketing team picked the metro and the week. Not us, and not a coin.
      • +
    +
    +
    +
    The one number you do not have
    + What the treated metro would have sold over those 20 weeks with no campaign. It is nowhere in the file, i.e. the counterfactual.
    +
    +
    +
    +
    +
    +
    ▶ LIVE 30 markets, one treated at week 40
    +
    +
    + + + +
    +
    +
    +
    +
      +
    • Before/after: post minus pre-launch mean, /week.
      • +
    • Subtract the crowd: remove the shared wave and the step is ≈ /week.
      • +
    • The noise floor: pre-launch sd /week around a /week base.
      • +
    +
    Which number decides?
    +
      +
    • Is this all due to the campaign?
      • +
    • What's the right number to take a decision?
      • +
    +
    +
    +
    ▸ Where each of these three readings comes back later
    +
      +
    • Before/after is the estimator \(\hat\tau^{\text{BA}}\) in the zoo you will grade: it assumes \(Y_{1t}(0)\) simply holds at the pre-launch mean.
      • +
    • Subtracting the crowd strips out the shared wave \(\gamma_j^{\top} f_t\) of the next slide's model, which is exactly what treated-vs-control and difference-in-differences try to do.
      • +
    • The noise floor is that model's own \(\varepsilon_{jt}\) term, the wobble every estimator must out-shout with a single treated unit.
      • +
    +
    +
    +
    + + +
    +
    Act I · The question
    +

    Commit to a number

    +
    +
    +
    ✋ Poll
    +
    Before/after on the treated metro: sales averaged ·/week before launch and ·/week after, a bump of ·/week (about +6%). What is the campaign's true weekly effect?
    +
    + + + +
    + +
    C. The true effect is ·/week, a 12% lift on a ·/week baseline. On the previous slide you already saw the naive bump grow once the shared crowd was removed: the first sign it understates. Before/after books barely half the truth because the shared macro wave happened to dip after launch: the tide can inflate a campaign's credit, or, as here, hide it. A naive comparison is not "roughly right plus noise"; it is charged with everything else that changed in the world, in whichever direction that happened to be. Making the counterfactual visible is the whole method.
    +
    +
    +
    + + +
    +
    Act I · The question
    +

    Causal inference is a missing-data problem

    +
    +
    +

    Same panel as the data slide, one new idea: the treated metro (market 1) has two week-\(t\) sales, only one of which ever happens. \(Y_{1t}(1)\) with the campaign, \(Y_{1t}(0)\) without. The causal effect is the gap between those two worlds:

    +
    + \[ \tau_t \;=\; Y_{1t}(1)-Y_{1t}(0), \qquad \tau \;=\; \sum_{t\ge T_0}\tau_t \] +
    +
      +
    • \(\tau_t\) vs \(\tau\): the weekly effect, and the 20-week total the business actually buys.
      • +
    • Consistency: what we record is the realised arm, \(Y_{1t}=Y_{1t}(1)\) after launch and \(Y_{1t}=Y_{1t}(0)\) before it, one world per week, never both.
      • +
    • The one missing cell: \(Y_{1t}(0)\) for \(t\ge T_0\) is never observed, the fundamental problem of causal inference, and every method fills exactly that cell.
      • +
    +
    Define the estimand first
    + Per-week effect and 20-week total are different estimands with different uncertainties, and the rollout decision turns on the total.
    +
    +
    + + + + + + + +
    pre (t < 40)post (t ≥ 40)
    treated · \(Y_{1t}(1)\)·observed ✓
    treated · \(Y_{1t}(0)\)observed ✓missing: the counterfactual
    donors · \(Y_{jt}(0)\)observed ✓observed ✓
    +
    Three of four cells are data; the donors' untreated post-period is the live information a synthetic \(Y_{1t}(0)\) is rebuilt from.
    +
    ▸ Reading the notation (observed vs potential)
    +
      +
    • \(Y_{jt}\) from the data slide is what you observe; \(Y_{1t}(1),\,Y_{1t}(0)\) are the treated metro's two potential outcomes.
      • +
    • Subscript says which market, argument says which world (1 = campaign, 0 = none): distinct axes that happen to share the digit 1.
      • +
    • \(w_j\): the donor weights the synthetic control will choose so \(\sum_{j\ge 2} w_j Y_{jt}\) rebuilds the missing \(Y_{1t}(0)\).
      • +
    +

    "What would this metro have sold anyway?" is an estimation target, not a rhetorical question.

    +
    +
    +
    + + + + +
    +
    Act I · The question, and the world behind the data
    +

    The model that generated the data

    +
    We simulate the data: here the counterfactual is known, so every estimator can be graded.
    +
    +

    Each market's no-campaign sales are its level \(\alpha_j\), its exposures \(\gamma_j\) to three shared factors \(f_t\), plus noise: the whole world in one line (all €000).

    +
    + \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} + \;=\; \alpha_j \;+\; \gamma_{j1}\underbrace{\,0.4\,t\,}_{\text{trend}} \;+\; \gamma_{j2}\underbrace{\bigl[\,8\sin(2\pi t/26)+4\sin(2\pi t/13)\,\bigr]}_{\text{seasonality}} \;+\; \gamma_{j3}\underbrace{\textstyle\sum_{s=1}^{t}\eta_s}_{\text{macro walk }m_t} \;+\; \varepsilon_{jt} \] +
    +
      +
    • The macro walk \(m_t=\sum_{s\le t}\eta_s\): a running sum of shocks that wanders like an economy, its variance \(t\,\sigma_\eta^2\) growing every week.
      • +
    • The exposures \(\gamma_j\sim\mathrm{U}(1{-}s,1{+}s)\): each market responds its own amount, and the spread \(s\) is what kills parallel trends.
      • +
    +
    ▸ All five ingredients, one by one
    +
      +
    • Trend \(0.4\,t\): steady growth of 0.4 (€000) per week. Deterministic.
      • +
    • Seasonality \(8\sin(2\pi t/26)+4\sin(2\pi t/13)\): a 26-week half-year cycle of amplitude 8 and a 13-week quarterly cycle of amplitude 4. Identical for every market.
      • +
    • The macro walk \(m_t\): one fresh shock \(\eta_s\sim\mathcal{N}(0,\sigma_\eta^2)\) per week, none ever forgotten. This is "the economy".
      • +
    • Exposures \(\gamma_j\): each market multiplies each ingredient by its own loading.
      • +
    • Noise \(\varepsilon_{jt}\sim\mathcal{N}(0,\sigma_\varepsilon^2)\): the market's own weekly wobble, \(\sigma_\varepsilon\approx 3\).
      • +
    +
    +
    Which of these is ever observed?
    + Only \(Y_{jt}\) reaches is observed. The level, loadings, factors and noise (market size, seasonal sensitivity, the economy) are real but recorded nowhere. Every estimator that follows is a way to cope with that.
    +
    The planted truth, in business units
    + On top of the world above we switch on a 12% lift at launch week \(T_0=40\): ·/week on average, · over the 20 campaign weeks.
    +
    ▸ The planted lift, written out
    +
    + \[ \tau_t \;=\; 0.12\,\bigl(\alpha_1+\gamma_1^{\top}f_t\bigr)\,\mathbf{1}_{\{t\ge T_0\}} \] + 12% of the treated metro's own baseline \(\alpha_1+\gamma_1^{\top}f_t\), switched on from week \(T_0=40\). +
    +
    +
    +
    + + +
    +
    Act I · The question, and the world behind the data
    +

    Simulate the world yourself

    +
    Take a look at the counterfactual.
    +
    +
    +
    ▶ LIVE The equations of the previous slide. Grey: donor markets. White: the treated metro.
    +
    +
    + + + + + + + + +
    +
    Dashed blue: the treated market's \(Y_{1t}(0)\), the potential outcome itself (not an estimate), which in real data does not exist.
    +
    +
    +
    +
      +
    • True lift: the planted 12% rate averaged over the 20 post-launch weeks (its euro value varies week to week).
      • +
    • Before/after: the treated series' post-launch mean minus its pre-launch mean.
      • +
    • Its error: the % gap between them, because before/after charges the whole tide to the campaign.
      • +
    +
    +
    +
    What the sliders teach
    +
      +
    • Loading spread \(s\to 0\): every market reacts alike, trends run parallel, DiD lives.
      • +
    • Macro shock \(\sigma_\eta\) up: the shared wave dominates, before/after becomes garbage.
      • +
    • Noise \(\sigma_\varepsilon\) up: the weekly wobble grows, and any one estimate gets harder to trust.
      • +
    +
    +
    +
    +
    + +
    +
    Act II · The counterfactual problem
    +

    Abadie's idea: if no twin exists, build one

    +
    Idea: manufacture a treated market's twin using a blend of donor markets.
    +
    +

    Assume untreated sales follow the factor structure of the DGP slide, now taken as the assumption rather than the recipe: a level \(\alpha_j\), loadings \(\gamma_j\) on shared latent factors \(f_t\), plus noise (equation in the fold below).

    +
      +
    • \(f_t\), the latent factors: the shared shocks (macro wave rising, season turning) that every market feels, but not equally.
      • +
    • \(\gamma_j\), the loadings: how strongly market \(j\) responds to each factor (\(\alpha_j\) is its level); in real data none of these are observed.
      • +
    • The method never names them: in the sim \(f_t\) is trend + season + macro walk, but on real data it works without knowing what the factors are.
      • +
    +

    Blend the donors so their level and loadings equal the treated metro's:

    +
    + \[ \sum_j w_j\,(\alpha_j,\;\gamma_j) = (\alpha_1,\;\gamma_1) \;\Longrightarrow\; \sum_j w_j\,Y_{jt} \approx Y_{1t}(0)\ \ \text{for all } t \] +
    +
      +
    • Same loadings, same response to every factor movement, including ones that have not happened yet: that forward guarantee is the whole trick.
      • +
    +
    Note
    + DiD matches a level while synthetic control matches the response to shocks, which is why it survives the non-parallel trends that break DiD.
    +
    ▸ The factor equation, pedigree, and the papers' notation
    +
    + \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} \] + the assumed structure the blend must match, level and loadings, week by week +
    + Abadie & Gardeazabal (2003); Abadie, Diamond & Hainmueller (2010): "arguably the most important innovation in the policy-evaluation literature in the last 15 years" (Athey & Imbens 2017). The literature writes the factors \(\lambda_t\) and the loadings \(\mu_j\), and adds a common shock \(\delta_t\) felt identically by every market; any cross-market comparison cancels \(\delta_t\), so nothing changes here. +
    +
    +
    + +
    +
    Act II · The counterfactual problem
    +

    The estimator, precisely

    +
    A constrained least square optimization.
    +
    +

    Fit the weights on the pre-period only, constrained to the simplex:

    +
    + \[ \hat w \;=\; \arg\min_{w\in\Delta}\; \sum_{t \lt T_0}\Bigl(Y_{1t}-\sum_j w_j Y_{jt}\Bigr)^{2}, \qquad \Delta=\Bigl\{w : w_j\ge 0,\ \textstyle\sum_j w_j=1\Bigr\} \] +
    +
      +
    • Least squares on the 40 pre-launch weeks: the post-period is never shown to the optimiser, so the fit cannot learn the effect it will measure.
      • +
    • The simplex \(\Delta\): weights are non-negative and sum to one, so the synthetic is an interpolation, never "−80% of Milan + 190% of Rome": the weights-on-the-table version of what Colgate's projection did in time.
      • +
    • No likelihood, no standard error: constrained least squares (SLSQP) ships a point estimate only, and inference comes later, from placebos.
      • +
    +
    ▸ Reading the effect off the gap
    +
    + \[ \hat\tau_t \;=\; Y_{1t}-\sum_j \hat w_j Y_{jt}, \qquad\qquad \hat\tau \;=\; \sum_{t\ge T_0}\hat\tau_t \] + weekly gap between the treated metro and its synthetic twin, summed over the 20 post-launch weeks +
    +
    +
    Why constrain at all?
    + The simplex buys three things ordinary regression cannot: interpretability, regularisation, and an off-switch.
    +
      +
    • Interpretability: "40% dma_08 + 32% dma_20 + 17% dma_03" is a sentence a planner can act on.
      • +
    • Regularisation: most weights land on exactly zero.
      • +
    • An off-switch: a market no blend can match fails loudly instead of extrapolating.
      • +
    +
    +
    + + +
    +
    Act II · The counterfactual problem
    +

    What the simplex buys, geometrically: stay inside the hull

    +
    A simplex blend can only land inside the donors' convex hull while OLS can leave it.
    +
    +
    +
    +
    +
    ▶ LIVE Drag the treated market in and out of the donors' hull
    +
    +
    + + +
    +
    +
    +
    +
      +
    • With \(w_j\ge 0\) and \(\sum_j w_j=1\), every point a synthetic market can reach lies inside the shaded convex hull.
      • +
    • Inside the hull: a real blend reproduces the treated market's fingerprint, so a clean pre-period fit keeps working out of sample.
      • +
    • Outside the hull: no non-negative blend reaches it; the simplex stops at the frontier, and only extrapolation (negative weights) closes the gap.
      • +
    +
    Why constrain the weights
    + The simplex forces the synthetic to be an interpolation of markets you actually observed, so a good fit is a real blend, not an extrapolation that happens to line up.
    +
    When the treated market sits outside
    + That is the signal to widen the donor pool, not to trust an OLS fit that only matches by leaving the hull. The refusal is a feature. Ask any vendor: when does your method refuse to answer? If never, walk away.
    +
    +
    +
    +
    + + + +
    +
    Act II · The counterfactual problem
    +

    What the constraint buys: drop it and see

    +
    What if we used a simple unconstrained OLS?
    +
    +
    +
    ▶ LIVE Two synthetics against the truth only a simulation can draw
    +
    +
    OLS hugs the pre-period tighter ( vs ) yet drifts further from the true \(Y_{1t}(0)\) after launch ( vs ): overfitting the simplex refuses.
    +
    +
    +
    +
    The weights OLS chose
    +
    +
    OLS puts negative weight on donors (down to ) and gross weight , where the simplex uses exactly 1.
    +
    +
    +
    + \[ n_{\mathrm{eff}} \;=\; \frac{1}{\sum_j \hat w_j^{2}} \] + Effective number of donors (inverse Herfindahl): one donor \(\Rightarrow n_{\mathrm{eff}}=1\), all 29 equally \(\Rightarrow 29\). Ours: 3.3. +
    +
    What n_eff = 3.3 tells you
    +
      +
    • The synthetic leans on about three donor markets, not a fuzzy mix of all 29.
      • +
    • Sparse weights keep the blend interpretable and stable out of sample: fewer donors carrying real weight means less room to overfit noise.
      • +
    • It is a health check: near 1 the twin is hostage to a single market, near 29 it is averaging everything into mush. 3.3 is concentrated but not fragile.
      • +
    +
    +
    +
    +
    + + +
    +
    Act II · The counterfactual problem, versus machine learning
    +

    "Why not gradient boosting / Prophet / an LSTM?"

    +
    Is this a forecasting exercise?
    +
    +
    +
    ▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
    +
    +
    Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
    +
    +
      +
    • In-sample the forecaster fits tighter: pre-launch error €k/week vs the simplex's €k/week. More flexibility always buys a closer fit to the past.
      • +
    • Yet their campaign estimates are a wash: the forecaster reads and the simplex for the 20-week effect, both a touch under the €284k the simulation planted (the truth we can see only because this is a simulation). On this one dataset you cannot say which is better.
      • +
    • The gap shows only in the sweep: across 24 fresh worlds the unconstrained fit reconstructs the missing \(Y_{1t}(0)\) about {{nb07.hull_oos_ratio}}× worse out of sample on average, and its weights leave nothing to inspect.
      • +
    +
    +
    +
    Principle 1: a counterfactual, not a forecast
    + A forecaster asks what comes next in a world like the training data. The campaign asks what this one market would have done in a world that never happened: extrapolation to an unobserved regime, not prediction.
    +
      +
    • Principle 2: better in-sample fit is the trap, not the goal. More flexibility means more room to chase noise and extrapolate off-support.
      • +
    • Principle 3: you cannot validate the answer. With one treated unit and one post-period there is no held-out counterfactual, so cross-validation scores prediction error, never the missing cell.
      • +
    • Principle 4: the simplex ships an inspectable claim. Non-negative weights summing to one are an auditable blend of real markets while a boosted tree is a black box.
      • +
    +
    The forecaster is better at prediction. The simplex is better at the causal job.
    +
    +
    +
    + +
    +
    Act II · The counterfactual problem
    +

    Compare the estimators

    +
    Every "obvious" estimator is a claim about \(Y_{1t}(0)\).
    +
    +
    +
    +
    Four "obvious" estimators, four hidden assumptions
    + Each is a different guess at the missing \(Y_{1t}(0)\), unbiased only in its own special world (equations in the fold below).
    +
      +
    • Before/after: unbiased only in a flat world, where the shared factors have equal pre- and post-means.
      • +
    • Treated vs average control: unbiased only if controls match the treated in level and loadings; the level gap survives even at spread \(s\to0\).
      • +
    • Difference-in-differences: removes the level gap, unbiased iff trends are parallel (\(\gamma_1=\bar\gamma\)), which \(s\to0\) delivers.
      • +
    • Synthetic control: unbiased iff a fitted blend matches the treated's loadings (\(\sum_j w_j\gamma_j=\gamma_1\), the treated inside the hull).
      • +
    +
    +
    +
    ▶ LIVE Four estimates, one truth: the world's dials exposed
    +
    +
    + + + + +
    +
    +
    +
    ▸ The four estimators, written out with their bias conditions
    +
    + \[ \hat\tau^{\text{BA}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{1,\text{pre}} \] + Before/after: unbiased only in a flat world, equal shared-factor means pre and post (the macro-shock dial drives its error). +
    +
    + \[ \hat\tau^{\text{T-C}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{\text{ctrl},\text{post}} \] + Treated vs average control: unbiased only if controls match the treated in level and loadings; the level gap survives even at spread \(s\to 0\). +
    +
    + \[ \hat\tau^{\text{DiD}} \;=\; \Delta_{\text{treated}} - \Delta_{\text{ctrl}}, \quad \Delta = \bar Y_{\text{post}} - \bar Y_{\text{pre}} \] + Difference-in-differences: removes the level gap; unbiased iff trends are parallel (equal loadings \(\gamma_1=\bar\gamma\)), which spread \(s\to 0\) delivers. +
    +
    + \[ \hat\tau^{\text{SC}} \;=\; \bar Y_{1,\text{post}} - \textstyle\sum_j w_j\, \bar Y_{j,\text{post}} \] + Synthetic control (the one you fitted): unbiased iff a fitted blend matches the treated's loadings (\(\sum_j w_j\gamma_j=\gamma_1\), the treated inside the hull). +
    +
    +
    ▸ How to read the dials (red = error against the truth)
    +
      +
    • Loading spread → 0: markets react identically, trends become parallel, DiD's error collapses to noise. Treated-minus-control keeps its error: the level gap never dies.
      • +
    • Macro shock up, with spread > 0: before/after's error explodes, DiD's grows with the walk, and synthetic control stays pinned near the truth: it matches loadings instead of hoping they are equal.
      • +
    +
    +
    None of these numbers ships a warning label
    + Each error is a wrong claim about \(Y_{1t}(0)\), and a wrong claim never improves with more data: the cure is a better claim, not a bigger sample.
    +
    +
    + + +
    +
    Act II · The counterfactual problem
    +

    Interesting limits for the simpler estimators

    +
    Where before/after and treated/control estimators break.
    +
    +
    +
    +
    Before/after · one unit, across time
    + The treated metro's post-average minus its pre-average.
    +
      +
    • The level \(\alpha_1\) cancels (same market, subtracted from itself), but nothing subtracts the shared wave.
      • +
    • The full factor drift \(\Delta\bar f\) lands at the metro's own loadings \(\gamma_1\approx1\), not at a mismatch: the largest of the three biases, and it grows with the horizon through the trend term \(0.4\,t\).
      • +
    +
    +
    +
    Treated-vs-control · one period, across units
    + The treated metro minus the control average, both in the post window.
    +
      +
    • Time-differencing is gone, so the level gap \(\alpha_1-\bar\alpha_C\) survives, and market sizes differ a lot: this term dominates.
      • +
    • The loading gap now multiplies the factor level \(\bar f_{\text{post}}\), not its drift: this is why raw sales are never compared across cities.
      • +
    +
    +
    +
    ▸ The two bias decompositions, term by term
    +
    + \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] + the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. +
    +
    + \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] + no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. +
    +
    +
    +
    + + +
    +
    Act II · The counterfactual problem
    +

    Interesting limits for DiD

    +
    Where the DiD estimator breaks.
    +
    +
    The one question
    + DiD subtracts the control trend from the treated trend, so it only works if the two would have drifted together. When is that true, and what does it cost when it is not? Two dials answer it (loading spread \(s\), macro shock \(\sigma_\eta\)); the algebra is in the fold.
    +
    Synthetic control solves it
    + The one bias DiD cannot shed, \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\), comes entirely from its equal weights \(\bar\gamma_C\): it must hope the spread \(s\) is small. Synthetic control chooses the weights instead, so \(\sum_j w_j\gamma_j=\gamma_1\) and \(B\to0\) at any spread. The residual DiD lives with, the twin removes by construction.
    +
    ▸ The DiD bias, decomposed (and why more data cannot shrink it)
    +
    + \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] +
    +
      +
    • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
      • +
    • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
      • +
    +
    + \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] +
    +
      +
    • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
      • +
    • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
      • +
    +
    +
    ▸ The two limits, dial by dial: send a knob to zero and read what dies
    +
    +
    Limit 1 · loading spread \(s \to 0\): sufficient on its own
    +
      +
    • The prefactor \(s^2/3\) kills the whole bracket at once.
      • +
    • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
      • +
    • DiD then recovers \(\bar\tau\) up to noise.
      • +
    +
    Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
    +
      +
    • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
      • +
    • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
      • +
    • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
      • +
    +
    +
    +
    ▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
    + Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. +
    +
    +
    + + +
    +
    Act II · The counterfactual problem
    +

    What must be true for the gap to be causal

    +
    The four assumptions that must hold.
    +
    + + + + + + + + + + + + + + + + + + + + +
    AssumptionWhat it demandsThe checkVerdict
    ① Pre-fit qualityThe blend tracks the treated metro before launch: pre-RMSE below a third of the weekly lift we must detect.The gate: 3.2 vs a bar of 4.4.PASS
    ② No anticipationSales must not react before the campaign (no stockpiling, no leaked launch).Placebo-in-time: back-date the launch 10 weeks, the fake "effect" is ≈ €1.7k, inside noise.PASS
    ③ No spillover (SUTVA)The campaign must not touch donor markets (no travel, no online leakage): Colgate's second-launch leak, in space.No statistical check exists. A design duty: keep test and control regions apart.UNTESTABLE
    ④ Hull & stable loadingsThe treated metro is a feasible mix of donors, and exposures stay put through the window.Pre-fit gate + leave-one-out: drop any donor, the estimate stays in €239–276k.PASS
    +
    The referee principle
    + Three of the four are testable and the verdict is PASS. The one that is not, no-spillover, is exactly why geo-test design matters more than any statistic computed afterwards.
    +
    +
    + + +
    +
    Act III · Is it real, and how big?
    +

    €260k. Real, or a lucky metro? Build the null yourself

    +
    +
    +
    ✋ Poll
    +
    The usual t-test needs many treated units (its standard error is a spread across treated units); with one treated metro it is not weak, it is undefined. So build the test yourself: run the whole synthetic-control pipeline 29 more times, each time pretending one untreated donor was the treated market ("placebo"). That yields 30 estimated "effects", 29 of them in markets where no campaign ran. Where does the real treated metro's €260k rank among the 30?
    +
    + + + +
    + +
    A: rank 1 of 30. And the rank is the inference. If the campaign did nothing, the treated metro is just another market, exchangeable with its donors, so P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test, the standard error the t-test could not supply.
    +
    +
    +
    + + +
    +
    Act III · Is it real, and how big?
    +

    Placebo-in-space: measure the luck directly

    +
    Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
    +
    +
    +
    +
    +
    ▶ LIVE Refit the estimator on every donor as if it were treated
    +
    +
    The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
    +
    +
    +
    +
    The null hypothesis, stated
    + \(H_0\): the campaign did nothing, \(\tau_t=0\) for every \(t\ge T_0\). Then the treated metro is just one more untreated market, exchangeable with its 29 donors, so its gap is equally likely to hold any of the 30 ranks.
    +
    + \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] +
    +
    +
    +
    What the p-value means (and what it is not)
    + The probability, if \(H_0\) were true, of a gap as extreme as our €260k where nothing happened: a rank, not a bell-curve tail (no normality, independence, or asymptotics assumed). Nothing beat us, so \(p\) hits its floor \(1/(J+1)=1/30\), set by the donor count, not the weeks of data.
    +
    ▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
    +
      +
    • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
      • +
    • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
      • +
    +
    Conclusion: the €260k lift is real, at \(p\approx0.033\)
    + Our metro ranks 1 of 30, clear of the whole cloud of luck, so we reject \(H_0\): a real campaign effect, not a lucky draw, and the rank holds (p = 0.033 every time) when shaky placebos are dropped at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": that verdict waits for Act IV.
    +
    +
    + + +
    +
    Act III · Is it real, and how big?
    +

    From a test to an interval: inversion

    +
    We measured one number, €260k. Which true lifts could plausibly have produced it? We test every candidate against the placebos and keep the survivors, with no bell curve anywhere.
    +
    +
    +
    +
    +
    ▶ LIVE Drag \(H\), a guess at the truth. Green line = our data, fixed. Blue cloud = the guess, it slides.
    +
    +
    + + +
    +
    Axis: 20-week total gap (€000). A the estimator's error at truth 0 · B that same error slid to \(H\), with its middle 90% · C every \(H\) whose band still covers €260k, collected: the interval. Drag past an edge to reject.
    +
    +
    +
    +
    The whole idea
    + Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
    +
      +
    • ① Measure the method's error. The 29 donor markets ran no campaign, yet refitting the estimator on each still reports a small fake "effect". Together they make a cloud centred on zero, spread about ±€50k: how far off the method typically lands when the truth is really zero.
      • +
    • ② Guess a true lift \(H\). \(H\) is a candidate for the truth, not our estimate: the dial you turn. If the real lift were \(H\), the method would report \(H\) plus that same error cloud, the blue cloud in the figure slid onto \(H\).
      • +
    • ③ Keep or reject \(H\). Keep \(H\) if our €260k lands inside the middle 90% of its cloud; reject it if €260k falls out in a tail (too far from \(H\) to be believable).
      • +
    • ④ Sweep and collect. Repeat for every \(H\). The smallest and largest that survive are €195k and €335k, so the interval is [€195k, €335k], read straight off the placebos.
      • +
    +
    +
    +
    ▸ The inversion, in one line of algebra
    +
    + \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} + \qquad\Longleftrightarrow\qquad + \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] + \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. +
    +
    +
    ▸ Why this interval is the referee for the rest of the lecture
    + It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. +
    +
    +
    + + +
    +
    Act III · Is it real, and how big? Stress-test the estimate
    +

    Falsification 1: a launch that did not happen

    +
    The anticipation check.
    +
    +
    How it is computed
    + Refit using only weeks 0-29, pretend the launch happened at week 30, and read the "effect" in weeks 30-39, where the truth is zero.
    +
    ▶ LIVE The weekly gap (treated minus synthetic) around a back-dated launch
    +
    +
    + + +
    +
    Clean world: the gap hugs zero through the fake window (average ≈ €1.7k) and lifts only after the real launch. Tick the box for a leaked launch: the gap climbs before week 40 and the fake window lights up.
    +
    +
    What passing rules out
    + Stockpiling, pre-announcement, a leaked launch: anything that makes sales react before the campaign, which a synthetic control that "detects" an effect before treatment would reveal.
    +
    +
    + + +
    +
    Act III · Is it real, and how big? Stress-test the estimate
    +

    Falsification 2: the assumption no placebo can see

    +
    The no-spillover check: the disease you named for Colgate, now in space.
    +
    +
    The assumption (SUTVA)
    + Every method so far assumed the campaign moved only the treated metro. Advertising ignores borders, so we break that on purpose and measure the damage.
    +
    +
    ▶ LIVE Four worlds, rebuilt with a bigger and bigger leak
    +
    +
    Each point is a complete re-simulation and refit, not a correction applied afterwards.
    +
    +
    +
    The concrete story
    + Your metro's TV and outdoor ads reach the commuter belt. Those neighbouring markets are in your donor pool. So the campaign you are trying to measure quietly raises the very series you are using as "untreated" comparison.
    +
    + \(\varphi\) is the leaked share: to each of 5 neighbour donors we add a fraction \(\varphi\) of the treated metro's own weekly lift \(\Delta_t\), then refit. Lifting the donors lifts the synthetic, so the measured gap shrinks. +
    +
    +
    +
      +
    • The sweep: \(\varphi\in\{0,0.1,0.25,0.5\}\), from a clean world to half the lift leaking. Each point a full re-simulation and refit.
      • +
    • The result: the reported total slides from €260k (clean) to €236k, and the bias is always toward zero.
      • +
    • Dangerous because invisible: the leak starts at launch, so the pre-fit gate stays green and every placebo still passes, leaving no fingerprint to catch.
      • +
    • What drives the decision: because the bias can only attenuate, the number is a conservative floor: if it already clears the cost, spillover cannot overturn the call. The defence is design, keep donors off the treated metro's media footprint.
      • +
    +
    +
    + + +
    +
    Act III · Is it real, and how big?
    +

    Statistics done. Three numbers.

    +
    Questions ① and ② from the boardroom slide are now answered.
    +
    + + + + + + + +
    QuestionAnswerTool that answered it
    Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
    How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
    Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
    +
    Truth check (only a simulation allows it)
    + The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
    +
    The sentence that loses money
    + "The campaign drove €260k of sales for €75k, a 3.5× return. Roll it out." Every number is true and the conclusion still does not follow: question ③, is the profit worth the price and with what confidence, is not a statistics question.
    +
    +
    + + +
    +
    Act IV · The decision in euros
    +

    Now the money

    +
    +
    +
    ✋ Poll
    +
    Cost €{{nb07.cost}}k, incremental sales €{{nb07.cl_total}}k, gross margin {{nb07.margin}}%. What did the campaign earn?
    +
    + + + + +
    + +
    C: about €16k net. Profit = margin × lift − cost = 0.35 × 260 − 75 ≈ +€16k. Option A is the boardroom classic: it treats revenue as profit. The campaign returned €3.47 per €1 spent against a break-even of 2.86 at this margin: it paid, in expectation, with room to spare. Whether that expectation carries enough certainty to act on is the question the rest of this act prices.
    +
    +
    +
    + + +
    +
    Act IV · The decision in euros
    +

    The margin trap

    +
    Compare profit to cost, not revenue to cost.
    +
    +
    +
    +
    \[ \text{profit} \;=\; m\,\tau \;-\; c \]
    +
    • Revenue is not cash: the business keeps only the gross margin \(m=35\%\) of the lift \(\tau\) yet pays the cost \(c\) in full, so comparing \(\tau\) to \(c\) treats revenue as profit.
    +
    +
    +
    \[ \tau_{\mathrm{BE}} = \frac{c}{m} = \frac{75}{0.35} = \text{€}214\mathrm{k}, \qquad \mathrm{iROAS}_{\mathrm{BE}} = \frac{1}{m} = 2.86 \]
    +
    • Break-even is €214k of sales, not €75k: our €260k clears it by €46k of revenue (≈ €16k of profit), an iROAS of 3.47 against the \(1/m=2.86\) bar.
      • +
    • Hold onto €214k: it returns as the hinge of the €4M decision.
    +
    +
    +
    +
    ▶ LIVE Where the €260k goes: slide the margin
    +
    +
    + + +
    +
    "iROAS > 1" is a vanity bar; the real bar is 1/margin. Software (90% margin) breaks even at 1.1; grocery (20%) needs 5.0. Same campaign, opposite verdicts.
    +
    +
    +
    + + +
    +
    Act IV · The decision in euros
    +

    The same evidence at every price

    +
    When is the campaign profitable?
    +
    +
    +
    • Line 1, the point estimate: expected net \(m\hat\tau-c\) crosses zero at \(c=0.35\times260=\text{€}91\mathrm{k}\); below that price it looks profitable on average.
    +
    • Line 2, the uncertainty zone: the profit interval \(0.35\times[195,335]=[\text{€}68\mathrm{k},\text{€}117\mathrm{k}]\), the straddle zone the evidence cannot separate from break-even.
    +
    +
    +
    ▶ LIVE Slide the campaign price and watch the verdict move
    +
    +
    + + +
    +
    Between the lines live the dangerous campaigns: "looks profitable" (left of €91k) but not provably so. Ours at €75k is one of them, a comfortable mean the interval refuses to bless.
    +
    + + + +
    PriceBreak-even liftNet at \(\hat\tau\)VerdictThe lesson
    +
    +
    + + +
    +
    Act IV · The decision in euros, delivered
    +

    The pilot verdict, delivered to the board

    +
    This slide answers the €75k pilot. The €4M is a separate question, next.
    +
    +
    +

    Three questions about the €75k pilot, all answered by the permutation machinery:

    +
      +
    • ① Real? Yes. Rank 1 of 30, \(p=0.033\).
      • +
    • ② Big enough? Yes, on the point estimate. \(\hat\tau = \text{€}260\mathrm{k}\) against a break-even of €214k: €46k of revenue to spare, about €16k of net profit.
      • +
    • ③ Confident? No. The profit interval [€68k, €117k] still straddles the €75k price. The upside is real but a loss is still inside the evidence.
      • +
    +
    Pilot verdict, banked
    + At €75k, probably profitable but not a confident buy: renegotiate the price or buy certainty with a bigger test. This says nothing yet about the €4M, which is not the pilot multiplied.
    +
    +
    + + + + + + + + + + +
    Board numberValueSource
    Incremental sales, 20 wk€260ksynthetic-control gap
    Real, or luck?p = 0.033placebo permutation, rank 1/30
    90% interval on sales[€195k, €335k]test inversion
    iROAS vs break-even3.47 vs 2.86P&L: break-even = 1/margin
    Gross profit vs price€91k vs €75k0.35 × 260 − 75
    Pilot recommendationNOT A CONFIDENT BUYinterval straddles the price
    +
    The one question this memo cannot answer
    + Everything above is a yes/no about where an interval sits: it cannot say "78% chance it pays", name the highest price this evidence would still buy, or whether a follow-up study is worth funding. And it is only about €75k: the €4M rollout is the next two slides.
    +
    +
    +
    + + +
    +
    Act IV · The decision in euros, scaled up
    +

    The €4M is not provably profitable

    +
    Even at its best case, the profit interval straddles zero.
    +
    +
    +
    +
    +
    ▶ LIVE Rollout profit at €4M, carrying the 90% interval, as a share \(\delta\) of the pilot survives
    +
    +
    + + +
    +
    The point says +€860k at \(\delta=1\), but the 90% band [−€360k, +€2.25M] straddles zero even there. Drag \(\delta\) down (the real world, where each euro does less) and it only sinks; below \(\delta=0.64\) it wholly loses.
    +
    +
    +
    +
      +
    • ① The marketing number is a point: 53 pilot-budgets × €260k × 35% margin − €4M ≈ +€860k. Every step correct.
      • +
    • ② Carry the interval, and be generous: assume the pilot transports perfectly (\(\delta=1\)). Even then the 90% profit interval is [−€360k, +€2.25M]: it straddles zero.
      • +
    • ③ And \(\delta=1\) is a ceiling: €4M is €133k per market (1.78× the pilot, so each euro does less) and one metro is not the nation, so real \(\delta<1\) and the band only sinks.
      • +
    +
    The frequentist verdict, no probability needed
    + At its most generous the €4M is not provably profitable, and below \(\delta\approx0.64\) it provably loses. Do not commit €4M on this evidence.
    +
    +
    +
    ▸ Where 0.82 and 0.64 come from, and a defensible estimate of δ
    +
    +
    +
    The bar: required δ (arithmetic)
    +
    \[ \delta^{*} = \frac{c}{m\,\tau} = \frac{\tau_{\mathrm{BE}}}{\tau} = \frac{214}{\tau} \]reach \(R=C/c\), so the budget \(C\) cancels: the bar is set by cost, margin and lift only
    +
      +
    • Across the lift interval [€195k, €335k], required δ ∈ [0.64, 1.10], central 0.82 (82%).
      • +
    • Below 0.64 even the top of the interval loses: that is the "provably loses" edge.
      • +
    +
    +
    +
    The estimate: plausible δ (one assumption)
    +
    \[ \delta \;\approx\; 1.78^{\,\beta-1} \]per-euro lift at 1.78× intensity, up a concave response curve of elasticity \(\beta\)
    +
      +
    • \(\beta\) from the client's MMM, or "double the spend → ≈1.6× response" → \(\beta\approx0.68\) → δ ≈ 0.83.
      • +
    • \(\beta\) 0.6–0.8 gives [0.79, 0.89], central 0.84 (84%); being one metro only widens this; ours sits at the median (effect CV ≈ 0.14).
      • +
    +
    +
    +
    Why we measure, not estimate
    + The estimate 0.84 lands on the bar 0.82: profit at €4M runs −€1.11M / about €0 / +€1.57M across the pessimistic / central / optimistic corners. Defensible enough to refuse €4M, too close to the line to commit it: only a multi-market test separates the two.
    +
    +
    +
    + + +
    +
    Act IV · The decision in euros, scaled up
    +

    Measure before you commit

    +
    One treated market made this interval; only more will shrink it.
    +
    +
    +
    +
    The call
    + Even the best case straddled zero, so on this evidence do not commit €4M. The rollout is not dead: the evidence simply cannot size it.
    +
      +
    • Why analysis cannot rescue it: the interval is wide because there is one treated market. Re-running the model does not narrow it; only more treated markets do.
      • +
    • Step zero, first: check the drawer: prior pilots, the media-mix model, agency benchmarks may already narrow the range for the cost of one meeting.
      • +
    +
    +
    +
    The fix: measure the national effect directly
    + Run the campaign in 8 markets chosen to look like the nation, at full intensity (~€133k each, €1.07M). Because the cells span the country, the measured lift is the national lift, with an interval tight enough to clear or kill the €4M. HelloFresh's calibration loop, bought once.
    +
    The rule, written before the data lands
    + Release the remaining €2.9M only if the measured 90% profit interval clears zero. Written first, or the test is theatre.
    +
    The recommendation
    + ① Check the drawer. ② Do not commit €4M: the best case is not provably profitable. ③ Run the €1.07M, 8-market test at national intensity, randomised. ④ Separately: renegotiate the pilot toward €{{nb07.headroom}}k.
    +
    +
    +
    ▸ Sizing the test, and why 8 markets
    +
    \[ k \;\ge\; \Bigl[\tfrac{(z_{0.95}+z_{0.80})\,\mathrm{CV}}{\Delta}\Bigr]^{2} = \Bigl[\tfrac{2.49\times0.30}{0.36}\Bigr]^{2} \approx 4.3 \]
    +
      +
    • Sized from the decision: distinguish δ = 1 from δ = 0.64 (a 36% shortfall) against between-market spread; ≈ 5 markets on power, 8 to span the mix, and CV 0.4 pushes power itself to ≈ 8.
      • +
    • Constraints: randomise in matched pairs (that alone frees the p-value from the single-metro floor, 0.033 → ≈ 0.004); test at deployment intensity; keep controls off the media footprint.
      • +
    • Worth it: the Bayesian layer of this course prices this information at ≈ €320k, far above the test's incremental cost, since the €1.07M is real advertising working while it measures.
      • +
    +
    +
    +
    + + + +
    +
    Closing · Provenance
    +

    The tools were the product too

    +
    +
      +
    • CausalPy: {{labs.causalpy_methods}} in one open-source package: the method you just learned and its quasi-experimental family, industrialized by PyMC Labs. The IV estimator that closes this session joined the package later.
      • +
    • Its launch example: individual exposure to a TV campaign {{labs.causalpy_tv}}, yet its causal impact remains a core business need: the sentence this whole session opened with.
      • +
    • pymc-marketing: the MMM library behind Case 2's calibration story; one client's {{labs.bolt_pr}} came back as a pull request (Bolt).
      • +
    • Webinars and content: the consultancy's own webinar walks geo-experimentation, MMM, synthetic control, difference-in-differences, regression discontinuity and {{labs.webinar_agenda}}: this session's syllabus.
      • +
    +
    + CausalPy + PyMC-Marketing +
    +
    +
    + +
    +
    Closing
    +

    The pattern in every engagement

    +
    +
      +
    • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
      • +
    • An experiment anchors every observational model: calibration is the product, not a luxury.
      • +
    • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
      • +
    + + + + + + +
    Agent, on adversarial MMM dataResult
    Vanilla coding agentFit a model, recommended budget reallocations. {{labs.dl_vanilla}}
    PyMC Labs' Decision Lab{{labs.dl_explored}} Returned: "{{labs.dl_verdict}}"
    +
    Even the machines know the punchline
    + The honest system's best answer was Part 2's closing advice: run the experiment.
    +
    +
    + +
    +
    Closing
    +

    One breath

    +
    +
    The pattern to take home
    + The toolkit a Bayesian consultancy sells: counterfactuals, calibrated by experiments, priced as probabilities.
    +
      +
    • Read the cases: pymc-labs.com/blog-posts: every number in this deck is pinned to a post, listed on the next slide.
      • +
    • Say hello: both authors consult for PyMC Labs; the notebooks behind this session are the course repository.
      • +
    +
    +
    + + +
    +
    Backup
    + Backup · Sources +

    Every number, pinned

    +
    Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
    +
    + + + + + +
    SourceFacts pinned
    +
    +
    + +
    + + + +
    + Causal Marketing · SDA Bocconi · Causal Inference in the Wild + + + + 1 / 1 +
    +
    +

    Contents

      + + + + diff --git a/causal-marketing-pymc/apps/verify_labs_deck.py b/causal-marketing-pymc/apps/verify_labs_deck.py index 1b0c0ff..82a5333 100644 --- a/causal-marketing-pymc/apps/verify_labs_deck.py +++ b/causal-marketing-pymc/apps/verify_labs_deck.py @@ -41,8 +41,8 @@ CHAR_BUDGET_HARD = True -def load_deck(): - html = DECK.read_text() +def load_deck(deck: Path = DECK): + html = deck.read_text() i = html.index("/*__DATA__*/") j = html.index("{", i) data, _ = json.JSONDecoder().raw_decode(html, j) @@ -83,8 +83,8 @@ def norm(s): return re.sub(r"[\s  ]+", " ", s) -def main() -> int: - html, data, slides, slides_visible = load_deck() +def main(deck: Path = DECK) -> int: + html, data, slides, slides_visible = load_deck(deck) pins = json.loads(DATAFILE.read_text()) env = make_env(data) claims = yaml.safe_load(CLAIMS.read_text()) @@ -166,4 +166,9 @@ def report(claim_id, ok, msg): if __name__ == "__main__": - sys.exit(main()) + import argparse + + ap = argparse.ArgumentParser(description="Verify a labs-claims deck against apps/labs_claims.yaml.") + ap.add_argument("--deck", type=Path, default=DECK, help="deck to verify (default: labs_slides.html)") + a = ap.parse_args() + sys.exit(main(a.deck.resolve())) diff --git a/causal-marketing-pymc/apps/verify_unified_deck.py b/causal-marketing-pymc/apps/verify_unified_deck.py new file mode 100644 index 0000000..3e4e4c0 --- /dev/null +++ b/causal-marketing-pymc/apps/verify_unified_deck.py @@ -0,0 +1,36 @@ +"""Verify the unified session deck (apps/unified_slides.html) end to end. + +Two registries, one deck: + - apps/geo_claims.yaml via verify_geo_deck (--allow-missing-slides semantics: claims + anchored to full-deck slides absent from the short arc are skipped, exactly as for + apps/geo_lift_sh.html); + - apps/labs_claims.yaml via verify_labs_deck (also carries the deck-wide sweeps: token + residue, ZERO em-dashes outside MathJax, no raw currency inside math, per-slide + visible-length budget, pin hygiene). + +Run: .venv/bin/python apps/verify_unified_deck.py (exit 1 on any FAIL) +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import verify_geo_deck as vg +import verify_labs_deck as vl + +HERE = Path(__file__).resolve().parent +DECK = HERE / "unified_slides.html" + + +def main() -> int: + if not DECK.exists(): + sys.exit(f"FAIL: {DECK} missing — run apps/build_unified_slides.py first.") + print("== geo claims (short-arc subset) " + "=" * 46) + rc_geo = vg.main(deck=DECK, allow_missing_slides=True) + print("== labs claims + deck-wide sweeps " + "=" * 45) + rc_labs = vl.main(deck=DECK) + return 1 if (rc_geo or rc_labs) else 0 + + +if __name__ == "__main__": + sys.exit(main()) From c88205b5e0ae5b7aedae2c6a428f6bde16c79613 Mon Sep 17 00:00:00 2001 From: Francesco Muia Date: Tue, 21 Jul 2026 16:25:43 +0200 Subject: [PATCH 2/3] Last changes --- .../apps/build_geo_slides.py | 55 + .../apps/build_iv_slides_sh.py | 19 + .../apps/build_unified_slides.py | 34 +- causal-marketing-pymc/apps/geo_lift_sh.html | 30 +- .../apps/geo_lift_slides.html | 30 +- .../apps/geo_slides_sh_src.html | 28 +- .../apps/geo_slides_src.html | 28 +- .../apps/iv_slides_REVISION_PLAN.md | 125 + causal-marketing-pymc/apps/iv_slides_sh.html | 1841 ++++++++++++++ .../apps/iv_slides_sh_src.html | 1841 ++++++++++++++ .../apps/unified_slides.html | 2139 +++++++++++++++-- .../apps/unified_slides_src.html | 2137 ++++++++++++++-- 12 files changed, 7767 insertions(+), 540 deletions(-) create mode 100644 causal-marketing-pymc/apps/build_iv_slides_sh.py create mode 100644 causal-marketing-pymc/apps/iv_slides_sh.html create mode 100644 causal-marketing-pymc/apps/iv_slides_sh_src.html diff --git a/causal-marketing-pymc/apps/build_geo_slides.py b/causal-marketing-pymc/apps/build_geo_slides.py index e7c6f02..07f9305 100644 --- a/causal-marketing-pymc/apps/build_geo_slides.py +++ b/causal-marketing-pymc/apps/build_geo_slides.py @@ -126,6 +126,54 @@ def naive_grid() -> dict: "diff-in-differences", "synthetic control"]} +def dgp_parts(labels_top: list, treated: list, donors_top: list) -> dict: + """Bake the seed-5 STANDARDIZED draws behind cmp.dgp.geo_panel so the DGP-sandbox + slide can re-assemble the SAME world at any dial setting. + + geo_panel is exactly linear in every dial the slide exposes — normal(0, sd) = sd*z, + uniform(1-s, 1+s) = 1 + s*u, and the lift enters as lift*(level + g.F) — so shipping + the unit variates (z, u, e) plus the levels lets the browser rebuild + Y_jt = a_j + (1+s*u_j).F_t(sigma_eta) + sigma_eps*e_jt for arbitrary dials. At the case + dials that reconstruction IS the seed-5 panel (asserted below), so the picture deforms + continuously from the case instead of jumping to a different-seed toy world. + Only the markets the deck draws are shipped: treated first, then the top donors in + the bundle's own donor_labels_top order. + """ + import numpy as np + from cmp import dgp + + n_weeks, n_dmas, launch = 60, 30, 40 + # replicate geo_panel(seed=5)'s draw stream call-for-call, recording unit variates + rng = np.random.default_rng(5) + z = rng.normal(0, 1.2, n_weeks) / 1.2 # unit macro increments + levels = rng.uniform(80, 140, n_dmas) + u = (rng.uniform(0.6, 1.4, (n_dmas, 3)) - 1.0) / 0.4 # unit loading deviations + e = rng.normal(0, 3, (n_dmas, n_weeks)) / 3.0 # unit noise + zc = np.cumsum(z) + + # the reconstruction at the case dials must equal geo_panel(seed=5) to float precision + t = np.arange(n_weeks) + trend = 0.4 * t + season = 8 * np.sin(2 * np.pi * t / 26) + 4 * np.sin(2 * np.pi * t / 13) + F = np.column_stack([trend, season, 1.2 * zc]) + loads = 1.0 + 0.4 * u + sales = levels[:, None] + loads @ F.T + 3.0 * e + eff = 0.12 * (levels[0] + loads[0] @ F.T) * (t >= launch) + df, eff_ref, launch_ref, lab = dgp.geo_panel(seed=5) + assert launch_ref == launch and np.allclose(eff, eff_ref, atol=1e-9) + assert np.allclose(sales[0] + eff, df[lab].values, atol=1e-9) + assert np.allclose(sales[1:], df.values.T[1:], atol=1e-9) + + # and match the bundle's (rounded-to-0.1) shipped series within rounding + idx = [0] + [int(l.split("_")[1]) for l in labels_top] + assert np.abs(sales[0] + eff - np.asarray(treated)).max() < 0.051 + for k, l_idx in enumerate(idx[1:]): + assert np.abs(sales[l_idx] - np.asarray(donors_top[k])).max() < 0.051 + + r = lambda a: np.round(np.asarray(a), 5).tolist() + return {"levels": r(levels[idx]), "u": r(u[idx]), "e": r(e[idx]), "zc": r(zc)} + + def build_bundle() -> dict: """Assemble the full DATA bundle (nb07 lecture bundle + shard scalars + aliases + real-data bundle + extras + the naive grid). Shared with build_unified_slides.py.""" @@ -180,6 +228,13 @@ def build_bundle() -> dict: bundle_meta["naive_grid"] = naive_grid() print(f"naive grid: {len(bundle_meta['naive_grid']['s'])}×" f"{len(bundle_meta['naive_grid']['sd'])} cells from cmp.dgp.geo_panel") + + # the DGP-sandbox slide: the seed-5 unit draws, so the live figure re-assembles the + # SAME world at off-case dials instead of re-simulating a different-seed toy world. + bundle_meta["dgp_parts"] = dgp_parts(bundle_meta["donor_labels_top"], + bundle_meta["treated"], + bundle_meta["donor_series_top"]) + print(f"dgp parts: {len(bundle_meta['dgp_parts']['levels'])} markets' seed-5 unit draws") return bundle_meta diff --git a/causal-marketing-pymc/apps/build_iv_slides_sh.py b/causal-marketing-pymc/apps/build_iv_slides_sh.py new file mode 100644 index 0000000..4b36c58 --- /dev/null +++ b/causal-marketing-pymc/apps/build_iv_slides_sh.py @@ -0,0 +1,19 @@ +"""Generate apps/iv_slides_sh.html, the management-first rework of the Ch. 13 IV deck. + +Same pipeline as build_iv_slides.py (tokens from the executed shards, DATA bundle at +/*__DATA__*/, vendored MathJax at /*__MATHJAX__*/), different template and output: + + Template: apps/iv_slides_sh_src.html (editable source) + Output: apps/iv_slides_sh.html (generated, self-contained) + +Build: .venv/bin/python apps/build_iv_slides_sh.py +""" +from __future__ import annotations + +import build_iv_slides as base + +base.SRC = base.HERE / "iv_slides_sh_src.html" +base.OUT = base.HERE / "iv_slides_sh.html" + +if __name__ == "__main__": + base.main() diff --git a/causal-marketing-pymc/apps/build_unified_slides.py b/causal-marketing-pymc/apps/build_unified_slides.py index d2c7789..4a4acb8 100644 --- a/causal-marketing-pymc/apps/build_unified_slides.py +++ b/causal-marketing-pymc/apps/build_unified_slides.py @@ -1,19 +1,21 @@ """Generate the UNIFIED SDA session deck: apps/unified_slides.html. Part 1 ("Causal Inference in the Wild": real PyMC Labs engagements, the business hook) -spliced in FRONT of the short classical geo deck (synthetic control, Acts I-IV), closing -with the labs synthesis slides. One file, one chrome (the geo deck's, a strict superset), -one DATA bundle. The narrative this deck implements: apps/geo_lift_lecture_emphasis.md; +spliced in FRONT of the short classical geo deck (synthetic control, Acts I-IV), then +Part 3 (the short instrumental-variables deck) after the geo Act IV closer, closing with +the labs synthesis slides. One file, one chrome (the geo deck's, a strict superset), one +DATA bundle. The narrative this deck implements: apps/geo_lift_lecture_emphasis.md; the spoken script: apps/geo_lift_lecture_speech.md. Inputs, none hand-typed: - tokens : {{nb07*.*}} from the executed notebook shards (build_geo_slides.load_tokens) - PLUS {{labs.*}} from apps/labs_deck_data.json (blog-pinned facts with source - URLs; build_labs_slides.load_pins). The two prefixes are disjoint (asserted). + tokens : {{nb07*.*}} from the executed geo shards (build_geo_slides.load_tokens), + {{labs.*}} from apps/labs_deck_data.json (blog-pinned facts with source URLs; + build_labs_slides.load_pins), and {{nb11*.*}} from the executed IV shards + (build_iv_slides.load_tokens). The three prefixes are disjoint (asserted). DATA : the geo bundle (build_geo_slides.build_bundle) merged with the labs chart keys - (the deterministic counterfactual schematic + the pinned ROAS pair). The key - sets are disjoint (asserted), so the labs figure code reads the same DATA - global as the geo figures. + (deterministic counterfactual schematic + pinned ROAS pair; disjoint, asserted), + plus the IV bundle nested under DATA.iv (its dgp/scalars keys collide with geo's, + so the spliced IV figure code was rebased to read DATA.iv.* by the splicer). SOURCES: the labs Sources backup table rows at . MATHJAX: the vendored tex-svg build, inlined once at /*__MATHJAX__*/. @@ -29,6 +31,7 @@ from pathlib import Path import build_geo_slides as bg +import build_iv_slides as bi import build_labs_slides as bl HERE = Path(__file__).resolve().parent @@ -51,6 +54,13 @@ def main() -> None: sys.exit("FAIL: token collision between shards and labs pins: " + ", ".join(sorted(dup))) tokens |= labs_tokens + # Part 3 (IV) prose tokens: {{nb11.*}} / {{nb11b.*}} from the executed IV shards + iv_tokens = bi.load_tokens() + dup_iv = set(tokens) & set(iv_tokens) + if dup_iv: + sys.exit("FAIL: token collision between existing tokens and IV shards: " + ", ".join(sorted(dup_iv))) + tokens |= iv_tokens + used, missing = set(), [] def sub_token(m: re.Match) -> str: @@ -73,6 +83,12 @@ def sub_token(m: re.Match) -> str: if overlap: sys.exit("FAIL: DATA key overlap between geo and labs bundles: " + ", ".join(sorted(overlap))) bundle.update(labs_bundle) + + # Part 3 (IV) chart data: nested under a single key (its dgp/scalars/... collide with geo's, + # so the IV figure code was rebased to read DATA.iv.* by the splicer). + if "iv" in bundle: + sys.exit("FAIL: DATA key 'iv' already present; cannot nest the IV bundle.") + bundle["iv"] = bi.bake_bundle() html = bg.inject_data(html, bundle) html = bg.inject_nums(html, tokens) diff --git a/causal-marketing-pymc/apps/geo_lift_sh.html b/causal-marketing-pymc/apps/geo_lift_sh.html index 717ffa8..d0d46c3 100644 --- a/causal-marketing-pymc/apps/geo_lift_sh.html +++ b/causal-marketing-pymc/apps/geo_lift_sh.html @@ -1210,7 +1210,7 @@

      Measure before you commit

      + + + +
      + + +
      +
      +
      +
      Causal Inference & XAI for Business · SDA Bocconi
      +

      Instrumental Variables

      +
      What is one ad exposure worth, when the platform already picked who sees the ads?
      +
      A guest lecture for Prof. Michele Russo's course
      +
      +
      +
      Francesco Muia
      +
      PhD in Theoretical Physics, EMBA.
      Consultant for PyMC Labs and Brown University.
      +
      francesco.muia@pymc-labs.com
      +
      +
      +
      Alexander Fengler
      +
      PhD in Computational Cognitive Science.
      Postdoc at Brown University, consultant for PyMC Labs.
      +
      alexander.fengler@pymc-labs.com
      +
      +
      +
      + PyMC + PyMC Labs +
      +
      +
      + +
      Where the lecture lands: the dashboard's tight interval misses the true effect of an ad exposure, while the interval built from a lottery covers it, and still clears the price.
      +
      +
      +
      + + +
      +
      Part 1 · The problem
      +

      The case

      +
      One retailer, one ad platform, one decision to make.
      +
      +
      +
      The product
      An online retailer pays an ad platform to show its display ads. One exposure = one user actually saw an ad.
      +
      The price
      Each exposure costs €10, billed by the platform.
      +
      The assignment
      The platform decides, in a real-time auction, which users see the ads.
      +
      The dashboard
      Users who saw an ad brought in €23.7 more than users who did not.
      +
      +
      The platform's pitch
      + "An exposure is worth €23.7 and costs €10. Raise the budget."
      +
      The decision on the table
      + Keep paying €10 per exposure, pay more, or stop. One number settles it: how many euros of extra sales one exposure brings in. All we have is the platform's logs: who saw an ad, and what every user spent.
      +

      The question for the poll: is the dashboard's €23.7 that number?

      +
      +
      + +
      +
      Part 1 · The problem
      +

      Poll · the pitch

      +
      +
      +
      ✋ Poll
      +
      The dashboard reports that users who saw an ad brought in €23.7 more than users who did not, and an exposure costs €10. The platform concludes: "raise the budget". What do you think of that claim?
      +
      + + + + +
      + +
      C. The arithmetic is correct and, at this sample size, comfortably significant, so A and B both miss the point. The platform chose who saw the ads, and it is paid to pick users who already look ready to buy. If the two groups differed before any ad ran, the €23.7 mixes the ad's effect with that pre-existing difference, in unknown proportions. D is backwards: averaging over thousands of users is exactly how noise is handled.
      +
      +
      +
      + +
      +
      Part 1 · The problem · the variables, drawn
      +

      The variables and the confounder

      +
      The three quantities of the case, defined through their causal diagram.
      +
      +
      +

      The platform's logs cover 3,000 customers, indexed by \(i\). Three quantities per customer:

      +
        +
      • \(Y_i\) · sales: contribution euros earned from customer \(i\) in the window (sales net of product cost); "sales" for short. Noisy: the standard deviation across customers is €17.1.
        • +
      • \(X_i\) · exposure: 1 if customer \(i\) saw the ad, 0 if not. Assigned by the platform's auction, not by us.
        • +
      • \(U_i\) · intent: how ready \(i\) was to buy before any ad. Real, and it drives buying, but it is a feeling in a shopper's head: in no log file, never observed.
        • +
      +
      Definition · confounder
      + A confounder is a variable that influences both the treatment and the outcome. Here it is intent \(U\): it raises the chance the platform shows the ad (the targeting), and it raises sales on its own (ready buyers buy, ad or not).
      +
      Definition · backdoor path
      + In the diagram, \(X \leftarrow U \rightarrow Y\) is a backdoor path: a second route connecting exposure and sales that runs behind the treatment. Association flows along it even if the ad does nothing.
      +
      +
      +
      the situation, as a diagram
      +
      +
      \(U\) points at both \(X\) and \(Y\): the fork every claim in this lecture has to get past.
      +
      +
      +
      + +
      +
      Part 1 · The problem
      +

      The naive comparison

      +
      The dashboard's €23.7, written down as a formula.
      +
      +
      +

      Write \(Y_i(1)\) for what customer \(i\) spends with the ad and \(Y_i(0)\) for what the same customer spends without it. Each customer shows us exactly one of the two:

      + + + + + + +
      group\(Y_i(1)\) · with ad\(Y_i(0)\) · without ad
      exposed (\(X_i=1\))observedN/A
      unexposed (\(X_i=0\))N/Aobserved
      +
      \[ \widehat{\Delta}_{\text{naive}} \;=\; \bar{Y}_{\text{exposed}} \;-\; \bar{Y}_{\text{unexposed}} \;=\; \text{€}23.7 \]
      +
        +
      • What it averages: the two observed cells, one from each row. The arithmetic is right.
        • +
      • What it pretends: that the unexposed row's \(Y(0)\) is a fair stand-in for the exposed row's missing \(Y(0)\). Fair only if the two groups are comparable.
        • +
      • The auction works against us: the platform is paid to find likely buyers, so the exposed would have spent more with no ad at all.
        • +
      +
      +
      +
      +
      ▶ LIVE exposed users were already different
      +
      +
      + + +
      +
      The gap compares two groups the auction built to differ. The hidden column shows by how much.
      +
      +
      +
      +
      + +
      +
      Part 1 · The problem
      +

      Poll · read the gap

      +
      +
      +
      ✋ Poll
      +
      The naive comparison gives \(\widehat{\Delta}_{\text{naive}} = \) €23.7. Which reading of that number is defensible?
      +
      + + + + +
      + +
      B. The arithmetic is sound, so C is out. A and D each pick one extreme of the same unknown split: A books all of it to the ad, D books all of it to targeting, and the dashboard alone cannot justify either. The honest statement is the decomposition: €23.7 = the ad's effect + selection bias, with nothing on the dashboard saying how it splits. Measuring that split is the technical problem this lecture solves.
      +
      +
      +
      + +
      +
      Part 1 · The problem · the poll's answer, formalized
      +

      Selection bias

      +
      The decomposition behind the poll's answer.
      +
      +
      +

      Split the naive gap into the two things it adds together:

      +
      \[ \Delta_{\text{naive}} \;=\; \underbrace{\mathbb{E}[Y(1)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}1]}_{\text{effect of the ad on the exposed}} \;+\; \underbrace{\mathbb{E}[Y(0)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}0]}_{\text{selection bias}} \]
      +
        +
      • Read the first term: the exposed users with the ad versus the same users without it. This is what we want to buy, and its second entry is the table's N/A cell.
        • +
      • Read the second term: the gap that would exist with no ads at all, because the picked users differ from the skipped ones.
        • +
      • Here it is positive: the auction picks ready buyers, so the exposed would have out-spent the unexposed anyway.
        • +
      • The dashboard reports only the total: €23.7. Nothing on it says how the total splits.
        • +
      +
      Definition · selection bias
      + When treated and untreated groups differed before the treatment, the difference of their averages includes that pre-existing gap. The pre-existing part is selection bias.
      +
      +
      +
      +
      the dashboard's number is a sum, and the split is invisible
      +
      +
      The dashboard hands us the full bar. Where the boundary sits, nobody can see from the dashboard alone. Finding it is the rest of the lecture.
      +
      +
      +
      +
      + +
      +
      Part 1 · The problem · a world we can grade
      +

      The simulated world

      +
      A world where the true effect is planted, so every method can be graded.
      +
      +
      +
      +
      ▶ LIVE the world the equations generate, as data
      +
      +
      + + +
      +
      Sales distributions for exposed and unexposed users, drawn fresh from the equations. Push \(\kappa\) up, so that intent leaks more strongly into sales, and the two groups drift apart with the planted effect unchanged: the dashboard's gap grows while the truth stands still.
      +
      +
      +
      +
      Why simulate at all
      + Three equations generate the dashboard you saw. We planted an ad effect of \(\beta =\) €15, so every method in this lecture can be graded against a known answer.
      +
      \[ U \sim \mathcal{N}(0,1) \]
      +
      \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \lambda\,U\right) \]
      +
      \[ Y \;=\; \mu + \beta\,X \;+\; \kappa\,U \;+\; \varepsilon \] + \(\mu\): baseline sales  ·  \(\alpha_0\): baseline exposure odds  ·  \(\varepsilon\): noise
      +
        +
      • Interpretation of line 2: who sees an ad rises with intent (\(\lambda U\)): the targeting. \(\sigma\) is the logistic function, turning any score into a probability.
        • +
      • Interpretation of line 3: sales are the ad effect \(\beta\) plus intent leaking straight into spending (\(\kappa U\)).
        • +
      • The math is here for the record. The point is on the left: one hidden cause, \(U\), sits in both equations.
        • +
      +
      +
      +
      + +
      +
      Part 1 · The problem · the disease, stated exactly
      +

      Endogeneity

      +
      The whole problem, stated as one inequality, then shown as a picture.
      +
      +
      +

      Collect everything the sales equation leaves out into a single error term \(v\):

      +
      \[ Y \;=\; \mu + \beta\,X + v, \qquad v \;=\; \kappa\,U + \varepsilon \]
      +
      \[ \operatorname{Cov}(X,\, v) \;>\; 0 \]
      +
        +
      • In words: \(v\) is "every other reason this customer spent money", and exposure is not independent of those reasons, because the auction targets exactly the customers with reasons to spend.
        • +
      • The picture on the right is the inequality: walk up the intent scale and both bars rise together. That joint rise is \(\operatorname{Cov}(X,v) > 0\).
        • +
      +
      Definition · endogeneity
      + A treatment correlated with the other drivers of the outcome is endogenous. Ours is, and everything else in this lecture is a response to that fact.
      +
      The sharp edge
      + The variable driving the correlation was never written to disk.
      +
      +
      +
      +
      one hidden cause pushes both
      +
      +
      Customers grouped by buying intent (which nobody observes). Higher-intent customers see more ads and spend more, ad or no ad. The formula on the left is this picture, in symbols.
      +
      +
      +
      +
      + +
      +
      Part 1 · The problem · the bias, computed
      +

      The size of the bias

      +
      The dashboard's error is not vague. It has a formula, and the formula has three lessons.
      +
      +
      +
      \[ \widehat{\Delta}_{\text{naive}} \;\longrightarrow\; \beta \;+\; \underbrace{\kappa\,\frac{\operatorname{Cov}(X, U)}{\operatorname{Var}(X)}}_{\text{selection bias}} \]
      +
        +
      • Predicted vs delivered: the formula predicts a naive gap near €23.5. The data delivered €23.7, against a truth of €15.
        • +
      +
      What the formula teaches
      + 1. The error is systematic, not bad luck: both factors are positive, so the dashboard can only overstate.
      + 2. It grows with the targeting (\(\lambda\), how strongly intent drives exposure) and with intent's pull on sales (\(\kappa\)): the better the platform targets, the worse the dashboard lies.
      + 3. Sample size is absent: more rows shrink noise, never bias. The dashboard's band, €22.9 to €24.5, is the tightest in this lecture and misses €15 completely.
      +
      What the dashboard measures
      + Not the ad's power to create buyers but the platform's skill at finding them, reported as one number.
      +
      +
      +
      +
      ▶ LIVE the same bar, at different targeting strengths
      +
      +
      + + +
      +
      The dashboard's bar, split by the formula: the true effect (blue) never moves, while the selection bias (orange) grows as intent leaks more strongly into sales. In this simulated world we can draw the boundary, because we planted it.
      +
      +
      +
      +
      + +
      +
      Part 1 · The problem · closing the obvious exit
      +

      The limits of adjustment

      +
      The standard fix works, when you have the column it needs. We do not.
      +
      +
      +
      The standard fix: close the backdoor
      + Compare users at the same level of the confounder. Within a group of equal-intent users, intent no longer separates exposed from unexposed, and any remaining gap belongs to the ad. Controls in a regression, matching, and propensity scores are all versions of this one move.
      +
      +
      ▶ LIVE the fix works, if the column exists
      +
      +
      + + +
      +
      Tick the box: comparing exposed to unexposed within equal-intent groups recovers the planted truth to within half a euro. The method is fine. Untick it: in the real logs the column does not exist, and no amount of modelling recreates it.
      +
      +
      Why the engine stalls here
      + You cannot adjust for what you did not record.
      +
      +
      +

      Two customers, identical on every recorded feature \(W\):

      + + + + + + +
      customerpages, device, history \(W\)intent \(U\)sees the ad?
      Annaidenticalhigh, unrecordedpicked
      Beaidenticallow, unrecordedskipped
      +
      what conditioning blocks, and what it cannot
      +
      +
      Holding \(W\) fixed (the box) blocks the backdoors that run through it. The arrows themselves never move: conditioning blocks paths, it does not delete causes. The path through \(U\) stays open, because \(U\) is in no file to hold fixed.
      +
      +
      +
      + +
      +
      Part 1 · The problem
      +

      Poll · what would fix it?

      +
      +
      +
      ✋ Poll
      +
      Adjustment failed because intent was never recorded. Your team can request one addition to the data. Which one would actually let you measure what an exposure causes?
      +
      + + + + +
      + +
      C. A tells you the wrong number more precisely: bias is not noise, and the formula for it does not contain the sample size. B and D are more controls, and the auction reacts to live signals that no exported feature set fully contains, so the backdoor stays open. C is different in kind: it changes who assigns the treatment, and the assignment was the disease all along. The rest of the lecture is about getting C, or the closest thing to it the system already contains.
      +
      +
      +
      + + +
      +
      Part 2 · The idea · what we wish we could do
      +

      The ideal experiment

      +
      What randomizing exposure would buy, and why we cannot do it.
      +
      +
      +
      randomization cuts the arrow that causes the trouble
      +
      +
      Left: today's world, where intent drives both exposure and sales. Right: the experiment we wish we could run, where a randomizer assigns exposure and intent's arrow into it is removed, not merely blocked: exposure no longer listens to intent at all. With that arrow gone, the naive comparison becomes the right comparison.
      +
      \[ X \text{ randomized} \;\;\Rightarrow\;\; \operatorname{Cov}(X,\, v) = 0 \;\;\Rightarrow\;\; \widehat{\Delta}_{\text{naive}} \longrightarrow \beta \]
      +
      +
      +
        +
      • The wish: option C from the poll: show the ad to a random half ourselves.
        • +
      • The wall: the platform's auction decides exposure in milliseconds, using exactly the intent signals that cause the problem. It will not hand us the controls.
        • +
      +
      The idea that saves us
      + If we cannot inject randomness into exposure, find the randomness the system already contains and use only that part.
      +
        +
      • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
        • +
      +
      The plan for Part 2
      + 1. Give that random lever a name: an instrument.
      + 2. State the conditions it must satisfy.
      + 3. Check which of them the data can verify.
      +
      +
      +
      + +
      +
      Part 2 · The idea · the randomness we ran
      +

      The lottery

      +
      The randomness our system already contains, because we put it there.
      +
      +
      +
        +
      • What we ran: before the campaign, a serving-priority lottery. Our own random number generator picked half the users and gave them a small priority boost in the platform's ad auction.
        • +
      • What it does: the lottery shows nobody an ad. It only raises the odds of seeing one.
        • +
      • A new column joins the data:
        • +
      + + + + + +
      quantitymeaningtypewho assigns itobserved?
      \(Z_i\) · the lottery1 if the lottery boosted \(i\)'s priority in the ad auctionbinarya random number generator we controlobserved
      +
      \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \gamma\,Z + \lambda\,U\right) \] + the simulated world ran the lottery too: the exposure equation gains \(\gamma Z\), the lottery's push on the odds of seeing an ad
      +
        +
      • Why it matters: \(Z\) moves exposure, and nothing else about the customer. It is the one source of variation in \(X\) that the targeting cannot touch.
        • +
      +
      +
      +
      the fork, and the way around it
      +
      +
      Top: the confounded fork. Bottom: the lottery \(Z\) pushes on \(X\) from outside. No arrow from \(U\) into \(Z\) (it is a random draw), and no arrow from \(Z\) straight to \(Y\) (a queue bump shows the user nothing).
      +
      +
      +
      + +
      +
      Part 2 · The idea · the central definition
      +

      The instrument

      +
      What we just built has a name.
      +
      +
      Definition · instrument
      + An instrument \(Z\) moves the treatment, is as good as random with respect to the hidden confounder, and affects the outcome only through the treatment. Four conditions make that precise, and they grade our lottery as follows.
      + + + + + + + + + + + + + +
      conditionthe claimcan it be checked?what it buys
      1 · Relevancethe lottery actually moves exposure \(X\)testable measured by the first stagea push that really happened
      2 · Exogeneitythe lottery is blind to intent \(U\)by design if the draw is genuinewith 1: a clean experiment on the lottery
      3 · Exclusionthe lottery touches sales only through exposureuntestable argued, never checkedturns the lottery's effect into the ad's
      4 · Monotonicitythe lottery never blocks an exposure that would have happened without ituntestable argued from the mechanismsays whose effect we measured
      +
      An instrument is the opposite of a control
      + You do not hold \(Z\) fixed. You use the variation it created.
      +

      The genius is never the estimator, it is spotting the randomness your system already contains: a rollout order, a capacity limit, an upstream A/B test, a rounding rule.

      +
      +
      + +
      +
      Part 2 · The idea · condition 1, measured
      +

      Condition 1 · relevance

      +
      The first stage: did the lottery actually move exposure?
      +
      +
      +

      Relevance is a claim about the data, so the data can answer it.

      +
      Definition · first stage
      + The first stage is the effect of the instrument on the treatment: how much exposure the lottery itself created. It is the coefficient \(\pi\) in the regression below.
      +
      +
      \[ X \;=\; b_0 + \pi\,Z + u \] + \(\pi\): the first stage  ·  \(b_0\): exposure rate among lottery losers  ·  \(u\): every other driver of exposure, intent included
      +
      +
      +
      \[ \pi \;=\; \Pr(X{=}1 \mid Z{=}1) \;-\; \Pr(X{=}1 \mid Z{=}0) \;=\; 0.2106 \] + the measured first stage: what the lottery did to exposure
      +
        +
      • Interpretation of \(\pi\): winning the lottery raises a user's chance of seeing an ad by +21 percentage points, from 56% to 77%.
        • +
      • It is causal, full stop: \(Z\) is random, so nothing hidden can explain the step.
        • +
      +
      +
      +
      exposure rate, by the lottery
      +
      +
      The step between the bars is the first stage \(\pi\): the slice of exposure that the lottery, not targeting, created.
      +
      The gate: is the push strong enough?
      + Like every estimate in this lecture, \(\pi\) comes with a significance test: the first-stage \(F\). Trust the coming division only when \(F > 10\), well past bare significance, because a \(\pi\) that is merely nonzero still leaves the division unstable. Here \(F = 156\): the push is unmistakably real. We quote such tests, not derive them.
      +
      +
      +
      + +
      +
      Part 2 · The idea · conditions 2 and 3
      +

      Conditions 2 and 3 · exogeneity and exclusion

      +
      The lottery must be blind to intent, and silent about sales.
      +
      +
      +
      Blind · exogeneity
      + The lottery must not favour high-intent users. Ours is a genuine randomizer, so this holds by design.
      +
      Silent · exclusion
      + The lottery must not touch sales through any channel except the ad itself. A side-channel (faster pages for boosted users, a shown price) would poison the method:
      +
      \[ \hat\beta_{\text{IV}} \;=\; \beta + \frac{s}{\pi} \] + \(\hat\beta_{\text{IV}}\): the estimate the lottery-based method reports  ·  \(\beta\): the true effect of one exposure on sales, the number we are hunting  ·  \(s\): the side-channel's own lift on sales  ·  \(\pi\): the first stage
      +
      No test will catch it
      + Exclusion is a claim about a path that should not exist, so no output can verify it. You defend it by design.
      +
      +
      +
      ▶ LIVE break the silence on purpose
      +
      +
      + + +
      +
      What the method reports, split into the true effect (blue) and the contamination \(s/\pi\) (red). Slide \(s\) up: a small leak becomes a large error.
      +
      +
        +
      • Why a tiny leak is a big error: the side-channel \(s\) gets divided by the small first stage \(\pi\), so even €1 of leak moves the estimate by €1 / 0.2106.
        • +
      • Our defence: a queue bump is content-free, sells nothing, and the user never learns it happened.
        • +
      • The counterexample: an email with a discount code would move spending on its own, and everything after this slide would collapse.
        • +
      +
      +
      +
      + + +
      +
      Part 3 · The estimator · the second ingredient
      +

      The reduced form

      +
      The same comparison as the first stage, now for sales.
      +
      +
      +
      Definition · reduced form
      + The reduced form is the effect of the instrument on the outcome: the sales gap between the lottery groups. It is the \(\delta\) measured below.
      +
      \[ \delta \;=\; \mathbb{E}[Y \mid Z{=}1] \;-\; \mathbb{E}[Y \mid Z{=}0] \;=\; \text{€}3.48 \] + \(\delta\): the reduced form  ·  \(\beta\): the true effect of one exposure on sales, still the number we are hunting
      +
        +
      • Interpretation of \(\delta\): winning the lottery adds €3.48 of sales for the average user. Causal for the same reason \(\pi\) was: a random draw cannot favour ready buyers.
        • +
      • The intuition that carries the day: the lottery can only reach sales through the ad. One lottery win buys \(\pi\) extra exposures, and each exposure is worth \(\beta\) euros. So the lottery's effect on sales must be \(\beta \times \pi\):
        • +
      +
      \[ \delta \;=\; \beta \times \pi \]
      +
      Both numbers are correct. Neither answers the question.
      + \(\pi\) is exposures per lottery win and \(\delta\) is euros per lottery win, but the client asked for euros per exposure. Since \(\delta = \beta\pi\), one step is left.
      +
      +
      +
      average sales, by the lottery
      +
      +
      The step is small but clean: a random draw cannot favour ready buyers, so selection cannot explain it.
      +

      For the record, the algebra behind the intuition: substitute \(X = b_0 + \pi Z + u\) into the sales equation and the coefficient multiplying \(Z\) is exactly \(\beta\pi\).

      +
      +
      +
      + +
      +
      Part 3 · The estimator · the whole method in one division
      +

      The IV estimate

      +
      Euros per lottery win, divided by exposures per lottery win.
      +
      +
      +
      the whole method, as arithmetic on two measured numbers
      +
      +
      One lottery win buys 0.2106 extra exposures and €3.48 of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
      +
      +
      +
      +
      \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ 3.48 }{ 0.2106 } \;=\; \text{€}16.5 \]
      +
        +
      • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
        • +
      • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
        • +
      +
      +
      +
      What just happened
      + We priced the ad using only the random slice of exposure: the dashboard said €23.7, the lottery says €16.5, against a planted truth of €15.
      +

      The method never needed a model of intent, controls, or machine learning: two averages and a division.

      +
      +
      +
      ▸ Deep dive · the confidence interval around €16.5
      +
      +
      +

      \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

      +
      \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; 16.5 \;\pm\; 1.645 \times 2.31 \;=\; [\,12.7,\; 20.4\,] \] + the 90% confidence interval: the range of effect sizes the data support
      +
        +
      • Interpretation of the interval: any effect between €12.7 and €20.4 is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
        • +
      • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
        • +
      +
      +
      +
      two intervals, graded against the planted truth
      +
      +
      The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
      +
      +
      +
      +
      +
      + +
      +
      Part 3 · The estimator · hands on
      +

      Why the division is forced

      +
      The division is not a modelling choice. It is the only effect size the two measurements allow.
      +
      +
      +
      ▶ LIVE every candidate effect makes a prediction. One matches.
      +
      +
      + + +
      +
      The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €3.48. Move your guess to the crossing and you have priced the ad.
      +
      +
      +

      Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

      +
        +
      • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's 0.2106 extra exposures per win should create \(\hat\beta \times 0.2106\) euros per win.
        • +
      • The fact: the lottery actually created €3.48 per win.
        • +
      • The verdict: every candidate except €16.5 contradicts a number we measured. The division is the only survivor, not a choice.
        • +
      +
      \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
      +
      Not a black box
      + Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
      +
      +
      +
      + + +
      +
      Part 4 · When it breaks · the dangerous failure
      +

      Weak instruments

      +
      A weak instrument is worse than no instrument.
      +
      +
      +
      ▶ LIVE what the estimator does as the first stage dies
      +
      +
      + + +
      +
      Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
      +
      +
      +

      The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

      +
        +
      • The honest failure: the interval balloons, and the data admit they know little.
        • +
      • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
        • +
      • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
        • +
      +
      Report the first-stage \(F\), always
      + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
      +
      +
      +
      ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
      +
      +
      +
      ▶ LIVE test each candidate \(\beta_0\), keep the survivors
      +
      +
      + + +
      +
      The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [12.6, 20.2]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
      +
      +
      +
        +
      • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
        • +
      • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
        • +
      +
      \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
      +
        +
      • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
        • +
      • Here, a good sign: the lottery is strong, so AR [12.6, 20.2] nearly matches the usual [12.7, 20.4]. When \(F\) is small the two part company, and AR is the one still telling the truth.
        • +
      +
      +
      +
      +
      +
      + +
      +
      Part 4 · When it breaks · whose effect it is
      +

      Compliers and the LATE

      +
      Whom does the €16.5 describe? Only the users the lottery could move.
      +
      +
      ▶ LIVE the user base, split by how they respond to the lottery
      +
      +
      + + +
      +
      Push \(\gamma\): the complier slice grows, because the complier share is the first stage. Shares are drawn live from one simulated batch, so they can differ from the printed figures by a rounding step.
      +
      +
      +
        +
      • Always-takers (56.5%): the auction shows them the ad with or without the lottery. It changes nothing for them, so the data say nothing about them.
        • +
      • Never-takers (22.5%): never see the ad either way. Same silence.
        • +
      • Compliers (21.0%): see the ad only because the lottery favoured them. Every euro of the lottery's lift \(\delta\) came from them.
        • +
      • The caveat, monotonicity: we assume no defiers, users who would see the ad only when the lottery does not favour them. A nudge that never repels.
        • +
      +
      +
      +
      Definition · LATE (local average treatment effect)
      + The LATE is the average effect of the ad on the compliers alone, and it is what the division estimates: a local answer, not a statement about every user.
      +
      \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;\;\text{ estimates }\;\; \mathbb{E}[\,Y(1)-Y(0)\mid \text{complier}\,] \] + the average of each complier's personal effect, the same \(Y(1)-Y(0)\) contrast the naive slide could not touch
      +
      +
      +
      Why a manager should love this fine print
      + Compliers are the same kind of marginal user a higher bid would newly reach: the closest thing in the data to the customer a bid change buys. That makes €16.5 a price for the marginal customer, measured on the margin rather than on the average.
      +
      +
      + +
      +
      Part 4 · When it breaks · what must hold, on one page
      +

      The checklist

      +
      The four assumptions, and which ones the data can check.
      +
      + + + + + + + + +
      AssumptionWhat it saysIn the caseStatus
      Relevance\(Z\) moves \(X\)\(F = 156\), far above 10TESTABLE, passes
      Exogeneity\(Z \perp U\)the lottery is a genuine random drawBY DESIGN
      Exclusion\(Z \to Y\) only via \(X\)a queue bump shows the user nothingUNTESTABLE
      Monotonicityno defiersa nudge never repelsUNTESTABLE, plausible
      +
      +
      +
      The honest scorecard, for any IV study you are shown
      + One measured number (the first-stage \(F\)), one design guarantee (the randomization), two arguments (exclusion, monotonicity). Ask for all four before you accept the estimate.
      +
        +
      • What we can now defend: an exposure causes about €16.5 of sales for the users a bid can actually move.
        • +
      +
      +
      The sentence that loses money
      + "Exposed users are worth €23.7 each, so raise the bid": every word true, conclusion wrong. It books the platform's targeting as advertising.
      +
      +
      +
      + + + +
      +
      Part 5 · The decision · euros at last
      +

      The price map

      +
      The estimate becomes a decision only when it meets the price.
      +
      +
      +
      ▶ LIVE the verdict as the price moves
      +
      +
      + + +
      +
      Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
      +
      +
      +

      One estimate gives not one answer but a map from any price to a verdict:

      + + + + + + + +
      Price zoneVerdictWhy
      below €12.7GOeven the most pessimistic supported effect pays
      €12.7 to €20.4TESTthe data straddle the price: negotiate, or measure more
      above €20.4NO-GOno supported effect pays
      +
        +
      • Today's rate, €10, sits in the GO zone, below the whole interval.
        • +
      • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €16.5 − €10 = €6.5. Even read at the interval floor, €12.7 against €10, the exposure still pays.
        • +
      +
      Why boards like this framing
      + "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
      +
      +
      +
      + +
      +
      Part 5 · The decision
      +

      Poll · the negotiation

      +
      +
      +
      ✋ Poll
      +
      The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €16.5 per exposure, 90% interval [12.7, 20.4]. What is the highest rate at which you would still sign "buy" without further study?
      +
      + + + + +
      + +
      C. Below €12.7, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
      +
      +
      +
      + +
      +
      Part 5 · The decision · banked
      +

      The verdict and the recommendation

      +
      The complete answer, assembled from everything measured so far.
      +
      +
      + + + + + + + + + +
      QuantityValueSource
      effect of one exposure€16.5the division δ/π
      90% interval[12.7, 20.4]classical, and AR agrees
      first-stage F156the lottery is strong
      price€10the platform's rate card
      net per exposure€6.5β − c, at the point estimate
      +
      The verdict
      + BUY  Keep buying at €10: the entire defensible range clears the price.
      +
      +
      +
      The recommendation, in three lines
      + 1. Keep buying at the €10 rate: even the interval's most pessimistic effect pays.
      + 2. Cap the bid at the interval's lower end, €12.7: up to there, every effect the data support still clears the price.
      + 3. Measure again only if the rate card climbs toward €12.7: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
      +
        +
      • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
        • +
      • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
        • +
      +
      +
      +
      + + + +
      + + + +
      + Causal Marketing · SDA Bocconi · Instrumental Variables + + + + 1 / 1 +
      +
      +

      Contents

        + + + + diff --git a/causal-marketing-pymc/apps/iv_slides_sh_src.html b/causal-marketing-pymc/apps/iv_slides_sh_src.html new file mode 100644 index 0000000..bdc349b --- /dev/null +++ b/causal-marketing-pymc/apps/iv_slides_sh_src.html @@ -0,0 +1,1841 @@ + + + + + + +Instrumental Variables: what is one ad exposure worth? (slides) + + + + + +
        + + +
        +
        +
        +
        Causal Inference & XAI for Business · SDA Bocconi
        +

        Instrumental Variables

        +
        What is one ad exposure worth, when the platform already picked who sees the ads?
        +
        A guest lecture for Prof. Michele Russo's course
        +
        +
        +
        Francesco Muia
        +
        PhD in Theoretical Physics, EMBA.
        Consultant for PyMC Labs and Brown University.
        +
        francesco.muia@pymc-labs.com
        +
        +
        +
        Alexander Fengler
        +
        PhD in Computational Cognitive Science.
        Postdoc at Brown University, consultant for PyMC Labs.
        +
        alexander.fengler@pymc-labs.com
        +
        +
        +
        + PyMC + PyMC Labs +
        +
        +
        + +
        Where the lecture lands: the dashboard's tight interval misses the true effect of an ad exposure, while the interval built from a lottery covers it, and still clears the price.
        +
        +
        +
        + + +
        +
        Part 1 · The problem
        +

        The case

        +
        One retailer, one ad platform, one decision to make.
        +
        +
        +
        The product
        An online retailer pays an ad platform to show its display ads. One exposure = one user actually saw an ad.
        +
        The price
        Each exposure costs €{{nb11.cost}}, billed by the platform.
        +
        The assignment
        The platform decides, in a real-time auction, which users see the ads.
        +
        The dashboard
        Users who saw an ad brought in €{{nb11.naive}} more than users who did not.
        +
        +
        The platform's pitch
        + "An exposure is worth €{{nb11.naive}} and costs €{{nb11.cost}}. Raise the budget."
        +
        The decision on the table
        + Keep paying €{{nb11.cost}} per exposure, pay more, or stop. One number settles it: how many euros of extra sales one exposure brings in. All we have is the platform's logs: who saw an ad, and what every user spent.
        +

        The question for the poll: is the dashboard's €{{nb11.naive}} that number?

        +
        +
        + +
        +
        Part 1 · The problem
        +

        Poll · the pitch

        +
        +
        +
        ✋ Poll
        +
        The dashboard reports that users who saw an ad brought in €{{nb11.naive}} more than users who did not, and an exposure costs €{{nb11.cost}}. The platform concludes: "raise the budget". What do you think of that claim?
        +
        + + + + +
        + +
        C. The arithmetic is correct and, at this sample size, comfortably significant, so A and B both miss the point. The platform chose who saw the ads, and it is paid to pick users who already look ready to buy. If the two groups differed before any ad ran, the €{{nb11.naive}} mixes the ad's effect with that pre-existing difference, in unknown proportions. D is backwards: averaging over thousands of users is exactly how noise is handled.
        +
        +
        +
        + +
        +
        Part 1 · The problem · the variables, drawn
        +

        The variables and the confounder

        +
        The three quantities of the case, defined through their causal diagram.
        +
        +
        +

        The platform's logs cover {{nb11.n}} customers, indexed by \(i\). Three quantities per customer:

        +
          +
        • \(Y_i\) · sales: contribution euros earned from customer \(i\) in the window (sales net of product cost); "sales" for short. Noisy: the standard deviation across customers is €{{nb11.ppc_obs_sd}}.
          • +
        • \(X_i\) · exposure: 1 if customer \(i\) saw the ad, 0 if not. Assigned by the platform's auction, not by us.
          • +
        • \(U_i\) · intent: how ready \(i\) was to buy before any ad. Real, and it drives buying, but it is a feeling in a shopper's head: in no log file, never observed.
          • +
        +
        Definition · confounder
        + A confounder is a variable that influences both the treatment and the outcome. Here it is intent \(U\): it raises the chance the platform shows the ad (the targeting), and it raises sales on its own (ready buyers buy, ad or not).
        +
        Definition · backdoor path
        + In the diagram, \(X \leftarrow U \rightarrow Y\) is a backdoor path: a second route connecting exposure and sales that runs behind the treatment. Association flows along it even if the ad does nothing.
        +
        +
        +
        the situation, as a diagram
        +
        +
        \(U\) points at both \(X\) and \(Y\): the fork every claim in this lecture has to get past.
        +
        +
        +
        + +
        +
        Part 1 · The problem
        +

        The naive comparison

        +
        The dashboard's €{{nb11.naive}}, written down as a formula.
        +
        +
        +

        Write \(Y_i(1)\) for what customer \(i\) spends with the ad and \(Y_i(0)\) for what the same customer spends without it. Each customer shows us exactly one of the two:

        + + + + + + +
        group\(Y_i(1)\) · with ad\(Y_i(0)\) · without ad
        exposed (\(X_i=1\))observedN/A
        unexposed (\(X_i=0\))N/Aobserved
        +
        \[ \widehat{\Delta}_{\text{naive}} \;=\; \bar{Y}_{\text{exposed}} \;-\; \bar{Y}_{\text{unexposed}} \;=\; \text{€}{{nb11.naive}} \]
        +
          +
        • What it averages: the two observed cells, one from each row. The arithmetic is right.
          • +
        • What it pretends: that the unexposed row's \(Y(0)\) is a fair stand-in for the exposed row's missing \(Y(0)\). Fair only if the two groups are comparable.
          • +
        • The auction works against us: the platform is paid to find likely buyers, so the exposed would have spent more with no ad at all.
          • +
        +
        +
        +
        +
        ▶ LIVE exposed users were already different
        +
        +
        + + +
        +
        The gap compares two groups the auction built to differ. The hidden column shows by how much.
        +
        +
        +
        +
        + +
        +
        Part 1 · The problem
        +

        Poll · read the gap

        +
        +
        +
        ✋ Poll
        +
        The naive comparison gives \(\widehat{\Delta}_{\text{naive}} = \) €{{nb11.naive}}. Which reading of that number is defensible?
        +
        + + + + +
        + +
        B. The arithmetic is sound, so C is out. A and D each pick one extreme of the same unknown split: A books all of it to the ad, D books all of it to targeting, and the dashboard alone cannot justify either. The honest statement is the decomposition: €{{nb11.naive}} = the ad's effect + selection bias, with nothing on the dashboard saying how it splits. Measuring that split is the technical problem this lecture solves.
        +
        +
        +
        + +
        +
        Part 1 · The problem · the poll's answer, formalized
        +

        Selection bias

        +
        The decomposition behind the poll's answer.
        +
        +
        +

        Split the naive gap into the two things it adds together:

        +
        \[ \Delta_{\text{naive}} \;=\; \underbrace{\mathbb{E}[Y(1)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}1]}_{\text{effect of the ad on the exposed}} \;+\; \underbrace{\mathbb{E}[Y(0)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}0]}_{\text{selection bias}} \]
        +
          +
        • Read the first term: the exposed users with the ad versus the same users without it. This is what we want to buy, and its second entry is the table's N/A cell.
          • +
        • Read the second term: the gap that would exist with no ads at all, because the picked users differ from the skipped ones.
          • +
        • Here it is positive: the auction picks ready buyers, so the exposed would have out-spent the unexposed anyway.
          • +
        • The dashboard reports only the total: €{{nb11.naive}}. Nothing on it says how the total splits.
          • +
        +
        Definition · selection bias
        + When treated and untreated groups differed before the treatment, the difference of their averages includes that pre-existing gap. The pre-existing part is selection bias.
        +
        +
        +
        +
        the dashboard's number is a sum, and the split is invisible
        +
        +
        The dashboard hands us the full bar. Where the boundary sits, nobody can see from the dashboard alone. Finding it is the rest of the lecture.
        +
        +
        +
        +
        + +
        +
        Part 1 · The problem · a world we can grade
        +

        The simulated world

        +
        A world where the true effect is planted, so every method can be graded.
        +
        +
        +
        +
        ▶ LIVE the world the equations generate, as data
        +
        +
        + + +
        +
        Sales distributions for exposed and unexposed users, drawn fresh from the equations. Push \(\kappa\) up, so that intent leaks more strongly into sales, and the two groups drift apart with the planted effect unchanged: the dashboard's gap grows while the truth stands still.
        +
        +
        +
        +
        Why simulate at all
        + Three equations generate the dashboard you saw. We planted an ad effect of \(\beta =\) €{{nb11.true}}, so every method in this lecture can be graded against a known answer.
        +
        \[ U \sim \mathcal{N}(0,1) \]
        +
        \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \lambda\,U\right) \]
        +
        \[ Y \;=\; \mu + \beta\,X \;+\; \kappa\,U \;+\; \varepsilon \] + \(\mu\): baseline sales  ·  \(\alpha_0\): baseline exposure odds  ·  \(\varepsilon\): noise
        +
          +
        • Interpretation of line 2: who sees an ad rises with intent (\(\lambda U\)): the targeting. \(\sigma\) is the logistic function, turning any score into a probability.
          • +
        • Interpretation of line 3: sales are the ad effect \(\beta\) plus intent leaking straight into spending (\(\kappa U\)).
          • +
        • The math is here for the record. The point is on the left: one hidden cause, \(U\), sits in both equations.
          • +
        +
        +
        +
        + +
        +
        Part 1 · The problem · the disease, stated exactly
        +

        Endogeneity

        +
        The whole problem, stated as one inequality, then shown as a picture.
        +
        +
        +

        Collect everything the sales equation leaves out into a single error term \(v\):

        +
        \[ Y \;=\; \mu + \beta\,X + v, \qquad v \;=\; \kappa\,U + \varepsilon \]
        +
        \[ \operatorname{Cov}(X,\, v) \;>\; 0 \]
        +
          +
        • In words: \(v\) is "every other reason this customer spent money", and exposure is not independent of those reasons, because the auction targets exactly the customers with reasons to spend.
          • +
        • The picture on the right is the inequality: walk up the intent scale and both bars rise together. That joint rise is \(\operatorname{Cov}(X,v) > 0\).
          • +
        +
        Definition · endogeneity
        + A treatment correlated with the other drivers of the outcome is endogenous. Ours is, and everything else in this lecture is a response to that fact.
        +
        The sharp edge
        + The variable driving the correlation was never written to disk.
        +
        +
        +
        +
        one hidden cause pushes both
        +
        +
        Customers grouped by buying intent (which nobody observes). Higher-intent customers see more ads and spend more, ad or no ad. The formula on the left is this picture, in symbols.
        +
        +
        +
        +
        + +
        +
        Part 1 · The problem · the bias, computed
        +

        The size of the bias

        +
        The dashboard's error is not vague. It has a formula, and the formula has three lessons.
        +
        +
        +
        \[ \widehat{\Delta}_{\text{naive}} \;\longrightarrow\; \beta \;+\; \underbrace{\kappa\,\frac{\operatorname{Cov}(X, U)}{\operatorname{Var}(X)}}_{\text{selection bias}} \]
        +
          +
        • Predicted vs delivered: the formula predicts a naive gap near €{{nb11.ols_predicted}}. The data delivered €{{nb11.naive}}, against a truth of €{{nb11.true}}.
          • +
        +
        What the formula teaches
        + 1. The error is systematic, not bad luck: both factors are positive, so the dashboard can only overstate.
        + 2. It grows with the targeting (\(\lambda\), how strongly intent drives exposure) and with intent's pull on sales (\(\kappa\)): the better the platform targets, the worse the dashboard lies.
        + 3. Sample size is absent: more rows shrink noise, never bias. The dashboard's band, €{{nb11.naive_lo}} to €{{nb11.naive_hi}}, is the tightest in this lecture and misses €{{nb11.true}} completely.
        +
        What the dashboard measures
        + Not the ad's power to create buyers but the platform's skill at finding them, reported as one number.
        +
        +
        +
        +
        ▶ LIVE the same bar, at different targeting strengths
        +
        +
        + + +
        +
        The dashboard's bar, split by the formula: the true effect (blue) never moves, while the selection bias (orange) grows as intent leaks more strongly into sales. In this simulated world we can draw the boundary, because we planted it.
        +
        +
        +
        +
        + +
        +
        Part 1 · The problem · closing the obvious exit
        +

        The limits of adjustment

        +
        The standard fix works, when you have the column it needs. We do not.
        +
        +
        +
        The standard fix: close the backdoor
        + Compare users at the same level of the confounder. Within a group of equal-intent users, intent no longer separates exposed from unexposed, and any remaining gap belongs to the ad. Controls in a regression, matching, and propensity scores are all versions of this one move.
        +
        +
        ▶ LIVE the fix works, if the column exists
        +
        +
        + + +
        +
        Tick the box: comparing exposed to unexposed within equal-intent groups recovers the planted truth to within half a euro. The method is fine. Untick it: in the real logs the column does not exist, and no amount of modelling recreates it.
        +
        +
        Why the engine stalls here
        + You cannot adjust for what you did not record.
        +
        +
        +

        Two customers, identical on every recorded feature \(W\):

        + + + + + + +
        customerpages, device, history \(W\)intent \(U\)sees the ad?
        Annaidenticalhigh, unrecordedpicked
        Beaidenticallow, unrecordedskipped
        +
        what conditioning blocks, and what it cannot
        +
        +
        Holding \(W\) fixed (the box) blocks the backdoors that run through it. The arrows themselves never move: conditioning blocks paths, it does not delete causes. The path through \(U\) stays open, because \(U\) is in no file to hold fixed.
        +
        +
        +
        + +
        +
        Part 1 · The problem
        +

        Poll · what would fix it?

        +
        +
        +
        ✋ Poll
        +
        Adjustment failed because intent was never recorded. Your team can request one addition to the data. Which one would actually let you measure what an exposure causes?
        +
        + + + + +
        + +
        C. A tells you the wrong number more precisely: bias is not noise, and the formula for it does not contain the sample size. B and D are more controls, and the auction reacts to live signals that no exported feature set fully contains, so the backdoor stays open. C is different in kind: it changes who assigns the treatment, and the assignment was the disease all along. The rest of the lecture is about getting C, or the closest thing to it the system already contains.
        +
        +
        +
        + + +
        +
        Part 2 · The idea · what we wish we could do
        +

        The ideal experiment

        +
        What randomizing exposure would buy, and why we cannot do it.
        +
        +
        +
        randomization cuts the arrow that causes the trouble
        +
        +
        Left: today's world, where intent drives both exposure and sales. Right: the experiment we wish we could run, where a randomizer assigns exposure and intent's arrow into it is removed, not merely blocked: exposure no longer listens to intent at all. With that arrow gone, the naive comparison becomes the right comparison.
        +
        \[ X \text{ randomized} \;\;\Rightarrow\;\; \operatorname{Cov}(X,\, v) = 0 \;\;\Rightarrow\;\; \widehat{\Delta}_{\text{naive}} \longrightarrow \beta \]
        +
        +
        +
          +
        • The wish: option C from the poll: show the ad to a random half ourselves.
          • +
        • The wall: the platform's auction decides exposure in milliseconds, using exactly the intent signals that cause the problem. It will not hand us the controls.
          • +
        +
        The idea that saves us
        + If we cannot inject randomness into exposure, find the randomness the system already contains and use only that part.
        +
          +
        • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
          • +
        +
        The plan for Part 2
        + 1. Give that random lever a name: an instrument.
        + 2. State the conditions it must satisfy.
        + 3. Check which of them the data can verify.
        +
        +
        +
        + +
        +
        Part 2 · The idea · the randomness we ran
        +

        The lottery

        +
        The randomness our system already contains, because we put it there.
        +
        +
        +
          +
        • What we ran: before the campaign, a serving-priority lottery. Our own random number generator picked half the users and gave them a small priority boost in the platform's ad auction.
          • +
        • What it does: the lottery shows nobody an ad. It only raises the odds of seeing one.
          • +
        • A new column joins the data:
          • +
        + + + + + +
        quantitymeaningtypewho assigns itobserved?
        \(Z_i\) · the lottery1 if the lottery boosted \(i\)'s priority in the ad auctionbinarya random number generator we controlobserved
        +
        \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \gamma\,Z + \lambda\,U\right) \] + the simulated world ran the lottery too: the exposure equation gains \(\gamma Z\), the lottery's push on the odds of seeing an ad
        +
          +
        • Why it matters: \(Z\) moves exposure, and nothing else about the customer. It is the one source of variation in \(X\) that the targeting cannot touch.
          • +
        +
        +
        +
        the fork, and the way around it
        +
        +
        Top: the confounded fork. Bottom: the lottery \(Z\) pushes on \(X\) from outside. No arrow from \(U\) into \(Z\) (it is a random draw), and no arrow from \(Z\) straight to \(Y\) (a queue bump shows the user nothing).
        +
        +
        +
        + +
        +
        Part 2 · The idea · the central definition
        +

        The instrument

        +
        What we just built has a name.
        +
        +
        Definition · instrument
        + An instrument \(Z\) moves the treatment, is as good as random with respect to the hidden confounder, and affects the outcome only through the treatment. Four conditions make that precise, and they grade our lottery as follows.
        + + + + + + + + + + + + + +
        conditionthe claimcan it be checked?what it buys
        1 · Relevancethe lottery actually moves exposure \(X\)testable measured by the first stagea push that really happened
        2 · Exogeneitythe lottery is blind to intent \(U\)by design if the draw is genuinewith 1: a clean experiment on the lottery
        3 · Exclusionthe lottery touches sales only through exposureuntestable argued, never checkedturns the lottery's effect into the ad's
        4 · Monotonicitythe lottery never blocks an exposure that would have happened without ituntestable argued from the mechanismsays whose effect we measured
        +
        An instrument is the opposite of a control
        + You do not hold \(Z\) fixed. You use the variation it created.
        +

        The genius is never the estimator, it is spotting the randomness your system already contains: a rollout order, a capacity limit, an upstream A/B test, a rounding rule.

        +
        +
        + +
        +
        Part 2 · The idea · condition 1, measured
        +

        Condition 1 · relevance

        +
        The first stage: did the lottery actually move exposure?
        +
        +
        +

        Relevance is a claim about the data, so the data can answer it.

        +
        Definition · first stage
        + The first stage is the effect of the instrument on the treatment: how much exposure the lottery itself created. It is the coefficient \(\pi\) in the regression below.
        +
        +
        \[ X \;=\; b_0 + \pi\,Z + u \] + \(\pi\): the first stage  ·  \(b_0\): exposure rate among lottery losers  ·  \(u\): every other driver of exposure, intent included
        +
        +
        +
        \[ \pi \;=\; \Pr(X{=}1 \mid Z{=}1) \;-\; \Pr(X{=}1 \mid Z{=}0) \;=\; {{nb11.first}} \] + the measured first stage: what the lottery did to exposure
        +
          +
        • Interpretation of \(\pi\): winning the lottery raises a user's chance of seeing an ad by {{nb11.fs_shift}} percentage points, from {{nb11.fs_lo}}% to {{nb11.fs_hi}}%.
          • +
        • It is causal, full stop: \(Z\) is random, so nothing hidden can explain the step.
          • +
        +
        +
        +
        exposure rate, by the lottery
        +
        +
        The step between the bars is the first stage \(\pi\): the slice of exposure that the lottery, not targeting, created.
        +
        The gate: is the push strong enough?
        + Like every estimate in this lecture, \(\pi\) comes with a significance test: the first-stage \(F\). Trust the coming division only when \(F > 10\), well past bare significance, because a \(\pi\) that is merely nonzero still leaves the division unstable. Here \(F = {{nb11.f_stat}}\): the push is unmistakably real. We quote such tests, not derive them.
        +
        +
        +
        + +
        +
        Part 2 · The idea · conditions 2 and 3
        +

        Conditions 2 and 3 · exogeneity and exclusion

        +
        The lottery must be blind to intent, and silent about sales.
        +
        +
        +
        Blind · exogeneity
        + The lottery must not favour high-intent users. Ours is a genuine randomizer, so this holds by design.
        +
        Silent · exclusion
        + The lottery must not touch sales through any channel except the ad itself. A side-channel (faster pages for boosted users, a shown price) would poison the method:
        +
        \[ \hat\beta_{\text{IV}} \;=\; \beta + \frac{s}{\pi} \] + \(\hat\beta_{\text{IV}}\): the estimate the lottery-based method reports  ·  \(\beta\): the true effect of one exposure on sales, the number we are hunting  ·  \(s\): the side-channel's own lift on sales  ·  \(\pi\): the first stage
        +
        No test will catch it
        + Exclusion is a claim about a path that should not exist, so no output can verify it. You defend it by design.
        +
        +
        +
        ▶ LIVE break the silence on purpose
        +
        +
        + + +
        +
        What the method reports, split into the true effect (blue) and the contamination \(s/\pi\) (red). Slide \(s\) up: a small leak becomes a large error.
        +
        +
          +
        • Why a tiny leak is a big error: the side-channel \(s\) gets divided by the small first stage \(\pi\), so even €1 of leak moves the estimate by €1 / {{nb11.first}}.
          • +
        • Our defence: a queue bump is content-free, sells nothing, and the user never learns it happened.
          • +
        • The counterexample: an email with a discount code would move spending on its own, and everything after this slide would collapse.
          • +
        +
        +
        +
        + + +
        +
        Part 3 · The estimator · the second ingredient
        +

        The reduced form

        +
        The same comparison as the first stage, now for sales.
        +
        +
        +
        Definition · reduced form
        + The reduced form is the effect of the instrument on the outcome: the sales gap between the lottery groups. It is the \(\delta\) measured below.
        +
        \[ \delta \;=\; \mathbb{E}[Y \mid Z{=}1] \;-\; \mathbb{E}[Y \mid Z{=}0] \;=\; \text{€}{{nb11.reduced}} \] + \(\delta\): the reduced form  ·  \(\beta\): the true effect of one exposure on sales, still the number we are hunting
        +
          +
        • Interpretation of \(\delta\): winning the lottery adds €{{nb11.reduced}} of sales for the average user. Causal for the same reason \(\pi\) was: a random draw cannot favour ready buyers.
          • +
        • The intuition that carries the day: the lottery can only reach sales through the ad. One lottery win buys \(\pi\) extra exposures, and each exposure is worth \(\beta\) euros. So the lottery's effect on sales must be \(\beta \times \pi\):
          • +
        +
        \[ \delta \;=\; \beta \times \pi \]
        +
        Both numbers are correct. Neither answers the question.
        + \(\pi\) is exposures per lottery win and \(\delta\) is euros per lottery win, but the client asked for euros per exposure. Since \(\delta = \beta\pi\), one step is left.
        +
        +
        +
        average sales, by the lottery
        +
        +
        The step is small but clean: a random draw cannot favour ready buyers, so selection cannot explain it.
        +

        For the record, the algebra behind the intuition: substitute \(X = b_0 + \pi Z + u\) into the sales equation and the coefficient multiplying \(Z\) is exactly \(\beta\pi\).

        +
        +
        +
        + +
        +
        Part 3 · The estimator · the whole method in one division
        +

        The IV estimate

        +
        Euros per lottery win, divided by exposures per lottery win.
        +
        +
        +
        the whole method, as arithmetic on two measured numbers
        +
        +
        One lottery win buys {{nb11.first}} extra exposures and €{{nb11.reduced}} of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
        +
        +
        +
        +
        \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ {{nb11.reduced}} }{ {{nb11.first}} } \;=\; \text{€}{{nb11.iv_est}} \]
        +
          +
        • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
          • +
        • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
          • +
        +
        +
        +
        What just happened
        + We priced the ad using only the random slice of exposure: the dashboard said €{{nb11.naive}}, the lottery says €{{nb11.iv_est}}, against a planted truth of €{{nb11.true}}.
        +

        The method never needed a model of intent, controls, or machine learning: two averages and a division.

        +
        +
        +
        ▸ Deep dive · the confidence interval around €{{nb11.iv_est}}
        +
        +
        +

        \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

        +
        \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; {{nb11.iv_est}} \;\pm\; 1.645 \times {{nb11.iv_se}} \;=\; [\,{{nb11.iv_lo}},\; {{nb11.iv_hi}}\,] \] + the 90% confidence interval: the range of effect sizes the data support
        +
          +
        • Interpretation of the interval: any effect between €{{nb11.iv_lo}} and €{{nb11.iv_hi}} is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
          • +
        • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
          • +
        +
        +
        +
        two intervals, graded against the planted truth
        +
        +
        The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
        +
        +
        +
        +
        +
        + +
        +
        Part 3 · The estimator · hands on
        +

        Why the division is forced

        +
        The division is not a modelling choice. It is the only effect size the two measurements allow.
        +
        +
        +
        ▶ LIVE every candidate effect makes a prediction. One matches.
        +
        +
        + + +
        +
        The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €{{nb11.reduced}}. Move your guess to the crossing and you have priced the ad.
        +
        +
        +

        Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

        +
          +
        • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's {{nb11.first}} extra exposures per win should create \(\hat\beta \times {{nb11.first}}\) euros per win.
          • +
        • The fact: the lottery actually created €{{nb11.reduced}} per win.
          • +
        • The verdict: every candidate except €{{nb11.iv_est}} contradicts a number we measured. The division is the only survivor, not a choice.
          • +
        +
        \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
        +
        Not a black box
        + Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
        +
        +
        +
        + + +
        +
        Part 4 · When it breaks · the dangerous failure
        +

        Weak instruments

        +
        A weak instrument is worse than no instrument.
        +
        +
        +
        ▶ LIVE what the estimator does as the first stage dies
        +
        +
        + + +
        +
        Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
        +
        +
        +

        The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

        +
          +
        • The honest failure: the interval balloons, and the data admit they know little.
          • +
        • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
          • +
        • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
          • +
        +
        Report the first-stage \(F\), always
        + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
        +
        +
        +
        ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
        +
        +
        +
        ▶ LIVE test each candidate \(\beta_0\), keep the survivors
        +
        +
        + + +
        +
        The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [{{nb11.ar_lo}}, {{nb11.ar_hi}}]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
        +
        +
        +
          +
        • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
          • +
        • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
          • +
        +
        \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
        +
          +
        • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
          • +
        • Here, a good sign: the lottery is strong, so AR [{{nb11.ar_lo}}, {{nb11.ar_hi}}] nearly matches the usual [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. When \(F\) is small the two part company, and AR is the one still telling the truth.
          • +
        +
        +
        +
        +
        +
        + +
        +
        Part 4 · When it breaks · whose effect it is
        +

        Compliers and the LATE

        +
        Whom does the €{{nb11.iv_est}} describe? Only the users the lottery could move.
        +
        +
        ▶ LIVE the user base, split by how they respond to the lottery
        +
        +
        + + +
        +
        Push \(\gamma\): the complier slice grows, because the complier share is the first stage. Shares are drawn live from one simulated batch, so they can differ from the printed figures by a rounding step.
        +
        +
        +
          +
        • Always-takers ({{nb11.st_share_always}}%): the auction shows them the ad with or without the lottery. It changes nothing for them, so the data say nothing about them.
          • +
        • Never-takers ({{nb11.st_share_never}}%): never see the ad either way. Same silence.
          • +
        • Compliers ({{nb11.st_share_complier}}%): see the ad only because the lottery favoured them. Every euro of the lottery's lift \(\delta\) came from them.
          • +
        • The caveat, monotonicity: we assume no defiers, users who would see the ad only when the lottery does not favour them. A nudge that never repels.
          • +
        +
        +
        +
        Definition · LATE (local average treatment effect)
        + The LATE is the average effect of the ad on the compliers alone, and it is what the division estimates: a local answer, not a statement about every user.
        +
        \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;\;\text{ estimates }\;\; \mathbb{E}[\,Y(1)-Y(0)\mid \text{complier}\,] \] + the average of each complier's personal effect, the same \(Y(1)-Y(0)\) contrast the naive slide could not touch
        +
        +
        +
        Why a manager should love this fine print
        + Compliers are the same kind of marginal user a higher bid would newly reach: the closest thing in the data to the customer a bid change buys. That makes €{{nb11.iv_est}} a price for the marginal customer, measured on the margin rather than on the average.
        +
        +
        + +
        +
        Part 4 · When it breaks · what must hold, on one page
        +

        The checklist

        +
        The four assumptions, and which ones the data can check.
        +
        + + + + + + + + +
        AssumptionWhat it saysIn the caseStatus
        Relevance\(Z\) moves \(X\)\(F = {{nb11.f_stat}}\), far above 10TESTABLE, passes
        Exogeneity\(Z \perp U\)the lottery is a genuine random drawBY DESIGN
        Exclusion\(Z \to Y\) only via \(X\)a queue bump shows the user nothingUNTESTABLE
        Monotonicityno defiersa nudge never repelsUNTESTABLE, plausible
        +
        +
        +
        The honest scorecard, for any IV study you are shown
        + One measured number (the first-stage \(F\)), one design guarantee (the randomization), two arguments (exclusion, monotonicity). Ask for all four before you accept the estimate.
        +
          +
        • What we can now defend: an exposure causes about €{{nb11.iv_est}} of sales for the users a bid can actually move.
          • +
        +
        +
        The sentence that loses money
        + "Exposed users are worth €{{nb11.naive}} each, so raise the bid": every word true, conclusion wrong. It books the platform's targeting as advertising.
        +
        +
        +
        + + + +
        +
        Part 5 · The decision · euros at last
        +

        The price map

        +
        The estimate becomes a decision only when it meets the price.
        +
        +
        +
        ▶ LIVE the verdict as the price moves
        +
        +
        + + +
        +
        Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
        +
        +
        +

        One estimate gives not one answer but a map from any price to a verdict:

        + + + + + + + +
        Price zoneVerdictWhy
        below €{{nb11.iv_lo}}GOeven the most pessimistic supported effect pays
        €{{nb11.iv_lo}} to €{{nb11.iv_hi}}TESTthe data straddle the price: negotiate, or measure more
        above €{{nb11.iv_hi}}NO-GOno supported effect pays
        +
          +
        • Today's rate, €{{nb11.cost}}, sits in the GO zone, below the whole interval.
          • +
        • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €{{nb11.iv_est}} − €{{nb11.cost}} = €{{nb11.net}}. Even read at the interval floor, €{{nb11.iv_lo}} against €{{nb11.cost}}, the exposure still pays.
          • +
        +
        Why boards like this framing
        + "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
        +
        +
        +
        + +
        +
        Part 5 · The decision
        +

        Poll · the negotiation

        +
        +
        +
        ✋ Poll
        +
        The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €{{nb11.iv_est}} per exposure, 90% interval [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. What is the highest rate at which you would still sign "buy" without further study?
        +
        + + + + +
        + +
        C. Below €{{nb11.iv_lo}}, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
        +
        +
        +
        + +
        +
        Part 5 · The decision · banked
        +

        The verdict and the recommendation

        +
        The complete answer, assembled from everything measured so far.
        +
        +
        + + + + + + + + + +
        QuantityValueSource
        effect of one exposure€{{nb11.iv_est}}the division δ/π
        90% interval[{{nb11.iv_lo}}, {{nb11.iv_hi}}]classical, and AR agrees
        first-stage F{{nb11.f_stat}}the lottery is strong
        price€{{nb11.cost}}the platform's rate card
        net per exposure€{{nb11.net}}β − c, at the point estimate
        +
        The verdict
        + BUY  Keep buying at €{{nb11.cost}}: the entire defensible range clears the price.
        +
        +
        +
        The recommendation, in three lines
        + 1. Keep buying at the €{{nb11.cost}} rate: even the interval's most pessimistic effect pays.
        + 2. Cap the bid at the interval's lower end, €{{nb11.iv_lo}}: up to there, every effect the data support still clears the price.
        + 3. Measure again only if the rate card climbs toward €{{nb11.iv_lo}}: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
        +
          +
        • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
          • +
        • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
          • +
        +
        +
        +
        + + + +
        + + + +
        + Causal Marketing · SDA Bocconi · Instrumental Variables + + + + 1 / 1 +
        +
        +

        Contents

          + + + + diff --git a/causal-marketing-pymc/apps/unified_slides.html b/causal-marketing-pymc/apps/unified_slides.html index a512501..11037dd 100644 --- a/causal-marketing-pymc/apps/unified_slides.html +++ b/causal-marketing-pymc/apps/unified_slides.html @@ -110,7 +110,9 @@ /* polls */ .poll{border:1.5px solid var(--orange); border-radius:12px; margin:.7rem 0; overflow:hidden; box-shadow:var(--shadow); background:var(--bg); max-width:980px;} -.poll .ph{background:color-mix(in srgb,var(--orange) 12%,var(--bg)); padding:.5rem 1rem; font-weight:800; color:#b9740f; font-size:.78rem; letter-spacing:.02em; text-transform:uppercase;} +.poll .ph{background:color-mix(in srgb,var(--orange) 12%,var(--bg)); padding:.5rem 1rem; font-weight:800; color:#b9740f; font-size:.78rem; letter-spacing:.02em; text-transform:uppercase; display:flex; align-items:center; gap:.6rem;} +.poll .ph .poll-reset{display:none; margin-left:auto; font:inherit; font-size:.72rem; font-weight:700; text-transform:none; letter-spacing:.02em; color:inherit; background:none; border:1px solid color-mix(in srgb,var(--orange) 45%,transparent); border-radius:7px; padding:.1rem .55rem; cursor:pointer;} +.poll.revealed .ph .poll-reset{display:inline-block;} :root[data-theme="dark"] .poll .ph{color:#e6b968;} .poll .pq{padding:.8rem 1rem .15rem; font-size:1.1rem; font-weight:600; color:var(--navy);} .poll .opts{padding:.5rem 1rem .7rem; display:grid; gap:.42rem;} @@ -121,13 +123,18 @@ .poll .opt.sel{border-color:var(--orange); background:color-mix(in srgb,var(--orange) 10%,var(--panel));} .poll .opt.correct{border-color:var(--green); background:color-mix(in srgb,var(--green) 13%,var(--panel));} .poll .opt.correct .let{color:var(--green);} -.poll .opt.wrong{opacity:.55;} +.poll .opt.wrong{opacity:.75;} +.poll .opt{flex-wrap:wrap;} +.poll .opt .why{display:none; flex-basis:100%; margin-left:1.8em; font-size:.85rem; font-weight:400; color:var(--faint); line-height:1.35;} +.poll.revealed .opt.wrong .why{display:block;} .poll .reveal-btn{margin:0 1rem .8rem; font:inherit; font-weight:700; font-size:.88rem; cursor:pointer; color:#fff; background:var(--orange); border:none; border-radius:8px; padding:.42rem .9rem;} .poll .answer{display:none; margin:0 1rem .9rem; padding:.75rem .95rem; border-radius:9px; background:var(--panel2); border-left:4px solid var(--green); font-size:.98rem;} .poll.revealed .answer{display:block;} .poll .answer b{color:var(--green);} +.post-reveal{display:none;} +.post-reveal.shown{display:block;} /* live figs: full-width by default, charts BIG */ .fig{margin:.6rem 0; border:1px solid var(--rule); border-radius:12px; background:var(--bg); box-shadow:var(--shadow); overflow:hidden;} @@ -234,16 +241,16 @@

          PyMC Labs: what we do

          A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
            -
          • What we sell: senior modeling expertise, not seats or software licenses: revenue is services-led.
            • -
          • Open source is the top of the funnel: the libraries build trust and reach; the consulting monetizes the deep expertise behind them.
            • -
          • How we work: small teams of senior people who start from the client's actual decision, encode domain knowledge and uncertainty explicitly, and hand back models the client can own and extend, not black boxes.
            • +
          • What we sell: senior modeling expertise, not software licenses.
            • +
          • Open source is the top of the funnel: the libraries build reach; the consulting monetizes the expertise behind them.
            • +
          • How we work: small senior teams, starting from the client's decision; the client keeps the model, not a black box.
          - - - + + +
          EngagementWhat it isTypical client
          Project consultingFixed-scope build of a custom model: a marketing-mix model, a demand forecaster, a pricing modelA specific, high-value decision problem
          Advisory / retainerOngoing access to our modelers, guiding an in-house teamFirms building their own capability
          Enablement & trainingWorkshops and embedded upskilling on Bayesian methods and our toolingAnalytics teams standardizing on PyMC
          Project consultingFixed-scope build of one custom model (marketing mix, forecasting, pricing)A specific, high-value decision problem
          Advisory / retainerOur modelers guiding an in-house teamFirms building their own capability
          Enablement & trainingWorkshops and upskilling on Bayesian methods and the toolingAnalytics teams standardizing on PyMC
          The clients in this session
          @@ -257,6 +264,93 @@

          PyMC Labs: what we do

          +
          +
          Opening · A note on method
          +

          Bayesian vs frequentist, in one slide

          +
          Same data, two philosophies.
          +
          +
          +
          Frequentist
          + Reports how a procedure behaves over many hypothetical repetitions of the study.
          +
          Bayesian
          + Reports a full probability distribution over the unknown, given the data in hand.
          +
          +
          +
          + + + + + + + + +
          AspectFrequentistBayesian
          You geta point estimate and a confidence intervala full distribution over the unknown effect
          It answers"how would this procedure behave if I repeated the study many times?""given the data I have, how probable is each value?"
          Shines whensamples are large, the design is standard, speed mattersdata is small or messy, structure is layered, prior knowledge exists
          The catchthe 95% interval is not "95% probability the truth is inside"you must state a prior, and it costs computation
          +
          Where Bayes earns its keep
          + It answers the decision directly: not "is the lift real?" but a number you can act on, P(lift beats breakeven).
          +
          +
          +
          +
          SCHEMATICThe Bayesian answer is a shape, not a point
          +
          + + + + + -2 + 0 + 2 + 4 + 6 + + estimated lift (effect size) + + + no effect + + + + + posterior + P(lift > 0) = 0.92 + + + + + + classical: estimate + symmetric 95% CI + +
          +
          The posterior is skewed, so mode, mean, and the symmetric classical interval no longer coincide, and that interval even dips below zero, yet P(lift > 0) still reads straight off the shaded mass.
          +
          +
          +
          +
          Why marketing analytics is going Bayesian
          + Marketing data is short, noisy, and highly correlated, and every model ends in a spend decision, which is exactly where priors and full uncertainty pay off.
          +
          For today, this is a curiosity
          + The entire lecture runs on classical methods, and every number you will see is frequentist; we surface the Bayesian read only as a flavour at the edges, never as the load-bearing tool.
          +
          ▸ Where it is already the default (and where it is not)
          +
            +
          • Marketing mix modeling: the modern open-source MMM tools are Bayesian (Google's Meridian, PyMC-Marketing); with only two or three years of weekly data and correlated channels, plain regression is unstable, and priors on adstock, saturation, and ROI stabilize it.
            • +
          • Calibrating models to experiments: geo-lift and A/B results enter as priors on channel ROI, so the model and the experiments agree, the HelloFresh loop you will see today.
            • +
          • Decisions under uncertainty: you get a full distribution over each channel's ROAS, so budget optimization and statements like P(channel A beats B) fall out directly.
            • +
          • Many thin segments: hierarchical models borrow strength across regions, SKUs, or cohorts too small to estimate on their own.
            • +
          • Signal loss: cookie deprecation and privacy changes push measurement back toward aggregate MMM, which is Bayesian home turf.
            • +
          • Not everywhere: for large-sample, standard A/B tests and simple designs, classical methods are faster and perfectly adequate.
            • +
          +
          +
          ▸ Pros and cons, plainly
          + + + + + + +
          ProsCons
          Frequentistfast, familiar, few choices to defend, excellent for large samples and standard designsp-values and confidence intervals are routinely misread, and awkward once models get layered or data gets thin
          Bayesiandirect probability statements, priors fold in domain knowledge, hierarchical models borrow strength across groups, uncertainty flows straight into a decisionyou must choose priors (and defend them), sampling can be slow, and it demands more modeling care
          +
          +
          +
          +
          Case 1 · Colgate-Palmolive
          @@ -267,13 +361,16 @@

          You are the consultant

          ✋ Poll
          Colgate-Palmolive calls: "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it stealing share from competitors, or from our own brands?" Which tool do you reach for first?
          - - + + - +
          -
          C. A national launch leaves nothing to randomize and no market to difference against: A and B need a control group that does not exist, and D's instrument does not exist for a shelf that changed everywhere at once. What remains is the move this whole session builds, here run in time: fit the world before the launch, project it forward, read the gap. PyMC Labs sold exactly that projection; the next slide shows it.
          +
          C. A national launch leaves nothing to randomize and nothing to difference against: A, B and D all need a comparison that does not exist. What remains is this session's one move: fit the world before the launch, project it forward, read the gap. PyMC Labs sold exactly that; next slide.
          @@ -287,16 +384,16 @@

          Colgate-Palmolive: incremental, or cannibalistic?

          The launch, and the world without it schematic
          -
          Illustrative shape of the engagement's counterfactual read, not client data: fit the pre-period, project it forward, price the gap.
          +
          Illustrative shape, not client data: fit the pre-period, project, price the gap.
          The brief, in their words
          "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
            -
          • The client: Colgate-Palmolive came to PyMC Labs in 2023, in a market estimated at $20.8 billion in 2023.
            • -
          • The method: a multivariate Bayesian interrupted time series: the pre-launch world projected forward, pointed at a product; later extended to a nested-logit choice model.
            • -
          • The grading: on simulated data the model recovers a planted 50% incrementality as a 94% interval of 49-59%: the recover-the-truth contract of this whole session, run commercially.
            • +
          • The client: Colgate-Palmolive, 2023; a market estimated at $20.8 billion in 2023.
            • +
          • The method: a multivariate Bayesian interrupted time series: project the pre-launch world forward.
            • +
          • The grading: a planted 50% recovered as a 94% interval of 49-59%: recover a known truth first, then be believed.
          @@ -305,26 +402,40 @@

          Colgate-Palmolive: incremental, or cannibalistic?

          Case 1 · Colgate-Palmolive · open floor

          What would break it?

          -
          -
          +
          +
          🗣 Open floor · 2 minutes
          You are Colgate's CMO. The incrementality estimate you just saw (the share of the new product's sales that are genuinely new, not cannibalized) decides the launch review. Name one real-world event that would make it wrong.
          -
          A second launch. When another product entered the estimation window, the same machinery reported 64-76% incrementality against a planted truth of 100%: the counterfactual absorbed part of the very effect it was meant to isolate. An honest consultancy publishes exactly this: the 64-76% miss is printed in the same post as the win. In Part 2 the same disease returns with its formal name, spillover; the defence is design, not statistics.
          -
            -
          • What the model learns: everything before the launch defines "normal growth", and the projection (red) extrapolates that normal forward.
            • -
          • What a second launch does: inside the window it becomes part of "normal", so the projection rises too fast and under-credits the true lift; after the launch it inflates the observed line instead, and over-credits.
            • -
          -
          -
          Break it yourself: slide a second launch into the window schematic
          -
          -
          - - - +
          +
          +
          Learning 1 · what breaks it
          + A second launch inside the fit window redefines "normal growth", so the counterfactual absorbs part of the very effect it is meant to isolate.
          +
          Learning 2 · the lesson that runs the whole session
          + A counterfactual is only as good as a clean window: in Part 2 this disease returns with its formal name, spillover, and the defence is design, not statistics.
          +
          +
          +
          +
          Break it yourself: slide a second launch into the window schematic
          +
          +
          + + + +
          +
          Schematic, not client data: the red projection is refit live from whatever the pre-launch months show.
          +
          +
          +
          How to read the plot
          +
            +
          • Three lines: solid blue is observed sales, grey dashed is the true no-launch world, red dotted is the model's guess of that world, fit on the pre-launch months only.
            • +
          • The clean read: red sits exactly on grey and the readout says 100%: the model recovers the true lift.
            • +
          • Break it: tick the box; a second launch before month 0 pulls red off grey and the readout falls (under-credit), after month 0 it climbs past 100% (over-credit).
            • +
          • The real case: Colgate's model read 64-76% when the planted truth was 100%.
            • +
          -
          Same schematic world as the previous slide. The real case reported 64-76% against a truth of 100%.
          +
          @@ -333,43 +444,45 @@

          What would break it?

          Case 2 · HelloFresh · the tool, and its failure mode

          Why calibrate? A model alone can rank channels backwards

          -
          Before the HelloFresh story, the tool it relies on: a warm-up from PyMC Labs' published calibration tutorial.
          +
          A public tutorial that shows the real job clients hire us for: making a company's ad-budget model trustworthy.
          -
            -
          • The tool: a marketing-mix model (MMM) explains total sales as the sum of per-channel contributions, fit on observational spend data: no experiment anywhere in it.
            • -
          • The grading: the tutorial plants a truth: return on ad spend (ROAS, sales per unit of spend) of 93.39 for channel x1 against 171.41 for x2, so x2 is nearly twice as effective.
            • -
          • The experiment: a lift test nudges one channel's spend by a known amount and measures the sales change it causes: a small randomized ground-truth reading for that channel.
            • -
          • The repair: two lift tests per channel, entered into the likelihood, recover both values: the experiment is the model's anchor, the theme of everything that follows.
            • -
          +
          +
          The settinga company spreads its budget across many ad channels (TV, search, social) and needs to know which ones actually pay back.
          +
          The everyday toola model reads years of spend-and-sales history and scores each channel's return on ad spend, cheaply and always-on.
          +
          The catch, and our jobhistory is not an experiment; a company spends more exactly when demand is already high, so the model can credit the season instead of the channel and rank them backwards.
          +
          What we adda real experiment: nudge one channel's budget by a known amount, measure the sales it truly causes, and anchor the model to that number. This is calibration.
          +
          -
          One MMM on observational spend alone, one planted truth, one inversion baked from the tutorial
          -
          +
          The model's channel ranking, before and after a real experiment baked from the tutorial
          +
          -
          Left: the ranking the uncalibrated model reported. Right: the planted truth the experiments recover.
          +
          Left: what the history-only model reported. Right: the true answer the experiments recover.
          -
          The inversion
          - Fit on observational data alone, the baseline model ranked x1 above x2: the direct opposite of the truth.
          +
          The inversion, on a known answer
          + True returns were 93.39 versus 171.41, so channel 2 is nearly twice as effective; yet the uncalibrated model reported the direct opposite of the truth, and two lift tests per channel fixed it.
          Case 2 · HelloFresh

          HelloFresh runs the loop, at industrial scale

          +
          The same idea, not on a tutorial now but in production at a company you know.
          -
            -
          • The loop: MMM priors fed by field experiments such as lift or incrementality tests; a 60% cut in prediction variance.
            • -
          • On stage: the panel's own agenda: Bayesian MMM can be calibrated to ensure consistency with incrementality measurements.
            • -
          • The experiment supply: a pipeline handling thousands of concurrent tests: A/B, ABC, and ABCD campaigns run simultaneously, the overnight batch down from 5–6 hours to 5–6 minutes; the Criteo experiment from the session's IV close (13,979,592 users) sits in exactly this regime.
            • -
          -
          The supply chain, for the IV close
          - The experiments a company already runs are its instrument supply: a randomized encouragement is the instrument for the exposure you cannot randomize.
          +
          +
          Not a toyHelloFresh, the meal-kit company, runs this exact model-plus-experiment loop in production.
          +
          The loopthe model steers the budget always-on, while a steady stream of lift or incrementality tests keeps re-anchoring it, cutting prediction variance by 60%.
          +
          Calibration, put plainlya Bayesian budget model should be "calibrated to ensure consistency with incrementality measurements".
          +
          The scalethousands of concurrent tests (A/B, ABC, and ABCD designs), each model fit batched down from 5–6 hours to 5–6 minutes; one experiment alone logged 13,979,592 users.
          +
          +
          The takeaway
          + The experiment is not a one-off audit; it is a supply line that keeps the budget model honest, campaign after campaign.
          The loop, in one picture
          -
          The model runs always-on; the experiment disciplines it; the counterfactual reads the experiment out.
          +
          The model runs always-on; the experiments keep correcting it; together they steer the budget.
          @@ -385,13 +498,16 @@

          Price the engagement

          ✋ Poll
          Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over June through November of model-guided spend, cost per lead (CPL) moved by how much?
          - - + + - +
          -
          C. "This year we were able to drive the CPL down by more than 27%, which is very, very good" (Philip Herp, Nürnberger Versicherung). The mechanism is the lesson: under GDPR, customer journeys appeared artificially shortened, so last-touch under-credited the upper funnel and budget followed attribution mechanics instead of incremental business impact. The funnel model measured what video spend causes downstream, and the client is scaling it into full production for 2026.
          +
          C. "This year we were able to drive the CPL down by more than 27%, which is very, very good" (Philip Herp, Nürnberger Versicherung). Under GDPR, customer journeys appeared artificially shortened, so last-touch under-credited the upper funnel and budget followed attribution mechanics instead of incremental business impact. The client is scaling into full production for 2026.
          The client's bar for belief
          "Trust is not created by R² values. It is created when business reality matches model expectations."
          @@ -439,7 +555,7 @@

          Synthetic Control

          Act I · The question

          The boardroom question

          -
          A campaign ran in one region. Sales rose. Marketing wants €4M to go national. You decide. The Colgate question again: what would have happened anyway, now with €4M riding on it.
          +
          A campaign ran in one region. Sales rose. Marketing wants €4M to go national. The Colgate question again: what would have happened anyway?
          @@ -462,15 +578,13 @@

          The boardroom question

          The method this data calls for
            -
          • One treated unit, an observational time series, no A/B test possible.
            • -
          • This is synthetic control's home ground.
            • +
          • One treated unit, no A/B test possible: synthetic control's home ground.
          • It is the engine under Meta's GeoLift and Google's geo methodology.
          The €4M hinge: the transportability discount
          • δ = the share of the pilot's per-euro lift that survives national rollout.
            • -
          • Measuring this one metro is unbiased as the fit uses its own pre-launch record.
            • -
          • The €4M call reduces to one question: is δ large enough for the national rollout to still clear its cost?
            • +
          • The €4M call reduces to one question: is δ large enough to still clear the cost?
          ▸ What "unbiased" means here, precisely, and where it would break

          The estimator subtracts a reconstructed counterfactual from the treated metro's post-launch sales, and bias is its expected gap from the true lift:

          @@ -496,14 +610,14 @@

          The data you actually get

            -
          • \(Y_{jt}\): weekly sales of market \(j\) in week \(t\) (€000): 30 markets (treated metro = market 1, donors \(j=2,\dots,30\)), 60 weeks (\(t=0,\dots,59\)) = 1,800 observed numbers.
            • -
          • The design: the treated metro ran a €75k campaign from week \(T_0=40\) for 20 weeks at a 35% margin; the other 29 ran nothing.
            • -
          • The choice: the firm's marketing team picked the metro and the week. Not us, and not a coin.
            • +
          • \(Y_{jt}\): weekly sales (€000): 30 markets, 60 weeks (\(t=0,\dots,59\)); treated metro = market 1.
            • +
          • The design: a €75k campaign from week \(T_0=40\), 20 weeks, 35% margin; the other 29 ran nothing.
            • +
          • The choice: marketing picked the metro and the week; not a coin.
          The one number you do not have
          - What the treated metro would have sold over those 20 weeks with no campaign. It is nowhere in the file, i.e. the counterfactual.
          + What the metro would have sold with no campaign: nowhere in the file. The counterfactual.
          @@ -550,12 +664,14 @@

          Commit to a number

          ✋ Poll
          Before/after on the treated metro: sales averaged ·/week before launch and ·/week after, a bump of ·/week (about +6%). What is the campaign's true weekly effect?
          - - + +
          -
          C. The true effect is ·/week, a 12% lift on a ·/week baseline. On the previous slide you already saw the naive bump grow once the shared crowd was removed: the first sign it understates. Before/after books barely half the truth because the shared macro wave happened to dip after launch: the tide can inflate a campaign's credit, or, as here, hide it. A naive comparison is not "roughly right plus noise"; it is charged with everything else that changed in the world, in whichever direction that happened to be. Making the counterfactual visible is the whole method.
          +
          C. The true effect is ·/week: a 12% lift on a ·/week baseline. Before/after books barely half of it: the macro wave dipped after launch. The tide can inflate a campaign's credit or hide it; a naive comparison is charged with everything else that changed.
          @@ -566,17 +682,17 @@

          Commit to a number

          Causal inference is a missing-data problem

          -

          Same panel as the data slide, one new idea: the treated metro (market 1) has two week-\(t\) sales, only one of which ever happens. \(Y_{1t}(1)\) with the campaign, \(Y_{1t}(0)\) without. The causal effect is the gap between those two worlds:

          +

          The treated metro has two week-\(t\) sales, only one of which ever happens: \(Y_{1t}(1)\) with the campaign, \(Y_{1t}(0)\) without. The effect is the gap:

          \[ \tau_t \;=\; Y_{1t}(1)-Y_{1t}(0), \qquad \tau \;=\; \sum_{t\ge T_0}\tau_t \]
          • \(\tau_t\) vs \(\tau\): the weekly effect, and the 20-week total the business actually buys.
            • -
          • Consistency: what we record is the realised arm, \(Y_{1t}=Y_{1t}(1)\) after launch and \(Y_{1t}=Y_{1t}(0)\) before it, one world per week, never both.
            • -
          • The one missing cell: \(Y_{1t}(0)\) for \(t\ge T_0\) is never observed, the fundamental problem of causal inference, and every method fills exactly that cell.
            • +
          • Consistency: we record one world per week, never both.
            • +
          • The missing cell: \(Y_{1t}(0)\) for \(t\ge T_0\): every method fills exactly that cell.
          Define the estimand first
          - Per-week effect and 20-week total are different estimands with different uncertainties, and the rollout decision turns on the total.
          + Per-week and 20-week total are different estimands; the €4M turns on the total.
          @@ -587,7 +703,7 @@

          Causal inference is a missing-data problem

          donors · \(Y_{jt}(0)\)observed ✓observed ✓
          -
          Three of four cells are data; the donors' untreated post-period is the live information a synthetic \(Y_{1t}(0)\) is rebuilt from.
          +
          Three cells are data; the donors' post-period is what a synthetic \(Y_{1t}(0)\) is rebuilt from.
          ▸ Reading the notation (observed vs potential)
          • \(Y_{jt}\) from the data slide is what you observe; \(Y_{1t}(1),\,Y_{1t}(0)\) are the treated metro's two potential outcomes.
            • @@ -607,14 +723,14 @@

            Causal inference is a missing-data problem

            The model that generated the data

            We simulate the data: here the counterfactual is known, so every estimator can be graded.
            -

            Each market's no-campaign sales are its level \(\alpha_j\), its exposures \(\gamma_j\) to three shared factors \(f_t\), plus noise: the whole world in one line (all €000).

            +

            Each market: level \(\alpha_j\) + exposures \(\gamma_j^{\top} f_t\) to three shared factors + noise (all €000).

            \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} \;=\; \alpha_j \;+\; \gamma_{j1}\underbrace{\,0.4\,t\,}_{\text{trend}} \;+\; \gamma_{j2}\underbrace{\bigl[\,8\sin(2\pi t/26)+4\sin(2\pi t/13)\,\bigr]}_{\text{seasonality}} \;+\; \gamma_{j3}\underbrace{\textstyle\sum_{s=1}^{t}\eta_s}_{\text{macro walk }m_t} \;+\; \varepsilon_{jt} \]
              -
            • The macro walk \(m_t=\sum_{s\le t}\eta_s\): a running sum of shocks that wanders like an economy, its variance \(t\,\sigma_\eta^2\) growing every week.
              • -
            • The exposures \(\gamma_j\sim\mathrm{U}(1{-}s,1{+}s)\): each market responds its own amount, and the spread \(s\) is what kills parallel trends.
              • +
            • The macro walk \(m_t\): a running sum of shocks that wanders like an economy.
              • +
            • The exposures \(\gamma_j\): each market responds its own amount; the spread \(s\) kills parallel trends.
            ▸ All five ingredients, one by one
              @@ -626,9 +742,9 @@

              The model that generated the data

            Which of these is ever observed?
            - Only \(Y_{jt}\) reaches is observed. The level, loadings, factors and noise (market size, seasonal sensitivity, the economy) are real but recorded nowhere. Every estimator that follows is a way to cope with that.
            + Only \(Y_{jt}\) is observed; levels, loadings, factors and noise are real but recorded nowhere. Every estimator copes with that.
            The planted truth, in business units
            - On top of the world above we switch on a 12% lift at launch week \(T_0=40\): ·/week on average, · over the 20 campaign weeks.
            + A 12% lift switches on at week 40: ·/week, · over 20 weeks. Every method is graded against it.
          ▸ The planted lift, written out
          \[ \tau_t \;=\; 0.12\,\bigl(\alpha_1+\gamma_1^{\top}f_t\bigr)\,\mathbf{1}_{\{t\ge T_0\}} \] @@ -657,22 +773,22 @@

          Simulate the world yourself

          -
          Dashed blue: the treated market's \(Y_{1t}(0)\), the potential outcome itself (not an estimate), which in real data does not exist.
          +
          Dashed blue: \(Y_{1t}(0)\) itself, the line that in real data does not exist.
            -
          • True lift: the planted 12% rate averaged over the 20 post-launch weeks (its euro value varies week to week).
            • +
          • True lift: the planted 12%, averaged over the 20 post weeks.
          • Before/after: the treated series' post-launch mean minus its pre-launch mean.
            • -
          • Its error: the % gap between them, because before/after charges the whole tide to the campaign.
            • +
          • Its error: the tide, charged entirely to the campaign.
          What the sliders teach
            -
          • Loading spread \(s\to 0\): every market reacts alike, trends run parallel, DiD lives.
            • -
          • Macro shock \(\sigma_\eta\) up: the shared wave dominates, before/after becomes garbage.
            • -
          • Noise \(\sigma_\varepsilon\) up: the weekly wobble grows, and any one estimate gets harder to trust.
            • +
          • Spread \(s\to 0\): parallel trends, DiD lives.
            • +
          • Macro \(\sigma_\eta\) up: the tide dominates, before/after dies.
            • +
          • Noise \(\sigma_\varepsilon\) up: every estimate gets shakier.
          @@ -684,11 +800,11 @@

          Simulate the world yourself

          Abadie's idea: if no twin exists, build one

          Idea: manufacture a treated market's twin using a blend of donor markets.
          -

          Assume untreated sales follow the factor structure of the DGP slide, now taken as the assumption rather than the recipe: a level \(\alpha_j\), loadings \(\gamma_j\) on shared latent factors \(f_t\), plus noise (equation in the fold below).

          +

          Assume untreated sales follow the factor structure: level, loadings on shared factors, noise (equation in the fold).

            -
          • \(f_t\), the latent factors: the shared shocks (macro wave rising, season turning) that every market feels, but not equally.
            • -
          • \(\gamma_j\), the loadings: how strongly market \(j\) responds to each factor (\(\alpha_j\) is its level); in real data none of these are observed.
            • -
          • The method never names them: in the sim \(f_t\) is trend + season + macro walk, but on real data it works without knowing what the factors are.
            • +
          • \(f_t\), the latent factors: the shared shocks every market feels, unequally.
            • +
          • \(\gamma_j\), the loadings: how strongly market \(j\) responds; never observed in real data.
            • +
          • The method never names them: it works without knowing what the factors are.

          Blend the donors so their level and loadings equal the treated metro's:

          @@ -697,8 +813,8 @@

          Abadie's idea: if no twin exists, build one

          • Same loadings, same response to every factor movement, including ones that have not happened yet: that forward guarantee is the whole trick.
          -
          Note
          - DiD matches a level while synthetic control matches the response to shocks, which is why it survives the non-parallel trends that break DiD.
          +
          Why it survives storms
          + DiD matches a level; synthetic control matches the response to shocks, so it survives what breaks DiD.
          ▸ The factor equation, pedigree, and the papers' notation
          \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} \] @@ -719,9 +835,9 @@

          The estimator, precisely

          \[ \hat w \;=\; \arg\min_{w\in\Delta}\; \sum_{t \lt T_0}\Bigl(Y_{1t}-\sum_j w_j Y_{jt}\Bigr)^{2}, \qquad \Delta=\Bigl\{w : w_j\ge 0,\ \textstyle\sum_j w_j=1\Bigr\} \]
            -
          • Least squares on the 40 pre-launch weeks: the post-period is never shown to the optimiser, so the fit cannot learn the effect it will measure.
            • -
          • The simplex \(\Delta\): weights are non-negative and sum to one, so the synthetic is an interpolation, never "−80% of Milan + 190% of Rome": the weights-on-the-table version of what Colgate's projection did in time.
            • -
          • No likelihood, no standard error: constrained least squares (SLSQP) ships a point estimate only, and inference comes later, from placebos.
            • +
          • Pre-launch only: the optimiser never sees the post-period, so it cannot cheat.
            • +
          • The simplex \(\Delta\): a readable recipe, never "−80% of Milan + 190% of Rome": Colgate's projection with the weights on the table.
            • +
          • No standard error: a point estimate only; inference comes later, from placebos.
          ▸ Reading the effect off the gap
          @@ -759,13 +875,13 @@

          What the simplex buys, geometrically: stay inside the hull

          • With \(w_j\ge 0\) and \(\sum_j w_j=1\), every point a synthetic market can reach lies inside the shaded convex hull.
            • -
          • Inside the hull: a real blend reproduces the treated market's fingerprint, so a clean pre-period fit keeps working out of sample.
            • -
          • Outside the hull: no non-negative blend reaches it; the simplex stops at the frontier, and only extrapolation (negative weights) closes the gap.
            • +
          • Inside: a real blend reproduces the metro; the fit keeps working out of sample.
            • +
          • Outside: no real blend reaches it; only extrapolation (negative weights) closes the gap.
          Why constrain the weights
          - The simplex forces the synthetic to be an interpolation of markets you actually observed, so a good fit is a real blend, not an extrapolation that happens to line up.
          + A good fit is a real blend of observed markets, not an extrapolation that happens to line up.
          When the treated market sits outside
          - That is the signal to widen the donor pool, not to trust an OLS fit that only matches by leaving the hull. The refusal is a feature. Ask any vendor: when does your method refuse to answer? If never, walk away.
          + Widen the donor pool; do not force a fit. The refusal is a feature. Ask any vendor: when does your method refuse? If never, walk away.
          @@ -797,8 +913,8 @@

          What the constraint buys: drop it and see

          What n_eff = 3.3 tells you
          • The synthetic leans on about three donor markets, not a fuzzy mix of all 29.
            • -
          • Sparse weights keep the blend interpretable and stable out of sample: fewer donors carrying real weight means less room to overfit noise.
            • -
          • It is a health check: near 1 the twin is hostage to a single market, near 29 it is averaging everything into mush. 3.3 is concentrated but not fragile.
            • +
          • Sparse weights: interpretable, and less room to overfit.
            • +
          • Near 1: hostage to one market. Near 29: mush. 3.3: concentrated, not fragile.
          @@ -817,18 +933,18 @@

          "Why not gradient boosting / Prophet / an LSTM?"

          Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
            -
          • In-sample the forecaster fits tighter: pre-launch error €k/week vs the simplex's €k/week. More flexibility always buys a closer fit to the past.
            • -
          • Yet their campaign estimates are a wash: the forecaster reads and the simplex for the 20-week effect, both a touch under the €284k the simulation planted (the truth we can see only because this is a simulation). On this one dataset you cannot say which is better.
            • -
          • The gap shows only in the sweep: across 24 fresh worlds the unconstrained fit reconstructs the missing \(Y_{1t}(0)\) about 1.6× worse out of sample on average, and its weights leave nothing to inspect.
            • +
          • In-sample it fits tighter: €k vs €k per week: flexibility always buys the past.
            • +
          • Yet the estimates are a wash: vs , both near the planted €284k. One dataset cannot rank them.
            • +
          • The sweep decides: across 24 fresh worlds, 1.6× worse out of sample, with nothing to inspect.
          Principle 1: a counterfactual, not a forecast
          - A forecaster asks what comes next in a world like the training data. The campaign asks what this one market would have done in a world that never happened: extrapolation to an unobserved regime, not prediction.
          + A forecaster asks what comes next. We ask what this market would have done in a world that never happened.
            -
          • Principle 2: better in-sample fit is the trap, not the goal. More flexibility means more room to chase noise and extrapolate off-support.
            • -
          • Principle 3: you cannot validate the answer. With one treated unit and one post-period there is no held-out counterfactual, so cross-validation scores prediction error, never the missing cell.
            • -
          • Principle 4: the simplex ships an inspectable claim. Non-negative weights summing to one are an auditable blend of real markets while a boosted tree is a black box.
            • +
          • Principle 2: in-sample fit is the trap, not the goal.
            • +
          • Principle 3: you cannot cross-validate the counterfactual. The truth is never observed.
            • +
          • Principle 4: the simplex ships an auditable claim; a boosted tree is a black box.
          The forecaster is better at prediction. The simplex is better at the causal job.
          @@ -845,10 +961,10 @@

          Compare the estimators

          Four "obvious" estimators, four hidden assumptions
          Each is a different guess at the missing \(Y_{1t}(0)\), unbiased only in its own special world (equations in the fold below).
            -
          • Before/after: unbiased only in a flat world, where the shared factors have equal pre- and post-means.
            • -
          • Treated vs average control: unbiased only if controls match the treated in level and loadings; the level gap survives even at spread \(s\to0\).
            • -
          • Difference-in-differences: removes the level gap, unbiased iff trends are parallel (\(\gamma_1=\bar\gamma\)), which \(s\to0\) delivers.
            • -
          • Synthetic control: unbiased iff a fitted blend matches the treated's loadings (\(\sum_j w_j\gamma_j=\gamma_1\), the treated inside the hull).
            • +
          • Before/after: unbiased only in a flat world.
            • +
          • Treated vs average control: needs matching level and loadings; the level gap survives.
            • +
          • Difference-in-differences: unbiased iff trends are parallel (\(\gamma_1=\bar\gamma\)).
            • +
          • Synthetic control: unbiased iff the blend matches the loadings (treated inside the hull).
          @@ -902,16 +1018,16 @@

          Interesting limits for the simpler estimators

          Before/after · one unit, across time
          The treated metro's post-average minus its pre-average.
            -
          • The level \(\alpha_1\) cancels (same market, subtracted from itself), but nothing subtracts the shared wave.
            • -
          • The full factor drift \(\Delta\bar f\) lands at the metro's own loadings \(\gamma_1\approx1\), not at a mismatch: the largest of the three biases, and it grows with the horizon through the trend term \(0.4\,t\).
            • +
          • The level cancels; nothing subtracts the shared wave.
            • +
          • The whole drift \(\Delta\bar f\) lands on the metro: the largest bias, growing with the horizon.
          Treated-vs-control · one period, across units
          The treated metro minus the control average, both in the post window.
            -
          • Time-differencing is gone, so the level gap \(\alpha_1-\bar\alpha_C\) survives, and market sizes differ a lot: this term dominates.
            • -
          • The loading gap now multiplies the factor level \(\bar f_{\text{post}}\), not its drift: this is why raw sales are never compared across cities.
            • +
          • The level gap \(\alpha_1-\bar\alpha_C\) survives, and sizes differ a lot: it dominates.
            • +
          • This is why raw sales are never compared across cities.
          @@ -935,9 +1051,9 @@

          Interesting limits for DiD

          Where the DiD estimator breaks.
          The one question
          - DiD subtracts the control trend from the treated trend, so it only works if the two would have drifted together. When is that true, and what does it cost when it is not? Two dials answer it (loading spread \(s\), macro shock \(\sigma_\eta\)); the algebra is in the fold.
          + DiD only works if treated and controls would have drifted together. Two dials decide (spread \(s\), macro \(\sigma_\eta\)); the algebra is in the fold.
          Synthetic control solves it
          - The one bias DiD cannot shed, \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\), comes entirely from its equal weights \(\bar\gamma_C\): it must hope the spread \(s\) is small. Synthetic control chooses the weights instead, so \(\sum_j w_j\gamma_j=\gamma_1\) and \(B\to0\) at any spread. The residual DiD lives with, the twin removes by construction.
          + DiD's residual bias \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\) comes from its equal weights: it hopes \(s\) is small. Synthetic control chooses the weights, so \(B\to0\) at any spread.
          ▸ The DiD bias, decomposed (and why more data cannot shrink it)
          \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] @@ -986,25 +1102,25 @@

          What must be true for the gap to be causal

          AssumptionWhat it demandsThe checkVerdict ① Pre-fit quality - The blend tracks the treated metro before launch: pre-RMSE below a third of the weekly lift we must detect. + The blend tracks the metro before launch. The gate: 3.2 vs a bar of 4.4. PASS ② No anticipation - Sales must not react before the campaign (no stockpiling, no leaked launch). - Placebo-in-time: back-date the launch 10 weeks, the fake "effect" is ≈ €1.7k, inside noise. + No reaction before the campaign (no stockpiling, no leak). + Placebo-in-time: the fake launch finds ≈ €1.7k, inside noise. PASS ③ No spillover (SUTVA) - The campaign must not touch donor markets (no travel, no online leakage): Colgate's second-launch leak, in space. + The campaign must not touch donors: Colgate's second-launch leak, in space. No statistical check exists. A design duty: keep test and control regions apart. UNTESTABLE ④ Hull & stable loadings - The treated metro is a feasible mix of donors, and exposures stay put through the window. - Pre-fit gate + leave-one-out: drop any donor, the estimate stays in €239–276k. + The metro is a feasible mix of donors; exposures stay put. + Leave-one-out: drop any donor, the estimate stays in €239–276k. PASS
          The referee principle
          - Three of the four are testable and the verdict is PASS. The one that is not, no-spillover, is exactly why geo-test design matters more than any statistic computed afterwards.
          + Three testable, all PASS: checks ② and ③ are run in Act III, once the luck is priced. The untestable one, no-spillover, is why geo-test design beats any statistic computed afterwards.
          @@ -1015,14 +1131,16 @@

          €260k. Real, or a lucky metro? Build the null yourself

          ✋ Poll
          -
          The usual t-test needs many treated units (its standard error is a spread across treated units); with one treated metro it is not weak, it is undefined. So build the test yourself: run the whole synthetic-control pipeline 29 more times, each time pretending one untreated donor was the treated market ("placebo"). That yields 30 estimated "effects", 29 of them in markets where no campaign ran. Where does the real treated metro's €260k rank among the 30?
          +
          With one treated metro a t-test is not weak, it is undefined. So build the test yourself: rerun the whole pipeline 29 more times, each donor pretended treated ("placebo"): 30 "effects", 29 where nothing ran. Where does our €260k rank?
          - - + +
          -
          A: rank 1 of 30. And the rank is the inference. If the campaign did nothing, the treated metro is just another market, exchangeable with its donors, so P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test, the standard error the t-test could not supply.
          +
          A: rank 1 of 30. The rank is the inference: if the campaign did nothing, P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test.
          @@ -1043,21 +1161,21 @@

          Placebo-in-space: measure the luck directly

          The null hypothesis, stated
          - \(H_0\): the campaign did nothing, \(\tau_t=0\) for every \(t\ge T_0\). Then the treated metro is just one more untreated market, exchangeable with its 29 donors, so its gap is equally likely to hold any of the 30 ranks.
          + \(H_0\): the campaign did nothing. Then the metro is exchangeable with its donors: any of the 30 ranks is equally likely.
          \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \]
          What the p-value means (and what it is not)
          - The probability, if \(H_0\) were true, of a gap as extreme as our €260k where nothing happened: a rank, not a bell-curve tail (no normality, independence, or asymptotics assumed). Nothing beat us, so \(p\) hits its floor \(1/(J+1)=1/30\), set by the donor count, not the weeks of data.
          + The probability, if \(H_0\) were true, of a gap this extreme: a rank, no bell curve assumed. \(p\) sits at its floor \(1/30\), set by the donor count, not the weeks.
          ▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
          • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
          • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
          Conclusion: the €260k lift is real, at \(p\approx0.033\)
          - Our metro ranks 1 of 30, clear of the whole cloud of luck, so we reject \(H_0\): a real campaign effect, not a lucky draw, and the rank holds (p = 0.033 every time) when shaky placebos are dropped at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": that verdict waits for Act IV.
          + Rank 1 of 30, clear of the cloud: reject \(H_0\). The rank holds (p = 0.033 every time) dropping shaky placebos at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": Act IV. @@ -1065,28 +1183,28 @@

          Placebo-in-space: measure the luck directly

          Act III · Is it real, and how big?

          From a test to an interval: inversion

          -
          We measured one number, €260k. Which true lifts could plausibly have produced it? We test every candidate against the placebos and keep the survivors, with no bell curve anywhere.
          +
          Which true lifts could plausibly have produced our €260k? Keep the survivors: no bell curve anywhere.
          -
          ▶ LIVE Drag \(H\), a guess at the truth. Green line = our data, fixed. Blue cloud = the guess, it slides.
          +
          ▶ LIVE Drag \(H\), a guess at the truth: the error cloud slides with it; our €260k never moves.
          -
          Axis: 20-week total gap (€000). A the estimator's error at truth 0 · B that same error slid to \(H\), with its middle 90% · C every \(H\) whose band still covers €260k, collected: the interval. Drag past an edge to reject.
          +
          Axis: 20-week total gap (€000). Top: the placebo errors at truth 0. Middle: the same errors slid to \(H\); \(H\) survives if the green line sits inside the shaded middle 90%. Bottom: the survivors, collected: the interval. Drag past an edge to reject.
          The whole idea
          Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
          -
            -
          • ① Measure the method's error. The 29 donor markets ran no campaign, yet refitting the estimator on each still reports a small fake "effect". Together they make a cloud centred on zero, spread about ±€50k: how far off the method typically lands when the truth is really zero.
            • -
          • ② Guess a true lift \(H\). \(H\) is a candidate for the truth, not our estimate: the dial you turn. If the real lift were \(H\), the method would report \(H\) plus that same error cloud, the blue cloud in the figure slid onto \(H\).
            • -
          • ③ Keep or reject \(H\). Keep \(H\) if our €260k lands inside the middle 90% of its cloud; reject it if €260k falls out in a tail (too far from \(H\) to be believable).
            • -
          • ④ Sweep and collect. Repeat for every \(H\). The smallest and largest that survive are €195k and €335k, so the interval is [€195k, €335k], read straight off the placebos.
            • +
              +
            • ① Measure the error. The 29 placebo "effects" form a cloud around zero, spread about ±€50k.
              • +
            • ② Guess a truth \(H\). If the lift were \(H\), we would see \(H\) plus that same cloud.
              • +
            • ③ Keep or reject. Keep \(H\) if €260k sits inside its middle 90%.
              • +
            • ④ Sweep. The survivors run €195k to €335k: the interval is [€195k, €335k].
          @@ -1108,20 +1226,20 @@

          From a test to an interval: inversion

          Act III · Is it real, and how big? Stress-test the estimate

          Falsification 1: a launch that did not happen

          -
          The anticipation check.
          +
          The anticipation check: unlike Colgate's simulated grading, this needs no known answer, because a launch that has not happened yet must have zero effect.
          How it is computed
          - Refit using only weeks 0-29, pretend the launch happened at week 30, and read the "effect" in weeks 30-39, where the truth is zero.
          + Refit using only weeks 0-29, back-date the launch to week 30, and read the "effect" in weeks 30-39: a window before the real launch, so its true effect is zero by the calendar, not by assumption.
          ▶ LIVE The weekly gap (treated minus synthetic) around a back-dated launch
          -
          Clean world: the gap hugs zero through the fake window (average ≈ €1.7k) and lifts only after the real launch. Tick the box for a leaked launch: the gap climbs before week 40 and the fake window lights up.
          +
          Clean world: the gap hugs zero in the fake window (≈ €1.7k). Tick the box: a leak climbs before week 40.
          What passing rules out
          - Stockpiling, pre-announcement, a leaked launch: anything that makes sales react before the campaign, which a synthetic control that "detects" an effect before treatment would reveal.
          + Stockpiling, pre-announcement, a leaked launch: anything that moves sales before the campaign.
          @@ -1140,17 +1258,17 @@

          Falsification 2: the assumption no placebo can see

          The concrete story
          - Your metro's TV and outdoor ads reach the commuter belt. Those neighbouring markets are in your donor pool. So the campaign you are trying to measure quietly raises the very series you are using as "untreated" comparison.
          + TV and outdoor reach the commuter belt, and those markets are in your donor pool: the campaign quietly raises your "untreated" comparison.
          - \(\varphi\) is the leaked share: to each of 5 neighbour donors we add a fraction \(\varphi\) of the treated metro's own weekly lift \(\Delta_t\), then refit. Lifting the donors lifts the synthetic, so the measured gap shrinks. + \(\varphi\) = the share of the treated lift leaking into 5 neighbour donors, then refit: lifting the donors shrinks the measured gap.
            -
          • The sweep: \(\varphi\in\{0,0.1,0.25,0.5\}\), from a clean world to half the lift leaking. Each point a full re-simulation and refit.
            • -
          • The result: the reported total slides from €260k (clean) to €236k, and the bias is always toward zero.
            • -
          • Dangerous because invisible: the leak starts at launch, so the pre-fit gate stays green and every placebo still passes, leaving no fingerprint to catch.
            • -
          • What drives the decision: because the bias can only attenuate, the number is a conservative floor: if it already clears the cost, spillover cannot overturn the call. The defence is design, keep donors off the treated metro's media footprint.
            • +
          • The sweep: \(\varphi\) from 0 to 0.5: clean world to half the lift leaking.
            • +
          • The result: the reported total slides from €260k (clean) to €236k, always toward zero, exactly how a second launch bent Colgate's counterfactual: a contaminated baseline, now in space instead of time.
            • +
          • Dangerous because invisible: the leak starts at launch; every check stays green.
            • +
          • What drives the decision: the bias only shrinks the number: a conservative floor. The defence is design: donors off the media footprint.
          @@ -1172,7 +1290,7 @@

          Statistics done. Three numbers.

          Truth check (only a simulation allows it)
          The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
          The sentence that loses money
          - "The campaign drove €260k of sales for €75k, a 3.5× return. Roll it out." Every number is true and the conclusion still does not follow: question ③, is the profit worth the price and with what confidence, is not a statistics question.
          + "€260k of sales for €75k, a 3.5× return. Roll it out." Every number true; the conclusion does not follow. Question ③ is not a statistics question. @@ -1185,13 +1303,16 @@

          Now the money

          ✋ Poll
          Cost €75k, incremental sales €260k, gross margin 35%. What did the campaign earn?
          - - + + - +
          -
          C: about €16k net. Profit = margin × lift − cost = 0.35 × 260 − 75 ≈ +€16k. Option A is the boardroom classic: it treats revenue as profit. The campaign returned €3.47 per €1 spent against a break-even of 2.86 at this margin: it paid, in expectation, with room to spare. Whether that expectation carries enough certainty to act on is the question the rest of this act prices.
          +
          C: about €16k net. Profit = margin × lift − cost = 0.35 × 260 − 75 ≈ +€16k. It returned €3.47 per €1 against a break-even of 2.86: it paid, in expectation. With how much certainty is what this act prices.
          @@ -1205,11 +1326,11 @@

          The margin trap

          \[ \text{profit} \;=\; m\,\tau \;-\; c \]
          -
          • Revenue is not cash: the business keeps only the gross margin \(m=35\%\) of the lift \(\tau\) yet pays the cost \(c\) in full, so comparing \(\tau\) to \(c\) treats revenue as profit.
          +
          • Revenue is not cash: you keep the margin \(m=35\%\) of the lift; you pay the cost in full.
          \[ \tau_{\mathrm{BE}} = \frac{c}{m} = \frac{75}{0.35} = \text{€}214\mathrm{k}, \qquad \mathrm{iROAS}_{\mathrm{BE}} = \frac{1}{m} = 2.86 \]
          -
          • Break-even is €214k of sales, not €75k: our €260k clears it by €46k of revenue (≈ €16k of profit), an iROAS of 3.47 against the \(1/m=2.86\) bar.
            • +
            • Break-even is €214k of sales, not €75k: €260k clears it by €46k (≈ €16k of profit): iROAS 3.47 vs the \(1/m=2.86\) bar.
            • Hold onto €214k: it returns as the hinge of the €4M decision.
          @@ -1232,8 +1353,8 @@

          The same evidence at every price

          When is the campaign profitable?
          -
          • Line 1, the point estimate: expected net \(m\hat\tau-c\) crosses zero at \(c=0.35\times260=\text{€}91\mathrm{k}\); below that price it looks profitable on average.
          -
          • Line 2, the uncertainty zone: the profit interval \(0.35\times[195,335]=[\text{€}68\mathrm{k},\text{€}117\mathrm{k}]\), the straddle zone the evidence cannot separate from break-even.
          +
          • Line 1, the point: net \(m\hat\tau-c\) crosses zero at \(c=\text{€}91\mathrm{k}\).
          +
          • Line 2, the straddle zone: the profit interval \([\text{€}68\mathrm{k},\text{€}117\mathrm{k}]\), inseparable from break-even.
          ▶ LIVE Slide the campaign price and watch the verdict move
          @@ -1242,7 +1363,7 @@

          The same evidence at every price

          -
          Between the lines live the dangerous campaigns: "looks profitable" (left of €91k) but not provably so. Ours at €75k is one of them, a comfortable mean the interval refuses to bless.
          +
          Between the lines: "looks profitable" (left of €91k), not provably so. Ours at €75k is one of them.
          @@ -1259,10 +1380,10 @@

          The pilot verdict, delivered to the board

          Three questions about the €75k pilot, all answered by the permutation machinery:

          -
            +
            • ① Real? Yes. Rank 1 of 30, \(p=0.033\).
              • -
            • ② Big enough? Yes, on the point estimate. \(\hat\tau = \text{€}260\mathrm{k}\) against a break-even of €214k: €46k of revenue to spare, about €16k of net profit.
              • -
            • ③ Confident? No. The profit interval [€68k, €117k] still straddles the €75k price. The upside is real but a loss is still inside the evidence.
              • +
            • ② Big enough? On the point, yes: €260k vs break-even €214k: €46k to spare, ≈ €16k net.
              • +
            • ③ Confident? No. Profit [€68k, €117k] still straddles the €75k price.
            Pilot verdict, banked
            At €75k, probably profitable but not a confident buy: renegotiate the price or buy certainty with a bigger test. This says nothing yet about the €4M, which is not the pilot multiplied.
            @@ -1280,7 +1401,7 @@

            The pilot verdict, delivered to the board

          PriceBreak-even liftNet at \(\hat\tau\)VerdictThe lesson
          The one question this memo cannot answer
          - Everything above is a yes/no about where an interval sits: it cannot say "78% chance it pays", name the highest price this evidence would still buy, or whether a follow-up study is worth funding. And it is only about €75k: the €4M rollout is the next two slides.
          + A classical memo is yes/no about where an interval sits: it cannot say "78% chance it pays", price the evidence, or value a follow-up study. And it is only about €75k: the €4M is next. @@ -1304,7 +1425,7 @@

          The €4M is not provably profitable

          -
            +
            • ① The marketing number is a point: 53 pilot-budgets × €260k × 35% margin − €4M ≈ +€860k. Every step correct.
            • ② Carry the interval, and be generous: assume the pilot transports perfectly (\(\delta=1\)). Even then the 90% profit interval is [−€360k, +€2.25M]: it straddles zero.
            • ③ And \(\delta=1\) is a ceiling: €4M is €133k per market (1.78× the pilot, so each euro does less) and one metro is not the nation, so real \(\delta<1\) and the band only sinks.
              • @@ -1350,12 +1471,12 @@

              Measure before you commit

              Even the best case straddled zero, so on this evidence do not commit €4M. The rollout is not dead: the evidence simply cannot size it.
          • Why analysis cannot rescue it: the interval is wide because there is one treated market. Re-running the model does not narrow it; only more treated markets do.
            • -
          • Step zero, first: check the drawer: prior pilots, the media-mix model, agency benchmarks may already narrow the range for the cost of one meeting.
            • +
          • Step zero, first: check the drawer: prior pilots, the media-mix model, benchmarks may already answer it.
          The fix: measure the national effect directly
          - Run the campaign in 8 markets chosen to look like the nation, at full intensity (~€133k each, €1.07M). Because the cells span the country, the measured lift is the national lift, with an interval tight enough to clear or kill the €4M. HelloFresh's calibration loop, bought once.
          + Run it in 8 markets chosen to look like the nation, at full intensity (~€133k each, €1.07M). The cells span the country, so the measured lift is the national lift. HelloFresh's loop, bought once.
          The rule, written before the data lands
          Release the remaining €2.9M only if the measured 90% profit interval clears zero. Written first, or the test is theatre.
          The recommendation
          @@ -1374,16 +1495,790 @@

          Measure before you commit

          - + + +
          +
          +
          +
          Part 3 · The natural experiment
          +

          Instrumental Variables

          +
          What is one ad exposure worth, when the platform already picked who sees the ads?
          +
          +
          + +
          Where this part lands: the dashboard's tight interval misses the true effect of an ad exposure, while the interval built from a lottery covers it, and still clears the price.
          +
          +
          +
          + + +
          +
          IV · The problem
          +

          The case

          +
          One retailer, one ad platform, one decision to make.
          +
          +
          +
          The product
          An online retailer pays an ad platform to show its display ads. One exposure = one user actually saw an ad.
          +
          The price
          Each exposure costs €10, billed by the platform.
          +
          The assignment
          The platform decides, in a real-time auction, which users see the ads.
          +
          The dashboard
          Users who saw an ad brought in €23.7 more than users who did not.
          +
          +
          The platform's pitch
          + "An exposure is worth €23.7 and costs €10. Raise the budget."
          +
          The decision on the table
          + Keep paying €10 per exposure, pay more, or stop. One number settles it: how many euros of extra sales one exposure brings in. All we have is the platform's logs: who saw an ad, and what every user spent.
          +

          The question for the poll: is the dashboard's €23.7 that number?

          +
          +
          + +
          +
          IV · The problem
          +

          Poll · the pitch

          +
          +
          +
          ✋ Poll
          +
          The dashboard reports that users who saw an ad brought in €23.7 more than users who did not, and an exposure costs €10. The platform concludes: "raise the budget". What do you think of that claim?
          +
          + + + + +
          + +
          C. The arithmetic is correct and, at this sample size, comfortably significant, so A and B both miss the point. The platform chose who saw the ads, and it is paid to pick users who already look ready to buy. If the two groups differed before any ad ran, the €23.7 mixes the ad's effect with that pre-existing difference, in unknown proportions. D is backwards: averaging over thousands of users is exactly how noise is handled.
          +
          +
          +
          + +
          +
          IV · The problem · the variables, drawn
          +

          The variables and the confounder

          +
          The three quantities of the case, defined through their causal diagram.
          +
          +
          +

          The platform's logs cover 3,000 customers, indexed by \(i\). Three quantities per customer:

          +
            +
          • \(Y_i\) · sales: contribution euros earned from customer \(i\) in the window (sales net of product cost); "sales" for short. Noisy: the standard deviation across customers is €17.1.
            • +
          • \(X_i\) · exposure: 1 if customer \(i\) saw the ad, 0 if not. Assigned by the platform's auction, not by us.
            • +
          • \(U_i\) · intent: how ready \(i\) was to buy before any ad. Real, and it drives buying, but it is a feeling in a shopper's head: in no log file, never observed.
            • +
          +
          Definition · confounder
          + A confounder is a variable that influences both the treatment and the outcome. Here it is intent \(U\): it raises the chance the platform shows the ad (the targeting), and it raises sales on its own (ready buyers buy, ad or not).
          +
          Definition · backdoor path
          + In the diagram, \(X \leftarrow U \rightarrow Y\) is a backdoor path: a second route connecting exposure and sales that runs behind the treatment. Association flows along it even if the ad does nothing.
          +
          +
          +
          the situation, as a diagram
          +
          +
          \(U\) points at both \(X\) and \(Y\): the fork every claim in this lecture has to get past.
          +
          +
          +
          + +
          +
          IV · The problem
          +

          The naive comparison

          +
          The dashboard's €23.7, written down as a formula.
          +
          +
          +

          Write \(Y_i(1)\) for what customer \(i\) spends with the ad and \(Y_i(0)\) for what the same customer spends without it. Each customer shows us exactly one of the two:

          + + + + + + +
          group\(Y_i(1)\) · with ad\(Y_i(0)\) · without ad
          exposed (\(X_i=1\))observedN/A
          unexposed (\(X_i=0\))N/Aobserved
          +
          \[ \widehat{\Delta}_{\text{naive}} \;=\; \bar{Y}_{\text{exposed}} \;-\; \bar{Y}_{\text{unexposed}} \;=\; \text{€}23.7 \]
          +
            +
          • What it averages: the two observed cells, one from each row. The arithmetic is right.
            • +
          • What it pretends: that the unexposed row's \(Y(0)\) is a fair stand-in for the exposed row's missing \(Y(0)\). Fair only if the two groups are comparable.
            • +
          • The auction works against us: the platform is paid to find likely buyers, so the exposed would have spent more with no ad at all.
            • +
          +
          +
          +
          +
          ▶ LIVE exposed users were already different
          +
          +
          + + +
          +
          The gap compares two groups the auction built to differ. The hidden column shows by how much.
          +
          +
          +
          +
          + +
          +
          IV · The problem
          +

          Poll · read the gap

          +
          +
          +
          ✋ Poll
          +
          The naive comparison gives \(\widehat{\Delta}_{\text{naive}} = \) €23.7. Which reading of that number is defensible?
          +
          + + + + +
          + +
          B. The arithmetic is sound, so C is out. A and D each pick one extreme of the same unknown split: A books all of it to the ad, D books all of it to targeting, and the dashboard alone cannot justify either. The honest statement is the decomposition: €23.7 = the ad's effect + selection bias, with nothing on the dashboard saying how it splits. Measuring that split is the technical problem this lecture solves.
          +
          +
          +
          + +
          +
          IV · The problem · the poll's answer, formalized
          +

          Selection bias

          +
          The decomposition behind the poll's answer.
          +
          +
          +

          Split the naive gap into the two things it adds together:

          +
          \[ \Delta_{\text{naive}} \;=\; \underbrace{\mathbb{E}[Y(1)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}1]}_{\text{effect of the ad on the exposed}} \;+\; \underbrace{\mathbb{E}[Y(0)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}0]}_{\text{selection bias}} \]
          +
            +
          • Read the first term: the exposed users with the ad versus the same users without it. This is what we want to buy, and its second entry is the table's N/A cell.
            • +
          • Read the second term: the gap that would exist with no ads at all, because the picked users differ from the skipped ones.
            • +
          • Here it is positive: the auction picks ready buyers, so the exposed would have out-spent the unexposed anyway.
            • +
          • The dashboard reports only the total: €23.7. Nothing on it says how the total splits.
            • +
          +
          Definition · selection bias
          + When treated and untreated groups differed before the treatment, the difference of their averages includes that pre-existing gap. The pre-existing part is selection bias.
          +
          +
          +
          +
          the dashboard's number is a sum, and the split is invisible
          +
          +
          The dashboard hands us the full bar. Where the boundary sits, nobody can see from the dashboard alone. Finding it is the rest of the lecture.
          +
          +
          +
          +
          + +
          +
          IV · The problem · a world we can grade
          +

          The simulated world

          +
          A world where the true effect is planted, so every method can be graded.
          +
          +
          +
          +
          ▶ LIVE the world the equations generate, as data
          +
          +
          + + +
          +
          Sales distributions for exposed and unexposed users, drawn fresh from the equations. Push \(\kappa\) up, so that intent leaks more strongly into sales, and the two groups drift apart with the planted effect unchanged: the dashboard's gap grows while the truth stands still.
          +
          +
          +
          +
          Why simulate at all
          + Three equations generate the dashboard you saw. We planted an ad effect of \(\beta =\) €15, so every method in this lecture can be graded against a known answer.
          +
          \[ U \sim \mathcal{N}(0,1) \]
          +
          \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \lambda\,U\right) \]
          +
          \[ Y \;=\; \mu + \beta\,X \;+\; \kappa\,U \;+\; \varepsilon \] + \(\mu\): baseline sales  ·  \(\alpha_0\): baseline exposure odds  ·  \(\varepsilon\): noise
          +
            +
          • Interpretation of line 2: who sees an ad rises with intent (\(\lambda U\)): the targeting. \(\sigma\) is the logistic function, turning any score into a probability.
            • +
          • Interpretation of line 3: sales are the ad effect \(\beta\) plus intent leaking straight into spending (\(\kappa U\)).
            • +
          • The math is here for the record. The point is on the left: one hidden cause, \(U\), sits in both equations.
            • +
          +
          +
          +
          + +
          +
          IV · The problem · the disease, stated exactly
          +

          Endogeneity

          +
          The whole problem, stated as one inequality, then shown as a picture.
          +
          +
          +

          Collect everything the sales equation leaves out into a single error term \(v\):

          +
          \[ Y \;=\; \mu + \beta\,X + v, \qquad v \;=\; \kappa\,U + \varepsilon \]
          +
          \[ \operatorname{Cov}(X,\, v) \;>\; 0 \]
          +
            +
          • In words: \(v\) is "every other reason this customer spent money", and exposure is not independent of those reasons, because the auction targets exactly the customers with reasons to spend.
            • +
          • The picture on the right is the inequality: walk up the intent scale and both bars rise together. That joint rise is \(\operatorname{Cov}(X,v) > 0\).
            • +
          +
          Definition · endogeneity
          + A treatment correlated with the other drivers of the outcome is endogenous. Ours is, and everything else in this lecture is a response to that fact.
          +
          The sharp edge
          + The variable driving the correlation was never written to disk.
          +
          +
          +
          +
          one hidden cause pushes both
          +
          +
          Customers grouped by buying intent (which nobody observes). Higher-intent customers see more ads and spend more, ad or no ad. The formula on the left is this picture, in symbols.
          +
          +
          +
          +
          + +
          +
          IV · The problem · the bias, computed
          +

          The size of the bias

          +
          The dashboard's error is not vague. It has a formula, and the formula has three lessons.
          +
          +
          +
          \[ \widehat{\Delta}_{\text{naive}} \;\longrightarrow\; \beta \;+\; \underbrace{\kappa\,\frac{\operatorname{Cov}(X, U)}{\operatorname{Var}(X)}}_{\text{selection bias}} \]
          +
            +
          • Predicted vs delivered: the formula predicts a naive gap near €23.5. The data delivered €23.7, against a truth of €15.
            • +
          +
          What the formula teaches
          + 1. The error is systematic, not bad luck: both factors are positive, so the dashboard can only overstate.
          + 2. It grows with the targeting (\(\lambda\), how strongly intent drives exposure) and with intent's pull on sales (\(\kappa\)): the better the platform targets, the worse the dashboard lies.
          + 3. Sample size is absent: more rows shrink noise, never bias. The dashboard's band, €22.9 to €24.5, is the tightest in this lecture and misses €15 completely.
          +
          What the dashboard measures
          + Not the ad's power to create buyers but the platform's skill at finding them, reported as one number.
          +
          +
          +
          +
          ▶ LIVE the same bar, at different targeting strengths
          +
          +
          + + +
          +
          The dashboard's bar, split by the formula: the true effect (blue) never moves, while the selection bias (orange) grows as intent leaks more strongly into sales. In this simulated world we can draw the boundary, because we planted it.
          +
          +
          +
          +
          + +
          +
          IV · The problem · closing the obvious exit
          +

          The limits of adjustment

          +
          The standard fix works, when you have the column it needs. We do not.
          +
          +
          +
          The standard fix: close the backdoor
          + Compare users at the same level of the confounder. Within a group of equal-intent users, intent no longer separates exposed from unexposed, and any remaining gap belongs to the ad. Controls in a regression, matching, and propensity scores are all versions of this one move.
          +
          +
          ▶ LIVE the fix works, if the column exists
          +
          +
          + + +
          +
          Tick the box: comparing exposed to unexposed within equal-intent groups recovers the planted truth to within half a euro. The method is fine. Untick it: in the real logs the column does not exist, and no amount of modelling recreates it.
          +
          +
          Why the engine stalls here
          + You cannot adjust for what you did not record.
          +
          +
          +

          Two customers, identical on every recorded feature \(W\):

          + + + + + + +
          customerpages, device, history \(W\)intent \(U\)sees the ad?
          Annaidenticalhigh, unrecordedpicked
          Beaidenticallow, unrecordedskipped
          +
          what conditioning blocks, and what it cannot
          +
          +
          Holding \(W\) fixed (the box) blocks the backdoors that run through it. The arrows themselves never move: conditioning blocks paths, it does not delete causes. The path through \(U\) stays open, because \(U\) is in no file to hold fixed.
          +
          +
          +
          + +
          +
          IV · The problem
          +

          Poll · what would fix it?

          +
          +
          +
          ✋ Poll
          +
          Adjustment failed because intent was never recorded. Your team can request one addition to the data. Which one would actually let you measure what an exposure causes?
          +
          + + + + +
          + +
          C. A tells you the wrong number more precisely: bias is not noise, and the formula for it does not contain the sample size. B and D are more controls, and the auction reacts to live signals that no exported feature set fully contains, so the backdoor stays open. C is different in kind: it changes who assigns the treatment, and the assignment was the disease all along. The rest of the lecture is about getting C, or the closest thing to it the system already contains.
          +
          +
          +
          + + +
          +
          IV · The idea · what we wish we could do
          +

          The ideal experiment

          +
          What randomizing exposure would buy, and why we cannot do it.
          +
          +
          +
          randomization cuts the arrow that causes the trouble
          +
          +
          Left: today's world, where intent drives both exposure and sales. Right: the experiment we wish we could run, where a randomizer assigns exposure and intent's arrow into it is removed, not merely blocked: exposure no longer listens to intent at all. With that arrow gone, the naive comparison becomes the right comparison.
          +
          \[ X \text{ randomized} \;\;\Rightarrow\;\; \operatorname{Cov}(X,\, v) = 0 \;\;\Rightarrow\;\; \widehat{\Delta}_{\text{naive}} \longrightarrow \beta \]
          +
          +
          +
            +
          • The wish: option C from the poll: show the ad to a random half ourselves.
            • +
          • The wall: the platform's auction decides exposure in milliseconds, using exactly the intent signals that cause the problem. It will not hand us the controls.
            • +
          +
          The idea that saves us
          + If we cannot inject randomness into exposure, find the randomness the system already contains and use only that part.
          +
            +
          • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
            • +
          +
          The plan for Part 2
          + 1. Give that random lever a name: an instrument.
          + 2. State the conditions it must satisfy.
          + 3. Check which of them the data can verify.
          +
          +
          +
          + +
          +
          IV · The idea · the randomness we ran
          +

          The lottery

          +
          The randomness our system already contains, because we put it there.
          +
          +
          +
            +
          • What we ran: before the campaign, a serving-priority lottery. Our own random number generator picked half the users and gave them a small priority boost in the platform's ad auction.
            • +
          • What it does: the lottery shows nobody an ad. It only raises the odds of seeing one.
            • +
          • A new column joins the data:
            • +
          + + + + + +
          quantitymeaningtypewho assigns itobserved?
          \(Z_i\) · the lottery1 if the lottery boosted \(i\)'s priority in the ad auctionbinarya random number generator we controlobserved
          +
          \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \gamma\,Z + \lambda\,U\right) \] + the simulated world ran the lottery too: the exposure equation gains \(\gamma Z\), the lottery's push on the odds of seeing an ad
          +
            +
          • Why it matters: \(Z\) moves exposure, and nothing else about the customer. It is the one source of variation in \(X\) that the targeting cannot touch.
            • +
          +
          +
          +
          the fork, and the way around it
          +
          +
          Top: the confounded fork. Bottom: the lottery \(Z\) pushes on \(X\) from outside. No arrow from \(U\) into \(Z\) (it is a random draw), and no arrow from \(Z\) straight to \(Y\) (a queue bump shows the user nothing).
          +
          +
          +
          + +
          +
          IV · The idea · the central definition
          +

          The instrument

          +
          What we just built has a name.
          +
          +
          Definition · instrument
          + An instrument \(Z\) moves the treatment, is as good as random with respect to the hidden confounder, and affects the outcome only through the treatment. Four conditions make that precise, and they grade our lottery as follows.
          + + + + + + + + + + + + + +
          conditionthe claimcan it be checked?what it buys
          1 · Relevancethe lottery actually moves exposure \(X\)testable measured by the first stagea push that really happened
          2 · Exogeneitythe lottery is blind to intent \(U\)by design if the draw is genuinewith 1: a clean experiment on the lottery
          3 · Exclusionthe lottery touches sales only through exposureuntestable argued, never checkedturns the lottery's effect into the ad's
          4 · Monotonicitythe lottery never blocks an exposure that would have happened without ituntestable argued from the mechanismsays whose effect we measured
          +
          An instrument is the opposite of a control
          + You do not hold \(Z\) fixed. You use the variation it created.
          +

          The genius is never the estimator, it is spotting the randomness your system already contains: a rollout order, a capacity limit, an upstream A/B test, a rounding rule.

          +
          +
          + +
          +
          IV · The idea · condition 1, measured
          +

          Condition 1 · relevance

          +
          The first stage: did the lottery actually move exposure?
          +
          +
          +

          Relevance is a claim about the data, so the data can answer it.

          +
          Definition · first stage
          + The first stage is the effect of the instrument on the treatment: how much exposure the lottery itself created. It is the coefficient \(\pi\) in the regression below.
          +
          +
          \[ X \;=\; b_0 + \pi\,Z + u \] + \(\pi\): the first stage  ·  \(b_0\): exposure rate among lottery losers  ·  \(u\): every other driver of exposure, intent included
          +
          +
          +
          \[ \pi \;=\; \Pr(X{=}1 \mid Z{=}1) \;-\; \Pr(X{=}1 \mid Z{=}0) \;=\; 0.2106 \] + the measured first stage: what the lottery did to exposure
          +
            +
          • Interpretation of \(\pi\): winning the lottery raises a user's chance of seeing an ad by +21 percentage points, from 56% to 77%.
            • +
          • It is causal, full stop: \(Z\) is random, so nothing hidden can explain the step.
            • +
          +
          +
          +
          exposure rate, by the lottery
          +
          +
          The step between the bars is the first stage \(\pi\): the slice of exposure that the lottery, not targeting, created.
          +
          The gate: is the push strong enough?
          + Like every estimate in this lecture, \(\pi\) comes with a significance test: the first-stage \(F\). Trust the coming division only when \(F > 10\), well past bare significance, because a \(\pi\) that is merely nonzero still leaves the division unstable. Here \(F = 156\): the push is unmistakably real. We quote such tests, not derive them.
          +
          +
          +
          + +
          +
          IV · The idea · conditions 2 and 3
          +

          Conditions 2 and 3 · exogeneity and exclusion

          +
          The lottery must be blind to intent, and silent about sales.
          +
          +
          +
          Blind · exogeneity
          + The lottery must not favour high-intent users. Ours is a genuine randomizer, so this holds by design.
          +
          Silent · exclusion
          + The lottery must not touch sales through any channel except the ad itself. A side-channel (faster pages for boosted users, a shown price) would poison the method:
          +
          \[ \hat\beta_{\text{IV}} \;=\; \beta + \frac{s}{\pi} \] + \(\hat\beta_{\text{IV}}\): the estimate the lottery-based method reports  ·  \(\beta\): the true effect of one exposure on sales, the number we are hunting  ·  \(s\): the side-channel's own lift on sales  ·  \(\pi\): the first stage
          +
          No test will catch it
          + Exclusion is a claim about a path that should not exist, so no output can verify it. You defend it by design.
          +
          +
          +
          ▶ LIVE break the silence on purpose
          +
          +
          + + +
          +
          What the method reports, split into the true effect (blue) and the contamination \(s/\pi\) (red). Slide \(s\) up: a small leak becomes a large error.
          +
          +
            +
          • Why a tiny leak is a big error: the side-channel \(s\) gets divided by the small first stage \(\pi\), so even €1 of leak moves the estimate by €1 / 0.2106.
            • +
          • Our defence: a queue bump is content-free, sells nothing, and the user never learns it happened.
            • +
          • The counterexample: an email with a discount code would move spending on its own, and everything after this slide would collapse.
            • +
          +
          +
          +
          + + +
          +
          IV · The estimator · the second ingredient
          +

          The reduced form

          +
          The same comparison as the first stage, now for sales.
          +
          +
          +
          Definition · reduced form
          + The reduced form is the effect of the instrument on the outcome: the sales gap between the lottery groups. It is the \(\delta\) measured below.
          +
          \[ \delta \;=\; \mathbb{E}[Y \mid Z{=}1] \;-\; \mathbb{E}[Y \mid Z{=}0] \;=\; \text{€}3.48 \] + \(\delta\): the reduced form  ·  \(\beta\): the true effect of one exposure on sales, still the number we are hunting
          +
            +
          • Interpretation of \(\delta\): winning the lottery adds €3.48 of sales for the average user. Causal for the same reason \(\pi\) was: a random draw cannot favour ready buyers.
            • +
          • The intuition that carries the day: the lottery can only reach sales through the ad. One lottery win buys \(\pi\) extra exposures, and each exposure is worth \(\beta\) euros. So the lottery's effect on sales must be \(\beta \times \pi\):
            • +
          +
          \[ \delta \;=\; \beta \times \pi \]
          +
          Both numbers are correct. Neither answers the question.
          + \(\pi\) is exposures per lottery win and \(\delta\) is euros per lottery win, but the client asked for euros per exposure. Since \(\delta = \beta\pi\), one step is left.
          +
          +
          +
          average sales, by the lottery
          +
          +
          The step is small but clean: a random draw cannot favour ready buyers, so selection cannot explain it.
          +

          For the record, the algebra behind the intuition: substitute \(X = b_0 + \pi Z + u\) into the sales equation and the coefficient multiplying \(Z\) is exactly \(\beta\pi\).

          +
          +
          +
          + +
          +
          IV · The estimator · the whole method in one division
          +

          The IV estimate

          +
          Euros per lottery win, divided by exposures per lottery win.
          +
          +
          +
          the whole method, as arithmetic on two measured numbers
          +
          +
          One lottery win buys 0.2106 extra exposures and €3.48 of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
          +
          +
          +
          +
          \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ 3.48 }{ 0.2106 } \;=\; \text{€}16.5 \]
          +
            +
          • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
            • +
          • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
            • +
          +
          +
          +
          What just happened
          + We priced the ad using only the random slice of exposure: the dashboard said €23.7, the lottery says €16.5, against a planted truth of €15.
          +

          The method never needed a model of intent, controls, or machine learning: two averages and a division.

          +
          +
          +
          ▸ Deep dive · the confidence interval around €16.5
          +
          +
          +

          \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

          +
          \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; 16.5 \;\pm\; 1.645 \times 2.31 \;=\; [\,12.7,\; 20.4\,] \] + the 90% confidence interval: the range of effect sizes the data support
          +
            +
          • Interpretation of the interval: any effect between €12.7 and €20.4 is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
            • +
          • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
            • +
          +
          +
          +
          two intervals, graded against the planted truth
          +
          +
          The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
          +
          +
          +
          +
          +
          + +
          +
          IV · The estimator · hands on
          +

          Why the division is forced

          +
          The division is not a modelling choice. It is the only effect size the two measurements allow.
          +
          +
          +
          ▶ LIVE every candidate effect makes a prediction. One matches.
          +
          +
          + + +
          +
          The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €3.48. Move your guess to the crossing and you have priced the ad.
          +
          +
          +

          Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

          +
            +
          • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's 0.2106 extra exposures per win should create \(\hat\beta \times 0.2106\) euros per win.
            • +
          • The fact: the lottery actually created €3.48 per win.
            • +
          • The verdict: every candidate except €16.5 contradicts a number we measured. The division is the only survivor, not a choice.
            • +
          +
          \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
          +
          Not a black box
          + Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
          +
          +
          +
          + + +
          +
          IV · When it breaks · the dangerous failure
          +

          Weak instruments

          +
          A weak instrument is worse than no instrument.
          +
          +
          +
          ▶ LIVE what the estimator does as the first stage dies
          +
          +
          + + +
          +
          Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
          +
          +
          +

          The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

          +
            +
          • The honest failure: the interval balloons, and the data admit they know little.
            • +
          • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
            • +
          • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
            • +
          +
          Report the first-stage \(F\), always
          + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
          +
          +
          +
          ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
          +
          +
          +
          ▶ LIVE test each candidate \(\beta_0\), keep the survivors
          +
          +
          + + +
          +
          The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [12.6, 20.2]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
          +
          +
          +
            +
          • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
            • +
          • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
            • +
          +
          \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
          +
            +
          • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
            • +
          • Here, a good sign: the lottery is strong, so AR [12.6, 20.2] nearly matches the usual [12.7, 20.4]. When \(F\) is small the two part company, and AR is the one still telling the truth.
            • +
          +
          +
          +
          +
          +
          + +
          +
          IV · When it breaks · whose effect it is
          +

          Compliers and the LATE

          +
          Whom does the €16.5 describe? Only the users the lottery could move.
          +
          +
          ▶ LIVE the user base, split by how they respond to the lottery
          +
          +
          + + +
          +
          Push \(\gamma\): the complier slice grows, because the complier share is the first stage. Shares are drawn live from one simulated batch, so they can differ from the printed figures by a rounding step.
          +
          +
          +
            +
          • Always-takers (56.5%): the auction shows them the ad with or without the lottery. It changes nothing for them, so the data say nothing about them.
            • +
          • Never-takers (22.5%): never see the ad either way. Same silence.
            • +
          • Compliers (21.0%): see the ad only because the lottery favoured them. Every euro of the lottery's lift \(\delta\) came from them.
            • +
          • The caveat, monotonicity: we assume no defiers, users who would see the ad only when the lottery does not favour them. A nudge that never repels.
            • +
          +
          +
          +
          Definition · LATE (local average treatment effect)
          + The LATE is the average effect of the ad on the compliers alone, and it is what the division estimates: a local answer, not a statement about every user.
          +
          \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;\;\text{ estimates }\;\; \mathbb{E}[\,Y(1)-Y(0)\mid \text{complier}\,] \] + the average of each complier's personal effect, the same \(Y(1)-Y(0)\) contrast the naive slide could not touch
          +
          +
          +
          Why a manager should love this fine print
          + Compliers are the same kind of marginal user a higher bid would newly reach: the closest thing in the data to the customer a bid change buys. That makes €16.5 a price for the marginal customer, measured on the margin rather than on the average.
          +
          +
          + +
          +
          IV · When it breaks · what must hold, on one page
          +

          The checklist

          +
          The four assumptions, and which ones the data can check.
          +
          + + + + + + + + +
          AssumptionWhat it saysIn the caseStatus
          Relevance\(Z\) moves \(X\)\(F = 156\), far above 10TESTABLE, passes
          Exogeneity\(Z \perp U\)the lottery is a genuine random drawBY DESIGN
          Exclusion\(Z \to Y\) only via \(X\)a queue bump shows the user nothingUNTESTABLE
          Monotonicityno defiersa nudge never repelsUNTESTABLE, plausible
          +
          +
          +
          The honest scorecard, for any IV study you are shown
          + One measured number (the first-stage \(F\)), one design guarantee (the randomization), two arguments (exclusion, monotonicity). Ask for all four before you accept the estimate.
          +
            +
          • What we can now defend: an exposure causes about €16.5 of sales for the users a bid can actually move.
            • +
          +
          +
          The sentence that loses money
          + "Exposed users are worth €23.7 each, so raise the bid": every word true, conclusion wrong. It books the platform's targeting as advertising.
          +
          +
          +
          + + + +
          +
          IV · The decision · euros at last
          +

          The price map

          +
          The estimate becomes a decision only when it meets the price.
          +
          +
          +
          ▶ LIVE the verdict as the price moves
          +
          +
          + + +
          +
          Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
          +
          +
          +

          One estimate gives not one answer but a map from any price to a verdict:

          + + + + + + + +
          Price zoneVerdictWhy
          below €12.7GOeven the most pessimistic supported effect pays
          €12.7 to €20.4TESTthe data straddle the price: negotiate, or measure more
          above €20.4NO-GOno supported effect pays
          +
            +
          • Today's rate, €10, sits in the GO zone, below the whole interval.
            • +
          • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €16.5 − €10 = €6.5. Even read at the interval floor, €12.7 against €10, the exposure still pays.
            • +
          +
          Why boards like this framing
          + "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
          +
          +
          +
          + +
          +
          IV · The decision
          +

          Poll · the negotiation

          +
          +
          +
          ✋ Poll
          +
          The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €16.5 per exposure, 90% interval [12.7, 20.4]. What is the highest rate at which you would still sign "buy" without further study?
          +
          + + + + +
          + +
          C. Below €12.7, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
          +
          +
          +
          + +
          +
          IV · The decision · banked
          +

          The verdict and the recommendation

          +
          The complete answer, assembled from everything measured so far.
          +
          +
          + + + + + + + + + +
          QuantityValueSource
          effect of one exposure€16.5the division δ/π
          90% interval[12.7, 20.4]classical, and AR agrees
          first-stage F156the lottery is strong
          price€10the platform's rate card
          net per exposure€6.5β − c, at the point estimate
          +
          The verdict
          + BUY  Keep buying at €10: the entire defensible range clears the price.
          +
          +
          +
          The recommendation, in three lines
          + 1. Keep buying at the €10 rate: even the interval's most pessimistic effect pays.
          + 2. Cap the bid at the interval's lower end, €12.7: up to there, every effect the data support still clears the price.
          + 3. Measure again only if the rate card climbs toward €12.7: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
          +
            +
          • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
            • +
          • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
            • +
          +
          +
          +
          + +
          Closing · Provenance

          The tools were the product too

            -
          • CausalPy: synthetic control, interrupted time series, difference in differences, and regression discontinuity in one open-source package: the method you just learned and its quasi-experimental family, industrialized by PyMC Labs. The IV estimator that closes this session joined the package later.
            • +
          • CausalPy: synthetic control, interrupted time series, difference in differences, and regression discontinuity in one open-source package. The IV estimator that closes this session joined later.
          • Its launch example: individual exposure to a TV campaign cannot be randomised, yet its causal impact remains a core business need: the sentence this whole session opened with.
          • pymc-marketing: the MMM library behind Case 2's calibration story; one client's budget allocation approach to PyMC-Marketing came back as a pull request (Bolt).
            • -
          • Webinars and content: the consultancy's own webinar walks geo-experimentation, MMM, synthetic control, difference-in-differences, regression discontinuity and Instrumental Variables: this session's syllabus.
            • +
          • Webinars and content: the consultancy's own webinar agenda is this session's syllabus, Instrumental Variables included.
          CausalPy @@ -1470,7 +2365,7 @@

          Every number, pinned