diff --git a/causal-marketing-pymc/Makefile b/causal-marketing-pymc/Makefile index f448423..ced052c 100644 --- a/causal-marketing-pymc/Makefile +++ b/causal-marketing-pymc/Makefile @@ -80,6 +80,10 @@ html-iv-slides: ## build the Ch.13 interactive slide deck (apps/iv_slides.html) # /*__DATA__*/, inlines MathJax. Edit apps/iv_slides_src.html, then rebuild. $(CORE)/python apps/build_iv_slides.py +html-iv-slides-sh: ## build the management-first rework of the Ch.13 deck (apps/iv_slides_sh.html) + # Same pipeline as html-iv-slides, different template/output. Edit apps/iv_slides_sh_src.html. + $(CORE)/python apps/build_iv_slides_sh.py + html-geo: ## build the standalone Ch.9 slide deck (apps/geo_lift_slides.html) — one offline file # Assembles apps/geo_slides_src.html: shard-token substitution (numbers never retyped), # the nb07 lecture bundle for the live charts, book figures inlined as PNGs, MathJax vendored. @@ -98,6 +102,13 @@ html-geo-sh: ## build the SHORT Ch.9 deck (apps/geo_lift_sh.html) — classical $(CORE)/python apps/build_geo_slides.py --src apps/geo_slides_sh_src.html --out apps/geo_lift_sh.html $(CORE)/python apps/verify_geo_deck.py --deck apps/geo_lift_sh.html --allow-missing-slides +html-unified: ## build the UNIFIED session deck (apps/unified_slides.html) — labs hook + short geo arc + # Part 1 (real PyMC Labs engagements) in front of the short classical geo deck, one file. + # Tokens from BOTH sources ({{labs.*}} pins + {{nb07*.*}} shards), one merged DATA bundle. + # Edit apps/unified_slides_src.html, then rebuild. Narrative: apps/geo_lift_lecture_emphasis.md. + $(CORE)/python apps/build_unified_slides.py + $(CORE)/python apps/verify_unified_deck.py + site: ## export the WASM-safe apps to static, in-browser pages under site/ $(CORE)/marimo export html-wasm apps/experiment_design.py -o site/experiment_design --mode run $(CORE)/marimo export html-wasm apps/iv_lecture.py -o site/iv_lecture --mode run diff --git a/causal-marketing-pymc/apps/build_geo_slides.py b/causal-marketing-pymc/apps/build_geo_slides.py index a91d5b3..07f9305 100644 --- a/causal-marketing-pymc/apps/build_geo_slides.py +++ b/causal-marketing-pymc/apps/build_geo_slides.py @@ -126,24 +126,57 @@ def naive_grid() -> dict: "diff-in-differences", "synthetic control"]} -def main(src: Path = SRC, out: Path = OUT) -> None: - html = src.read_text() - tokens = load_tokens() - - # 1 · scalar tokens ------------------------------------------------------------------ - used, missing = set(), [] - def sub_token(m: re.Match) -> str: - key = m.group(1).strip() - if key not in tokens: - missing.append(key) - return m.group(0) - used.add(key) - return tokens[key] - html = re.sub(r"\{\{([a-z0-9_.]+)\}\}", sub_token, html) - if missing: - sys.exit("FAIL: unknown tokens (not in the shards): " + ", ".join(sorted(set(missing)))) - - # 2 · the data bundle ---------------------------------------------------------------- +def dgp_parts(labels_top: list, treated: list, donors_top: list) -> dict: + """Bake the seed-5 STANDARDIZED draws behind cmp.dgp.geo_panel so the DGP-sandbox + slide can re-assemble the SAME world at any dial setting. + + geo_panel is exactly linear in every dial the slide exposes — normal(0, sd) = sd*z, + uniform(1-s, 1+s) = 1 + s*u, and the lift enters as lift*(level + g.F) — so shipping + the unit variates (z, u, e) plus the levels lets the browser rebuild + Y_jt = a_j + (1+s*u_j).F_t(sigma_eta) + sigma_eps*e_jt for arbitrary dials. At the case + dials that reconstruction IS the seed-5 panel (asserted below), so the picture deforms + continuously from the case instead of jumping to a different-seed toy world. + Only the markets the deck draws are shipped: treated first, then the top donors in + the bundle's own donor_labels_top order. + """ + import numpy as np + from cmp import dgp + + n_weeks, n_dmas, launch = 60, 30, 40 + # replicate geo_panel(seed=5)'s draw stream call-for-call, recording unit variates + rng = np.random.default_rng(5) + z = rng.normal(0, 1.2, n_weeks) / 1.2 # unit macro increments + levels = rng.uniform(80, 140, n_dmas) + u = (rng.uniform(0.6, 1.4, (n_dmas, 3)) - 1.0) / 0.4 # unit loading deviations + e = rng.normal(0, 3, (n_dmas, n_weeks)) / 3.0 # unit noise + zc = np.cumsum(z) + + # the reconstruction at the case dials must equal geo_panel(seed=5) to float precision + t = np.arange(n_weeks) + trend = 0.4 * t + season = 8 * np.sin(2 * np.pi * t / 26) + 4 * np.sin(2 * np.pi * t / 13) + F = np.column_stack([trend, season, 1.2 * zc]) + loads = 1.0 + 0.4 * u + sales = levels[:, None] + loads @ F.T + 3.0 * e + eff = 0.12 * (levels[0] + loads[0] @ F.T) * (t >= launch) + df, eff_ref, launch_ref, lab = dgp.geo_panel(seed=5) + assert launch_ref == launch and np.allclose(eff, eff_ref, atol=1e-9) + assert np.allclose(sales[0] + eff, df[lab].values, atol=1e-9) + assert np.allclose(sales[1:], df.values.T[1:], atol=1e-9) + + # and match the bundle's (rounded-to-0.1) shipped series within rounding + idx = [0] + [int(l.split("_")[1]) for l in labels_top] + assert np.abs(sales[0] + eff - np.asarray(treated)).max() < 0.051 + for k, l_idx in enumerate(idx[1:]): + assert np.abs(sales[l_idx] - np.asarray(donors_top[k])).max() < 0.051 + + r = lambda a: np.round(np.asarray(a), 5).tolist() + return {"levels": r(levels[idx]), "u": r(u[idx]), "e": r(e[idx]), "zc": r(zc)} + + +def build_bundle() -> dict: + """Assemble the full DATA bundle (nb07 lecture bundle + shard scalars + aliases + + real-data bundle + extras + the naive grid). Shared with build_unified_slides.py.""" if not BUNDLE.exists(): sys.exit(f"FAIL: {BUNDLE} missing — re-execute nb07 (its lecture-bundle cell writes it).") bundle_meta = json.loads(BUNDLE.read_text()) @@ -196,25 +229,62 @@ def sub_token(m: re.Match) -> str: print(f"naive grid: {len(bundle_meta['naive_grid']['s'])}×" f"{len(bundle_meta['naive_grid']['sd'])} cells from cmp.dgp.geo_panel") + # the DGP-sandbox slide: the seed-5 unit draws, so the live figure re-assembles the + # SAME world at off-case dials instead of re-simulating a different-seed toy world. + bundle_meta["dgp_parts"] = dgp_parts(bundle_meta["donor_labels_top"], + bundle_meta["treated"], + bundle_meta["donor_series_top"]) + print(f"dgp parts: {len(bundle_meta['dgp_parts']['levels'])} markets' seed-5 unit draws") + return bundle_meta + + +def inject_data(html: str, bundle_meta: dict) -> str: + """Replace the JSON object after the /*__DATA__*/ marker with the baked bundle.""" if "/*__DATA__*/" not in html: sys.exit("FAIL: template has no /*__DATA__*/ marker.") j = html.index("{", html.index("/*__DATA__*/")) _, end = json.JSONDecoder().raw_decode(html, j) - html = html[:j] + json.dumps(bundle_meta, separators=(",", ":")) + html[end:] + return html[:j] + json.dumps(bundle_meta, separators=(",", ":")) + html[end:] - # 2b · the N map: formatted number strings the deck's JS writes into prose spans. - # The template's /*__NUMS__*/{...}/*__ENDNUMS__*/ map supplies the KEYS; every value is - # re-derived from the shards (nb07.), so a number can no more go stale here than in - # the {{token}} prose layer. A key missing from the shards is a build error. + +def inject_nums(html: str, tokens: dict[str, str]) -> str: + """The N map: formatted number strings the deck's JS writes into prose spans. + The template's /*__NUMS__*/{...}/*__ENDNUMS__*/ map supplies the KEYS; every value is + re-derived from the shards (nb07.), so a number can no more go stale here than in + the {{token}} prose layer. A key missing from the shards is a build error.""" nm = re.search(r"/\*__NUMS__\*/(\{.*?\})/\*__ENDNUMS__\*/", html, re.S) - if nm: - n_keys = list(json.loads(nm.group(1))) - missing_n = [k for k in n_keys if f"nb07.{k}" not in tokens and f"nb07b.{k}" not in tokens] - if missing_n: - sys.exit("FAIL: N-map keys not in the shards: " + ", ".join(missing_n)) - nmap = {k: tokens.get(f"nb07.{k}", tokens.get(f"nb07b.{k}")) for k in n_keys} - html = html[:nm.start(1)] + json.dumps(nmap, separators=(",", ":")) + html[nm.end(1):] - print(f"N map: {len(nmap)} formatted numbers injected from shards") + if not nm: + return html + n_keys = list(json.loads(nm.group(1))) + missing_n = [k for k in n_keys if f"nb07.{k}" not in tokens and f"nb07b.{k}" not in tokens] + if missing_n: + sys.exit("FAIL: N-map keys not in the shards: " + ", ".join(missing_n)) + nmap = {k: tokens.get(f"nb07.{k}", tokens.get(f"nb07b.{k}")) for k in n_keys} + html = html[:nm.start(1)] + json.dumps(nmap, separators=(",", ":")) + html[nm.end(1):] + print(f"N map: {len(nmap)} formatted numbers injected from shards") + return html + + +def main(src: Path = SRC, out: Path = OUT) -> None: + html = src.read_text() + tokens = load_tokens() + + # 1 · scalar tokens ------------------------------------------------------------------ + used, missing = set(), [] + def sub_token(m: re.Match) -> str: + key = m.group(1).strip() + if key not in tokens: + missing.append(key) + return m.group(0) + used.add(key) + return tokens[key] + html = re.sub(r"\{\{([a-z0-9_.]+)\}\}", sub_token, html) + if missing: + sys.exit("FAIL: unknown tokens (not in the shards): " + ", ".join(sorted(set(missing)))) + + # 2 · the data bundle, 2b · the N map ----------------------------------------------- + html = inject_data(html, build_bundle()) + html = inject_nums(html, tokens) # 3 · book figures ------------------------------------------------------------------- fig_names = sorted(set(re.findall(r"", html))) diff --git a/causal-marketing-pymc/apps/build_iv_slides_sh.py b/causal-marketing-pymc/apps/build_iv_slides_sh.py new file mode 100644 index 0000000..4b36c58 --- /dev/null +++ b/causal-marketing-pymc/apps/build_iv_slides_sh.py @@ -0,0 +1,19 @@ +"""Generate apps/iv_slides_sh.html, the management-first rework of the Ch. 13 IV deck. + +Same pipeline as build_iv_slides.py (tokens from the executed shards, DATA bundle at +/*__DATA__*/, vendored MathJax at /*__MATHJAX__*/), different template and output: + + Template: apps/iv_slides_sh_src.html (editable source) + Output: apps/iv_slides_sh.html (generated, self-contained) + +Build: .venv/bin/python apps/build_iv_slides_sh.py +""" +from __future__ import annotations + +import build_iv_slides as base + +base.SRC = base.HERE / "iv_slides_sh_src.html" +base.OUT = base.HERE / "iv_slides_sh.html" + +if __name__ == "__main__": + base.main() diff --git a/causal-marketing-pymc/apps/build_unified_slides.py b/causal-marketing-pymc/apps/build_unified_slides.py new file mode 100644 index 0000000..4a4acb8 --- /dev/null +++ b/causal-marketing-pymc/apps/build_unified_slides.py @@ -0,0 +1,113 @@ +"""Generate the UNIFIED SDA session deck: apps/unified_slides.html. + +Part 1 ("Causal Inference in the Wild": real PyMC Labs engagements, the business hook) +spliced in FRONT of the short classical geo deck (synthetic control, Acts I-IV), then +Part 3 (the short instrumental-variables deck) after the geo Act IV closer, closing with +the labs synthesis slides. One file, one chrome (the geo deck's, a strict superset), one +DATA bundle. The narrative this deck implements: apps/geo_lift_lecture_emphasis.md; +the spoken script: apps/geo_lift_lecture_speech.md. + +Inputs, none hand-typed: + tokens : {{nb07*.*}} from the executed geo shards (build_geo_slides.load_tokens), + {{labs.*}} from apps/labs_deck_data.json (blog-pinned facts with source URLs; + build_labs_slides.load_pins), and {{nb11*.*}} from the executed IV shards + (build_iv_slides.load_tokens). The three prefixes are disjoint (asserted). + DATA : the geo bundle (build_geo_slides.build_bundle) merged with the labs chart keys + (deterministic counterfactual schematic + pinned ROAS pair; disjoint, asserted), + plus the IV bundle nested under DATA.iv (its dgp/scalars keys collide with geo's, + so the spliced IV figure code was rebased to read DATA.iv.* by the splicer). + SOURCES: the labs Sources backup table rows at . + MATHJAX: the vendored tex-svg build, inlined once at /*__MATHJAX__*/. + +Build: .venv/bin/python apps/build_unified_slides.py (or `make html-unified`) +Template: apps/unified_slides_src.html (canonical; edit it, then rebuild). +Verify: .venv/bin/python apps/verify_unified_deck.py (geo claims subset + labs claims) +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import build_geo_slides as bg +import build_iv_slides as bi +import build_labs_slides as bl + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent +SRC = HERE / "unified_slides_src.html" +OUT = HERE / "unified_slides.html" + + +def main() -> None: + if not SRC.exists(): + sys.exit(f"FAIL: {SRC} missing.") + html = SRC.read_text() + + # 1 · scalar/quote tokens from BOTH sources ------------------------------------------ + pins = bl.load_pins() + tokens = bg.load_tokens() + labs_tokens = {k: str(v["text"]) for k, v in pins.items()} + dup = set(tokens) & set(labs_tokens) + if dup: + sys.exit("FAIL: token collision between shards and labs pins: " + ", ".join(sorted(dup))) + tokens |= labs_tokens + + # Part 3 (IV) prose tokens: {{nb11.*}} / {{nb11b.*}} from the executed IV shards + iv_tokens = bi.load_tokens() + dup_iv = set(tokens) & set(iv_tokens) + if dup_iv: + sys.exit("FAIL: token collision between existing tokens and IV shards: " + ", ".join(sorted(dup_iv))) + tokens |= iv_tokens + + used, missing = set(), [] + + def sub_token(m: re.Match) -> str: + key = m.group(1).strip() + if key not in tokens: + missing.append(key) + return m.group(0) + used.add(key) + return tokens[key] + + html = re.sub(r"\{\{([a-z0-9_.]+)\}\}", sub_token, html) + if missing: + sys.exit("FAIL: unknown tokens (neither shards nor labs pins): " + + ", ".join(sorted(set(missing)))) + + # 2 · ONE data bundle (geo + labs chart keys), 2b · the N map ------------------------ + bundle = bg.build_bundle() + labs_bundle = bl.bake_bundle(pins) + overlap = set(bundle) & set(labs_bundle) + if overlap: + sys.exit("FAIL: DATA key overlap between geo and labs bundles: " + ", ".join(sorted(overlap))) + bundle.update(labs_bundle) + + # Part 3 (IV) chart data: nested under a single key (its dgp/scalars/... collide with geo's, + # so the IV figure code was rebased to read DATA.iv.* by the splicer). + if "iv" in bundle: + sys.exit("FAIL: DATA key 'iv' already present; cannot nest the IV bundle.") + bundle["iv"] = bi.bake_bundle() + html = bg.inject_data(html, bundle) + html = bg.inject_nums(html, tokens) + + # 3 · the labs Sources backup slide -------------------------------------------------- + if "" not in html: + sys.exit("FAIL: template has no marker.") + html = html.replace("", bl.sources_rows(pins), 1) + + # 4 · MathJax ------------------------------------------------------------------------ + if "/*__MATHJAX__*/" not in html: + sys.exit("FAIL: template has no /*__MATHJAX__*/ marker.") + html = html.replace("/*__MATHJAX__*/", bg.mathjax_js(), 1) + + OUT.write_text(html) + size = OUT.stat().st_size / 1e6 + n_slides = html.count('
` blocks. It STOPS at the classical verdict ("measure first"). Each act calls back +to Part 1's cases (🔗). Slide numbers below are the unified deck.* + +### 10 · Part 2 divider / Title — "Synthetic Control" +- 🎯 Reset the frame: "One treated market, no experiment, €4M on the line: did the campaign work?" Keep the + hero figure. 💬 "Colgate at least had a clean before-and-after. Take even that away — one market, no + control, no experiment. This is the general case, and it's the most common one." + +## ACT I — The question (~9 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 11 | The boardroom question | 🎯 Three questions IN ORDER: attribution → significance → decision. The metro was CHOSEN, not randomized — that's why it's hard. 💬 "Most companies jump to question 3 with an unchecked answer to question 1." 🔗 "Colgate's launch was national; here it's one metro — same missing number." | 📐 "unbiased" fold-out (only if asked). | +| 12 | The data (live) | 🎯 The counterfactual: the one number in no file — what the metro would have sold anyway. Show live that the naive bump changes when you subtract the crowd. 🖼 Rooster-takes-credit-for-sunrise ("the campaign made sales rise"). | 📐 The three technical readings, folded. | +| 13 | Poll: the 12% bump ✋ | 🎯 Naive before/after isn't "roughly right + noise": the tide can HIDE a real effect (as here) or inflate a fake one. Run the poll — commitment makes the teaching moment. | — | +| 14 | Potential outcomes | 🎯 Causal effect = gap between two histories; only one is observed. The 2×2: 3 cells are data, 1 is missing, every method fills that cell. Per-week vs 20-week total are different targets; the €4M rides on the total. 🖼 Doctor Strange "14,000,605 futures, we observe one." | 📐 Potential-outcome notation box — fold. | +| 15 | The data-generating model | 🎯 ONE honest confession: data is simulated SO every method can be graded against a known truth (~€284k) — the same recover-the-truth contract as Colgate. | 📐 The whole equation. "Four ingredients incl. per-market sensitivity" and move. | +| 16 | Simulate the world (live) | 🎯 ONE slider (macro shock): turbulence destroys before/after. 💬 "Compared to what?" 🖼 optional: stormy-sea "a rising tide lifts all boats" visual. | ⏭ Other sliders; the full "what sliders teach" list. | + +## ACT II — The counterfactual (~11 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 17 | Abadie's idea: the synthetic twin | 🎯 THE core idea: no twin exists, so blend markets into one. Twin matches RESPONSES to shocks, not levels → survives storms. Anchors: Meta GeoLift, Google geo tools. 🖼 "assemble a twin from parts" cartoon, tasteful. | 📐 Factor-model equation; pedigree fold-out (one clause). | +| 18 | The estimator | 🎯 3 business facts: (1) fitted ONLY pre-launch = no peeking, (2) weights are a readable recipe (40%+32%+17%), (3) effect = subtraction. 🔗 "The honest, weights-on-the-table version of what Colgate's model did in time." | 📐 argmin / simplex formulas — folded on the slide. | +| 19 | Inside the convex hull (live) | 🎯 Coffee+milk can't make orange juice: a blend can't reach outside the donors' range → the method can REFUSE. 💬 SIGNATURE MANAGEMENT LINE: "Ask any vendor — when does your method refuse to answer? If 'never', walk away." 🖼 big red OFF switch / "computer says no". | 📐 "simplex", "hull" — use the demo, not the words. | +| 20 | What the constraint buys (live) | 🎯 Better fit to the past ≠ better model. Drop the constraint: OLS fits tighter, predicts worse, uses absurd weights (−80%). 💬 "Anyone selling a model on past fit is selling the wrong metric." 🖼 the overfitting meme (squiggly line through every point). | 📐 n_eff (inverse Herfindahl) block — fold. | +| 21 | Why not just forecast it? | 🎯 Forecasting ≠ counterfactual: "what comes next" vs "what would have happened in a world that never happened." You cannot cross-validate the counterfactual → pick defendable assumptions, not demo accuracy. Auditability is a business feature. 🖼 confidently-wrong-AI / "trust me bro" robot. | The sweep numbers (one clause: "clearly worse across 24 worlds"). | +| 22 | Every shortcut is an assumption | 🎯 KEY SLIDE of Act II. Every method = a hidden claim about the counterfactual. Live dials: DiD dies when markets react differently. 💬 "A wrong method does not look wrong, and more data does not fix it." | 📐 The formulas — point at the dials instead. | +| 23–24 | Before DiD: two one-difference estimators / When does DiD work? (algebra) | ⏭ Compress BOTH to: before/after fails when the world moves; controls fail on size; DiD needs parallel reactions and dies silently; SC chooses weights instead of hoping. 💬 KEEP: "More weeks of data buy precision around the same wrong number." | 📐 All derivations, Var(B), both fold-outs. | +| 25 | Identification: what must hold | 🎯 Due-diligence framing: 4 assumptions, 3 testable (PASS), 1 untestable (spillover) — flag it, it returns at SUTVA. 🔗 "The untestable one is the second-launch leak you named for Colgate." Managers get checklists and audits. | 📐 Gate arithmetic. | + +## ACT III — Is it real, how big? (~8 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 26 | Poll: is it real? ✋ | 🎯 Open with the absorbed line: with ONE treated metro a t-test is not weak, it is UNDEFINED. Then let THEM invent permutation inference: 29 fake "treatments" measure the method's luck. Rank 1/30 → p ≈ 3%. 💬 "You just reinvented a rigorous test using common sense." 🖼 "is this a pigeon? / is this a real effect?" | The word "exchangeable". | +| 27 | Placebo-in-space (live) | 🎯 The picture: green line clear of the grey luck-cloud. 💬 "Real ≠ profitable" — hold applause for Act IV. | Abadie hygiene rule (one clause for analysts). | +| 28 | Interval by test inversion (live) | 🎯 Only the OUTPUT and its role: [€195k, €335k], built with NO model assumptions → it is the REFEREE all later models must answer to. | 📐 The whole inversion mechanics (steps ①–④, quantile algebra) — fold. | +| 29 | Falsification 1: placebo-in-time | 🎯 Fake launch 10 weeks early → finds nothing → pass. 💬 "Any vendor claiming lift should show their method finds zero where nothing happened." | Broken-world checkbox unless time allows. | +| 30 | Falsification 2: spillover (SUTVA) | 🎯 MANAGEMENT SLIDE, don't rush: the untestable leak (ads reach the commuter belt = your controls). 🔗 "This is the second-launch failure you named for Colgate, in space — and the endogeneity we meet head-on in Part 3." Two takeaways: bias only SHRINKS the number (conservative floor), and 💬 "Design beats analysis" — keep controls off the media footprint at TEST-DESIGN time. | 📐 φ sweep mechanics. | +| 31 | What the statistics says | 🎯 Bank the 3 numbers (p≈0.033 / €260k / [195, 335]). Then the trap: 💬 "260 for 75, 3.5×, roll it out — every number true, conclusion doesn't follow." Cliffhanger into money. | Truth-check detail (one clause: truth 284 is inside the interval). | + +## ACT IV — The decision in euros (~12 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 32 | Poll: the margin ✋ | 🎯 Profit = margin × lift − cost ≈ €16k net. Option A (sales − cost = 185) is THE boardroom classic: revenue treated as profit. 🖼 "revenue vs profit — corporate needs you to find the difference" (callback to Nürnberger). | — | +| 33 | The margin trap (live) | 🎯 CHEAPEST LESSON OF THE DAY: break-even ROAS = 1/margin, not 1. Slide the margin: software 1.1 vs grocery 5.0 — same campaign, opposite verdicts. 💬 "ROAS > 1 is a vanity bar." | 📐 iROAS notation. | +| 34 | The decision space (live) | 🎯 "Looks profitable on average" ≠ "provably profitable": profit range [68, 117] straddles the 75 price. Our campaign lives in the dangerous zone. 🖼 the statistician who drowned crossing a river "1m deep on average." | 📐 The price-sweep table. | +| 35 | What you say to the board (classical) | 🎯 This memo answers the €75k PILOT only: ① real (p=0.033) ② big enough on the point (€16k net) ③ NOT confident (profit [68,117] straddles 75). Verdict: renegotiate or buy certainty with a bigger test. 💬 "This says nothing yet about the €4M — it is not the pilot multiplied." The one question it CAN'T answer ("78% chance it pays") is the forward tease to the probability view (slide 3's Bayesian word) we add later. | Table row-by-row. | +| 36 | What €4M actually buys | 🎯 THE FREQUENTIST VERDICT, no probability needed. Carry the INTERVAL not the point, be maximally generous (δ=1, perfect transport): even then the 90% profit band is [−€360k, +€2.25M] → STRADDLES ZERO. 💬 "At the most optimistic assumption we can write down, we cannot show the €4M pays." Below δ≈0.64 it provably loses. ✋ ENGAGEMENT: run naive +€860k and poll for approval BEFORE carrying the interval. | 📐 δ-decomposition (already folded in a `
` on the slide — leave it folded). | +| 37 | Measure it, don't guess it | 🎯 MOST PROTECTED SLIDE — never cut. The call: even the best case straddled zero → do NOT commit €4M on this evidence ("not yet, not on this"). WHY analysis can't fix it: the interval is wide because there is ONE treated market; only more treated markets narrow it. THE FIX: (1) 💬 "before you spend a million on evidence, spend an hour checking you haven't already bought it"; (2) MEASURE — 8 markets that look like the nation, national intensity, €1.07M, so measured lift IS national lift; (3) pre-commit the release rule or the test is theatre. 🔗 "This is HelloFresh's loop, bought once — the counterfactual, anchored by an experiment." ✋ "so what would you spend?" → honest "not €4M yet." | 📐 Sizing arithmetic, p-floor — folded. | + +### BRIDGE — Part 2 → Part 3 (say this before the break, ~30s) +💬 "Synthetic control needed one thing to work: a clean group of untreated markets to build the twin from. +When you have that, you can rescue a decision even without an experiment. But sometimes the thing you want +to measure was assigned by someone whose whole job is to NOT be random — a platform choosing who sees an +ad. No clean control exists, and you can't run the test. After the break: the last and hardest case, and +its own tool." *(The finale now sits AFTER Part 3; Part 2 hands off to IV, not to the close.)* + +--- +--- + +# PART 3 — INSTRUMENTAL VARIABLES (the last tool · ~30 min) + +*The SHORT IV deck (`iv_slides_sh.html`): one retailer, one ad platform, one decision — what is one ad +exposure worth when the platform picked who sees the ads? Entirely CLASSICAL (two averages, one division, +one F, one interval). Sections mirror the deck's data-sec groups: The problem → The idea → The estimator → +When it breaks → The decision. This is the endogeneity case Part 1's Colgate probe and Part 2's SUTVA +slide both pointed at. Slide numbers are the unified deck.* + +### 38 · Part 3 divider / Title — "Instrumental Variables" +- 🎯 Reset, and pose the part in one image: the dashboard's number is TIGHT and WRONG; the lottery's number + is WIDER and RIGHT, and still clears the price. Kicker "Part 3 · The natural experiment". 💬 "Same + discipline — counterfactual, anchor, price — but the anchor comes from a surprising place: a scrap of + randomness we already ran." +- 🖼 The hero figure (dashboard interval misses the truth; lottery interval covers it) carries it. No meme yet. + +## IV · THE PROBLEM — why the dashboard lies (~13 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 39 | The case | 🎯 Set the decision cleanly: exposure = one user saw an ad, costs €10; the auction (not us) picks who; dashboard says exposed spent ~€24 more; pitch = "raise the budget"; decision = keep €10 / pay more / stop. One number settles it: euros of extra sales ONE exposure causes. 🔗 "Colgate's disease, a third time — compared to what?" | 📐 nothing; pure business setup. | +| 40 | Poll · the pitch ✋ | 🎯 Correct = C (not sound: exposed users may differ from unexposed, ad or no ad). A/B miss because the arithmetic IS right and it IS significant — that's not the problem. 💬 "The platform's whole job is to find buyers, so the exposed group was more likely to buy before any ad ran." 🖼 a pushy-salesperson / "trust me, raise the budget" meme. | — | +| 41 | Confounding, defined | 🎯 Name the three quantities: sales (outcome), exposure (0/1, auction-assigned), and INTENT — readiness to buy, real, drives buying AND drives who gets an ad, and NEVER logged. A hidden driver of both = a CONFOUNDER. That word is the takeaway. | 📐 SKIPPABLE MATH: the DAG + formal definitions — fold; say "confounder" and move. | +| 42 | The naive comparison (live) | 🎯 Write €24 down honestly: €68 − €44, two OBSERVED cells; it pretends the unexposed group's spend is a fair stand-in for the exposed group's missing "without-ad" spend. LIVE: tick "reveal the hidden driver" → the groups already differed in intent. 💬 "The arithmetic is right. The comparison is wrong." | 📐 potential-outcomes table Y(1)/Y(0) — on-slide, keep it visual. | +| 43 | Poll · read the gap ✋ | 🎯 Correct = B (€24 = the ad's effect + the groups' pre-existing difference, in unknown proportions). A books it all to the ad, D all to targeting; the data alone justify neither. 💬 "The only honest statement is the split — and splitting it is the whole rest of the part." | — | +| 44 | Selection bias, defined | 🎯 The split has a formula: naive gap = (effect of the ad on the exposed) + (SELECTION BIAS = the gap that would exist with NO ads, because the platform picked buyers). Our job = kill the second term. | 📐 SKIPPABLE MATH: the E[Y(0)|X] decomposition — fold; say it in words. | +| 45 | The simulated world | 🎯 Same honesty as geo: we PLANTED the true effect at €15, so every method is graded; the dashboard's €24 is already wrong by €9. LIVE (κ slider): push intent's pull up → groups drift apart, dashboard gap grows, planted truth never moves. ⏭ Fast (~45s). | 📐 the three generating equations — fold. | +| 46 | Endogeneity, precisely | 🎯 The whole disease as ONE inequality, translated: exposure is correlated with "every other reason this person spent" (intent), because the auction targets exactly those people. Name = ENDOGENEITY. 🔗 "This is the formal name for Colgate's 'leak into the control' and geo's spillover — a thing CHOSEN using info you can't see." | 📐 SKIPPABLE MATH: Cov(X,v)>0, v = κU+ε — fold; the picture (both bars rise with intent) carries it. | +| 47 | The size of the bias (live) | 🎯 The lie is computable, not vague: formula predicts ~€23.5, data delivered €23.7, truth €15. THREE lessons: (1) SYSTEMATIC not luck — dashboard can only OVERSTATE; (2) grows with targeting and with intent's pull — better platform, bigger lie; (3) sample size is NOWHERE in it → more data can't fix bias. | 📐 SKIPPABLE MATH: the bias formula — the live readout is the point. | +| 48 | The limits of adjustment | 🎯 NEVER CUT — the pivot. The standard fix (control/match on the confounder) is the RIGHT move and normally works: LIVE tick "pretend intent were logged" → recovers €15. Untick → intent is a feeling in a head, not in the logs, never will be. 💬 "You cannot control for a column you do not have." The toolkit runs out of road HERE. | 📐 the boxed-W backdoor DAG — on-slide. | +| 49 | Poll · what would fix it ✋ | 🎯 NEVER CUT — the hinge into IV. Correct = C (a batch of users whose exposure we assign at random). A = wrong number more precisely (bias isn't noise); B/D = more controls, auction reacts to live signals, backdoor stays open. 💬 "The instinct is right — but you can't randomize the ads; the platform owns that auction. So we need randomness we CAN get our hands on." | — | + +## IV · THE IDEA — the instrument (~9 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 50 | The ideal experiment | 🎯 NEVER CUT. What randomizing exposure would buy: it doesn't BLOCK intent's arrow into exposure, it DELETES it → the naive comparison becomes right. We can't randomize exposure directly — but we only need something random that NUDGES it. 🔗 "Option C from the poll, made real." | 📐 X randomized ⇒ Cov(X,v)=0 ⇒ naive → β — one line. | +| 51 | The lottery | 🎯 NEVER CUT — the heart of the method. Before the campaign our own RNG ran a serving-priority LOTTERY: picked half the users, gave them a small auction-priority boost. Subtle and crucial: it shows NOBODY an ad — it only raises the ODDS of one. Winners chosen by our coin flip → blind to intent. 🖼 a "golden ticket" / rigged-in-our-favour-coin visual. | 📐 Pr(X=1)=σ(α₀+γZ+λU); new column Z — fold. | +| 52 | The instrument, defined | 🎯 The scrap of randomness has a name: an INSTRUMENT moves the treatment, is as-good-as-random vs the hidden confounder, and hits the outcome ONLY through the treatment. Four conditions: Relevance (testable), Exogeneity (by design), Exclusion (argued), Monotonicity (argued). 💬 "Two we get for free because we built the lottery; two we must argue. That scorecard is how you judge ANY IV study." | 📐 the four-row condition table — present the four names, fold the fine print. | +| 53 | Condition 1 · relevance | 🎯 The one condition DATA can settle: the FIRST STAGE = extra exposure the lottery created. Winners exposed ~21 points more than losers → π ≈ 0.21. Real, measured, strong. "If this were ~0 the method is dead — hold that for weak instruments." | 📐 SKIPPABLE MATH: X = b₀+πZ+u, π = 0.2106 — the two bars carry it. | +| 54 | Conditions 2 and 3 (live) | 🎯 The two you must ARGUE. Exogeneity (blind to intent) = free, we drew it. EXCLUSION is the honest one: the lottery must touch sales ONLY via exposure — a side-channel (faster page, shown price) leaks straight into the answer. 🔗 "The untestable assumption — exactly like geo's spillover. You DESIGN it out, you argue it in words." | 📐 the side-channel bias β̂_IV = β + s/π — LIVE demo is the point. | + +## IV · THE ESTIMATOR — the division (~5 min, but slide 56 is the climax) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 55 | The reduced form | 🎯 Second measured number, same shape: the REDUCED FORM = the lottery's effect on SALES = the raw win/lose sales gap ≈ €3.48. Clean for the same reason π was (a random draw can't favour ready buyers). Hold the two clean numbers: 0.21 extra exposures, €3.48 extra sales per win. | 📐 SKIPPABLE MATH: δ = E[Y|Z=1]−E[Y|Z=0] = €3.48 — say it in words. | +| 56 | The IV estimate | 🎯 THE PAYOFF — NEVER CUT. One division: €3.48 of sales per win ÷ 0.21 exposures per win = **€16.5 per exposure**. Units ARE the intuition (euros/win ÷ exposures/win = euros/exposure). Intent divides out (can't correlate with a coin flip). 💬 "Dashboard said 24, truth is 15, the lottery says 16½ — the first number all session not poisoned by who the platform picked." | 📐 β̂_IV = δ/π — the arithmetic IS the slide. | +| 57 | Why the division is forced (live) | 🎯 ⏭ Fast reinforcement: the division isn't a modelling choice, it's the ONLY effect the two numbers allow. LIVE: drag a guess; rising line = its prediction (β̂×π), flat line = the fact (€3.48); they meet only at €16.5. | — | + +## IV · WHEN IT BREAKS — the caveats (~4 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 58 | Weak instruments (live) | 🎯 The failure to RECOGNIZE: a WEAK instrument (tiny first stage) is worse than none. LIVE (drag lottery strength down): the interval explodes AND the centre creeps back toward the naive lie (dividing by ~0). 💬 "Looks like a method while quietly handing back the dashboard's number." The defence = the first-stage F (rule of thumb >10; ours 156). Always ask for it. | 📐 the F-vs-estimate sweep — the live band is the point. | +| 59 | Compliers and the LATE (live) | 🎯 Whose number is €16.5? Only the movable — the COMPLIERS (saw the ad because they won the lottery). Always-takers / never-takers: the lottery is silent about them. 💬 "The movable are exactly who your next euro of budget reaches — the right number for the decision." Name = LATE. | 📐 SKIPPABLE MATH: the three-slice split; "local average treatment effect" — fold. | +| 60 | The checklist | 🎯 NEVER CUT — the reusable scorecard. Four assumptions: Relevance (F=156, TESTABLE, passes) · Exogeneity (by design) · Exclusion (UNTESTABLE, argue) · Monotonicity (UNTESTABLE, plausible). 💬 "One number you can check, one design guarantee, two arguments in words. Ask a vendor for all four — if they show only the estimate, they showed you the least important part." | — | + +## IV · THE DECISION — euros at last (~3 min) + +| # | Slide (short-deck title) | Emphasize | Fold / skip | +|---|-------|-----------|-------------| +| 61 | The price map (live) | 🎯 NEVER CUT — the cleanest decision slide. One estimate → a MAP from price to verdict: below €12.7 GO (even the most pessimistic supported effect pays), €12.7–€20.4 TEST (interval straddles the price), above €20.4 NO-GO. LIVE: drag the price and watch the verdict flip. Today's €10 sits comfortably in GO. | 📐 the interval band + zones — the live figure carries it. | +| 62 | Poll · the negotiation ✋ | 🎯 Correct = C (€12.7, the interval FLOOR = the no-regret bid cap: below it every supported effect pays, whichever is true). B (€16.5, the point) is a coin-flip gamble. 💬 "You just priced a bid cap off a confidence interval." | — | +| 63 | The verdict and the recommendation | 🎯 NEVER CUT — the close of Part 3. On one page: effect €16.5 (δ/π), 90% [12.7, 20.4] (classical, AR agrees — the placebo-style referee, exactly as in geo), F=156, price €10, net €6.5. Verdict BUY. Three lines: keep buying at €10 · cap the bid at €12.7 · re-measure only if the rate climbs toward the floor. 💬 "Entirely classical, and it answered what the dashboard could not — by finding one scrap of randomness we controlled." | 📐 the summary table — on-slide. | + +### BRIDGE — Part 3 → FINALE (say this, ~15s) +💬 "Three real cases, then two builds — a €4M rollout you learned to refuse, and an ad price you learned +to trust. Different tools, one pattern: a counterfactual, an anchor, a price. That's not a synthetic-control +trick or an IV trick — it's the shape under every real engagement, including the three we opened with." + +--- +--- + +# FINALE — the pattern, and the close (~2.5 min) + +*The labs closing slides come HOME here — written as a synthesis, they belong at the very end. This is the +bookend: we opened with three cases, we close on the one pattern under all of them. Now that Part 3 has +run, this finale is the true session close.* + +### 64 · The tools are open-source — and they are the syllabus (moved from Part 1) +- 🎯 Provenance beat, ~45s: CausalPy (synthetic control, ITS, DiD, RDD — AND the IV estimator you just used + to price the ad exposure) and pymc-marketing are the tools behind everything you saw; one client's + budget-allocation approach came back as a pull request (Bolt); the consultancy's own webinar agenda is + literally this session's syllabus (geo tests, MMM, synthetic control, diff-in-diff, instrumental variables). +- Placement rationale: lands better here than in the hook — after both builds the audience knows what the + tools actually DO. Cuttable first if time runs out. +- 🖼 Package logos (already in deck). No meme. + +### 65 · The pattern in every engagement +- 🎯 Collapse the whole session to three lines: the deliverable is a counterfactual; an experiment anchors + every observational model — and when you can't run one, you hunt a scrap of randomness the world already + made (the instrument); uncertainty prices the decision (boards act on the interval and the headroom, not + a point). 💬 "Twice today the honest interval overturned the confident point." +- 🖼 The Decision Lab gag table already in the labs deck: vanilla agent → "Confidently wrong."; PyMC Labs' + honest system → "No valid model found. Run a geo-holdout experiment." 💬 "Even the machine's best answer + was this lecture's advice: measure it." + +### 66 · One breath (close) +- 🎯 Deliver the one-breath summary and STOP. 💬 "The toolkit a Bayesian consultancy sells: counterfactuals, + anchored by experiments, decided with their uncertainty on the table. You've watched real clients buy it, + and built two yourself — a €4M rollout you learned to refuse, and an ad exposure you learned to trust." + Do not add content after it. +- Say-hello / repo line stays. This IS the end of the 1.5 hours. + +### Backup · Sources (67) +- Auto-generated pin table from labs_deck_data.json + geo/IV claims. Backup only. + +--- +--- + +## The management take-homes (if you must cut, protect these) + +1. **The counterfactual** (slides 5, 12, 14, 42): a campaign's — or an ad's — value is a missing number; "compared to what?" is always the question. +2. **Every method is a hidden claim** (slide 22): a wrong claim doesn't look wrong, and more data doesn't fix it. +3. **Calibration is the product** (slides 7–8, 37): an experiment anchors every model; it is not a luxury. +4. **When does the method refuse?** (slide 19): a tool that never says "I can't answer" is the one to distrust. +5. **Design beats analysis** (slides 25, 30, 54): the untestable assumption (spillover / exclusion) is controlled when you DESIGN the test or the instrument, not after. +6. **The margin trap** (slides 32–33): break-even ROAS = 1/margin. The single cheapest, most reusable lesson. +7. **Significance ≠ worthiness** (slides 31, 35, 37): "real" and "worth it" are different questions; the expensive mistakes live between them. +8. **The interval is the recommendation** (slides 36–37, 61): even the BEST case for the €4M straddles zero (measure first); the IV interval's FLOOR is the no-regret bid cap. A range beats a point. +9. **Endogeneity, and its cure** (slides 46, 48–49, 51, 56): when the thing you measure was CHOSEN using what you can't see, controls fail — you need a scrap of randomness (an instrument), and the whole method is then one honest division. +10. **A weak instrument is worse than none** (slide 58): near-zero first stage → the estimate quietly reverts to the biased dashboard number. Always ask for the first-stage F. + +## If you are told "you have 60 minutes" on the day (Parts 1–2, hold Part 3 for a follow-up) + +Cut in this order: finale slide 64 (provenance) → geo 23–24 (keep the one quoted line) → 20 → 15–16 (merge +to one sentence each) → 34 (fold its point into 35) → the second poll of whichever act is running long. +Never cut: 2–7 (who we are + the hook), 11–14, 17, 19, 22, 26–27, 30, 32–33, 35–37 (36–37 above all), 65–66. + +## If Part 3 runs tight (~30 min for IV) + +Fast/fold: 41 (name the confounder, fold the DAG), 44 (words not algebra), 45, 46 (one-line endogeneity), +47, 53 ("21 points more exposure"), 55 ("€3.48 per win"), 57, 59 (one sentence on "whose number"). +Never cut: 39, 42, 48–52, 56 (the division), 60 (checklist), 61, 63 (the verdict). + +## Meme / visual budget (keep it to ~8–10 across the whole 1.5h session — scarcity keeps them funny) + +Highest-value, lowest-risk spots: slide 4 ("this is fine" CMO, no holdout) · slide 5 (two-Spider-Men +counterfactual) · slide 7 (overfitting / target-around-arrow) · slide 14 (Doctor Strange one-future) · +slide 19 (big red OFF switch — the method refuses) · slide 33 (revenue vs profit "same picture") · slide 34 +(river 1m deep on average) · slide 40 (pushy salesperson "raise the budget") · slide 51 (golden ticket / +rigged coin — the lottery in our favour). Everything else stays a clean figure. All optional; the polls and +live widgets are the primary engagement, memes are seasoning. diff --git a/causal-marketing-pymc/apps/geo_lift_lecture_speech.md b/causal-marketing-pymc/apps/geo_lift_lecture_speech.md new file mode 100644 index 0000000..583ecbc --- /dev/null +++ b/causal-marketing-pymc/apps/geo_lift_lecture_speech.md @@ -0,0 +1,1062 @@ +# Unified Lecture — Full Speech (Management-Class Version) + +The spoken script for `apps/unified_slides.html` (67 slides): Part 1 "Causal Inference in the +Wild" (real PyMC Labs engagements, the hook) → Part 2 "Synthetic Control" (the short classical +arc, Acts I–IV, a €4M rollout) → Part 3 "Instrumental Variables" (what one ad exposure is worth +when the platform picked who sees it) → the finale (the pattern, one breath). Slide numbers match +the deck's counter. The per-slide priorities live in `apps/geo_lift_lecture_emphasis.md`; this file +is what you say. + +**Audience adjustment.** Written for a general management class, not an analytics class. Every +slide is presented as a **business decision problem first**. The math on the slides is optional +depth: whenever a slide is math-heavy, the script gives a short business translation to say out +loud, plus one sentence to hand the math to the curious ("the formula is in the fold for those +who want it"). You never need to derive anything live. + +**Language.** Short sentences. Plain words. No idioms. Read it as written or in your own words. + +**Timing guide (~90 min, the full 1.5h session).** Part 1 ~11 min · Part 2 ~44 min (Act I ~9, +Act II ~11, Act III ~8, Act IV ~12 — the €4M pair, slides 36–37, is the climax: protect it) · +[break] · Part 3 ~30 min (the disease slides 39–48 ~13, the instrument 49–55 ~9, the estimate and +its limits 56–60 ~5, the decision 61–63 ~3 — the division, slide 56, and the verdict, slide 63, are +the climax) · Finale ~2.5 min. If there is no break, Part 3 folds straight on after slide 37. + +Slides marked **[FAST]** can be covered in 30–60 seconds for this audience. +Slides marked **[SKIPPABLE MATH]** can be summarized in one sentence; the fold stays closed. + +--- +--- + +# PART 1 — CAUSAL INFERENCE IN THE WILD + +## Slide 1 — Title + +> Good morning. Before any theory, I want to show you what this subject looks like when real +> companies pay for it. Three client stories, ten minutes. Then we take the hardest version of +> their problem and build the machinery ourselves, end to end — twice. First on a four-million-euro +> rollout decision. Then, after the break, on a subtler one: what a single ad exposure is worth +> when someone else already chose who sees it. +> +> Nothing today needs a statistics background. It needs the willingness to ask one question, +> stubbornly: compared to what? + +## Slide 2 — PyMC Labs: what we do + +> One minute on who is talking, because it explains why these stories exist. +> +> PyMC Labs is a Bayesian modeling consultancy. Two things at once: we build custom +> decision-making models for companies where off-the-shelf tools fall short, and we maintain +> the open-source libraries this field runs on — PyMC, PyMC-Marketing, CausalPy. +> +> The business model matters for how you read the cases. We sell senior modeling expertise, not +> software licenses. Engagements come in three shapes — you see them in the table: a fixed-scope +> project when a company has one high-value decision problem; an advisory retainer when a firm +> is building its own team; and training when an analytics team is standardizing on the tools. +> The libraries are free. They build trust and reach. The consulting monetizes the expertise +> behind them. Small senior teams, no junior pyramid, and the client keeps the model at the end — +> not a black box. +> +> The client list for today is on the slide: Colgate-Palmolive, HelloFresh, a German insurer +> called Nürnberger Versicherung. You meet three of them in the next ten minutes. [The fold has +> the full areas-of-interest list, for later.] + +## Slide 3 — Bayesian vs frequentist, in one slide **[FAST]** + +> One word of vocabulary before the cases, because you will hear two words all session, and the +> difference decides what kind of answer you are allowed to ask for. +> +> **Frequentist** thinking asks: how would this procedure behave if I repeated the whole study +> many times? It hands you a point estimate and a confidence interval — a range. **Bayesian** +> thinking asks a more natural question: given the exact data I have in hand, how probable is +> each possible value of the effect? It hands you a full probability distribution — the language +> of "a 78 percent chance this pays". +> +> Why it matters for a manager: you allocate money in the Bayesian language — probabilities and +> odds — but the fastest, most standard tools speak the frequentist one. Today's two builds stop +> at the honest frequentist answer: a range, and whether the price sits inside it. That is +> already worth real money, as you will see. The probability layer — "what are the odds it pays" +> — is where this course goes next. Keep both words; I will tell you which one we are speaking. + +## Slide 4 — Poll: you are the consultant + +> Now you are the consultant, and the phone rings. [Read the poll.] Colgate-Palmolive: +> "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it +> stealing share from competitors, or from our own brands?" Four instruments on the slide. +> Which do you reach for first? +> +> [Collect hands on each option. Then reveal.] +> +> C. And here is why the others fail. An A/B test needs something left to randomize — the +> launch already happened, everywhere. Difference-in-differences needs a region without the +> product — there is none. An instrument for shelf placement — the shelf changed in every store +> at once. When the world gives you no comparison group, one move remains: learn the world +> before the launch, project it forward, and read the gap. That projection is called a +> counterfactual, and it is the single product every case today has in common. +> +> If you were not sure which to pick — good. That gap is exactly what this session is for. + +## Slide 5 — Colgate: incremental, or cannibalistic? + +> Here is the brief in the client's own words: "estimate the counterfactual sales of all +> products, had the new product not been introduced." Read that twice. The client is asking to +> buy a number that exists in no file, no dashboard, no report anywhere on earth: what would +> have happened without the launch. +> +> Why does it matter? Two words on the slide. **Incremental** sales are won from competitors or +> category growth — those justify the launch. **Cannibalistic** sales are stolen from your own +> shelf — those just moved euros between your own products. The launch verdict is the split. +> +> And notice the professional discipline, because it returns all day: before trusting the model +> on real data, they graded it on simulated data where the answer was planted. Planted truth: +> fifty percent incremental. The model's interval: 49 to 59 percent. It recovers the truth, so +> it earns the right to be believed. Remember this contract — recover a known truth first. +> We will hold every method today to it. +> +> [If asked for the method's name: "a multivariate Bayesian interrupted time series, later a +> nested-logit choice model — the names matter less than the projection idea."] + +## Slide 6 — What would break it? (open floor + live widget) + +> Before you trust that number, earn your fee. You are Colgate's CMO. Name one real-world event +> that would make this estimate wrong. [Open floor, two minutes. Typical answers: a competitor +> launch, a price change, a pandemic, a second launch of our own.] +> +> [Reveal.] The model itself confessed one: **a second product launch** inside the estimation +> window. When that happened, the same machinery reported 64 to 76 percent against a planted +> truth of one hundred. Why? The counterfactual learned "normal growth" from a window that +> already contained another launch — it absorbed part of the very effect it was meant to isolate. +> +> [Hand a student the widget: tick "a second product launches nearby", slide the month.] Watch +> the readout bend: inside the window it under-credits; near the edge it over-credits. Same +> machinery, wrong world, wrong number — and no error message anywhere. +> +> Two things to keep. First: an honest consultancy prints this failure in the same post as the +> win. When you buy analytics, ask for the failure cases; if there are none, walk away. Second: +> hold onto the phrase "something leaked into my comparison". It comes back this afternoon with +> a formal name, and it is the one assumption no statistical test can rescue. + +## Slide 7 — Why calibrate? A model alone can rank channels backwards + +> Second case, and first a warm-up from the tools themselves. A marketing-mix model — MMM on +> the slide — explains total sales as the sum of each channel's contribution, fitted on the +> spend history you already have. Every large advertiser runs one. +> +> Here is the published tutorial demo. Fit on observational spend alone, the model reported +> channel one earning 93 and channel two earning 171 — sorry, the reverse: it ranked them +> **backwards**. The truth was the direct opposite of what the dashboard said. And nothing on +> the dashboard warns you. The fit was good. The intervals were tight. The ranking was wrong. +> +> [Tick the toggle: "add two lift tests per channel".] A lift test is a small experiment: nudge +> one channel's spend by a known amount, measure the sales it causes. Feed two of those per +> channel into the model, and the ranking flips to the truth. +> +> The lesson of the whole session in one line: **the experiment is the model's anchor.** A model +> without an anchor can be confidently wrong. Hold that thought; the entire second half is about +> how to build the anchor when you only get one. + +## Slide 8 — HelloFresh runs the loop, at industrial scale **[FAST]** + +> Does anyone run this loop for real? HelloFresh does, at industrial scale. An always-on +> marketing-mix model, calibrated by a continuous stream of field experiments — the panel's own +> agenda says "calibrated to ensure consistency with incrementality measurements" — and the +> calibration cut prediction variance by sixty percent. The picture is the whole slide: the +> model asks for ground truth, the experiment supplies it, the counterfactual reads it out. +> +> One sentence to remember for the last part of this session: the experiments a company +> already runs are its **instrument supply**. When you cannot randomize the thing you care +> about, a randomized nudge nearby can stand in for it. That is the doorway to instrumental +> variables, and we walk through it after the break. + +## Slide 9 — Poll: price the engagement (Nürnberger) + +> Last case, and this one has a number with an invoice attached. Nürnberger Versicherung, a +> German insurer. Their steering metric was last-touch attribution — credit the click that came +> last. They replaced it with a funnel-aware causal model. [Read the poll.] Over six months of +> model-guided spend, what happened to their cost per lead? +> +> [Collect. Reveal.] +> +> Down more than 27 percent. In the client's words, "very, very good". The mechanism is the +> management lesson: under GDPR consent rules, customer journeys appeared artificially +> shortened, so last-touch systematically under-credited the upper funnel — video, display — +> and budget chased the wrong clicks. The causal model measured what the upper funnel actually +> causes downstream, and the client is scaling it into full production. +> +> And listen to the client's bar for belief, on the slide: "Trust is not created by R-squared +> values. It is created when business reality matches model expectations." No statistical +> pitch ever beats that sentence. Keep it. +> +> **[The bridge — say this over the slide before advancing.]** Three cases, one shape. A +> counterfactual — the world without the launch, the campaign, the exposure. An experiment that +> anchors the model. And a decision priced in euros, with its uncertainty on the table. Colgate +> built the counterfactual in *time*: before the launch versus after. The hardest and most +> common version of the problem is built in *space*: one market got the campaign, and you must +> decide about a nation. No experiment, real money. For the next forty minutes we build exactly +> that one, end to end, and every idea from these three cases will come back with its proper name. + +--- +--- + +# PART 2 — SYNTHETIC CONTROL + +## Slide 10 — Part 2 divider + +> Now the deep dive. I will not teach you statistics. I will teach you how to make one specific +> business decision: whether to spend four million euros on a national marketing campaign, when +> the only evidence you have is one regional pilot. The method is called synthetic control. It +> is the engine inside the geo-testing tools used by Meta and Google. But the method is not the +> point. The point is the decision. +> +> Colgate at least had a clean national before-and-after. Now take even that away. + +# ACT I — THE QUESTION + +## Slide 11 — The boardroom question + +> Here is the situation. You are on the board. A campaign ran in one metro region for twenty +> weeks. It cost 75 thousand euros. Sales went up. Marketing is happy, and now they ask for +> 4 million euros to take the campaign national. You must decide. +> +> What do you actually have? Thirty markets, sixty weeks of weekly sales data. One metro got +> the campaign. The other twenty-nine did nothing. And one detail that matters a lot: the metro +> was **chosen** by the marketing team. It was not picked at random. This is not an A/B test. +> +> To decide, you must answer three questions, in this order. +> One: how much of the sales increase did the campaign actually **cause**? That is attribution — +> the Colgate question again: what would have happened anyway? +> Two: is that increase more than luck? That is significance. +> Three: is the profit worth the price? That is the decision. +> +> Notice the order. Most companies jump straight to question three with a number from question +> one that was never checked. Today we do it properly, and each question needs a different tool. + +*[If asked about the fold on "unbiased": "The slide has a precise note on when this measurement +can be trusted. Short version: the method learns the metro's normal behavior from before the +launch, so it does not matter why the metro was chosen — unless the choice used secret knowledge +about the future. We test for that later."]* + +## Slide 12 — The data (live) + +> This is the raw data. Every line is one market's weekly sales. The white line is our metro. +> At week 40 the campaign starts. +> +> [Run the live demo: show the before/after means, then subtract the cross-market average.] +> +> The simplest possible analysis: compare average sales before the launch with after. You get a +> bump. But watch what happens when I subtract the average of all the other markets — the crowd. +> The bump changes size. That is the whole problem in one picture: part of what you see is the +> campaign, and part is the tide — the economy, the season, everything that moved every market +> at once. +> +> And here is the one number you do not have, and never will: what this metro **would have sold +> without the campaign**. In no file, no dashboard, no report. Everything in Part 2 is a way to +> estimate that one missing number. You already know its name: the counterfactual. + +## Slide 13 — Poll: the 12% bump + +> Before I show you any method, commit. [Read the poll.] Sales averaged a certain level before +> launch and a higher level after — a bump of about six percent. What is the campaign's true +> weekly effect? Around 2 thousand per week — most of the bump is just trend? Around 7 thousand +> — the simple comparison is roughly right? Or around 14 thousand — double the bump? +> +> [Collect hands. Reveal.] +> +> About double. The economy happened to **dip** right after the launch. The tide went out, and +> it hid half of the campaign's effect. This is the first management lesson of the day: a naive +> before/after comparison is not "roughly right with some noise". It is charged with everything +> else that happened in the world. This time the tide hid a real effect. Next time it will +> flatter a dead one, and you will pay for a campaign that did nothing. + +## Slide 14 — Potential outcomes + +> The one piece of theory you need today. It fits on this slide. +> +> Our metro has two possible histories for the campaign weeks. History one: sales **with** the +> campaign. History two: sales **without** it. The campaign's true effect is the gap between +> the two histories. That is all "causal effect" means. +> +> The problem: only one history ever happens. Look at the table: three of the four cells are +> real data. The fourth — the metro's sales without the campaign, after launch — is missing. +> Every method you have ever heard of, A/B tests included, is a different way to fill that one +> missing cell. +> +> One more distinction that matters for the money: the effect **per week** and the **total** +> over twenty weeks are different numbers with different uncertainties. The 4-million decision +> rides on the total. Keep that in mind. + +*[SKIPPABLE MATH — the notation fold: "precise definitions for those who like them; you will +not need them to follow."]* + +## Slide 15 — The data-generating model **[FAST / SKIPPABLE MATH]** + +> One honest confession: today's data is simulated. We built the world ourselves. Why? Because +> in a simulated world we **know** the true answer — we planted a 12 percent lift — so we can +> grade every method against the truth. It is exactly Colgate's discipline from this morning: +> recover a planted truth first, then trust the method. +> +> The formula says the world has four ingredients: each market has its own size, a shared trend, +> a shared season, a shared economic wave — and each market feels these shared forces with its +> **own sensitivity**. That last part makes the problem hard, and you will see why. +> +> What matters for today: the planted truth is about 284 thousand euros of extra sales over +> twenty weeks. Remember 284. Every method gets graded against it. + +## Slide 16 — Simulate the world (live) **[FAST]** + +> [Play with one slider only — the macro shock.] +> +> Ten seconds of demonstration. The dashed blue line is the counterfactual itself — the metro +> in the world without the campaign. In real life this line does not exist. Watch when I make +> the economy more turbulent: the naive before/after estimate becomes garbage, because the tide +> dominates. "Sales went up after we launched" is not evidence. The question is always: +> compared to what? + +# ACT II — THE COUNTERFACTUAL + +## Slide 17 — Abadie's idea: the synthetic twin + +> So how do we fill the missing cell? The idea is genuinely simple. +> +> If our metro had an identical twin — same size, same seasonality, same sensitivity to the +> economy — we could just watch the twin. The twin got no campaign. Its sales during those +> twenty weeks are our answer. +> +> No single market is a perfect twin. But a **blend** can be. Forty percent of market eight, +> plus 32 percent of market twenty, plus 17 percent of market three behaves almost exactly like +> our metro. That blend is the synthetic control — a manufactured twin. +> +> The key property, in business terms: the twin is built to **react to shocks** the same way +> our metro reacts. When the economy dips or the season turns — even in ways nobody predicted — +> the twin dips and turns with it. Whatever gap remains between metro and twin is the campaign. +> +> Strong pedigree: called the most important innovation in policy evaluation in fifteen years, +> and it is the engine inside Meta's GeoLift and Google's geo tools — the tools your media +> agencies already use. + +## Slide 18 — The estimator **[SKIPPABLE MATH]** + +> This slide is the recipe in formulas. Three business facts, then you may ignore the algebra. +> +> Fact one: the twin is fitted **only on data from before the launch**. The method never sees +> the campaign period while it learns, so it cannot cheat. Same discipline you demand from any +> forecast: no peeking. +> +> Fact two: the blend must be a real recipe — weights positive, summing to one hundred percent. +> You get statements like "40 percent market eight plus 32 percent market twenty". A manager +> can read that, question it, audit it. No market enters at minus 80 percent. It is the +> weights-on-the-table version of what Colgate's projection did in time. +> +> Fact three: after fitting, the effect is a subtraction — actual sales minus twin sales, +> summed over the campaign weeks. Nothing exotic. + +## Slide 19 — Inside the convex hull (live) **[FAST]** + +> [Drag the treated market in and out of the hull — thirty seconds.] +> +> One picture on why the "real recipe" rule matters. A blend of real markets can only reproduce +> a market that lives **inside** the range of its donors. You can mix coffee and milk; no mix of +> coffee and milk gives you orange juice. When the target sits outside that range, an +> unconstrained model still produces a number — by extrapolating. The constrained one **refuses**. +> +> And the refusal is a feature, not a bug. It is the method telling you: widen the donor pool, +> do not trust me here. So make this a procurement question. Ask any vendor: when does your +> method refuse to answer? If the answer is "never", walk away. + +## Slide 20 — What the constraint buys (live) **[FAST]** + +> Here we removed the recipe rule and let an ordinary regression do what it wants. Look: the +> unconstrained model fits the past **better** — and predicts the missing counterfactual +> **worse**. It used absurd weights: minus 80 percent of one market, 190 percent of another. +> It memorized noise. +> +> The management lesson: a tighter fit to history is not a better model, and is often a worse +> one. Anyone selling you a model on "look how well it fits the past" is selling the wrong +> metric. You saw the same inversion this morning — the MMM with the backwards ranking fitted +> its history beautifully. + +## Slide 21 — Why not just forecast it? + +> Somebody always asks: why this old-fashioned blend? Why not gradient boosting, Prophet, a +> neural network? We ran that race. The fancy forecaster fits the past better — again. Its +> campaign estimate lands in the same place as the simple twin. And across twenty-four fresh +> simulated worlds it reconstructs the missing counterfactual clearly worse on average, with +> weights no planner can audit. +> +> The deep reason is a business reason. A forecaster answers: "what comes next, in a world like +> the past?" Our question is: "what would **this** market have done in a world that never +> happened?" You cannot cross-validate that answer — the truth is never observed. So accuracy +> on the past cannot pick your tool. You must pick the tool whose **assumptions you can defend +> in the boardroom**. The simple auditable blend wins the causal job even when it loses the +> forecasting beauty contest. + +## Slide 22 — Every shortcut is an assumption (live) + +> The slide I most want you to remember from Act II. Four common ways to measure the same +> campaign: before/after; treated versus the average of other markets; difference-in- +> differences — the consultant's favorite; and synthetic control. +> +> Each is really a hidden **claim** about the missing counterfactual. Before/after claims the +> world would have stood still. Treated-versus-control claims other markets are just like +> yours. Difference-in-differences claims all markets ride the economy in parallel. Synthetic +> control claims a blend of markets can mimic yours. +> +> [Live demo: move the two dials.] +> +> Watch the errors as I change the world. When markets react to the economy differently — the +> normal case — before/after and diff-in-diff drift away from the truth. Synthetic control +> stays close, because it matched how the metro **responds**, not just its level. +> +> And the uncomfortable fact: none of these numbers arrives with a warning label. A wrong +> method does not look wrong. More data does not fix it — a wrong claim stays wrong at any +> sample size. The cure is a better claim, not a bigger spreadsheet. + +## Slides 23–24 — The one-difference estimators / DiD algebra **[SKIPPABLE MATH]** + +> These two slides do in algebra what the dials just showed in pictures: exactly when +> before/after, treated-versus-control, and difference-in-differences give the wrong answer. +> The executive summary is enough: +> +> Before/after fails whenever the world does not stand still — so, almost always. Comparing to +> other markets fails because markets differ in size. Difference-in-differences fixes size but +> still requires every market to ride the economy in parallel — and that assumption dies +> quietly, no alarm. The one bias diff-in-diff cannot remove is precisely the one synthetic +> control removes by construction, because it **chooses** the comparison weights instead of +> hoping equal weights work. +> +> One line worth repeating to any analyst who reports to you: more weeks of data buy precision +> around the **same wrong number**. If the method is biased, patience does not help. Ask which +> assumption the comparison rests on. No answer — the number is decoration. + +## Slide 25 — Identification: what must hold + +> Every method rests on assumptions. The professional standard: list them, test what can be +> tested. Read this as a due-diligence checklist. +> +> One: the twin must track the metro before launch. Testable — it passes, with a quantitative +> gate. Two: sales must not react **before** the campaign — no leaked launch, no stockpiling. +> Testable with a placebo you will see shortly — passes. Three: the campaign must not touch the +> comparison markets. Ads do not respect borders. This one is **untestable** with statistics — +> it is Colgate's second-launch leak, in space. Mark it; it returns in ten minutes. Four: the +> metro must be reconstructable from the donors, and no single market may hold the answer +> hostage — drop any donor, the answer barely moves. Passes. +> +> Three passes, one honest "cannot test". Remember which one. + +# ACT III — IS IT REAL, AND HOW BIG? + +## Slide 26 — Poll: is it real? + +> The twin says: about 260 thousand euros of extra sales over twenty weeks. New question: real, +> or a lucky metro? Normally you would run a significance test. Here you cannot: every standard +> test measures luck across **many treated units**, and we have one treated metro. You cannot +> measure variation across one thing. The standard machinery is not weak here — it does not +> exist. +> +> And you can invent the replacement yourselves. [Read the poll.] Take each of the 29 markets +> that ran **no campaign**. Pretend, one at a time, that it was the treated one. Run the whole +> twin machinery on it. It should show roughly zero — nothing happened there. Whatever "effect" +> it shows anyway is pure luck: the noise level of our method. +> +> Thirty markets, thirty measured "effects", twenty-nine of them known to be luck. Where does +> our metro's 260 thousand rank? +> +> [Collect. Reveal.] +> +> Rank one of thirty. The most extreme of all. And the rank **is** the test: if the campaign +> truly did nothing, our metro would be just another market, and the chance of ranking first by +> luck is one in thirty — about three percent. You have just reinvented a rigorous statistical +> test using nothing but common sense. + +## Slide 27 — Placebo-in-space (live) + +> The idea, drawn. The grey cloud: 29 fake "effects" — the luck of the method, measured +> directly. The green line: our metro at 260, standing clear outside the whole cloud. +> +> Conclusion: the effect is real, with about a three percent chance we are fooling ourselves. +> One caveat for your analysts: a comparison market that fits badly can fake a big effect, so +> bad-fit placebos are dropped before ranking — the standard hygiene rule is on the slide. +> +> But hold the applause. "Real" is not "profitable". That is question three, and we are not +> there yet. + +## Slide 28 — Interval by test inversion (live) **[SKIPPABLE MATH]** + +> One more product from the same placebo idea, then money. We have a point estimate, 260. A +> board should never accept a point without a range. The range: 195 to 335 thousand, with 90 +> percent confidence. +> +> How it is built, in one sentence: for every possible "true" effect, we ask whether that truth +> could plausibly have produced our 260 given the luck we measured — and the survivors form the +> range. The mechanics are on the slide for the curious; the demo lets you drag the hypothesis. +> +> Why care about this particular range: it was built with **no model of the noise** — no bell +> curve, no software assumptions, only the placebo logic you invented by hand. That makes it +> the referee for every fancier number that comes later. Any model that contradicts this range +> has explaining to do. + +## Slide 29 — Falsification 1: placebo-in-time **[FAST]** + +> Two stress tests before the money. First: pretend the launch happened ten weeks earlier, and +> re-run everything. A real method should find **nothing** there — nothing happened. It finds +> nothing: about 1.7 thousand, inside noise. Pass. +> +> What this rules out: leaked launches, stockpiling, sales reacting early. Make it a habit: any +> vendor claiming a lift should show you their method finds zero where nothing happened. + +## Slide 30 — Falsification 2: spillover (SUTVA) + +> The second stress test is the one I flagged as untestable — the disease you named for Colgate +> this morning, now in space. Our metro's TV and outdoor ads reach the commuter belt. Those +> neighboring markets are in our comparison pool. So the campaign quietly lifts the very +> markets we use as "untreated". The twin gets inflated, and the measured effect **shrinks**. +> +> We cannot test for this in the data — the leak starts exactly at launch, so every check stays +> green. What we can do is simulate it deliberately and measure the damage: with a realistic +> leak, our 260 becomes 236. Two takeaways. +> +> First: the bias only pushes the number **down**. Our estimate is a conservative floor — if the +> campaign clears its cost at 260, a leak cannot overturn the decision. +> Second, and more important: the fix is not statistical, it is managerial. When you **design** +> a geo test, keep the control markets off the campaign's media footprint. The best analysis +> cannot repair a badly designed test. Design beats analysis. Every time. + +## Slide 31 — What the statistics says + +> Act III complete. Three numbers to carry. +> Is it real? Yes — three percent chance of self-deception. +> How big? 260 thousand euros of incremental sales. +> Give or take? 195 to 335, at 90 percent confidence. +> +> And a satisfying check only a simulation allows: the planted truth, 284, sits inside the +> range. The machinery is honest. +> +> Now — the sentence that loses money in real boardrooms: "The campaign drove 260 thousand of +> sales for 75 thousand of spend. That is a 3.5x return. Roll it out." Every number in that +> sentence is true. The conclusion still does not follow. Why not? Act IV. + +# ACT IV — THE DECISION IN EUROS + +## Slide 32 — Poll: the margin + +> [Read the poll.] The campaign cost 75 thousand. It generated 260 thousand of incremental +> sales. Gross margin is 35 percent. What did the campaign **earn**? +> +> [Collect. Reveal.] +> +> About 16 thousand net. Margin times lift, minus cost: 35 percent of 260 is 91, minus 75 — +> about 16. If you answered "185 — sales minus cost", you made the boardroom classic: you +> treated revenue as profit. The company keeps only the margin on the extra sales; it pays the +> campaign invoice in full. Nürnberger's last-touch trap was a cousin of this one: the +> comfortable metric, wrong by construction. + +## Slide 33 — The margin trap (live) + +> The general rule — the cheapest lesson of the day. "Return on ad spend" compares incremental +> **sales** to cost, so its break-even is not 1. It is 1 divided by your gross margin. At our +> 35 percent margin: 2.86. You need 214 thousand of incremental sales just to get your 75 back. +> +> [Move the margin slider.] +> +> A software company at 90 percent margin breaks even at 1.1. A grocery chain at 20 percent +> needs 5.0. The **same campaign with the same measured lift** is a good buy for one business +> and a money-loser for the other. If your agency reports "ROAS above 1" as success, that is a +> vanity bar. The real bar is 1 over margin. Ours: 3.47 achieved against 2.86 required. It paid. + +## Slide 34 — The decision space (live) **[FAST]** + +> Now bring the uncertainty back. Expected profit says the campaign pays at any price below 91 +> thousand. But the 90-percent range on profit runs 68 to 117 — and our price, 75, sits +> **inside** it. Translation: probably paid, but a loss is still consistent with the evidence. +> [Slide the price control.] Between "looks profitable on average" and "provably profitable" +> there is a wide dangerous zone, and our campaign lives in it. + +## Slide 35 — What you say to the board (classical) + +> The memo a competent analyst delivers with classical tools only. +> Real? Yes. Big enough on the point estimate? Yes — about 16 thousand net. +> Confident? **No.** The profit range straddles the price. +> +> Verdict on the €75k pilot: probably a profitable buy, not a confident one. Renegotiate the +> price, or buy certainty with a bigger test. And say the second sentence out loud: this memo +> says **nothing yet about the €4M** — the rollout is not the pilot multiplied. +> +> One more thing, in the last box. There are questions this memo cannot answer, in principle: +> "what is the **probability** the campaign pays?", "what is the highest price this evidence +> would still justify?", "is a follow-up study worth its cost?". Classical statistics answers +> yes/no about where a range sits; it has no language for "78 percent chance" — the Bayesian +> word from slide 3. Managers allocate money in exactly that language. That language exists — it +> is the Bayesian layer of this same toolkit, and it is where this course goes next. Today we +> stop at the honest classical verdict, which is already worth real money — watch. + +## Slide 36 — What €4M actually buys + +> We still owe the board the real question: the four million. Here is the number in the +> marketing deck: 4 million is 53 pilot budgets; 53 times the pilot's lift, at our margin, +> nets about **plus 860 thousand**. Every step is correct. +> +> [Poll the room: who approves the €4M? Same device as slides 13 and 32 — let them commit.] +> +> Now do the one thing the marketing deck did not: carry the **interval**, not the point. And +> be maximally generous — assume the campaign works nationally exactly as well as in the metro. +> Perfect transport, every euro. Even then, the 90 percent profit interval on the 4 million +> runs from **minus 360 thousand to plus 2.25 million**. It straddles zero. At the most +> optimistic assumption we can write down, we cannot show the 4 million pays. +> +> And that assumption is a ceiling. Two forces only pull it down: 4 million over 30 markets is +> 133 thousand per market — nearly double the pilot's intensity, and advertising has +> diminishing returns; and one metro is not the country. [Drag δ down.] Below about 64 percent +> surviving, the rollout provably loses. +> +> That is the whole frequentist verdict, and it needs no probability model: at its best case +> the 4 million is not provably profitable, and it can provably lose. **Do not commit the 4 +> million on this evidence.** The next slide is what you do instead. + +## Slide 37 — Measure it, don't guess it + +> The answer is not "yes" and not "no" — it is "not on this evidence". And notice why the +> evidence cannot decide: the interval is wide because there is **one** treated market. You +> cannot think your way out of that. Re-running the model, adding weeks, arguing about +> assumptions — none of it narrows the interval. Only more treated markets do. +> +> [Ask: so what would you spend? Take answers. The honest one is "not four million yet" — that +> is the lesson, not a dodge.] +> +> Two moves. First, step zero — it gets a laugh because it is so obvious and so often skipped: +> before you spend a million on evidence, spend an hour checking you have not already bought +> it. A firm with a 4-million budget has prior pilots, a media-mix model, agency benchmarks. +> The answer may be in a drawer. +> +> Second, if it is not: measure it, do not model it. Run the campaign in **eight markets chosen +> to look like the nation**, at full national intensity — about 133 thousand each, 1.07 million +> total. Because the cells span the country, the measured lift **is** the national lift, with +> an interval tight enough to clear the 4 million or kill it. And recognize the shape: this is +> HelloFresh's loop from this morning, bought once — the model asked for ground truth, and we +> went and bought the experiment. +> +> The discipline that separates a test from theatre: write the release rule **before** the data +> lands. "Release the remaining 2.9 million only if the measured profit interval clears zero." +> One quarter later you have a national campaign at a size the evidence supports. +> +> [Delivery notes: "transportability" is classroom language — in a client room say "how much of +> this travels". The fold prices this information at about 320 thousand euros using the course's +> Bayesian machinery — far above the test's true cost — but the decision to test was already +> made here, classically.] + +### BRIDGE — Part 2 → Part 3 (say this before the break, ~30s) + +> Look at what synthetic control needed to work: a clean group of untreated markets to build the +> twin from. When you have that, you can rescue a decision even without an experiment. But +> sometimes you do not have it. Sometimes the thing you want to measure was assigned by someone +> whose whole job is to *not* be random — a platform picking who sees an ad, a doctor choosing +> who gets a drug, a bank deciding who gets a loan. No clean control group exists, and you cannot +> run the test yourself. That is the last and hardest case, and it has its own tool. After the +> break: what one ad exposure is worth, when the platform already chose who sees it. + +--- +--- + +# PART 3 — INSTRUMENTAL VARIABLES + +## Slide 38 — Part 3 divider + +> Welcome back. One more decision, one more tool, and then we are done. Same discipline all the +> way through — a counterfactual, an anchor, a price — but the anchor comes from a surprising +> place. +> +> The picture on the right is the whole part in one image. A dashboard hands you a **tight** +> number and it is **wrong**. A lottery — you will see which lottery — hands you a **wider** +> number that is **right**, and still clears the price. By the end you will know exactly why the +> tight number is the dangerous one. + +## Slide 39 — The case + +> The setup. An online retailer pays an ad platform to show display ads. One **exposure** is one +> user who actually saw an ad, and each exposure costs 10 euros, billed by the platform. Who +> sees an ad is decided by the platform, in a real-time auction — not by us. +> +> The dashboard says: users who saw an ad brought in about 24 euros more than users who did not. +> So the platform's pitch is the obvious one: "an exposure is worth 24, it costs 10 — raise the +> budget." Your decision is three-way: keep paying 10 per exposure, pay more, or stop. One +> number settles it: how many euros of extra sales does **one exposure** actually cause? And all +> you have is the platform's own logs — who saw an ad, and what every user spent. +> +> Hold that against the pitch. Is 24 euros that number? + +## Slide 40 — Poll · the pitch **[✋]** + +> [Read the poll.] The dashboard's 24-euro gap is real, the arithmetic is right, and at this +> sample size it is comfortably significant. Given that, what do you make of "raise the budget"? +> +> [Collect. Reveal.] +> +> C. The problem is not the arithmetic and not the sample size — so A and B both miss it. The +> problem is who is being compared. The platform **chose** who saw the ads: its whole job is to +> find the users most likely to buy. So the exposed group was more likely to buy **before any ad +> ran**. We are comparing willing buyers to the general public and calling the difference "the +> ad". This is Colgate's disease and the geo metro's disease, a third time: compared to what? + +## Slide 41 — Confounding, defined **[SKIPPABLE MATH]** + +> The case has exactly three quantities, and naming them cleanly is half the battle. On 3,000 +> customers: **sales** — what each customer spent, our outcome. **Exposure** — one or zero, did +> they see an ad, assigned by the auction. And a third one, the villain: **intent** — how ready +> a customer was to buy before any ad was ever shown. +> +> Intent is real, it drives buying, and it is nowhere in the data — it is a feeling in a +> shopper's head, in no log file, never observed. And it does two things at once: it makes a +> customer more likely to buy, **and** it makes the auction more likely to show them an ad. A +> hidden thing that drives both the treatment and the outcome has a name — a **confounder** — and +> it is the reason the dashboard's 24 is not the ad's effect. The diagram is on the slide; the +> word to keep is confounder. + +## Slide 42 — The naive comparison (live) + +> Let us write the dashboard's number down honestly and watch it break. [Run the live figure.] +> +> Each customer has two possible spends: what they would spend **with** an ad, and what they +> would spend **without** one. We only ever see one of the two. The exposed group shows us their +> "with" number; the unexposed show us their "without". The dashboard subtracts those two +> averages — 68 minus 44, about 24 — and quietly pretends the unexposed group's spending is a +> fair stand-in for what the exposed group **would** have spent with no ad. +> +> [Tick "reveal the hidden driver".] There it is: the two groups already differed in intent +> before any ad. The gap is part ad, part pre-existing difference — and the dashboard cannot +> tell you the split. The arithmetic is right. The comparison is wrong. + +## Slide 43 — Poll · read the gap **[✋]** + +> [Read the poll.] So which reading of the 24 is actually defensible? +> +> [Collect. Reveal.] +> +> B. The number is not meaningless — the arithmetic is sound, so C is out. But A books the whole +> 24 to the ad, and D books the whole 24 to targeting, and the honest truth is that it is a +> **mix of the two in unknown proportions**. That is the only statement the data alone support. +> Everything from here is one question: how do we split the 24 into the ad's part and the +> pre-existing part? + +## Slide 44 — Selection bias, defined **[SKIPPABLE MATH]** + +> The split has a name and a formula, and the formula is worth ten seconds. The naive gap is +> exactly two things added together: the **effect of the ad** on the people who saw it, plus a +> **selection bias** — the gap that would have existed with no ads at all, purely because the +> platform picked buyers. +> +> Read the second term slowly: it is the exposed group's "without-ad" spending minus the +> unexposed group's "without-ad" spending. If the platform picked well, that is a big positive +> number all by itself — and it is contaminating every euro of the 24. Our whole job is to kill +> that second term. + +## Slide 45 — The simulated world **[FAST]** + +> Same honesty move as the geo half: we simulate, so we can grade. [Run the figure, push one +> slider.] We planted the true ad effect at exactly 15 euros. So every method in this part has a +> known answer to be graded against — and the dashboard's 24 is already wrong by nine. +> +> Watch the slider: as I make intent pull harder on sales, the two groups drift further apart — +> the dashboard's gap grows — while the planted truth, 15, never moves. Bigger gap, same truth. +> That is confounding, live. + +## Slide 46 — Endogeneity, precisely **[SKIPPABLE MATH]** + +> One inequality states the entire problem, and I will translate it. Collect everything the ad +> does not explain about a customer's spending into one bucket — call it "every other reason +> this person spent money". Intent is the big item in that bucket. +> +> The trouble is that exposure is **correlated** with that bucket — because the auction +> deliberately targets the people with the most other reasons to spend. The math writes that as +> a covariance greater than zero; the name for it is **endogeneity** — the treatment is tangled +> up with the error term. Whenever the thing you are measuring was **chosen** using information +> you cannot see, you have it. It is the formal disease behind all three of today's cases. + +## Slide 47 — The size of the bias (live) **[SKIPPABLE MATH]** + +> The error is not vague hand-waving — it has a formula, and the formula teaches three things. +> [Show the live readout.] It predicts the naive gap should land near 23.5; the data delivered +> 23.7, against a truth of 15. So we can literally compute the lie. +> +> Three lessons. One: the bias is **systematic**, not bad luck — both pieces are positive, so the +> dashboard can only ever **overstate** the ad. Two: it grows with how hard the platform targets +> and with how much intent drives sales — the better the ad platform, the bigger the lie. Three, +> the one that ends the first act: sample size is nowhere in that formula. More data does not +> shrink this. It never will. + +## Slide 48 — The limits of adjustment + +> The natural objection — surely we just control for intent? — and why it fails here. The +> standard fix is exactly that: compare exposed to unexposed **within groups of equal intent**, +> so intent can no longer separate them. Controls in a regression, matching, propensity scores — +> all the same move, and normally the right one. +> +> [Tick the box: "pretend intent were logged".] If we could see intent, it works — recovers the +> planted 15 almost exactly. [Untick it.] But intent is a feeling in a shopper's head. It is not +> in the logs, and it never will be. You cannot control for a column you do not have. The +> standard toolkit runs out of road right here — and that is what makes this the hard case. + +## Slide 49 — Poll · what would fix it **[✋]** + +> [Read the poll.] Adjustment failed because intent was never recorded. Your team can request +> **one** addition to the data. Which one actually lets you measure what an exposure causes? +> +> [Collect — this one genuinely splits a room. Reveal.] +> +> C. A batch of users whose exposure we assign **at random**. Here is why the others lose. Ten +> times more of the same logs (A) gives you the wrong number more precisely — remember, sample +> size is not in the bias formula. The platform's targeting features (B) or last year's spending +> (D) are just more controls, and the auction reacts to live signals no export fully captures, +> so the backdoor stays open. Only randomness breaks the link between exposure and intent. The +> instinct is right. The problem is you cannot randomize the ads — the platform owns that +> auction. So we need randomness we can get our hands on. + +## Slide 50 — The ideal experiment + +> Let us be precise about what randomization would buy, so we know what to hunt for. In today's +> world, intent points at two things: it drives sales, and it drives who gets an ad. [Show the +> two diagrams.] If a coin flip decided exposure instead of the auction, that second arrow — from +> intent into exposure — is not blocked, it is **gone**. Exposure would no longer listen to +> intent at all, and the naive comparison would become the right comparison. +> +> That is the target. We cannot randomize exposure directly. But we do not have to. We only need +> something random that **nudges** exposure — and it turns out we already ran one. + +## Slide 51 — The lottery + +> Here is the move, and it is the heart of the method. Before the campaign, our own system ran a +> **serving-priority lottery**. A random number generator we control picked half the users and +> gave them a small priority boost in the ad auction. Nothing more. +> +> Read what the lottery does carefully, because it is subtle: it shows **nobody** an ad. It only +> raises the **odds** that the auction serves you one. Winners are a bit more likely to be +> exposed; that is all. But the winners were chosen by our own coin flip — so lottery status is +> completely blind to intent. We have manufactured a scrap of pure randomness, sitting right +> next to the thing we cannot randomize. That scrap has a name. + +## Slide 52 — The instrument, defined + +> The scrap of randomness is called an **instrument**. The definition, in words: an instrument is +> something that moves the treatment, is as good as random with respect to the hidden confounder, +> and touches the outcome **only** through the treatment. +> +> Four conditions make that precise, and the slide grades our lottery against each. **Relevance**: +> the lottery really moves exposure — testable, and we will measure it. **Exogeneity**: the +> lottery is blind to intent — true by design, because we drew it. **Exclusion**: the lottery +> affects sales only by changing exposure, nothing else — an argument, not a test. **Monotonicity**: +> the nudge never pushes anyone the wrong way. Two we can lean on for free because we built the +> lottery; two we have to argue. Keep that scorecard — it is how you judge any IV study you are +> ever shown. + +## Slide 53 — Condition 1 · relevance **[SKIPPABLE MATH]** + +> The one condition the data can settle, so let us settle it. Relevance asks: did the lottery +> actually move exposure? The measure is called the **first stage** — the extra exposure the +> lottery created. [Point at the two bars.] +> +> Winners were exposed about 21 percentage points more often than losers. That is the whole first +> stage: 0.21 extra exposures per lottery win. It is real, it is measured, and it is strong — the +> handle we will turn in a moment. If this number had been near zero, the method would be dead on +> arrival; hold that thought for the "weak instrument" slide. + +## Slide 54 — Conditions 2 and 3 (live) + +> The two conditions we have to **argue**, because no test can prove them. **Exogeneity** — the +> lottery is blind to intent — we get for free: we drew it with our own random number generator, +> so it cannot favour ready buyers. That one is solid. +> +> **Exclusion** is the one that keeps you honest. The lottery must touch sales **only** by +> changing whether someone sees the ad — through no other door. [Show the live side-channel.] If +> winning the lottery also, say, loaded the page a little faster, or flashed a different price, +> that side effect would leak straight into our answer — the formula on the slide shows the bias +> it injects. This is the untestable assumption, the exact cousin of the geo spillover. You do +> not test it. You **design** it out and you argue it in plain words: a queue-priority bump shows +> the user nothing different but the ad itself. + +## Slide 55 — The reduced form **[SKIPPABLE MATH]** + +> Second measured number, same shape as the first. The first stage was the lottery's effect on +> exposure. The **reduced form** is the lottery's effect on **sales**: the raw sales gap between +> lottery winners and losers. +> +> It is about 3.48 euros. Winning the lottery is worth three and a half euros of extra sales for +> the average user — and it is clean for the very same reason the first stage was: a random draw +> cannot favour ready buyers, so no intent contaminates it. Now hold the two clean numbers side +> by side: a lottery win buys 0.21 extra exposures, and 3.48 euros of extra sales. + +## Slide 56 — The IV estimate + +> This is the payoff of the whole part, and it is one division. A lottery win caused 0.21 extra +> exposures. That same lottery win caused 3.48 euros of extra sales. If those extra sales came +> **only** through the extra exposures — which is exactly what exclusion guarantees — then the +> value of one exposure is forced: 3.48 divided by 0.21, about **16 and a half euros**. +> +> Check the units — they are the intuition: euros per win, divided by exposures per win, leaves +> euros per exposure. And notice what just happened to intent: it cannot correlate with a random +> draw, so it contributes nothing to the top and nothing to the bottom. It has been divided out. +> The dashboard said 24. The truth we planted was 15. The lottery says 16 and a half — the honest +> number, and the first one all session that is not poisoned by who the platform picked. + +## Slide 57 — Why the division is forced (live) **[FAST]** + +> Thirty seconds to feel why that division is not a modelling choice but the **only** answer the +> two numbers allow. [Hand a student the guess slider.] Pick any candidate value for one +> exposure. It makes a prediction: an effect that size, times the 0.21 exposures the lottery +> created, is how much the lottery should have moved sales. The flat line is the fact — the +> lottery moved sales by 3.48. Only one guess makes the prediction meet the fact, and it is +> sitting at 16 and a half. Every other number contradicts something we measured. + +## Slide 58 — Weak instruments (live) + +> The most important failure mode to recognize, because it is the one that quietly bites. [Drag +> the lottery-strength slider down.] A **weak** instrument is one that barely moves exposure — a +> tiny first stage. Watch what happens: the interval explodes, and — this is the dangerous part — +> the centre creeps **back toward the naive number**. You are dividing by a number close to zero, +> and that resurrects exactly the bias you came to remove. +> +> A weak instrument is worse than no instrument, because it looks like a method while quietly +> handing back the dashboard's lie. The defence is one number, the first-stage F. The rule of +> thumb is above ten; ours, you will see, is 156. Always ask for it. + +## Slide 59 — Compliers and the LATE (live) **[SKIPPABLE MATH]** + +> One honest caveat about **whose** number the 16 and a half is. [Show the three slices.] The +> lottery only teaches us about the people it could actually move — the **compliers**, who saw an +> ad because they won and would not have otherwise. Some users are always shown the ad regardless +> of the lottery; some never are. The lottery is silent about both of those groups, because it +> changed nothing for them. +> +> So the estimate is the effect **for the movable**. In plain business terms, that is a feature, +> not a bug: the movable users are exactly the ones your next euro of budget will reach. It is +> the right number for the decision on the table. The name, for the curious, is the local average +> treatment effect — the LATE. + +## Slide 60 — The checklist + +> Compress the whole method to a scorecard you can use on anyone else's IV study. Four +> assumptions. **Relevance** — is the instrument strong? Measured: F of 156, far above 10. Passes, +> and it is the only one you can test. **Exogeneity** — is the instrument really random? Ours is, +> by design. **Exclusion** — does it touch the outcome only through the treatment? Untestable; you +> argue it. **Monotonicity** — no one pushed the wrong way? Untestable, but plausible. +> +> That is the honest shape of every instrumental-variables claim: one number you can check, one +> design guarantee, and two arguments you must make in words. When a vendor shows you an IV +> result, ask for all four. If they only show you the estimate, you have been shown the least +> important part. + +## Slide 61 — The price map (live) + +> Now the estimate becomes a decision, and it is the cleanest decision slide of the day. [Drag +> the price.] One estimate does not give one answer — it gives a **map** from any price to a +> verdict. Below the interval's floor, 12.7, even the most pessimistic supported effect pays: +> that is the **GO** zone. Between 12.7 and 20.4 the interval straddles the price: **TEST** — +> negotiate or measure more. Above 20.4, no supported effect pays: **NO-GO**. +> +> And where does today's rate sit? Ten euros — comfortably inside GO, below even the floor. On +> this evidence, the ad pays. + +## Slide 62 — Poll · the negotiation **[✋]** + +> [Read the poll.] The platform wants to renegotiate the rate up. Your analyst hands you the +> causal read: best estimate 16.5 per exposure, 90 percent interval 12.7 to 20.4. What is the +> highest rate at which you would still sign "buy" **without** commissioning more study? +> +> [Collect. Reveal.] +> +> C — 12.7, the floor of the interval. That is the no-regret cap: below it, **every** effect the +> data support still pays, whichever value inside the interval turns out to be the truth. Paying +> the point estimate, 16.5, is a coin-flip gamble — half the supported effects lose. The floor is +> the number a careful manager signs. You have just priced a bid cap off a confidence interval. + +## Slide 63 — The verdict and the recommendation + +> The whole part on one page. Effect of one exposure: 16 and a half euros, from the division. The +> 90 percent interval: 12.7 to 20.4 — and note the classical placebo-style referee, the AR +> interval, agrees, exactly as it refereed the geo number. First-stage F of 156, so the lottery +> is strong. Price, 10. Net, about six and a half euros per exposure at the point estimate. +> +> The verdict is **buy**: keep paying 10, because the entire defensible range clears the price. +> The recommendation in three lines: keep buying at 10; cap the bid at 12.7, the no-regret floor; +> and go back to measure again only if the platform's rate ever climbs toward that floor. Notice +> what this whole part was: entirely classical — two averages, one division, one F, one interval — +> and it answered a question the dashboard could not, by finding one scrap of randomness we +> controlled. That is instrumental variables. + +--- +--- + +# FINALE — THE PATTERN + +## Slide 64 — The tools were the product too **[FAST]** + +> Almost done. Three footnotes that are really credentials. +> +> Everything you just used exists as open source, industrialized by the same consultancy: +> CausalPy carries synthetic control, interrupted time series, difference-in-differences, +> regression discontinuity — and the instrumental-variables estimator you just used to price the +> ad exposure. Its launch example is the sentence this session opened with: exposure you cannot +> randomize, causal impact you still need. PyMC-Marketing is the MMM library behind the +> calibration story — and one client's budget-allocation approach came back as a pull request, +> which is what a healthy open-source business looks like. And the consultancy's own webinar +> agenda — geo tests, MMM, synthetic control, diff-in-diff, instrumental variables — is literally +> this session's syllabus. You did not take a course *about* the industry today. You took the +> industry's course. + +## Slide 65 — The pattern in every engagement + +> Compress the whole session to three lines. +> The deliverable is a **counterfactual** — a world minus the launch, the campaign, the +> exposure — priced in euros. +> An **experiment anchors** every observational model. And when you cannot run one, you hunt for +> a scrap of randomness the world already made: that was the instrument. Calibration is the +> product, not a luxury. +> And **uncertainty prices the decision** — a board acts on the interval and on the headroom, not +> on a point estimate. Twice today the honest interval overturned the confident point. +> +> One last exhibit, because it is 2026: we ran a vanilla coding agent on deliberately +> adversarial marketing data. It fit a model and confidently recommended budget reallocations. +> Confidently wrong. The honest system explored eleven model specifications, none converged, +> and returned: "No valid model found. Run a geo-holdout experiment." Even the machines, at +> their best, arrive at the advice under everything today: measure it. + +## Slide 66 — One breath + +> One breath, and we stop. What a Bayesian consultancy actually sells: counterfactuals, +> calibrated by experiments, priced as probabilities. You have now watched real clients buy it, +> and you have built two yourself — a four-million-euro rollout you learned to refuse, and the +> price of an ad exposure you learned to trust. +> +> Every number today is pinned to a public source or baked from the executed course notebooks — +> the list is on the backup slide. The notebooks behind this session are the course repository. +> Both of us consult for PyMC Labs: say hello. + +## Slide 67 — Sources (backup) + +> [Only if asked.] Every fact from Part 1, pinned to its public post with the exact quote and +> retrieval date; Part 2's and Part 3's numbers are baked from the executed course notebooks and +> verified against a claims registry at build time. Nothing on these slides was typed by hand. + +--- +--- + +# If you are told "you have 60 minutes" on the day (Parts 1–2 only, hold Part 3 for a follow-up) + +Cut in this order: slide 64 (provenance) → 23–24 to their one quoted line → 20 → 15–16 to one +sentence each → 34 (fold its point into 35) → the second poll of whichever act runs long. +Never cut: 2–7 (who we are + the hook), 11–14, 17, 19, 22, 26–27, 30, 32–33, 35–37 (36–37 +above all), 65–66. + +# If Part 3 runs and you are tight (~30 min for IV) + +Fast or fold: 41 (name the confounder, fold the DAG), 44 (say "ad part plus targeting part", +fold the algebra), 45, 46 (say "endogeneity = chosen by what you can't see", fold), 47, 53 +(just "21 points more exposure"), 55 (just "3.48 per win"), 57, 59 (one sentence on "whose +number"). Never cut: 39, 42, 48–52, 56 (the division), 60 (the checklist), 61, 63 (the verdict). + +# The engagement beats (do not silently skip) + +Polls: 4 (route the call) · 9 (price the engagement) · 13 (the 12% bump) · 26 (rank the metro) +· 32 (the margin) · 36 (approve the €4M — hands, before the interval) · 40 (the pitch) · 43 +(read the gap) · 49 (what would fix it) · 62 (the bid cap). Open floor: 6 (what would break it). +Live widgets to hand to a student: 6 (second launch), 7 (lift-test toggle), 16 (macro shock), +22 (estimator dials), 28 (drag the hypothesis), 33 (margin slider), 36 (drag δ), 42 (reveal +intent), 45 (κ slider), 47 (bias readout), 48 (log intent on/off), 54 (side-channel), 57 (guess +the effect), 58 (weaken the lottery), 59 (complier slices), 61 (drag the price). The polls are +the lecture's spine: commitment before revelation, every time. diff --git a/causal-marketing-pymc/apps/geo_lift_sh.html b/causal-marketing-pymc/apps/geo_lift_sh.html index 717ffa8..d0d46c3 100644 --- a/causal-marketing-pymc/apps/geo_lift_sh.html +++ b/causal-marketing-pymc/apps/geo_lift_sh.html @@ -1210,7 +1210,7 @@

Measure before you commit

+ + + +
+ + +
+
+
+
Causal Inference & XAI for Business · SDA Bocconi
+

Instrumental Variables

+
What is one ad exposure worth, when the platform already picked who sees the ads?
+
A guest lecture for Prof. Michele Russo's course
+
+
+
Francesco Muia
+
PhD in Theoretical Physics, EMBA.
Consultant for PyMC Labs and Brown University.
+
francesco.muia@pymc-labs.com
+
+
+
Alexander Fengler
+
PhD in Computational Cognitive Science.
Postdoc at Brown University, consultant for PyMC Labs.
+
alexander.fengler@pymc-labs.com
+
+
+
+ PyMC + PyMC Labs +
+
+
+ +
Where the lecture lands: the dashboard's tight interval misses the true effect of an ad exposure, while the interval built from a lottery covers it, and still clears the price.
+
+
+
+ + +
+
Part 1 · The problem
+

The case

+
One retailer, one ad platform, one decision to make.
+
+
+
The product
An online retailer pays an ad platform to show its display ads. One exposure = one user actually saw an ad.
+
The price
Each exposure costs €10, billed by the platform.
+
The assignment
The platform decides, in a real-time auction, which users see the ads.
+
The dashboard
Users who saw an ad brought in €23.7 more than users who did not.
+
+
The platform's pitch
+ "An exposure is worth €23.7 and costs €10. Raise the budget."
+
The decision on the table
+ Keep paying €10 per exposure, pay more, or stop. One number settles it: how many euros of extra sales one exposure brings in. All we have is the platform's logs: who saw an ad, and what every user spent.
+

The question for the poll: is the dashboard's €23.7 that number?

+
+
+ +
+
Part 1 · The problem
+

Poll · the pitch

+
+
+
✋ Poll
+
The dashboard reports that users who saw an ad brought in €23.7 more than users who did not, and an exposure costs €10. The platform concludes: "raise the budget". What do you think of that claim?
+
+ + + + +
+ +
C. The arithmetic is correct and, at this sample size, comfortably significant, so A and B both miss the point. The platform chose who saw the ads, and it is paid to pick users who already look ready to buy. If the two groups differed before any ad ran, the €23.7 mixes the ad's effect with that pre-existing difference, in unknown proportions. D is backwards: averaging over thousands of users is exactly how noise is handled.
+
+
+
+ +
+
Part 1 · The problem · the variables, drawn
+

The variables and the confounder

+
The three quantities of the case, defined through their causal diagram.
+
+
+

The platform's logs cover 3,000 customers, indexed by \(i\). Three quantities per customer:

+
    +
  • \(Y_i\) · sales: contribution euros earned from customer \(i\) in the window (sales net of product cost); "sales" for short. Noisy: the standard deviation across customers is €17.1.
    • +
  • \(X_i\) · exposure: 1 if customer \(i\) saw the ad, 0 if not. Assigned by the platform's auction, not by us.
    • +
  • \(U_i\) · intent: how ready \(i\) was to buy before any ad. Real, and it drives buying, but it is a feeling in a shopper's head: in no log file, never observed.
    • +
+
Definition · confounder
+ A confounder is a variable that influences both the treatment and the outcome. Here it is intent \(U\): it raises the chance the platform shows the ad (the targeting), and it raises sales on its own (ready buyers buy, ad or not).
+
Definition · backdoor path
+ In the diagram, \(X \leftarrow U \rightarrow Y\) is a backdoor path: a second route connecting exposure and sales that runs behind the treatment. Association flows along it even if the ad does nothing.
+
+
+
the situation, as a diagram
+
+
\(U\) points at both \(X\) and \(Y\): the fork every claim in this lecture has to get past.
+
+
+
+ +
+
Part 1 · The problem
+

The naive comparison

+
The dashboard's €23.7, written down as a formula.
+
+
+

Write \(Y_i(1)\) for what customer \(i\) spends with the ad and \(Y_i(0)\) for what the same customer spends without it. Each customer shows us exactly one of the two:

+ + + + + + +
group\(Y_i(1)\) · with ad\(Y_i(0)\) · without ad
exposed (\(X_i=1\))observedN/A
unexposed (\(X_i=0\))N/Aobserved
+
\[ \widehat{\Delta}_{\text{naive}} \;=\; \bar{Y}_{\text{exposed}} \;-\; \bar{Y}_{\text{unexposed}} \;=\; \text{€}23.7 \]
+
    +
  • What it averages: the two observed cells, one from each row. The arithmetic is right.
    • +
  • What it pretends: that the unexposed row's \(Y(0)\) is a fair stand-in for the exposed row's missing \(Y(0)\). Fair only if the two groups are comparable.
    • +
  • The auction works against us: the platform is paid to find likely buyers, so the exposed would have spent more with no ad at all.
    • +
+
+
+
+
▶ LIVE exposed users were already different
+
+
+ + +
+
The gap compares two groups the auction built to differ. The hidden column shows by how much.
+
+
+
+
+ +
+
Part 1 · The problem
+

Poll · read the gap

+
+
+
✋ Poll
+
The naive comparison gives \(\widehat{\Delta}_{\text{naive}} = \) €23.7. Which reading of that number is defensible?
+
+ + + + +
+ +
B. The arithmetic is sound, so C is out. A and D each pick one extreme of the same unknown split: A books all of it to the ad, D books all of it to targeting, and the dashboard alone cannot justify either. The honest statement is the decomposition: €23.7 = the ad's effect + selection bias, with nothing on the dashboard saying how it splits. Measuring that split is the technical problem this lecture solves.
+
+
+
+ +
+
Part 1 · The problem · the poll's answer, formalized
+

Selection bias

+
The decomposition behind the poll's answer.
+
+
+

Split the naive gap into the two things it adds together:

+
\[ \Delta_{\text{naive}} \;=\; \underbrace{\mathbb{E}[Y(1)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}1]}_{\text{effect of the ad on the exposed}} \;+\; \underbrace{\mathbb{E}[Y(0)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}0]}_{\text{selection bias}} \]
+
    +
  • Read the first term: the exposed users with the ad versus the same users without it. This is what we want to buy, and its second entry is the table's N/A cell.
    • +
  • Read the second term: the gap that would exist with no ads at all, because the picked users differ from the skipped ones.
    • +
  • Here it is positive: the auction picks ready buyers, so the exposed would have out-spent the unexposed anyway.
    • +
  • The dashboard reports only the total: €23.7. Nothing on it says how the total splits.
    • +
+
Definition · selection bias
+ When treated and untreated groups differed before the treatment, the difference of their averages includes that pre-existing gap. The pre-existing part is selection bias.
+
+
+
+
the dashboard's number is a sum, and the split is invisible
+
+
The dashboard hands us the full bar. Where the boundary sits, nobody can see from the dashboard alone. Finding it is the rest of the lecture.
+
+
+
+
+ +
+
Part 1 · The problem · a world we can grade
+

The simulated world

+
A world where the true effect is planted, so every method can be graded.
+
+
+
+
▶ LIVE the world the equations generate, as data
+
+
+ + +
+
Sales distributions for exposed and unexposed users, drawn fresh from the equations. Push \(\kappa\) up, so that intent leaks more strongly into sales, and the two groups drift apart with the planted effect unchanged: the dashboard's gap grows while the truth stands still.
+
+
+
+
Why simulate at all
+ Three equations generate the dashboard you saw. We planted an ad effect of \(\beta =\) €15, so every method in this lecture can be graded against a known answer.
+
\[ U \sim \mathcal{N}(0,1) \]
+
\[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \lambda\,U\right) \]
+
\[ Y \;=\; \mu + \beta\,X \;+\; \kappa\,U \;+\; \varepsilon \] + \(\mu\): baseline sales  ·  \(\alpha_0\): baseline exposure odds  ·  \(\varepsilon\): noise
+
    +
  • Interpretation of line 2: who sees an ad rises with intent (\(\lambda U\)): the targeting. \(\sigma\) is the logistic function, turning any score into a probability.
    • +
  • Interpretation of line 3: sales are the ad effect \(\beta\) plus intent leaking straight into spending (\(\kappa U\)).
    • +
  • The math is here for the record. The point is on the left: one hidden cause, \(U\), sits in both equations.
    • +
+
+
+
+ +
+
Part 1 · The problem · the disease, stated exactly
+

Endogeneity

+
The whole problem, stated as one inequality, then shown as a picture.
+
+
+

Collect everything the sales equation leaves out into a single error term \(v\):

+
\[ Y \;=\; \mu + \beta\,X + v, \qquad v \;=\; \kappa\,U + \varepsilon \]
+
\[ \operatorname{Cov}(X,\, v) \;>\; 0 \]
+
    +
  • In words: \(v\) is "every other reason this customer spent money", and exposure is not independent of those reasons, because the auction targets exactly the customers with reasons to spend.
    • +
  • The picture on the right is the inequality: walk up the intent scale and both bars rise together. That joint rise is \(\operatorname{Cov}(X,v) > 0\).
    • +
+
Definition · endogeneity
+ A treatment correlated with the other drivers of the outcome is endogenous. Ours is, and everything else in this lecture is a response to that fact.
+
The sharp edge
+ The variable driving the correlation was never written to disk.
+
+
+
+
one hidden cause pushes both
+
+
Customers grouped by buying intent (which nobody observes). Higher-intent customers see more ads and spend more, ad or no ad. The formula on the left is this picture, in symbols.
+
+
+
+
+ +
+
Part 1 · The problem · the bias, computed
+

The size of the bias

+
The dashboard's error is not vague. It has a formula, and the formula has three lessons.
+
+
+
\[ \widehat{\Delta}_{\text{naive}} \;\longrightarrow\; \beta \;+\; \underbrace{\kappa\,\frac{\operatorname{Cov}(X, U)}{\operatorname{Var}(X)}}_{\text{selection bias}} \]
+
    +
  • Predicted vs delivered: the formula predicts a naive gap near €23.5. The data delivered €23.7, against a truth of €15.
    • +
+
What the formula teaches
+ 1. The error is systematic, not bad luck: both factors are positive, so the dashboard can only overstate.
+ 2. It grows with the targeting (\(\lambda\), how strongly intent drives exposure) and with intent's pull on sales (\(\kappa\)): the better the platform targets, the worse the dashboard lies.
+ 3. Sample size is absent: more rows shrink noise, never bias. The dashboard's band, €22.9 to €24.5, is the tightest in this lecture and misses €15 completely.
+
What the dashboard measures
+ Not the ad's power to create buyers but the platform's skill at finding them, reported as one number.
+
+
+
+
▶ LIVE the same bar, at different targeting strengths
+
+
+ + +
+
The dashboard's bar, split by the formula: the true effect (blue) never moves, while the selection bias (orange) grows as intent leaks more strongly into sales. In this simulated world we can draw the boundary, because we planted it.
+
+
+
+
+ +
+
Part 1 · The problem · closing the obvious exit
+

The limits of adjustment

+
The standard fix works, when you have the column it needs. We do not.
+
+
+
The standard fix: close the backdoor
+ Compare users at the same level of the confounder. Within a group of equal-intent users, intent no longer separates exposed from unexposed, and any remaining gap belongs to the ad. Controls in a regression, matching, and propensity scores are all versions of this one move.
+
+
▶ LIVE the fix works, if the column exists
+
+
+ + +
+
Tick the box: comparing exposed to unexposed within equal-intent groups recovers the planted truth to within half a euro. The method is fine. Untick it: in the real logs the column does not exist, and no amount of modelling recreates it.
+
+
Why the engine stalls here
+ You cannot adjust for what you did not record.
+
+
+

Two customers, identical on every recorded feature \(W\):

+ + + + + + +
customerpages, device, history \(W\)intent \(U\)sees the ad?
Annaidenticalhigh, unrecordedpicked
Beaidenticallow, unrecordedskipped
+
what conditioning blocks, and what it cannot
+
+
Holding \(W\) fixed (the box) blocks the backdoors that run through it. The arrows themselves never move: conditioning blocks paths, it does not delete causes. The path through \(U\) stays open, because \(U\) is in no file to hold fixed.
+
+
+
+ +
+
Part 1 · The problem
+

Poll · what would fix it?

+
+
+
✋ Poll
+
Adjustment failed because intent was never recorded. Your team can request one addition to the data. Which one would actually let you measure what an exposure causes?
+
+ + + + +
+ +
C. A tells you the wrong number more precisely: bias is not noise, and the formula for it does not contain the sample size. B and D are more controls, and the auction reacts to live signals that no exported feature set fully contains, so the backdoor stays open. C is different in kind: it changes who assigns the treatment, and the assignment was the disease all along. The rest of the lecture is about getting C, or the closest thing to it the system already contains.
+
+
+
+ + +
+
Part 2 · The idea · what we wish we could do
+

The ideal experiment

+
What randomizing exposure would buy, and why we cannot do it.
+
+
+
randomization cuts the arrow that causes the trouble
+
+
Left: today's world, where intent drives both exposure and sales. Right: the experiment we wish we could run, where a randomizer assigns exposure and intent's arrow into it is removed, not merely blocked: exposure no longer listens to intent at all. With that arrow gone, the naive comparison becomes the right comparison.
+
\[ X \text{ randomized} \;\;\Rightarrow\;\; \operatorname{Cov}(X,\, v) = 0 \;\;\Rightarrow\;\; \widehat{\Delta}_{\text{naive}} \longrightarrow \beta \]
+
+
+
    +
  • The wish: option C from the poll: show the ad to a random half ourselves.
    • +
  • The wall: the platform's auction decides exposure in milliseconds, using exactly the intent signals that cause the problem. It will not hand us the controls.
    • +
+
The idea that saves us
+ If we cannot inject randomness into exposure, find the randomness the system already contains and use only that part.
+
    +
  • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
    • +
+
The plan for Part 2
+ 1. Give that random lever a name: an instrument.
+ 2. State the conditions it must satisfy.
+ 3. Check which of them the data can verify.
+
+
+
+ +
+
Part 2 · The idea · the randomness we ran
+

The lottery

+
The randomness our system already contains, because we put it there.
+
+
+
    +
  • What we ran: before the campaign, a serving-priority lottery. Our own random number generator picked half the users and gave them a small priority boost in the platform's ad auction.
    • +
  • What it does: the lottery shows nobody an ad. It only raises the odds of seeing one.
    • +
  • A new column joins the data:
    • +
+ + + + + +
quantitymeaningtypewho assigns itobserved?
\(Z_i\) · the lottery1 if the lottery boosted \(i\)'s priority in the ad auctionbinarya random number generator we controlobserved
+
\[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \gamma\,Z + \lambda\,U\right) \] + the simulated world ran the lottery too: the exposure equation gains \(\gamma Z\), the lottery's push on the odds of seeing an ad
+
    +
  • Why it matters: \(Z\) moves exposure, and nothing else about the customer. It is the one source of variation in \(X\) that the targeting cannot touch.
    • +
+
+
+
the fork, and the way around it
+
+
Top: the confounded fork. Bottom: the lottery \(Z\) pushes on \(X\) from outside. No arrow from \(U\) into \(Z\) (it is a random draw), and no arrow from \(Z\) straight to \(Y\) (a queue bump shows the user nothing).
+
+
+
+ +
+
Part 2 · The idea · the central definition
+

The instrument

+
What we just built has a name.
+
+
Definition · instrument
+ An instrument \(Z\) moves the treatment, is as good as random with respect to the hidden confounder, and affects the outcome only through the treatment. Four conditions make that precise, and they grade our lottery as follows.
+ + + + + + + + + + + + + +
conditionthe claimcan it be checked?what it buys
1 · Relevancethe lottery actually moves exposure \(X\)testable measured by the first stagea push that really happened
2 · Exogeneitythe lottery is blind to intent \(U\)by design if the draw is genuinewith 1: a clean experiment on the lottery
3 · Exclusionthe lottery touches sales only through exposureuntestable argued, never checkedturns the lottery's effect into the ad's
4 · Monotonicitythe lottery never blocks an exposure that would have happened without ituntestable argued from the mechanismsays whose effect we measured
+
An instrument is the opposite of a control
+ You do not hold \(Z\) fixed. You use the variation it created.
+

The genius is never the estimator, it is spotting the randomness your system already contains: a rollout order, a capacity limit, an upstream A/B test, a rounding rule.

+
+
+ +
+
Part 2 · The idea · condition 1, measured
+

Condition 1 · relevance

+
The first stage: did the lottery actually move exposure?
+
+
+

Relevance is a claim about the data, so the data can answer it.

+
Definition · first stage
+ The first stage is the effect of the instrument on the treatment: how much exposure the lottery itself created. It is the coefficient \(\pi\) in the regression below.
+
+
\[ X \;=\; b_0 + \pi\,Z + u \] + \(\pi\): the first stage  ·  \(b_0\): exposure rate among lottery losers  ·  \(u\): every other driver of exposure, intent included
+
+
+
\[ \pi \;=\; \Pr(X{=}1 \mid Z{=}1) \;-\; \Pr(X{=}1 \mid Z{=}0) \;=\; 0.2106 \] + the measured first stage: what the lottery did to exposure
+
    +
  • Interpretation of \(\pi\): winning the lottery raises a user's chance of seeing an ad by +21 percentage points, from 56% to 77%.
    • +
  • It is causal, full stop: \(Z\) is random, so nothing hidden can explain the step.
    • +
+
+
+
exposure rate, by the lottery
+
+
The step between the bars is the first stage \(\pi\): the slice of exposure that the lottery, not targeting, created.
+
The gate: is the push strong enough?
+ Like every estimate in this lecture, \(\pi\) comes with a significance test: the first-stage \(F\). Trust the coming division only when \(F > 10\), well past bare significance, because a \(\pi\) that is merely nonzero still leaves the division unstable. Here \(F = 156\): the push is unmistakably real. We quote such tests, not derive them.
+
+
+
+ +
+
Part 2 · The idea · conditions 2 and 3
+

Conditions 2 and 3 · exogeneity and exclusion

+
The lottery must be blind to intent, and silent about sales.
+
+
+
Blind · exogeneity
+ The lottery must not favour high-intent users. Ours is a genuine randomizer, so this holds by design.
+
Silent · exclusion
+ The lottery must not touch sales through any channel except the ad itself. A side-channel (faster pages for boosted users, a shown price) would poison the method:
+
\[ \hat\beta_{\text{IV}} \;=\; \beta + \frac{s}{\pi} \] + \(\hat\beta_{\text{IV}}\): the estimate the lottery-based method reports  ·  \(\beta\): the true effect of one exposure on sales, the number we are hunting  ·  \(s\): the side-channel's own lift on sales  ·  \(\pi\): the first stage
+
No test will catch it
+ Exclusion is a claim about a path that should not exist, so no output can verify it. You defend it by design.
+
+
+
▶ LIVE break the silence on purpose
+
+
+ + +
+
What the method reports, split into the true effect (blue) and the contamination \(s/\pi\) (red). Slide \(s\) up: a small leak becomes a large error.
+
+
    +
  • Why a tiny leak is a big error: the side-channel \(s\) gets divided by the small first stage \(\pi\), so even €1 of leak moves the estimate by €1 / 0.2106.
    • +
  • Our defence: a queue bump is content-free, sells nothing, and the user never learns it happened.
    • +
  • The counterexample: an email with a discount code would move spending on its own, and everything after this slide would collapse.
    • +
+
+
+
+ + +
+
Part 3 · The estimator · the second ingredient
+

The reduced form

+
The same comparison as the first stage, now for sales.
+
+
+
Definition · reduced form
+ The reduced form is the effect of the instrument on the outcome: the sales gap between the lottery groups. It is the \(\delta\) measured below.
+
\[ \delta \;=\; \mathbb{E}[Y \mid Z{=}1] \;-\; \mathbb{E}[Y \mid Z{=}0] \;=\; \text{€}3.48 \] + \(\delta\): the reduced form  ·  \(\beta\): the true effect of one exposure on sales, still the number we are hunting
+
    +
  • Interpretation of \(\delta\): winning the lottery adds €3.48 of sales for the average user. Causal for the same reason \(\pi\) was: a random draw cannot favour ready buyers.
    • +
  • The intuition that carries the day: the lottery can only reach sales through the ad. One lottery win buys \(\pi\) extra exposures, and each exposure is worth \(\beta\) euros. So the lottery's effect on sales must be \(\beta \times \pi\):
    • +
+
\[ \delta \;=\; \beta \times \pi \]
+
Both numbers are correct. Neither answers the question.
+ \(\pi\) is exposures per lottery win and \(\delta\) is euros per lottery win, but the client asked for euros per exposure. Since \(\delta = \beta\pi\), one step is left.
+
+
+
average sales, by the lottery
+
+
The step is small but clean: a random draw cannot favour ready buyers, so selection cannot explain it.
+

For the record, the algebra behind the intuition: substitute \(X = b_0 + \pi Z + u\) into the sales equation and the coefficient multiplying \(Z\) is exactly \(\beta\pi\).

+
+
+
+ +
+
Part 3 · The estimator · the whole method in one division
+

The IV estimate

+
Euros per lottery win, divided by exposures per lottery win.
+
+
+
the whole method, as arithmetic on two measured numbers
+
+
One lottery win buys 0.2106 extra exposures and €3.48 of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
+
+
+
+
\[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ 3.48 }{ 0.2106 } \;=\; \text{€}16.5 \]
+
    +
  • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
    • +
  • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
    • +
+
+
+
What just happened
+ We priced the ad using only the random slice of exposure: the dashboard said €23.7, the lottery says €16.5, against a planted truth of €15.
+

The method never needed a model of intent, controls, or machine learning: two averages and a division.

+
+
+
▸ Deep dive · the confidence interval around €16.5
+
+
+

\(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

+
\[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; 16.5 \;\pm\; 1.645 \times 2.31 \;=\; [\,12.7,\; 20.4\,] \] + the 90% confidence interval: the range of effect sizes the data support
+
    +
  • Interpretation of the interval: any effect between €12.7 and €20.4 is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
    • +
  • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
    • +
+
+
+
two intervals, graded against the planted truth
+
+
The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
+
+
+
+
+
+ +
+
Part 3 · The estimator · hands on
+

Why the division is forced

+
The division is not a modelling choice. It is the only effect size the two measurements allow.
+
+
+
▶ LIVE every candidate effect makes a prediction. One matches.
+
+
+ + +
+
The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €3.48. Move your guess to the crossing and you have priced the ad.
+
+
+

Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

+
    +
  • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's 0.2106 extra exposures per win should create \(\hat\beta \times 0.2106\) euros per win.
    • +
  • The fact: the lottery actually created €3.48 per win.
    • +
  • The verdict: every candidate except €16.5 contradicts a number we measured. The division is the only survivor, not a choice.
    • +
+
\[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
+
Not a black box
+ Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
+
+
+
+ + +
+
Part 4 · When it breaks · the dangerous failure
+

Weak instruments

+
A weak instrument is worse than no instrument.
+
+
+
▶ LIVE what the estimator does as the first stage dies
+
+
+ + +
+
Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
+
+
+

The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

+
    +
  • The honest failure: the interval balloons, and the data admit they know little.
    • +
  • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
    • +
  • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
    • +
+
Report the first-stage \(F\), always
+ Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
+
+
+
▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
+
+
+
▶ LIVE test each candidate \(\beta_0\), keep the survivors
+
+
+ + +
+
The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [12.6, 20.2]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
+
+
+
    +
  • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
    • +
  • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
    • +
+
\[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
+
    +
  • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
    • +
  • Here, a good sign: the lottery is strong, so AR [12.6, 20.2] nearly matches the usual [12.7, 20.4]. When \(F\) is small the two part company, and AR is the one still telling the truth.
    • +
+
+
+
+
+
+ +
+
Part 4 · When it breaks · whose effect it is
+

Compliers and the LATE

+
Whom does the €16.5 describe? Only the users the lottery could move.
+
+
▶ LIVE the user base, split by how they respond to the lottery
+
+
+ + +
+
Push \(\gamma\): the complier slice grows, because the complier share is the first stage. Shares are drawn live from one simulated batch, so they can differ from the printed figures by a rounding step.
+
+
+
    +
  • Always-takers (56.5%): the auction shows them the ad with or without the lottery. It changes nothing for them, so the data say nothing about them.
    • +
  • Never-takers (22.5%): never see the ad either way. Same silence.
    • +
  • Compliers (21.0%): see the ad only because the lottery favoured them. Every euro of the lottery's lift \(\delta\) came from them.
    • +
  • The caveat, monotonicity: we assume no defiers, users who would see the ad only when the lottery does not favour them. A nudge that never repels.
    • +
+
+
+
Definition · LATE (local average treatment effect)
+ The LATE is the average effect of the ad on the compliers alone, and it is what the division estimates: a local answer, not a statement about every user.
+
\[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;\;\text{ estimates }\;\; \mathbb{E}[\,Y(1)-Y(0)\mid \text{complier}\,] \] + the average of each complier's personal effect, the same \(Y(1)-Y(0)\) contrast the naive slide could not touch
+
+
+
Why a manager should love this fine print
+ Compliers are the same kind of marginal user a higher bid would newly reach: the closest thing in the data to the customer a bid change buys. That makes €16.5 a price for the marginal customer, measured on the margin rather than on the average.
+
+
+ +
+
Part 4 · When it breaks · what must hold, on one page
+

The checklist

+
The four assumptions, and which ones the data can check.
+
+ + + + + + + + +
AssumptionWhat it saysIn the caseStatus
Relevance\(Z\) moves \(X\)\(F = 156\), far above 10TESTABLE, passes
Exogeneity\(Z \perp U\)the lottery is a genuine random drawBY DESIGN
Exclusion\(Z \to Y\) only via \(X\)a queue bump shows the user nothingUNTESTABLE
Monotonicityno defiersa nudge never repelsUNTESTABLE, plausible
+
+
+
The honest scorecard, for any IV study you are shown
+ One measured number (the first-stage \(F\)), one design guarantee (the randomization), two arguments (exclusion, monotonicity). Ask for all four before you accept the estimate.
+
    +
  • What we can now defend: an exposure causes about €16.5 of sales for the users a bid can actually move.
    • +
+
+
The sentence that loses money
+ "Exposed users are worth €23.7 each, so raise the bid": every word true, conclusion wrong. It books the platform's targeting as advertising.
+
+
+
+ + + +
+
Part 5 · The decision · euros at last
+

The price map

+
The estimate becomes a decision only when it meets the price.
+
+
+
▶ LIVE the verdict as the price moves
+
+
+ + +
+
Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
+
+
+

One estimate gives not one answer but a map from any price to a verdict:

+ + + + + + + +
Price zoneVerdictWhy
below €12.7GOeven the most pessimistic supported effect pays
€12.7 to €20.4TESTthe data straddle the price: negotiate, or measure more
above €20.4NO-GOno supported effect pays
+
    +
  • Today's rate, €10, sits in the GO zone, below the whole interval.
    • +
  • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €16.5 − €10 = €6.5. Even read at the interval floor, €12.7 against €10, the exposure still pays.
    • +
+
Why boards like this framing
+ "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
+
+
+
+ +
+
Part 5 · The decision
+

Poll · the negotiation

+
+
+
✋ Poll
+
The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €16.5 per exposure, 90% interval [12.7, 20.4]. What is the highest rate at which you would still sign "buy" without further study?
+
+ + + + +
+ +
C. Below €12.7, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
+
+
+
+ +
+
Part 5 · The decision · banked
+

The verdict and the recommendation

+
The complete answer, assembled from everything measured so far.
+
+
+ + + + + + + + + +
QuantityValueSource
effect of one exposure€16.5the division δ/π
90% interval[12.7, 20.4]classical, and AR agrees
first-stage F156the lottery is strong
price€10the platform's rate card
net per exposure€6.5β − c, at the point estimate
+
The verdict
+ BUY  Keep buying at €10: the entire defensible range clears the price.
+
+
+
The recommendation, in three lines
+ 1. Keep buying at the €10 rate: even the interval's most pessimistic effect pays.
+ 2. Cap the bid at the interval's lower end, €12.7: up to there, every effect the data support still clears the price.
+ 3. Measure again only if the rate card climbs toward €12.7: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
+
    +
  • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
    • +
  • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
    • +
+
+
+
+ + + +
+ + + +
+ Causal Marketing · SDA Bocconi · Instrumental Variables + + + + 1 / 1 +
+
+

Contents

    + + + + diff --git a/causal-marketing-pymc/apps/iv_slides_sh_src.html b/causal-marketing-pymc/apps/iv_slides_sh_src.html new file mode 100644 index 0000000..bdc349b --- /dev/null +++ b/causal-marketing-pymc/apps/iv_slides_sh_src.html @@ -0,0 +1,1841 @@ + + + + + + +Instrumental Variables: what is one ad exposure worth? (slides) + + + + + +
    + + +
    +
    +
    +
    Causal Inference & XAI for Business · SDA Bocconi
    +

    Instrumental Variables

    +
    What is one ad exposure worth, when the platform already picked who sees the ads?
    +
    A guest lecture for Prof. Michele Russo's course
    +
    +
    +
    Francesco Muia
    +
    PhD in Theoretical Physics, EMBA.
    Consultant for PyMC Labs and Brown University.
    +
    francesco.muia@pymc-labs.com
    +
    +
    +
    Alexander Fengler
    +
    PhD in Computational Cognitive Science.
    Postdoc at Brown University, consultant for PyMC Labs.
    +
    alexander.fengler@pymc-labs.com
    +
    +
    +
    + PyMC + PyMC Labs +
    +
    +
    + +
    Where the lecture lands: the dashboard's tight interval misses the true effect of an ad exposure, while the interval built from a lottery covers it, and still clears the price.
    +
    +
    +
    + + +
    +
    Part 1 · The problem
    +

    The case

    +
    One retailer, one ad platform, one decision to make.
    +
    +
    +
    The product
    An online retailer pays an ad platform to show its display ads. One exposure = one user actually saw an ad.
    +
    The price
    Each exposure costs €{{nb11.cost}}, billed by the platform.
    +
    The assignment
    The platform decides, in a real-time auction, which users see the ads.
    +
    The dashboard
    Users who saw an ad brought in €{{nb11.naive}} more than users who did not.
    +
    +
    The platform's pitch
    + "An exposure is worth €{{nb11.naive}} and costs €{{nb11.cost}}. Raise the budget."
    +
    The decision on the table
    + Keep paying €{{nb11.cost}} per exposure, pay more, or stop. One number settles it: how many euros of extra sales one exposure brings in. All we have is the platform's logs: who saw an ad, and what every user spent.
    +

    The question for the poll: is the dashboard's €{{nb11.naive}} that number?

    +
    +
    + +
    +
    Part 1 · The problem
    +

    Poll · the pitch

    +
    +
    +
    ✋ Poll
    +
    The dashboard reports that users who saw an ad brought in €{{nb11.naive}} more than users who did not, and an exposure costs €{{nb11.cost}}. The platform concludes: "raise the budget". What do you think of that claim?
    +
    + + + + +
    + +
    C. The arithmetic is correct and, at this sample size, comfortably significant, so A and B both miss the point. The platform chose who saw the ads, and it is paid to pick users who already look ready to buy. If the two groups differed before any ad ran, the €{{nb11.naive}} mixes the ad's effect with that pre-existing difference, in unknown proportions. D is backwards: averaging over thousands of users is exactly how noise is handled.
    +
    +
    +
    + +
    +
    Part 1 · The problem · the variables, drawn
    +

    The variables and the confounder

    +
    The three quantities of the case, defined through their causal diagram.
    +
    +
    +

    The platform's logs cover {{nb11.n}} customers, indexed by \(i\). Three quantities per customer:

    +
      +
    • \(Y_i\) · sales: contribution euros earned from customer \(i\) in the window (sales net of product cost); "sales" for short. Noisy: the standard deviation across customers is €{{nb11.ppc_obs_sd}}.
      • +
    • \(X_i\) · exposure: 1 if customer \(i\) saw the ad, 0 if not. Assigned by the platform's auction, not by us.
      • +
    • \(U_i\) · intent: how ready \(i\) was to buy before any ad. Real, and it drives buying, but it is a feeling in a shopper's head: in no log file, never observed.
      • +
    +
    Definition · confounder
    + A confounder is a variable that influences both the treatment and the outcome. Here it is intent \(U\): it raises the chance the platform shows the ad (the targeting), and it raises sales on its own (ready buyers buy, ad or not).
    +
    Definition · backdoor path
    + In the diagram, \(X \leftarrow U \rightarrow Y\) is a backdoor path: a second route connecting exposure and sales that runs behind the treatment. Association flows along it even if the ad does nothing.
    +
    +
    +
    the situation, as a diagram
    +
    +
    \(U\) points at both \(X\) and \(Y\): the fork every claim in this lecture has to get past.
    +
    +
    +
    + +
    +
    Part 1 · The problem
    +

    The naive comparison

    +
    The dashboard's €{{nb11.naive}}, written down as a formula.
    +
    +
    +

    Write \(Y_i(1)\) for what customer \(i\) spends with the ad and \(Y_i(0)\) for what the same customer spends without it. Each customer shows us exactly one of the two:

    + + + + + + +
    group\(Y_i(1)\) · with ad\(Y_i(0)\) · without ad
    exposed (\(X_i=1\))observedN/A
    unexposed (\(X_i=0\))N/Aobserved
    +
    \[ \widehat{\Delta}_{\text{naive}} \;=\; \bar{Y}_{\text{exposed}} \;-\; \bar{Y}_{\text{unexposed}} \;=\; \text{€}{{nb11.naive}} \]
    +
      +
    • What it averages: the two observed cells, one from each row. The arithmetic is right.
      • +
    • What it pretends: that the unexposed row's \(Y(0)\) is a fair stand-in for the exposed row's missing \(Y(0)\). Fair only if the two groups are comparable.
      • +
    • The auction works against us: the platform is paid to find likely buyers, so the exposed would have spent more with no ad at all.
      • +
    +
    +
    +
    +
    ▶ LIVE exposed users were already different
    +
    +
    + + +
    +
    The gap compares two groups the auction built to differ. The hidden column shows by how much.
    +
    +
    +
    +
    + +
    +
    Part 1 · The problem
    +

    Poll · read the gap

    +
    +
    +
    ✋ Poll
    +
    The naive comparison gives \(\widehat{\Delta}_{\text{naive}} = \) €{{nb11.naive}}. Which reading of that number is defensible?
    +
    + + + + +
    + +
    B. The arithmetic is sound, so C is out. A and D each pick one extreme of the same unknown split: A books all of it to the ad, D books all of it to targeting, and the dashboard alone cannot justify either. The honest statement is the decomposition: €{{nb11.naive}} = the ad's effect + selection bias, with nothing on the dashboard saying how it splits. Measuring that split is the technical problem this lecture solves.
    +
    +
    +
    + +
    +
    Part 1 · The problem · the poll's answer, formalized
    +

    Selection bias

    +
    The decomposition behind the poll's answer.
    +
    +
    +

    Split the naive gap into the two things it adds together:

    +
    \[ \Delta_{\text{naive}} \;=\; \underbrace{\mathbb{E}[Y(1)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}1]}_{\text{effect of the ad on the exposed}} \;+\; \underbrace{\mathbb{E}[Y(0)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}0]}_{\text{selection bias}} \]
    +
      +
    • Read the first term: the exposed users with the ad versus the same users without it. This is what we want to buy, and its second entry is the table's N/A cell.
      • +
    • Read the second term: the gap that would exist with no ads at all, because the picked users differ from the skipped ones.
      • +
    • Here it is positive: the auction picks ready buyers, so the exposed would have out-spent the unexposed anyway.
      • +
    • The dashboard reports only the total: €{{nb11.naive}}. Nothing on it says how the total splits.
      • +
    +
    Definition · selection bias
    + When treated and untreated groups differed before the treatment, the difference of their averages includes that pre-existing gap. The pre-existing part is selection bias.
    +
    +
    +
    +
    the dashboard's number is a sum, and the split is invisible
    +
    +
    The dashboard hands us the full bar. Where the boundary sits, nobody can see from the dashboard alone. Finding it is the rest of the lecture.
    +
    +
    +
    +
    + +
    +
    Part 1 · The problem · a world we can grade
    +

    The simulated world

    +
    A world where the true effect is planted, so every method can be graded.
    +
    +
    +
    +
    ▶ LIVE the world the equations generate, as data
    +
    +
    + + +
    +
    Sales distributions for exposed and unexposed users, drawn fresh from the equations. Push \(\kappa\) up, so that intent leaks more strongly into sales, and the two groups drift apart with the planted effect unchanged: the dashboard's gap grows while the truth stands still.
    +
    +
    +
    +
    Why simulate at all
    + Three equations generate the dashboard you saw. We planted an ad effect of \(\beta =\) €{{nb11.true}}, so every method in this lecture can be graded against a known answer.
    +
    \[ U \sim \mathcal{N}(0,1) \]
    +
    \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \lambda\,U\right) \]
    +
    \[ Y \;=\; \mu + \beta\,X \;+\; \kappa\,U \;+\; \varepsilon \] + \(\mu\): baseline sales  ·  \(\alpha_0\): baseline exposure odds  ·  \(\varepsilon\): noise
    +
      +
    • Interpretation of line 2: who sees an ad rises with intent (\(\lambda U\)): the targeting. \(\sigma\) is the logistic function, turning any score into a probability.
      • +
    • Interpretation of line 3: sales are the ad effect \(\beta\) plus intent leaking straight into spending (\(\kappa U\)).
      • +
    • The math is here for the record. The point is on the left: one hidden cause, \(U\), sits in both equations.
      • +
    +
    +
    +
    + +
    +
    Part 1 · The problem · the disease, stated exactly
    +

    Endogeneity

    +
    The whole problem, stated as one inequality, then shown as a picture.
    +
    +
    +

    Collect everything the sales equation leaves out into a single error term \(v\):

    +
    \[ Y \;=\; \mu + \beta\,X + v, \qquad v \;=\; \kappa\,U + \varepsilon \]
    +
    \[ \operatorname{Cov}(X,\, v) \;>\; 0 \]
    +
      +
    • In words: \(v\) is "every other reason this customer spent money", and exposure is not independent of those reasons, because the auction targets exactly the customers with reasons to spend.
      • +
    • The picture on the right is the inequality: walk up the intent scale and both bars rise together. That joint rise is \(\operatorname{Cov}(X,v) > 0\).
      • +
    +
    Definition · endogeneity
    + A treatment correlated with the other drivers of the outcome is endogenous. Ours is, and everything else in this lecture is a response to that fact.
    +
    The sharp edge
    + The variable driving the correlation was never written to disk.
    +
    +
    +
    +
    one hidden cause pushes both
    +
    +
    Customers grouped by buying intent (which nobody observes). Higher-intent customers see more ads and spend more, ad or no ad. The formula on the left is this picture, in symbols.
    +
    +
    +
    +
    + +
    +
    Part 1 · The problem · the bias, computed
    +

    The size of the bias

    +
    The dashboard's error is not vague. It has a formula, and the formula has three lessons.
    +
    +
    +
    \[ \widehat{\Delta}_{\text{naive}} \;\longrightarrow\; \beta \;+\; \underbrace{\kappa\,\frac{\operatorname{Cov}(X, U)}{\operatorname{Var}(X)}}_{\text{selection bias}} \]
    +
      +
    • Predicted vs delivered: the formula predicts a naive gap near €{{nb11.ols_predicted}}. The data delivered €{{nb11.naive}}, against a truth of €{{nb11.true}}.
      • +
    +
    What the formula teaches
    + 1. The error is systematic, not bad luck: both factors are positive, so the dashboard can only overstate.
    + 2. It grows with the targeting (\(\lambda\), how strongly intent drives exposure) and with intent's pull on sales (\(\kappa\)): the better the platform targets, the worse the dashboard lies.
    + 3. Sample size is absent: more rows shrink noise, never bias. The dashboard's band, €{{nb11.naive_lo}} to €{{nb11.naive_hi}}, is the tightest in this lecture and misses €{{nb11.true}} completely.
    +
    What the dashboard measures
    + Not the ad's power to create buyers but the platform's skill at finding them, reported as one number.
    +
    +
    +
    +
    ▶ LIVE the same bar, at different targeting strengths
    +
    +
    + + +
    +
    The dashboard's bar, split by the formula: the true effect (blue) never moves, while the selection bias (orange) grows as intent leaks more strongly into sales. In this simulated world we can draw the boundary, because we planted it.
    +
    +
    +
    +
    + +
    +
    Part 1 · The problem · closing the obvious exit
    +

    The limits of adjustment

    +
    The standard fix works, when you have the column it needs. We do not.
    +
    +
    +
    The standard fix: close the backdoor
    + Compare users at the same level of the confounder. Within a group of equal-intent users, intent no longer separates exposed from unexposed, and any remaining gap belongs to the ad. Controls in a regression, matching, and propensity scores are all versions of this one move.
    +
    +
    ▶ LIVE the fix works, if the column exists
    +
    +
    + + +
    +
    Tick the box: comparing exposed to unexposed within equal-intent groups recovers the planted truth to within half a euro. The method is fine. Untick it: in the real logs the column does not exist, and no amount of modelling recreates it.
    +
    +
    Why the engine stalls here
    + You cannot adjust for what you did not record.
    +
    +
    +

    Two customers, identical on every recorded feature \(W\):

    + + + + + + +
    customerpages, device, history \(W\)intent \(U\)sees the ad?
    Annaidenticalhigh, unrecordedpicked
    Beaidenticallow, unrecordedskipped
    +
    what conditioning blocks, and what it cannot
    +
    +
    Holding \(W\) fixed (the box) blocks the backdoors that run through it. The arrows themselves never move: conditioning blocks paths, it does not delete causes. The path through \(U\) stays open, because \(U\) is in no file to hold fixed.
    +
    +
    +
    + +
    +
    Part 1 · The problem
    +

    Poll · what would fix it?

    +
    +
    +
    ✋ Poll
    +
    Adjustment failed because intent was never recorded. Your team can request one addition to the data. Which one would actually let you measure what an exposure causes?
    +
    + + + + +
    + +
    C. A tells you the wrong number more precisely: bias is not noise, and the formula for it does not contain the sample size. B and D are more controls, and the auction reacts to live signals that no exported feature set fully contains, so the backdoor stays open. C is different in kind: it changes who assigns the treatment, and the assignment was the disease all along. The rest of the lecture is about getting C, or the closest thing to it the system already contains.
    +
    +
    +
    + + +
    +
    Part 2 · The idea · what we wish we could do
    +

    The ideal experiment

    +
    What randomizing exposure would buy, and why we cannot do it.
    +
    +
    +
    randomization cuts the arrow that causes the trouble
    +
    +
    Left: today's world, where intent drives both exposure and sales. Right: the experiment we wish we could run, where a randomizer assigns exposure and intent's arrow into it is removed, not merely blocked: exposure no longer listens to intent at all. With that arrow gone, the naive comparison becomes the right comparison.
    +
    \[ X \text{ randomized} \;\;\Rightarrow\;\; \operatorname{Cov}(X,\, v) = 0 \;\;\Rightarrow\;\; \widehat{\Delta}_{\text{naive}} \longrightarrow \beta \]
    +
    +
    +
      +
    • The wish: option C from the poll: show the ad to a random half ourselves.
      • +
    • The wall: the platform's auction decides exposure in milliseconds, using exactly the intent signals that cause the problem. It will not hand us the controls.
      • +
    +
    The idea that saves us
    + If we cannot inject randomness into exposure, find the randomness the system already contains and use only that part.
    +
      +
    • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
      • +
    +
    The plan for Part 2
    + 1. Give that random lever a name: an instrument.
    + 2. State the conditions it must satisfy.
    + 3. Check which of them the data can verify.
    +
    +
    +
    + +
    +
    Part 2 · The idea · the randomness we ran
    +

    The lottery

    +
    The randomness our system already contains, because we put it there.
    +
    +
    +
      +
    • What we ran: before the campaign, a serving-priority lottery. Our own random number generator picked half the users and gave them a small priority boost in the platform's ad auction.
      • +
    • What it does: the lottery shows nobody an ad. It only raises the odds of seeing one.
      • +
    • A new column joins the data:
      • +
    + + + + + +
    quantitymeaningtypewho assigns itobserved?
    \(Z_i\) · the lottery1 if the lottery boosted \(i\)'s priority in the ad auctionbinarya random number generator we controlobserved
    +
    \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \gamma\,Z + \lambda\,U\right) \] + the simulated world ran the lottery too: the exposure equation gains \(\gamma Z\), the lottery's push on the odds of seeing an ad
    +
      +
    • Why it matters: \(Z\) moves exposure, and nothing else about the customer. It is the one source of variation in \(X\) that the targeting cannot touch.
      • +
    +
    +
    +
    the fork, and the way around it
    +
    +
    Top: the confounded fork. Bottom: the lottery \(Z\) pushes on \(X\) from outside. No arrow from \(U\) into \(Z\) (it is a random draw), and no arrow from \(Z\) straight to \(Y\) (a queue bump shows the user nothing).
    +
    +
    +
    + +
    +
    Part 2 · The idea · the central definition
    +

    The instrument

    +
    What we just built has a name.
    +
    +
    Definition · instrument
    + An instrument \(Z\) moves the treatment, is as good as random with respect to the hidden confounder, and affects the outcome only through the treatment. Four conditions make that precise, and they grade our lottery as follows.
    + + + + + + + + + + + + + +
    conditionthe claimcan it be checked?what it buys
    1 · Relevancethe lottery actually moves exposure \(X\)testable measured by the first stagea push that really happened
    2 · Exogeneitythe lottery is blind to intent \(U\)by design if the draw is genuinewith 1: a clean experiment on the lottery
    3 · Exclusionthe lottery touches sales only through exposureuntestable argued, never checkedturns the lottery's effect into the ad's
    4 · Monotonicitythe lottery never blocks an exposure that would have happened without ituntestable argued from the mechanismsays whose effect we measured
    +
    An instrument is the opposite of a control
    + You do not hold \(Z\) fixed. You use the variation it created.
    +

    The genius is never the estimator, it is spotting the randomness your system already contains: a rollout order, a capacity limit, an upstream A/B test, a rounding rule.

    +
    +
    + +
    +
    Part 2 · The idea · condition 1, measured
    +

    Condition 1 · relevance

    +
    The first stage: did the lottery actually move exposure?
    +
    +
    +

    Relevance is a claim about the data, so the data can answer it.

    +
    Definition · first stage
    + The first stage is the effect of the instrument on the treatment: how much exposure the lottery itself created. It is the coefficient \(\pi\) in the regression below.
    +
    +
    \[ X \;=\; b_0 + \pi\,Z + u \] + \(\pi\): the first stage  ·  \(b_0\): exposure rate among lottery losers  ·  \(u\): every other driver of exposure, intent included
    +
    +
    +
    \[ \pi \;=\; \Pr(X{=}1 \mid Z{=}1) \;-\; \Pr(X{=}1 \mid Z{=}0) \;=\; {{nb11.first}} \] + the measured first stage: what the lottery did to exposure
    +
      +
    • Interpretation of \(\pi\): winning the lottery raises a user's chance of seeing an ad by {{nb11.fs_shift}} percentage points, from {{nb11.fs_lo}}% to {{nb11.fs_hi}}%.
      • +
    • It is causal, full stop: \(Z\) is random, so nothing hidden can explain the step.
      • +
    +
    +
    +
    exposure rate, by the lottery
    +
    +
    The step between the bars is the first stage \(\pi\): the slice of exposure that the lottery, not targeting, created.
    +
    The gate: is the push strong enough?
    + Like every estimate in this lecture, \(\pi\) comes with a significance test: the first-stage \(F\). Trust the coming division only when \(F > 10\), well past bare significance, because a \(\pi\) that is merely nonzero still leaves the division unstable. Here \(F = {{nb11.f_stat}}\): the push is unmistakably real. We quote such tests, not derive them.
    +
    +
    +
    + +
    +
    Part 2 · The idea · conditions 2 and 3
    +

    Conditions 2 and 3 · exogeneity and exclusion

    +
    The lottery must be blind to intent, and silent about sales.
    +
    +
    +
    Blind · exogeneity
    + The lottery must not favour high-intent users. Ours is a genuine randomizer, so this holds by design.
    +
    Silent · exclusion
    + The lottery must not touch sales through any channel except the ad itself. A side-channel (faster pages for boosted users, a shown price) would poison the method:
    +
    \[ \hat\beta_{\text{IV}} \;=\; \beta + \frac{s}{\pi} \] + \(\hat\beta_{\text{IV}}\): the estimate the lottery-based method reports  ·  \(\beta\): the true effect of one exposure on sales, the number we are hunting  ·  \(s\): the side-channel's own lift on sales  ·  \(\pi\): the first stage
    +
    No test will catch it
    + Exclusion is a claim about a path that should not exist, so no output can verify it. You defend it by design.
    +
    +
    +
    ▶ LIVE break the silence on purpose
    +
    +
    + + +
    +
    What the method reports, split into the true effect (blue) and the contamination \(s/\pi\) (red). Slide \(s\) up: a small leak becomes a large error.
    +
    +
      +
    • Why a tiny leak is a big error: the side-channel \(s\) gets divided by the small first stage \(\pi\), so even €1 of leak moves the estimate by €1 / {{nb11.first}}.
      • +
    • Our defence: a queue bump is content-free, sells nothing, and the user never learns it happened.
      • +
    • The counterexample: an email with a discount code would move spending on its own, and everything after this slide would collapse.
      • +
    +
    +
    +
    + + +
    +
    Part 3 · The estimator · the second ingredient
    +

    The reduced form

    +
    The same comparison as the first stage, now for sales.
    +
    +
    +
    Definition · reduced form
    + The reduced form is the effect of the instrument on the outcome: the sales gap between the lottery groups. It is the \(\delta\) measured below.
    +
    \[ \delta \;=\; \mathbb{E}[Y \mid Z{=}1] \;-\; \mathbb{E}[Y \mid Z{=}0] \;=\; \text{€}{{nb11.reduced}} \] + \(\delta\): the reduced form  ·  \(\beta\): the true effect of one exposure on sales, still the number we are hunting
    +
      +
    • Interpretation of \(\delta\): winning the lottery adds €{{nb11.reduced}} of sales for the average user. Causal for the same reason \(\pi\) was: a random draw cannot favour ready buyers.
      • +
    • The intuition that carries the day: the lottery can only reach sales through the ad. One lottery win buys \(\pi\) extra exposures, and each exposure is worth \(\beta\) euros. So the lottery's effect on sales must be \(\beta \times \pi\):
      • +
    +
    \[ \delta \;=\; \beta \times \pi \]
    +
    Both numbers are correct. Neither answers the question.
    + \(\pi\) is exposures per lottery win and \(\delta\) is euros per lottery win, but the client asked for euros per exposure. Since \(\delta = \beta\pi\), one step is left.
    +
    +
    +
    average sales, by the lottery
    +
    +
    The step is small but clean: a random draw cannot favour ready buyers, so selection cannot explain it.
    +

    For the record, the algebra behind the intuition: substitute \(X = b_0 + \pi Z + u\) into the sales equation and the coefficient multiplying \(Z\) is exactly \(\beta\pi\).

    +
    +
    +
    + +
    +
    Part 3 · The estimator · the whole method in one division
    +

    The IV estimate

    +
    Euros per lottery win, divided by exposures per lottery win.
    +
    +
    +
    the whole method, as arithmetic on two measured numbers
    +
    +
    One lottery win buys {{nb11.first}} extra exposures and €{{nb11.reduced}} of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
    +
    +
    +
    +
    \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ {{nb11.reduced}} }{ {{nb11.first}} } \;=\; \text{€}{{nb11.iv_est}} \]
    +
      +
    • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
      • +
    • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
      • +
    +
    +
    +
    What just happened
    + We priced the ad using only the random slice of exposure: the dashboard said €{{nb11.naive}}, the lottery says €{{nb11.iv_est}}, against a planted truth of €{{nb11.true}}.
    +

    The method never needed a model of intent, controls, or machine learning: two averages and a division.

    +
    +
    +
    ▸ Deep dive · the confidence interval around €{{nb11.iv_est}}
    +
    +
    +

    \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

    +
    \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; {{nb11.iv_est}} \;\pm\; 1.645 \times {{nb11.iv_se}} \;=\; [\,{{nb11.iv_lo}},\; {{nb11.iv_hi}}\,] \] + the 90% confidence interval: the range of effect sizes the data support
    +
      +
    • Interpretation of the interval: any effect between €{{nb11.iv_lo}} and €{{nb11.iv_hi}} is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
      • +
    • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
      • +
    +
    +
    +
    two intervals, graded against the planted truth
    +
    +
    The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
    +
    +
    +
    +
    +
    + +
    +
    Part 3 · The estimator · hands on
    +

    Why the division is forced

    +
    The division is not a modelling choice. It is the only effect size the two measurements allow.
    +
    +
    +
    ▶ LIVE every candidate effect makes a prediction. One matches.
    +
    +
    + + +
    +
    The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €{{nb11.reduced}}. Move your guess to the crossing and you have priced the ad.
    +
    +
    +

    Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

    +
      +
    • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's {{nb11.first}} extra exposures per win should create \(\hat\beta \times {{nb11.first}}\) euros per win.
      • +
    • The fact: the lottery actually created €{{nb11.reduced}} per win.
      • +
    • The verdict: every candidate except €{{nb11.iv_est}} contradicts a number we measured. The division is the only survivor, not a choice.
      • +
    +
    \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
    +
    Not a black box
    + Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
    +
    +
    +
    + + +
    +
    Part 4 · When it breaks · the dangerous failure
    +

    Weak instruments

    +
    A weak instrument is worse than no instrument.
    +
    +
    +
    ▶ LIVE what the estimator does as the first stage dies
    +
    +
    + + +
    +
    Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
    +
    +
    +

    The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

    +
      +
    • The honest failure: the interval balloons, and the data admit they know little.
      • +
    • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
      • +
    • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
      • +
    +
    Report the first-stage \(F\), always
    + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
    +
    +
    +
    ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
    +
    +
    +
    ▶ LIVE test each candidate \(\beta_0\), keep the survivors
    +
    +
    + + +
    +
    The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [{{nb11.ar_lo}}, {{nb11.ar_hi}}]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
    +
    +
    +
      +
    • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
      • +
    • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
      • +
    +
    \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
    +
      +
    • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
      • +
    • Here, a good sign: the lottery is strong, so AR [{{nb11.ar_lo}}, {{nb11.ar_hi}}] nearly matches the usual [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. When \(F\) is small the two part company, and AR is the one still telling the truth.
      • +
    +
    +
    +
    +
    +
    + +
    +
    Part 4 · When it breaks · whose effect it is
    +

    Compliers and the LATE

    +
    Whom does the €{{nb11.iv_est}} describe? Only the users the lottery could move.
    +
    +
    ▶ LIVE the user base, split by how they respond to the lottery
    +
    +
    + + +
    +
    Push \(\gamma\): the complier slice grows, because the complier share is the first stage. Shares are drawn live from one simulated batch, so they can differ from the printed figures by a rounding step.
    +
    +
    +
      +
    • Always-takers ({{nb11.st_share_always}}%): the auction shows them the ad with or without the lottery. It changes nothing for them, so the data say nothing about them.
      • +
    • Never-takers ({{nb11.st_share_never}}%): never see the ad either way. Same silence.
      • +
    • Compliers ({{nb11.st_share_complier}}%): see the ad only because the lottery favoured them. Every euro of the lottery's lift \(\delta\) came from them.
      • +
    • The caveat, monotonicity: we assume no defiers, users who would see the ad only when the lottery does not favour them. A nudge that never repels.
      • +
    +
    +
    +
    Definition · LATE (local average treatment effect)
    + The LATE is the average effect of the ad on the compliers alone, and it is what the division estimates: a local answer, not a statement about every user.
    +
    \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;\;\text{ estimates }\;\; \mathbb{E}[\,Y(1)-Y(0)\mid \text{complier}\,] \] + the average of each complier's personal effect, the same \(Y(1)-Y(0)\) contrast the naive slide could not touch
    +
    +
    +
    Why a manager should love this fine print
    + Compliers are the same kind of marginal user a higher bid would newly reach: the closest thing in the data to the customer a bid change buys. That makes €{{nb11.iv_est}} a price for the marginal customer, measured on the margin rather than on the average.
    +
    +
    + +
    +
    Part 4 · When it breaks · what must hold, on one page
    +

    The checklist

    +
    The four assumptions, and which ones the data can check.
    +
    + + + + + + + + +
    AssumptionWhat it saysIn the caseStatus
    Relevance\(Z\) moves \(X\)\(F = {{nb11.f_stat}}\), far above 10TESTABLE, passes
    Exogeneity\(Z \perp U\)the lottery is a genuine random drawBY DESIGN
    Exclusion\(Z \to Y\) only via \(X\)a queue bump shows the user nothingUNTESTABLE
    Monotonicityno defiersa nudge never repelsUNTESTABLE, plausible
    +
    +
    +
    The honest scorecard, for any IV study you are shown
    + One measured number (the first-stage \(F\)), one design guarantee (the randomization), two arguments (exclusion, monotonicity). Ask for all four before you accept the estimate.
    +
      +
    • What we can now defend: an exposure causes about €{{nb11.iv_est}} of sales for the users a bid can actually move.
      • +
    +
    +
    The sentence that loses money
    + "Exposed users are worth €{{nb11.naive}} each, so raise the bid": every word true, conclusion wrong. It books the platform's targeting as advertising.
    +
    +
    +
    + + + +
    +
    Part 5 · The decision · euros at last
    +

    The price map

    +
    The estimate becomes a decision only when it meets the price.
    +
    +
    +
    ▶ LIVE the verdict as the price moves
    +
    +
    + + +
    +
    Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
    +
    +
    +

    One estimate gives not one answer but a map from any price to a verdict:

    + + + + + + + +
    Price zoneVerdictWhy
    below €{{nb11.iv_lo}}GOeven the most pessimistic supported effect pays
    €{{nb11.iv_lo}} to €{{nb11.iv_hi}}TESTthe data straddle the price: negotiate, or measure more
    above €{{nb11.iv_hi}}NO-GOno supported effect pays
    +
      +
    • Today's rate, €{{nb11.cost}}, sits in the GO zone, below the whole interval.
      • +
    • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €{{nb11.iv_est}} − €{{nb11.cost}} = €{{nb11.net}}. Even read at the interval floor, €{{nb11.iv_lo}} against €{{nb11.cost}}, the exposure still pays.
      • +
    +
    Why boards like this framing
    + "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
    +
    +
    +
    + +
    +
    Part 5 · The decision
    +

    Poll · the negotiation

    +
    +
    +
    ✋ Poll
    +
    The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €{{nb11.iv_est}} per exposure, 90% interval [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. What is the highest rate at which you would still sign "buy" without further study?
    +
    + + + + +
    + +
    C. Below €{{nb11.iv_lo}}, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
    +
    +
    +
    + +
    +
    Part 5 · The decision · banked
    +

    The verdict and the recommendation

    +
    The complete answer, assembled from everything measured so far.
    +
    +
    + + + + + + + + + +
    QuantityValueSource
    effect of one exposure€{{nb11.iv_est}}the division δ/π
    90% interval[{{nb11.iv_lo}}, {{nb11.iv_hi}}]classical, and AR agrees
    first-stage F{{nb11.f_stat}}the lottery is strong
    price€{{nb11.cost}}the platform's rate card
    net per exposure€{{nb11.net}}β − c, at the point estimate
    +
    The verdict
    + BUY  Keep buying at €{{nb11.cost}}: the entire defensible range clears the price.
    +
    +
    +
    The recommendation, in three lines
    + 1. Keep buying at the €{{nb11.cost}} rate: even the interval's most pessimistic effect pays.
    + 2. Cap the bid at the interval's lower end, €{{nb11.iv_lo}}: up to there, every effect the data support still clears the price.
    + 3. Measure again only if the rate card climbs toward €{{nb11.iv_lo}}: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
    +
      +
    • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
      • +
    • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
      • +
    +
    +
    +
    + + + +
    + + + +
    + Causal Marketing · SDA Bocconi · Instrumental Variables + + + + 1 / 1 +
    +
    +

    Contents

      + + + + diff --git a/causal-marketing-pymc/apps/unified_slides.html b/causal-marketing-pymc/apps/unified_slides.html new file mode 100644 index 0000000..66854bf --- /dev/null +++ b/causal-marketing-pymc/apps/unified_slides.html @@ -0,0 +1,4357 @@ + + + + + + +Causal Inference in the Wild: from real engagements to two decisions in euros + + + + + +
      + + + +
      +
      Causal Inference & XAI for Business · SDA Bocconi
      +

      Causal Inference in the Wild

      +
      Real PyMC Labs engagements, then we build the machinery ourselves twice: a €4M rollout decision, and the price of one ad exposure.
      +
      A guest lecture for Prof. Michele Russo's course
      +
      +
      +
      Francesco Muia
      +
      PhD in Theoretical Physics, EMBA.
      Consultant for PyMC Labs and Brown University.
      +
      francesco.muia@pymc-labs.com
      +
      francesco.muia@ai-and-analytics-solutions.com
      +
      +
      +
      Alexander Fengler
      +
      PhD in Computational Cognitive Science.
      Postdoc at Brown University, consultant for PyMC Labs.
      +
      alexander.fengler@pymc-labs.com
      +
      +
      +
      + PyMC + PyMC Labs +
      +
      + + +
      +
      Opening · Who is talking
      +

      PyMC Labs: what we do

      +
      A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
      +
      +
        +
      • What we sell: senior modeling expertise, not software licenses.
        • +
      • Open source is the top of the funnel: the libraries build reach; the consulting monetizes the expertise behind them.
        • +
      • How we work: small senior teams, starting from the client's decision; the client keeps the model, not a black box.
        • +
      + + + + + + + +
      EngagementWhat it isTypical client
      Project consultingFixed-scope build of one custom model (marketing mix, forecasting, pricing)A specific, high-value decision problem
      Advisory / retainerOur modelers guiding an in-house teamFirms building their own capability
      Enablement & trainingWorkshops and upskilling on Bayesian methods and the toolingAnalytics teams standardizing on PyMC
      +
      The clients in this session
      + Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
      +
      ▸ Areas of interest, and where we work
      +
        +
      • Areas: marketing-mix modeling and media measurement, causal inference, demand forecasting and pricing, experimentation and A/B testing at scale, applied Bayesian modeling.
        • +
      • Sectors: consumer/CPG, retail and e-commerce, tech, and finance.
        • +
      +
      +
      +
      + +
      +
      Opening · A note on method
      +

      Bayesian vs frequentist, in one slide

      +
      Same data, two philosophies.
      +
      +
      +
      Frequentist
      + Reports how a procedure behaves over many hypothetical repetitions of the study.
      +
      Bayesian
      + Reports a full probability distribution over the unknown, given the data in hand.
      +
      +
      +
      + + + + + + + + +
      AspectFrequentistBayesian
      You geta point estimate and a confidence intervala full distribution over the unknown effect
      It answers"how would this procedure behave if I repeated the study many times?""given the data I have, how probable is each value?"
      Shines whensamples are large, the design is standard, speed mattersdata is small or messy, structure is layered, prior knowledge exists
      The catchthe 95% interval is not "95% probability the truth is inside"you must state a prior, and it costs computation
      +
      Where Bayes earns its keep
      + It answers the decision directly: not "is the lift real?" but a number you can act on, P(lift beats breakeven).
      +
      +
      +
      +
      SCHEMATICThe Bayesian answer is a shape, not a point
      +
      + + + + + -2 + 0 + 2 + 4 + 6 + + estimated lift (effect size) + + + no effect + + + + + posterior + P(lift > 0) = 0.92 + + + + + + classical: estimate + symmetric 95% CI + +
      +
      The posterior is skewed, so mode, mean, and the symmetric classical interval no longer coincide, and that interval even dips below zero, yet P(lift > 0) still reads straight off the shaded mass.
      +
      +
      +
      +
      Why marketing analytics is going Bayesian
      + Marketing data is short, noisy, and highly correlated, and every model ends in a spend decision, which is exactly where priors and full uncertainty pay off.
      +
      For today, this is a curiosity
      + The entire lecture runs on classical methods, and every number you will see is frequentist; we surface the Bayesian read only as a flavour at the edges, never as the load-bearing tool.
      +
      ▸ Where it is already the default (and where it is not)
      +
        +
      • Marketing mix modeling: the modern open-source MMM tools are Bayesian (Google's Meridian, PyMC-Marketing); with only two or three years of weekly data and correlated channels, plain regression is unstable, and priors on adstock, saturation, and ROI stabilize it.
        • +
      • Calibrating models to experiments: geo-lift and A/B results enter as priors on channel ROI, so the model and the experiments agree, the HelloFresh loop you will see today.
        • +
      • Decisions under uncertainty: you get a full distribution over each channel's ROAS, so budget optimization and statements like P(channel A beats B) fall out directly.
        • +
      • Many thin segments: hierarchical models borrow strength across regions, SKUs, or cohorts too small to estimate on their own.
        • +
      • Signal loss: cookie deprecation and privacy changes push measurement back toward aggregate MMM, which is Bayesian home turf.
        • +
      • Not everywhere: for large-sample, standard A/B tests and simple designs, classical methods are faster and perfectly adequate.
        • +
      +
      +
      ▸ Pros and cons, plainly
      + + + + + + +
      ProsCons
      Frequentistfast, familiar, few choices to defend, excellent for large samples and standard designsp-values and confidence intervals are routinely misread, and awkward once models get layered or data gets thin
      Bayesiandirect probability statements, priors fold in domain knowledge, hierarchical models borrow strength across groups, uncertainty flows straight into a decisionyou must choose priors (and defend them), sampling can be slow, and it demands more modeling care
      +
      +
      +
      + + +
      +
      Case 1 · Colgate-Palmolive
      +

      You are the consultant

      +
      The counterfactuals this session builds are what clients buy. First test: route a real call, on instinct.
      +
      +
      +
      ✋ Poll
      +
      Colgate-Palmolive calls: "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it stealing share from competitors, or from our own brands?" Which tool do you reach for first?
      +
      + + + + +
      + +
      C. A national launch leaves nothing to randomize and nothing to difference against: A, B and D all need a comparison that does not exist. What remains is this session's one move: fit the world before the launch, project it forward, read the gap. PyMC Labs sold exactly that; next slide.
      +
      +
      +
      + +
      +
      Case 1 · Colgate-Palmolive
      +

      Colgate-Palmolive: incremental, or cannibalistic?

      +
      Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The launch verdict is the split.
      +
      +
      +
      +
      The launch, and the world without it schematic
      +
      +
      Illustrative shape, not client data: fit the pre-period, project, price the gap.
      +
      +
      +
      +
      The brief, in their words
      + "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
      +
        +
      • The client: Colgate-Palmolive, 2023; a market estimated at $20.8 billion in 2023.
        • +
      • The method: a multivariate Bayesian interrupted time series: project the pre-launch world forward.
        • +
      • The grading: a planted 50% recovered as a 94% interval of 49-59%: recover a known truth first, then be believed.
        • +
      +
      +
      +
      + +
      +
      Case 1 · Colgate-Palmolive · open floor
      +

      What would break it?

      +
      +
      +
      🗣 Open floor · 2 minutes
      +
      You are Colgate's CMO. The incrementality estimate you just saw (the share of the new product's sales that are genuinely new, not cannibalized) decides the launch review. Name one real-world event that would make it wrong.
      + +
      +
      +
      +
      Learning 1 · what breaks it
      + A second launch inside the fit window redefines "normal growth", so the counterfactual absorbs part of the very effect it is meant to isolate.
      +
      Learning 2 · the lesson that runs the whole session
      + A counterfactual is only as good as a clean window: in Part 2 this disease returns with its formal name, spillover, and the defence is design, not statistics.
      +
      +
      +
      +
      Break it yourself: slide a second launch into the window schematic
      +
      +
      + + + +
      +
      Schematic, not client data: the red projection is refit live from whatever the pre-launch months show.
      +
      +
      +
      How to read the plot
      +
        +
      • Three lines: solid blue is observed sales, grey dashed is the true no-launch world, red dotted is the model's guess of that world, fit on the pre-launch months only.
        • +
      • The clean read: red sits exactly on grey and the readout says 100%: the model recovers the true lift.
        • +
      • Break it: tick the box; a second launch before month 0 pulls red off grey and the readout falls (under-credit), after month 0 it climbs past 100% (over-credit).
        • +
      • The real case: Colgate's model read 64-76% when the planted truth was 100%.
        • +
      +
      +
      +
      +
      +
      + + +
      +
      Case 2 · HelloFresh · the tool, and its failure mode
      +

      Why calibrate? A model alone can rank channels backwards

      +
      A public tutorial that shows the real job clients hire us for: making a company's ad-budget model trustworthy.
      +
      +
      +
      The settinga company spreads its budget across many ad channels (TV, search, social) and needs to know which ones actually pay back.
      +
      The everyday toola model reads years of spend-and-sales history and scores each channel's return on ad spend, cheaply and always-on.
      +
      The catch, and our jobhistory is not an experiment; a company spends more exactly when demand is already high, so the model can credit the season instead of the channel and rank them backwards.
      +
      What we adda real experiment: nudge one channel's budget by a known amount, measure the sales it truly causes, and anchor the model to that number. This is calibration.
      +
      +
      +
      The model's channel ranking, before and after a real experiment baked from the tutorial
      +
      +
      +
      Left: what the history-only model reported. Right: the true answer the experiments recover.
      +
      +
      The inversion, on a known answer
      + True returns were 93.39 versus 171.41, so channel 2 is nearly twice as effective; yet the uncalibrated model reported the direct opposite of the truth, and two lift tests per channel fixed it.
      +
      +
      + +
      +
      Case 2 · HelloFresh
      +

      HelloFresh runs the loop, at industrial scale

      +
      The same idea, not on a tutorial now but in production at a company you know.
      +
      +
      +
      +
      Not a toyHelloFresh, the meal-kit company, runs this exact model-plus-experiment loop in production.
      +
      The loopthe model steers the budget always-on, while a steady stream of lift or incrementality tests keeps re-anchoring it, cutting prediction variance by 60%.
      +
      Calibration, put plainlya Bayesian budget model should be "calibrated to ensure consistency with incrementality measurements".
      +
      The scalethousands of concurrent tests (A/B, ABC, and ABCD designs), each model fit batched down from 5–6 hours to 5–6 minutes; one experiment alone logged 13,979,592 users.
      +
      +
      The takeaway
      + The experiment is not a one-off audit; it is a supply line that keeps the budget model honest, campaign after campaign.
      +
      +
      +
      +
      The loop, in one picture
      +
      +
      The model runs always-on; the experiments keep correcting it; together they steer the budget.
      +
      +
      +
      +
      + + +
      +
      Case 3 · Nürnberger Versicherung
      +

      Price the engagement

      +
      A German insurer, last-touch attribution, and a funnel-aware causal MMM, in production.
      +
      +
      +
      ✋ Poll
      +
      Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over June through November of model-guided spend, cost per lead (CPL) moved by how much?
      +
      + + + + +
      + +
      C. "This year we were able to drive the CPL down by more than 27%, which is very, very good" (Philip Herp, Nürnberger Versicherung). Under GDPR, customer journeys appeared artificially shortened, so last-touch under-credited the upper funnel and budget followed attribution mechanics instead of incremental business impact. The client is scaling into full production for 2026.
      +
      +
      The client's bar for belief
      + "Trust is not created by R² values. It is created when business reality matches model expectations."
      +
      +
      + + +
      +
      +
      +
      Part 2 · The deep dive
      +

      Synthetic Control

      +
      One treated market, no experiment, €4M on the line: did the campaign work?
      +
      +
      + +
      Where the lecture lands: a synthetic twin rebuilds the metro's counterfactual, and the gap beyond break-even is the campaign's profit.
      +
      +
      +
      + + +
      +
      Act I · The question
      +

      The boardroom question

      +
      A campaign ran in one region. Sales rose. Marketing wants €4M to go national. The Colgate question again: what would have happened anyway?
      +
      +
      +
      +

      What you have

      +
        +
      • 30 markets, 60 weeks of weekly sales.
        • +
      • One metro, week 40: a €75k campaign for 20 weeks; the other 29 did nothing.
        • +
      • No randomisation: the metro was chosen, not drawn.
        • +
      • 35% gross margin on incremental sales.
        • +
      +
      +
      +

      What the data forces you to answer, in order

      +
      +
      ① How much did the campaign actually cause? attribution
      +
      ② Is that more than luck? significance
      +
      ③ Is the profit worth the price? the decision
      +
      +
      +
      +
      The method this data calls for
      +
        +
      • One treated unit, no A/B test possible: synthetic control's home ground.
        • +
      • It is the engine under Meta's GeoLift and Google's geo methodology.
        • +
      +
      The €4M hinge: the transportability discount
      +
        +
      • δ = the share of the pilot's per-euro lift that survives national rollout.
        • +
      • The €4M call reduces to one question: is δ large enough to still clear the cost?
        • +
      +
      ▸ What "unbiased" means here, precisely, and where it would break
      +

      The estimator subtracts a reconstructed counterfactual from the treated metro's post-launch sales, and bias is its expected gap from the true lift:

      +
      + \[ \hat\tau \;=\; \bar Y_{1,\text{post}} - \hat Y_{1,\text{post}}(0), \qquad \text{bias} \;\equiv\; \mathbb{E}[\hat\tau] - \tau \] +
      +
        +
      • The counterfactual is the synthetic twin, built to reproduce the metro's pre-launch record (its level and its factor loadings) week by week.
        • +
      • Unbiasedness rests on one assumption: that pre-launch match would have continued through the post window had the campaign never run.
        • +
      • This needs no story about why the metro was chosen. Whatever pre-launch trait drove the pick, the twin already carries it, because it was fitted to match it: selection on anything visible before launch cannot tilt the estimate.
        • +
      • Bias enters only through what the pre-launch record cannot see: if the launch were timed on private knowledge of a coming local boom, the twin could not anticipate it and the gap would credit the boom to the campaign. The later placebo-in-time test hunts exactly that.
        • +
      +
      +
      +
      + + +
      +
      Act I · The question
      +

      The data you actually get

      +
      The simulated world.
      +
      +
      +
      +
        +
      • \(Y_{jt}\): weekly sales (€000): 30 markets, 60 weeks (\(t=0,\dots,59\)); treated metro = market 1.
        • +
      • The design: a €75k campaign from week \(T_0=40\), 20 weeks, 35% margin; the other 29 ran nothing.
        • +
      • The choice: marketing picked the metro and the week; not a coin.
        • +
      +
      +
      +
      The one number you do not have
      + What the metro would have sold with no campaign: nowhere in the file. The counterfactual.
      +
      +
      +
      +
      +
      +
      ▶ LIVE 30 markets, one treated at week 40
      +
      +
      + + + +
      +
      +
      +
      +
        +
      • Before/after: post minus pre-launch mean, /week.
        • +
      • Subtract the crowd: remove the shared wave and the step is ≈ /week.
        • +
      • The noise floor: pre-launch sd /week around a /week base.
        • +
      +
      Which number decides?
      +
        +
      • Is this all due to the campaign?
        • +
      • What's the right number to take a decision?
        • +
      +
      +
      +
      ▸ Where each of these three readings comes back later
      +
        +
      • Before/after is the estimator \(\hat\tau^{\text{BA}}\) in the zoo you will grade: it assumes \(Y_{1t}(0)\) simply holds at the pre-launch mean.
        • +
      • Subtracting the crowd strips out the shared wave \(\gamma_j^{\top} f_t\) of the next slide's model, which is exactly what treated-vs-control and difference-in-differences try to do.
        • +
      • The noise floor is that model's own \(\varepsilon_{jt}\) term, the wobble every estimator must out-shout with a single treated unit.
        • +
      +
      +
      +
      + + +
      +
      Act I · The question
      +

      Commit to a number

      +
      +
      +
      ✋ Poll
      +
      Before/after on the treated metro: sales averaged ·/week before launch and ·/week after, a bump of ·/week (about +6%). What is the campaign's true weekly effect?
      +
      + + + +
      + +
      C. The true effect is ·/week: a 12% lift on a ·/week baseline. Before/after books barely half of it: the macro wave dipped after launch. The tide can inflate a campaign's credit or hide it; a naive comparison is charged with everything else that changed.
      +
      +
      +
      + + +
      +
      Act I · The question
      +

      Causal inference is a missing-data problem

      +
      +
      +

      The treated metro has two week-\(t\) sales, only one of which ever happens: \(Y_{1t}(1)\) with the campaign, \(Y_{1t}(0)\) without. The effect is the gap:

      +
      + \[ \tau_t \;=\; Y_{1t}(1)-Y_{1t}(0), \qquad \tau \;=\; \sum_{t\ge T_0}\tau_t \] +
      +
        +
      • \(\tau_t\) vs \(\tau\): the weekly effect, and the 20-week total the business actually buys.
        • +
      • Consistency: we record one world per week, never both.
        • +
      • The missing cell: \(Y_{1t}(0)\) for \(t\ge T_0\): every method fills exactly that cell.
        • +
      +
      Define the estimand first
      + Per-week and 20-week total are different estimands; the €4M turns on the total.
      +
      +
      + + + + + + + +
      pre (t < 40)post (t ≥ 40)
      treated · \(Y_{1t}(1)\)·observed ✓
      treated · \(Y_{1t}(0)\)observed ✓missing: the counterfactual
      donors · \(Y_{jt}(0)\)observed ✓observed ✓
      +
      Three cells are data; the donors' post-period is what a synthetic \(Y_{1t}(0)\) is rebuilt from.
      +
      ▸ Reading the notation (observed vs potential)
      +
        +
      • \(Y_{jt}\) from the data slide is what you observe; \(Y_{1t}(1),\,Y_{1t}(0)\) are the treated metro's two potential outcomes.
        • +
      • Subscript says which market, argument says which world (1 = campaign, 0 = none): distinct axes that happen to share the digit 1.
        • +
      • \(w_j\): the donor weights the synthetic control will choose so \(\sum_{j\ge 2} w_j Y_{jt}\) rebuilds the missing \(Y_{1t}(0)\).
        • +
      +

      "What would this metro have sold anyway?" is an estimation target, not a rhetorical question.

      +
      +
      +
      + + + + +
      +
      Act I · The question, and the world behind the data
      +

      The model that generated the data

      +
      We simulate the data: here the counterfactual is known, so every estimator can be graded.
      +
      +

      Each market: level \(\alpha_j\) + exposures \(\gamma_j^{\top} f_t\) to three shared factors + noise (all €000).

      +
      + \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} + \;=\; \alpha_j \;+\; \gamma_{j1}\underbrace{\,0.4\,t\,}_{\text{trend}} \;+\; \gamma_{j2}\underbrace{\bigl[\,8\sin(2\pi t/26)+4\sin(2\pi t/13)\,\bigr]}_{\text{seasonality}} \;+\; \gamma_{j3}\underbrace{\textstyle\sum_{s=1}^{t}\eta_s}_{\text{macro walk }m_t} \;+\; \varepsilon_{jt} \] +
      +
        +
      • The macro walk \(m_t\): a running sum of shocks that wanders like an economy.
        • +
      • The exposures \(\gamma_j\): each market responds its own amount; the spread \(s\) kills parallel trends.
        • +
      +
      ▸ All five ingredients, one by one
      +
        +
      • Trend \(0.4\,t\): steady growth of 0.4 (€000) per week. Deterministic.
        • +
      • Seasonality \(8\sin(2\pi t/26)+4\sin(2\pi t/13)\): a 26-week half-year cycle of amplitude 8 and a 13-week quarterly cycle of amplitude 4. Identical for every market.
        • +
      • The macro walk \(m_t\): one fresh shock \(\eta_s\sim\mathcal{N}(0,\sigma_\eta^2)\) per week, none ever forgotten. This is "the economy".
        • +
      • Exposures \(\gamma_j\): each market multiplies each ingredient by its own loading.
        • +
      • Noise \(\varepsilon_{jt}\sim\mathcal{N}(0,\sigma_\varepsilon^2)\): the market's own weekly wobble, \(\sigma_\varepsilon\approx 3\).
        • +
      +
      +
      Which of these is ever observed?
      + Only \(Y_{jt}\) is observed; levels, loadings, factors and noise are real but recorded nowhere. Every estimator copes with that.
      +
      The planted truth, in business units
      + A 12% lift switches on at week 40: ·/week, · over 20 weeks. Every method is graded against it.
      +
      ▸ The planted lift, written out
      +
      + \[ \tau_t \;=\; 0.12\,\bigl(\alpha_1+\gamma_1^{\top}f_t\bigr)\,\mathbf{1}_{\{t\ge T_0\}} \] + 12% of the treated metro's own baseline \(\alpha_1+\gamma_1^{\top}f_t\), switched on from week \(T_0=40\). +
      +
      +
      +
      + + +
      +
      Act I · The question, and the world behind the data
      +

      Simulate the world yourself

      +
      Take a look at the counterfactual.
      +
      +
      +
      ▶ LIVE The equations of the previous slide. Grey: donor markets. White: the treated metro.
      +
      +
      + + + + + + + + +
      +
      Dashed blue: \(Y_{1t}(0)\) itself, the line that in real data does not exist.
      +
      +
      +
      +
        +
      • True lift: the planted 12%, averaged over the 20 post weeks.
        • +
      • Before/after: the treated series' post-launch mean minus its pre-launch mean.
        • +
      • Its error: the tide, charged entirely to the campaign.
        • +
      +
      +
      +
      What the sliders teach
      +
        +
      • Spread \(s\to 0\): parallel trends, DiD lives.
        • +
      • Macro \(\sigma_\eta\) up: the tide dominates, before/after dies.
        • +
      • Noise \(\sigma_\varepsilon\) up: every estimate gets shakier.
        • +
      +
      +
      +
      +
      + +
      +
      Act II · The counterfactual problem
      +

      Abadie's idea: if no twin exists, build one

      +
      Idea: manufacture a treated market's twin using a blend of donor markets.
      +
      +

      Assume untreated sales follow the factor structure: level, loadings on shared factors, noise (equation in the fold).

      +
        +
      • \(f_t\), the latent factors: the shared shocks every market feels, unequally.
        • +
      • \(\gamma_j\), the loadings: how strongly market \(j\) responds; never observed in real data.
        • +
      • The method never names them: it works without knowing what the factors are.
        • +
      +

      Blend the donors so their level and loadings equal the treated metro's:

      +
      + \[ \sum_j w_j\,(\alpha_j,\;\gamma_j) = (\alpha_1,\;\gamma_1) \;\Longrightarrow\; \sum_j w_j\,Y_{jt} \approx Y_{1t}(0)\ \ \text{for all } t \] +
      +
        +
      • Same loadings, same response to every factor movement, including ones that have not happened yet: that forward guarantee is the whole trick.
        • +
      +
      Why it survives storms
      + DiD matches a level; synthetic control matches the response to shocks, so it survives what breaks DiD.
      +
      ▸ The factor equation, pedigree, and the papers' notation
      +
      + \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} \] + the assumed structure the blend must match, level and loadings, week by week +
      + Abadie & Gardeazabal (2003); Abadie, Diamond & Hainmueller (2010): "arguably the most important innovation in the policy-evaluation literature in the last 15 years" (Athey & Imbens 2017). The literature writes the factors \(\lambda_t\) and the loadings \(\mu_j\), and adds a common shock \(\delta_t\) felt identically by every market; any cross-market comparison cancels \(\delta_t\), so nothing changes here. +
      +
      +
      + +
      +
      Act II · The counterfactual problem
      +

      The estimator, precisely

      +
      A constrained least square optimization.
      +
      +

      Fit the weights on the pre-period only, constrained to the simplex:

      +
      + \[ \hat w \;=\; \arg\min_{w\in\Delta}\; \sum_{t \lt T_0}\Bigl(Y_{1t}-\sum_j w_j Y_{jt}\Bigr)^{2}, \qquad \Delta=\Bigl\{w : w_j\ge 0,\ \textstyle\sum_j w_j=1\Bigr\} \] +
      +
        +
      • Pre-launch only: the optimiser never sees the post-period, so it cannot cheat.
        • +
      • The simplex \(\Delta\): a readable recipe, never "−80% of Milan + 190% of Rome": Colgate's projection with the weights on the table.
        • +
      • No standard error: a point estimate only; inference comes later, from placebos.
        • +
      +
      ▸ Reading the effect off the gap
      +
      + \[ \hat\tau_t \;=\; Y_{1t}-\sum_j \hat w_j Y_{jt}, \qquad\qquad \hat\tau \;=\; \sum_{t\ge T_0}\hat\tau_t \] + weekly gap between the treated metro and its synthetic twin, summed over the 20 post-launch weeks +
      +
      +
      Why constrain at all?
      + The simplex buys three things ordinary regression cannot: interpretability, regularisation, and an off-switch.
      +
        +
      • Interpretability: "40% dma_08 + 32% dma_20 + 17% dma_03" is a sentence a planner can act on.
        • +
      • Regularisation: most weights land on exactly zero.
        • +
      • An off-switch: a market no blend can match fails loudly instead of extrapolating.
        • +
      +
      +
      + + +
      +
      Act II · The counterfactual problem
      +

      What the simplex buys, geometrically: stay inside the hull

      +
      A simplex blend can only land inside the donors' convex hull while OLS can leave it.
      +
      +
      +
      +
      +
      ▶ LIVE Drag the treated market in and out of the donors' hull
      +
      +
      + + +
      +
      +
      +
      +
        +
      • With \(w_j\ge 0\) and \(\sum_j w_j=1\), every point a synthetic market can reach lies inside the shaded convex hull.
        • +
      • Inside: a real blend reproduces the metro; the fit keeps working out of sample.
        • +
      • Outside: no real blend reaches it; only extrapolation (negative weights) closes the gap.
        • +
      +
      Why constrain the weights
      + A good fit is a real blend of observed markets, not an extrapolation that happens to line up.
      +
      When the treated market sits outside
      + Widen the donor pool; do not force a fit. The refusal is a feature. Ask any vendor: when does your method refuse? If never, walk away.
      +
      +
      +
      +
      + + + +
      +
      Act II · The counterfactual problem
      +

      What the constraint buys: drop it and see

      +
      What if we used a simple unconstrained OLS?
      +
      +
      +
      ▶ LIVE Two synthetics against the truth only a simulation can draw
      +
      +
      OLS hugs the pre-period tighter ( vs ) yet drifts further from the true \(Y_{1t}(0)\) after launch ( vs ): overfitting the simplex refuses.
      +
      +
      +
      +
      The weights OLS chose
      +
      +
      OLS puts negative weight on donors (down to ) and gross weight , where the simplex uses exactly 1.
      +
      +
      +
      + \[ n_{\mathrm{eff}} \;=\; \frac{1}{\sum_j \hat w_j^{2}} \] + Effective number of donors (inverse Herfindahl): one donor \(\Rightarrow n_{\mathrm{eff}}=1\), all 29 equally \(\Rightarrow 29\). Ours: 3.3. +
      +
      What n_eff = 3.3 tells you
      +
        +
      • The synthetic leans on about three donor markets, not a fuzzy mix of all 29.
        • +
      • Sparse weights: interpretable, and less room to overfit.
        • +
      • Near 1: hostage to one market. Near 29: mush. 3.3: concentrated, not fragile.
        • +
      +
      +
      +
      +
      + + +
      +
      Act II · The counterfactual problem, versus machine learning
      +

      "Why not gradient boosting / Prophet / an LSTM?"

      +
      Is this a forecasting exercise?
      +
      +
      +
      ▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
      +
      +
      Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
      +
      +
        +
      • In-sample it fits tighter: €k vs €k per week: flexibility always buys the past.
        • +
      • Yet the estimates are a wash: vs , both near the planted €284k. One dataset cannot rank them.
        • +
      • The sweep decides: across 24 fresh worlds, 1.6× worse out of sample, with nothing to inspect.
        • +
      +
      +
      +
      Principle 1: a counterfactual, not a forecast
      + A forecaster asks what comes next. We ask what this market would have done in a world that never happened.
      +
        +
      • Principle 2: in-sample fit is the trap, not the goal.
        • +
      • Principle 3: you cannot cross-validate the counterfactual. The truth is never observed.
        • +
      • Principle 4: the simplex ships an auditable claim; a boosted tree is a black box.
        • +
      +
      The forecaster is better at prediction. The simplex is better at the causal job.
      +
      +
      +
      + +
      +
      Act II · The counterfactual problem
      +

      Compare the estimators

      +
      Every "obvious" estimator is a claim about \(Y_{1t}(0)\).
      +
      +
      +
      +
      Four "obvious" estimators, four hidden assumptions
      + Each is a different guess at the missing \(Y_{1t}(0)\), unbiased only in its own special world (equations in the fold below).
      +
        +
      • Before/after: unbiased only in a flat world.
        • +
      • Treated vs average control: needs matching level and loadings; the level gap survives.
        • +
      • Difference-in-differences: unbiased iff trends are parallel (\(\gamma_1=\bar\gamma\)).
        • +
      • Synthetic control: unbiased iff the blend matches the loadings (treated inside the hull).
        • +
      +
      +
      +
      ▶ LIVE Four estimates, one truth: the world's dials exposed
      +
      +
      + + + + +
      +
      +
      +
      ▸ The four estimators, written out with their bias conditions
      +
      + \[ \hat\tau^{\text{BA}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{1,\text{pre}} \] + Before/after: unbiased only in a flat world, equal shared-factor means pre and post (the macro-shock dial drives its error). +
      +
      + \[ \hat\tau^{\text{T-C}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{\text{ctrl},\text{post}} \] + Treated vs average control: unbiased only if controls match the treated in level and loadings; the level gap survives even at spread \(s\to 0\). +
      +
      + \[ \hat\tau^{\text{DiD}} \;=\; \Delta_{\text{treated}} - \Delta_{\text{ctrl}}, \quad \Delta = \bar Y_{\text{post}} - \bar Y_{\text{pre}} \] + Difference-in-differences: removes the level gap; unbiased iff trends are parallel (equal loadings \(\gamma_1=\bar\gamma\)), which spread \(s\to 0\) delivers. +
      +
      + \[ \hat\tau^{\text{SC}} \;=\; \bar Y_{1,\text{post}} - \textstyle\sum_j w_j\, \bar Y_{j,\text{post}} \] + Synthetic control (the one you fitted): unbiased iff a fitted blend matches the treated's loadings (\(\sum_j w_j\gamma_j=\gamma_1\), the treated inside the hull). +
      +
      +
      ▸ How to read the dials (red = error against the truth)
      +
        +
      • Loading spread → 0: markets react identically, trends become parallel, DiD's error collapses to noise. Treated-minus-control keeps its error: the level gap never dies.
        • +
      • Macro shock up, with spread > 0: before/after's error explodes, DiD's grows with the walk, and synthetic control stays pinned near the truth: it matches loadings instead of hoping they are equal.
        • +
      +
      +
      None of these numbers ships a warning label
      + Each error is a wrong claim about \(Y_{1t}(0)\), and a wrong claim never improves with more data: the cure is a better claim, not a bigger sample.
      +
      +
      + + +
      +
      Act II · The counterfactual problem
      +

      Interesting limits for the simpler estimators

      +
      Where before/after and treated/control estimators break.
      +
      +
      +
      +
      Before/after · one unit, across time
      + The treated metro's post-average minus its pre-average.
      +
        +
      • The level cancels; nothing subtracts the shared wave.
        • +
      • The whole drift \(\Delta\bar f\) lands on the metro: the largest bias, growing with the horizon.
        • +
      +
      +
      +
      Treated-vs-control · one period, across units
      + The treated metro minus the control average, both in the post window.
      +
        +
      • The level gap \(\alpha_1-\bar\alpha_C\) survives, and sizes differ a lot: it dominates.
        • +
      • This is why raw sales are never compared across cities.
        • +
      +
      +
      +
      ▸ The two bias decompositions, term by term
      +
      + \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] + the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. +
      +
      + \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] + no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. +
      +
      +
      +
      + + +
      +
      Act II · The counterfactual problem
      +

      Interesting limits for DiD

      +
      Where the DiD estimator breaks.
      +
      +
      The one question
      + DiD only works if treated and controls would have drifted together. Two dials decide (spread \(s\), macro \(\sigma_\eta\)); the algebra is in the fold.
      +
      Synthetic control solves it
      + DiD's residual bias \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\) comes from its equal weights: it hopes \(s\) is small. Synthetic control chooses the weights, so \(B\to0\) at any spread.
      +
      ▸ The DiD bias, decomposed (and why more data cannot shrink it)
      +
      + \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] +
      +
        +
      • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
        • +
      • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
        • +
      +
      + \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] +
      +
        +
      • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
        • +
      • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
        • +
      +
      +
      ▸ The two limits, dial by dial: send a knob to zero and read what dies
      +
      +
      Limit 1 · loading spread \(s \to 0\): sufficient on its own
      +
        +
      • The prefactor \(s^2/3\) kills the whole bracket at once.
        • +
      • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
        • +
      • DiD then recovers \(\bar\tau\) up to noise.
        • +
      +
      Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
      +
        +
      • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
        • +
      • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
        • +
      • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
        • +
      +
      +
      +
      ▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
      + Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. +
      +
      +
      + + +
      +
      Act II · The counterfactual problem
      +

      What must be true for the gap to be causal

      +
      The four assumptions that must hold.
      +
      + + + + + + + + + + + + + + + + + + + + +
      AssumptionWhat it demandsThe checkVerdict
      ① Pre-fit qualityThe blend tracks the metro before launch.The gate: 3.2 vs a bar of 4.4.PASS
      ② No anticipationNo reaction before the campaign (no stockpiling, no leak).Placebo-in-time: the fake launch finds ≈ €1.7k, inside noise.PASS
      ③ No spillover (SUTVA)The campaign must not touch donors: Colgate's second-launch leak, in space.No statistical check exists. A design duty: keep test and control regions apart.UNTESTABLE
      ④ Hull & stable loadingsThe metro is a feasible mix of donors; exposures stay put.Leave-one-out: drop any donor, the estimate stays in €239–276k.PASS
      +
      The referee principle
      + Three testable, all PASS: checks ② and ③ are run in Act III, once the luck is priced. The untestable one, no-spillover, is why geo-test design beats any statistic computed afterwards.
      +
      +
      + + +
      +
      Act III · Is it real, and how big?
      +

      €260k. Real, or a lucky metro? Build the null yourself

      +
      +
      +
      ✋ Poll
      +
      With one treated metro a t-test is not weak, it is undefined. So build the test yourself: rerun the whole pipeline 29 more times, each donor pretended treated ("placebo"): 30 "effects", 29 where nothing ran. Where does our €260k rank?
      +
      + + + +
      + +
      A: rank 1 of 30. The rank is the inference: if the campaign did nothing, P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test.
      +
      +
      +
      + + +
      +
      Act III · Is it real, and how big?
      +

      Placebo-in-space: measure the luck directly

      +
      Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
      +
      +
      +
      +
      +
      ▶ LIVE Refit the estimator on every donor as if it were treated
      +
      +
      The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
      +
      +
      +
      +
      The null hypothesis, stated
      + \(H_0\): the campaign did nothing. Then the metro is exchangeable with its donors: any of the 30 ranks is equally likely.
      +
      + \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] +
      +
      +
      +
      What the p-value means (and what it is not)
      + The probability, if \(H_0\) were true, of a gap this extreme: a rank, no bell curve assumed. \(p\) sits at its floor \(1/30\), set by the donor count, not the weeks.
      +
      ▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
      +
        +
      • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
        • +
      • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
        • +
      +
      Conclusion: the €260k lift is real, at \(p\approx0.033\)
      + Rank 1 of 30, clear of the cloud: reject \(H_0\). The rank holds (p = 0.033 every time) dropping shaky placebos at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": Act IV.
      +
      +
      + + +
      +
      Act III · Is it real, and how big?
      +

      From a test to an interval: inversion

      +
      Which true lifts could plausibly have produced our €260k? Keep the survivors: no bell curve anywhere.
      +
      +
      +
      +
      +
      ▶ LIVE Drag \(H\), a guess at the truth: the error cloud slides with it; our €260k never moves.
      +
      +
      + + +
      +
      Axis: 20-week total gap (€000). Top: the placebo errors at truth 0. Middle: the same errors slid to \(H\); \(H\) survives if the green line sits inside the shaded middle 90%. Bottom: the survivors, collected: the interval. Drag past an edge to reject.
      +
      +
      +
      +
      The whole idea
      + Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
      +
        +
      • ① Measure the error. The 29 placebo "effects" form a cloud around zero, spread about ±€50k.
        • +
      • ② Guess a truth \(H\). If the lift were \(H\), we would see \(H\) plus that same cloud.
        • +
      • ③ Keep or reject. Keep \(H\) if €260k sits inside its middle 90%.
        • +
      • ④ Sweep. The survivors run €195k to €335k: the interval is [€195k, €335k].
        • +
      +
      +
      +
      ▸ The inversion, in one line of algebra
      +
      + \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} + \qquad\Longleftrightarrow\qquad + \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] + \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. +
      +
      +
      ▸ Why this interval is the referee for the rest of the lecture
      + It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. +
      +
      +
      + + +
      +
      Act III · Is it real, and how big? Stress-test the estimate
      +

      Falsification 1: a launch that did not happen

      +
      The anticipation check: unlike Colgate's simulated grading, this needs no known answer, because a launch that has not happened yet must have zero effect.
      +
      +
      How it is computed
      + Refit using only weeks 0-29, back-date the launch to week 30, and read the "effect" in weeks 30-39: a window before the real launch, so its true effect is zero by the calendar, not by assumption.
      +
      ▶ LIVE The weekly gap (treated minus synthetic) around a back-dated launch
      +
      +
      + + +
      +
      Clean world: the gap hugs zero in the fake window (≈ €1.7k). Tick the box: a leak climbs before week 40.
      +
      +
      What passing rules out
      + Stockpiling, pre-announcement, a leaked launch: anything that moves sales before the campaign.
      +
      +
      + + +
      +
      Act III · Is it real, and how big? Stress-test the estimate
      +

      Falsification 2: the assumption no placebo can see

      +
      The no-spillover check: the disease you named for Colgate, now in space.
      +
      +
      The assumption (SUTVA)
      + Every method so far assumed the campaign moved only the treated metro. Advertising ignores borders, so we break that on purpose and measure the damage.
      +
      +
      ▶ LIVE Four worlds, rebuilt with a bigger and bigger leak
      +
      +
      Each point is a complete re-simulation and refit, not a correction applied afterwards.
      +
      +
      +
      The concrete story
      + TV and outdoor reach the commuter belt, and those markets are in your donor pool: the campaign quietly raises your "untreated" comparison.
      +
      + \(\varphi\) = the share of the treated lift leaking into 5 neighbour donors, then refit: lifting the donors shrinks the measured gap. +
      +
      +
      +
        +
      • The sweep: \(\varphi\) from 0 to 0.5: clean world to half the lift leaking.
        • +
      • The result: the reported total slides from €260k (clean) to €236k, always toward zero, exactly how a second launch bent Colgate's counterfactual: a contaminated baseline, now in space instead of time.
        • +
      • Dangerous because invisible: the leak starts at launch; every check stays green.
        • +
      • What drives the decision: the bias only shrinks the number: a conservative floor. The defence is design: donors off the media footprint.
        • +
      +
      +
      + + +
      +
      Act III · Is it real, and how big?
      +

      Statistics done. Three numbers.

      +
      Questions ① and ② from the boardroom slide are now answered.
      +
      + + + + + + + +
      QuestionAnswerTool that answered it
      Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
      How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
      Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
      +
      Truth check (only a simulation allows it)
      + The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
      +
      The sentence that loses money
      + "€260k of sales for €75k, a 3.5× return. Roll it out." Every number true; the conclusion does not follow. Question ③ is not a statistics question.
      +
      +
      + + +
      +
      Act IV · The decision in euros
      +

      Now the money

      +
      +
      +
      ✋ Poll
      +
      Cost €75k, incremental sales €260k, gross margin 35%. What did the campaign earn?
      +
      + + + + +
      + +
      C: about €16k net. Profit = margin × lift − cost = 0.35 × 260 − 75 ≈ +€16k. It returned €3.47 per €1 against a break-even of 2.86: it paid, in expectation. With how much certainty is what this act prices.
      +
      +
      +
      + + +
      +
      Act IV · The decision in euros
      +

      The margin trap

      +
      Compare profit to cost, not revenue to cost.
      +
      +
      +
      +
      \[ \text{profit} \;=\; m\,\tau \;-\; c \]
      +
      • Revenue is not cash: you keep the margin \(m=35\%\) of the lift; you pay the cost in full.
      +
      +
      +
      \[ \tau_{\mathrm{BE}} = \frac{c}{m} = \frac{75}{0.35} = \text{€}214\mathrm{k}, \qquad \mathrm{iROAS}_{\mathrm{BE}} = \frac{1}{m} = 2.86 \]
      +
      • Break-even is €214k of sales, not €75k: €260k clears it by €46k (≈ €16k of profit): iROAS 3.47 vs the \(1/m=2.86\) bar.
        • +
      • Hold onto €214k: it returns as the hinge of the €4M decision.
      +
      +
      +
      +
      ▶ LIVE Where the €260k goes: slide the margin
      +
      +
      + + +
      +
      "iROAS > 1" is a vanity bar; the real bar is 1/margin. Software (90% margin) breaks even at 1.1; grocery (20%) needs 5.0. Same campaign, opposite verdicts.
      +
      +
      +
      + + +
      +
      Act IV · The decision in euros
      +

      The same evidence at every price

      +
      When is the campaign profitable?
      +
      +
      +
      • Line 1, the point: net \(m\hat\tau-c\) crosses zero at \(c=\text{€}91\mathrm{k}\).
      +
      • Line 2, the straddle zone: the profit interval \([\text{€}68\mathrm{k},\text{€}117\mathrm{k}]\), inseparable from break-even.
      +
      +
      +
      ▶ LIVE Slide the campaign price and watch the verdict move
      +
      +
      + + +
      +
      Between the lines: "looks profitable" (left of €91k), not provably so. Ours at €75k is one of them.
      +
      + + + +
      PriceBreak-even liftNet at \(\hat\tau\)VerdictThe lesson
      +
      +
      + + +
      +
      Act IV · The decision in euros, delivered
      +

      The pilot verdict, delivered to the board

      +
      This slide answers the €75k pilot. The €4M is a separate question, next.
      +
      +
      +

      Three questions about the €75k pilot, all answered by the permutation machinery:

      +
        +
      • ① Real? Yes. Rank 1 of 30, \(p=0.033\).
        • +
      • ② Big enough? On the point, yes: €260k vs break-even €214k: €46k to spare, ≈ €16k net.
        • +
      • ③ Confident? No. Profit [€68k, €117k] still straddles the €75k price.
        • +
      +
      Pilot verdict, banked
      + At €75k, probably profitable but not a confident buy: renegotiate the price or buy certainty with a bigger test. This says nothing yet about the €4M, which is not the pilot multiplied.
      +
      +
      + + + + + + + + + + +
      Board numberValueSource
      Incremental sales, 20 wk€260ksynthetic-control gap
      Real, or luck?p = 0.033placebo permutation, rank 1/30
      90% interval on sales[€195k, €335k]test inversion
      iROAS vs break-even3.47 vs 2.86P&L: break-even = 1/margin
      Gross profit vs price€91k vs €75k0.35 × 260 − 75
      Pilot recommendationNOT A CONFIDENT BUYinterval straddles the price
      +
      The one question this memo cannot answer
      + A classical memo is yes/no about where an interval sits: it cannot say "78% chance it pays", price the evidence, or value a follow-up study. And it is only about €75k: the €4M is next.
      +
      +
      +
      + + +
      +
      Act IV · The decision in euros, scaled up
      +

      The €4M is not provably profitable

      +
      Even at its best case, the profit interval straddles zero.
      +
      +
      +
      +
      +
      ▶ LIVE Rollout profit at €4M, carrying the 90% interval, as a share \(\delta\) of the pilot survives
      +
      +
      + + +
      +
      The point says +€860k at \(\delta=1\), but the 90% band [−€360k, +€2.25M] straddles zero even there. Drag \(\delta\) down (the real world, where each euro does less) and it only sinks; below \(\delta=0.64\) it wholly loses.
      +
      +
      +
      +
        +
      • ① The marketing number is a point: 53 pilot-budgets × €260k × 35% margin − €4M ≈ +€860k. Every step correct.
        • +
      • ② Carry the interval, and be generous: assume the pilot transports perfectly (\(\delta=1\)). Even then the 90% profit interval is [−€360k, +€2.25M]: it straddles zero.
        • +
      • ③ And \(\delta=1\) is a ceiling: €4M is €133k per market (1.78× the pilot, so each euro does less) and one metro is not the nation, so real \(\delta<1\) and the band only sinks.
        • +
      +
      The frequentist verdict, no probability needed
      + At its most generous the €4M is not provably profitable, and below \(\delta\approx0.64\) it provably loses. Do not commit €4M on this evidence.
      +
      +
      +
      ▸ Where 0.82 and 0.64 come from, and a defensible estimate of δ
      +
      +
      +
      The bar: required δ (arithmetic)
      +
      \[ \delta^{*} = \frac{c}{m\,\tau} = \frac{\tau_{\mathrm{BE}}}{\tau} = \frac{214}{\tau} \]reach \(R=C/c\), so the budget \(C\) cancels: the bar is set by cost, margin and lift only
      +
        +
      • Across the lift interval [€195k, €335k], required δ ∈ [0.64, 1.10], central 0.82 (82%).
        • +
      • Below 0.64 even the top of the interval loses: that is the "provably loses" edge.
        • +
      +
      +
      +
      The estimate: plausible δ (one assumption)
      +
      \[ \delta \;\approx\; 1.78^{\,\beta-1} \]per-euro lift at 1.78× intensity, up a concave response curve of elasticity \(\beta\)
      +
        +
      • \(\beta\) from the client's MMM, or "double the spend → ≈1.6× response" → \(\beta\approx0.68\) → δ ≈ 0.83.
        • +
      • \(\beta\) 0.6–0.8 gives [0.79, 0.89], central 0.84 (84%); being one metro only widens this; ours sits at the median (effect CV ≈ 0.14).
        • +
      +
      +
      +
      Why we measure, not estimate
      + The estimate 0.84 lands on the bar 0.82: profit at €4M runs −€1.11M / about €0 / +€1.57M across the pessimistic / central / optimistic corners. Defensible enough to refuse €4M, too close to the line to commit it: only a multi-market test separates the two.
      +
      +
      +
      + + +
      +
      Act IV · The decision in euros, scaled up
      +

      Measure before you commit

      +
      One treated market made this interval; only more will shrink it.
      +
      +
      +
      +
      The call
      + Even the best case straddled zero, so on this evidence do not commit €4M. The rollout is not dead: the evidence simply cannot size it.
      +
        +
      • Why analysis cannot rescue it: the interval is wide because there is one treated market. Re-running the model does not narrow it; only more treated markets do.
        • +
      • Step zero, first: check the drawer: prior pilots, the media-mix model, benchmarks may already answer it.
        • +
      +
      +
      +
      The fix: measure the national effect directly
      + Run it in 8 markets chosen to look like the nation, at full intensity (~€133k each, €1.07M). The cells span the country, so the measured lift is the national lift. HelloFresh's loop, bought once.
      +
      The rule, written before the data lands
      + Release the remaining €2.9M only if the measured 90% profit interval clears zero. Written first, or the test is theatre.
      +
      The recommendation
      + ① Check the drawer. ② Do not commit €4M: the best case is not provably profitable. ③ Run the €1.07M, 8-market test at national intensity, randomised. ④ Separately: renegotiate the pilot toward €67k.
      +
      +
      +
      ▸ Sizing the test, and why 8 markets
      +
      \[ k \;\ge\; \Bigl[\tfrac{(z_{0.95}+z_{0.80})\,\mathrm{CV}}{\Delta}\Bigr]^{2} = \Bigl[\tfrac{2.49\times0.30}{0.36}\Bigr]^{2} \approx 4.3 \]
      +
        +
      • Sized from the decision: distinguish δ = 1 from δ = 0.64 (a 36% shortfall) against between-market spread; ≈ 5 markets on power, 8 to span the mix, and CV 0.4 pushes power itself to ≈ 8.
        • +
      • Constraints: randomise in matched pairs (that alone frees the p-value from the single-metro floor, 0.033 → ≈ 0.004); test at deployment intensity; keep controls off the media footprint.
        • +
      • Worth it: the Bayesian layer of this course prices this information at ≈ €320k, far above the test's incremental cost, since the €1.07M is real advertising working while it measures.
        • +
      +
      +
      +
      + + + + +
      +
      +
      +
      Part 3 · The natural experiment
      +

      Instrumental Variables

      +
      What is one ad exposure worth, when the platform already picked who sees the ads?
      +
      +
      + +
      Where this part lands: the dashboard's tight interval misses the true effect of an ad exposure, while the interval built from a lottery covers it, and still clears the price.
      +
      +
      +
      + + +
      +
      IV · The problem
      +

      The case

      +
      One retailer, one ad platform, one decision to make.
      +
      +
      +
      The product
      An online retailer pays an ad platform to show its display ads. One exposure = one user actually saw an ad.
      +
      The price
      Each exposure costs €10, billed by the platform.
      +
      The assignment
      The platform decides, in a real-time auction, which users see the ads.
      +
      The dashboard
      Users who saw an ad brought in €23.7 more than users who did not.
      +
      +
      The platform's pitch
      + "An exposure is worth €23.7 and costs €10. Raise the budget."
      +
      The decision on the table
      + Keep paying €10 per exposure, pay more, or stop. One number settles it: how many euros of extra sales one exposure brings in. All we have is the platform's logs: who saw an ad, and what every user spent.
      +

      The question for the poll: is the dashboard's €23.7 that number?

      +
      +
      + +
      +
      IV · The problem
      +

      Poll · the pitch

      +
      +
      +
      ✋ Poll
      +
      The dashboard reports that users who saw an ad brought in €23.7 more than users who did not, and an exposure costs €10. The platform concludes: "raise the budget". What do you think of that claim?
      +
      + + + + +
      + +
      C. The arithmetic is correct and, at this sample size, comfortably significant, so A and B both miss the point. The platform chose who saw the ads, and it is paid to pick users who already look ready to buy. If the two groups differed before any ad ran, the €23.7 mixes the ad's effect with that pre-existing difference, in unknown proportions. D is backwards: averaging over thousands of users is exactly how noise is handled.
      +
      +
      +
      + +
      +
      IV · The problem · the variables, drawn
      +

      The variables and the confounder

      +
      The three quantities of the case, defined through their causal diagram.
      +
      +
      +

      The platform's logs cover 3,000 customers, indexed by \(i\). Three quantities per customer:

      +
        +
      • \(Y_i\) · sales: contribution euros earned from customer \(i\) in the window (sales net of product cost); "sales" for short. Noisy: the standard deviation across customers is €17.1.
        • +
      • \(X_i\) · exposure: 1 if customer \(i\) saw the ad, 0 if not. Assigned by the platform's auction, not by us.
        • +
      • \(U_i\) · intent: how ready \(i\) was to buy before any ad. Real, and it drives buying, but it is a feeling in a shopper's head: in no log file, never observed.
        • +
      +
      Definition · confounder
      + A confounder is a variable that influences both the treatment and the outcome. Here it is intent \(U\): it raises the chance the platform shows the ad (the targeting), and it raises sales on its own (ready buyers buy, ad or not).
      +
      Definition · backdoor path
      + In the diagram, \(X \leftarrow U \rightarrow Y\) is a backdoor path: a second route connecting exposure and sales that runs behind the treatment. Association flows along it even if the ad does nothing.
      +
      +
      +
      the situation, as a diagram
      +
      +
      \(U\) points at both \(X\) and \(Y\): the fork every claim in this lecture has to get past.
      +
      +
      +
      + +
      +
      IV · The problem
      +

      The naive comparison

      +
      The dashboard's €23.7, written down as a formula.
      +
      +
      +

      Write \(Y_i(1)\) for what customer \(i\) spends with the ad and \(Y_i(0)\) for what the same customer spends without it. Each customer shows us exactly one of the two:

      + + + + + + +
      group\(Y_i(1)\) · with ad\(Y_i(0)\) · without ad
      exposed (\(X_i=1\))observedN/A
      unexposed (\(X_i=0\))N/Aobserved
      +
      \[ \widehat{\Delta}_{\text{naive}} \;=\; \bar{Y}_{\text{exposed}} \;-\; \bar{Y}_{\text{unexposed}} \;=\; \text{€}23.7 \]
      +
        +
      • What it averages: the two observed cells, one from each row. The arithmetic is right.
        • +
      • What it pretends: that the unexposed row's \(Y(0)\) is a fair stand-in for the exposed row's missing \(Y(0)\). Fair only if the two groups are comparable.
        • +
      • The auction works against us: the platform is paid to find likely buyers, so the exposed would have spent more with no ad at all.
        • +
      +
      +
      +
      +
      ▶ LIVE exposed users were already different
      +
      +
      + + +
      +
      The gap compares two groups the auction built to differ. The hidden column shows by how much.
      +
      +
      +
      +
      + +
      +
      IV · The problem
      +

      Poll · read the gap

      +
      +
      +
      ✋ Poll
      +
      The naive comparison gives \(\widehat{\Delta}_{\text{naive}} = \) €23.7. Which reading of that number is defensible?
      +
      + + + + +
      + +
      B. The arithmetic is sound, so C is out. A and D each pick one extreme of the same unknown split: A books all of it to the ad, D books all of it to targeting, and the dashboard alone cannot justify either. The honest statement is the decomposition: €23.7 = the ad's effect + selection bias, with nothing on the dashboard saying how it splits. Measuring that split is the technical problem this lecture solves.
      +
      +
      +
      + +
      +
      IV · The problem · the poll's answer, formalized
      +

      Selection bias

      +
      The decomposition behind the poll's answer.
      +
      +
      +

      Split the naive gap into the two things it adds together:

      +
      \[ \Delta_{\text{naive}} \;=\; \underbrace{\mathbb{E}[Y(1)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}1]}_{\text{effect of the ad on the exposed}} \;+\; \underbrace{\mathbb{E}[Y(0)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}0]}_{\text{selection bias}} \]
      +
        +
      • Read the first term: the exposed users with the ad versus the same users without it. This is what we want to buy, and its second entry is the table's N/A cell.
        • +
      • Read the second term: the gap that would exist with no ads at all, because the picked users differ from the skipped ones.
        • +
      • Here it is positive: the auction picks ready buyers, so the exposed would have out-spent the unexposed anyway.
        • +
      • The dashboard reports only the total: €23.7. Nothing on it says how the total splits.
        • +
      +
      Definition · selection bias
      + When treated and untreated groups differed before the treatment, the difference of their averages includes that pre-existing gap. The pre-existing part is selection bias.
      +
      +
      +
      +
      the dashboard's number is a sum, and the split is invisible
      +
      +
      The dashboard hands us the full bar. Where the boundary sits, nobody can see from the dashboard alone. Finding it is the rest of the lecture.
      +
      +
      +
      +
      + +
      +
      IV · The problem · a world we can grade
      +

      The simulated world

      +
      A world where the true effect is planted, so every method can be graded.
      +
      +
      +
      +
      ▶ LIVE the world the equations generate, as data
      +
      +
      + + +
      +
      Sales distributions for exposed and unexposed users, drawn fresh from the equations. Push \(\kappa\) up, so that intent leaks more strongly into sales, and the two groups drift apart with the planted effect unchanged: the dashboard's gap grows while the truth stands still.
      +
      +
      +
      +
      Why simulate at all
      + Three equations generate the dashboard you saw. We planted an ad effect of \(\beta =\) €15, so every method in this lecture can be graded against a known answer.
      +
      \[ U \sim \mathcal{N}(0,1) \]
      +
      \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \lambda\,U\right) \]
      +
      \[ Y \;=\; \mu + \beta\,X \;+\; \kappa\,U \;+\; \varepsilon \] + \(\mu\): baseline sales  ·  \(\alpha_0\): baseline exposure odds  ·  \(\varepsilon\): noise
      +
        +
      • Interpretation of line 2: who sees an ad rises with intent (\(\lambda U\)): the targeting. \(\sigma\) is the logistic function, turning any score into a probability.
        • +
      • Interpretation of line 3: sales are the ad effect \(\beta\) plus intent leaking straight into spending (\(\kappa U\)).
        • +
      • The math is here for the record. The point is on the left: one hidden cause, \(U\), sits in both equations.
        • +
      +
      +
      +
      + +
      +
      IV · The problem · the disease, stated exactly
      +

      Endogeneity

      +
      The whole problem, stated as one inequality, then shown as a picture.
      +
      +
      +

      Collect everything the sales equation leaves out into a single error term \(v\):

      +
      \[ Y \;=\; \mu + \beta\,X + v, \qquad v \;=\; \kappa\,U + \varepsilon \]
      +
      \[ \operatorname{Cov}(X,\, v) \;>\; 0 \]
      +
        +
      • In words: \(v\) is "every other reason this customer spent money", and exposure is not independent of those reasons, because the auction targets exactly the customers with reasons to spend.
        • +
      • The picture on the right is the inequality: walk up the intent scale and both bars rise together. That joint rise is \(\operatorname{Cov}(X,v) > 0\).
        • +
      +
      Definition · endogeneity
      + A treatment correlated with the other drivers of the outcome is endogenous. Ours is, and everything else in this lecture is a response to that fact.
      +
      The sharp edge
      + The variable driving the correlation was never written to disk.
      +
      +
      +
      +
      one hidden cause pushes both
      +
      +
      Customers grouped by buying intent (which nobody observes). Higher-intent customers see more ads and spend more, ad or no ad. The formula on the left is this picture, in symbols.
      +
      +
      +
      +
      + +
      +
      IV · The problem · the bias, computed
      +

      The size of the bias

      +
      The dashboard's error is not vague. It has a formula, and the formula has three lessons.
      +
      +
      +
      \[ \widehat{\Delta}_{\text{naive}} \;\longrightarrow\; \beta \;+\; \underbrace{\kappa\,\frac{\operatorname{Cov}(X, U)}{\operatorname{Var}(X)}}_{\text{selection bias}} \]
      +
        +
      • Predicted vs delivered: the formula predicts a naive gap near €23.5. The data delivered €23.7, against a truth of €15.
        • +
      +
      What the formula teaches
      + 1. The error is systematic, not bad luck: both factors are positive, so the dashboard can only overstate.
      + 2. It grows with the targeting (\(\lambda\), how strongly intent drives exposure) and with intent's pull on sales (\(\kappa\)): the better the platform targets, the worse the dashboard lies.
      + 3. Sample size is absent: more rows shrink noise, never bias. The dashboard's band, €22.9 to €24.5, is the tightest in this lecture and misses €15 completely.
      +
      What the dashboard measures
      + Not the ad's power to create buyers but the platform's skill at finding them, reported as one number.
      +
      +
      +
      +
      ▶ LIVE the same bar, at different targeting strengths
      +
      +
      + + +
      +
      The dashboard's bar, split by the formula: the true effect (blue) never moves, while the selection bias (orange) grows as intent leaks more strongly into sales. In this simulated world we can draw the boundary, because we planted it.
      +
      +
      +
      +
      + +
      +
      IV · The problem · closing the obvious exit
      +

      The limits of adjustment

      +
      The standard fix works, when you have the column it needs. We do not.
      +
      +
      +
      The standard fix: close the backdoor
      + Compare users at the same level of the confounder. Within a group of equal-intent users, intent no longer separates exposed from unexposed, and any remaining gap belongs to the ad. Controls in a regression, matching, and propensity scores are all versions of this one move.
      +
      +
      ▶ LIVE the fix works, if the column exists
      +
      +
      + + +
      +
      Tick the box: comparing exposed to unexposed within equal-intent groups recovers the planted truth to within half a euro. The method is fine. Untick it: in the real logs the column does not exist, and no amount of modelling recreates it.
      +
      +
      Why the engine stalls here
      + You cannot adjust for what you did not record.
      +
      +
      +

      Two customers, identical on every recorded feature \(W\):

      + + + + + + +
      customerpages, device, history \(W\)intent \(U\)sees the ad?
      Annaidenticalhigh, unrecordedpicked
      Beaidenticallow, unrecordedskipped
      +
      what conditioning blocks, and what it cannot
      +
      +
      Holding \(W\) fixed (the box) blocks the backdoors that run through it. The arrows themselves never move: conditioning blocks paths, it does not delete causes. The path through \(U\) stays open, because \(U\) is in no file to hold fixed.
      +
      +
      +
      + +
      +
      IV · The problem
      +

      Poll · what would fix it?

      +
      +
      +
      ✋ Poll
      +
      Adjustment failed because intent was never recorded. Your team can request one addition to the data. Which one would actually let you measure what an exposure causes?
      +
      + + + + +
      + +
      C. A tells you the wrong number more precisely: bias is not noise, and the formula for it does not contain the sample size. B and D are more controls, and the auction reacts to live signals that no exported feature set fully contains, so the backdoor stays open. C is different in kind: it changes who assigns the treatment, and the assignment was the disease all along. The rest of the lecture is about getting C, or the closest thing to it the system already contains.
      +
      +
      +
      + + +
      +
      IV · The idea · what we wish we could do
      +

      The ideal experiment

      +
      What randomizing exposure would buy, and why we cannot do it.
      +
      +
      +
      randomization cuts the arrow that causes the trouble
      +
      +
      Left: today's world, where intent drives both exposure and sales. Right: the experiment we wish we could run, where a randomizer assigns exposure and intent's arrow into it is removed, not merely blocked: exposure no longer listens to intent at all. With that arrow gone, the naive comparison becomes the right comparison.
      +
      \[ X \text{ randomized} \;\;\Rightarrow\;\; \operatorname{Cov}(X,\, v) = 0 \;\;\Rightarrow\;\; \widehat{\Delta}_{\text{naive}} \longrightarrow \beta \]
      +
      +
      +
        +
      • The wish: option C from the poll: show the ad to a random half ourselves.
        • +
      • The wall: the platform's auction decides exposure in milliseconds, using exactly the intent signals that cause the problem. It will not hand us the controls.
        • +
      +
      The idea that saves us
      + If we cannot inject randomness into exposure, find the randomness the system already contains and use only that part.
      +
        +
      • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
        • +
      +
      The plan for Part 2
      + 1. Give that random lever a name: an instrument.
      + 2. State the conditions it must satisfy.
      + 3. Check which of them the data can verify.
      +
      +
      +
      + +
      +
      IV · The idea · the randomness we ran
      +

      The lottery

      +
      The randomness our system already contains, because we put it there.
      +
      +
      +
        +
      • What we ran: before the campaign, a serving-priority lottery. Our own random number generator picked half the users and gave them a small priority boost in the platform's ad auction.
        • +
      • What it does: the lottery shows nobody an ad. It only raises the odds of seeing one.
        • +
      • A new column joins the data:
        • +
      + + + + + +
      quantitymeaningtypewho assigns itobserved?
      \(Z_i\) · the lottery1 if the lottery boosted \(i\)'s priority in the ad auctionbinarya random number generator we controlobserved
      +
      \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \gamma\,Z + \lambda\,U\right) \] + the simulated world ran the lottery too: the exposure equation gains \(\gamma Z\), the lottery's push on the odds of seeing an ad
      +
        +
      • Why it matters: \(Z\) moves exposure, and nothing else about the customer. It is the one source of variation in \(X\) that the targeting cannot touch.
        • +
      +
      +
      +
      the fork, and the way around it
      +
      +
      Top: the confounded fork. Bottom: the lottery \(Z\) pushes on \(X\) from outside. No arrow from \(U\) into \(Z\) (it is a random draw), and no arrow from \(Z\) straight to \(Y\) (a queue bump shows the user nothing).
      +
      +
      +
      + +
      +
      IV · The idea · the central definition
      +

      The instrument

      +
      What we just built has a name.
      +
      +
      Definition · instrument
      + An instrument \(Z\) moves the treatment, is as good as random with respect to the hidden confounder, and affects the outcome only through the treatment. Four conditions make that precise, and they grade our lottery as follows.
      + + + + + + + + + + + + + +
      conditionthe claimcan it be checked?what it buys
      1 · Relevancethe lottery actually moves exposure \(X\)testable measured by the first stagea push that really happened
      2 · Exogeneitythe lottery is blind to intent \(U\)by design if the draw is genuinewith 1: a clean experiment on the lottery
      3 · Exclusionthe lottery touches sales only through exposureuntestable argued, never checkedturns the lottery's effect into the ad's
      4 · Monotonicitythe lottery never blocks an exposure that would have happened without ituntestable argued from the mechanismsays whose effect we measured
      +
      An instrument is the opposite of a control
      + You do not hold \(Z\) fixed. You use the variation it created.
      +

      The genius is never the estimator, it is spotting the randomness your system already contains: a rollout order, a capacity limit, an upstream A/B test, a rounding rule.

      +
      +
      + +
      +
      IV · The idea · condition 1, measured
      +

      Condition 1 · relevance

      +
      The first stage: did the lottery actually move exposure?
      +
      +
      +

      Relevance is a claim about the data, so the data can answer it.

      +
      Definition · first stage
      + The first stage is the effect of the instrument on the treatment: how much exposure the lottery itself created. It is the coefficient \(\pi\) in the regression below.
      +
      +
      \[ X \;=\; b_0 + \pi\,Z + u \] + \(\pi\): the first stage  ·  \(b_0\): exposure rate among lottery losers  ·  \(u\): every other driver of exposure, intent included
      +
      +
      +
      \[ \pi \;=\; \Pr(X{=}1 \mid Z{=}1) \;-\; \Pr(X{=}1 \mid Z{=}0) \;=\; 0.2106 \] + the measured first stage: what the lottery did to exposure
      +
        +
      • Interpretation of \(\pi\): winning the lottery raises a user's chance of seeing an ad by +21 percentage points, from 56% to 77%.
        • +
      • It is causal, full stop: \(Z\) is random, so nothing hidden can explain the step.
        • +
      +
      +
      +
      exposure rate, by the lottery
      +
      +
      The step between the bars is the first stage \(\pi\): the slice of exposure that the lottery, not targeting, created.
      +
      The gate: is the push strong enough?
      + Like every estimate in this lecture, \(\pi\) comes with a significance test: the first-stage \(F\). Trust the coming division only when \(F > 10\), well past bare significance, because a \(\pi\) that is merely nonzero still leaves the division unstable. Here \(F = 156\): the push is unmistakably real. We quote such tests, not derive them.
      +
      +
      +
      + +
      +
      IV · The idea · conditions 2 and 3
      +

      Conditions 2 and 3 · exogeneity and exclusion

      +
      The lottery must be blind to intent, and silent about sales.
      +
      +
      +
      Blind · exogeneity
      + The lottery must not favour high-intent users. Ours is a genuine randomizer, so this holds by design.
      +
      Silent · exclusion
      + The lottery must not touch sales through any channel except the ad itself. A side-channel (faster pages for boosted users, a shown price) would poison the method:
      +
      \[ \hat\beta_{\text{IV}} \;=\; \beta + \frac{s}{\pi} \] + \(\hat\beta_{\text{IV}}\): the estimate the lottery-based method reports  ·  \(\beta\): the true effect of one exposure on sales, the number we are hunting  ·  \(s\): the side-channel's own lift on sales  ·  \(\pi\): the first stage
      +
      No test will catch it
      + Exclusion is a claim about a path that should not exist, so no output can verify it. You defend it by design.
      +
      +
      +
      ▶ LIVE break the silence on purpose
      +
      +
      + + +
      +
      What the method reports, split into the true effect (blue) and the contamination \(s/\pi\) (red). Slide \(s\) up: a small leak becomes a large error.
      +
      +
        +
      • Why a tiny leak is a big error: the side-channel \(s\) gets divided by the small first stage \(\pi\), so even €1 of leak moves the estimate by €1 / 0.2106.
        • +
      • Our defence: a queue bump is content-free, sells nothing, and the user never learns it happened.
        • +
      • The counterexample: an email with a discount code would move spending on its own, and everything after this slide would collapse.
        • +
      +
      +
      +
      + + +
      +
      IV · The estimator · the second ingredient
      +

      The reduced form

      +
      The same comparison as the first stage, now for sales.
      +
      +
      +
      Definition · reduced form
      + The reduced form is the effect of the instrument on the outcome: the sales gap between the lottery groups. It is the \(\delta\) measured below.
      +
      \[ \delta \;=\; \mathbb{E}[Y \mid Z{=}1] \;-\; \mathbb{E}[Y \mid Z{=}0] \;=\; \text{€}3.48 \] + \(\delta\): the reduced form  ·  \(\beta\): the true effect of one exposure on sales, still the number we are hunting
      +
        +
      • Interpretation of \(\delta\): winning the lottery adds €3.48 of sales for the average user. Causal for the same reason \(\pi\) was: a random draw cannot favour ready buyers.
        • +
      • The intuition that carries the day: the lottery can only reach sales through the ad. One lottery win buys \(\pi\) extra exposures, and each exposure is worth \(\beta\) euros. So the lottery's effect on sales must be \(\beta \times \pi\):
        • +
      +
      \[ \delta \;=\; \beta \times \pi \]
      +
      Both numbers are correct. Neither answers the question.
      + \(\pi\) is exposures per lottery win and \(\delta\) is euros per lottery win, but the client asked for euros per exposure. Since \(\delta = \beta\pi\), one step is left.
      +
      +
      +
      average sales, by the lottery
      +
      +
      The step is small but clean: a random draw cannot favour ready buyers, so selection cannot explain it.
      +

      For the record, the algebra behind the intuition: substitute \(X = b_0 + \pi Z + u\) into the sales equation and the coefficient multiplying \(Z\) is exactly \(\beta\pi\).

      +
      +
      +
      + +
      +
      IV · The estimator · the whole method in one division
      +

      The IV estimate

      +
      Euros per lottery win, divided by exposures per lottery win.
      +
      +
      +
      the whole method, as arithmetic on two measured numbers
      +
      +
      One lottery win buys 0.2106 extra exposures and €3.48 of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
      +
      +
      +
      +
      \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ 3.48 }{ 0.2106 } \;=\; \text{€}16.5 \]
      +
        +
      • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
        • +
      • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
        • +
      +
      +
      +
      What just happened
      + We priced the ad using only the random slice of exposure: the dashboard said €23.7, the lottery says €16.5, against a planted truth of €15.
      +

      The method never needed a model of intent, controls, or machine learning: two averages and a division.

      +
      +
      +
      ▸ Deep dive · the confidence interval around €16.5
      +
      +
      +

      \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

      +
      \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; 16.5 \;\pm\; 1.645 \times 2.31 \;=\; [\,12.7,\; 20.4\,] \] + the 90% confidence interval: the range of effect sizes the data support
      +
        +
      • Interpretation of the interval: any effect between €12.7 and €20.4 is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
        • +
      • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
        • +
      +
      +
      +
      two intervals, graded against the planted truth
      +
      +
      The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
      +
      +
      +
      +
      +
      + +
      +
      IV · The estimator · hands on
      +

      Why the division is forced

      +
      The division is not a modelling choice. It is the only effect size the two measurements allow.
      +
      +
      +
      ▶ LIVE every candidate effect makes a prediction. One matches.
      +
      +
      + + +
      +
      The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €3.48. Move your guess to the crossing and you have priced the ad.
      +
      +
      +

      Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

      +
        +
      • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's 0.2106 extra exposures per win should create \(\hat\beta \times 0.2106\) euros per win.
        • +
      • The fact: the lottery actually created €3.48 per win.
        • +
      • The verdict: every candidate except €16.5 contradicts a number we measured. The division is the only survivor, not a choice.
        • +
      +
      \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
      +
      Not a black box
      + Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
      +
      +
      +
      + + +
      +
      IV · When it breaks · the dangerous failure
      +

      Weak instruments

      +
      A weak instrument is worse than no instrument.
      +
      +
      +
      ▶ LIVE what the estimator does as the first stage dies
      +
      +
      + + +
      +
      Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
      +
      +
      +

      The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

      +
        +
      • The honest failure: the interval balloons, and the data admit they know little.
        • +
      • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
        • +
      • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
        • +
      +
      Report the first-stage \(F\), always
      + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
      +
      +
      +
      ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
      +
      +
      +
      ▶ LIVE test each candidate \(\beta_0\), keep the survivors
      +
      +
      + + +
      +
      The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [12.6, 20.2]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
      +
      +
      +
        +
      • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
        • +
      • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
        • +
      +
      \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
      +
        +
      • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
        • +
      • Here, a good sign: the lottery is strong, so AR [12.6, 20.2] nearly matches the usual [12.7, 20.4]. When \(F\) is small the two part company, and AR is the one still telling the truth.
        • +
      +
      +
      +
      +
      +
      + +
      +
      IV · When it breaks · whose effect it is
      +

      Compliers and the LATE

      +
      Whom does the €16.5 describe? Only the users the lottery could move.
      +
      +
      ▶ LIVE the user base, split by how they respond to the lottery
      +
      +
      + + +
      +
      Push \(\gamma\): the complier slice grows, because the complier share is the first stage. Shares are drawn live from one simulated batch, so they can differ from the printed figures by a rounding step.
      +
      +
      +
        +
      • Always-takers (56.5%): the auction shows them the ad with or without the lottery. It changes nothing for them, so the data say nothing about them.
        • +
      • Never-takers (22.5%): never see the ad either way. Same silence.
        • +
      • Compliers (21.0%): see the ad only because the lottery favoured them. Every euro of the lottery's lift \(\delta\) came from them.
        • +
      • The caveat, monotonicity: we assume no defiers, users who would see the ad only when the lottery does not favour them. A nudge that never repels.
        • +
      +
      +
      +
      Definition · LATE (local average treatment effect)
      + The LATE is the average effect of the ad on the compliers alone, and it is what the division estimates: a local answer, not a statement about every user.
      +
      \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;\;\text{ estimates }\;\; \mathbb{E}[\,Y(1)-Y(0)\mid \text{complier}\,] \] + the average of each complier's personal effect, the same \(Y(1)-Y(0)\) contrast the naive slide could not touch
      +
      +
      +
      Why a manager should love this fine print
      + Compliers are the same kind of marginal user a higher bid would newly reach: the closest thing in the data to the customer a bid change buys. That makes €16.5 a price for the marginal customer, measured on the margin rather than on the average.
      +
      +
      + +
      +
      IV · When it breaks · what must hold, on one page
      +

      The checklist

      +
      The four assumptions, and which ones the data can check.
      +
      + + + + + + + + +
      AssumptionWhat it saysIn the caseStatus
      Relevance\(Z\) moves \(X\)\(F = 156\), far above 10TESTABLE, passes
      Exogeneity\(Z \perp U\)the lottery is a genuine random drawBY DESIGN
      Exclusion\(Z \to Y\) only via \(X\)a queue bump shows the user nothingUNTESTABLE
      Monotonicityno defiersa nudge never repelsUNTESTABLE, plausible
      +
      +
      +
      The honest scorecard, for any IV study you are shown
      + One measured number (the first-stage \(F\)), one design guarantee (the randomization), two arguments (exclusion, monotonicity). Ask for all four before you accept the estimate.
      +
        +
      • What we can now defend: an exposure causes about €16.5 of sales for the users a bid can actually move.
        • +
      +
      +
      The sentence that loses money
      + "Exposed users are worth €23.7 each, so raise the bid": every word true, conclusion wrong. It books the platform's targeting as advertising.
      +
      +
      +
      + + + +
      +
      IV · The decision · euros at last
      +

      The price map

      +
      The estimate becomes a decision only when it meets the price.
      +
      +
      +
      ▶ LIVE the verdict as the price moves
      +
      +
      + + +
      +
      Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
      +
      +
      +

      One estimate gives not one answer but a map from any price to a verdict:

      + + + + + + + +
      Price zoneVerdictWhy
      below €12.7GOeven the most pessimistic supported effect pays
      €12.7 to €20.4TESTthe data straddle the price: negotiate, or measure more
      above €20.4NO-GOno supported effect pays
      +
        +
      • Today's rate, €10, sits in the GO zone, below the whole interval.
        • +
      • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €16.5 − €10 = €6.5. Even read at the interval floor, €12.7 against €10, the exposure still pays.
        • +
      +
      Why boards like this framing
      + "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
      +
      +
      +
      + +
      +
      IV · The decision
      +

      Poll · the negotiation

      +
      +
      +
      ✋ Poll
      +
      The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €16.5 per exposure, 90% interval [12.7, 20.4]. What is the highest rate at which you would still sign "buy" without further study?
      +
      + + + + +
      + +
      C. Below €12.7, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
      +
      +
      +
      + +
      +
      IV · The decision · banked
      +

      The verdict and the recommendation

      +
      The complete answer, assembled from everything measured so far.
      +
      +
      + + + + + + + + + +
      QuantityValueSource
      effect of one exposure€16.5the division δ/π
      90% interval[12.7, 20.4]classical, and AR agrees
      first-stage F156the lottery is strong
      price€10the platform's rate card
      net per exposure€6.5β − c, at the point estimate
      +
      The verdict
      + BUY  Keep buying at €10: the entire defensible range clears the price.
      +
      +
      +
      The recommendation, in three lines
      + 1. Keep buying at the €10 rate: even the interval's most pessimistic effect pays.
      + 2. Cap the bid at the interval's lower end, €12.7: up to there, every effect the data support still clears the price.
      + 3. Measure again only if the rate card climbs toward €12.7: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
      +
        +
      • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
        • +
      • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
        • +
      +
      +
      +
      + + +
      +
      Closing · Provenance
      +

      The tools were the product too

      +
      +
        +
      • CausalPy: synthetic control, interrupted time series, difference in differences, and regression discontinuity in one open-source package. The IV estimator that closes this session joined later.
        • +
      • Its launch example: individual exposure to a TV campaign cannot be randomised, yet its causal impact remains a core business need: the sentence this whole session opened with.
        • +
      • pymc-marketing: the MMM library behind Case 2's calibration story; one client's budget allocation approach to PyMC-Marketing came back as a pull request (Bolt).
        • +
      • Webinars and content: the consultancy's own webinar agenda is this session's syllabus, Instrumental Variables included.
        • +
      +
      + CausalPy + PyMC-Marketing +
      +
      +
      + +
      +
      Closing
      +

      The pattern in every engagement

      +
      +
        +
      • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
        • +
      • An experiment anchors every observational model: calibration is the product, not a luxury.
        • +
      • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
        • +
      + + + + + + +
      Agent, on adversarial MMM dataResult
      Vanilla coding agentFit a model, recommended budget reallocations. Confidently wrong.
      PyMC Labs' Decision LabExplored 11 approaches, 0 converged. Returned: "No valid model found. Run a geo-holdout experiment."
      +
      Even the machines know the punchline
      + The honest system's best answer was Part 2's closing advice: run the experiment.
      +
      +
      + +
      +
      Closing
      +

      One breath

      +
      +
      The pattern to take home
      + The toolkit a Bayesian consultancy sells: counterfactuals, calibrated by experiments, priced as probabilities.
      +
        +
      • Read the cases: pymc-labs.com/blog-posts: every number in this deck is pinned to a post, listed on the next slide.
        • +
      • Say hello: both authors consult for PyMC Labs; the notebooks behind this session are the course repository.
        • +
      +
      +
      + + +
      +
      Backup
      + Backup · Sources +

      Every number, pinned

      +
      Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
      +
      + + + + + + + + + + + + + + + + + +
      SourceFacts pinned
      ailab.criteo.com · criteo-uplift-prediction-datasetcriteo_rows
      pymc-labs.com · 2022-11-11-HelloFreshhf_panel_calibration
      pymc-labs.com · 2023-06-20-juan-marketing-analyticswebinar_agenda
      pymc-labs.com · bayes-is-slow-speeding-up-hellofreshs-bayesian-ab-tests-by-60xhf_batch, hf_test_types, hf_thousands
      pymc-labs.com · bayesian-media-mix-modeling-for-marketing-optimizationhf_priors_experiments
      pymc-labs.com · causal-sales-analytics-are-my-sales-incremental-or-cannibalisticcolgate_ci, colgate_ci_level, colgate_truth, colgate_year, fail_range, fail_truth, market
      pymc-labs.com · causal-sales-analytics-discrete-choice-modelingcolgate_counterfactual_quote
      pymc-labs.com · causalpy-a-new-package-for-bayesian-causal-inference-for-quasi-experimentscausalpy_methods, causalpy_tv
      pymc-labs.com · funnel-aware-mmmcpl, cpl_window, gdpr_sentence, herp_attribution_quote, nurn_2026, trust_quote
      pymc-labs.com · marketing-mix-modeling-a-complete-guidebolt_pr
      pymc-labs.com · mmm_roas_liftlift_tests_n, roas_gap_words, roas_wrong_ranking, roas_x1, roas_x2
      pymc-labs.com · open-sourcing-decision-lab-scaling-ai-judgment-data-sciencedl_explored, dl_vanilla, dl_verdict
      pymc-labs.com · reducing-customer-acquisition-costs-how-we-helped-optimizing-hellofreshs-marketing-budgethf_var
      +
      +
      + +
      + + + +
      + Causal Marketing · SDA Bocconi · Causal Inference in the Wild + + + + 1 / 1 +
      +
      +

      Contents

        + + + + diff --git a/causal-marketing-pymc/apps/unified_slides_src.html b/causal-marketing-pymc/apps/unified_slides_src.html new file mode 100644 index 0000000..c6c3896 --- /dev/null +++ b/causal-marketing-pymc/apps/unified_slides_src.html @@ -0,0 +1,4345 @@ + + + + + + +Causal Inference in the Wild: from real engagements to two decisions in euros + + + + + +
        + + + +
        +
        Causal Inference & XAI for Business · SDA Bocconi
        +

        Causal Inference in the Wild

        +
        Real PyMC Labs engagements, then we build the machinery ourselves twice: a €4M rollout decision, and the price of one ad exposure.
        +
        A guest lecture for Prof. Michele Russo's course
        +
        +
        +
        Francesco Muia
        +
        PhD in Theoretical Physics, EMBA.
        Consultant for PyMC Labs and Brown University.
        +
        francesco.muia@pymc-labs.com
        +
        francesco.muia@ai-and-analytics-solutions.com
        +
        +
        +
        Alexander Fengler
        +
        PhD in Computational Cognitive Science.
        Postdoc at Brown University, consultant for PyMC Labs.
        +
        alexander.fengler@pymc-labs.com
        +
        +
        +
        + PyMC + PyMC Labs +
        +
        + + +
        +
        Opening · Who is talking
        +

        PyMC Labs: what we do

        +
        A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
        +
        +
          +
        • What we sell: senior modeling expertise, not software licenses.
          • +
        • Open source is the top of the funnel: the libraries build reach; the consulting monetizes the expertise behind them.
          • +
        • How we work: small senior teams, starting from the client's decision; the client keeps the model, not a black box.
          • +
        + + + + + + + +
        EngagementWhat it isTypical client
        Project consultingFixed-scope build of one custom model (marketing mix, forecasting, pricing)A specific, high-value decision problem
        Advisory / retainerOur modelers guiding an in-house teamFirms building their own capability
        Enablement & trainingWorkshops and upskilling on Bayesian methods and the toolingAnalytics teams standardizing on PyMC
        +
        The clients in this session
        + Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
        +
        ▸ Areas of interest, and where we work
        +
          +
        • Areas: marketing-mix modeling and media measurement, causal inference, demand forecasting and pricing, experimentation and A/B testing at scale, applied Bayesian modeling.
          • +
        • Sectors: consumer/CPG, retail and e-commerce, tech, and finance.
          • +
        +
        +
        +
        + +
        +
        Opening · A note on method
        +

        Bayesian vs frequentist, in one slide

        +
        Same data, two philosophies.
        +
        +
        +
        Frequentist
        + Reports how a procedure behaves over many hypothetical repetitions of the study.
        +
        Bayesian
        + Reports a full probability distribution over the unknown, given the data in hand.
        +
        +
        +
        + + + + + + + + +
        AspectFrequentistBayesian
        You geta point estimate and a confidence intervala full distribution over the unknown effect
        It answers"how would this procedure behave if I repeated the study many times?""given the data I have, how probable is each value?"
        Shines whensamples are large, the design is standard, speed mattersdata is small or messy, structure is layered, prior knowledge exists
        The catchthe 95% interval is not "95% probability the truth is inside"you must state a prior, and it costs computation
        +
        Where Bayes earns its keep
        + It answers the decision directly: not "is the lift real?" but a number you can act on, P(lift beats breakeven).
        +
        +
        +
        +
        SCHEMATICThe Bayesian answer is a shape, not a point
        +
        + + + + + -2 + 0 + 2 + 4 + 6 + + estimated lift (effect size) + + + no effect + + + + + posterior + P(lift > 0) = 0.92 + + + + + + classical: estimate + symmetric 95% CI + +
        +
        The posterior is skewed, so mode, mean, and the symmetric classical interval no longer coincide, and that interval even dips below zero, yet P(lift > 0) still reads straight off the shaded mass.
        +
        +
        +
        +
        Why marketing analytics is going Bayesian
        + Marketing data is short, noisy, and highly correlated, and every model ends in a spend decision, which is exactly where priors and full uncertainty pay off.
        +
        For today, this is a curiosity
        + The entire lecture runs on classical methods, and every number you will see is frequentist; we surface the Bayesian read only as a flavour at the edges, never as the load-bearing tool.
        +
        ▸ Where it is already the default (and where it is not)
        +
          +
        • Marketing mix modeling: the modern open-source MMM tools are Bayesian (Google's Meridian, PyMC-Marketing); with only two or three years of weekly data and correlated channels, plain regression is unstable, and priors on adstock, saturation, and ROI stabilize it.
          • +
        • Calibrating models to experiments: geo-lift and A/B results enter as priors on channel ROI, so the model and the experiments agree, the HelloFresh loop you will see today.
          • +
        • Decisions under uncertainty: you get a full distribution over each channel's ROAS, so budget optimization and statements like P(channel A beats B) fall out directly.
          • +
        • Many thin segments: hierarchical models borrow strength across regions, SKUs, or cohorts too small to estimate on their own.
          • +
        • Signal loss: cookie deprecation and privacy changes push measurement back toward aggregate MMM, which is Bayesian home turf.
          • +
        • Not everywhere: for large-sample, standard A/B tests and simple designs, classical methods are faster and perfectly adequate.
          • +
        +
        +
        ▸ Pros and cons, plainly
        + + + + + + +
        ProsCons
        Frequentistfast, familiar, few choices to defend, excellent for large samples and standard designsp-values and confidence intervals are routinely misread, and awkward once models get layered or data gets thin
        Bayesiandirect probability statements, priors fold in domain knowledge, hierarchical models borrow strength across groups, uncertainty flows straight into a decisionyou must choose priors (and defend them), sampling can be slow, and it demands more modeling care
        +
        +
        +
        + + +
        +
        Case 1 · Colgate-Palmolive
        +

        You are the consultant

        +
        The counterfactuals this session builds are what clients buy. First test: route a real call, on instinct.
        +
        +
        +
        ✋ Poll
        +
        Colgate-Palmolive calls: "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it stealing share from competitors, or from our own brands?" Which tool do you reach for first?
        +
        + + + + +
        + +
        C. A national launch leaves nothing to randomize and nothing to difference against: A, B and D all need a comparison that does not exist. What remains is this session's one move: fit the world before the launch, project it forward, read the gap. PyMC Labs sold exactly that; next slide.
        +
        +
        +
        + +
        +
        Case 1 · Colgate-Palmolive
        +

        Colgate-Palmolive: incremental, or cannibalistic?

        +
        Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The launch verdict is the split.
        +
        +
        +
        +
        The launch, and the world without it schematic
        +
        +
        Illustrative shape, not client data: fit the pre-period, project, price the gap.
        +
        +
        +
        +
        The brief, in their words
        + "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
        +
          +
        • The client: Colgate-Palmolive, {{labs.colgate_year}}; a market estimated at {{labs.market}}.
          • +
        • The method: a multivariate Bayesian interrupted time series: project the pre-launch world forward.
          • +
        • The grading: a planted {{labs.colgate_truth}} recovered as a {{labs.colgate_ci_level}} interval of {{labs.colgate_ci}}: recover a known truth first, then be believed.
          • +
        +
        +
        +
        + +
        +
        Case 1 · Colgate-Palmolive · open floor
        +

        What would break it?

        +
        +
        +
        🗣 Open floor · 2 minutes
        +
        You are Colgate's CMO. The incrementality estimate you just saw (the share of the new product's sales that are genuinely new, not cannibalized) decides the launch review. Name one real-world event that would make it wrong.
        + +
        +
        +
        +
        Learning 1 · what breaks it
        + A second launch inside the fit window redefines "normal growth", so the counterfactual absorbs part of the very effect it is meant to isolate.
        +
        Learning 2 · the lesson that runs the whole session
        + A counterfactual is only as good as a clean window: in Part 2 this disease returns with its formal name, spillover, and the defence is design, not statistics.
        +
        +
        +
        +
        Break it yourself: slide a second launch into the window schematic
        +
        +
        + + + +
        +
        Schematic, not client data: the red projection is refit live from whatever the pre-launch months show.
        +
        +
        +
        How to read the plot
        +
          +
        • Three lines: solid blue is observed sales, grey dashed is the true no-launch world, red dotted is the model's guess of that world, fit on the pre-launch months only.
          • +
        • The clean read: red sits exactly on grey and the readout says 100%: the model recovers the true lift.
          • +
        • Break it: tick the box; a second launch before month 0 pulls red off grey and the readout falls (under-credit), after month 0 it climbs past 100% (over-credit).
          • +
        • The real case: Colgate's model read {{labs.fail_range}} when the planted truth was {{labs.fail_truth}}.
          • +
        +
        +
        +
        +
        +
        + + +
        +
        Case 2 · HelloFresh · the tool, and its failure mode
        +

        Why calibrate? A model alone can rank channels backwards

        +
        A public tutorial that shows the real job clients hire us for: making a company's ad-budget model trustworthy.
        +
        +
        +
        The settinga company spreads its budget across many ad channels (TV, search, social) and needs to know which ones actually pay back.
        +
        The everyday toola model reads years of spend-and-sales history and scores each channel's return on ad spend, cheaply and always-on.
        +
        The catch, and our jobhistory is not an experiment; a company spends more exactly when demand is already high, so the model can credit the season instead of the channel and rank them backwards.
        +
        What we adda real experiment: nudge one channel's budget by a known amount, measure the sales it truly causes, and anchor the model to that number. This is calibration.
        +
        +
        +
        The model's channel ranking, before and after a real experiment baked from the tutorial
        +
        +
        +
        Left: what the history-only model reported. Right: the true answer the experiments recover.
        +
        +
        The inversion, on a known answer
        + True returns were {{labs.roas_x1}} versus {{labs.roas_x2}}, so channel 2 is {{labs.roas_gap_words}}; yet the uncalibrated model reported {{labs.roas_wrong_ranking}}, and {{labs.lift_tests_n}} per channel fixed it.
        +
        +
        + +
        +
        Case 2 · HelloFresh
        +

        HelloFresh runs the loop, at industrial scale

        +
        The same idea, not on a tutorial now but in production at a company you know.
        +
        +
        +
        +
        Not a toyHelloFresh, the meal-kit company, runs this exact model-plus-experiment loop in production.
        +
        The loopthe model steers the budget always-on, while a steady stream of {{labs.hf_priors_experiments}} keeps re-anchoring it, cutting prediction variance by {{labs.hf_var}}.
        +
        Calibration, put plainlya Bayesian budget model should be "{{labs.hf_panel_calibration}}".
        +
        The scale{{labs.hf_thousands}} ({{labs.hf_test_types}} designs), each model fit batched down from {{labs.hf_batch}}; one experiment alone logged {{labs.criteo_rows}} users.
        +
        +
        The takeaway
        + The experiment is not a one-off audit; it is a supply line that keeps the budget model honest, campaign after campaign.
        +
        +
        +
        +
        The loop, in one picture
        +
        +
        The model runs always-on; the experiments keep correcting it; together they steer the budget.
        +
        +
        +
        +
        + + +
        +
        Case 3 · Nürnberger Versicherung
        +

        Price the engagement

        +
        A German insurer, last-touch attribution, and a funnel-aware causal MMM, in production.
        +
        +
        +
        ✋ Poll
        +
        Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over {{labs.cpl_window}} of model-guided spend, cost per lead (CPL) moved by how much?
        +
        + + + + +
        + +
        C. "This year we were able to drive the CPL down by {{labs.cpl}}, which is very, very good" (Philip Herp, Nürnberger Versicherung). Under GDPR, {{labs.gdpr_sentence}}, so last-touch under-credited the upper funnel and budget followed {{labs.herp_attribution_quote}}. The client is scaling into {{labs.nurn_2026}}.
        +
        +
        The client's bar for belief
        + "{{labs.trust_quote}}"
        +
        +
        + + +
        +
        +
        +
        Part 2 · The deep dive
        +

        Synthetic Control

        +
        One treated market, no experiment, €4M on the line: did the campaign work?
        +
        +
        + +
        Where the lecture lands: a synthetic twin rebuilds the metro's counterfactual, and the gap beyond break-even is the campaign's profit.
        +
        +
        +
        + + +
        +
        Act I · The question
        +

        The boardroom question

        +
        A campaign ran in one region. Sales rose. Marketing wants €4M to go national. The Colgate question again: what would have happened anyway?
        +
        +
        +
        +

        What you have

        +
          +
        • 30 markets, 60 weeks of weekly sales.
          • +
        • One metro, week 40: a €75k campaign for 20 weeks; the other 29 did nothing.
          • +
        • No randomisation: the metro was chosen, not drawn.
          • +
        • 35% gross margin on incremental sales.
          • +
        +
        +
        +

        What the data forces you to answer, in order

        +
        +
        ① How much did the campaign actually cause? attribution
        +
        ② Is that more than luck? significance
        +
        ③ Is the profit worth the price? the decision
        +
        +
        +
        +
        The method this data calls for
        +
          +
        • One treated unit, no A/B test possible: synthetic control's home ground.
          • +
        • It is the engine under Meta's GeoLift and Google's geo methodology.
          • +
        +
        The €4M hinge: the transportability discount
        +
          +
        • δ = the share of the pilot's per-euro lift that survives national rollout.
          • +
        • The €4M call reduces to one question: is δ large enough to still clear the cost?
          • +
        +
        ▸ What "unbiased" means here, precisely, and where it would break
        +

        The estimator subtracts a reconstructed counterfactual from the treated metro's post-launch sales, and bias is its expected gap from the true lift:

        +
        + \[ \hat\tau \;=\; \bar Y_{1,\text{post}} - \hat Y_{1,\text{post}}(0), \qquad \text{bias} \;\equiv\; \mathbb{E}[\hat\tau] - \tau \] +
        +
          +
        • The counterfactual is the synthetic twin, built to reproduce the metro's pre-launch record (its level and its factor loadings) week by week.
          • +
        • Unbiasedness rests on one assumption: that pre-launch match would have continued through the post window had the campaign never run.
          • +
        • This needs no story about why the metro was chosen. Whatever pre-launch trait drove the pick, the twin already carries it, because it was fitted to match it: selection on anything visible before launch cannot tilt the estimate.
          • +
        • Bias enters only through what the pre-launch record cannot see: if the launch were timed on private knowledge of a coming local boom, the twin could not anticipate it and the gap would credit the boom to the campaign. The later placebo-in-time test hunts exactly that.
          • +
        +
        +
        +
        + + +
        +
        Act I · The question
        +

        The data you actually get

        +
        The simulated world.
        +
        +
        +
        +
          +
        • \(Y_{jt}\): weekly sales (€000): 30 markets, 60 weeks (\(t=0,\dots,59\)); treated metro = market 1.
          • +
        • The design: a €75k campaign from week \(T_0=40\), 20 weeks, 35% margin; the other 29 ran nothing.
          • +
        • The choice: marketing picked the metro and the week; not a coin.
          • +
        +
        +
        +
        The one number you do not have
        + What the metro would have sold with no campaign: nowhere in the file. The counterfactual.
        +
        +
        +
        +
        +
        +
        ▶ LIVE 30 markets, one treated at week 40
        +
        +
        + + + +
        +
        +
        +
        +
          +
        • Before/after: post minus pre-launch mean, /week.
          • +
        • Subtract the crowd: remove the shared wave and the step is ≈ /week.
          • +
        • The noise floor: pre-launch sd /week around a /week base.
          • +
        +
        Which number decides?
        +
          +
        • Is this all due to the campaign?
          • +
        • What's the right number to take a decision?
          • +
        +
        +
        +
        ▸ Where each of these three readings comes back later
        +
          +
        • Before/after is the estimator \(\hat\tau^{\text{BA}}\) in the zoo you will grade: it assumes \(Y_{1t}(0)\) simply holds at the pre-launch mean.
          • +
        • Subtracting the crowd strips out the shared wave \(\gamma_j^{\top} f_t\) of the next slide's model, which is exactly what treated-vs-control and difference-in-differences try to do.
          • +
        • The noise floor is that model's own \(\varepsilon_{jt}\) term, the wobble every estimator must out-shout with a single treated unit.
          • +
        +
        +
        +
        + + +
        +
        Act I · The question
        +

        Commit to a number

        +
        +
        +
        ✋ Poll
        +
        Before/after on the treated metro: sales averaged ·/week before launch and ·/week after, a bump of ·/week (about +6%). What is the campaign's true weekly effect?
        +
        + + + +
        + +
        C. The true effect is ·/week: a 12% lift on a ·/week baseline. Before/after books barely half of it: the macro wave dipped after launch. The tide can inflate a campaign's credit or hide it; a naive comparison is charged with everything else that changed.
        +
        +
        +
        + + +
        +
        Act I · The question
        +

        Causal inference is a missing-data problem

        +
        +
        +

        The treated metro has two week-\(t\) sales, only one of which ever happens: \(Y_{1t}(1)\) with the campaign, \(Y_{1t}(0)\) without. The effect is the gap:

        +
        + \[ \tau_t \;=\; Y_{1t}(1)-Y_{1t}(0), \qquad \tau \;=\; \sum_{t\ge T_0}\tau_t \] +
        +
          +
        • \(\tau_t\) vs \(\tau\): the weekly effect, and the 20-week total the business actually buys.
          • +
        • Consistency: we record one world per week, never both.
          • +
        • The missing cell: \(Y_{1t}(0)\) for \(t\ge T_0\): every method fills exactly that cell.
          • +
        +
        Define the estimand first
        + Per-week and 20-week total are different estimands; the €4M turns on the total.
        +
        +
        + + + + + + + +
        pre (t < 40)post (t ≥ 40)
        treated · \(Y_{1t}(1)\)·observed ✓
        treated · \(Y_{1t}(0)\)observed ✓missing: the counterfactual
        donors · \(Y_{jt}(0)\)observed ✓observed ✓
        +
        Three cells are data; the donors' post-period is what a synthetic \(Y_{1t}(0)\) is rebuilt from.
        +
        ▸ Reading the notation (observed vs potential)
        +
          +
        • \(Y_{jt}\) from the data slide is what you observe; \(Y_{1t}(1),\,Y_{1t}(0)\) are the treated metro's two potential outcomes.
          • +
        • Subscript says which market, argument says which world (1 = campaign, 0 = none): distinct axes that happen to share the digit 1.
          • +
        • \(w_j\): the donor weights the synthetic control will choose so \(\sum_{j\ge 2} w_j Y_{jt}\) rebuilds the missing \(Y_{1t}(0)\).
          • +
        +

        "What would this metro have sold anyway?" is an estimation target, not a rhetorical question.

        +
        +
        +
        + + + + +
        +
        Act I · The question, and the world behind the data
        +

        The model that generated the data

        +
        We simulate the data: here the counterfactual is known, so every estimator can be graded.
        +
        +

        Each market: level \(\alpha_j\) + exposures \(\gamma_j^{\top} f_t\) to three shared factors + noise (all €000).

        +
        + \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} + \;=\; \alpha_j \;+\; \gamma_{j1}\underbrace{\,0.4\,t\,}_{\text{trend}} \;+\; \gamma_{j2}\underbrace{\bigl[\,8\sin(2\pi t/26)+4\sin(2\pi t/13)\,\bigr]}_{\text{seasonality}} \;+\; \gamma_{j3}\underbrace{\textstyle\sum_{s=1}^{t}\eta_s}_{\text{macro walk }m_t} \;+\; \varepsilon_{jt} \] +
        +
          +
        • The macro walk \(m_t\): a running sum of shocks that wanders like an economy.
          • +
        • The exposures \(\gamma_j\): each market responds its own amount; the spread \(s\) kills parallel trends.
          • +
        +
        ▸ All five ingredients, one by one
        +
          +
        • Trend \(0.4\,t\): steady growth of 0.4 (€000) per week. Deterministic.
          • +
        • Seasonality \(8\sin(2\pi t/26)+4\sin(2\pi t/13)\): a 26-week half-year cycle of amplitude 8 and a 13-week quarterly cycle of amplitude 4. Identical for every market.
          • +
        • The macro walk \(m_t\): one fresh shock \(\eta_s\sim\mathcal{N}(0,\sigma_\eta^2)\) per week, none ever forgotten. This is "the economy".
          • +
        • Exposures \(\gamma_j\): each market multiplies each ingredient by its own loading.
          • +
        • Noise \(\varepsilon_{jt}\sim\mathcal{N}(0,\sigma_\varepsilon^2)\): the market's own weekly wobble, \(\sigma_\varepsilon\approx 3\).
          • +
        +
        +
        Which of these is ever observed?
        + Only \(Y_{jt}\) is observed; levels, loadings, factors and noise are real but recorded nowhere. Every estimator copes with that.
        +
        The planted truth, in business units
        + A 12% lift switches on at week 40: ·/week, · over 20 weeks. Every method is graded against it.
        +
        ▸ The planted lift, written out
        +
        + \[ \tau_t \;=\; 0.12\,\bigl(\alpha_1+\gamma_1^{\top}f_t\bigr)\,\mathbf{1}_{\{t\ge T_0\}} \] + 12% of the treated metro's own baseline \(\alpha_1+\gamma_1^{\top}f_t\), switched on from week \(T_0=40\). +
        +
        +
        +
        + + +
        +
        Act I · The question, and the world behind the data
        +

        Simulate the world yourself

        +
        Take a look at the counterfactual.
        +
        +
        +
        ▶ LIVE The equations of the previous slide. Grey: donor markets. White: the treated metro.
        +
        +
        + + + + + + + + +
        +
        Dashed blue: \(Y_{1t}(0)\) itself, the line that in real data does not exist.
        +
        +
        +
        +
          +
        • True lift: the planted 12%, averaged over the 20 post weeks.
          • +
        • Before/after: the treated series' post-launch mean minus its pre-launch mean.
          • +
        • Its error: the tide, charged entirely to the campaign.
          • +
        +
        +
        +
        What the sliders teach
        +
          +
        • Spread \(s\to 0\): parallel trends, DiD lives.
          • +
        • Macro \(\sigma_\eta\) up: the tide dominates, before/after dies.
          • +
        • Noise \(\sigma_\varepsilon\) up: every estimate gets shakier.
          • +
        +
        +
        +
        +
        + +
        +
        Act II · The counterfactual problem
        +

        Abadie's idea: if no twin exists, build one

        +
        Idea: manufacture a treated market's twin using a blend of donor markets.
        +
        +

        Assume untreated sales follow the factor structure: level, loadings on shared factors, noise (equation in the fold).

        +
          +
        • \(f_t\), the latent factors: the shared shocks every market feels, unequally.
          • +
        • \(\gamma_j\), the loadings: how strongly market \(j\) responds; never observed in real data.
          • +
        • The method never names them: it works without knowing what the factors are.
          • +
        +

        Blend the donors so their level and loadings equal the treated metro's:

        +
        + \[ \sum_j w_j\,(\alpha_j,\;\gamma_j) = (\alpha_1,\;\gamma_1) \;\Longrightarrow\; \sum_j w_j\,Y_{jt} \approx Y_{1t}(0)\ \ \text{for all } t \] +
        +
          +
        • Same loadings, same response to every factor movement, including ones that have not happened yet: that forward guarantee is the whole trick.
          • +
        +
        Why it survives storms
        + DiD matches a level; synthetic control matches the response to shocks, so it survives what breaks DiD.
        +
        ▸ The factor equation, pedigree, and the papers' notation
        +
        + \[ Y_{jt}(0) \;=\; \alpha_j \;+\; \gamma_j^{\top} f_t \;+\; \varepsilon_{jt} \] + the assumed structure the blend must match, level and loadings, week by week +
        + Abadie & Gardeazabal (2003); Abadie, Diamond & Hainmueller (2010): "arguably the most important innovation in the policy-evaluation literature in the last 15 years" (Athey & Imbens 2017). The literature writes the factors \(\lambda_t\) and the loadings \(\mu_j\), and adds a common shock \(\delta_t\) felt identically by every market; any cross-market comparison cancels \(\delta_t\), so nothing changes here. +
        +
        +
        + +
        +
        Act II · The counterfactual problem
        +

        The estimator, precisely

        +
        A constrained least square optimization.
        +
        +

        Fit the weights on the pre-period only, constrained to the simplex:

        +
        + \[ \hat w \;=\; \arg\min_{w\in\Delta}\; \sum_{t \lt T_0}\Bigl(Y_{1t}-\sum_j w_j Y_{jt}\Bigr)^{2}, \qquad \Delta=\Bigl\{w : w_j\ge 0,\ \textstyle\sum_j w_j=1\Bigr\} \] +
        +
          +
        • Pre-launch only: the optimiser never sees the post-period, so it cannot cheat.
          • +
        • The simplex \(\Delta\): a readable recipe, never "−80% of Milan + 190% of Rome": Colgate's projection with the weights on the table.
          • +
        • No standard error: a point estimate only; inference comes later, from placebos.
          • +
        +
        ▸ Reading the effect off the gap
        +
        + \[ \hat\tau_t \;=\; Y_{1t}-\sum_j \hat w_j Y_{jt}, \qquad\qquad \hat\tau \;=\; \sum_{t\ge T_0}\hat\tau_t \] + weekly gap between the treated metro and its synthetic twin, summed over the 20 post-launch weeks +
        +
        +
        Why constrain at all?
        + The simplex buys three things ordinary regression cannot: interpretability, regularisation, and an off-switch.
        +
          +
        • Interpretability: "40% dma_08 + 32% dma_20 + 17% dma_03" is a sentence a planner can act on.
          • +
        • Regularisation: most weights land on exactly zero.
          • +
        • An off-switch: a market no blend can match fails loudly instead of extrapolating.
          • +
        +
        +
        + + +
        +
        Act II · The counterfactual problem
        +

        What the simplex buys, geometrically: stay inside the hull

        +
        A simplex blend can only land inside the donors' convex hull while OLS can leave it.
        +
        +
        +
        +
        +
        ▶ LIVE Drag the treated market in and out of the donors' hull
        +
        +
        + + +
        +
        +
        +
        +
          +
        • With \(w_j\ge 0\) and \(\sum_j w_j=1\), every point a synthetic market can reach lies inside the shaded convex hull.
          • +
        • Inside: a real blend reproduces the metro; the fit keeps working out of sample.
          • +
        • Outside: no real blend reaches it; only extrapolation (negative weights) closes the gap.
          • +
        +
        Why constrain the weights
        + A good fit is a real blend of observed markets, not an extrapolation that happens to line up.
        +
        When the treated market sits outside
        + Widen the donor pool; do not force a fit. The refusal is a feature. Ask any vendor: when does your method refuse? If never, walk away.
        +
        +
        +
        +
        + + + +
        +
        Act II · The counterfactual problem
        +

        What the constraint buys: drop it and see

        +
        What if we used a simple unconstrained OLS?
        +
        +
        +
        ▶ LIVE Two synthetics against the truth only a simulation can draw
        +
        +
        OLS hugs the pre-period tighter ( vs ) yet drifts further from the true \(Y_{1t}(0)\) after launch ( vs ): overfitting the simplex refuses.
        +
        +
        +
        +
        The weights OLS chose
        +
        +
        OLS puts negative weight on donors (down to ) and gross weight , where the simplex uses exactly 1.
        +
        +
        +
        + \[ n_{\mathrm{eff}} \;=\; \frac{1}{\sum_j \hat w_j^{2}} \] + Effective number of donors (inverse Herfindahl): one donor \(\Rightarrow n_{\mathrm{eff}}=1\), all 29 equally \(\Rightarrow 29\). Ours: 3.3. +
        +
        What n_eff = 3.3 tells you
        +
          +
        • The synthetic leans on about three donor markets, not a fuzzy mix of all 29.
          • +
        • Sparse weights: interpretable, and less room to overfit.
          • +
        • Near 1: hostage to one market. Near 29: mush. 3.3: concentrated, not fragile.
          • +
        +
        +
        +
        +
        + + +
        +
        Act II · The counterfactual problem, versus machine learning
        +

        "Why not gradient boosting / Prophet / an LSTM?"

        +
        Is this a forecasting exercise?
        +
        +
        +
        ▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
        +
        +
        Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
        +
        +
          +
        • In-sample it fits tighter: €k vs €k per week: flexibility always buys the past.
          • +
        • Yet the estimates are a wash: vs , both near the planted €284k. One dataset cannot rank them.
          • +
        • The sweep decides: across 24 fresh worlds, {{nb07.hull_oos_ratio}}× worse out of sample, with nothing to inspect.
          • +
        +
        +
        +
        Principle 1: a counterfactual, not a forecast
        + A forecaster asks what comes next. We ask what this market would have done in a world that never happened.
        +
          +
        • Principle 2: in-sample fit is the trap, not the goal.
          • +
        • Principle 3: you cannot cross-validate the counterfactual. The truth is never observed.
          • +
        • Principle 4: the simplex ships an auditable claim; a boosted tree is a black box.
          • +
        +
        The forecaster is better at prediction. The simplex is better at the causal job.
        +
        +
        +
        + +
        +
        Act II · The counterfactual problem
        +

        Compare the estimators

        +
        Every "obvious" estimator is a claim about \(Y_{1t}(0)\).
        +
        +
        +
        +
        Four "obvious" estimators, four hidden assumptions
        + Each is a different guess at the missing \(Y_{1t}(0)\), unbiased only in its own special world (equations in the fold below).
        +
          +
        • Before/after: unbiased only in a flat world.
          • +
        • Treated vs average control: needs matching level and loadings; the level gap survives.
          • +
        • Difference-in-differences: unbiased iff trends are parallel (\(\gamma_1=\bar\gamma\)).
          • +
        • Synthetic control: unbiased iff the blend matches the loadings (treated inside the hull).
          • +
        +
        +
        +
        ▶ LIVE Four estimates, one truth: the world's dials exposed
        +
        +
        + + + + +
        +
        +
        +
        ▸ The four estimators, written out with their bias conditions
        +
        + \[ \hat\tau^{\text{BA}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{1,\text{pre}} \] + Before/after: unbiased only in a flat world, equal shared-factor means pre and post (the macro-shock dial drives its error). +
        +
        + \[ \hat\tau^{\text{T-C}} \;=\; \bar Y_{1,\text{post}} - \bar Y_{\text{ctrl},\text{post}} \] + Treated vs average control: unbiased only if controls match the treated in level and loadings; the level gap survives even at spread \(s\to 0\). +
        +
        + \[ \hat\tau^{\text{DiD}} \;=\; \Delta_{\text{treated}} - \Delta_{\text{ctrl}}, \quad \Delta = \bar Y_{\text{post}} - \bar Y_{\text{pre}} \] + Difference-in-differences: removes the level gap; unbiased iff trends are parallel (equal loadings \(\gamma_1=\bar\gamma\)), which spread \(s\to 0\) delivers. +
        +
        + \[ \hat\tau^{\text{SC}} \;=\; \bar Y_{1,\text{post}} - \textstyle\sum_j w_j\, \bar Y_{j,\text{post}} \] + Synthetic control (the one you fitted): unbiased iff a fitted blend matches the treated's loadings (\(\sum_j w_j\gamma_j=\gamma_1\), the treated inside the hull). +
        +
        +
        ▸ How to read the dials (red = error against the truth)
        +
          +
        • Loading spread → 0: markets react identically, trends become parallel, DiD's error collapses to noise. Treated-minus-control keeps its error: the level gap never dies.
          • +
        • Macro shock up, with spread > 0: before/after's error explodes, DiD's grows with the walk, and synthetic control stays pinned near the truth: it matches loadings instead of hoping they are equal.
          • +
        +
        +
        None of these numbers ships a warning label
        + Each error is a wrong claim about \(Y_{1t}(0)\), and a wrong claim never improves with more data: the cure is a better claim, not a bigger sample.
        +
        +
        + + +
        +
        Act II · The counterfactual problem
        +

        Interesting limits for the simpler estimators

        +
        Where before/after and treated/control estimators break.
        +
        +
        +
        +
        Before/after · one unit, across time
        + The treated metro's post-average minus its pre-average.
        +
          +
        • The level cancels; nothing subtracts the shared wave.
          • +
        • The whole drift \(\Delta\bar f\) lands on the metro: the largest bias, growing with the horizon.
          • +
        +
        +
        +
        Treated-vs-control · one period, across units
        + The treated metro minus the control average, both in the post window.
        +
          +
        • The level gap \(\alpha_1-\bar\alpha_C\) survives, and sizes differ a lot: it dominates.
          • +
        • This is why raw sales are never compared across cities.
          • +
        +
        +
        +
        ▸ The two bias decompositions, term by term
        +
        + \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] + the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. +
        +
        + \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] + no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. +
        +
        +
        +
        + + +
        +
        Act II · The counterfactual problem
        +

        Interesting limits for DiD

        +
        Where the DiD estimator breaks.
        +
        +
        The one question
        + DiD only works if treated and controls would have drifted together. Two dials decide (spread \(s\), macro \(\sigma_\eta\)); the algebra is in the fold.
        +
        Synthetic control solves it
        + DiD's residual bias \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\) comes from its equal weights: it hopes \(s\) is small. Synthetic control chooses the weights, so \(B\to0\) at any spread.
        +
        ▸ The DiD bias, decomposed (and why more data cannot shrink it)
        +
        + \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] +
        +
          +
        • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
          • +
        • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
          • +
        +
        + \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] +
        +
          +
        • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
          • +
        • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
          • +
        +
        +
        ▸ The two limits, dial by dial: send a knob to zero and read what dies
        +
        +
        Limit 1 · loading spread \(s \to 0\): sufficient on its own
        +
          +
        • The prefactor \(s^2/3\) kills the whole bracket at once.
          • +
        • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
          • +
        • DiD then recovers \(\bar\tau\) up to noise.
          • +
        +
        Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
        +
          +
        • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
          • +
        • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
          • +
        • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
          • +
        +
        +
        +
        ▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
        + Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. +
        +
        +
        + + +
        +
        Act II · The counterfactual problem
        +

        What must be true for the gap to be causal

        +
        The four assumptions that must hold.
        +
        + + + + + + + + + + + + + + + + + + + + +
        AssumptionWhat it demandsThe checkVerdict
        ① Pre-fit qualityThe blend tracks the metro before launch.The gate: 3.2 vs a bar of 4.4.PASS
        ② No anticipationNo reaction before the campaign (no stockpiling, no leak).Placebo-in-time: the fake launch finds ≈ €1.7k, inside noise.PASS
        ③ No spillover (SUTVA)The campaign must not touch donors: Colgate's second-launch leak, in space.No statistical check exists. A design duty: keep test and control regions apart.UNTESTABLE
        ④ Hull & stable loadingsThe metro is a feasible mix of donors; exposures stay put.Leave-one-out: drop any donor, the estimate stays in €239–276k.PASS
        +
        The referee principle
        + Three testable, all PASS: checks ② and ③ are run in Act III, once the luck is priced. The untestable one, no-spillover, is why geo-test design beats any statistic computed afterwards.
        +
        +
        + + +
        +
        Act III · Is it real, and how big?
        +

        €260k. Real, or a lucky metro? Build the null yourself

        +
        +
        +
        ✋ Poll
        +
        With one treated metro a t-test is not weak, it is undefined. So build the test yourself: rerun the whole pipeline 29 more times, each donor pretended treated ("placebo"): 30 "effects", 29 where nothing ran. Where does our €260k rank?
        +
        + + + +
        + +
        A: rank 1 of 30. The rank is the inference: if the campaign did nothing, P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test.
        +
        +
        +
        + + +
        +
        Act III · Is it real, and how big?
        +

        Placebo-in-space: measure the luck directly

        +
        Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
        +
        +
        +
        +
        +
        ▶ LIVE Refit the estimator on every donor as if it were treated
        +
        +
        The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
        +
        +
        +
        +
        The null hypothesis, stated
        + \(H_0\): the campaign did nothing. Then the metro is exchangeable with its donors: any of the 30 ranks is equally likely.
        +
        + \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] +
        +
        +
        +
        What the p-value means (and what it is not)
        + The probability, if \(H_0\) were true, of a gap this extreme: a rank, no bell curve assumed. \(p\) sits at its floor \(1/30\), set by the donor count, not the weeks.
        +
        ▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
        +
          +
        • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
          • +
        • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
          • +
        +
        Conclusion: the €260k lift is real, at \(p\approx0.033\)
        + Rank 1 of 30, clear of the cloud: reject \(H_0\). The rank holds (p = 0.033 every time) dropping shaky placebos at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": Act IV.
        +
        +
        + + +
        +
        Act III · Is it real, and how big?
        +

        From a test to an interval: inversion

        +
        Which true lifts could plausibly have produced our €260k? Keep the survivors: no bell curve anywhere.
        +
        +
        +
        +
        +
        ▶ LIVE Drag \(H\), a guess at the truth: the error cloud slides with it; our €260k never moves.
        +
        +
        + + +
        +
        Axis: 20-week total gap (€000). Top: the placebo errors at truth 0. Middle: the same errors slid to \(H\); \(H\) survives if the green line sits inside the shaded middle 90%. Bottom: the survivors, collected: the interval. Drag past an edge to reject.
        +
        +
        +
        +
        The whole idea
        + Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
        +
          +
        • ① Measure the error. The 29 placebo "effects" form a cloud around zero, spread about ±€50k.
          • +
        • ② Guess a truth \(H\). If the lift were \(H\), we would see \(H\) plus that same cloud.
          • +
        • ③ Keep or reject. Keep \(H\) if €260k sits inside its middle 90%.
          • +
        • ④ Sweep. The survivors run €195k to €335k: the interval is [€195k, €335k].
          • +
        +
        +
        +
        ▸ The inversion, in one line of algebra
        +
        + \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} + \qquad\Longleftrightarrow\qquad + \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] + \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. +
        +
        +
        ▸ Why this interval is the referee for the rest of the lecture
        + It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. +
        +
        +
        + + +
        +
        Act III · Is it real, and how big? Stress-test the estimate
        +

        Falsification 1: a launch that did not happen

        +
        The anticipation check: unlike Colgate's simulated grading, this needs no known answer, because a launch that has not happened yet must have zero effect.
        +
        +
        How it is computed
        + Refit using only weeks 0-29, back-date the launch to week 30, and read the "effect" in weeks 30-39: a window before the real launch, so its true effect is zero by the calendar, not by assumption.
        +
        ▶ LIVE The weekly gap (treated minus synthetic) around a back-dated launch
        +
        +
        + + +
        +
        Clean world: the gap hugs zero in the fake window (≈ €1.7k). Tick the box: a leak climbs before week 40.
        +
        +
        What passing rules out
        + Stockpiling, pre-announcement, a leaked launch: anything that moves sales before the campaign.
        +
        +
        + + +
        +
        Act III · Is it real, and how big? Stress-test the estimate
        +

        Falsification 2: the assumption no placebo can see

        +
        The no-spillover check: the disease you named for Colgate, now in space.
        +
        +
        The assumption (SUTVA)
        + Every method so far assumed the campaign moved only the treated metro. Advertising ignores borders, so we break that on purpose and measure the damage.
        +
        +
        ▶ LIVE Four worlds, rebuilt with a bigger and bigger leak
        +
        +
        Each point is a complete re-simulation and refit, not a correction applied afterwards.
        +
        +
        +
        The concrete story
        + TV and outdoor reach the commuter belt, and those markets are in your donor pool: the campaign quietly raises your "untreated" comparison.
        +
        + \(\varphi\) = the share of the treated lift leaking into 5 neighbour donors, then refit: lifting the donors shrinks the measured gap. +
        +
        +
        +
          +
        • The sweep: \(\varphi\) from 0 to 0.5: clean world to half the lift leaking.
          • +
        • The result: the reported total slides from €260k (clean) to €236k, always toward zero, exactly how a second launch bent Colgate's counterfactual: a contaminated baseline, now in space instead of time.
          • +
        • Dangerous because invisible: the leak starts at launch; every check stays green.
          • +
        • What drives the decision: the bias only shrinks the number: a conservative floor. The defence is design: donors off the media footprint.
          • +
        +
        +
        + + +
        +
        Act III · Is it real, and how big?
        +

        Statistics done. Three numbers.

        +
        Questions ① and ② from the boardroom slide are now answered.
        +
        + + + + + + + +
        QuestionAnswerTool that answered it
        Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
        How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
        Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
        +
        Truth check (only a simulation allows it)
        + The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
        +
        The sentence that loses money
        + "€260k of sales for €75k, a 3.5× return. Roll it out." Every number true; the conclusion does not follow. Question ③ is not a statistics question.
        +
        +
        + + +
        +
        Act IV · The decision in euros
        +

        Now the money

        +
        +
        +
        ✋ Poll
        +
        Cost €{{nb07.cost}}k, incremental sales €{{nb07.cl_total}}k, gross margin {{nb07.margin}}%. What did the campaign earn?
        +
        + + + + +
        + +
        C: about €16k net. Profit = margin × lift − cost = 0.35 × 260 − 75 ≈ +€16k. It returned €3.47 per €1 against a break-even of 2.86: it paid, in expectation. With how much certainty is what this act prices.
        +
        +
        +
        + + +
        +
        Act IV · The decision in euros
        +

        The margin trap

        +
        Compare profit to cost, not revenue to cost.
        +
        +
        +
        +
        \[ \text{profit} \;=\; m\,\tau \;-\; c \]
        +
        • Revenue is not cash: you keep the margin \(m=35\%\) of the lift; you pay the cost in full.
        +
        +
        +
        \[ \tau_{\mathrm{BE}} = \frac{c}{m} = \frac{75}{0.35} = \text{€}214\mathrm{k}, \qquad \mathrm{iROAS}_{\mathrm{BE}} = \frac{1}{m} = 2.86 \]
        +
        • Break-even is €214k of sales, not €75k: €260k clears it by €46k (≈ €16k of profit): iROAS 3.47 vs the \(1/m=2.86\) bar.
          • +
        • Hold onto €214k: it returns as the hinge of the €4M decision.
        +
        +
        +
        +
        ▶ LIVE Where the €260k goes: slide the margin
        +
        +
        + + +
        +
        "iROAS > 1" is a vanity bar; the real bar is 1/margin. Software (90% margin) breaks even at 1.1; grocery (20%) needs 5.0. Same campaign, opposite verdicts.
        +
        +
        +
        + + +
        +
        Act IV · The decision in euros
        +

        The same evidence at every price

        +
        When is the campaign profitable?
        +
        +
        +
        • Line 1, the point: net \(m\hat\tau-c\) crosses zero at \(c=\text{€}91\mathrm{k}\).
        +
        • Line 2, the straddle zone: the profit interval \([\text{€}68\mathrm{k},\text{€}117\mathrm{k}]\), inseparable from break-even.
        +
        +
        +
        ▶ LIVE Slide the campaign price and watch the verdict move
        +
        +
        + + +
        +
        Between the lines: "looks profitable" (left of €91k), not provably so. Ours at €75k is one of them.
        +
        + + + +
        PriceBreak-even liftNet at \(\hat\tau\)VerdictThe lesson
        +
        +
        + + +
        +
        Act IV · The decision in euros, delivered
        +

        The pilot verdict, delivered to the board

        +
        This slide answers the €75k pilot. The €4M is a separate question, next.
        +
        +
        +

        Three questions about the €75k pilot, all answered by the permutation machinery:

        +
          +
        • ① Real? Yes. Rank 1 of 30, \(p=0.033\).
          • +
        • ② Big enough? On the point, yes: €260k vs break-even €214k: €46k to spare, ≈ €16k net.
          • +
        • ③ Confident? No. Profit [€68k, €117k] still straddles the €75k price.
          • +
        +
        Pilot verdict, banked
        + At €75k, probably profitable but not a confident buy: renegotiate the price or buy certainty with a bigger test. This says nothing yet about the €4M, which is not the pilot multiplied.
        +
        +
        + + + + + + + + + + +
        Board numberValueSource
        Incremental sales, 20 wk€260ksynthetic-control gap
        Real, or luck?p = 0.033placebo permutation, rank 1/30
        90% interval on sales[€195k, €335k]test inversion
        iROAS vs break-even3.47 vs 2.86P&L: break-even = 1/margin
        Gross profit vs price€91k vs €75k0.35 × 260 − 75
        Pilot recommendationNOT A CONFIDENT BUYinterval straddles the price
        +
        The one question this memo cannot answer
        + A classical memo is yes/no about where an interval sits: it cannot say "78% chance it pays", price the evidence, or value a follow-up study. And it is only about €75k: the €4M is next.
        +
        +
        +
        + + +
        +
        Act IV · The decision in euros, scaled up
        +

        The €4M is not provably profitable

        +
        Even at its best case, the profit interval straddles zero.
        +
        +
        +
        +
        +
        ▶ LIVE Rollout profit at €4M, carrying the 90% interval, as a share \(\delta\) of the pilot survives
        +
        +
        + + +
        +
        The point says +€860k at \(\delta=1\), but the 90% band [−€360k, +€2.25M] straddles zero even there. Drag \(\delta\) down (the real world, where each euro does less) and it only sinks; below \(\delta=0.64\) it wholly loses.
        +
        +
        +
        +
          +
        • ① The marketing number is a point: 53 pilot-budgets × €260k × 35% margin − €4M ≈ +€860k. Every step correct.
          • +
        • ② Carry the interval, and be generous: assume the pilot transports perfectly (\(\delta=1\)). Even then the 90% profit interval is [−€360k, +€2.25M]: it straddles zero.
          • +
        • ③ And \(\delta=1\) is a ceiling: €4M is €133k per market (1.78× the pilot, so each euro does less) and one metro is not the nation, so real \(\delta<1\) and the band only sinks.
          • +
        +
        The frequentist verdict, no probability needed
        + At its most generous the €4M is not provably profitable, and below \(\delta\approx0.64\) it provably loses. Do not commit €4M on this evidence.
        +
        +
        +
        ▸ Where 0.82 and 0.64 come from, and a defensible estimate of δ
        +
        +
        +
        The bar: required δ (arithmetic)
        +
        \[ \delta^{*} = \frac{c}{m\,\tau} = \frac{\tau_{\mathrm{BE}}}{\tau} = \frac{214}{\tau} \]reach \(R=C/c\), so the budget \(C\) cancels: the bar is set by cost, margin and lift only
        +
          +
        • Across the lift interval [€195k, €335k], required δ ∈ [0.64, 1.10], central 0.82 (82%).
          • +
        • Below 0.64 even the top of the interval loses: that is the "provably loses" edge.
          • +
        +
        +
        +
        The estimate: plausible δ (one assumption)
        +
        \[ \delta \;\approx\; 1.78^{\,\beta-1} \]per-euro lift at 1.78× intensity, up a concave response curve of elasticity \(\beta\)
        +
          +
        • \(\beta\) from the client's MMM, or "double the spend → ≈1.6× response" → \(\beta\approx0.68\) → δ ≈ 0.83.
          • +
        • \(\beta\) 0.6–0.8 gives [0.79, 0.89], central 0.84 (84%); being one metro only widens this; ours sits at the median (effect CV ≈ 0.14).
          • +
        +
        +
        +
        Why we measure, not estimate
        + The estimate 0.84 lands on the bar 0.82: profit at €4M runs −€1.11M / about €0 / +€1.57M across the pessimistic / central / optimistic corners. Defensible enough to refuse €4M, too close to the line to commit it: only a multi-market test separates the two.
        +
        +
        +
        + + +
        +
        Act IV · The decision in euros, scaled up
        +

        Measure before you commit

        +
        One treated market made this interval; only more will shrink it.
        +
        +
        +
        +
        The call
        + Even the best case straddled zero, so on this evidence do not commit €4M. The rollout is not dead: the evidence simply cannot size it.
        +
          +
        • Why analysis cannot rescue it: the interval is wide because there is one treated market. Re-running the model does not narrow it; only more treated markets do.
          • +
        • Step zero, first: check the drawer: prior pilots, the media-mix model, benchmarks may already answer it.
          • +
        +
        +
        +
        The fix: measure the national effect directly
        + Run it in 8 markets chosen to look like the nation, at full intensity (~€133k each, €1.07M). The cells span the country, so the measured lift is the national lift. HelloFresh's loop, bought once.
        +
        The rule, written before the data lands
        + Release the remaining €2.9M only if the measured 90% profit interval clears zero. Written first, or the test is theatre.
        +
        The recommendation
        + ① Check the drawer. ② Do not commit €4M: the best case is not provably profitable. ③ Run the €1.07M, 8-market test at national intensity, randomised. ④ Separately: renegotiate the pilot toward €{{nb07.headroom}}k.
        +
        +
        +
        ▸ Sizing the test, and why 8 markets
        +
        \[ k \;\ge\; \Bigl[\tfrac{(z_{0.95}+z_{0.80})\,\mathrm{CV}}{\Delta}\Bigr]^{2} = \Bigl[\tfrac{2.49\times0.30}{0.36}\Bigr]^{2} \approx 4.3 \]
        +
          +
        • Sized from the decision: distinguish δ = 1 from δ = 0.64 (a 36% shortfall) against between-market spread; ≈ 5 markets on power, 8 to span the mix, and CV 0.4 pushes power itself to ≈ 8.
          • +
        • Constraints: randomise in matched pairs (that alone frees the p-value from the single-metro floor, 0.033 → ≈ 0.004); test at deployment intensity; keep controls off the media footprint.
          • +
        • Worth it: the Bayesian layer of this course prices this information at ≈ €320k, far above the test's incremental cost, since the €1.07M is real advertising working while it measures.
          • +
        +
        +
        +
        + + + + +
        +
        +
        +
        Part 3 · The natural experiment
        +

        Instrumental Variables

        +
        What is one ad exposure worth, when the platform already picked who sees the ads?
        +
        +
        + +
        Where this part lands: the dashboard's tight interval misses the true effect of an ad exposure, while the interval built from a lottery covers it, and still clears the price.
        +
        +
        +
        + + +
        +
        IV · The problem
        +

        The case

        +
        One retailer, one ad platform, one decision to make.
        +
        +
        +
        The product
        An online retailer pays an ad platform to show its display ads. One exposure = one user actually saw an ad.
        +
        The price
        Each exposure costs €{{nb11.cost}}, billed by the platform.
        +
        The assignment
        The platform decides, in a real-time auction, which users see the ads.
        +
        The dashboard
        Users who saw an ad brought in €{{nb11.naive}} more than users who did not.
        +
        +
        The platform's pitch
        + "An exposure is worth €{{nb11.naive}} and costs €{{nb11.cost}}. Raise the budget."
        +
        The decision on the table
        + Keep paying €{{nb11.cost}} per exposure, pay more, or stop. One number settles it: how many euros of extra sales one exposure brings in. All we have is the platform's logs: who saw an ad, and what every user spent.
        +

        The question for the poll: is the dashboard's €{{nb11.naive}} that number?

        +
        +
        + +
        +
        IV · The problem
        +

        Poll · the pitch

        +
        +
        +
        ✋ Poll
        +
        The dashboard reports that users who saw an ad brought in €{{nb11.naive}} more than users who did not, and an exposure costs €{{nb11.cost}}. The platform concludes: "raise the budget". What do you think of that claim?
        +
        + + + + +
        + +
        C. The arithmetic is correct and, at this sample size, comfortably significant, so A and B both miss the point. The platform chose who saw the ads, and it is paid to pick users who already look ready to buy. If the two groups differed before any ad ran, the €{{nb11.naive}} mixes the ad's effect with that pre-existing difference, in unknown proportions. D is backwards: averaging over thousands of users is exactly how noise is handled.
        +
        +
        +
        + +
        +
        IV · The problem · the variables, drawn
        +

        The variables and the confounder

        +
        The three quantities of the case, defined through their causal diagram.
        +
        +
        +

        The platform's logs cover {{nb11.n}} customers, indexed by \(i\). Three quantities per customer:

        +
          +
        • \(Y_i\) · sales: contribution euros earned from customer \(i\) in the window (sales net of product cost); "sales" for short. Noisy: the standard deviation across customers is €{{nb11.ppc_obs_sd}}.
          • +
        • \(X_i\) · exposure: 1 if customer \(i\) saw the ad, 0 if not. Assigned by the platform's auction, not by us.
          • +
        • \(U_i\) · intent: how ready \(i\) was to buy before any ad. Real, and it drives buying, but it is a feeling in a shopper's head: in no log file, never observed.
          • +
        +
        Definition · confounder
        + A confounder is a variable that influences both the treatment and the outcome. Here it is intent \(U\): it raises the chance the platform shows the ad (the targeting), and it raises sales on its own (ready buyers buy, ad or not).
        +
        Definition · backdoor path
        + In the diagram, \(X \leftarrow U \rightarrow Y\) is a backdoor path: a second route connecting exposure and sales that runs behind the treatment. Association flows along it even if the ad does nothing.
        +
        +
        +
        the situation, as a diagram
        +
        +
        \(U\) points at both \(X\) and \(Y\): the fork every claim in this lecture has to get past.
        +
        +
        +
        + +
        +
        IV · The problem
        +

        The naive comparison

        +
        The dashboard's €{{nb11.naive}}, written down as a formula.
        +
        +
        +

        Write \(Y_i(1)\) for what customer \(i\) spends with the ad and \(Y_i(0)\) for what the same customer spends without it. Each customer shows us exactly one of the two:

        + + + + + + +
        group\(Y_i(1)\) · with ad\(Y_i(0)\) · without ad
        exposed (\(X_i=1\))observedN/A
        unexposed (\(X_i=0\))N/Aobserved
        +
        \[ \widehat{\Delta}_{\text{naive}} \;=\; \bar{Y}_{\text{exposed}} \;-\; \bar{Y}_{\text{unexposed}} \;=\; \text{€}{{nb11.naive}} \]
        +
          +
        • What it averages: the two observed cells, one from each row. The arithmetic is right.
          • +
        • What it pretends: that the unexposed row's \(Y(0)\) is a fair stand-in for the exposed row's missing \(Y(0)\). Fair only if the two groups are comparable.
          • +
        • The auction works against us: the platform is paid to find likely buyers, so the exposed would have spent more with no ad at all.
          • +
        +
        +
        +
        +
        ▶ LIVE exposed users were already different
        +
        +
        + + +
        +
        The gap compares two groups the auction built to differ. The hidden column shows by how much.
        +
        +
        +
        +
        + +
        +
        IV · The problem
        +

        Poll · read the gap

        +
        +
        +
        ✋ Poll
        +
        The naive comparison gives \(\widehat{\Delta}_{\text{naive}} = \) €{{nb11.naive}}. Which reading of that number is defensible?
        +
        + + + + +
        + +
        B. The arithmetic is sound, so C is out. A and D each pick one extreme of the same unknown split: A books all of it to the ad, D books all of it to targeting, and the dashboard alone cannot justify either. The honest statement is the decomposition: €{{nb11.naive}} = the ad's effect + selection bias, with nothing on the dashboard saying how it splits. Measuring that split is the technical problem this lecture solves.
        +
        +
        +
        + +
        +
        IV · The problem · the poll's answer, formalized
        +

        Selection bias

        +
        The decomposition behind the poll's answer.
        +
        +
        +

        Split the naive gap into the two things it adds together:

        +
        \[ \Delta_{\text{naive}} \;=\; \underbrace{\mathbb{E}[Y(1)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}1]}_{\text{effect of the ad on the exposed}} \;+\; \underbrace{\mathbb{E}[Y(0)\mid X{=}1]\;-\;\mathbb{E}[Y(0)\mid X{=}0]}_{\text{selection bias}} \]
        +
          +
        • Read the first term: the exposed users with the ad versus the same users without it. This is what we want to buy, and its second entry is the table's N/A cell.
          • +
        • Read the second term: the gap that would exist with no ads at all, because the picked users differ from the skipped ones.
          • +
        • Here it is positive: the auction picks ready buyers, so the exposed would have out-spent the unexposed anyway.
          • +
        • The dashboard reports only the total: €{{nb11.naive}}. Nothing on it says how the total splits.
          • +
        +
        Definition · selection bias
        + When treated and untreated groups differed before the treatment, the difference of their averages includes that pre-existing gap. The pre-existing part is selection bias.
        +
        +
        +
        +
        the dashboard's number is a sum, and the split is invisible
        +
        +
        The dashboard hands us the full bar. Where the boundary sits, nobody can see from the dashboard alone. Finding it is the rest of the lecture.
        +
        +
        +
        +
        + +
        +
        IV · The problem · a world we can grade
        +

        The simulated world

        +
        A world where the true effect is planted, so every method can be graded.
        +
        +
        +
        +
        ▶ LIVE the world the equations generate, as data
        +
        +
        + + +
        +
        Sales distributions for exposed and unexposed users, drawn fresh from the equations. Push \(\kappa\) up, so that intent leaks more strongly into sales, and the two groups drift apart with the planted effect unchanged: the dashboard's gap grows while the truth stands still.
        +
        +
        +
        +
        Why simulate at all
        + Three equations generate the dashboard you saw. We planted an ad effect of \(\beta =\) €{{nb11.true}}, so every method in this lecture can be graded against a known answer.
        +
        \[ U \sim \mathcal{N}(0,1) \]
        +
        \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \lambda\,U\right) \]
        +
        \[ Y \;=\; \mu + \beta\,X \;+\; \kappa\,U \;+\; \varepsilon \] + \(\mu\): baseline sales  ·  \(\alpha_0\): baseline exposure odds  ·  \(\varepsilon\): noise
        +
          +
        • Interpretation of line 2: who sees an ad rises with intent (\(\lambda U\)): the targeting. \(\sigma\) is the logistic function, turning any score into a probability.
          • +
        • Interpretation of line 3: sales are the ad effect \(\beta\) plus intent leaking straight into spending (\(\kappa U\)).
          • +
        • The math is here for the record. The point is on the left: one hidden cause, \(U\), sits in both equations.
          • +
        +
        +
        +
        + +
        +
        IV · The problem · the disease, stated exactly
        +

        Endogeneity

        +
        The whole problem, stated as one inequality, then shown as a picture.
        +
        +
        +

        Collect everything the sales equation leaves out into a single error term \(v\):

        +
        \[ Y \;=\; \mu + \beta\,X + v, \qquad v \;=\; \kappa\,U + \varepsilon \]
        +
        \[ \operatorname{Cov}(X,\, v) \;>\; 0 \]
        +
          +
        • In words: \(v\) is "every other reason this customer spent money", and exposure is not independent of those reasons, because the auction targets exactly the customers with reasons to spend.
          • +
        • The picture on the right is the inequality: walk up the intent scale and both bars rise together. That joint rise is \(\operatorname{Cov}(X,v) > 0\).
          • +
        +
        Definition · endogeneity
        + A treatment correlated with the other drivers of the outcome is endogenous. Ours is, and everything else in this lecture is a response to that fact.
        +
        The sharp edge
        + The variable driving the correlation was never written to disk.
        +
        +
        +
        +
        one hidden cause pushes both
        +
        +
        Customers grouped by buying intent (which nobody observes). Higher-intent customers see more ads and spend more, ad or no ad. The formula on the left is this picture, in symbols.
        +
        +
        +
        +
        + +
        +
        IV · The problem · the bias, computed
        +

        The size of the bias

        +
        The dashboard's error is not vague. It has a formula, and the formula has three lessons.
        +
        +
        +
        \[ \widehat{\Delta}_{\text{naive}} \;\longrightarrow\; \beta \;+\; \underbrace{\kappa\,\frac{\operatorname{Cov}(X, U)}{\operatorname{Var}(X)}}_{\text{selection bias}} \]
        +
          +
        • Predicted vs delivered: the formula predicts a naive gap near €{{nb11.ols_predicted}}. The data delivered €{{nb11.naive}}, against a truth of €{{nb11.true}}.
          • +
        +
        What the formula teaches
        + 1. The error is systematic, not bad luck: both factors are positive, so the dashboard can only overstate.
        + 2. It grows with the targeting (\(\lambda\), how strongly intent drives exposure) and with intent's pull on sales (\(\kappa\)): the better the platform targets, the worse the dashboard lies.
        + 3. Sample size is absent: more rows shrink noise, never bias. The dashboard's band, €{{nb11.naive_lo}} to €{{nb11.naive_hi}}, is the tightest in this lecture and misses €{{nb11.true}} completely.
        +
        What the dashboard measures
        + Not the ad's power to create buyers but the platform's skill at finding them, reported as one number.
        +
        +
        +
        +
        ▶ LIVE the same bar, at different targeting strengths
        +
        +
        + + +
        +
        The dashboard's bar, split by the formula: the true effect (blue) never moves, while the selection bias (orange) grows as intent leaks more strongly into sales. In this simulated world we can draw the boundary, because we planted it.
        +
        +
        +
        +
        + +
        +
        IV · The problem · closing the obvious exit
        +

        The limits of adjustment

        +
        The standard fix works, when you have the column it needs. We do not.
        +
        +
        +
        The standard fix: close the backdoor
        + Compare users at the same level of the confounder. Within a group of equal-intent users, intent no longer separates exposed from unexposed, and any remaining gap belongs to the ad. Controls in a regression, matching, and propensity scores are all versions of this one move.
        +
        +
        ▶ LIVE the fix works, if the column exists
        +
        +
        + + +
        +
        Tick the box: comparing exposed to unexposed within equal-intent groups recovers the planted truth to within half a euro. The method is fine. Untick it: in the real logs the column does not exist, and no amount of modelling recreates it.
        +
        +
        Why the engine stalls here
        + You cannot adjust for what you did not record.
        +
        +
        +

        Two customers, identical on every recorded feature \(W\):

        + + + + + + +
        customerpages, device, history \(W\)intent \(U\)sees the ad?
        Annaidenticalhigh, unrecordedpicked
        Beaidenticallow, unrecordedskipped
        +
        what conditioning blocks, and what it cannot
        +
        +
        Holding \(W\) fixed (the box) blocks the backdoors that run through it. The arrows themselves never move: conditioning blocks paths, it does not delete causes. The path through \(U\) stays open, because \(U\) is in no file to hold fixed.
        +
        +
        +
        + +
        +
        IV · The problem
        +

        Poll · what would fix it?

        +
        +
        +
        ✋ Poll
        +
        Adjustment failed because intent was never recorded. Your team can request one addition to the data. Which one would actually let you measure what an exposure causes?
        +
        + + + + +
        + +
        C. A tells you the wrong number more precisely: bias is not noise, and the formula for it does not contain the sample size. B and D are more controls, and the auction reacts to live signals that no exported feature set fully contains, so the backdoor stays open. C is different in kind: it changes who assigns the treatment, and the assignment was the disease all along. The rest of the lecture is about getting C, or the closest thing to it the system already contains.
        +
        +
        +
        + + +
        +
        IV · The idea · what we wish we could do
        +

        The ideal experiment

        +
        What randomizing exposure would buy, and why we cannot do it.
        +
        +
        +
        randomization cuts the arrow that causes the trouble
        +
        +
        Left: today's world, where intent drives both exposure and sales. Right: the experiment we wish we could run, where a randomizer assigns exposure and intent's arrow into it is removed, not merely blocked: exposure no longer listens to intent at all. With that arrow gone, the naive comparison becomes the right comparison.
        +
        \[ X \text{ randomized} \;\;\Rightarrow\;\; \operatorname{Cov}(X,\, v) = 0 \;\;\Rightarrow\;\; \widehat{\Delta}_{\text{naive}} \longrightarrow \beta \]
        +
        +
        +
          +
        • The wish: option C from the poll: show the ad to a random half ourselves.
          • +
        • The wall: the platform's auction decides exposure in milliseconds, using exactly the intent signals that cause the problem. It will not hand us the controls.
          • +
        +
        The idea that saves us
        + If we cannot inject randomness into exposure, find the randomness the system already contains and use only that part.
        +
          +
        • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
          • +
        +
        The plan for Part 2
        + 1. Give that random lever a name: an instrument.
        + 2. State the conditions it must satisfy.
        + 3. Check which of them the data can verify.
        +
        +
        +
        + +
        +
        IV · The idea · the randomness we ran
        +

        The lottery

        +
        The randomness our system already contains, because we put it there.
        +
        +
        +
          +
        • What we ran: before the campaign, a serving-priority lottery. Our own random number generator picked half the users and gave them a small priority boost in the platform's ad auction.
          • +
        • What it does: the lottery shows nobody an ad. It only raises the odds of seeing one.
          • +
        • A new column joins the data:
          • +
        + + + + + +
        quantitymeaningtypewho assigns itobserved?
        \(Z_i\) · the lottery1 if the lottery boosted \(i\)'s priority in the ad auctionbinarya random number generator we controlobserved
        +
        \[ \Pr(X{=}1) \;=\; \sigma\!\left(\alpha_0 + \gamma\,Z + \lambda\,U\right) \] + the simulated world ran the lottery too: the exposure equation gains \(\gamma Z\), the lottery's push on the odds of seeing an ad
        +
          +
        • Why it matters: \(Z\) moves exposure, and nothing else about the customer. It is the one source of variation in \(X\) that the targeting cannot touch.
          • +
        +
        +
        +
        the fork, and the way around it
        +
        +
        Top: the confounded fork. Bottom: the lottery \(Z\) pushes on \(X\) from outside. No arrow from \(U\) into \(Z\) (it is a random draw), and no arrow from \(Z\) straight to \(Y\) (a queue bump shows the user nothing).
        +
        +
        +
        + +
        +
        IV · The idea · the central definition
        +

        The instrument

        +
        What we just built has a name.
        +
        +
        Definition · instrument
        + An instrument \(Z\) moves the treatment, is as good as random with respect to the hidden confounder, and affects the outcome only through the treatment. Four conditions make that precise, and they grade our lottery as follows.
        + + + + + + + + + + + + + +
        conditionthe claimcan it be checked?what it buys
        1 · Relevancethe lottery actually moves exposure \(X\)testable measured by the first stagea push that really happened
        2 · Exogeneitythe lottery is blind to intent \(U\)by design if the draw is genuinewith 1: a clean experiment on the lottery
        3 · Exclusionthe lottery touches sales only through exposureuntestable argued, never checkedturns the lottery's effect into the ad's
        4 · Monotonicitythe lottery never blocks an exposure that would have happened without ituntestable argued from the mechanismsays whose effect we measured
        +
        An instrument is the opposite of a control
        + You do not hold \(Z\) fixed. You use the variation it created.
        +

        The genius is never the estimator, it is spotting the randomness your system already contains: a rollout order, a capacity limit, an upstream A/B test, a rounding rule.

        +
        +
        + +
        +
        IV · The idea · condition 1, measured
        +

        Condition 1 · relevance

        +
        The first stage: did the lottery actually move exposure?
        +
        +
        +

        Relevance is a claim about the data, so the data can answer it.

        +
        Definition · first stage
        + The first stage is the effect of the instrument on the treatment: how much exposure the lottery itself created. It is the coefficient \(\pi\) in the regression below.
        +
        +
        \[ X \;=\; b_0 + \pi\,Z + u \] + \(\pi\): the first stage  ·  \(b_0\): exposure rate among lottery losers  ·  \(u\): every other driver of exposure, intent included
        +
        +
        +
        \[ \pi \;=\; \Pr(X{=}1 \mid Z{=}1) \;-\; \Pr(X{=}1 \mid Z{=}0) \;=\; {{nb11.first}} \] + the measured first stage: what the lottery did to exposure
        +
          +
        • Interpretation of \(\pi\): winning the lottery raises a user's chance of seeing an ad by {{nb11.fs_shift}} percentage points, from {{nb11.fs_lo}}% to {{nb11.fs_hi}}%.
          • +
        • It is causal, full stop: \(Z\) is random, so nothing hidden can explain the step.
          • +
        +
        +
        +
        exposure rate, by the lottery
        +
        +
        The step between the bars is the first stage \(\pi\): the slice of exposure that the lottery, not targeting, created.
        +
        The gate: is the push strong enough?
        + Like every estimate in this lecture, \(\pi\) comes with a significance test: the first-stage \(F\). Trust the coming division only when \(F > 10\), well past bare significance, because a \(\pi\) that is merely nonzero still leaves the division unstable. Here \(F = {{nb11.f_stat}}\): the push is unmistakably real. We quote such tests, not derive them.
        +
        +
        +
        + +
        +
        IV · The idea · conditions 2 and 3
        +

        Conditions 2 and 3 · exogeneity and exclusion

        +
        The lottery must be blind to intent, and silent about sales.
        +
        +
        +
        Blind · exogeneity
        + The lottery must not favour high-intent users. Ours is a genuine randomizer, so this holds by design.
        +
        Silent · exclusion
        + The lottery must not touch sales through any channel except the ad itself. A side-channel (faster pages for boosted users, a shown price) would poison the method:
        +
        \[ \hat\beta_{\text{IV}} \;=\; \beta + \frac{s}{\pi} \] + \(\hat\beta_{\text{IV}}\): the estimate the lottery-based method reports  ·  \(\beta\): the true effect of one exposure on sales, the number we are hunting  ·  \(s\): the side-channel's own lift on sales  ·  \(\pi\): the first stage
        +
        No test will catch it
        + Exclusion is a claim about a path that should not exist, so no output can verify it. You defend it by design.
        +
        +
        +
        ▶ LIVE break the silence on purpose
        +
        +
        + + +
        +
        What the method reports, split into the true effect (blue) and the contamination \(s/\pi\) (red). Slide \(s\) up: a small leak becomes a large error.
        +
        +
          +
        • Why a tiny leak is a big error: the side-channel \(s\) gets divided by the small first stage \(\pi\), so even €1 of leak moves the estimate by €1 / {{nb11.first}}.
          • +
        • Our defence: a queue bump is content-free, sells nothing, and the user never learns it happened.
          • +
        • The counterexample: an email with a discount code would move spending on its own, and everything after this slide would collapse.
          • +
        +
        +
        +
        + + +
        +
        IV · The estimator · the second ingredient
        +

        The reduced form

        +
        The same comparison as the first stage, now for sales.
        +
        +
        +
        Definition · reduced form
        + The reduced form is the effect of the instrument on the outcome: the sales gap between the lottery groups. It is the \(\delta\) measured below.
        +
        \[ \delta \;=\; \mathbb{E}[Y \mid Z{=}1] \;-\; \mathbb{E}[Y \mid Z{=}0] \;=\; \text{€}{{nb11.reduced}} \] + \(\delta\): the reduced form  ·  \(\beta\): the true effect of one exposure on sales, still the number we are hunting
        +
          +
        • Interpretation of \(\delta\): winning the lottery adds €{{nb11.reduced}} of sales for the average user. Causal for the same reason \(\pi\) was: a random draw cannot favour ready buyers.
          • +
        • The intuition that carries the day: the lottery can only reach sales through the ad. One lottery win buys \(\pi\) extra exposures, and each exposure is worth \(\beta\) euros. So the lottery's effect on sales must be \(\beta \times \pi\):
          • +
        +
        \[ \delta \;=\; \beta \times \pi \]
        +
        Both numbers are correct. Neither answers the question.
        + \(\pi\) is exposures per lottery win and \(\delta\) is euros per lottery win, but the client asked for euros per exposure. Since \(\delta = \beta\pi\), one step is left.
        +
        +
        +
        average sales, by the lottery
        +
        +
        The step is small but clean: a random draw cannot favour ready buyers, so selection cannot explain it.
        +

        For the record, the algebra behind the intuition: substitute \(X = b_0 + \pi Z + u\) into the sales equation and the coefficient multiplying \(Z\) is exactly \(\beta\pi\).

        +
        +
        +
        + +
        +
        IV · The estimator · the whole method in one division
        +

        The IV estimate

        +
        Euros per lottery win, divided by exposures per lottery win.
        +
        +
        +
        the whole method, as arithmetic on two measured numbers
        +
        +
        One lottery win buys {{nb11.first}} extra exposures and €{{nb11.reduced}} of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
        +
        +
        +
        +
        \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ {{nb11.reduced}} }{ {{nb11.first}} } \;=\; \text{€}{{nb11.iv_est}} \]
        +
          +
        • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
          • +
        • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
          • +
        +
        +
        +
        What just happened
        + We priced the ad using only the random slice of exposure: the dashboard said €{{nb11.naive}}, the lottery says €{{nb11.iv_est}}, against a planted truth of €{{nb11.true}}.
        +

        The method never needed a model of intent, controls, or machine learning: two averages and a division.

        +
        +
        +
        ▸ Deep dive · the confidence interval around €{{nb11.iv_est}}
        +
        +
        +

        \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

        +
        \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; {{nb11.iv_est}} \;\pm\; 1.645 \times {{nb11.iv_se}} \;=\; [\,{{nb11.iv_lo}},\; {{nb11.iv_hi}}\,] \] + the 90% confidence interval: the range of effect sizes the data support
        +
          +
        • Interpretation of the interval: any effect between €{{nb11.iv_lo}} and €{{nb11.iv_hi}} is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
          • +
        • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
          • +
        +
        +
        +
        two intervals, graded against the planted truth
        +
        +
        The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
        +
        +
        +
        +
        +
        + +
        +
        IV · The estimator · hands on
        +

        Why the division is forced

        +
        The division is not a modelling choice. It is the only effect size the two measurements allow.
        +
        +
        +
        ▶ LIVE every candidate effect makes a prediction. One matches.
        +
        +
        + + +
        +
        The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €{{nb11.reduced}}. Move your guess to the crossing and you have priced the ad.
        +
        +
        +

        Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

        +
          +
        • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's {{nb11.first}} extra exposures per win should create \(\hat\beta \times {{nb11.first}}\) euros per win.
          • +
        • The fact: the lottery actually created €{{nb11.reduced}} per win.
          • +
        • The verdict: every candidate except €{{nb11.iv_est}} contradicts a number we measured. The division is the only survivor, not a choice.
          • +
        +
        \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
        +
        Not a black box
        + Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
        +
        +
        +
        + + +
        +
        IV · When it breaks · the dangerous failure
        +

        Weak instruments

        +
        A weak instrument is worse than no instrument.
        +
        +
        +
        ▶ LIVE what the estimator does as the first stage dies
        +
        +
        + + +
        +
        Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
        +
        +
        +

        The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

        +
          +
        • The honest failure: the interval balloons, and the data admit they know little.
          • +
        • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
          • +
        • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
          • +
        +
        Report the first-stage \(F\), always
        + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
        +
        +
        +
        ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
        +
        +
        +
        ▶ LIVE test each candidate \(\beta_0\), keep the survivors
        +
        +
        + + +
        +
        The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [{{nb11.ar_lo}}, {{nb11.ar_hi}}]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
        +
        +
        +
          +
        • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
          • +
        • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
          • +
        +
        \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
        +
          +
        • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
          • +
        • Here, a good sign: the lottery is strong, so AR [{{nb11.ar_lo}}, {{nb11.ar_hi}}] nearly matches the usual [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. When \(F\) is small the two part company, and AR is the one still telling the truth.
          • +
        +
        +
        +
        +
        +
        + +
        +
        IV · When it breaks · whose effect it is
        +

        Compliers and the LATE

        +
        Whom does the €{{nb11.iv_est}} describe? Only the users the lottery could move.
        +
        +
        ▶ LIVE the user base, split by how they respond to the lottery
        +
        +
        + + +
        +
        Push \(\gamma\): the complier slice grows, because the complier share is the first stage. Shares are drawn live from one simulated batch, so they can differ from the printed figures by a rounding step.
        +
        +
        +
          +
        • Always-takers ({{nb11.st_share_always}}%): the auction shows them the ad with or without the lottery. It changes nothing for them, so the data say nothing about them.
          • +
        • Never-takers ({{nb11.st_share_never}}%): never see the ad either way. Same silence.
          • +
        • Compliers ({{nb11.st_share_complier}}%): see the ad only because the lottery favoured them. Every euro of the lottery's lift \(\delta\) came from them.
          • +
        • The caveat, monotonicity: we assume no defiers, users who would see the ad only when the lottery does not favour them. A nudge that never repels.
          • +
        +
        +
        +
        Definition · LATE (local average treatment effect)
        + The LATE is the average effect of the ad on the compliers alone, and it is what the division estimates: a local answer, not a statement about every user.
        +
        \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;\;\text{ estimates }\;\; \mathbb{E}[\,Y(1)-Y(0)\mid \text{complier}\,] \] + the average of each complier's personal effect, the same \(Y(1)-Y(0)\) contrast the naive slide could not touch
        +
        +
        +
        Why a manager should love this fine print
        + Compliers are the same kind of marginal user a higher bid would newly reach: the closest thing in the data to the customer a bid change buys. That makes €{{nb11.iv_est}} a price for the marginal customer, measured on the margin rather than on the average.
        +
        +
        + +
        +
        IV · When it breaks · what must hold, on one page
        +

        The checklist

        +
        The four assumptions, and which ones the data can check.
        +
        + + + + + + + + +
        AssumptionWhat it saysIn the caseStatus
        Relevance\(Z\) moves \(X\)\(F = {{nb11.f_stat}}\), far above 10TESTABLE, passes
        Exogeneity\(Z \perp U\)the lottery is a genuine random drawBY DESIGN
        Exclusion\(Z \to Y\) only via \(X\)a queue bump shows the user nothingUNTESTABLE
        Monotonicityno defiersa nudge never repelsUNTESTABLE, plausible
        +
        +
        +
        The honest scorecard, for any IV study you are shown
        + One measured number (the first-stage \(F\)), one design guarantee (the randomization), two arguments (exclusion, monotonicity). Ask for all four before you accept the estimate.
        +
          +
        • What we can now defend: an exposure causes about €{{nb11.iv_est}} of sales for the users a bid can actually move.
          • +
        +
        +
        The sentence that loses money
        + "Exposed users are worth €{{nb11.naive}} each, so raise the bid": every word true, conclusion wrong. It books the platform's targeting as advertising.
        +
        +
        +
        + + + +
        +
        IV · The decision · euros at last
        +

        The price map

        +
        The estimate becomes a decision only when it meets the price.
        +
        +
        +
        ▶ LIVE the verdict as the price moves
        +
        +
        + + +
        +
        Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
        +
        +
        +

        One estimate gives not one answer but a map from any price to a verdict:

        + + + + + + + +
        Price zoneVerdictWhy
        below €{{nb11.iv_lo}}GOeven the most pessimistic supported effect pays
        €{{nb11.iv_lo}} to €{{nb11.iv_hi}}TESTthe data straddle the price: negotiate, or measure more
        above €{{nb11.iv_hi}}NO-GOno supported effect pays
        +
          +
        • Today's rate, €{{nb11.cost}}, sits in the GO zone, below the whole interval.
          • +
        • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €{{nb11.iv_est}} − €{{nb11.cost}} = €{{nb11.net}}. Even read at the interval floor, €{{nb11.iv_lo}} against €{{nb11.cost}}, the exposure still pays.
          • +
        +
        Why boards like this framing
        + "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
        +
        +
        +
        + +
        +
        IV · The decision
        +

        Poll · the negotiation

        +
        +
        +
        ✋ Poll
        +
        The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €{{nb11.iv_est}} per exposure, 90% interval [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. What is the highest rate at which you would still sign "buy" without further study?
        +
        + + + + +
        + +
        C. Below €{{nb11.iv_lo}}, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
        +
        +
        +
        + +
        +
        IV · The decision · banked
        +

        The verdict and the recommendation

        +
        The complete answer, assembled from everything measured so far.
        +
        +
        + + + + + + + + + +
        QuantityValueSource
        effect of one exposure€{{nb11.iv_est}}the division δ/π
        90% interval[{{nb11.iv_lo}}, {{nb11.iv_hi}}]classical, and AR agrees
        first-stage F{{nb11.f_stat}}the lottery is strong
        price€{{nb11.cost}}the platform's rate card
        net per exposure€{{nb11.net}}β − c, at the point estimate
        +
        The verdict
        + BUY  Keep buying at €{{nb11.cost}}: the entire defensible range clears the price.
        +
        +
        +
        The recommendation, in three lines
        + 1. Keep buying at the €{{nb11.cost}} rate: even the interval's most pessimistic effect pays.
        + 2. Cap the bid at the interval's lower end, €{{nb11.iv_lo}}: up to there, every effect the data support still clears the price.
        + 3. Measure again only if the rate card climbs toward €{{nb11.iv_lo}}: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
        +
          +
        • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
          • +
        • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
          • +
        +
        +
        +
        + + +
        +
        Closing · Provenance
        +

        The tools were the product too

        +
        +
          +
        • CausalPy: {{labs.causalpy_methods}} in one open-source package. The IV estimator that closes this session joined later.
          • +
        • Its launch example: individual exposure to a TV campaign {{labs.causalpy_tv}}, yet its causal impact remains a core business need: the sentence this whole session opened with.
          • +
        • pymc-marketing: the MMM library behind Case 2's calibration story; one client's {{labs.bolt_pr}} came back as a pull request (Bolt).
          • +
        • Webinars and content: the consultancy's own webinar agenda is this session's syllabus, {{labs.webinar_agenda}} included.
          • +
        +
        + CausalPy + PyMC-Marketing +
        +
        +
        + +
        +
        Closing
        +

        The pattern in every engagement

        +
        +
          +
        • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
          • +
        • An experiment anchors every observational model: calibration is the product, not a luxury.
          • +
        • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
          • +
        + + + + + + +
        Agent, on adversarial MMM dataResult
        Vanilla coding agentFit a model, recommended budget reallocations. {{labs.dl_vanilla}}
        PyMC Labs' Decision Lab{{labs.dl_explored}} Returned: "{{labs.dl_verdict}}"
        +
        Even the machines know the punchline
        + The honest system's best answer was Part 2's closing advice: run the experiment.
        +
        +
        + +
        +
        Closing
        +

        One breath

        +
        +
        The pattern to take home
        + The toolkit a Bayesian consultancy sells: counterfactuals, calibrated by experiments, priced as probabilities.
        +
          +
        • Read the cases: pymc-labs.com/blog-posts: every number in this deck is pinned to a post, listed on the next slide.
          • +
        • Say hello: both authors consult for PyMC Labs; the notebooks behind this session are the course repository.
          • +
        +
        +
        + + +
        +
        Backup
        + Backup · Sources +

        Every number, pinned

        +
        Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
        +
        + + + + + +
        SourceFacts pinned
        +
        +
        + +
        + + + +
        + Causal Marketing · SDA Bocconi · Causal Inference in the Wild + + + + 1 / 1 +
        +
        +

        Contents

          + + + + diff --git a/causal-marketing-pymc/apps/verify_labs_deck.py b/causal-marketing-pymc/apps/verify_labs_deck.py index 1b0c0ff..82a5333 100644 --- a/causal-marketing-pymc/apps/verify_labs_deck.py +++ b/causal-marketing-pymc/apps/verify_labs_deck.py @@ -41,8 +41,8 @@ CHAR_BUDGET_HARD = True -def load_deck(): - html = DECK.read_text() +def load_deck(deck: Path = DECK): + html = deck.read_text() i = html.index("/*__DATA__*/") j = html.index("{", i) data, _ = json.JSONDecoder().raw_decode(html, j) @@ -83,8 +83,8 @@ def norm(s): return re.sub(r"[\s  ]+", " ", s) -def main() -> int: - html, data, slides, slides_visible = load_deck() +def main(deck: Path = DECK) -> int: + html, data, slides, slides_visible = load_deck(deck) pins = json.loads(DATAFILE.read_text()) env = make_env(data) claims = yaml.safe_load(CLAIMS.read_text()) @@ -166,4 +166,9 @@ def report(claim_id, ok, msg): if __name__ == "__main__": - sys.exit(main()) + import argparse + + ap = argparse.ArgumentParser(description="Verify a labs-claims deck against apps/labs_claims.yaml.") + ap.add_argument("--deck", type=Path, default=DECK, help="deck to verify (default: labs_slides.html)") + a = ap.parse_args() + sys.exit(main(a.deck.resolve())) diff --git a/causal-marketing-pymc/apps/verify_unified_deck.py b/causal-marketing-pymc/apps/verify_unified_deck.py new file mode 100644 index 0000000..3e4e4c0 --- /dev/null +++ b/causal-marketing-pymc/apps/verify_unified_deck.py @@ -0,0 +1,36 @@ +"""Verify the unified session deck (apps/unified_slides.html) end to end. + +Two registries, one deck: + - apps/geo_claims.yaml via verify_geo_deck (--allow-missing-slides semantics: claims + anchored to full-deck slides absent from the short arc are skipped, exactly as for + apps/geo_lift_sh.html); + - apps/labs_claims.yaml via verify_labs_deck (also carries the deck-wide sweeps: token + residue, ZERO em-dashes outside MathJax, no raw currency inside math, per-slide + visible-length budget, pin hygiene). + +Run: .venv/bin/python apps/verify_unified_deck.py (exit 1 on any FAIL) +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import verify_geo_deck as vg +import verify_labs_deck as vl + +HERE = Path(__file__).resolve().parent +DECK = HERE / "unified_slides.html" + + +def main() -> int: + if not DECK.exists(): + sys.exit(f"FAIL: {DECK} missing — run apps/build_unified_slides.py first.") + print("== geo claims (short-arc subset) " + "=" * 46) + rc_geo = vg.main(deck=DECK, allow_missing_slides=True) + print("== labs claims + deck-wide sweeps " + "=" * 45) + rc_labs = vl.main(deck=DECK) + return 1 if (rc_geo or rc_labs) else 0 + + +if __name__ == "__main__": + sys.exit(main())