From 9c66d3c36b77893f379405737d5cc86e95c93b99 Mon Sep 17 00:00:00 2001 From: Alexander Fengler Date: Tue, 21 Jul 2026 17:10:30 -0400 Subject: [PATCH 1/5] some improvements --- .../scratchpad/assemble_labs_src.py | 80 +++++++ .../scratchpad/labs_figures_fragment.js | 172 ++++++++++++++ .../scratchpad/labs_slides_fragment.html | 223 ++++++++++++++++++ 3 files changed, 475 insertions(+) create mode 100644 causal-marketing-pymc/scratchpad/assemble_labs_src.py create mode 100644 causal-marketing-pymc/scratchpad/labs_figures_fragment.js create mode 100644 causal-marketing-pymc/scratchpad/labs_slides_fragment.html diff --git a/causal-marketing-pymc/scratchpad/assemble_labs_src.py b/causal-marketing-pymc/scratchpad/assemble_labs_src.py new file mode 100644 index 0000000..ac5a0da --- /dev/null +++ b/causal-marketing-pymc/scratchpad/assemble_labs_src.py @@ -0,0 +1,80 @@ +"""One-time assembler for apps/labs_slides_src.html (the business-applications closer deck). + +Splices the SHARED CHROME out of apps/iv_slides_src.html (CSS, MathJax config, footer/nav/TOC +DOM, svg helpers, poll wiring, deck-navigation IIFE, the two base64 logo chips) and combines it +with the labs deck's own slides (scratchpad/labs_slides_fragment.html) and figures +(scratchpad/labs_figures_fragment.js). + +After this runs once, apps/labs_slides_src.html is the CANONICAL editable source; edit it +directly and rebuild with `make html-labs`. Re-running this assembler OVERWRITES it from the +fragments, so only re-run if you deliberately want to re-derive the chrome. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +APPS = HERE.parent / "apps" +IV = (APPS / "iv_slides_src.html").read_text() +SLIDES = (HERE / "labs_slides_fragment.html").read_text() +FIGURES = (HERE / "labs_figures_fragment.js").read_text() +OUT = APPS / "labs_slides_src.html" + + +def cut(s: str, start: str, end: str, *, incl_start=True, incl_end=False) -> str: + i = s.index(start) + j = s.index(end, i) + a = i if incl_start else i + len(start) + b = j + len(end) if incl_end else j + return s[a:b] + + +# 1 · head + CSS + MathJax config, up to (excluding) the first slide section +head = IV[: IV.index("")] +head = head.replace( + "Instrumental Variables: what is one ad exposure worth? (slides)", + "Causal Inference in the Wild: PyMC Labs cases (slides)", +) + +# 2 · the two logo chips (giant base64 lines on the IV title slide), spliced verbatim +logo_lines = [ln.strip() for ln in IV.splitlines() if '' in ln] +if len(logo_lines) != 2: + sys.exit(f"FAIL: expected 2 logo-chip lines in iv_slides_src.html, found {len(logo_lines)}") +slides = SLIDES.replace("__PYMC_LOGO__", " " + logo_lines[0]).replace( + "__LABS_LOGO__", " " + logo_lines[1] +) +if "__" in re.sub(r"", "", slides.replace("__SOURCES", "")): # loose guard + pass # data-* attrs etc. are fine; real placeholder misses are caught by the build + +# 3 · chrome DOM after SLIDES-END: deck close, navzones, footer, pbar, tocOv, script open, +# DATA marker line (stop before the IV N-map line) +mid = cut(IV, " +
+
Causal inference for marketing · SDA Bocconi · closing act
+

Causal Inference in the Wild

+
Drawing from real PyMC Labs client engagements.
+
+
Francesco Muia
PhD in Theoretical Physics, EMBA. Consultant for PyMC Labs and Brown University.
francesco.muia@pymc-labs.com
+
Alexander Fengler
PhD in Statistics. Postdoc at Brown University and consultant for PyMC Labs.
alexander.fengler@pymc-labs.com
+
+
+__PYMC_LOGO__ +__LABS_LOGO__ +
+
+ + +
+
Case 1 · Colgate-Palmolive
+

You are the consultant

+
The counterfactuals you built today are what clients buy. First test: route a real call.
+
+
+
✋ Poll
+
Colgate-Palmolive calls: "Our new toothpaste launched nationally last quarter. No holdout, no test market. Is it stealing share from competitors, or from our own brands?" Which tool from today do you reach for first?
+
+ + + + +
+ +
C. A national launch leaves nothing to randomize and no market to difference against: A and B need a control group that does not exist, and D's instrument does not exist for a shelf that changed everywhere at once. What remains is this morning's move, run in time instead of space: fit the world before the launch, project it forward, read the gap. PyMC Labs sold exactly that projection; the next slide shows it.
+
+
+
+ +
+
Case 1 · Colgate-Palmolive
+

Colgate-Palmolive: incremental, or cannibalistic?

+
Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The launch verdict is the split.
+
+
+
+
The launch, and the world without it schematic
+
+
Illustrative shape of the engagement's counterfactual read, not client data: fit the pre-period, project it forward, price the gap.
+
+
+
+
The brief, in their words
+ "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
+
    +
  • The client: Colgate-Palmolive came to PyMC Labs in {{labs.colgate_year}}, in a market estimated at {{labs.market}}.
    • +
  • The method: a multivariate Bayesian interrupted time series: nb10's machinery pointed at a product instead of a market, later extended to a nested-logit choice model.
    • +
  • The grading: on simulated data the model recovers a planted {{labs.colgate_truth}} incrementality as a {{labs.colgate_ci_level}} interval of {{labs.colgate_ci}}: the recover-the-truth contract you saw all day, run commercially.
    • +
+
+
+
+ +
+
Case 1 · Colgate-Palmolive · open floor
+

What would break it?

+
+
+
🗣 Open floor · 2 minutes
+
You are Colgate's CMO. The incrementality estimate you just saw (the share of the new product's sales that are genuinely new, not cannibalized) decides the launch review. Name one real-world event that would make it wrong.
+ +
A second launch. When another product entered the estimation window, the same machinery reported {{labs.fail_range}} incrementality against a planted truth of {{labs.fail_truth}}: the counterfactual absorbed part of the very effect it was meant to isolate. An honest consultancy publishes exactly this: the {{labs.fail_range}} miss is printed in the same post as the win. This morning's version of the disease was spillover; the defence is design, not statistics.
+
+
    +
  • What the model learns: everything before the launch defines "normal growth", and the projection (red) extrapolates that normal forward.
    • +
  • What a second launch does: inside the window it becomes part of "normal", so the projection rises too fast and under-credits the true lift; after the launch it inflates the observed line instead, and over-credits.
    • +
+
+
Break it yourself: slide a second launch into the window schematic
+
+
+ + + +
+
Same schematic world as the previous slide. The real case reported {{labs.fail_range}} against a truth of {{labs.fail_truth}}.
+
+
+
+ + +
+
Case 2 · HelloFresh · the tool, and its failure mode
+

Why calibrate? A model alone can rank channels backwards

+
Before the HelloFresh story, the tool it relies on: a warm-up from PyMC Labs' published calibration tutorial.
+
+
    +
  • The tool: a marketing-mix model (MMM) explains total sales as the sum of per-channel contributions, fit on observational spend data: no experiment anywhere in it.
    • +
  • The grading: the tutorial plants a truth: return on ad spend (ROAS, sales per unit of spend) of {{labs.roas_x1}} for channel x1 against {{labs.roas_x2}} for x2, so x2 is {{labs.roas_gap_words}}.
    • +
  • The experiment: a lift test nudges one channel's spend by a known amount and measures the sales change it causes: a small randomized ground-truth reading for that channel.
    • +
  • The repair: {{labs.lift_tests_n}} per channel, entered into the likelihood, recover both values: the experiment is the model's anchor, exactly Ch 9's role.
    • +
+
+
One MMM on observational spend alone, one planted truth, one inversion baked from the tutorial
+
+
+
Left: the ranking the uncalibrated model reported. Right: the planted truth the experiments recover.
+
+
The inversion
+ Fit on observational data alone, the baseline model ranked x1 above x2: {{labs.roas_wrong_ranking}}.
+
+
+ +
+
Case 2 · HelloFresh
+

HelloFresh runs the loop, at industrial scale

+
+
+
    +
  • The loop: MMM priors fed by field experiments such as {{labs.hf_priors_experiments}}; a {{labs.hf_var}} cut in prediction variance.
    • +
  • On stage: the panel's own agenda: Bayesian MMM can be {{labs.hf_panel_calibration}}.
    • +
  • The experiment supply: a pipeline handling {{labs.hf_thousands}}: {{labs.hf_test_types}} campaigns run simultaneously, the overnight batch down from {{labs.hf_batch}}; the Criteo experiment you saw this afternoon ({{labs.criteo_rows}} users) sits in exactly this regime.
    • +
+
The supply chain, for Ch 13
+ The experiments a company already runs are its instrument supply: a randomized encouragement is the instrument for the exposure you cannot randomize.
+
+
+
+
The loop you learned today
+
+
The model runs always-on; the experiment disciplines it; the counterfactual reads the experiment out.
+
+
+
+
+ + +
+
Case 3 · Nürnberger Versicherung
+

Price the engagement

+
A German insurer, last-touch attribution, and a funnel-aware causal MMM: nb04's mediation chain, in production.
+
+
+
✋ Poll
+
Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over {{labs.cpl_window}} of model-guided spend, cost per lead (CPL) moved by how much?
+
+ + + + +
+ +
C. "This year we were able to drive the CPL down by {{labs.cpl}}, which is very, very good" (Philip Herp, Nürnberger Versicherung). The mechanism is the lesson: under GDPR, {{labs.gdpr_sentence}}, so last-touch under-credited the upper funnel and budget followed {{labs.herp_attribution_quote}}. The funnel model measured what video spend causes downstream, and the client is scaling it into {{labs.nurn_2026}}.
+
+
The client's bar for belief
+ "{{labs.trust_quote}}"
+
+
+ + +
+
Closing · Provenance
+

The tools were the product too

+
+
    +
  • CausalPy: {{labs.causalpy_methods}} in one open-source package: this morning's method and its quasi-experimental family, industrialized by PyMC Labs. The IV estimator you ran this afternoon joined the package later.
    • +
  • Its launch example: individual exposure to a TV campaign {{labs.causalpy_tv}}, yet its causal impact remains a core business need: the sentence both of today's lectures opened with.
    • +
  • pymc-marketing: nb06's MMM library; one client's {{labs.bolt_pr}} came back as a pull request (Bolt).
    • +
  • Webinars and content: the consultancy's own webinar walks geo-experimentation, MMM, synthetic control, difference-in-differences, regression discontinuity and {{labs.webinar_agenda}}: the syllabus you just finished.
    • +
+
+ CausalPy + PyMC-Marketing +
+
+
+ +
+
Closing
+

The pattern in every engagement

+
+
    +
  • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
    • +
  • An experiment anchors every observational model: calibration is the product, not a luxury.
    • +
  • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
    • +
+ + + + + + +
Agent, on adversarial MMM dataResult
Vanilla coding agentFit a model, recommended budget reallocations. {{labs.dl_vanilla}}
PyMC Labs' Decision Lab{{labs.dl_explored}} Returned: "{{labs.dl_verdict}}"
+
Even the machines know the punchline
+ The honest system's best answer was this morning's closing advice: run the experiment.
+
+
+ +
+
Closing
+

One breath

+
+
What you now hold
+ The toolkit a Bayesian consultancy sells: counterfactuals, calibrated by experiments, priced as probabilities.
+
    +
  • Read the cases: pymc-labs.com/blog-posts: every number in this deck is pinned to a post, listed on the next slide.
    • +
  • Say hello: both authors consult for PyMC Labs; the notebooks behind all three decks are the course repository.
    • +
+
+
+ + +
+
Backup
+ Backup · Sources +

Every number, pinned

+
Facts retrieved and pinned 2026-07-19; apps/labs_deck_data.json carries the exact quote for each.
+
+ + + + + +
SourceFacts pinned
+
+
From 263c107f0d68f78a93ddaba923ef04985ec20ee2 Mon Sep 17 00:00:00 2001 From: Alexander Fengler Date: Tue, 21 Jul 2026 18:07:29 -0400 Subject: [PATCH 2/5] Unified deck slide 2: PyMC Labs logo chip + the ecosystem graph The "PyMC Labs: what we do" slide gains the Labs logo as a white chip top-right (absolutely positioned, costing no vertical space; explicit background so the dark wordmark survives dark mode) and a left-to-right ecosystem graph: PyTensor (compute engine) -> PyMC (probabilistic programming) -> PyMC-Marketing (MMM, CLV), CausalPy (quasi-experiments), and a dashed Bespoke node for customer-specific builds. The graph shares a row with the clients callout so the slide stays above the fold; theme-aware IIFE registered in __redraw. Build note: this checkout has no book/build shards, so the identical edit was applied to both the canonical src and the committed built deck (no tokens or DATA touched; verified byte-identical insertions). The next shard-full `make html-unified` reproduces it from src. Verified: labs claims + deck-wide sweeps 90 checks 0 failures; node --check OK; headless DOM 67 slides, 0 mjx-merror, 0 JS errors; slide 2 screenshotted in light and dark. Co-Authored-By: Claude Fable 5 --- .../apps/unified_slides.html | 38 ++++++++++++++++++- .../apps/unified_slides_src.html | 38 ++++++++++++++++++- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/causal-marketing-pymc/apps/unified_slides.html b/causal-marketing-pymc/apps/unified_slides.html index 11037dd..e9e6093 100644 --- a/causal-marketing-pymc/apps/unified_slides.html +++ b/causal-marketing-pymc/apps/unified_slides.html @@ -236,6 +236,7 @@

Causal Inference in the Wild

+ PyMC Labs
Opening · Who is talking

PyMC Labs: what we do

A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
@@ -253,8 +254,11 @@

PyMC Labs: what we do

Enablement & trainingWorkshops and upskilling on Bayesian methods and the toolingAnalytics teams standardizing on PyMC -
The clients in this session
- Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
+
+
+
The clients in this session
+ Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
+
▸ Areas of interest, and where we work
  • Areas: marketing-mix modeling and media measurement, causal inference, demand forecasting and pricing, experimentation and A/B testing at scale, applied Bayesian modeling.
    • @@ -3473,6 +3477,36 @@

    Every number, pinned

    draw(); window.__redraw.push(draw); })(); +/* ==== fig: the PyMC ecosystem stack (slide 2) ==== */ +(function(){ + const svg=document.getElementById('svgEco'); if(!svg)return; + function draw(){ + clr(svg); const c=COL(); + function node(x,y,w,label,sub,col,dash){ + const attrs={x:x-w/2,y:y-19,width:w,height:38,rx:9,fill:'none',stroke:col,'stroke-width':2}; + if(dash)attrs['stroke-dasharray']='5 4'; + svg.appendChild(el('rect',attrs)); + svg.appendChild(el('text',{x:x,y:y-1,'text-anchor':'middle','font-size':12,'font-weight':'700',fill:c.navy},label)); + if(sub)svg.appendChild(el('text',{x:x,y:y+13,'text-anchor':'middle','font-size':9,fill:c.muted},sub)); + } + function arrow(x1,y1,x2,y2){ + svg.appendChild(el('line',{x1,y1,x2,y2,stroke:c.faint,'stroke-width':1.6})); + const a=Math.atan2(y2-y1,x2-x1); + svg.appendChild(el('path',{d:`M${x2} ${y2} L${x2-8*Math.cos(a-0.4)} ${y2-8*Math.sin(a-0.4)} L${x2-8*Math.cos(a+0.4)} ${y2-8*Math.sin(a+0.4)} Z`,fill:c.faint})); + } + node(95,66,120,'PyTensor','compute engine',c.grey); + node(285,66,155,'PyMC','probabilistic programming',c.blue); + node(540,22,170,'PyMC-Marketing','MMM, CLV',c.green); + node(540,66,170,'CausalPy','quasi-experiments',c.orange); + node(540,110,170,'Bespoke','customer-specific builds',c.gold,true); + arrow(157,66,205,66); + arrow(365,60,453,26); + arrow(365,66,453,66); + arrow(365,72,453,106); + } + draw(); window.__redraw.push(draw); +})(); + /* ==== fig: break the counterfactual yourself (probe slide) ==== */ (function(){ const svg=document.getElementById('svgBreak'); if(!svg)return; diff --git a/causal-marketing-pymc/apps/unified_slides_src.html b/causal-marketing-pymc/apps/unified_slides_src.html index 942903e..b342acc 100644 --- a/causal-marketing-pymc/apps/unified_slides_src.html +++ b/causal-marketing-pymc/apps/unified_slides_src.html @@ -236,6 +236,7 @@

    Causal Inference in the Wild

    + PyMC Labs
    Opening · Who is talking

    PyMC Labs: what we do

    A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
    @@ -253,8 +254,11 @@

    PyMC Labs: what we do

    Enablement & trainingWorkshops and upskilling on Bayesian methods and the toolingAnalytics teams standardizing on PyMC -
    The clients in this session
    - Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
    +
    +
    +
    The clients in this session
    + Colgate-Palmolive, HelloFresh, Nürnberger Versicherung (and a Bolt cameo): you meet three of them in the next ten minutes.
    +
    ▸ Areas of interest, and where we work
    • Areas: marketing-mix modeling and media measurement, causal inference, demand forecasting and pricing, experimentation and A/B testing at scale, applied Bayesian modeling.
      • @@ -3461,6 +3465,36 @@

      Every number, pinned

      draw(); window.__redraw.push(draw); })(); +/* ==== fig: the PyMC ecosystem stack (slide 2) ==== */ +(function(){ + const svg=document.getElementById('svgEco'); if(!svg)return; + function draw(){ + clr(svg); const c=COL(); + function node(x,y,w,label,sub,col,dash){ + const attrs={x:x-w/2,y:y-19,width:w,height:38,rx:9,fill:'none',stroke:col,'stroke-width':2}; + if(dash)attrs['stroke-dasharray']='5 4'; + svg.appendChild(el('rect',attrs)); + svg.appendChild(el('text',{x:x,y:y-1,'text-anchor':'middle','font-size':12,'font-weight':'700',fill:c.navy},label)); + if(sub)svg.appendChild(el('text',{x:x,y:y+13,'text-anchor':'middle','font-size':9,fill:c.muted},sub)); + } + function arrow(x1,y1,x2,y2){ + svg.appendChild(el('line',{x1,y1,x2,y2,stroke:c.faint,'stroke-width':1.6})); + const a=Math.atan2(y2-y1,x2-x1); + svg.appendChild(el('path',{d:`M${x2} ${y2} L${x2-8*Math.cos(a-0.4)} ${y2-8*Math.sin(a-0.4)} L${x2-8*Math.cos(a+0.4)} ${y2-8*Math.sin(a+0.4)} Z`,fill:c.faint})); + } + node(95,66,120,'PyTensor','compute engine',c.grey); + node(285,66,155,'PyMC','probabilistic programming',c.blue); + node(540,22,170,'PyMC-Marketing','MMM, CLV',c.green); + node(540,66,170,'CausalPy','quasi-experiments',c.orange); + node(540,110,170,'Bespoke','customer-specific builds',c.gold,true); + arrow(157,66,205,66); + arrow(365,60,453,26); + arrow(365,66,453,66); + arrow(365,72,453,106); + } + draw(); window.__redraw.push(draw); +})(); + /* ==== fig: break the counterfactual yourself (probe slide) ==== */ (function(){ const svg=document.getElementById('svgBreak'); if(!svg)return; From 8d5bb507a04891a03cd233b8168cabcbce1d59f5 Mon Sep 17 00:00:00 2001 From: Alexander Fengler Date: Tue, 21 Jul 2026 18:24:07 -0400 Subject: [PATCH 3/5] Unified deck round 2 (Alexander): Case 3 fleshed out, loop de-confused, wording passes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - S2: "What we sell" and "How we work" bullets rewritten to the reviewer's text (end-to-end data science; the client keeps the model and surrounding functionality transparently, not the core libraries). - S3: the method note now reads "Our lectures will center around classical methods to stay didactically coherent. We will only hint at what the Bayesian approach affords." - S5 Colgate: "Bayesian" dropped from the method line (the time series carries the point); the grading bullet now explains recovery plainly (run on simulated sales with the truth fixed at 50% by construction, estimate 49-59%: find a planted answer before being trusted on the real one); the sub closes with "how much of its sales is which?" instead of "the launch verdict is the split". - S7: the everyday-tool card now names the tool: a marketing-mix model (MMM), so the loop figure's MMM node is defined before use. - S8: the loop figure loses the synthetic-control node (not yet taught at this point in the session); now a two-node MMM <-> geo-experiment loop, closing with "reading a geo experiment out is its own craft: that method is Part 2 of this session". - Case 3 fleshed out to a two-slide case: a new "Nürnberger: steering by the last click" card slide defines last-touch attribution and the funnel-aware causal MMM (GDPR mechanism + Herp quote) BEFORE the poll; the poll becomes concept-grounded ("which channels does last-touch systematically under-credit?", correct: upper-funnel awareness) with per-option why-notes, and the CPL outcome moves to the reveal. data-t kept as "Poll: price the engagement" so shared claims stay anchored across both decks; colgate_method claim relaxed to the fragment both decks share. Both files (src with tokens, built with resolved pins) edited identically; no shards needed. Verified: labs claims + sweeps green on BOTH decks (90/90 each); node --check OK; 68 slides, 0 mjx-merror, 0 JS errors; poll E end-to-end (wrong pick dimmed, B marked correct); changed slides screenshotted. Co-Authored-By: Claude Fable 5 --- causal-marketing-pymc/apps/labs_claims.yaml | 2 +- .../apps/unified_slides.html | 88 +++++++++++-------- .../apps/unified_slides_src.html | 88 +++++++++++-------- 3 files changed, 101 insertions(+), 77 deletions(-) diff --git a/causal-marketing-pymc/apps/labs_claims.yaml b/causal-marketing-pymc/apps/labs_claims.yaml index 4289e08..95bf41f 100644 --- a/causal-marketing-pymc/apps/labs_claims.yaml +++ b/causal-marketing-pymc/apps/labs_claims.yaml @@ -25,7 +25,7 @@ - id: colgate_method slides: ["Colgate: incremental or cannibalistic"] - text: "multivariate Bayesian interrupted time series" + text: "interrupted time series" source: labs.colgate_counterfactual_quote - id: colgate_recovery diff --git a/causal-marketing-pymc/apps/unified_slides.html b/causal-marketing-pymc/apps/unified_slides.html index e9e6093..37dc7df 100644 --- a/causal-marketing-pymc/apps/unified_slides.html +++ b/causal-marketing-pymc/apps/unified_slides.html @@ -242,9 +242,9 @@

      PyMC Labs: what we do

      A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
        -
      • What we sell: senior modeling expertise, not software licenses.
        • +
      • What we sell: Senior modeling expertise, end-to-end data science, not a software license.
      • Open source is the top of the funnel: the libraries build reach; the consulting monetizes the expertise behind them.
        • -
      • How we work: small senior teams, starting from the client's decision; the client keeps the model, not a black box.
        • +
      • How we work: small senior teams; the client keeps the model and surrounding functionality transparently, not the core libraries.
      @@ -332,7 +332,7 @@

      Bayesian vs frequentist, in one slide

      Why marketing analytics is going Bayesian
      Marketing data is short, noisy, and highly correlated, and every model ends in a spend decision, which is exactly where priors and full uncertainty pay off.
      For today, this is a curiosity
      - The entire lecture runs on classical methods, and every number you will see is frequentist; we surface the Bayesian read only as a flavour at the edges, never as the load-bearing tool.
      + Our lectures will center around classical methods to stay didactically coherent. We will only hint at what the Bayesian approach affords.
      ▸ Where it is already the default (and where it is not)
      • Marketing mix modeling: the modern open-source MMM tools are Bayesian (Google's Meridian, PyMC-Marketing); with only two or three years of weekly data and correlated channels, plain regression is unstable, and priors on adstock, saturation, and ROI stabilize it.
        • @@ -382,7 +382,7 @@

        You are the consultant

        Case 1 · Colgate-Palmolive

        Colgate-Palmolive: incremental, or cannibalistic?

        -
        Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The launch verdict is the split.
        +
        Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The question for the launch: how much of its sales is which?
        @@ -396,8 +396,8 @@

        Colgate-Palmolive: incremental, or cannibalistic?

        "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
        • The client: Colgate-Palmolive, 2023; a market estimated at $20.8 billion in 2023.
          • -
        • The method: a multivariate Bayesian interrupted time series: project the pre-launch world forward.
          • -
        • The grading: a planted 50% recovered as a 94% interval of 49-59%: recover a known truth first, then be believed.
          • +
        • The method: a multivariate interrupted time series: project the pre-launch world forward.
          • +
        • The grading: the model is first run on simulated sales where the true incrementality is fixed at 50% by construction; it estimates 49-59% (a 94% interval): it must find an answer we planted before we trust it on the real one.
        @@ -452,7 +452,7 @@

        Why calibrate? A model alone can rank channels backwards

        The settinga company spreads its budget across many ad channels (TV, search, social) and needs to know which ones actually pay back.
        -
        The everyday toola model reads years of spend-and-sales history and scores each channel's return on ad spend, cheaply and always-on.
        +
        The everyday tool: a marketing-mix model (MMM)it reads years of spend-and-sales history and scores each channel's return on ad spend, cheaply and always-on.
        The catch, and our jobhistory is not an experiment; a company spends more exactly when demand is already high, so the model can credit the season instead of the channel and rank them backwards.
        What we adda real experiment: nudge one channel's budget by a known amount, measure the sales it truly causes, and anchor the model to that number. This is calibration.
        @@ -493,25 +493,41 @@

        HelloFresh runs the loop, at industrial scale

        +
        +
        Case 3 · Nürnberger Versicherung
        +

        Nürnberger Versicherung: steering by the last click

        +
        A German insurance group, spending across the whole funnel: awareness video and demand generation up top, branded search at the bottom.
        +
        +
        +
        The old ruler: last-touch attributioncredit every sale to the last ad click before it; simple, standard, and blind to everything upstream of that click.
        +
        What broke itunder GDPR, customer journeys appeared artificially shortened: the last click was often the only click the tracker could still see.
        +
        The replacement: a funnel-aware causal MMMa marketing-mix model that encodes the funnel: upper-funnel spend creates demand that surfaces later in lower-funnel channels, so credit flows to the cause, not to the final click.
        +
        The stakesbudget follows the ruler: whatever the measurement under-credits, the spreadsheet de-funds.
        +
        +
        The client, before the fix
        + "We were optimizing for attribution mechanics instead of incremental business impact." (Philip Herp, Nürnberger Versicherung)
        +
        +
        +
        Case 3 · Nürnberger Versicherung
        -

        Price the engagement

        -
        A German insurer, last-touch attribution, and a funnel-aware causal MMM, in production.
        +

        Who does the last click cheat?

        +
        The terms are on the table; the poll asks you to use them.
        -
        +
        ✋ Poll
        -
        Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over June through November of model-guided spend, cost per lead (CPL) moved by how much?
        +
        Under GDPR-shortened tracking, which channels does last-touch attribution systematically under-credit?
        - - - - + + + +
        -
        C. "This year we were able to drive the CPL down by more than 27%, which is very, very good" (Philip Herp, Nürnberger Versicherung). Under GDPR, customer journeys appeared artificially shortened, so last-touch under-credited the upper funnel and budget followed attribution mechanics instead of incremental business impact. The client is scaling into full production for 2026.
        +
        B. Under GDPR, customer journeys appeared artificially shortened, so the last click was often the only visible step, the upper funnel vanished from the books, and budget followed attribution mechanics instead of incremental business impact. The funnel-aware model re-priced it: "This year we were able to drive the CPL down by more than 27%, which is very, very good" (Philip Herp), over June through November of model-guided spend, and cost per lead (CPL) is the price of one new customer lead. The client is scaling into full production for 2026.
        The client's bar for belief
        "Trust is not created by R² values. It is created when business reality matches model expectations."
        @@ -3444,35 +3460,31 @@

        Every number, pinned

        draw(); window.__redraw.push(draw); })(); -/* ==== fig: the MMM / experiment / synthetic-control loop (S7) ==== */ +/* ==== fig: the MMM / experiment calibration loop (S7) ==== */ (function(){ const svg=document.getElementById('svgLoop'); if(!svg)return; function draw(){ clr(svg); const c=COL(); - const nodes=[ - {x:170,y:52,w:150,label:'MMM',sub:'always-on model'}, - {x:88,y:212,w:150,label:'Geo experiment',sub:'episodic truth'}, - {x:252,y:212,w:150,label:'Synthetic control',sub:'reads it out'}, - ]; - function box(n,col){ - svg.appendChild(el('rect',{x:n.x-n.w/2,y:n.y-26,width:n.w,height:52,rx:10,fill:'none',stroke:col,'stroke-width':2})); - svg.appendChild(el('text',{x:n.x,y:n.y-5,'text-anchor':'middle','font-size':13,'font-weight':'700',fill:c.navy},n.label)); - svg.appendChild(el('text',{x:n.x,y:n.y+13,'text-anchor':'middle','font-size':10,fill:c.muted},n.sub)); + function box(x,y,w,label,sub,col){ + svg.appendChild(el('rect',{x:x-w/2,y:y-26,width:w,height:52,rx:10,fill:'none',stroke:col,'stroke-width':2})); + svg.appendChild(el('text',{x:x,y:y-5,'text-anchor':'middle','font-size':13,'font-weight':'700',fill:c.navy},label)); + svg.appendChild(el('text',{x:x,y:y+13,'text-anchor':'middle','font-size':10,fill:c.muted},sub)); } - box(nodes[0],c.blue); box(nodes[1],c.orange); box(nodes[2],c.green); + box(170,60,190,'MMM','always-on budget model',c.blue); + box(170,220,190,'Geo experiment','episodic ground truth',c.orange); function arrow(x1,y1,x2,y2){ svg.appendChild(el('line',{x1,y1,x2,y2,stroke:c.faint,'stroke-width':1.6})); const a=Math.atan2(y2-y1,x2-x1); svg.appendChild(el('path',{d:`M${x2} ${y2} L${x2-9*Math.cos(a-0.4)} ${y2-9*Math.sin(a-0.4)} L${x2-9*Math.cos(a+0.4)} ${y2-9*Math.sin(a+0.4)} Z`,fill:c.faint})); } - arrow(128,80,100,182); // MMM -> experiment (asks) - arrow(120,182,148,80); // experiment -> MMM (calibrates) - arrow(166,224,176,224); // experiment -> SC - svg.appendChild(el('text',{x:56,y:132,'font-size':10,fill:c.muted},'asks for')); - svg.appendChild(el('text',{x:56,y:144,'font-size':10,fill:c.muted},'ground truth')); - svg.appendChild(el('text',{x:152,y:132,'font-size':10,fill:c.muted},'calibrates')); - svg.appendChild(el('text',{x:152,y:144,'font-size':10,fill:c.muted},'(priors, lift tests)')); - svg.appendChild(el('text',{x:170,y:262,'text-anchor':'middle','font-size':10,fill:c.muted},'MMM: last slide · synthetic control: Part 2 · the loop: this session')); + arrow(120,86,120,194); + arrow(220,194,220,86); + svg.appendChild(el('text',{x:108,y:140,'font-size':10,fill:c.muted,'text-anchor':'end'},'asks for')); + svg.appendChild(el('text',{x:108,y:152,'font-size':10,fill:c.muted,'text-anchor':'end'},'ground truth')); + svg.appendChild(el('text',{x:232,y:140,'font-size':10,fill:c.muted},'calibrates')); + svg.appendChild(el('text',{x:232,y:152,'font-size':10,fill:c.muted},'(priors, lift tests)')); + svg.appendChild(el('text',{x:170,y:282,'text-anchor':'middle','font-size':10,fill:c.muted},'reading a geo experiment out is its own craft:')); + svg.appendChild(el('text',{x:170,y:294,'text-anchor':'middle','font-size':10,fill:c.muted},'that method is Part 2 of this session')); } draw(); window.__redraw.push(draw); })(); diff --git a/causal-marketing-pymc/apps/unified_slides_src.html b/causal-marketing-pymc/apps/unified_slides_src.html index b342acc..a46c509 100644 --- a/causal-marketing-pymc/apps/unified_slides_src.html +++ b/causal-marketing-pymc/apps/unified_slides_src.html @@ -242,9 +242,9 @@

        PyMC Labs: what we do

        A Bayesian modeling consultancy: custom decision-making models where off-the-shelf tools fall short, and the open-source libraries the field runs on (PyMC, PyMC-Marketing, CausalPy).
          -
        • What we sell: senior modeling expertise, not software licenses.
          • +
        • What we sell: Senior modeling expertise, end-to-end data science, not a software license.
        • Open source is the top of the funnel: the libraries build reach; the consulting monetizes the expertise behind them.
          • -
        • How we work: small senior teams, starting from the client's decision; the client keeps the model, not a black box.
          • +
        • How we work: small senior teams; the client keeps the model and surrounding functionality transparently, not the core libraries.
      EngagementWhat it isTypical client
      @@ -332,7 +332,7 @@

      Bayesian vs frequentist, in one slide

      Why marketing analytics is going Bayesian
      Marketing data is short, noisy, and highly correlated, and every model ends in a spend decision, which is exactly where priors and full uncertainty pay off.
      For today, this is a curiosity
      - The entire lecture runs on classical methods, and every number you will see is frequentist; we surface the Bayesian read only as a flavour at the edges, never as the load-bearing tool.
      + Our lectures will center around classical methods to stay didactically coherent. We will only hint at what the Bayesian approach affords.
      ▸ Where it is already the default (and where it is not)
      • Marketing mix modeling: the modern open-source MMM tools are Bayesian (Google's Meridian, PyMC-Marketing); with only two or three years of weekly data and correlated channels, plain regression is unstable, and priors on adstock, saturation, and ROI stabilize it.
        • @@ -382,7 +382,7 @@

        You are the consultant

        Case 1 · Colgate-Palmolive

        Colgate-Palmolive: incremental, or cannibalistic?

        -
        Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The launch verdict is the split.
        +
        Incremental: sales won from competitors or category growth. Cannibalistic: sales taken from your own products. The question for the launch: how much of its sales is which?
        @@ -396,8 +396,8 @@

        Colgate-Palmolive: incremental, or cannibalistic?

        "We need to estimate the counterfactual sales of all products would have been if the new product had not been introduced."
        • The client: Colgate-Palmolive, {{labs.colgate_year}}; a market estimated at {{labs.market}}.
          • -
        • The method: a multivariate Bayesian interrupted time series: project the pre-launch world forward.
          • -
        • The grading: a planted {{labs.colgate_truth}} recovered as a {{labs.colgate_ci_level}} interval of {{labs.colgate_ci}}: recover a known truth first, then be believed.
          • +
        • The method: a multivariate interrupted time series: project the pre-launch world forward.
          • +
        • The grading: the model is first run on simulated sales where the true incrementality is fixed at {{labs.colgate_truth}} by construction; it estimates {{labs.colgate_ci}} (a {{labs.colgate_ci_level}} interval): it must find an answer we planted before we trust it on the real one.
        @@ -452,7 +452,7 @@

        Why calibrate? A model alone can rank channels backwards

        The settinga company spreads its budget across many ad channels (TV, search, social) and needs to know which ones actually pay back.
        -
        The everyday toola model reads years of spend-and-sales history and scores each channel's return on ad spend, cheaply and always-on.
        +
        The everyday tool: a marketing-mix model (MMM)it reads years of spend-and-sales history and scores each channel's return on ad spend, cheaply and always-on.
        The catch, and our jobhistory is not an experiment; a company spends more exactly when demand is already high, so the model can credit the season instead of the channel and rank them backwards.
        What we adda real experiment: nudge one channel's budget by a known amount, measure the sales it truly causes, and anchor the model to that number. This is calibration.
        @@ -493,25 +493,41 @@

        HelloFresh runs the loop, at industrial scale

        +
        +
        Case 3 · Nürnberger Versicherung
        +

        Nürnberger Versicherung: steering by the last click

        +
        A German insurance group, spending across the whole funnel: awareness video and demand generation up top, branded search at the bottom.
        +
        +
        +
        The old ruler: last-touch attributioncredit every sale to the last ad click before it; simple, standard, and blind to everything upstream of that click.
        +
        What broke itunder GDPR, {{labs.gdpr_sentence}}: the last click was often the only click the tracker could still see.
        +
        The replacement: a funnel-aware causal MMMa marketing-mix model that encodes the funnel: upper-funnel spend creates demand that surfaces later in lower-funnel channels, so credit flows to the cause, not to the final click.
        +
        The stakesbudget follows the ruler: whatever the measurement under-credits, the spreadsheet de-funds.
        +
        +
        The client, before the fix
        + "We were optimizing for {{labs.herp_attribution_quote}}." (Philip Herp, Nürnberger Versicherung)
        +
        +
        +
        Case 3 · Nürnberger Versicherung
        -

        Price the engagement

        -
        A German insurer, last-touch attribution, and a funnel-aware causal MMM, in production.
        +

        Who does the last click cheat?

        +
        The terms are on the table; the poll asks you to use them.
        -
        +
        ✋ Poll
        -
        Nürnberger Versicherung replaced last-touch attribution steering with a funnel-aware causal MMM. Over {{labs.cpl_window}} of model-guided spend, cost per lead (CPL) moved by how much?
        +
        Under GDPR-shortened tracking, which channels does last-touch attribution systematically under-credit?
        - - - - + + + +
        -
        C. "This year we were able to drive the CPL down by {{labs.cpl}}, which is very, very good" (Philip Herp, Nürnberger Versicherung). Under GDPR, {{labs.gdpr_sentence}}, so last-touch under-credited the upper funnel and budget followed {{labs.herp_attribution_quote}}. The client is scaling into {{labs.nurn_2026}}.
        +
        B. Under GDPR, {{labs.gdpr_sentence}}, so the last click was often the only visible step, the upper funnel vanished from the books, and budget followed {{labs.herp_attribution_quote}}. The funnel-aware model re-priced it: "This year we were able to drive the CPL down by {{labs.cpl}}, which is very, very good" (Philip Herp), over {{labs.cpl_window}} of model-guided spend, and cost per lead (CPL) is the price of one new customer lead. The client is scaling into {{labs.nurn_2026}}.
        The client's bar for belief
        "{{labs.trust_quote}}"
        @@ -3432,35 +3448,31 @@

        Every number, pinned

        draw(); window.__redraw.push(draw); })(); -/* ==== fig: the MMM / experiment / synthetic-control loop (S7) ==== */ +/* ==== fig: the MMM / experiment calibration loop (S7) ==== */ (function(){ const svg=document.getElementById('svgLoop'); if(!svg)return; function draw(){ clr(svg); const c=COL(); - const nodes=[ - {x:170,y:52,w:150,label:'MMM',sub:'always-on model'}, - {x:88,y:212,w:150,label:'Geo experiment',sub:'episodic truth'}, - {x:252,y:212,w:150,label:'Synthetic control',sub:'reads it out'}, - ]; - function box(n,col){ - svg.appendChild(el('rect',{x:n.x-n.w/2,y:n.y-26,width:n.w,height:52,rx:10,fill:'none',stroke:col,'stroke-width':2})); - svg.appendChild(el('text',{x:n.x,y:n.y-5,'text-anchor':'middle','font-size':13,'font-weight':'700',fill:c.navy},n.label)); - svg.appendChild(el('text',{x:n.x,y:n.y+13,'text-anchor':'middle','font-size':10,fill:c.muted},n.sub)); + function box(x,y,w,label,sub,col){ + svg.appendChild(el('rect',{x:x-w/2,y:y-26,width:w,height:52,rx:10,fill:'none',stroke:col,'stroke-width':2})); + svg.appendChild(el('text',{x:x,y:y-5,'text-anchor':'middle','font-size':13,'font-weight':'700',fill:c.navy},label)); + svg.appendChild(el('text',{x:x,y:y+13,'text-anchor':'middle','font-size':10,fill:c.muted},sub)); } - box(nodes[0],c.blue); box(nodes[1],c.orange); box(nodes[2],c.green); + box(170,60,190,'MMM','always-on budget model',c.blue); + box(170,220,190,'Geo experiment','episodic ground truth',c.orange); function arrow(x1,y1,x2,y2){ svg.appendChild(el('line',{x1,y1,x2,y2,stroke:c.faint,'stroke-width':1.6})); const a=Math.atan2(y2-y1,x2-x1); svg.appendChild(el('path',{d:`M${x2} ${y2} L${x2-9*Math.cos(a-0.4)} ${y2-9*Math.sin(a-0.4)} L${x2-9*Math.cos(a+0.4)} ${y2-9*Math.sin(a+0.4)} Z`,fill:c.faint})); } - arrow(128,80,100,182); // MMM -> experiment (asks) - arrow(120,182,148,80); // experiment -> MMM (calibrates) - arrow(166,224,176,224); // experiment -> SC - svg.appendChild(el('text',{x:56,y:132,'font-size':10,fill:c.muted},'asks for')); - svg.appendChild(el('text',{x:56,y:144,'font-size':10,fill:c.muted},'ground truth')); - svg.appendChild(el('text',{x:152,y:132,'font-size':10,fill:c.muted},'calibrates')); - svg.appendChild(el('text',{x:152,y:144,'font-size':10,fill:c.muted},'(priors, lift tests)')); - svg.appendChild(el('text',{x:170,y:262,'text-anchor':'middle','font-size':10,fill:c.muted},'MMM: last slide · synthetic control: Part 2 · the loop: this session')); + arrow(120,86,120,194); + arrow(220,194,220,86); + svg.appendChild(el('text',{x:108,y:140,'font-size':10,fill:c.muted,'text-anchor':'end'},'asks for')); + svg.appendChild(el('text',{x:108,y:152,'font-size':10,fill:c.muted,'text-anchor':'end'},'ground truth')); + svg.appendChild(el('text',{x:232,y:140,'font-size':10,fill:c.muted},'calibrates')); + svg.appendChild(el('text',{x:232,y:152,'font-size':10,fill:c.muted},'(priors, lift tests)')); + svg.appendChild(el('text',{x:170,y:282,'text-anchor':'middle','font-size':10,fill:c.muted},'reading a geo experiment out is its own craft:')); + svg.appendChild(el('text',{x:170,y:294,'text-anchor':'middle','font-size':10,fill:c.muted},'that method is Part 2 of this session')); } draw(); window.__redraw.push(draw); })(); From 2de55f2e2efe5c47e82b6036ddcd8791192a88b9 Mon Sep 17 00:00:00 2001 From: Alexander Fengler Date: Tue, 21 Jul 2026 20:03:36 -0400 Subject: [PATCH 4/5] Unified deck round 3 (Alexander): trims, the metros picture, and The END MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Geo arc: - Boardroom slide gains a schematic metros picture (29 grey donor markets, the treated metro highlighted with campaign arcs); the "what unbiased means" fold is dropped (raised a question the slide never asked); the two callouts share a row to keep everything above the fold. - Potential-outcomes slide drops the "estimation target, not a rhetorical question" line. - Simulate-the-world: legend corrected to "Black: the treated metro"; the before/after means overlay is now ON by default. - "The estimator" slide deleted (Abadie's idea now flows straight to the convex hull). - "What the constraint buys" simplified into "The synthetic twin, graded against the truth": OLS synthetic and the effective-donors machinery removed from figure and prose; one figure showing the twin tracking the true counterfactual, with pre/post RMSE in the caption. - Act III reduced to the two falsification slides; poll, placebo-in-space, test inversion, and the statistics summary moved to Backup, so the falsifications now hand straight to the money act. - "Why not just forecast it?", the two DiD-algebra slides, and "Why the division is forced" also moved to Backup (kickers prefixed, TOC groups them). IV arc: - "The plan for Part 2" callout retitled "Going forward". - The IV-estimate slide drops the "no model of intent, controls, or machine learning" line. Closing: - The provenance slide's logo row now runs PyTensor · PyMC · PyMC-Marketing · CausalPy (PyTensor vector logo vendored from pymc-devs/pytensor, base64). - "One breath" replaced by a title-style "The END": both authors' contacts, the breadth message (Bayesian inference as a vast toolset beyond causal inference), and a get-in-touch callout. Deck: 68 -> 67 slides (one deletion, eight moves to Backup). Both files edited identically (no tokens in touched regions). Verified: labs claims + sweeps 90/90; node --check OK; 67 slides, 0 mjx-merror, 0 JS errors; changed slides screenshotted (metros, means-on simulate, twin-vs-truth, logos row, The END). Co-Authored-By: Claude Fable 5 --- .../apps/unified_slides.html | 875 +++++++++--------- .../apps/unified_slides_src.html | 851 +++++++++-------- 2 files changed, 840 insertions(+), 886 deletions(-) diff --git a/causal-marketing-pymc/apps/unified_slides.html b/causal-marketing-pymc/apps/unified_slides.html index 37dc7df..3586b48 100644 --- a/causal-marketing-pymc/apps/unified_slides.html +++ b/causal-marketing-pymc/apps/unified_slides.html @@ -596,6 +596,11 @@

        The boardroom question

      +
      +
      One metro got the campaign; twenty-nine watched schematic
      +
      +
      +
      The method this data calls for
      • One treated unit, no A/B test possible: synthetic control's home ground.
        • @@ -606,18 +611,8 @@

        The boardroom question

      • δ = the share of the pilot's per-euro lift that survives national rollout.
      • The €4M call reduces to one question: is δ large enough to still clear the cost?
      -
      ▸ What "unbiased" means here, precisely, and where it would break
      -

      The estimator subtracts a reconstructed counterfactual from the treated metro's post-launch sales, and bias is its expected gap from the true lift:

      -
      - \[ \hat\tau \;=\; \bar Y_{1,\text{post}} - \hat Y_{1,\text{post}}(0), \qquad \text{bias} \;\equiv\; \mathbb{E}[\hat\tau] - \tau \]
      -
        -
      • The counterfactual is the synthetic twin, built to reproduce the metro's pre-launch record (its level and its factor loadings) week by week.
        • -
      • Unbiasedness rests on one assumption: that pre-launch match would have continued through the post window had the campaign never run.
        • -
      • This needs no story about why the metro was chosen. Whatever pre-launch trait drove the pick, the twin already carries it, because it was fitted to match it: selection on anything visible before launch cannot tilt the estimate.
        • -
      • Bias enters only through what the pre-launch record cannot see: if the launch were timed on private knowledge of a coming local boom, the twin could not anticipate it and the gap would credit the boom to the campaign. The later placebo-in-time test hunts exactly that.
        • -
      -
      +
      @@ -730,7 +725,6 @@

      Causal inference is a missing-data problem

    • Subscript says which market, argument says which world (1 = campaign, 0 = none): distinct axes that happen to share the digit 1.
    • \(w_j\): the donor weights the synthetic control will choose so \(\sum_{j\ge 2} w_j Y_{jt}\) rebuilds the missing \(Y_{1t}(0)\).
      • -

      "What would this metro have sold anyway?" is an estimation target, not a rhetorical question.

      @@ -781,7 +775,7 @@

      Simulate the world yourself

      Take a look at the counterfactual.
      -
      ▶ LIVE The equations of the previous slide. Grey: donor markets. White: the treated metro.
      +
      ▶ LIVE The equations of the previous slide. Grey: donor markets. Black: the treated metro.
      @@ -789,7 +783,7 @@

      Simulate the world yourself

      - +
      @@ -845,36 +839,6 @@

      Abadie's idea: if no twin exists, build one

      -
      -
      Act II · The counterfactual problem
      -

      The estimator, precisely

      -
      A constrained least square optimization.
      -
      -

      Fit the weights on the pre-period only, constrained to the simplex:

      -
      - \[ \hat w \;=\; \arg\min_{w\in\Delta}\; \sum_{t \lt T_0}\Bigl(Y_{1t}-\sum_j w_j Y_{jt}\Bigr)^{2}, \qquad \Delta=\Bigl\{w : w_j\ge 0,\ \textstyle\sum_j w_j=1\Bigr\} \] -
      -
        -
      • Pre-launch only: the optimiser never sees the post-period, so it cannot cheat.
        • -
      • The simplex \(\Delta\): a readable recipe, never "−80% of Milan + 190% of Rome": Colgate's projection with the weights on the table.
        • -
      • No standard error: a point estimate only; inference comes later, from placebos.
        • -
      -
      ▸ Reading the effect off the gap
      -
      - \[ \hat\tau_t \;=\; Y_{1t}-\sum_j \hat w_j Y_{jt}, \qquad\qquad \hat\tau \;=\; \sum_{t\ge T_0}\hat\tau_t \] - weekly gap between the treated metro and its synthetic twin, summed over the 20 post-launch weeks -
      -
      -
      Why constrain at all?
      - The simplex buys three things ordinary regression cannot: interpretability, regularisation, and an off-switch.
      -
        -
      • Interpretability: "40% dma_08 + 32% dma_20 + 17% dma_03" is a sentence a planner can act on.
        • -
      • Regularisation: most weights land on exactly zero.
        • -
      • An off-switch: a market no blend can match fails loudly instead of extrapolating.
        • -
      -
      -
      -
      Act II · The counterfactual problem
      @@ -911,65 +875,18 @@

      What the simplex buys, geometrically: stay inside the hull

      Act II · The counterfactual problem
      -

      What the constraint buys: drop it and see

      -
      What if we used a simple unconstrained OLS?
      +

      The synthetic twin, graded against the truth

      +
      Only a simulation can draw the true counterfactual; hold the twin against it.
      -
      ▶ LIVE Two synthetics against the truth only a simulation can draw
      +
      ▶ LIVE The synthetic against the truth only a simulation can draw
      -
      OLS hugs the pre-period tighter ( vs ) yet drifts further from the true \(Y_{1t}(0)\) after launch ( vs ): overfitting the simplex refuses.
      -
      -
      -
      -
      The weights OLS chose
      -
      -
      OLS puts negative weight on donors (down to ) and gross weight , where the simplex uses exactly 1.
      -
      -
      -
      - \[ n_{\mathrm{eff}} \;=\; \frac{1}{\sum_j \hat w_j^{2}} \] - Effective number of donors (inverse Herfindahl): one donor \(\Rightarrow n_{\mathrm{eff}}=1\), all 29 equally \(\Rightarrow 29\). Ours: 3.3. -
      -
      What n_eff = 3.3 tells you
      -
        -
      • The synthetic leans on about three donor markets, not a fuzzy mix of all 29.
        • -
      • Sparse weights: interpretable, and less room to overfit.
        • -
      • Near 1: hostage to one market. Near 29: mush. 3.3: concentrated, not fragile.
        • -
      -
      +
      Pre-launch fit: RMSE . Post-launch gap to the true \(Y_{1t}(0)\): . The twin tracks a line it was never shown.
      -
      -
      Act II · The counterfactual problem, versus machine learning
      -

      "Why not gradient boosting / Prophet / an LSTM?"

      -
      Is this a forecasting exercise?
      -
      -
      -
      ▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
      -
      -
      Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
      -
      -
        -
      • In-sample it fits tighter: €k vs €k per week: flexibility always buys the past.
        • -
      • Yet the estimates are a wash: vs , both near the planted €284k. One dataset cannot rank them.
        • -
      • The sweep decides: across 24 fresh worlds, 1.6× worse out of sample, with nothing to inspect.
        • -
      -
      -
      -
      Principle 1: a counterfactual, not a forecast
      - A forecaster asks what comes next. We ask what this market would have done in a world that never happened.
      -
        -
      • Principle 2: in-sample fit is the trap, not the goal.
        • -
      • Principle 3: you cannot cross-validate the counterfactual. The truth is never observed.
        • -
      • Principle 4: the simplex ships an auditable claim; a boosted tree is a black box.
        • -
      -
      The forecaster is better at prediction. The simplex is better at the causal job.
      -
      -
      -
      Act II · The counterfactual problem
      @@ -1028,90 +945,7 @@

      Compare the estimators

      -
      -
      Act II · The counterfactual problem
      -

      Interesting limits for the simpler estimators

      -
      Where before/after and treated/control estimators break.
      -
      -
      -
      -
      Before/after · one unit, across time
      - The treated metro's post-average minus its pre-average.
      -
        -
      • The level cancels; nothing subtracts the shared wave.
        • -
      • The whole drift \(\Delta\bar f\) lands on the metro: the largest bias, growing with the horizon.
        • -
      -
      -
      -
      Treated-vs-control · one period, across units
      - The treated metro minus the control average, both in the post window.
      -
        -
      • The level gap \(\alpha_1-\bar\alpha_C\) survives, and sizes differ a lot: it dominates.
        • -
      • This is why raw sales are never compared across cities.
        • -
      -
      -
      -
      ▸ The two bias decompositions, term by term
      -
      - \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] - the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. -
      -
      - \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] - no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. -
      -
      -
      -
      - -
      -
      Act II · The counterfactual problem
      -

      Interesting limits for DiD

      -
      Where the DiD estimator breaks.
      -
      -
      The one question
      - DiD only works if treated and controls would have drifted together. Two dials decide (spread \(s\), macro \(\sigma_\eta\)); the algebra is in the fold.
      -
      Synthetic control solves it
      - DiD's residual bias \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\) comes from its equal weights: it hopes \(s\) is small. Synthetic control chooses the weights, so \(B\to0\) at any spread.
      -
      ▸ The DiD bias, decomposed (and why more data cannot shrink it)
      -
      - \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] -
      -
        -
      • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
        • -
      • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
        • -
      -
      - \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] -
      -
        -
      • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
        • -
      • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
        • -
      -
      -
      ▸ The two limits, dial by dial: send a knob to zero and read what dies
      -
      -
      Limit 1 · loading spread \(s \to 0\): sufficient on its own
      -
        -
      • The prefactor \(s^2/3\) kills the whole bracket at once.
        • -
      • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
        • -
      • DiD then recovers \(\bar\tau\) up to noise.
        • -
      -
      Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
      -
        -
      • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
        • -
      • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
        • -
      • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
        • -
      -
      -
      -
      ▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
      - Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. -
      -
      -
      -
      Act II · The counterfactual problem
      @@ -1145,103 +979,8 @@

      What must be true for the gap to be causal

      -
      -
      Act III · Is it real, and how big?
      -

      €260k. Real, or a lucky metro? Build the null yourself

      -
      -
      -
      ✋ Poll
      -
      With one treated metro a t-test is not weak, it is undefined. So build the test yourself: rerun the whole pipeline 29 more times, each donor pretended treated ("placebo"): 30 "effects", 29 where nothing ran. Where does our €260k rank?
      -
      - - - -
      - -
      A: rank 1 of 30. The rank is the inference: if the campaign did nothing, P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test.
      -
      -
      -
      - -
      -
      Act III · Is it real, and how big?
      -

      Placebo-in-space: measure the luck directly

      -
      Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
      -
      -
      -
      -
      -
      ▶ LIVE Refit the estimator on every donor as if it were treated
      -
      -
      The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
      -
      -
      -
      -
      The null hypothesis, stated
      - \(H_0\): the campaign did nothing. Then the metro is exchangeable with its donors: any of the 30 ranks is equally likely.
      -
      - \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] -
      -
      -
      -
      What the p-value means (and what it is not)
      - The probability, if \(H_0\) were true, of a gap this extreme: a rank, no bell curve assumed. \(p\) sits at its floor \(1/30\), set by the donor count, not the weeks.
      -
      ▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
      -
        -
      • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
        • -
      • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
        • -
      -
      Conclusion: the €260k lift is real, at \(p\approx0.033\)
      - Rank 1 of 30, clear of the cloud: reject \(H_0\). The rank holds (p = 0.033 every time) dropping shaky placebos at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": Act IV.
      -
      -
      - -
      -
      Act III · Is it real, and how big?
      -

      From a test to an interval: inversion

      -
      Which true lifts could plausibly have produced our €260k? Keep the survivors: no bell curve anywhere.
      -
      -
      -
      -
      -
      ▶ LIVE Drag \(H\), a guess at the truth: the error cloud slides with it; our €260k never moves.
      -
      -
      - - -
      -
      Axis: 20-week total gap (€000). Top: the placebo errors at truth 0. Middle: the same errors slid to \(H\); \(H\) survives if the green line sits inside the shaded middle 90%. Bottom: the survivors, collected: the interval. Drag past an edge to reject.
      -
      -
      -
      -
      The whole idea
      - Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
      -
        -
      • ① Measure the error. The 29 placebo "effects" form a cloud around zero, spread about ±€50k.
        • -
      • ② Guess a truth \(H\). If the lift were \(H\), we would see \(H\) plus that same cloud.
        • -
      • ③ Keep or reject. Keep \(H\) if €260k sits inside its middle 90%.
        • -
      • ④ Sweep. The survivors run €195k to €335k: the interval is [€195k, €335k].
        • -
      -
      -
      -
      ▸ The inversion, in one line of algebra
      -
      - \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} - \qquad\Longleftrightarrow\qquad - \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] - \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. -
      -
      -
      ▸ Why this interval is the referee for the rest of the lecture
      - It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. -
      -
      -
      -
      Act III · Is it real, and how big? Stress-test the estimate
      @@ -1294,26 +1033,6 @@

      Falsification 2: the assumption no placebo can see

      -
      -
      Act III · Is it real, and how big?
      -

      Statistics done. Three numbers.

      -
      Questions ① and ② from the boardroom slide are now answered.
      -
      -
      EngagementWhat it isTypical client
      - - - - - - -
      QuestionAnswerTool that answered it
      Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
      How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
      Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
      -
      Truth check (only a simulation allows it)
      - The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
      -
      The sentence that loses money
      - "€260k of sales for €75k, a 3.5× return. Roll it out." Every number true; the conclusion does not follow. Question ③ is not a statistics question.
      -
      -
    -
    Act IV · The decision in euros
    @@ -1857,7 +1576,7 @@

    The ideal experiment

    • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
    -
    The plan for Part 2
    +
    Going forward
    1. Give that random lever a name: an instrument.
    2. State the conditions it must satisfy.
    3. Check which of them the data can verify.
    @@ -2036,7 +1755,6 @@

    The IV estimate

    What just happened
    We priced the ad using only the random slice of exposure: the dashboard said €23.7, the lottery says €16.5, against a planted truth of €15.
    -

    The method never needed a model of intent, controls, or machine learning: two averages and a division.

    ▸ Deep dive · the confidence interval around €16.5
    @@ -2060,34 +1778,6 @@

    The IV estimate

    -
    -
    IV · The estimator · hands on
    -

    Why the division is forced

    -
    The division is not a modelling choice. It is the only effect size the two measurements allow.
    -
    -
    -
    ▶ LIVE every candidate effect makes a prediction. One matches.
    -
    -
    - - -
    -
    The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €3.48. Move your guess to the crossing and you have priced the ad.
    -
    -
    -

    Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

    -
      -
    • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's 0.2106 extra exposures per win should create \(\hat\beta \times 0.2106\) euros per win.
      • -
    • The fact: the lottery actually created €3.48 per win.
      • -
    • The verdict: every candidate except €16.5 contradicts a number we measured. The division is the only survivor, not a choice.
      • -
    -
    \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
    -
    Not a black box
    - Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
    -
    -
    -
    -
    IV · When it breaks · the dangerous failure
    @@ -2216,157 +1906,425 @@

    The price map

-
Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
- -
-

One estimate gives not one answer but a map from any price to a verdict:

- - - - - - - -
Price zoneVerdictWhy
below €12.7GOeven the most pessimistic supported effect pays
€12.7 to €20.4TESTthe data straddle the price: negotiate, or measure more
above €20.4NO-GOno supported effect pays
+
Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
+ +
+

One estimate gives not one answer but a map from any price to a verdict:

+ + + + + + + +
Price zoneVerdictWhy
below €12.7GOeven the most pessimistic supported effect pays
€12.7 to €20.4TESTthe data straddle the price: negotiate, or measure more
above €20.4NO-GOno supported effect pays
+
    +
  • Today's rate, €10, sits in the GO zone, below the whole interval.
    • +
  • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €16.5 − €10 = €6.5. Even read at the interval floor, €12.7 against €10, the exposure still pays.
    • +
+
Why boards like this framing
+ "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
+
+ +
+ +
+
IV · The decision
+

Poll · the negotiation

+
+
+
✋ Poll
+
The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €16.5 per exposure, 90% interval [12.7, 20.4]. What is the highest rate at which you would still sign "buy" without further study?
+
+ + + + +
+ +
C. Below €12.7, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
+
+
+
+ +
+
IV · The decision · banked
+

The verdict and the recommendation

+
The complete answer, assembled from everything measured so far.
+
+
+ + + + + + + + + +
QuantityValueSource
effect of one exposure€16.5the division δ/π
90% interval[12.7, 20.4]classical, and AR agrees
first-stage F156the lottery is strong
price€10the platform's rate card
net per exposure€6.5β − c, at the point estimate
+
The verdict
+ BUY  Keep buying at €10: the entire defensible range clears the price.
+
+
+
The recommendation, in three lines
+ 1. Keep buying at the €10 rate: even the interval's most pessimistic effect pays.
+ 2. Cap the bid at the interval's lower end, €12.7: up to there, every effect the data support still clears the price.
+ 3. Measure again only if the rate card climbs toward €12.7: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
+
    +
  • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
    • +
  • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
    • +
+
+
+
+ + +
+
Closing · Provenance
+

The tools were the product too

+
+
    +
  • CausalPy: synthetic control, interrupted time series, difference in differences, and regression discontinuity in one open-source package. The IV estimator that closes this session joined later.
    • +
  • Its launch example: individual exposure to a TV campaign cannot be randomised, yet its causal impact remains a core business need: the sentence this whole session opened with.
    • +
  • pymc-marketing: the MMM library behind Case 2's calibration story; one client's budget allocation approach to PyMC-Marketing came back as a pull request (Bolt).
    • +
  • Webinars and content: the consultancy's own webinar agenda is this session's syllabus, Instrumental Variables included.
    • +
+
+ PyTensor + PyMC + PyMC-Marketing + CausalPy +
+
+
+ +
+
Closing
+

The pattern in every engagement

+
+
    +
  • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
    • +
  • An experiment anchors every observational model: calibration is the product, not a luxury.
    • +
  • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
    • +
+ + + + + + +
Agent, on adversarial MMM dataResult
Vanilla coding agentFit a model, recommended budget reallocations. Confidently wrong.
PyMC Labs' Decision LabExplored 11 approaches, 0 converged. Returned: "No valid model found. Run a geo-holdout experiment."
+
Even the machines know the punchline
+ The honest system's best answer was Part 2's closing advice: run the experiment.
+
+
+ +
+
Closing
+

The END

+
Causal inference is one shelf of the toolbox: Bayesian inference is a vast set of tools, and PyMC Labs builds with all of it.
+
    +
  • Beyond today: demand forecasting, pricing, experimentation at scale, customer lifetime value, hierarchical models across markets: the same machinery, aimed at different decisions.
    • +
  • The cases: pymc-labs.com/blog-posts: every number in this session is pinned to a public post (sources on the next slide).
    • +
+
+
+
Francesco Muia
+
PhD in Theoretical Physics, EMBA.
Consultant for PyMC Labs and Brown University.
+
francesco.muia@pymc-labs.com
+
francesco.muia@ai-and-analytics-solutions.com
+
+
+
Alexander Fengler
+
PhD in Computational Cognitive Science.
Postdoc at Brown University, consultant for PyMC Labs.
+
alexander.fengler@pymc-labs.com
+
+
Get in touch
+ If a decision in your company leans on a number nobody quite trusts, write to us: those are exactly the problems we like.
+
+ + +
+
Backup
+ Backup · Sources +

Every number, pinned

+
Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
+
+ + + + + + + + + + + + + + + + + +
SourceFacts pinned
ailab.criteo.com · criteo-uplift-prediction-datasetcriteo_rows
pymc-labs.com · 2022-11-11-HelloFreshhf_panel_calibration
pymc-labs.com · 2023-06-20-juan-marketing-analyticswebinar_agenda
pymc-labs.com · bayes-is-slow-speeding-up-hellofreshs-bayesian-ab-tests-by-60xhf_batch, hf_test_types, hf_thousands
pymc-labs.com · bayesian-media-mix-modeling-for-marketing-optimizationhf_priors_experiments
pymc-labs.com · causal-sales-analytics-are-my-sales-incremental-or-cannibalisticcolgate_ci, colgate_ci_level, colgate_truth, colgate_year, fail_range, fail_truth, market
pymc-labs.com · causal-sales-analytics-discrete-choice-modelingcolgate_counterfactual_quote
pymc-labs.com · causalpy-a-new-package-for-bayesian-causal-inference-for-quasi-experimentscausalpy_methods, causalpy_tv
pymc-labs.com · funnel-aware-mmmcpl, cpl_window, gdpr_sentence, herp_attribution_quote, nurn_2026, trust_quote
pymc-labs.com · marketing-mix-modeling-a-complete-guidebolt_pr
pymc-labs.com · mmm_roas_liftlift_tests_n, roas_gap_words, roas_wrong_ranking, roas_x1, roas_x2
pymc-labs.com · open-sourcing-decision-lab-scaling-ai-judgment-data-sciencedl_explored, dl_vanilla, dl_verdict
pymc-labs.com · reducing-customer-acquisition-costs-how-we-helped-optimizing-hellofreshs-marketing-budgethf_var
+
+
+ +
+
Backup · Act II · The counterfactual problem, versus machine learning
+

"Why not gradient boosting / Prophet / an LSTM?"

+
Is this a forecasting exercise?
+
+
+
▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
+
+
Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
+
+
    +
  • In-sample it fits tighter: €k vs €k per week: flexibility always buys the past.
    • +
  • Yet the estimates are a wash: vs , both near the planted €284k. One dataset cannot rank them.
    • +
  • The sweep decides: across 24 fresh worlds, 1.6× worse out of sample, with nothing to inspect.
    • +
+
+
+
Principle 1: a counterfactual, not a forecast
+ A forecaster asks what comes next. We ask what this market would have done in a world that never happened.
+
    +
  • Principle 2: in-sample fit is the trap, not the goal.
    • +
  • Principle 3: you cannot cross-validate the counterfactual. The truth is never observed.
    • +
  • Principle 4: the simplex ships an auditable claim; a boosted tree is a black box.
    • +
+
The forecaster is better at prediction. The simplex is better at the causal job.
+
+
+
+ +
+
Backup · Act II · The counterfactual problem
+

Interesting limits for the simpler estimators

+
Where before/after and treated/control estimators break.
+
+
+
+
Before/after · one unit, across time
+ The treated metro's post-average minus its pre-average.
+
    +
  • The level cancels; nothing subtracts the shared wave.
    • +
  • The whole drift \(\Delta\bar f\) lands on the metro: the largest bias, growing with the horizon.
    • +
+
+
+
Treated-vs-control · one period, across units
+ The treated metro minus the control average, both in the post window.
+
    +
  • The level gap \(\alpha_1-\bar\alpha_C\) survives, and sizes differ a lot: it dominates.
    • +
  • This is why raw sales are never compared across cities.
    • +
+
+
+
▸ The two bias decompositions, term by term
+
+ \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] + the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. +
+
+ \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] + no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. +
+
+
+
+ +
+
Backup · Act II · The counterfactual problem
+

Interesting limits for DiD

+
Where the DiD estimator breaks.
+
+
The one question
+ DiD only works if treated and controls would have drifted together. Two dials decide (spread \(s\), macro \(\sigma_\eta\)); the algebra is in the fold.
+
Synthetic control solves it
+ DiD's residual bias \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\) comes from its equal weights: it hopes \(s\) is small. Synthetic control chooses the weights, so \(B\to0\) at any spread.
+
▸ The DiD bias, decomposed (and why more data cannot shrink it)
+
+ \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] +
+
    +
  • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
    • +
  • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
    • +
+
+ \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] +
    -
  • Today's rate, €10, sits in the GO zone, below the whole interval.
    • -
  • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €16.5 − €10 = €6.5. Even read at the interval floor, €12.7 against €10, the exposure still pays.
    • +
  • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
    • +
  • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
-
Why boards like this framing
- "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
+
+
▸ The two limits, dial by dial: send a knob to zero and read what dies
+
+
Limit 1 · loading spread \(s \to 0\): sufficient on its own
+
    +
  • The prefactor \(s^2/3\) kills the whole bracket at once.
    • +
  • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
    • +
  • DiD then recovers \(\bar\tau\) up to noise.
    • +
+
Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
+
    +
  • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
    • +
  • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
    • +
  • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
    • +
-
+ +
▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
+ Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. +
+
-
-
IV · The decision
-

Poll · the negotiation

-
-
+
+
Backup · Act III · Is it real, and how big?
+

€260k. Real, or a lucky metro? Build the null yourself

+
+
✋ Poll
-
The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €16.5 per exposure, 90% interval [12.7, 20.4]. What is the highest rate at which you would still sign "buy" without further study?
+
With one treated metro a t-test is not weak, it is undefined. So build the test yourself: rerun the whole pipeline 29 more times, each donor pretended treated ("placebo"): 30 "effects", 29 where nothing ran. Where does our €260k rank?
- - - - + + +
-
C. Below €12.7, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
+
A: rank 1 of 30. The rank is the inference: if the campaign did nothing, P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test.
-
-
IV · The decision · banked
-

The verdict and the recommendation

-
The complete answer, assembled from everything measured so far.
-
-
- - - - - - - - - -
QuantityValueSource
effect of one exposure€16.5the division δ/π
90% interval[12.7, 20.4]classical, and AR agrees
first-stage F156the lottery is strong
price€10the platform's rate card
net per exposure€6.5β − c, at the point estimate
-
The verdict
- BUY  Keep buying at €10: the entire defensible range clears the price.
-
-
-
The recommendation, in three lines
- 1. Keep buying at the €10 rate: even the interval's most pessimistic effect pays.
- 2. Cap the bid at the interval's lower end, €12.7: up to there, every effect the data support still clears the price.
- 3. Measure again only if the rate card climbs toward €12.7: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
-
    -
  • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
    • -
  • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
    • -
+
+
Backup · Act III · Is it real, and how big?
+

Placebo-in-space: measure the luck directly

+
Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
+
+
+
+
+
▶ LIVE Refit the estimator on every donor as if it were treated
+
+
The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
+
+
+
+
The null hypothesis, stated
+ \(H_0\): the campaign did nothing. Then the metro is exchangeable with its donors: any of the 30 ranks is equally likely.
+
+ \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] +
+
-
+
What the p-value means (and what it is not)
+ The probability, if \(H_0\) were true, of a gap this extreme: a rank, no bell curve assumed. \(p\) sits at its floor \(1/30\), set by the donor count, not the weeks.
+
▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
+
    +
  • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
    • +
  • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
    • +
+
Conclusion: the €260k lift is real, at \(p\approx0.033\)
+ Rank 1 of 30, clear of the cloud: reject \(H_0\). The rank holds (p = 0.033 every time) dropping shaky placebos at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": Act IV.
+
- -
-
Closing · Provenance
-

The tools were the product too

+
+
Backup · Act III · Is it real, and how big?
+

From a test to an interval: inversion

+
Which true lifts could plausibly have produced our €260k? Keep the survivors: no bell curve anywhere.
-
    -
  • CausalPy: synthetic control, interrupted time series, difference in differences, and regression discontinuity in one open-source package. The IV estimator that closes this session joined later.
    • -
  • Its launch example: individual exposure to a TV campaign cannot be randomised, yet its causal impact remains a core business need: the sentence this whole session opened with.
    • -
  • pymc-marketing: the MMM library behind Case 2's calibration story; one client's budget allocation approach to PyMC-Marketing came back as a pull request (Bolt).
    • -
  • Webinars and content: the consultancy's own webinar agenda is this session's syllabus, Instrumental Variables included.
    • -
-
- CausalPy - PyMC-Marketing +
+
+
+
▶ LIVE Drag \(H\), a guess at the truth: the error cloud slides with it; our €260k never moves.
+
+
+ + +
+
Axis: 20-week total gap (€000). Top: the placebo errors at truth 0. Middle: the same errors slid to \(H\); \(H\) survives if the green line sits inside the shaded middle 90%. Bottom: the survivors, collected: the interval. Drag past an edge to reject.
+
+
+
+
The whole idea
+ Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
+
    +
  • ① Measure the error. The 29 placebo "effects" form a cloud around zero, spread about ±€50k.
    • +
  • ② Guess a truth \(H\). If the lift were \(H\), we would see \(H\) plus that same cloud.
    • +
  • ③ Keep or reject. Keep \(H\) if €260k sits inside its middle 90%.
    • +
  • ④ Sweep. The survivors run €195k to €335k: the interval is [€195k, €335k].
    • +
+
+
▸ The inversion, in one line of algebra
+
+ \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} + \qquad\Longleftrightarrow\qquad + \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] + \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. +
+
+
▸ Why this interval is the referee for the rest of the lecture
+ It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. +
-
-
Closing
-

The pattern in every engagement

+
+
Backup · Act III · Is it real, and how big?
+

Statistics done. Three numbers.

+
Questions ① and ② from the boardroom slide are now answered.
-
    -
  • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
    • -
  • An experiment anchors every observational model: calibration is the product, not a luxury.
    • -
  • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
    • -
- - +
Agent, on adversarial MMM dataResult
+ - - + + +
QuestionAnswerTool that answered it
Vanilla coding agentFit a model, recommended budget reallocations. Confidently wrong.
PyMC Labs' Decision LabExplored 11 approaches, 0 converged. Returned: "No valid model found. Run a geo-holdout experiment."
Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
-
Even the machines know the punchline
- The honest system's best answer was Part 2's closing advice: run the experiment.
-
-
- -
-
Closing
-

One breath

-
-
The pattern to take home
- The toolkit a Bayesian consultancy sells: counterfactuals, calibrated by experiments, priced as probabilities.
-
    -
  • Read the cases: pymc-labs.com/blog-posts: every number in this deck is pinned to a post, listed on the next slide.
    • -
  • Say hello: both authors consult for PyMC Labs; the notebooks behind this session are the course repository.
    • -
+
Truth check (only a simulation allows it)
+ The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
+
The sentence that loses money
+ "€260k of sales for €75k, a 3.5× return. Roll it out." Every number true; the conclusion does not follow. Question ③ is not a statistics question.
- -
-
Backup
- Backup · Sources -

Every number, pinned

-
Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
-
- - - - - - - - - - - - - - - - - -
SourceFacts pinned
ailab.criteo.com · criteo-uplift-prediction-datasetcriteo_rows
pymc-labs.com · 2022-11-11-HelloFreshhf_panel_calibration
pymc-labs.com · 2023-06-20-juan-marketing-analyticswebinar_agenda
pymc-labs.com · bayes-is-slow-speeding-up-hellofreshs-bayesian-ab-tests-by-60xhf_batch, hf_test_types, hf_thousands
pymc-labs.com · bayesian-media-mix-modeling-for-marketing-optimizationhf_priors_experiments
pymc-labs.com · causal-sales-analytics-are-my-sales-incremental-or-cannibalisticcolgate_ci, colgate_ci_level, colgate_truth, colgate_year, fail_range, fail_truth, market
pymc-labs.com · causal-sales-analytics-discrete-choice-modelingcolgate_counterfactual_quote
pymc-labs.com · causalpy-a-new-package-for-bayesian-causal-inference-for-quasi-experimentscausalpy_methods, causalpy_tv
pymc-labs.com · funnel-aware-mmmcpl, cpl_window, gdpr_sentence, herp_attribution_quote, nurn_2026, trust_quote
pymc-labs.com · marketing-mix-modeling-a-complete-guidebolt_pr
pymc-labs.com · mmm_roas_liftlift_tests_n, roas_gap_words, roas_wrong_ranking, roas_x1, roas_x2
pymc-labs.com · open-sourcing-decision-lab-scaling-ai-judgment-data-sciencedl_explored, dl_vanilla, dl_verdict
pymc-labs.com · reducing-customer-acquisition-costs-how-we-helped-optimizing-hellofreshs-marketing-budgethf_var
-
+
+
Backup · IV · The estimator · hands on
+

Why the division is forced

+
The division is not a modelling choice. It is the only effect size the two measurements allow.
+
+
+
▶ LIVE every candidate effect makes a prediction. One matches.
+
+
+ + +
+
The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €3.48. Move your guess to the crossing and you have priced the ad.
+
+
+

Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

+
    +
  • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's 0.2106 extra exposures per win should create \(\hat\beta \times 0.2106\) euros per win.
    • +
  • The fact: the lottery actually created €3.48 per win.
    • +
  • The verdict: every candidate except €16.5 contradicts a number we measured. The division is the only survivor, not a choice.
    • +
+
\[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
+
Not a black box
+ Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
+
+
@@ -3129,16 +3087,15 @@

Every number, pinned

const e=document.getElementById(id); if(e)e.textContent=v;}); const tr=DATA.treated,y0=DATA.y0_true,scl=DATA.synth_cl,ols=DATA.ols_synth,L=DATA.launch,W=tr.length; function draw(){clr(svg);const c=COL();const Wp=720,H=235,mL=46,mR=14,mT=32,mB=26; - let mn=1e9,mx=-1e9;[tr,y0,scl,ols].forEach(a=>a.forEach(v=>{if(vmx)mx=v;})); + let mn=1e9,mx=-1e9;[tr,y0,scl].forEach(a=>a.forEach(v=>{if(vmx)mx=v;})); const x=lin(0,W-1,mL,Wp-mR),y=lin(mn-2,mx+2,H-mB,mT); svg.appendChild(el('line',{x1:x(L),x2:x(L),y1:mT,y2:H-mB,stroke:c.orange,'stroke-width':1.3})); svg.appendChild(el('text',{x:x(L)+4,y:H-mB-6,fill:c.orange,'font-size':10},'launch')); svg.appendChild(el('path',{d:path(tr.map((v,i)=>[x(i),y(v)])),fill:'none',stroke:c.ink,'stroke-width':1.6,opacity:.5})); svg.appendChild(el('path',{d:path(y0.slice(L-1).map((v,i)=>[x(L-1+i),y(v)])),fill:'none',stroke:c.ink,'stroke-width':2,'stroke-dasharray':'6 4'})); svg.appendChild(el('path',{d:path(scl.map((v,i)=>[x(i),y(v)])),fill:'none',stroke:c.blue,'stroke-width':1.9})); - svg.appendChild(el('path',{d:path(ols.map((v,i)=>[x(i),y(v)])),fill:'none',stroke:c.red,'stroke-width':1.9})); const lg=[[c.ink,'treated (observed)',1.6,'none',.5],[c.ink,'true Y(0), post-launch',2,'6 4',1], - [c.blue,'simplex synthetic',1.9,'none',1],[c.red,'OLS synthetic',1.9,'none',1]]; + [c.blue,'simplex synthetic',1.9,'none',1]]; lg.forEach(([col,lab,wd,dash,op],i)=>{const xx=mL+8+i*168; svg.appendChild(el('line',{x1:xx,x2:xx+24,y1:10,y2:10,stroke:col,'stroke-width':wd,'stroke-dasharray':dash,opacity:op})); svg.appendChild(el('text',{x:xx+29,y:14,fill:col,'font-size':10.5,opacity:Math.max(op,.8)},lab));}); @@ -3519,6 +3476,26 @@

Every number, pinned

draw(); window.__redraw.push(draw); })(); +/* ==== fig: the metros, one treated (boardroom slide) ==== */ +(function(){ + const svg=document.getElementById('svgMetros'); if(!svg)return; + function draw(){ + clr(svg); const c=COL(); + const donors=[[60,40,7],[80,120,9],[130,30,6],[210,125,7],[220,55,10],[255,95,6],[290,35,8], + [320,120,9],[350,70,7],[385,30,6],[400,105,8],[430,60,11],[465,115,6],[480,35,7],[510,85,9], + [545,40,6],[560,120,8],[590,70,7],[620,105,6],[640,35,9],[665,80,7],[75,70,5],[170,45,5], + [240,20,5],[365,115,5],[450,20,5],[530,115,5],[610,20,5],[680,120,5]]; + donors.forEach(([x,y,r])=>svg.appendChild(el('circle',{cx:x,cy:y,r:r,fill:c.grey,opacity:.28,stroke:c.grey,'stroke-width':1}))); + svg.appendChild(el('circle',{cx:150,cy:70,r:14,fill:c.orange,opacity:.3,stroke:c.orange,'stroke-width':2.2})); + svg.appendChild(el('path',{d:'M 166 52 A 24 24 0 0 1 174 70',fill:'none',stroke:c.orange,'stroke-width':1.6})); + svg.appendChild(el('path',{d:'M 170 45 A 32 32 0 0 1 181 70',fill:'none',stroke:c.orange,'stroke-width':1.2,opacity:.7})); + svg.appendChild(el('text',{x:150,y:105,'text-anchor':'middle','font-size':10.5,fill:c.orange,'font-weight':700},'the treated metro')); + svg.appendChild(el('text',{x:150,y:118,'text-anchor':'middle','font-size':9.5,fill:c.orange},'the €75k campaign, weeks 40-59')); + svg.appendChild(el('text',{x:545,y:16,'text-anchor':'middle','font-size':10.5,fill:c.muted,'font-weight':700},'29 donor markets · no campaign')); + } + draw(); window.__redraw.push(draw); +})(); + /* ==== fig: break the counterfactual yourself (probe slide) ==== */ (function(){ const svg=document.getElementById('svgBreak'); if(!svg)return; diff --git a/causal-marketing-pymc/apps/unified_slides_src.html b/causal-marketing-pymc/apps/unified_slides_src.html index a46c509..4a9e17b 100644 --- a/causal-marketing-pymc/apps/unified_slides_src.html +++ b/causal-marketing-pymc/apps/unified_slides_src.html @@ -596,6 +596,11 @@

The boardroom question

+
+
One metro got the campaign; twenty-nine watched schematic
+
+
+
The method this data calls for
  • One treated unit, no A/B test possible: synthetic control's home ground.
    • @@ -606,18 +611,8 @@

    The boardroom question

  • δ = the share of the pilot's per-euro lift that survives national rollout.
  • The €4M call reduces to one question: is δ large enough to still clear the cost?
-
▸ What "unbiased" means here, precisely, and where it would break
-

The estimator subtracts a reconstructed counterfactual from the treated metro's post-launch sales, and bias is its expected gap from the true lift:

-
- \[ \hat\tau \;=\; \bar Y_{1,\text{post}} - \hat Y_{1,\text{post}}(0), \qquad \text{bias} \;\equiv\; \mathbb{E}[\hat\tau] - \tau \]
-
    -
  • The counterfactual is the synthetic twin, built to reproduce the metro's pre-launch record (its level and its factor loadings) week by week.
    • -
  • Unbiasedness rests on one assumption: that pre-launch match would have continued through the post window had the campaign never run.
    • -
  • This needs no story about why the metro was chosen. Whatever pre-launch trait drove the pick, the twin already carries it, because it was fitted to match it: selection on anything visible before launch cannot tilt the estimate.
    • -
  • Bias enters only through what the pre-launch record cannot see: if the launch were timed on private knowledge of a coming local boom, the twin could not anticipate it and the gap would credit the boom to the campaign. The later placebo-in-time test hunts exactly that.
    • -
-
+
@@ -730,7 +725,6 @@

Causal inference is a missing-data problem

  • Subscript says which market, argument says which world (1 = campaign, 0 = none): distinct axes that happen to share the digit 1.
  • \(w_j\): the donor weights the synthetic control will choose so \(\sum_{j\ge 2} w_j Y_{jt}\) rebuilds the missing \(Y_{1t}(0)\).
    • -

    "What would this metro have sold anyway?" is an estimation target, not a rhetorical question.

    @@ -781,7 +775,7 @@

    Simulate the world yourself

    Take a look at the counterfactual.
    -
    ▶ LIVE The equations of the previous slide. Grey: donor markets. White: the treated metro.
    +
    ▶ LIVE The equations of the previous slide. Grey: donor markets. Black: the treated metro.
    @@ -789,7 +783,7 @@

    Simulate the world yourself

    - +
    @@ -845,36 +839,6 @@

    Abadie's idea: if no twin exists, build one

    -
    -
    Act II · The counterfactual problem
    -

    The estimator, precisely

    -
    A constrained least square optimization.
    -
    -

    Fit the weights on the pre-period only, constrained to the simplex:

    -
    - \[ \hat w \;=\; \arg\min_{w\in\Delta}\; \sum_{t \lt T_0}\Bigl(Y_{1t}-\sum_j w_j Y_{jt}\Bigr)^{2}, \qquad \Delta=\Bigl\{w : w_j\ge 0,\ \textstyle\sum_j w_j=1\Bigr\} \] -
    -
      -
    • Pre-launch only: the optimiser never sees the post-period, so it cannot cheat.
      • -
    • The simplex \(\Delta\): a readable recipe, never "−80% of Milan + 190% of Rome": Colgate's projection with the weights on the table.
      • -
    • No standard error: a point estimate only; inference comes later, from placebos.
      • -
    -
    ▸ Reading the effect off the gap
    -
    - \[ \hat\tau_t \;=\; Y_{1t}-\sum_j \hat w_j Y_{jt}, \qquad\qquad \hat\tau \;=\; \sum_{t\ge T_0}\hat\tau_t \] - weekly gap between the treated metro and its synthetic twin, summed over the 20 post-launch weeks -
    -
    -
    Why constrain at all?
    - The simplex buys three things ordinary regression cannot: interpretability, regularisation, and an off-switch.
    -
      -
    • Interpretability: "40% dma_08 + 32% dma_20 + 17% dma_03" is a sentence a planner can act on.
      • -
    • Regularisation: most weights land on exactly zero.
      • -
    • An off-switch: a market no blend can match fails loudly instead of extrapolating.
      • -
    -
    -
    -
    Act II · The counterfactual problem
    @@ -911,65 +875,18 @@

    What the simplex buys, geometrically: stay inside the hull

    Act II · The counterfactual problem
    -

    What the constraint buys: drop it and see

    -
    What if we used a simple unconstrained OLS?
    +

    The synthetic twin, graded against the truth

    +
    Only a simulation can draw the true counterfactual; hold the twin against it.
    -
    ▶ LIVE Two synthetics against the truth only a simulation can draw
    +
    ▶ LIVE The synthetic against the truth only a simulation can draw
    -
    OLS hugs the pre-period tighter ( vs ) yet drifts further from the true \(Y_{1t}(0)\) after launch ( vs ): overfitting the simplex refuses.
    -
    -
    -
    -
    The weights OLS chose
    -
    -
    OLS puts negative weight on donors (down to ) and gross weight , where the simplex uses exactly 1.
    -
    -
    -
    - \[ n_{\mathrm{eff}} \;=\; \frac{1}{\sum_j \hat w_j^{2}} \] - Effective number of donors (inverse Herfindahl): one donor \(\Rightarrow n_{\mathrm{eff}}=1\), all 29 equally \(\Rightarrow 29\). Ours: 3.3. -
    -
    What n_eff = 3.3 tells you
    -
      -
    • The synthetic leans on about three donor markets, not a fuzzy mix of all 29.
      • -
    • Sparse weights: interpretable, and less room to overfit.
      • -
    • Near 1: hostage to one market. Near 29: mush. 3.3: concentrated, not fragile.
      • -
    -
    +
    Pre-launch fit: RMSE . Post-launch gap to the true \(Y_{1t}(0)\): . The twin tracks a line it was never shown.
    -
    -
    Act II · The counterfactual problem, versus machine learning
    -

    "Why not gradient boosting / Prophet / an LSTM?"

    -
    Is this a forecasting exercise?
    -
    -
    -
    ▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
    -
    -
    Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
    -
    -
      -
    • In-sample it fits tighter: €k vs €k per week: flexibility always buys the past.
      • -
    • Yet the estimates are a wash: vs , both near the planted €284k. One dataset cannot rank them.
      • -
    • The sweep decides: across 24 fresh worlds, {{nb07.hull_oos_ratio}}× worse out of sample, with nothing to inspect.
      • -
    -
    -
    -
    Principle 1: a counterfactual, not a forecast
    - A forecaster asks what comes next. We ask what this market would have done in a world that never happened.
    -
      -
    • Principle 2: in-sample fit is the trap, not the goal.
      • -
    • Principle 3: you cannot cross-validate the counterfactual. The truth is never observed.
      • -
    • Principle 4: the simplex ships an auditable claim; a boosted tree is a black box.
      • -
    -
    The forecaster is better at prediction. The simplex is better at the causal job.
    -
    -
    -
    Act II · The counterfactual problem
    @@ -1028,90 +945,7 @@

    Compare the estimators

    -
    -
    Act II · The counterfactual problem
    -

    Interesting limits for the simpler estimators

    -
    Where before/after and treated/control estimators break.
    -
    -
    -
    -
    Before/after · one unit, across time
    - The treated metro's post-average minus its pre-average.
    -
      -
    • The level cancels; nothing subtracts the shared wave.
      • -
    • The whole drift \(\Delta\bar f\) lands on the metro: the largest bias, growing with the horizon.
      • -
    -
    -
    -
    Treated-vs-control · one period, across units
    - The treated metro minus the control average, both in the post window.
    -
      -
    • The level gap \(\alpha_1-\bar\alpha_C\) survives, and sizes differ a lot: it dominates.
      • -
    • This is why raw sales are never compared across cities.
      • -
    -
    -
    -
    ▸ The two bias decompositions, term by term
    -
    - \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] - the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. -
    -
    - \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] - no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. -
    -
    -
    -
    - -
    -
    Act II · The counterfactual problem
    -

    Interesting limits for DiD

    -
    Where the DiD estimator breaks.
    -
    -
    The one question
    - DiD only works if treated and controls would have drifted together. Two dials decide (spread \(s\), macro \(\sigma_\eta\)); the algebra is in the fold.
    -
    Synthetic control solves it
    - DiD's residual bias \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\) comes from its equal weights: it hopes \(s\) is small. Synthetic control chooses the weights, so \(B\to0\) at any spread.
    -
    ▸ The DiD bias, decomposed (and why more data cannot shrink it)
    -
    - \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] -
    -
      -
    • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
      • -
    • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
      • -
    -
    - \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] -
    -
      -
    • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
      • -
    • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
      • -
    -
    -
    ▸ The two limits, dial by dial: send a knob to zero and read what dies
    -
    -
    Limit 1 · loading spread \(s \to 0\): sufficient on its own
    -
      -
    • The prefactor \(s^2/3\) kills the whole bracket at once.
      • -
    • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
      • -
    • DiD then recovers \(\bar\tau\) up to noise.
      • -
    -
    Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
    -
      -
    • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
      • -
    • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
      • -
    • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
      • -
    -
    -
    -
    ▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
    - Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. -
    -
    -
    -
    Act II · The counterfactual problem
    @@ -1145,103 +979,8 @@

    What must be true for the gap to be causal

    -
    -
    Act III · Is it real, and how big?
    -

    €260k. Real, or a lucky metro? Build the null yourself

    -
    -
    -
    ✋ Poll
    -
    With one treated metro a t-test is not weak, it is undefined. So build the test yourself: rerun the whole pipeline 29 more times, each donor pretended treated ("placebo"): 30 "effects", 29 where nothing ran. Where does our €260k rank?
    -
    - - - -
    - -
    A: rank 1 of 30. The rank is the inference: if the campaign did nothing, P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test.
    -
    -
    -
    - -
    -
    Act III · Is it real, and how big?
    -

    Placebo-in-space: measure the luck directly

    -
    Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
    -
    -
    -
    -
    -
    ▶ LIVE Refit the estimator on every donor as if it were treated
    -
    -
    The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
    -
    -
    -
    -
    The null hypothesis, stated
    - \(H_0\): the campaign did nothing. Then the metro is exchangeable with its donors: any of the 30 ranks is equally likely.
    -
    - \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] -
    -
    -
    -
    What the p-value means (and what it is not)
    - The probability, if \(H_0\) were true, of a gap this extreme: a rank, no bell curve assumed. \(p\) sits at its floor \(1/30\), set by the donor count, not the weeks.
    -
    ▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
    -
      -
    • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
      • -
    • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
      • -
    -
    Conclusion: the €260k lift is real, at \(p\approx0.033\)
    - Rank 1 of 30, clear of the cloud: reject \(H_0\). The rank holds (p = 0.033 every time) dropping shaky placebos at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": Act IV.
    -
    -
    - -
    -
    Act III · Is it real, and how big?
    -

    From a test to an interval: inversion

    -
    Which true lifts could plausibly have produced our €260k? Keep the survivors: no bell curve anywhere.
    -
    -
    -
    -
    -
    ▶ LIVE Drag \(H\), a guess at the truth: the error cloud slides with it; our €260k never moves.
    -
    -
    - - -
    -
    Axis: 20-week total gap (€000). Top: the placebo errors at truth 0. Middle: the same errors slid to \(H\); \(H\) survives if the green line sits inside the shaded middle 90%. Bottom: the survivors, collected: the interval. Drag past an edge to reject.
    -
    -
    -
    -
    The whole idea
    - Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
    -
      -
    • ① Measure the error. The 29 placebo "effects" form a cloud around zero, spread about ±€50k.
      • -
    • ② Guess a truth \(H\). If the lift were \(H\), we would see \(H\) plus that same cloud.
      • -
    • ③ Keep or reject. Keep \(H\) if €260k sits inside its middle 90%.
      • -
    • ④ Sweep. The survivors run €195k to €335k: the interval is [€195k, €335k].
      • -
    -
    -
    -
    ▸ The inversion, in one line of algebra
    -
    - \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} - \qquad\Longleftrightarrow\qquad - \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] - \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. -
    -
    -
    ▸ Why this interval is the referee for the rest of the lecture
    - It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. -
    -
    -
    -
    Act III · Is it real, and how big? Stress-test the estimate
    @@ -1294,26 +1033,6 @@

    Falsification 2: the assumption no placebo can see

    -
    -
    Act III · Is it real, and how big?
    -

    Statistics done. Three numbers.

    -
    Questions ① and ② from the boardroom slide are now answered.
    -
    - - - - - - - -
    QuestionAnswerTool that answered it
    Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
    How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
    Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
    -
    Truth check (only a simulation allows it)
    - The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
    -
    The sentence that loses money
    - "€260k of sales for €75k, a 3.5× return. Roll it out." Every number true; the conclusion does not follow. Question ③ is not a statistics question.
    -
    -
    -
    Act IV · The decision in euros
    @@ -1857,7 +1576,7 @@

    The ideal experiment

    • Where it hides: a lottery, a rollout order, an arbitrary rule that moved exposure for reasons unrelated to intent.
    -
    The plan for Part 2
    +
    Going forward
    1. Give that random lever a name: an instrument.
    2. State the conditions it must satisfy.
    3. Check which of them the data can verify.
    @@ -2036,7 +1755,6 @@

    The IV estimate

    What just happened
    We priced the ad using only the random slice of exposure: the dashboard said €{{nb11.naive}}, the lottery says €{{nb11.iv_est}}, against a planted truth of €{{nb11.true}}.
    -

    The method never needed a model of intent, controls, or machine learning: two averages and a division.

    ▸ Deep dive · the confidence interval around €{{nb11.iv_est}}
    @@ -2060,34 +1778,6 @@

    The IV estimate

    -
    -
    IV · The estimator · hands on
    -

    Why the division is forced

    -
    The division is not a modelling choice. It is the only effect size the two measurements allow.
    -
    -
    -
    ▶ LIVE every candidate effect makes a prediction. One matches.
    -
    -
    - - -
    -
    The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €{{nb11.reduced}}. Move your guess to the crossing and you have priced the ad.
    -
    -
    -

    Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

    -
      -
    • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's {{nb11.first}} extra exposures per win should create \(\hat\beta \times {{nb11.first}}\) euros per win.
      • -
    • The fact: the lottery actually created €{{nb11.reduced}} per win.
      • -
    • The verdict: every candidate except €{{nb11.iv_est}} contradicts a number we measured. The division is the only survivor, not a choice.
      • -
    -
    \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
    -
    Not a black box
    - Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
    -
    -
    -
    -
    IV · When it breaks · the dangerous failure
    @@ -2216,145 +1906,413 @@

    The price map

    -
    Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
    - -
    -

    One estimate gives not one answer but a map from any price to a verdict:

    - - - - - - - -
    Price zoneVerdictWhy
    below €{{nb11.iv_lo}}GOeven the most pessimistic supported effect pays
    €{{nb11.iv_lo}} to €{{nb11.iv_hi}}TESTthe data straddle the price: negotiate, or measure more
    above €{{nb11.iv_hi}}NO-GOno supported effect pays
    +
    Blue: net value per exposure at each price. The orange band is the 90% interval, the zone where the data refuse to commit. Drag the price through the three zones and watch the verdict flip.
    + +
    +

    One estimate gives not one answer but a map from any price to a verdict:

    + + + + + + + +
    Price zoneVerdictWhy
    below €{{nb11.iv_lo}}GOeven the most pessimistic supported effect pays
    €{{nb11.iv_lo}} to €{{nb11.iv_hi}}TESTthe data straddle the price: negotiate, or measure more
    above €{{nb11.iv_hi}}NO-GOno supported effect pays
    +
      +
    • Today's rate, €{{nb11.cost}}, sits in the GO zone, below the whole interval.
      • +
    • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €{{nb11.iv_est}} − €{{nb11.cost}} = €{{nb11.net}}. Even read at the interval floor, €{{nb11.iv_lo}} against €{{nb11.cost}}, the exposure still pays.
      • +
    +
    Why boards like this framing
    + "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
    +
    + + + +
    +
    IV · The decision
    +

    Poll · the negotiation

    +
    +
    +
    ✋ Poll
    +
    The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €{{nb11.iv_est}} per exposure, 90% interval [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. What is the highest rate at which you would still sign "buy" without further study?
    +
    + + + + +
    + +
    C. Below €{{nb11.iv_lo}}, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
    +
    +
    +
    + +
    +
    IV · The decision · banked
    +

    The verdict and the recommendation

    +
    The complete answer, assembled from everything measured so far.
    +
    +
    + + + + + + + + + +
    QuantityValueSource
    effect of one exposure€{{nb11.iv_est}}the division δ/π
    90% interval[{{nb11.iv_lo}}, {{nb11.iv_hi}}]classical, and AR agrees
    first-stage F{{nb11.f_stat}}the lottery is strong
    price€{{nb11.cost}}the platform's rate card
    net per exposure€{{nb11.net}}β − c, at the point estimate
    +
    The verdict
    + BUY  Keep buying at €{{nb11.cost}}: the entire defensible range clears the price.
    +
    +
    +
    The recommendation, in three lines
    + 1. Keep buying at the €{{nb11.cost}} rate: even the interval's most pessimistic effect pays.
    + 2. Cap the bid at the interval's lower end, €{{nb11.iv_lo}}: up to there, every effect the data support still clears the price.
    + 3. Measure again only if the rate card climbs toward €{{nb11.iv_lo}}: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
    +
      +
    • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
      • +
    • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
      • +
    +
    +
    +
    + + +
    +
    Closing · Provenance
    +

    The tools were the product too

    +
    +
      +
    • CausalPy: {{labs.causalpy_methods}} in one open-source package. The IV estimator that closes this session joined later.
      • +
    • Its launch example: individual exposure to a TV campaign {{labs.causalpy_tv}}, yet its causal impact remains a core business need: the sentence this whole session opened with.
      • +
    • pymc-marketing: the MMM library behind Case 2's calibration story; one client's {{labs.bolt_pr}} came back as a pull request (Bolt).
      • +
    • Webinars and content: the consultancy's own webinar agenda is this session's syllabus, {{labs.webinar_agenda}} included.
      • +
    +
    + PyTensor + PyMC + PyMC-Marketing + CausalPy +
    +
    +
    + +
    +
    Closing
    +

    The pattern in every engagement

    +
    +
      +
    • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
      • +
    • An experiment anchors every observational model: calibration is the product, not a luxury.
      • +
    • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
      • +
    + + + + + + +
    Agent, on adversarial MMM dataResult
    Vanilla coding agentFit a model, recommended budget reallocations. {{labs.dl_vanilla}}
    PyMC Labs' Decision Lab{{labs.dl_explored}} Returned: "{{labs.dl_verdict}}"
    +
    Even the machines know the punchline
    + The honest system's best answer was Part 2's closing advice: run the experiment.
    +
    +
    + +
    +
    Closing
    +

    The END

    +
    Causal inference is one shelf of the toolbox: Bayesian inference is a vast set of tools, and PyMC Labs builds with all of it.
    +
      +
    • Beyond today: demand forecasting, pricing, experimentation at scale, customer lifetime value, hierarchical models across markets: the same machinery, aimed at different decisions.
      • +
    • The cases: pymc-labs.com/blog-posts: every number in this session is pinned to a public post (sources on the next slide).
      • +
    +
    +
    +
    Francesco Muia
    +
    PhD in Theoretical Physics, EMBA.
    Consultant for PyMC Labs and Brown University.
    +
    francesco.muia@pymc-labs.com
    +
    francesco.muia@ai-and-analytics-solutions.com
    +
    +
    +
    Alexander Fengler
    +
    PhD in Computational Cognitive Science.
    Postdoc at Brown University, consultant for PyMC Labs.
    +
    alexander.fengler@pymc-labs.com
    +
    +
    Get in touch
    + If a decision in your company leans on a number nobody quite trusts, write to us: those are exactly the problems we like.
    +
    + + +
    +
    Backup
    + Backup · Sources +

    Every number, pinned

    +
    Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
    +
    + + + + + +
    SourceFacts pinned
    +
    +
    + +
    +
    Backup · Act II · The counterfactual problem, versus machine learning
    +

    "Why not gradient boosting / Prophet / an LSTM?"

    +
    Is this a forecasting exercise?
    +
    +
    +
    ▶ LIVE A kitchen-sink forecaster (donors + trend + seasonality, fit pre-launch only)
    +
    +
    Solid: treated sales. Dashed blue: the forecaster. Dashed grey: the simplex twin. Both fit only the 40 pre-launch weeks.
    +
    +
      +
    • In-sample it fits tighter: €k vs €k per week: flexibility always buys the past.
      • +
    • Yet the estimates are a wash: vs , both near the planted €284k. One dataset cannot rank them.
      • +
    • The sweep decides: across 24 fresh worlds, {{nb07.hull_oos_ratio}}× worse out of sample, with nothing to inspect.
      • +
    +
    +
    +
    Principle 1: a counterfactual, not a forecast
    + A forecaster asks what comes next. We ask what this market would have done in a world that never happened.
    +
      +
    • Principle 2: in-sample fit is the trap, not the goal.
      • +
    • Principle 3: you cannot cross-validate the counterfactual. The truth is never observed.
      • +
    • Principle 4: the simplex ships an auditable claim; a boosted tree is a black box.
      • +
    +
    The forecaster is better at prediction. The simplex is better at the causal job.
    +
    +
    +
    + +
    +
    Backup · Act II · The counterfactual problem
    +

    Interesting limits for the simpler estimators

    +
    Where before/after and treated/control estimators break.
    +
    +
    +
    +
    Before/after · one unit, across time
    + The treated metro's post-average minus its pre-average.
    +
      +
    • The level cancels; nothing subtracts the shared wave.
      • +
    • The whole drift \(\Delta\bar f\) lands on the metro: the largest bias, growing with the horizon.
      • +
    +
    +
    +
    Treated-vs-control · one period, across units
    + The treated metro minus the control average, both in the post window.
    +
      +
    • The level gap \(\alpha_1-\bar\alpha_C\) survives, and sizes differ a lot: it dominates.
      • +
    • This is why raw sales are never compared across cities.
      • +
    +
    +
    +
    ▸ The two bias decompositions, term by term
    +
    + \[ \hat\tau^{\text{B/A}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{1,\text{pre}} \;=\; \bar\tau \;+\; \underbrace{\gamma_1^{\top}\,\Delta\bar f}_{\text{bias: the whole drift}} \;+\; \Delta\bar\varepsilon_1 \] + the level \(\alpha_1\) cancels, but the whole shared-factor drift \(\Delta\bar f\) lands at the metro's own loadings: the largest bias, and it grows with the horizon. +
    +
    + \[ \hat\tau^{\text{TC}} \;=\; \bar Y_{1,\text{post}}-\bar Y_{C,\text{post}} \;=\; \bar\tau \;+\; \underbrace{(\alpha_1-\bar\alpha_C)}_{\text{level gap}} \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\bar f_{\text{post}}}_{\text{loading gap}\,\times\,\text{level}} \;+\; \Delta\bar\varepsilon \] + no time-differencing, so the level gap survives and dominates; the loading gap now multiplies the factor level, not its drift. +
    +
    +
    +
    + +
    +
    Backup · Act II · The counterfactual problem
    +

    Interesting limits for DiD

    +
    Where the DiD estimator breaks.
    +
    +
    The one question
    + DiD only works if treated and controls would have drifted together. Two dials decide (spread \(s\), macro \(\sigma_\eta\)); the algebra is in the fold.
    +
    Synthetic control solves it
    + DiD's residual bias \(B=(\gamma_1-\bar\gamma_C)^{\top}\Delta\bar f\) comes from its equal weights: it hopes \(s\) is small. Synthetic control chooses the weights, so \(B\to0\) at any spread.
    +
    ▸ The DiD bias, decomposed (and why more data cannot shrink it)
    +
    + \[ \hat\tau^{\text{DiD}} \;=\; \bar\tau \;+\; \underbrace{(\gamma_1-\bar\gamma_C)^{\top}\,\Delta\bar f}_{\text{systematic bias }B} \;+\; \underbrace{\Delta\bar\varepsilon_1-\Delta\bar\varepsilon_C}_{\text{idiosyncratic noise}} \] +
      -
    • Today's rate, €{{nb11.cost}}, sits in the GO zone, below the whole interval.
      • -
    • The net, computed: \(Y\) is contribution euros, so one exposure nets \(\hat\beta_{\text{IV}} - c = \) €{{nb11.iv_est}} − €{{nb11.cost}} = €{{nb11.net}}. Even read at the interval floor, €{{nb11.iv_lo}} against €{{nb11.cost}}, the exposure still pays.
      • +
    • The pieces: \(\Delta\bar f\) is the post-minus-pre drift of the three factors, \(\bar\gamma_C\) the \(n\) controls' average loadings, \(\Delta\bar\varepsilon\) the change in idiosyncratic noise.
      • +
    • \(B\) is mean-zero over the exposure draw, but your world is one draw, so in it \(B\) is a fixed nonzero number whose typical size is:
    -
    Why boards like this framing
    - "Is the effect significant?" has no business answer. "Up to what price is this a buy?" has one, and it is the same question a bid cap asks.
    +
    + \[ \operatorname{Var}(B) \;=\; \frac{s^{2}}{3}\Bigl(1+\frac{1}{n}\Bigr)\Bigl[(\Delta\bar f_{\text{trend}})^{2}+(\Delta\bar f_{\text{season}})^{2}+(\Delta\bar f_{\text{walk}})^{2}\Bigr] \] +
    +
      +
    • \(B\) carries no term for the amount of data: \(\Delta\bar f\) is the world's drift, not sampling error, so more weeks shrink only noise and buy precision around the same biased number.
      • +
    • The noise never leaves: the idiosyncratic term carries \(\sigma_\varepsilon \approx 3\) whatever \(s\) and \(\sigma_\eta\) do, so the honest claim in any limit is "the systematic bias vanishes", never "DiD equals the truth".
      • +
    +
    +
    ▸ The two limits, dial by dial: send a knob to zero and read what dies
    +
    +
    Limit 1 · loading spread \(s \to 0\): sufficient on its own
    +
      +
    • The prefactor \(s^2/3\) kills the whole bracket at once.
      • +
    • Every loading \(\to 1\), so \(\gamma_1-\bar\gamma_C\to 0\) deterministically: exact parallel trends, macro walk included.
      • +
    • DiD then recovers \(\bar\tau\) up to noise.
      • +
    +
    Limit 2 · macro shock \(\sigma_\eta \to 0\): not sufficient
    +
      +
    • Only the walk's term \((\Delta\bar f_{\text{walk}})^2\) drops out.
      • +
    • Trend and seasonality still drift between the windows, and markets still weight them differently whenever \(s>0\): \(B \neq 0\).
      • +
    • What it buys: it removes the one stochastic, horizon-growing confounder; the bias that remains is at least deterministic.
      • +
    -
    + +
    ▸ Where the \(\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) prefactor comes from
    + Each loading is drawn from \(\mathrm{U}(1{-}s,\,1{+}s)\), a uniform of width \(2s\), whose variance is \((2s)^2/12 = s^2/3\). The treated metro contributes \(\operatorname{Var}(\gamma_1)=s^2/3\); the average of \(n\) independent controls contributes \(\operatorname{Var}(\bar\gamma_C)=s^2/(3n)\). Independent draws add, so \(\operatorname{Var}(\gamma_1-\bar\gamma_C)=\tfrac{s^2}{3}\bigl(1+\tfrac1n\bigr)\) per factor. Each factor's mismatch is then scaled by that factor's post-minus-pre drift, and the three squared drifts sum: the bracket. +
    +
    -
    -
    IV · The decision
    -

    Poll · the negotiation

    -
    -
    +
    +
    Backup · Act III · Is it real, and how big?
    +

    €260k. Real, or a lucky metro? Build the null yourself

    +
    +
    ✋ Poll
    -
    The platform wants to renegotiate the rate. Your analyst hands you the causal read: effect €{{nb11.iv_est}} per exposure, 90% interval [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. What is the highest rate at which you would still sign "buy" without further study?
    +
    With one treated metro a t-test is not weak, it is undefined. So build the test yourself: rerun the whole pipeline 29 more times, each donor pretended treated ("placebo"): 30 "effects", 29 where nothing ran. Where does our €260k rank?
    - - - - + + +
    -
    C. Below €{{nb11.iv_lo}}, every effect the data support pays: the interval's floor is a no-regret bid cap, defensible whichever value inside the interval turns out to be the truth. B is a break-even gamble: paying the point estimate wins or loses depending on which side of it the truth sits, acceptable only for a risk-neutral buyer averaging over many campaigns. A pays a price that only the single most optimistic supported effect can justify. D leaves money on the table: the whole interval sits well above today's rate.
    +
    A: rank 1 of 30. The rank is the inference: if the campaign did nothing, P(rank 1 by luck) = 1/30 ≈ 0.033. You just re-derived the permutation test.
    -
    -
    IV · The decision · banked
    -

    The verdict and the recommendation

    -
    The complete answer, assembled from everything measured so far.
    -
    -
    - - - - - - - - - -
    QuantityValueSource
    effect of one exposure€{{nb11.iv_est}}the division δ/π
    90% interval[{{nb11.iv_lo}}, {{nb11.iv_hi}}]classical, and AR agrees
    first-stage F{{nb11.f_stat}}the lottery is strong
    price€{{nb11.cost}}the platform's rate card
    net per exposure€{{nb11.net}}β − c, at the point estimate
    -
    The verdict
    - BUY  Keep buying at €{{nb11.cost}}: the entire defensible range clears the price.
    -
    -
    -
    The recommendation, in three lines
    - 1. Keep buying at the €{{nb11.cost}} rate: even the interval's most pessimistic effect pays.
    - 2. Cap the bid at the interval's lower end, €{{nb11.iv_lo}}: up to there, every effect the data support still clears the price.
    - 3. Measure again only if the rate card climbs toward €{{nb11.iv_lo}}: at today's price, no effect inside the interval changes the action, so more measurement is almost certain to leave the decision unchanged and is worth close to nothing here.
    -
      -
    • Everything above is classical: two averages, one division, one F statistic, one confidence interval.
      • -
    • The one debt on record: exclusion is untestable. The recommendation is conditional on the argued design, and says so.
      • -
    +
    +
    Backup · Act III · Is it real, and how big?
    +

    Placebo-in-space: measure the luck directly

    +
    Randomisation inference: the t-test had no standard error, so the 29 donors become the null distribution.
    +
    +
    +
    +
    +
    ▶ LIVE Refit the estimator on every donor as if it were treated
    +
    +
    The null: the "effects" the method reports where nothing happened (spread ·, best placebo ·). Green line: our €260k, outside the cloud.
    +
    +
    +
    +
    The null hypothesis, stated
    + \(H_0\): the campaign did nothing. Then the metro is exchangeable with its donors: any of the 30 ranks is equally likely.
    +
    + \[ p \;=\; \frac{1+\#\{\,j:\ |\hat\tau_j|\ge|\hat\tau_1|\,\}}{J+1} \;=\; \frac{1+0}{30}\;\approx\;0.033 \] +
    +
    -
    +
    What the p-value means (and what it is not)
    + The probability, if \(H_0\) were true, of a gap this extreme: a rank, no bell curve assumed. \(p\) sits at its floor \(1/30\), set by the donor count, not the weeks.
    +
    ▸ When the rank is valid, and when it lies (Abadie's hygiene rule)
    +
      +
    • Valid under \(H_0\) when the placebos are fair stand-ins: comparable pre-launch fit, and no campaign spillover onto donors (SUTVA), the two ways exchangeability can hold.
      • +
    • Lies when a placebo fits its own pre-period badly: it books fitting failure as a giant fake effect and fattens the tail. Abadie's rule: drop those placebos before ranking.
      • +
    +
    Conclusion: the €260k lift is real, at \(p\approx0.033\)
    + Rank 1 of 30, clear of the cloud: reject \(H_0\). The rank holds (p = 0.033 every time) dropping shaky placebos at 2×, 5×, 20× pre-fit error. But "real" is not "profitable": Act IV.
    +
    - -
    -
    Closing · Provenance
    -

    The tools were the product too

    +
    +
    Backup · Act III · Is it real, and how big?
    +

    From a test to an interval: inversion

    +
    Which true lifts could plausibly have produced our €260k? Keep the survivors: no bell curve anywhere.
    -
      -
    • CausalPy: {{labs.causalpy_methods}} in one open-source package. The IV estimator that closes this session joined later.
      • -
    • Its launch example: individual exposure to a TV campaign {{labs.causalpy_tv}}, yet its causal impact remains a core business need: the sentence this whole session opened with.
      • -
    • pymc-marketing: the MMM library behind Case 2's calibration story; one client's {{labs.bolt_pr}} came back as a pull request (Bolt).
      • -
    • Webinars and content: the consultancy's own webinar agenda is this session's syllabus, {{labs.webinar_agenda}} included.
      • -
    -
    - CausalPy - PyMC-Marketing +
    +
    +
    +
    ▶ LIVE Drag \(H\), a guess at the truth: the error cloud slides with it; our €260k never moves.
    +
    +
    + + +
    +
    Axis: 20-week total gap (€000). Top: the placebo errors at truth 0. Middle: the same errors slid to \(H\); \(H\) survives if the green line sits inside the shaded middle 90%. Bottom: the survivors, collected: the interval. Drag past an edge to reject.
    +
    +
    +
    +
    The whole idea
    + Ask of every possible true lift: could it plausibly have produced our €260k? Collect the ones that could, and that set of survivors is the interval.
    +
      +
    • ① Measure the error. The 29 placebo "effects" form a cloud around zero, spread about ±€50k.
      • +
    • ② Guess a truth \(H\). If the lift were \(H\), we would see \(H\) plus that same cloud.
      • +
    • ③ Keep or reject. Keep \(H\) if €260k sits inside its middle 90%.
      • +
    • ④ Sweep. The survivors run €195k to €335k: the interval is [€195k, €335k].
      • +
    +
    +
    ▸ The inversion, in one line of algebra
    +
    + \[ \underbrace{H + q_{0.05} \;\le\; 260 \;\le\; H + q_{0.95}}_{\text{step ③: €260k sits in \(H\)'s middle \(90\%\)}} + \qquad\Longleftrightarrow\qquad + \underbrace{260 - q_{0.95} \;\le\; H \;\le\; 260 - q_{0.05}}_{\text{step ④: the same line, solved for \(H\)}} \] + \(q_{0.05},q_{0.95}\) are just the low and high edges of the error cloud from step ①, here \(q_{0.05}\!=\!-75\) and \(q_{0.95}\!=\!+65\) (€000). Rearranging the left inequality into the right one is the whole trick, and it hands you the endpoints €195k and €335k directly. +
    +
    +
    ▸ Why this interval is the referee for the rest of the lecture
    + It assumed no normality, no independence, no error model, so every model-based interval later (the Bayesian posterior included) has to answer to it. +
    -
    -
    Closing
    -

    The pattern in every engagement

    +
    +
    Backup · Act III · Is it real, and how big?
    +

    Statistics done. Three numbers.

    +
    Questions ① and ② from the boardroom slide are now answered.
    -
      -
    • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
      • -
    • An experiment anchors every observational model: calibration is the product, not a luxury.
      • -
    • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
      • -
    - - +
    Agent, on adversarial MMM dataResult
    + - - + + +
    QuestionAnswerTool that answered it
    Vanilla coding agentFit a model, recommended budget reallocations. {{labs.dl_vanilla}}
    PyMC Labs' Decision Lab{{labs.dl_explored}} Returned: "{{labs.dl_verdict}}"
    Is the effect real?Yes, p = 0.033placebo-in-space permutation: rank 1 of 30
    How big?€260k of incremental salessynthetic-control gap, summed over 20 weeks
    Give or take?[€195k, €335k] at 90%test inversion over the placebo cloud
    -
    Even the machines know the punchline
    - The honest system's best answer was Part 2's closing advice: run the experiment.
    -
    -
    - -
    -
    Closing
    -

    One breath

    -
    -
    The pattern to take home
    - The toolkit a Bayesian consultancy sells: counterfactuals, calibrated by experiments, priced as probabilities.
    -
      -
    • Read the cases: pymc-labs.com/blog-posts: every number in this deck is pinned to a post, listed on the next slide.
      • -
    • Say hello: both authors consult for PyMC Labs; the notebooks behind this session are the course repository.
      • -
    +
    Truth check (only a simulation allows it)
    + The planted total €284k sits inside the interval, €24k above the estimate: the machinery works, and its self-reported uncertainty is honest.
    +
    The sentence that loses money
    + "€260k of sales for €75k, a 3.5× return. Roll it out." Every number true; the conclusion does not follow. Question ③ is not a statistics question.
    - -
    -
    Backup
    - Backup · Sources -

    Every number, pinned

    -
    Part 1 facts retrieved and pinned 2026-07-19 (apps/labs_deck_data.json carries the exact quote); Part 2 numbers are baked from the executed course notebooks (nb07/nb07b shards).
    -
    - - - - - -
    SourceFacts pinned
    -
    +
    +
    Backup · IV · The estimator · hands on
    +

    Why the division is forced

    +
    The division is not a modelling choice. It is the only effect size the two measurements allow.
    +
    +
    +
    ▶ LIVE every candidate effect makes a prediction. One matches.
    +
    +
    + + +
    +
    The rising line is the prediction: an effect of \(\hat\beta\) per exposure implies the lottery should have lifted sales by \(\hat\beta \times \pi\). The flat line is the fact: it lifted them by €{{nb11.reduced}}. Move your guess to the crossing and you have priced the ad.
    +
    +
    +

    Forget the formula and grade any candidate effect \(\hat\beta\) against the two numbers we own:

    +
      +
    • Its prediction: if one exposure were worth \(\hat\beta\), the lottery's {{nb11.first}} extra exposures per win should create \(\hat\beta \times {{nb11.first}}\) euros per win.
      • +
    • The fact: the lottery actually created €{{nb11.reduced}} per win.
      • +
    • The verdict: every candidate except €{{nb11.iv_est}} contradicts a number we measured. The division is the only survivor, not a choice.
      • +
    +
    \[ \hat\beta \times \pi \;\stackrel{!}{=}\; \delta \quad\Longleftrightarrow\quad \hat\beta \;=\; \frac{\delta}{\pi} \]
    +
    Not a black box
    + Every IV estimate is the effect size that makes the instrument's sales bump add up. If you cannot state yours as a ratio of two simple differences, you do not yet understand it.
    +
    +
    @@ -3117,16 +3075,15 @@

    Every number, pinned

    const e=document.getElementById(id); if(e)e.textContent=v;}); const tr=DATA.treated,y0=DATA.y0_true,scl=DATA.synth_cl,ols=DATA.ols_synth,L=DATA.launch,W=tr.length; function draw(){clr(svg);const c=COL();const Wp=720,H=235,mL=46,mR=14,mT=32,mB=26; - let mn=1e9,mx=-1e9;[tr,y0,scl,ols].forEach(a=>a.forEach(v=>{if(vmx)mx=v;})); + let mn=1e9,mx=-1e9;[tr,y0,scl].forEach(a=>a.forEach(v=>{if(vmx)mx=v;})); const x=lin(0,W-1,mL,Wp-mR),y=lin(mn-2,mx+2,H-mB,mT); svg.appendChild(el('line',{x1:x(L),x2:x(L),y1:mT,y2:H-mB,stroke:c.orange,'stroke-width':1.3})); svg.appendChild(el('text',{x:x(L)+4,y:H-mB-6,fill:c.orange,'font-size':10},'launch')); svg.appendChild(el('path',{d:path(tr.map((v,i)=>[x(i),y(v)])),fill:'none',stroke:c.ink,'stroke-width':1.6,opacity:.5})); svg.appendChild(el('path',{d:path(y0.slice(L-1).map((v,i)=>[x(L-1+i),y(v)])),fill:'none',stroke:c.ink,'stroke-width':2,'stroke-dasharray':'6 4'})); svg.appendChild(el('path',{d:path(scl.map((v,i)=>[x(i),y(v)])),fill:'none',stroke:c.blue,'stroke-width':1.9})); - svg.appendChild(el('path',{d:path(ols.map((v,i)=>[x(i),y(v)])),fill:'none',stroke:c.red,'stroke-width':1.9})); const lg=[[c.ink,'treated (observed)',1.6,'none',.5],[c.ink,'true Y(0), post-launch',2,'6 4',1], - [c.blue,'simplex synthetic',1.9,'none',1],[c.red,'OLS synthetic',1.9,'none',1]]; + [c.blue,'simplex synthetic',1.9,'none',1]]; lg.forEach(([col,lab,wd,dash,op],i)=>{const xx=mL+8+i*168; svg.appendChild(el('line',{x1:xx,x2:xx+24,y1:10,y2:10,stroke:col,'stroke-width':wd,'stroke-dasharray':dash,opacity:op})); svg.appendChild(el('text',{x:xx+29,y:14,fill:col,'font-size':10.5,opacity:Math.max(op,.8)},lab));}); @@ -3507,6 +3464,26 @@

    Every number, pinned

    draw(); window.__redraw.push(draw); })(); +/* ==== fig: the metros, one treated (boardroom slide) ==== */ +(function(){ + const svg=document.getElementById('svgMetros'); if(!svg)return; + function draw(){ + clr(svg); const c=COL(); + const donors=[[60,40,7],[80,120,9],[130,30,6],[210,125,7],[220,55,10],[255,95,6],[290,35,8], + [320,120,9],[350,70,7],[385,30,6],[400,105,8],[430,60,11],[465,115,6],[480,35,7],[510,85,9], + [545,40,6],[560,120,8],[590,70,7],[620,105,6],[640,35,9],[665,80,7],[75,70,5],[170,45,5], + [240,20,5],[365,115,5],[450,20,5],[530,115,5],[610,20,5],[680,120,5]]; + donors.forEach(([x,y,r])=>svg.appendChild(el('circle',{cx:x,cy:y,r:r,fill:c.grey,opacity:.28,stroke:c.grey,'stroke-width':1}))); + svg.appendChild(el('circle',{cx:150,cy:70,r:14,fill:c.orange,opacity:.3,stroke:c.orange,'stroke-width':2.2})); + svg.appendChild(el('path',{d:'M 166 52 A 24 24 0 0 1 174 70',fill:'none',stroke:c.orange,'stroke-width':1.6})); + svg.appendChild(el('path',{d:'M 170 45 A 32 32 0 0 1 181 70',fill:'none',stroke:c.orange,'stroke-width':1.2,opacity:.7})); + svg.appendChild(el('text',{x:150,y:105,'text-anchor':'middle','font-size':10.5,fill:c.orange,'font-weight':700},'the treated metro')); + svg.appendChild(el('text',{x:150,y:118,'text-anchor':'middle','font-size':9.5,fill:c.orange},'the €75k campaign, weeks 40-59')); + svg.appendChild(el('text',{x:545,y:16,'text-anchor':'middle','font-size':10.5,fill:c.muted,'font-weight':700},'29 donor markets · no campaign')); + } + draw(); window.__redraw.push(draw); +})(); + /* ==== fig: break the counterfactual yourself (probe slide) ==== */ (function(){ const svg=document.getElementById('svgBreak'); if(!svg)return; From 37bef25595f7d3a951ed2bfa2eb31fb8b8054664 Mon Sep 17 00:00:00 2001 From: Alexander Fengler Date: Tue, 21 Jul 2026 20:16:15 -0400 Subject: [PATCH 5/5] Unified deck round 4 (Alexander): three more slides to Backup, means-on for the data slide, GDPR defined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - "The IV estimate", "Weak instruments (live)" and "The pattern in every engagement" moved to Backup (kickers prefixed; the pattern slide keeps its claims anchor). - The data slide's before/after means overlay is now ON by default, matching the simulate-the-world slide. - GDPR defined at first use on the Nürnberger case slide: "the EU's General Data Protection Regulation, the 2018 privacy law that restricts user-level tracking" (pinned quote fragment kept verbatim for the claim check). Verified: labs claims + sweeps 90/90; node --check OK; 67 slides, 0 JS errors; slides 9 and 13 screenshotted. Co-Authored-By: Claude Fable 5 --- .../apps/unified_slides.html | 238 +++++++++--------- .../apps/unified_slides_src.html | 238 +++++++++--------- 2 files changed, 238 insertions(+), 238 deletions(-) diff --git a/causal-marketing-pymc/apps/unified_slides.html b/causal-marketing-pymc/apps/unified_slides.html index 3586b48..4237e09 100644 --- a/causal-marketing-pymc/apps/unified_slides.html +++ b/causal-marketing-pymc/apps/unified_slides.html @@ -500,7 +500,7 @@

    Nürnberger Versicherung: steering by the last click

    The old ruler: last-touch attributioncredit every sale to the last ad click before it; simple, standard, and blind to everything upstream of that click.
    -
    What broke itunder GDPR, customer journeys appeared artificially shortened: the last click was often the only click the tracker could still see.
    +
    What broke itunder the GDPR (the EU's General Data Protection Regulation, the 2018 privacy law that restricts user-level tracking), customer journeys appeared artificially shortened: the last click was often the only click the tracker could still see.
    The replacement: a funnel-aware causal MMMa marketing-mix model that encodes the funnel: upper-funnel spend creates demand that surfaces later in lower-funnel channels, so credit flows to the cause, not to the final click.
    The stakesbudget follows the ruler: whatever the measurement under-credits, the spreadsheet de-funds.
    @@ -642,7 +642,7 @@

    The data you actually get

    - +
    @@ -1734,103 +1734,7 @@

    The reduced form

    -
    -
    IV · The estimator · the whole method in one division
    -

    The IV estimate

    -
    Euros per lottery win, divided by exposures per lottery win.
    -
    -
    -
    the whole method, as arithmetic on two measured numbers
    -
    -
    One lottery win buys 0.2106 extra exposures and €3.48 of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
    -
    -
    -
    -
    \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ 3.48 }{ 0.2106 } \;=\; \text{€}16.5 \]
    -
      -
    • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
      • -
    • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
      • -
    -
    -
    -
    What just happened
    - We priced the ad using only the random slice of exposure: the dashboard said €23.7, the lottery says €16.5, against a planted truth of €15.
    -
    -
    -
    ▸ Deep dive · the confidence interval around €16.5
    -
    -
    -

    \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

    -
    \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; 16.5 \;\pm\; 1.645 \times 2.31 \;=\; [\,12.7,\; 20.4\,] \] - the 90% confidence interval: the range of effect sizes the data support
    -
      -
    • Interpretation of the interval: any effect between €12.7 and €20.4 is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
      • -
    • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
      • -
    -
    -
    -
    two intervals, graded against the planted truth
    -
    -
    The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
    -
    -
    -
    -
    -
    - -
    -
    IV · When it breaks · the dangerous failure
    -

    Weak instruments

    -
    A weak instrument is worse than no instrument.
    -
    -
    -
    ▶ LIVE what the estimator does as the first stage dies
    -
    -
    - - -
    -
    Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
    -
    -
    -

    The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

    -
      -
    • The honest failure: the interval balloons, and the data admit they know little.
      • -
    • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
      • -
    • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
      • -
    -
    Report the first-stage \(F\), always
    - Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
    -
    -
    -
    ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
    -
    -
    -
    ▶ LIVE test each candidate \(\beta_0\), keep the survivors
    -
    -
    - - -
    -
    The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [12.6, 20.2]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
    -
    -
    -
      -
    • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
      • -
    • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
      • -
    -
    \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
    -
      -
    • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
      • -
    • Here, a good sign: the lottery is strong, so AR [12.6, 20.2] nearly matches the usual [12.7, 20.4]. When \(F\) is small the two part company, and AR is the one still telling the truth.
      • -
    -
    -
    -
    -
    -
    -
    IV · When it breaks · whose effect it is

    Compliers and the LATE

    @@ -1999,27 +1903,6 @@

    The tools were the product too

    -
    -
    Closing
    -

    The pattern in every engagement

    -
    -
      -
    • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
      • -
    • An experiment anchors every observational model: calibration is the product, not a luxury.
      • -
    • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
      • -
    - - - - - - -
    Agent, on adversarial MMM dataResult
    Vanilla coding agentFit a model, recommended budget reallocations. Confidently wrong.
    PyMC Labs' Decision LabExplored 11 approaches, 0 converged. Returned: "No valid model found. Run a geo-holdout experiment."
    -
    Even the machines know the punchline
    - The honest system's best answer was Part 2's closing advice: run the experiment.
    -
    -
    -
    Closing

    The END

    @@ -2327,6 +2210,123 @@

    Why the division is forced

    +
    +
    Backup · IV · The estimator · the whole method in one division
    +

    The IV estimate

    +
    Euros per lottery win, divided by exposures per lottery win.
    +
    +
    +
    the whole method, as arithmetic on two measured numbers
    +
    +
    One lottery win buys 0.2106 extra exposures and €3.48 of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
    +
    +
    +
    +
    \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ 3.48 }{ 0.2106 } \;=\; \text{€}16.5 \]
    +
      +
    • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
      • +
    • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
      • +
    +
    +
    +
    What just happened
    + We priced the ad using only the random slice of exposure: the dashboard said €23.7, the lottery says €16.5, against a planted truth of €15.
    +
    +
    +
    ▸ Deep dive · the confidence interval around €16.5
    +
    +
    +

    \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

    +
    \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; 16.5 \;\pm\; 1.645 \times 2.31 \;=\; [\,12.7,\; 20.4\,] \] + the 90% confidence interval: the range of effect sizes the data support
    +
      +
    • Interpretation of the interval: any effect between €12.7 and €20.4 is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
      • +
    • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
      • +
    +
    +
    +
    two intervals, graded against the planted truth
    +
    +
    The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
    +
    +
    +
    +
    +
    + +
    +
    Backup · IV · When it breaks · the dangerous failure
    +

    Weak instruments

    +
    A weak instrument is worse than no instrument.
    +
    +
    +
    ▶ LIVE what the estimator does as the first stage dies
    +
    +
    + + +
    +
    Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
    +
    +
    +

    The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

    +
      +
    • The honest failure: the interval balloons, and the data admit they know little.
      • +
    • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
      • +
    • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
      • +
    +
    Report the first-stage \(F\), always
    + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
    +
    +
    +
    ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
    +
    +
    +
    ▶ LIVE test each candidate \(\beta_0\), keep the survivors
    +
    +
    + + +
    +
    The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [12.6, 20.2]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
    +
    +
    +
      +
    • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
      • +
    • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
      • +
    +
    \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
    +
      +
    • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
      • +
    • Here, a good sign: the lottery is strong, so AR [12.6, 20.2] nearly matches the usual [12.7, 20.4]. When \(F\) is small the two part company, and AR is the one still telling the truth.
      • +
    +
    +
    +
    +
    +
    + +
    +
    Backup · Closing
    +

    The pattern in every engagement

    +
    +
      +
    • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
      • +
    • An experiment anchors every observational model: calibration is the product, not a luxury.
      • +
    • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
      • +
    + + + + + + +
    Agent, on adversarial MMM dataResult
    Vanilla coding agentFit a model, recommended budget reallocations. Confidently wrong.
    PyMC Labs' Decision LabExplored 11 approaches, 0 converged. Returned: "No valid model found. Run a geo-holdout experiment."
    +
    Even the machines know the punchline
    + The honest system's best answer was Part 2's closing advice: run the experiment.
    +
    +
    + diff --git a/causal-marketing-pymc/apps/unified_slides_src.html b/causal-marketing-pymc/apps/unified_slides_src.html index 4a9e17b..26d91b0 100644 --- a/causal-marketing-pymc/apps/unified_slides_src.html +++ b/causal-marketing-pymc/apps/unified_slides_src.html @@ -500,7 +500,7 @@

    Nürnberger Versicherung: steering by the last click

    The old ruler: last-touch attributioncredit every sale to the last ad click before it; simple, standard, and blind to everything upstream of that click.
    -
    What broke itunder GDPR, {{labs.gdpr_sentence}}: the last click was often the only click the tracker could still see.
    +
    What broke itunder the GDPR (the EU's General Data Protection Regulation, the 2018 privacy law that restricts user-level tracking), {{labs.gdpr_sentence}}: the last click was often the only click the tracker could still see.
    The replacement: a funnel-aware causal MMMa marketing-mix model that encodes the funnel: upper-funnel spend creates demand that surfaces later in lower-funnel channels, so credit flows to the cause, not to the final click.
    The stakesbudget follows the ruler: whatever the measurement under-credits, the spreadsheet de-funds.
    @@ -642,7 +642,7 @@

    The data you actually get

    - +
    @@ -1734,103 +1734,7 @@

    The reduced form

    -
    -
    IV · The estimator · the whole method in one division
    -

    The IV estimate

    -
    Euros per lottery win, divided by exposures per lottery win.
    -
    -
    -
    the whole method, as arithmetic on two measured numbers
    -
    -
    One lottery win buys {{nb11.first}} extra exposures and €{{nb11.reduced}} of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
    -
    -
    -
    -
    \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ {{nb11.reduced}} }{ {{nb11.first}} } \;=\; \text{€}{{nb11.iv_est}} \]
    -
      -
    • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
      • -
    • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
      • -
    -
    -
    -
    What just happened
    - We priced the ad using only the random slice of exposure: the dashboard said €{{nb11.naive}}, the lottery says €{{nb11.iv_est}}, against a planted truth of €{{nb11.true}}.
    -
    -
    -
    ▸ Deep dive · the confidence interval around €{{nb11.iv_est}}
    -
    -
    -

    \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

    -
    \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; {{nb11.iv_est}} \;\pm\; 1.645 \times {{nb11.iv_se}} \;=\; [\,{{nb11.iv_lo}},\; {{nb11.iv_hi}}\,] \] - the 90% confidence interval: the range of effect sizes the data support
    -
      -
    • Interpretation of the interval: any effect between €{{nb11.iv_lo}} and €{{nb11.iv_hi}} is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
      • -
    • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
      • -
    -
    -
    -
    two intervals, graded against the planted truth
    -
    -
    The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
    -
    -
    -
    -
    -
    - -
    -
    IV · When it breaks · the dangerous failure
    -

    Weak instruments

    -
    A weak instrument is worse than no instrument.
    -
    -
    -
    ▶ LIVE what the estimator does as the first stage dies
    -
    -
    - - -
    -
    Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
    -
    -
    -

    The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

    -
      -
    • The honest failure: the interval balloons, and the data admit they know little.
      • -
    • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
      • -
    • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
      • -
    -
    Report the first-stage \(F\), always
    - Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
    -
    -
    -
    ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
    -
    -
    -
    ▶ LIVE test each candidate \(\beta_0\), keep the survivors
    -
    -
    - - -
    -
    The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [{{nb11.ar_lo}}, {{nb11.ar_hi}}]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
    -
    -
    -
      -
    • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
      • -
    • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
      • -
    -
    \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
    -
      -
    • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
      • -
    • Here, a good sign: the lottery is strong, so AR [{{nb11.ar_lo}}, {{nb11.ar_hi}}] nearly matches the usual [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. When \(F\) is small the two part company, and AR is the one still telling the truth.
      • -
    -
    -
    -
    -
    -
    -
    IV · When it breaks · whose effect it is

    Compliers and the LATE

    @@ -1999,27 +1903,6 @@

    The tools were the product too

    -
    -
    Closing
    -

    The pattern in every engagement

    -
    -
      -
    • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
      • -
    • An experiment anchors every observational model: calibration is the product, not a luxury.
      • -
    • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
      • -
    - - - - - - -
    Agent, on adversarial MMM dataResult
    Vanilla coding agentFit a model, recommended budget reallocations. {{labs.dl_vanilla}}
    PyMC Labs' Decision Lab{{labs.dl_explored}} Returned: "{{labs.dl_verdict}}"
    -
    Even the machines know the punchline
    - The honest system's best answer was Part 2's closing advice: run the experiment.
    -
    -
    -
    Closing

    The END

    @@ -2315,6 +2198,123 @@

    Why the division is forced

    +
    +
    Backup · IV · The estimator · the whole method in one division
    +

    The IV estimate

    +
    Euros per lottery win, divided by exposures per lottery win.
    +
    +
    +
    the whole method, as arithmetic on two measured numbers
    +
    +
    One lottery win buys {{nb11.first}} extra exposures and €{{nb11.reduced}} of extra sales. If each exposure is worth \(\beta\), those two facts only fit together for one \(\beta\): the division.
    +
    +
    +
    +
    \[ \hat\beta_{\text{IV}} \;=\; \frac{\delta}{\pi} \;=\; \frac{ {{nb11.reduced}} }{ {{nb11.first}} } \;=\; \text{€}{{nb11.iv_est}} \]
    +
      +
    • Check the units, they are the intuition: euros per win over exposures per win leaves euros per exposure.
      • +
    • Why intent never enters: it cannot correlate with a random draw, so it contributes zero to the numerator and to the denominator alike.
      • +
    +
    +
    +
    What just happened
    + We priced the ad using only the random slice of exposure: the dashboard said €{{nb11.naive}}, the lottery says €{{nb11.iv_est}}, against a planted truth of €{{nb11.true}}.
    +
    +
    +
    ▸ Deep dive · the confidence interval around €{{nb11.iv_est}}
    +
    +
    +

    \(\delta\) and \(\pi\) are both estimates, so both carry noise, and the division hands that noise to \(\hat\beta_{\text{IV}}\). The standard frequentist machinery prices it as a standard error, quoted like every test:

    +
    \[ \hat\beta_{\text{IV}} \;\pm\; 1.645 \times \text{SE} \;=\; {{nb11.iv_est}} \;\pm\; 1.645 \times {{nb11.iv_se}} \;=\; [\,{{nb11.iv_lo}},\; {{nb11.iv_hi}}\,] \] + the 90% confidence interval: the range of effect sizes the data support
    +
      +
    • Interpretation of the interval: any effect between €{{nb11.iv_lo}} and €{{nb11.iv_hi}} is compatible with the measured \(\delta\) and \(\pi\), while anything outside would make them an unlikely accident.
      • +
    • Why it is wider than the naive band: dividing by a first stage below 1 stretches the noise. The width is the honest price of answering the causal question. Precision is not correctness: the naive band is far tighter, and tight around the wrong number.
      • +
    +
    +
    +
    two intervals, graded against the planted truth
    +
    +
    The naive interval is narrow and wrong. The IV interval is wider and contains the truth. In the field the green line is invisible: you choose the method that earns the right to miss it rarely.
    +
    +
    +
    +
    +
    + +
    +
    Backup · IV · When it breaks · the dangerous failure
    +

    Weak instruments

    +
    A weak instrument is worse than no instrument.
    +
    +
    +
    ▶ LIVE what the estimator does as the first stage dies
    +
    +
    + + +
    +
    Each point is the median IV estimate across many repeats at that first-stage \(F\). The band spans the middle 90 percent of the repeats. Drag \(\gamma\) down: the band explodes, and the centre drifts back toward the naive number.
    +
    +
    +

    The method divides by \(\pi\), and as the lottery weakens the ratio fails in two ways:

    +
      +
    • The honest failure: the interval balloons, and the data admit they know little.
      • +
    • The quiet failure: the centre creeps back toward the naive answer. Dividing by a noisy near-zero resurrects exactly the bias the instrument was hired to remove.
      • +
    • No announcement: a weak instrument produces a plausible number with a plausible interval that is quietly wrong.
      • +
    +
    Report the first-stage \(F\), always
    + Below 10, stop: walk away, or use the Anderson and Rubin interval in the deep dive below, which stays honest at any strength.
    +
    +
    +
    ▸ Deep dive · the Anderson and Rubin interval: the repair that survives weakness
    +
    +
    +
    ▶ LIVE test each candidate \(\beta_0\), keep the survivors
    +
    +
    + + +
    +
    The green band is the set of effects the data cannot reject: the Anderson and Rubin confidence set, [{{nb11.ar_lo}}, {{nb11.ar_hi}}]. It never divides by \(\pi\), so a weak instrument cannot corrupt it.
    +
    +
    +
      +
    • The idea: interrogate every candidate price of an exposure, and keep the ones the data cannot call a liar.
      • +
    • The interrogation: if a candidate \(\beta_0\) were the truth, then sales minus \(\beta_0 \times\) exposure should carry no trace of the lottery:
      • +
    +
    \[ Y - \beta_0 X \;\perp\; Z \qquad \text{if } \beta_0 = \beta \]
    +
      +
    • The interval: run that check at 90% confidence for every \(\beta_0\). The survivors are the interval, and no division by \(\pi\) ever happens.
      • +
    • Here, a good sign: the lottery is strong, so AR [{{nb11.ar_lo}}, {{nb11.ar_hi}}] nearly matches the usual [{{nb11.iv_lo}}, {{nb11.iv_hi}}]. When \(F\) is small the two part company, and AR is the one still telling the truth.
      • +
    +
    +
    +
    +
    +
    + +
    +
    Backup · Closing
    +

    The pattern in every engagement

    +
    +
      +
    • The deliverable is a counterfactual: a world minus the launch, the campaign, the exposure: priced in euros.
      • +
    • An experiment anchors every observational model: calibration is the product, not a luxury.
      • +
    • Uncertainty prices the decision: boards act on P(pays) and headroom, not on a point estimate.
      • +
    + + + + + + +
    Agent, on adversarial MMM dataResult
    Vanilla coding agentFit a model, recommended budget reallocations. {{labs.dl_vanilla}}
    PyMC Labs' Decision Lab{{labs.dl_explored}} Returned: "{{labs.dl_verdict}}"
    +
    Even the machines know the punchline
    + The honest system's best answer was Part 2's closing advice: run the experiment.
    +
    +
    +