From b47346754ce4cb605d2a116a9e02f05eec46c618 Mon Sep 17 00:00:00 2001 From: ada-weld Date: Wed, 26 Aug 2026 09:29:59 +0200 Subject: [PATCH 1/2] Zoho CRM SQL templates Four core models over Weld's Zoho CRM connector, in both Weld and dbt dialects: deal pipeline (deal grain, stage classified), deal flow by month, rep activity and account 360. The connector's schema constrains what is buildable, and the models say so rather than papering over it: - Calls, tasks and events lose Zoho's What_Id / Who_Id, so activity is reported per rep and never per deal. Notes keep Parent_Id, which is why account_360 can report a last-touch date at all. - No is_won / is_closed flag, so won and lost are read out of the stage string. assert_deal_stages_are_classified surfaces custom closed stages that would otherwise sit in open pipeline. - No history tables and no stage-change audit, so nothing can reconstruct a past pipeline; the README points at materialising deal_pipeline daily instead. - call_duration is a string that Zoho documents as hh:mm and its UI renders as mm:ss, with no seconds column to settle it. Parsed as documented, flagged as unverified. All 38 files parse as BigQuery, and every raw column referenced was checked against the connector's published ERD. --- README.md | 1 + zoho-crm/README.md | 138 ++++++++++++++++++ zoho-crm/dbt/README.md | 56 +++++++ .../analytics__zoho_crm_account_360.sql | 11 ++ ...analytics__zoho_crm_deal_flow_by_month.sql | 7 + .../analytics__zoho_crm_deal_pipeline.sql | 12 ++ .../analytics__zoho_crm_rep_activity.sql | 11 ++ .../core/core_zoho_crm__account_360.sql | 126 ++++++++++++++++ .../core_zoho_crm__deal_flow_by_month.sql | 117 +++++++++++++++ .../core/core_zoho_crm__deal_pipeline.sql | 98 +++++++++++++ .../core/core_zoho_crm__rep_activity.sql | 119 +++++++++++++++ zoho-crm/dbt/models/staging/sources.yml | 31 ++++ .../models/staging/stg_zoho_crm__account.sql | 21 +++ .../dbt/models/staging/stg_zoho_crm__call.sql | 31 ++++ .../models/staging/stg_zoho_crm__contact.sql | 23 +++ .../dbt/models/staging/stg_zoho_crm__deal.sql | 26 ++++ .../models/staging/stg_zoho_crm__event.sql | 24 +++ .../dbt/models/staging/stg_zoho_crm__lead.sql | 24 +++ .../dbt/models/staging/stg_zoho_crm__note.sql | 27 ++++ .../dbt/models/staging/stg_zoho_crm__task.sql | 24 +++ .../dbt/models/staging/stg_zoho_crm__user.sql | 27 ++++ .../dbt/tests/assert_call_duration_parses.sql | 21 +++ .../assert_deal_stages_are_classified.sql | 25 ++++ .../dbt/tests/assert_no_duplicate_deals.sql | 17 +++ .../dbt/tests/assert_owner_ids_resolve.sql | 22 +++ zoho-crm/weld/analytics/account_360.sql | 9 ++ .../weld/analytics/deal_flow_by_month.sql | 5 + zoho-crm/weld/analytics/deal_pipeline.sql | 10 ++ zoho-crm/weld/analytics/rep_activity.sql | 9 ++ zoho-crm/weld/core/account_360.sql | 126 ++++++++++++++++ zoho-crm/weld/core/deal_flow_by_month.sql | 117 +++++++++++++++ zoho-crm/weld/core/deal_pipeline.sql | 96 ++++++++++++ zoho-crm/weld/core/rep_activity.sql | 119 +++++++++++++++ zoho-crm/weld/staging/account.sql | 21 +++ zoho-crm/weld/staging/call.sql | 31 ++++ zoho-crm/weld/staging/contact.sql | 23 +++ zoho-crm/weld/staging/deal.sql | 26 ++++ zoho-crm/weld/staging/event.sql | 24 +++ zoho-crm/weld/staging/lead.sql | 24 +++ zoho-crm/weld/staging/note.sql | 27 ++++ zoho-crm/weld/staging/task.sql | 24 +++ zoho-crm/weld/staging/user.sql | 27 ++++ 42 files changed, 1757 insertions(+) create mode 100644 zoho-crm/README.md create mode 100644 zoho-crm/dbt/README.md create mode 100644 zoho-crm/dbt/models/analytics/analytics__zoho_crm_account_360.sql create mode 100644 zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_flow_by_month.sql create mode 100644 zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_pipeline.sql create mode 100644 zoho-crm/dbt/models/analytics/analytics__zoho_crm_rep_activity.sql create mode 100644 zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql create mode 100644 zoho-crm/dbt/models/core/core_zoho_crm__deal_flow_by_month.sql create mode 100644 zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql create mode 100644 zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql create mode 100644 zoho-crm/dbt/models/staging/sources.yml create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__account.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__call.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__contact.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__deal.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__event.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__lead.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__note.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__task.sql create mode 100644 zoho-crm/dbt/models/staging/stg_zoho_crm__user.sql create mode 100644 zoho-crm/dbt/tests/assert_call_duration_parses.sql create mode 100644 zoho-crm/dbt/tests/assert_deal_stages_are_classified.sql create mode 100644 zoho-crm/dbt/tests/assert_no_duplicate_deals.sql create mode 100644 zoho-crm/dbt/tests/assert_owner_ids_resolve.sql create mode 100644 zoho-crm/weld/analytics/account_360.sql create mode 100644 zoho-crm/weld/analytics/deal_flow_by_month.sql create mode 100644 zoho-crm/weld/analytics/deal_pipeline.sql create mode 100644 zoho-crm/weld/analytics/rep_activity.sql create mode 100644 zoho-crm/weld/core/account_360.sql create mode 100644 zoho-crm/weld/core/deal_flow_by_month.sql create mode 100644 zoho-crm/weld/core/deal_pipeline.sql create mode 100644 zoho-crm/weld/core/rep_activity.sql create mode 100644 zoho-crm/weld/staging/account.sql create mode 100644 zoho-crm/weld/staging/call.sql create mode 100644 zoho-crm/weld/staging/contact.sql create mode 100644 zoho-crm/weld/staging/deal.sql create mode 100644 zoho-crm/weld/staging/event.sql create mode 100644 zoho-crm/weld/staging/lead.sql create mode 100644 zoho-crm/weld/staging/note.sql create mode 100644 zoho-crm/weld/staging/task.sql create mode 100644 zoho-crm/weld/staging/user.sql diff --git a/README.md b/README.md index c416bc6..49c7b86 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Each integration ships both dialects: | [`shopify/`](shopify/) | sales over time, product sales over time, SKU cost per day, sales by product | | [`amazon-seller-central/`](amazon-seller-central/) | sales over time, product sales over time, settlement ledger, ASIN profitability, traffic and conversion, inventory health | | [`amazon-vendor-central/`](amazon-vendor-central/) | sales over time, sales by ASIN, inventory health, forecast vs actuals | +| [`zoho-crm/`](zoho-crm/) | deal pipeline, deal flow by month, rep activity, account 360 | The rest of the library is at [weld.app/templates](https://weld.app/templates). diff --git a/zoho-crm/README.md b/zoho-crm/README.md new file mode 100644 index 0000000..53ee3b6 --- /dev/null +++ b/zoho-crm/README.md @@ -0,0 +1,138 @@ +# Zoho CRM templates + +Pipeline, activity and account reporting from Weld's Zoho CRM connector. BigQuery, +org reporting currency. + +| Model | Answers | Grain | +|---|---|---| +| [`weld/core/deal_pipeline.sql`](weld/core/deal_pipeline.sql) · [dbt](dbt/models/core/core_zoho_crm__deal_pipeline.sql) | What is open, won and lost, and whose it is | deal | +| [`weld/core/deal_flow_by_month.sql`](weld/core/deal_flow_by_month.sql) · [dbt](dbt/models/core/core_zoho_crm__deal_flow_by_month.sql) | Created / won / lost and win rate over time | month × owner | +| [`weld/core/rep_activity.sql`](weld/core/rep_activity.sql) · [dbt](dbt/models/core/core_zoho_crm__rep_activity.sql) | How much each rep is doing | day × owner | +| [`weld/core/account_360.sql`](weld/core/account_360.sql) · [dbt](dbt/models/core/core_zoho_crm__account_360.sql) | People, pipeline and last touch per account | account | + +## Read this first — what this connector cannot do + +The Zoho CRM connector syncs 12 modules as flat tables. Three limits change what is +worth building, and all three are properties of the connector, not of these models: + +**1. Activities have no parent record.** Zoho's Calls, Events and Tasks modules each +carry `What_Id` / `Who_Id` — the deal, account, contact or lead the activity belongs +to. Neither field is synced. `owner_id` is the only foreign key on all three +streams, so activity can be counted **per rep and never per deal**. There is no +honest "activities per won deal" from this connector alone. + +The exception is **Notes**, whose `Parent_Id` *is* synced. That single column is why +`account_360` can report a last-touch date at all. + +**2. There is no won/lost flag.** Zoho ships no `is_won` or `is_closed` boolean, no +probability and no forecast category. `deal_pipeline` derives the outcome from the +stage *string*. Zoho's own defaults are covered — including `Closed-Lost to +Competition`, which is why the match is `LIKE '%lost%'` and not an equality list — +but a custom stage called `Contract Signed` or `Churned` will read as open pipeline +until you add it. Run +[`assert_deal_stages_are_classified`](dbt/tests/assert_deal_stages_are_classified.sql); +this is the one edit almost every org has to make. + +**3. There is no history.** Weld's Zoho CRM connector does not support history +tables, and the deal stream carries no stage-change audit. Nothing here can tell +you what the pipeline looked like last month, how long a deal sat in Negotiation, +or when a deal actually closed — `closing_date` is the *expected* close date and +Zoho does not clear it on close. **If stage velocity or a pipeline trend matters, +materialise `deal_pipeline` on a daily schedule and keep the runs.** That is the +only route to history, and it only starts working from the day you set it up. + +Two smaller ones worth knowing: + +- **`call_duration` is a string.** Zoho's API reference documents it as `hh:mm`; the + CRM UI shows `mm:ss` for short calls, and the stream carries no + `Call_Duration_in_seconds` column to settle it. `staging/call.sql` parses the + documented `hh:mm`. Reconcile against one call of known length before reporting on + call time. +- **Deletes are not captured.** Zoho's records API does not report deletions, so a + deleted record stops receiving updates and stays in your warehouse. Run a ReSync + on the affected table when the destination has to match Zoho exactly. + +## Layers + +``` +raw.zoho_crm.* ELT output, untouched + ↓ +weld/staging/*.sql 9 thin wrappers: cast, rename, blanks to NULL + ↓ +weld/core/*.sql the reports - all the business logic lives here + ↓ +weld/analytics/*.sql BI-facing contracts, deliberately thin +``` + +**Staging** casts, renames and normalises, and nothing else. It also unpicks Zoho's +lookup flattening: a lookup field arrives as an `_id` / `_name` / `_email` triple, +so `Account_Name` on a deal becomes `account_name_id` — which is the account's id, +despite the name. Staging renames it to `account_id` and drops the denormalised +label, so a renamed rep or account does not leave stale copies of its old name +scattered across every module. + +**Core** holds the stage classification, the date spines and the joins. This is the +layer worth reviewing. `deal_pipeline` is deal-grain and the other three read it. + +**Analytics** exists for indirection, not transformation. Bind dashboards and +reverse-ETL syncs to `analytics.zoho_crm.deal_pipeline` so core stays free to be +renamed or re-grained. All four are `SELECT *` — that is the point, not an oversight. + +With [GitHub Sync](https://weld.app/docs/transformations/github-sync) a push deploys +all of them and Weld resolves the dependency order. Weld references map to folder +paths, so `{{staging.zoho_crm.deal}}` expects the model in a `staging > zoho_crm` +folder — mirror that structure in your synced repo, or adjust the refs. + +## Required tables + +| Model | Raw tables | +|---|---| +| deal pipeline | `deal`, `account`, `user` | +| deal flow by month | the same | +| rep activity | `call`, `event`, `task`, `user` | +| account 360 | `account`, `contact`, `deal`, `user`, `note` | + +**Sync the `user` stream even if you do not think you need it.** It is the only +dimension the other modules can join to, and without it every report groups by a +NULL rep name. [`assert_owner_ids_resolve`](dbt/tests/assert_owner_ids_resolve.sql) +is there to catch exactly that. + +Three synced modules are deliberately not modelled, because nothing in the schema +joins to them: **campaign** (no deal or lead key, and no budget or response +metrics), and **product** and **price_book** (no line items on deals, and no key +between the two). + +## What deal_pipeline returns + +Grain is one row per deal. `stage_status` is `Open`, `Won` or `Lost`, with +`is_open` / `is_won` / `is_lost` alongside it, and `amount` split into +`pipeline_amount`, `won_amount` and `lost_amount` so BI can sum a column instead of +repeating the CASE. + +`age_days` is time-to-close for a decided deal and time-in-pipeline for an open +one. `is_overdue` flags open deals whose expected close date has already passed — +the cheapest pipeline-hygiene number available, and usually the first thing a sales +lead asks for. Account (`account_name`, `industry`) and owner (`owner_name`, +`owner_email`, `owner_role`, `owner_is_active`) are joined on. A deal owned by a +deactivated rep is unmanaged pipeline; `owner_is_active` is how you find it. + +## Notes + +- **`win_rate` excludes open deals from the denominator.** Counting them as + not-yet-won drags every current month down and makes the trend look like a + collapse. Both a count-based and a value-based rate are returned. +- **`deal_flow_by_month` books wins on `closing_date`**, because no real close + timestamp is synced. `modified_time` would be worse: it moves every time anyone + edits the record, so a note added in August would move a June win. +- **Both time-series models are built on a spine**, so a rep with a quiet month + still returns a row of zeros instead of vanishing from the series and letting a + BI line chart interpolate straight over the gap. +- **Multi-org ready.** Every staging model emits a `zoho_org` label and it is part + of every join key. To add an org, `UNION ALL` a second block in each staging model + with a different label — and change nothing else. +- **Treat every model as unverified until you have reconciled it.** Open pipeline + against Zoho's own Deals view is the check that matters; if it disagrees, the + stage classification is where to look first. + +[Zoho CRM connector docs](https://weld.app/docs/applications/zoho-crm) · +[Full walkthrough](https://weld.app/blog/zoho-crm-connector-sql-reports) diff --git a/zoho-crm/dbt/README.md b/zoho-crm/dbt/README.md new file mode 100644 index 0000000..a515a8b --- /dev/null +++ b/zoho-crm/dbt/README.md @@ -0,0 +1,56 @@ +# Zoho CRM dbt models + +Drop-in model files. Not a runnable project — no `dbt_project.yml`, not run in CI. + +``` +models/staging/sources.yml one source, one Zoho org +models/staging/stg_zoho_crm__*.sql 9 thin wrappers over the raw tables +models/core/core_zoho_crm__deal_pipeline.sql every deal, classified - the business logic +models/core/core_zoho_crm__deal_flow_by_month.sql created / won / lost per month per rep +models/core/core_zoho_crm__rep_activity.sql calls, meetings, tasks per rep per day +models/core/core_zoho_crm__account_360.sql account grain: people, pipeline, notes +models/analytics/analytics__zoho_crm_*.sql thin BI-facing contracts +tests/assert_deal_stages_are_classified.sql closed stages hiding in open pipeline +tests/assert_owner_ids_resolve.sql deals whose owner is not in `user` +tests/assert_call_duration_parses.sql duration strings the regex rejected +tests/assert_no_duplicate_deals.sql one row per deal +``` + +Four layers, same as the Weld side: raw → staging → core → analytics. The analytics +models are `SELECT *` by design — dashboards bind to them so core stays free to +change. + +**These are the same models as [`../weld/`](../weld/), differing only in ref +syntax.** They are generated from the Weld versions, so the two dialects cannot +silently disagree about how a deal is classified as won. + +## You supply + +**Your schema.** Change `schema:` in `sources.yml` to wherever your loader lands +the Zoho tables. + +**Your closed stages.** Zoho CRM has no `is_won` or `is_closed` field, so +`core_zoho_crm__deal_pipeline` derives won and lost from the stage *string* +(`LIKE '%won%'` / `'%lost%'`). That covers Zoho's defaults, including +`Closed-Lost to Competition`. It does not cover a custom stage named +`Contract Signed`, `Churned` or `Dead`. Run +`tests/assert_deal_stages_are_classified.sql` and add whatever it surfaces to the +`CASE` in that model — this is the one edit almost every org has to make. + +No vars to set, no packages required — the tests are plain `SELECT`s, so +`dbt_utils` is not needed. + +## Notes + +- **BigQuery.** `GENERATE_DATE_ARRAY`, `COUNTIF`, `SAFE_DIVIDE`, `TIMESTAMP_DIFF` + and `REGEXP_CONTAINS` all need swapping for other engines. +- **`core_zoho_crm__deal_pipeline` is materialized as a table.** The three other + core models plus its own analytics contract all read it; as a view the warehouse + re-scans staging four times. +- **Plain SQL, no macros.** Multi-org works by `UNION ALL` in the staging models: + add a source block per org, union the blocks with different `zoho_org` labels, + and change nothing else. `zoho_org` is already part of every join key. +- **Not run end to end.** Reconcile open pipeline against Zoho's own Deals view + before relying on the output. + +[Full walkthrough](https://weld.app/blog/zoho-crm-connector-sql-reports) diff --git a/zoho-crm/dbt/models/analytics/analytics__zoho_crm_account_360.sql b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_account_360.sql new file mode 100644 index 0000000..a3c6a84 --- /dev/null +++ b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_account_360.sql @@ -0,0 +1,11 @@ +-- analytics__zoho_crm_account_360 +-- BI-facing contract over the core model. See analytics/deal_pipeline.sql for why +-- this layer exists even when it is a passthrough. +-- +-- This is the natural source for a reverse-ETL sync back into Zoho: writing +-- open_pipeline_amount or is_stale_with_open_pipeline onto the Account record puts +-- the warehouse's view in front of the reps who need it. + +{{ config(materialized='view') }} + +SELECT * FROM {{ ref('core_zoho_crm__account_360') }} diff --git a/zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_flow_by_month.sql b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_flow_by_month.sql new file mode 100644 index 0000000..60a13b5 --- /dev/null +++ b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_flow_by_month.sql @@ -0,0 +1,7 @@ +-- analytics__zoho_crm_deal_flow_by_month +-- BI-facing contract over the core model. See analytics/deal_pipeline.sql for why +-- this layer exists even when it is a passthrough. + +{{ config(materialized='view') }} + +SELECT * FROM {{ ref('core_zoho_crm__deal_flow_by_month') }} diff --git a/zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_pipeline.sql b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_pipeline.sql new file mode 100644 index 0000000..84c6ef0 --- /dev/null +++ b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_deal_pipeline.sql @@ -0,0 +1,12 @@ +-- analytics__zoho_crm_deal_pipeline +-- BI-facing contract over the core model. Dashboards, scheduled reports and +-- reverse-ETL syncs bind HERE, never to core, so core stays free to be renamed, +-- re-grained or split without breaking anything downstream. +-- +-- A passthrough is the correct content for this layer. Put BI-specific shaping +-- (renames for a semantic layer, row filters for a workspace) in this file rather +-- than in core. + +{{ config(materialized='view') }} + +SELECT * FROM {{ ref('core_zoho_crm__deal_pipeline') }} diff --git a/zoho-crm/dbt/models/analytics/analytics__zoho_crm_rep_activity.sql b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_rep_activity.sql new file mode 100644 index 0000000..8aab66b --- /dev/null +++ b/zoho-crm/dbt/models/analytics/analytics__zoho_crm_rep_activity.sql @@ -0,0 +1,11 @@ +-- analytics__zoho_crm_rep_activity +-- BI-facing contract over the core model. See analytics/deal_pipeline.sql for why +-- this layer exists even when it is a passthrough. +-- +-- Reminder for whoever binds a dashboard to this: activity here is per rep only. +-- There is no deal or account key on Zoho's call, task and event streams in this +-- connector, so do not label a tile "activity per opportunity". + +{{ config(materialized='view') }} + +SELECT * FROM {{ ref('core_zoho_crm__rep_activity') }} diff --git a/zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql b/zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql new file mode 100644 index 0000000..5091694 --- /dev/null +++ b/zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql @@ -0,0 +1,126 @@ +-- zoho_crm_account_360 - one row per account, with its people, pipeline and notes. +-- Grain: account. Currency: org reporting currency. +-- Depends on: stg_zoho_crm__account, .contact, .user, .note, +-- core_zoho_crm__deal_pipeline +-- +-- The account-level view the rest of the schema can actually support: who works +-- there, what is open, what has been won, and when anybody last wrote anything +-- down. This is the model to reverse-ETL back into Zoho, or to join to product +-- usage and billing data for a real customer view. +-- +-- NOT IN HERE: calls, meetings and tasks. They carry no account key in this +-- connector - see core_zoho_crm__rep_activity for why. last_note_at is the closest +-- honest proxy for account engagement the connector allows, and it only reflects +-- what reps bothered to write down. + +WITH contacts AS ( + SELECT + zoho_org, + account_id, + COUNT(*) AS contacts, + COUNTIF(email IS NOT NULL) AS contacts_with_email, + MAX(created_time) AS last_contact_added_at + FROM {{ ref('stg_zoho_crm__contact') }} + WHERE account_id IS NOT NULL + GROUP BY 1, 2 +), + +deals AS ( + SELECT + zoho_org, + account_id, + COUNT(*) AS deals_total, + COUNTIF(is_open) AS deals_open, + COUNTIF(is_won) AS deals_won, + COUNTIF(is_lost) AS deals_lost, + COUNTIF(is_overdue) AS deals_overdue, + SUM(pipeline_amount) AS open_pipeline_amount, + SUM(won_amount) AS won_amount, + SUM(lost_amount) AS lost_amount, + MIN(CASE WHEN is_open THEN closing_date END) AS next_expected_close, + MAX(created_time) AS last_deal_created_at + FROM {{ ref('core_zoho_crm__deal_pipeline') }} + WHERE account_id IS NOT NULL + GROUP BY 1, 2 +), + +-- Notes reach an account three ways: written on the account itself, on one of its +-- deals, or on one of its contacts. Zoho ids are globally unique across modules, +-- so one join per route resolves the parent without needing $se_module. +note_targets AS ( + SELECT a.zoho_org, a.account_id, a.account_id AS target_id + FROM {{ ref('stg_zoho_crm__account') }} a + UNION ALL + SELECT d.zoho_org, d.account_id, d.deal_id + FROM {{ ref('core_zoho_crm__deal_pipeline') }} d + WHERE d.account_id IS NOT NULL + UNION ALL + SELECT c.zoho_org, c.account_id, c.contact_id + FROM {{ ref('stg_zoho_crm__contact') }} c + WHERE c.account_id IS NOT NULL +), + +notes AS ( + SELECT + t.zoho_org, + t.account_id, + COUNT(*) AS notes, + MAX(n.created_time) AS last_note_at + FROM {{ ref('stg_zoho_crm__note') }} n + JOIN note_targets t + ON t.zoho_org = n.zoho_org + AND t.target_id = n.parent_id + GROUP BY 1, 2 +) + +SELECT + a.zoho_org, + a.account_id, + a.account_name, + a.industry, + a.website, + a.phone, + a.created_time AS account_created_time, + + a.owner_id, + u.full_name AS owner_name, + u.email AS owner_email, + u.is_active AS owner_is_active, + + COALESCE(c.contacts, 0) AS contacts, + COALESCE(c.contacts_with_email, 0) AS contacts_with_email, + + COALESCE(d.deals_total, 0) AS deals_total, + COALESCE(d.deals_open, 0) AS deals_open, + COALESCE(d.deals_won, 0) AS deals_won, + COALESCE(d.deals_lost, 0) AS deals_lost, + COALESCE(d.deals_overdue, 0) AS deals_overdue, + COALESCE(d.open_pipeline_amount, 0) AS open_pipeline_amount, + COALESCE(d.won_amount, 0) AS won_amount, + COALESCE(d.lost_amount, 0) AS lost_amount, + d.next_expected_close, + + COALESCE(n.notes, 0) AS notes, + n.last_note_at, + + -- Latest of anything datable on the account. Note the absence of calls and + -- meetings: this is "last recorded touch", not "last contact". + GREATEST( + COALESCE(n.last_note_at, TIMESTAMP '1970-01-01'), + COALESCE(d.last_deal_created_at, TIMESTAMP '1970-01-01'), + COALESCE(c.last_contact_added_at, TIMESTAMP '1970-01-01'), + a.created_time + ) AS last_recorded_activity_at, + + -- An account with open pipeline and nothing written on it in a quarter is the + -- report this model exists to produce. + COALESCE(d.deals_open, 0) > 0 + AND COALESCE(n.last_note_at, a.created_time) + < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY) AS is_stale_with_open_pipeline +FROM {{ ref('stg_zoho_crm__account') }} a +LEFT JOIN contacts c ON c.zoho_org = a.zoho_org AND c.account_id = a.account_id +LEFT JOIN deals d ON d.zoho_org = a.zoho_org AND d.account_id = a.account_id +LEFT JOIN notes n ON n.zoho_org = a.zoho_org AND n.account_id = a.account_id +LEFT JOIN {{ ref('stg_zoho_crm__user') }} u + ON u.zoho_org = a.zoho_org AND u.user_id = a.owner_id +ORDER BY open_pipeline_amount DESC diff --git a/zoho-crm/dbt/models/core/core_zoho_crm__deal_flow_by_month.sql b/zoho-crm/dbt/models/core/core_zoho_crm__deal_flow_by_month.sql new file mode 100644 index 0000000..99810f5 --- /dev/null +++ b/zoho-crm/dbt/models/core/core_zoho_crm__deal_flow_by_month.sql @@ -0,0 +1,117 @@ +-- zoho_crm_deal_flow_by_month - deals created, won and lost per month per rep. +-- Grain: month x owner. Currency: org reporting currency. +-- Depends on: core_zoho_crm__deal_pipeline +-- +-- Built on a month spine so a rep with no activity in a month still returns a row +-- of zeros rather than disappearing from the series - otherwise a BI line chart +-- interpolates straight over the quiet month and the gap is invisible. +-- +-- WHICH DATE EACH EVENT LANDS ON: +-- created -> created_time, which is a real audit timestamp. +-- won/lost -> closing_date, which is Zoho's EXPECTED close date. +-- +-- That second one is the compromise this model cannot avoid. The connector syncs +-- no actual close timestamp and no stage history, so closing_date is the only +-- close date available. If a rep sets a deal to Closed Won in June while its +-- closing_date still says March, this model books the win in March. modified_time +-- would move it to June, but modified_time changes on ANY edit, so a note added in +-- August would move the win again. A forecast date that is wrong once beats a date +-- that moves every time somebody touches the record. +-- +-- The fix is upstream, not here: materialise deal_pipeline daily and derive real +-- close dates from when stage_status first changed. + +WITH month_spine AS ( + SELECT month + FROM UNNEST(GENERATE_DATE_ARRAY( + -- Start the spine at the first deal rather than a hardcoded date, so the + -- series never carries years of empty leading months. + (SELECT DATE_TRUNC(MIN(created_date), MONTH) FROM {{ ref('core_zoho_crm__deal_pipeline') }}), + -- End a year out: closing_date is a forecast, so pipeline legitimately + -- sits in the future and must not be truncated away. + DATE_TRUNC(DATE_ADD(CURRENT_DATE(), INTERVAL 1 YEAR), MONTH), + INTERVAL 1 MONTH + )) AS month +), + +owners AS ( + -- Only reps who actually appear on a deal. Spining every Zoho user against + -- every month would pad the output with support and admin logins. + SELECT DISTINCT zoho_org, owner_id, owner_name + FROM {{ ref('core_zoho_crm__deal_pipeline') }} +), + +grid AS ( + SELECT s.month, o.zoho_org, o.owner_id, o.owner_name + FROM month_spine s + CROSS JOIN owners o +), + +created AS ( + SELECT + DATE_TRUNC(created_date, MONTH) AS month, + zoho_org, + owner_id, + COUNT(*) AS deals_created, + SUM(amount) AS deals_created_amount + FROM {{ ref('core_zoho_crm__deal_pipeline') }} + GROUP BY 1, 2, 3 +), + +closed AS ( + SELECT + DATE_TRUNC(closing_date, MONTH) AS month, + zoho_org, + owner_id, + COUNTIF(is_won) AS deals_won, + SUM(won_amount) AS deals_won_amount, + COUNTIF(is_lost) AS deals_lost, + SUM(lost_amount) AS deals_lost_amount, + -- Open deals forecast to close in this month. Not an outcome - a promise. + COUNTIF(is_open) AS deals_forecast, + SUM(pipeline_amount) AS pipeline_amount + FROM {{ ref('core_zoho_crm__deal_pipeline') }} + WHERE closing_date IS NOT NULL + GROUP BY 1, 2, 3 +) + +SELECT + g.month, + g.zoho_org, + g.owner_id, + g.owner_name, + + COALESCE(c.deals_created, 0) AS deals_created, + COALESCE(c.deals_created_amount, 0) AS deals_created_amount, + + COALESCE(x.deals_won, 0) AS deals_won, + COALESCE(x.deals_won_amount, 0) AS deals_won_amount, + COALESCE(x.deals_lost, 0) AS deals_lost, + COALESCE(x.deals_lost_amount, 0) AS deals_lost_amount, + + COALESCE(x.deals_forecast, 0) AS deals_forecast, + COALESCE(x.pipeline_amount, 0) AS pipeline_amount, + + -- Win rate by count, over decided deals only. Open deals are excluded from the + -- denominator: counting them as not-yet-won drags every current month down and + -- makes the trend look like a collapse. + SAFE_DIVIDE( + COALESCE(x.deals_won, 0), + COALESCE(x.deals_won, 0) + COALESCE(x.deals_lost, 0) + ) AS win_rate, + SAFE_DIVIDE( + COALESCE(x.deals_won_amount, 0), + COALESCE(x.deals_won_amount, 0) + COALESCE(x.deals_lost_amount, 0) + ) AS win_rate_by_value, + -- Average won deal size, the other half of any quota conversation. + SAFE_DIVIDE(COALESCE(x.deals_won_amount, 0), NULLIF(x.deals_won, 0)) AS average_won_deal_size +FROM grid g +LEFT JOIN created c + ON c.month = g.month + AND c.zoho_org = g.zoho_org + AND c.owner_id = g.owner_id +LEFT JOIN closed x + ON x.month = g.month + AND x.zoho_org = g.zoho_org + AND x.owner_id = g.owner_id +ORDER BY g.month, g.owner_name diff --git a/zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql b/zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql new file mode 100644 index 0000000..8cff8bb --- /dev/null +++ b/zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql @@ -0,0 +1,98 @@ +-- zoho_crm_deal_pipeline - every deal, classified, with its account and owner. +-- Grain: one row per deal. Currency: org reporting currency (see scope note). +-- Depends on: stg_zoho_crm__deal, .account, .user +-- +-- This is the deal-grain model the other reports read. Classify once here rather +-- than repeating the stage logic in every downstream query. +-- +-- SCOPE: current state, not history. Zoho CRM has no history tables in Weld, and +-- this connector syncs no stage-change audit, so there is no way to ask what the +-- pipeline looked like last Tuesday or how long a deal sat in Negotiation. Every +-- deal shows only where it stands now. If you need stage velocity or a pipeline +-- snapshot over time, materialise this model daily and keep the runs. +-- +-- Joins are written with explicit ON rather than USING: both deal and account +-- carry owner_id, so a USING(owner_id) join further down would be ambiguous. + +{{ config(materialized='table') }} + +WITH deal AS ( + SELECT + d.*, + -- Zoho ships no is_won / is_closed boolean, so won and lost have to be + -- read out of the stage string. Matched on a pattern, not an equality + -- list, because Zoho's own defaults include "Closed-Lost to Competition" + -- and most orgs add their own stages on top. + -- + -- Won is tested first so a stage that somehow contains both words is + -- counted once. tests/assert_deal_stages_are_classified.sql lists every + -- stage falling through to Open - read it before trusting the split. + CASE + WHEN LOWER(d.stage) LIKE '%won%' THEN 'Won' + WHEN LOWER(d.stage) LIKE '%lost%' THEN 'Lost' + ELSE 'Open' + END AS stage_status + FROM {{ ref('stg_zoho_crm__deal') }} d +) + +SELECT + d.zoho_org, + d.deal_id, + d.deal_name, + d.stage, + d.stage_status, + d.stage_status = 'Open' AS is_open, + d.stage_status = 'Won' AS is_won, + d.stage_status = 'Lost' AS is_lost, + + d.amount, + -- Split out so BI can sum a column instead of writing the CASE again. An open + -- deal contributes to pipeline_amount only; a closed one to won or lost. + CASE WHEN d.stage_status = 'Open' THEN d.amount END AS pipeline_amount, + CASE WHEN d.stage_status = 'Won' THEN d.amount END AS won_amount, + CASE WHEN d.stage_status = 'Lost' THEN d.amount END AS lost_amount, + + DATE(d.created_time) AS created_date, + d.closing_date, + d.created_time, + d.modified_time, + + -- Age of the deal. For a closed deal this is how long it took; for an open one + -- how long it has been sitting. closing_date is Zoho's EXPECTED close date and + -- it is not cleared when a deal closes, so on a won or lost deal it is the + -- forecast that was in place, not necessarily the day money changed hands. + DATE_DIFF( + COALESCE(CASE WHEN d.stage_status <> 'Open' THEN d.closing_date END, CURRENT_DATE()), + DATE(d.created_time), + DAY + ) AS age_days, + -- Open deals whose expected close date has already passed: the cheapest + -- pipeline-hygiene number there is, and usually the first thing a sales lead + -- asks for. + d.stage_status = 'Open' AND d.closing_date < CURRENT_DATE() AS is_overdue, + CASE + WHEN d.stage_status = 'Open' + THEN DATE_DIFF(d.closing_date, CURRENT_DATE(), DAY) + END AS days_to_expected_close, + + d.account_id, + a.account_name, + a.industry, + + d.owner_id, + u.full_name AS owner_name, + u.email AS owner_email, + u.role_name AS owner_role, + -- A deal owned by a deactivated rep is unmanaged pipeline. NULL here means the + -- owner_id did not resolve at all - see tests/assert_owner_ids_resolve.sql. + u.is_active AS owner_is_active +FROM deal d +LEFT JOIN {{ ref('stg_zoho_crm__account') }} a + ON a.zoho_org = d.zoho_org + AND a.account_id = d.account_id +-- LEFT, not INNER: an owner who has been deleted from Zoho no longer appears in +-- the user module, and an inner join would silently drop their deals from the +-- pipeline total. +LEFT JOIN {{ ref('stg_zoho_crm__user') }} u + ON u.zoho_org = d.zoho_org + AND u.user_id = d.owner_id diff --git a/zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql b/zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql new file mode 100644 index 0000000..9a74207 --- /dev/null +++ b/zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql @@ -0,0 +1,119 @@ +-- zoho_crm_rep_activity - calls, meetings and tasks logged per rep per day. +-- Grain: day x owner. +-- Depends on: stg_zoho_crm__call, .event, .task, .user +-- +-- READ THIS BEFORE USING IT. This model counts activity per REP, never per deal, +-- account, contact or lead. That is not a design choice - Zoho's Calls, Events and +-- Tasks modules all carry What_Id / Who_Id pointing at the record the activity +-- belongs to, and the Weld connector does not sync either field. owner_id is the +-- only foreign key on all three streams. +-- +-- So this model answers "how much is each rep doing" and cannot answer "how much +-- activity did we put into the deals we won". Any dashboard promising +-- activity-to-outcome attribution from this connector alone is inventing the link. +-- Until the connector syncs the parent ids, the join has to come from somewhere +-- else - a calendar or dialer source keyed on email, or Zoho's Notes module, whose +-- parent_id_id IS synced. +-- +-- WHY THERE IS NO tasks_completed COLUMN: the connector syncs is_completed but no +-- completion timestamp. Completions can be counted as a current-state total, never +-- placed on a day. Booking them on modified_time would move a task's completion +-- every time anyone edited it afterwards. + +WITH activity AS ( + SELECT zoho_org, owner_id, DATE(call_start_time) AS activity_date, + 1 AS calls_logged, call_duration_minutes AS call_minutes, + 0 AS events_held, 0 AS event_minutes, + 0 AS tasks_created + FROM {{ ref('stg_zoho_crm__call') }} + WHERE call_start_time IS NOT NULL + + UNION ALL + + SELECT zoho_org, owner_id, DATE(start_date_time) AS activity_date, + 0, 0, + 1 AS events_held, duration_minutes AS event_minutes, + 0 + FROM {{ ref('stg_zoho_crm__event') }} + WHERE start_date_time IS NOT NULL + + UNION ALL + + -- Tasks land on the day they were CREATED, not their due date. Due dates get + -- pushed; creation is when the rep actually did something. + SELECT zoho_org, owner_id, DATE(created_time) AS activity_date, + 0, 0, + 0, 0, + 1 AS tasks_created + FROM {{ ref('stg_zoho_crm__task') }} + WHERE created_time IS NOT NULL +), + +rolled_up AS ( + SELECT + activity_date, + zoho_org, + owner_id, + SUM(calls_logged) AS calls_logged, + SUM(call_minutes) AS call_minutes, + SUM(events_held) AS events_held, + SUM(event_minutes) AS event_minutes, + SUM(tasks_created) AS tasks_created + FROM activity + GROUP BY 1, 2, 3 +), + +date_spine AS ( + SELECT day AS activity_date + FROM UNNEST(GENERATE_DATE_ARRAY( + (SELECT MIN(activity_date) FROM rolled_up), + CURRENT_DATE(), + INTERVAL 1 DAY + )) AS day +), + +owners AS ( + -- Reps who have logged anything at all. Spining every Zoho user against every + -- day would bury the active team under admin and integration logins. + SELECT DISTINCT zoho_org, owner_id FROM rolled_up +), + +grid AS ( + SELECT s.activity_date, o.zoho_org, o.owner_id + FROM date_spine s + CROSS JOIN owners o +) + +SELECT + g.activity_date, + g.zoho_org, + g.owner_id, + u.full_name AS owner_name, + u.email AS owner_email, + u.role_name AS owner_role, + u.is_active AS owner_is_active, + + COALESCE(r.calls_logged, 0) AS calls_logged, + -- NULL rather than 0 when no call was logged: a zero here would drag down any + -- average-call-length metric computed over the column. + r.call_minutes, + COALESCE(r.events_held, 0) AS events_held, + r.event_minutes, + COALESCE(r.tasks_created, 0) AS tasks_created, + + COALESCE(r.calls_logged, 0) + + COALESCE(r.events_held, 0) + + COALESCE(r.tasks_created, 0) AS total_activities, + -- Touchpoints that involved a human on the other end, which is usually the + -- number a sales lead actually wants out of an activity report. + COALESCE(r.calls_logged, 0) + + COALESCE(r.events_held, 0) AS live_touchpoints +FROM grid g +LEFT JOIN rolled_up r + ON r.activity_date = g.activity_date + AND r.zoho_org = g.zoho_org + AND r.owner_id = g.owner_id +LEFT JOIN {{ ref('stg_zoho_crm__user') }} u + ON u.zoho_org = g.zoho_org + AND u.user_id = g.owner_id +ORDER BY g.activity_date, owner_name diff --git a/zoho-crm/dbt/models/staging/sources.yml b/zoho-crm/dbt/models/staging/sources.yml new file mode 100644 index 0000000..2a2169b --- /dev/null +++ b/zoho-crm/dbt/models/staging/sources.yml @@ -0,0 +1,31 @@ +# Point `schema` at whatever schema your loader lands the Zoho CRM tables in. +# One source block per Zoho org. The staging models label rows with zoho_org and +# core carries that through every join, so adding an org is: add a block here, +# UNION ALL it in the staging models with a different label, done. +# +# No history tables listed. Weld's Zoho CRM connector does not support them +# (history: false on the connector's feature table), which is why none of these +# models can reconstruct a past pipeline - see core_zoho_crm__deal_pipeline. + +version: 2 + +sources: + - name: zoho_crm + schema: raw_zoho_crm + tables: + - name: account + - name: contact + - name: deal + - name: lead + # `user` is not optional: it is the only dimension the other streams can + # join to, and without it every report groups by a NULL rep name. + - name: user + - name: call + - name: event + - name: task + # The only activity stream that keeps its parent record id. + - name: note + +# Synced by the connector but not modelled here, because nothing in the schema +# joins to them: campaign (no deal or lead key), product and price_book (no line +# items, and no key between the two). diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__account.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__account.sql new file mode 100644 index 0000000..f2bb59b --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__account.sql @@ -0,0 +1,21 @@ +-- stg_zoho_crm__account +-- Thin wrapper over raw `account`. Casts, renames, normalises blanks to NULL. +-- +-- Single Zoho org. To add another, UNION ALL a second block below pointing at that +-- org's connector with a different zoho_org label. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS account_id, + NULLIF(TRIM(CAST(account_name AS STRING)), '') AS account_name, + NULLIF(TRIM(CAST(industry AS STRING)), '') AS industry, + NULLIF(TRIM(CAST(phone AS STRING)), '') AS phone, + NULLIF(TRIM(CAST(website AS STRING)), '') AS website, + -- Zoho flattens its lookup fields into _id / _name / _email triples. Keep the + -- id for joining and drop the denormalised name - stg_zoho_crm__user is the + -- single source of truth for what a user is called, so a rep who is renamed in + -- Zoho does not leave stale labels scattered across every module. + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'account') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__call.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__call.sql new file mode 100644 index 0000000..ebf90a8 --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__call.sql @@ -0,0 +1,31 @@ +-- stg_zoho_crm__call +-- Thin wrapper over raw `call`. Casts, renames, parses the duration string. +-- +-- NO PARENT LINK. Same as task: Zoho's Calls module has What_Id / Who_Id, the +-- connector does not sync them. owner_id is the only key. +-- +-- DURATION IS A STRING, NOT A NUMBER. `call_duration` arrives as text with one +-- colon. Zoho's API reference documents the field as hh:mm; the CRM UI shows +-- mm:ss for short calls, and there is no Call_Duration_in_seconds column on this +-- stream to disambiguate. This model parses it as the DOCUMENTED hh:mm. +-- +-- Reconcile call_duration_minutes against one call of known length before +-- reporting on it. If your org turns out to emit mm:ss, divide by 60. +-- tests/assert_call_duration_parses.sql guards the shape, not the unit. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS call_id, + NULLIF(TRIM(CAST(subject AS STRING)), '') AS subject, + NULLIF(TRIM(CAST(call_type AS STRING)), '') AS call_type, + CAST(call_start_time AS TIMESTAMP) AS call_start_time, + CAST(call_duration AS STRING) AS call_duration_raw, + CASE + WHEN REGEXP_CONTAINS(CAST(call_duration AS STRING), r'^\s*\d+:\d{1,2}\s*$') + THEN SAFE_CAST(SPLIT(TRIM(CAST(call_duration AS STRING)), ':')[OFFSET(0)] AS INT64) * 60 + + SAFE_CAST(SPLIT(TRIM(CAST(call_duration AS STRING)), ':')[OFFSET(1)] AS INT64) + END AS call_duration_minutes, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'call') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__contact.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__contact.sql new file mode 100644 index 0000000..6d92f1b --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__contact.sql @@ -0,0 +1,23 @@ +-- stg_zoho_crm__contact +-- Thin wrapper over raw `contact`. Casts, renames, builds full_name. +-- +-- NOTE ON THE ACCOUNT KEY: Zoho's lookup field is called Account_Name, so the +-- connector emits `account_name_id` (the account's id) and `account_name_name` +-- (its label). The _id column is the real foreign key despite the name. It is +-- renamed to account_id here so core reads like a normal star schema. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS contact_id, + CAST(first_name AS STRING) AS first_name, + CAST(last_name AS STRING) AS last_name, + NULLIF(TRIM(CONCAT( + COALESCE(CAST(first_name AS STRING), ''), ' ', + COALESCE(CAST(last_name AS STRING), '') + )), '') AS full_name, + LOWER(NULLIF(TRIM(CAST(email AS STRING)), '')) AS email, + CAST(account_name_id AS STRING) AS account_id, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'contact') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__deal.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__deal.sql new file mode 100644 index 0000000..7c74168 --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__deal.sql @@ -0,0 +1,26 @@ +-- stg_zoho_crm__deal +-- Thin wrapper over raw `deal`. Casts, renames, normalises blanks. Stage +-- classification is business logic and lives in core_zoho_crm__deal_pipeline. +-- +-- ACCOUNT KEY: as on contact, Zoho's lookup is called Account_Name, so the raw +-- column is `account_name_id`. It holds the account's id and is renamed here. +-- +-- WHAT IS NOT HERE, because the connector does not sync it: no is_won / is_closed +-- boolean, no probability, no expected_revenue, no currency, no lead_source, no +-- contact_id, no campaign_id and no pipeline name. Won/lost has to be derived from +-- the stage string, and amount is in whatever single currency the org reports in. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS deal_id, + NULLIF(TRIM(CAST(deal_name AS STRING)), '') AS deal_name, + NULLIF(TRIM(CAST(stage AS STRING)), '') AS stage, + -- No currency column exists on the stream, so this is org-reporting currency. + -- A multi-currency Zoho org cannot be summed correctly from this connector. + CAST(amount AS NUMERIC) AS amount, + CAST(closing_date AS DATE) AS closing_date, + CAST(account_name_id AS STRING) AS account_id, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'deal') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__event.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__event.sql new file mode 100644 index 0000000..36cd391 --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__event.sql @@ -0,0 +1,24 @@ +-- stg_zoho_crm__event +-- Thin wrapper over raw `event` (Zoho's Meetings module). Casts, renames, derives +-- duration from the start and end timestamps. +-- +-- NO PARENT LINK. Same as task and call: owner_id is the only foreign key. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS event_id, + NULLIF(TRIM(CAST(event_title AS STRING)), '') AS event_title, + CAST(start_date_time AS TIMESTAMP) AS start_date_time, + CAST(end_date_time AS TIMESTAMP) AS end_date_time, + -- Unlike call, event carries both ends, so duration is computed rather than + -- parsed out of a string and the unit is unambiguous. + TIMESTAMP_DIFF( + CAST(end_date_time AS TIMESTAMP), + CAST(start_date_time AS TIMESTAMP), + MINUTE + ) AS duration_minutes, + NULLIF(TRIM(CAST(location AS STRING)), '') AS location, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'event') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__lead.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__lead.sql new file mode 100644 index 0000000..c35f2d1 --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__lead.sql @@ -0,0 +1,24 @@ +-- stg_zoho_crm__lead +-- Thin wrapper over raw `lead`. Casts, renames, builds full_name. +-- +-- Leads are a terminal island in this schema. The connector syncs no +-- Converted_Account / Converted_Contact / Converted_Deal fields, so a lead cannot +-- be followed into the deal it became. Lead reporting here is volume and status +-- only - see the README before promising a lead-to-deal conversion rate. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS lead_id, + CAST(first_name AS STRING) AS first_name, + CAST(last_name AS STRING) AS last_name, + NULLIF(TRIM(CONCAT( + COALESCE(CAST(first_name AS STRING), ''), ' ', + COALESCE(CAST(last_name AS STRING), '') + )), '') AS full_name, + LOWER(NULLIF(TRIM(CAST(email AS STRING)), '')) AS email, + NULLIF(TRIM(CAST(company AS STRING)), '') AS company, + NULLIF(TRIM(CAST(lead_status AS STRING)), '') AS lead_status, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'lead') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__note.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__note.sql new file mode 100644 index 0000000..1565e0b --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__note.sql @@ -0,0 +1,27 @@ +-- stg_zoho_crm__note +-- Thin wrapper over raw `note`. Casts, renames, normalises blanks. +-- +-- THE ONE STREAM WITH A REAL PARENT LINK. Every other activity module (call, task, +-- event) loses its What_Id / Who_Id in this connector, but note keeps its parent: +-- Zoho's Parent_Id lookup arrives as `parent_id_id`, and it is the only way to tie +-- anything a rep wrote down back to the deal, account, contact or lead it was +-- about. +-- +-- WHAT MODULE IS THE PARENT? Zoho's API exposes $se_module to say which one, and +-- the connector does not sync it. Record ids are globally unique across Zoho +-- modules though, so the module can be recovered by joining parent_id to each +-- module in turn and seeing which one matches - that is what +-- core_zoho_crm__account_360 does. `parent_name` is the label Zoho denormalised +-- onto the note and is kept only as a fallback for a parent that no longer exists. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS note_id, + NULLIF(TRIM(CAST(note_title AS STRING)), '') AS note_title, + NULLIF(TRIM(CAST(note_content AS STRING)), '') AS note_content, + CAST(parent_id_id AS STRING) AS parent_id, + NULLIF(TRIM(CAST(parent_id_name AS STRING)), '') AS parent_name, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'note') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__task.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__task.sql new file mode 100644 index 0000000..6615a17 --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__task.sql @@ -0,0 +1,24 @@ +-- stg_zoho_crm__task +-- Thin wrapper over raw `task`. Casts, renames, derives is_completed. +-- +-- NO PARENT LINK. Zoho's Tasks module has What_Id / Who_Id (the deal, account, +-- contact or lead the task hangs off), but the connector does not sync them. The +-- only foreign key on this stream is owner_id, so a task can be counted per rep +-- and per day and nothing else. Do not join it to a deal - there is no key to +-- join on. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS task_id, + NULLIF(TRIM(CAST(subject AS STRING)), '') AS subject, + NULLIF(TRIM(CAST(status AS STRING)), '') AS status, + NULLIF(TRIM(CAST(priority AS STRING)), '') AS priority, + CAST(due_date AS DATE) AS due_date, + -- Zoho's default Task statuses are Not Started, Deferred, In Progress, + -- Completed and Waiting for input. Matched on a pattern rather than equality + -- so a renamed or translated status does not silently read as incomplete. + LOWER(CAST(status AS STRING)) LIKE '%complet%' AS is_completed, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'task') }} diff --git a/zoho-crm/dbt/models/staging/stg_zoho_crm__user.sql b/zoho-crm/dbt/models/staging/stg_zoho_crm__user.sql new file mode 100644 index 0000000..2c2a2cd --- /dev/null +++ b/zoho-crm/dbt/models/staging/stg_zoho_crm__user.sql @@ -0,0 +1,27 @@ +-- stg_zoho_crm__user +-- Thin wrapper over raw `user`. Casts, renames, derives is_active. No other logic. +-- +-- This is the only dimension every other stream can join to: every record-owning +-- module carries owner_id, and nothing else in the Zoho schema carries a foreign +-- key. Sync this stream even if you think you do not need it. +-- +-- Single Zoho org. To add another, UNION ALL a second block below pointing at that +-- org's connector with a different zoho_org label. Keep it in staging so the core +-- models never have to know how many orgs there are. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS user_id, + NULLIF(TRIM(CAST(full_name AS STRING)), '') AS full_name, + CAST(first_name AS STRING) AS first_name, + CAST(last_name AS STRING) AS last_name, + LOWER(NULLIF(TRIM(CAST(email AS STRING)), '')) AS email, + CAST(status AS STRING) AS status, + -- Zoho keeps deactivated users in the module rather than deleting them, so + -- filtering on this is how you get "reps who could take a deal today". + LOWER(CAST(status AS STRING)) = 'active' AS is_active, + CAST(role_name AS STRING) AS role_name, + CAST(profile_name AS STRING) AS profile_name, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{ source('zoho_crm', 'user') }} diff --git a/zoho-crm/dbt/tests/assert_call_duration_parses.sql b/zoho-crm/dbt/tests/assert_call_duration_parses.sql new file mode 100644 index 0000000..c9a5b50 --- /dev/null +++ b/zoho-crm/dbt/tests/assert_call_duration_parses.sql @@ -0,0 +1,21 @@ +-- Calls whose duration string did not parse. +-- +-- `call_duration` arrives from Zoho as text, and stg_zoho_crm__call parses it with +-- a regex expecting one colon. Rows returned here had a value that did not match - +-- an empty string, a plain number of minutes with no colon, an hh:mm:ss triple, or +-- something localised. +-- +-- THIS CHECKS THE SHAPE, NOT THE UNIT. Zoho's API reference documents the field as +-- hh:mm and the CRM UI shows mm:ss for short calls; the stream carries no +-- Call_Duration_in_seconds column to settle it. Passing this test does not mean +-- call_duration_minutes is in minutes. Reconcile against one call of known length +-- before reporting on call time, and if your org emits mm:ss, divide by 60. + +SELECT + call_duration_raw, + COUNT(*) AS calls +FROM {{ ref('stg_zoho_crm__call') }} +WHERE call_duration_raw IS NOT NULL + AND call_duration_minutes IS NULL +GROUP BY 1 +ORDER BY calls DESC diff --git a/zoho-crm/dbt/tests/assert_deal_stages_are_classified.sql b/zoho-crm/dbt/tests/assert_deal_stages_are_classified.sql new file mode 100644 index 0000000..1ebd8e6 --- /dev/null +++ b/zoho-crm/dbt/tests/assert_deal_stages_are_classified.sql @@ -0,0 +1,25 @@ +-- Every distinct deal stage that fell through to 'Open'. +-- +-- A FAILURE HERE IS INFORMATION, NOT NECESSARILY A DEFECT. Zoho has no is_won or +-- is_closed flag, so core_zoho_crm__deal_pipeline reads won and lost out of the +-- stage string with LIKE '%won%' / '%lost%'. Genuinely open stages +-- (Qualification, Negotiation/Review, ...) are supposed to appear in this list. +-- +-- What you are looking for is a CLOSED stage hiding among them - "Contract +-- Signed", "Dead", "Churned", "Nurture", "Abandoned", anything in another +-- language. Every row of that kind is revenue being counted as open pipeline, or +-- a loss that never shows up in the win rate. +-- +-- Run it once when you deploy, then whenever sales ops adds a stage. Add anything +-- it surfaces to the CASE in the core model. + +SELECT + stage, + COUNT(*) AS deals, + SUM(amount) AS amount, + MIN(created_time) AS first_seen, + MAX(created_time) AS last_seen +FROM {{ ref('core_zoho_crm__deal_pipeline') }} +WHERE stage_status = 'Open' +GROUP BY 1 +ORDER BY amount DESC diff --git a/zoho-crm/dbt/tests/assert_no_duplicate_deals.sql b/zoho-crm/dbt/tests/assert_no_duplicate_deals.sql new file mode 100644 index 0000000..5c40a9f --- /dev/null +++ b/zoho-crm/dbt/tests/assert_no_duplicate_deals.sql @@ -0,0 +1,17 @@ +-- One row per deal, per org. +-- +-- The Zoho connector merges on `id`, so duplicates should be impossible. This +-- guards the two ways they appear anyway: a second Zoho org unioned into the +-- staging models without changing the zoho_org label, and a ReSync that lands +-- alongside the existing table instead of replacing it. +-- +-- Worth keeping even though it "cannot fail" - it is the cheapest test here and +-- silent row duplication doubles every amount in the pipeline. + +SELECT + zoho_org, + deal_id, + COUNT(*) AS rows_for_deal +FROM {{ ref('core_zoho_crm__deal_pipeline') }} +GROUP BY 1, 2 +HAVING COUNT(*) > 1 diff --git a/zoho-crm/dbt/tests/assert_owner_ids_resolve.sql b/zoho-crm/dbt/tests/assert_owner_ids_resolve.sql new file mode 100644 index 0000000..8c40c4c --- /dev/null +++ b/zoho-crm/dbt/tests/assert_owner_ids_resolve.sql @@ -0,0 +1,22 @@ +-- Deals whose owner_id does not exist in the user module. +-- +-- Catches two real situations. Either the `user` stream is not being synced at all +-- - in which case every row in the pipeline has a NULL owner_name and nobody +-- notices until a dashboard groups by rep and shows one enormous blank bucket - +-- or a rep has been deleted from Zoho outright, taking their name with them while +-- their deals stay in the pipeline. +-- +-- The core model joins user with a LEFT join precisely so these deals are not +-- silently dropped from the pipeline total. This test is how you find out they are +-- there. + +SELECT + owner_id, + COUNT(*) AS deals, + COUNTIF(is_open) AS deals_open, + SUM(pipeline_amount) AS unattributed_open_pipeline +FROM {{ ref('core_zoho_crm__deal_pipeline') }} +WHERE owner_id IS NOT NULL + AND owner_name IS NULL +GROUP BY 1 +ORDER BY unattributed_open_pipeline DESC diff --git a/zoho-crm/weld/analytics/account_360.sql b/zoho-crm/weld/analytics/account_360.sql new file mode 100644 index 0000000..be7784a --- /dev/null +++ b/zoho-crm/weld/analytics/account_360.sql @@ -0,0 +1,9 @@ +-- analytics.zoho_crm.account_360 +-- BI-facing contract over the core model. See analytics/deal_pipeline.sql for why +-- this layer exists even when it is a passthrough. +-- +-- This is the natural source for a reverse-ETL sync back into Zoho: writing +-- open_pipeline_amount or is_stale_with_open_pipeline onto the Account record puts +-- the warehouse's view in front of the reps who need it. + +SELECT * FROM {{core.zoho_crm.account_360}} diff --git a/zoho-crm/weld/analytics/deal_flow_by_month.sql b/zoho-crm/weld/analytics/deal_flow_by_month.sql new file mode 100644 index 0000000..b211b65 --- /dev/null +++ b/zoho-crm/weld/analytics/deal_flow_by_month.sql @@ -0,0 +1,5 @@ +-- analytics.zoho_crm.deal_flow_by_month +-- BI-facing contract over the core model. See analytics/deal_pipeline.sql for why +-- this layer exists even when it is a passthrough. + +SELECT * FROM {{core.zoho_crm.deal_flow_by_month}} diff --git a/zoho-crm/weld/analytics/deal_pipeline.sql b/zoho-crm/weld/analytics/deal_pipeline.sql new file mode 100644 index 0000000..20e9abe --- /dev/null +++ b/zoho-crm/weld/analytics/deal_pipeline.sql @@ -0,0 +1,10 @@ +-- analytics.zoho_crm.deal_pipeline +-- BI-facing contract over the core model. Dashboards, scheduled reports and +-- reverse-ETL syncs bind HERE, never to core, so core stays free to be renamed, +-- re-grained or split without breaking anything downstream. +-- +-- A passthrough is the correct content for this layer. Put BI-specific shaping +-- (renames for a semantic layer, row filters for a workspace) in this file rather +-- than in core. + +SELECT * FROM {{core.zoho_crm.deal_pipeline}} diff --git a/zoho-crm/weld/analytics/rep_activity.sql b/zoho-crm/weld/analytics/rep_activity.sql new file mode 100644 index 0000000..782227a --- /dev/null +++ b/zoho-crm/weld/analytics/rep_activity.sql @@ -0,0 +1,9 @@ +-- analytics.zoho_crm.rep_activity +-- BI-facing contract over the core model. See analytics/deal_pipeline.sql for why +-- this layer exists even when it is a passthrough. +-- +-- Reminder for whoever binds a dashboard to this: activity here is per rep only. +-- There is no deal or account key on Zoho's call, task and event streams in this +-- connector, so do not label a tile "activity per opportunity". + +SELECT * FROM {{core.zoho_crm.rep_activity}} diff --git a/zoho-crm/weld/core/account_360.sql b/zoho-crm/weld/core/account_360.sql new file mode 100644 index 0000000..54d7ab2 --- /dev/null +++ b/zoho-crm/weld/core/account_360.sql @@ -0,0 +1,126 @@ +-- zoho_crm_account_360 - one row per account, with its people, pipeline and notes. +-- Grain: account. Currency: org reporting currency. +-- Depends on: staging.zoho_crm.account, .contact, .user, .note, +-- core.zoho_crm.deal_pipeline +-- +-- The account-level view the rest of the schema can actually support: who works +-- there, what is open, what has been won, and when anybody last wrote anything +-- down. This is the model to reverse-ETL back into Zoho, or to join to product +-- usage and billing data for a real customer view. +-- +-- NOT IN HERE: calls, meetings and tasks. They carry no account key in this +-- connector - see core.zoho_crm.rep_activity for why. last_note_at is the closest +-- honest proxy for account engagement the connector allows, and it only reflects +-- what reps bothered to write down. + +WITH contacts AS ( + SELECT + zoho_org, + account_id, + COUNT(*) AS contacts, + COUNTIF(email IS NOT NULL) AS contacts_with_email, + MAX(created_time) AS last_contact_added_at + FROM {{staging.zoho_crm.contact}} + WHERE account_id IS NOT NULL + GROUP BY 1, 2 +), + +deals AS ( + SELECT + zoho_org, + account_id, + COUNT(*) AS deals_total, + COUNTIF(is_open) AS deals_open, + COUNTIF(is_won) AS deals_won, + COUNTIF(is_lost) AS deals_lost, + COUNTIF(is_overdue) AS deals_overdue, + SUM(pipeline_amount) AS open_pipeline_amount, + SUM(won_amount) AS won_amount, + SUM(lost_amount) AS lost_amount, + MIN(CASE WHEN is_open THEN closing_date END) AS next_expected_close, + MAX(created_time) AS last_deal_created_at + FROM {{core.zoho_crm.deal_pipeline}} + WHERE account_id IS NOT NULL + GROUP BY 1, 2 +), + +-- Notes reach an account three ways: written on the account itself, on one of its +-- deals, or on one of its contacts. Zoho ids are globally unique across modules, +-- so one join per route resolves the parent without needing $se_module. +note_targets AS ( + SELECT a.zoho_org, a.account_id, a.account_id AS target_id + FROM {{staging.zoho_crm.account}} a + UNION ALL + SELECT d.zoho_org, d.account_id, d.deal_id + FROM {{core.zoho_crm.deal_pipeline}} d + WHERE d.account_id IS NOT NULL + UNION ALL + SELECT c.zoho_org, c.account_id, c.contact_id + FROM {{staging.zoho_crm.contact}} c + WHERE c.account_id IS NOT NULL +), + +notes AS ( + SELECT + t.zoho_org, + t.account_id, + COUNT(*) AS notes, + MAX(n.created_time) AS last_note_at + FROM {{staging.zoho_crm.note}} n + JOIN note_targets t + ON t.zoho_org = n.zoho_org + AND t.target_id = n.parent_id + GROUP BY 1, 2 +) + +SELECT + a.zoho_org, + a.account_id, + a.account_name, + a.industry, + a.website, + a.phone, + a.created_time AS account_created_time, + + a.owner_id, + u.full_name AS owner_name, + u.email AS owner_email, + u.is_active AS owner_is_active, + + COALESCE(c.contacts, 0) AS contacts, + COALESCE(c.contacts_with_email, 0) AS contacts_with_email, + + COALESCE(d.deals_total, 0) AS deals_total, + COALESCE(d.deals_open, 0) AS deals_open, + COALESCE(d.deals_won, 0) AS deals_won, + COALESCE(d.deals_lost, 0) AS deals_lost, + COALESCE(d.deals_overdue, 0) AS deals_overdue, + COALESCE(d.open_pipeline_amount, 0) AS open_pipeline_amount, + COALESCE(d.won_amount, 0) AS won_amount, + COALESCE(d.lost_amount, 0) AS lost_amount, + d.next_expected_close, + + COALESCE(n.notes, 0) AS notes, + n.last_note_at, + + -- Latest of anything datable on the account. Note the absence of calls and + -- meetings: this is "last recorded touch", not "last contact". + GREATEST( + COALESCE(n.last_note_at, TIMESTAMP '1970-01-01'), + COALESCE(d.last_deal_created_at, TIMESTAMP '1970-01-01'), + COALESCE(c.last_contact_added_at, TIMESTAMP '1970-01-01'), + a.created_time + ) AS last_recorded_activity_at, + + -- An account with open pipeline and nothing written on it in a quarter is the + -- report this model exists to produce. + COALESCE(d.deals_open, 0) > 0 + AND COALESCE(n.last_note_at, a.created_time) + < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY) AS is_stale_with_open_pipeline +FROM {{staging.zoho_crm.account}} a +LEFT JOIN contacts c ON c.zoho_org = a.zoho_org AND c.account_id = a.account_id +LEFT JOIN deals d ON d.zoho_org = a.zoho_org AND d.account_id = a.account_id +LEFT JOIN notes n ON n.zoho_org = a.zoho_org AND n.account_id = a.account_id +LEFT JOIN {{staging.zoho_crm.user}} u + ON u.zoho_org = a.zoho_org AND u.user_id = a.owner_id +ORDER BY open_pipeline_amount DESC diff --git a/zoho-crm/weld/core/deal_flow_by_month.sql b/zoho-crm/weld/core/deal_flow_by_month.sql new file mode 100644 index 0000000..1e9dda1 --- /dev/null +++ b/zoho-crm/weld/core/deal_flow_by_month.sql @@ -0,0 +1,117 @@ +-- zoho_crm_deal_flow_by_month - deals created, won and lost per month per rep. +-- Grain: month x owner. Currency: org reporting currency. +-- Depends on: core.zoho_crm.deal_pipeline +-- +-- Built on a month spine so a rep with no activity in a month still returns a row +-- of zeros rather than disappearing from the series - otherwise a BI line chart +-- interpolates straight over the quiet month and the gap is invisible. +-- +-- WHICH DATE EACH EVENT LANDS ON: +-- created -> created_time, which is a real audit timestamp. +-- won/lost -> closing_date, which is Zoho's EXPECTED close date. +-- +-- That second one is the compromise this model cannot avoid. The connector syncs +-- no actual close timestamp and no stage history, so closing_date is the only +-- close date available. If a rep sets a deal to Closed Won in June while its +-- closing_date still says March, this model books the win in March. modified_time +-- would move it to June, but modified_time changes on ANY edit, so a note added in +-- August would move the win again. A forecast date that is wrong once beats a date +-- that moves every time somebody touches the record. +-- +-- The fix is upstream, not here: materialise deal_pipeline daily and derive real +-- close dates from when stage_status first changed. + +WITH month_spine AS ( + SELECT month + FROM UNNEST(GENERATE_DATE_ARRAY( + -- Start the spine at the first deal rather than a hardcoded date, so the + -- series never carries years of empty leading months. + (SELECT DATE_TRUNC(MIN(created_date), MONTH) FROM {{core.zoho_crm.deal_pipeline}}), + -- End a year out: closing_date is a forecast, so pipeline legitimately + -- sits in the future and must not be truncated away. + DATE_TRUNC(DATE_ADD(CURRENT_DATE(), INTERVAL 1 YEAR), MONTH), + INTERVAL 1 MONTH + )) AS month +), + +owners AS ( + -- Only reps who actually appear on a deal. Spining every Zoho user against + -- every month would pad the output with support and admin logins. + SELECT DISTINCT zoho_org, owner_id, owner_name + FROM {{core.zoho_crm.deal_pipeline}} +), + +grid AS ( + SELECT s.month, o.zoho_org, o.owner_id, o.owner_name + FROM month_spine s + CROSS JOIN owners o +), + +created AS ( + SELECT + DATE_TRUNC(created_date, MONTH) AS month, + zoho_org, + owner_id, + COUNT(*) AS deals_created, + SUM(amount) AS deals_created_amount + FROM {{core.zoho_crm.deal_pipeline}} + GROUP BY 1, 2, 3 +), + +closed AS ( + SELECT + DATE_TRUNC(closing_date, MONTH) AS month, + zoho_org, + owner_id, + COUNTIF(is_won) AS deals_won, + SUM(won_amount) AS deals_won_amount, + COUNTIF(is_lost) AS deals_lost, + SUM(lost_amount) AS deals_lost_amount, + -- Open deals forecast to close in this month. Not an outcome - a promise. + COUNTIF(is_open) AS deals_forecast, + SUM(pipeline_amount) AS pipeline_amount + FROM {{core.zoho_crm.deal_pipeline}} + WHERE closing_date IS NOT NULL + GROUP BY 1, 2, 3 +) + +SELECT + g.month, + g.zoho_org, + g.owner_id, + g.owner_name, + + COALESCE(c.deals_created, 0) AS deals_created, + COALESCE(c.deals_created_amount, 0) AS deals_created_amount, + + COALESCE(x.deals_won, 0) AS deals_won, + COALESCE(x.deals_won_amount, 0) AS deals_won_amount, + COALESCE(x.deals_lost, 0) AS deals_lost, + COALESCE(x.deals_lost_amount, 0) AS deals_lost_amount, + + COALESCE(x.deals_forecast, 0) AS deals_forecast, + COALESCE(x.pipeline_amount, 0) AS pipeline_amount, + + -- Win rate by count, over decided deals only. Open deals are excluded from the + -- denominator: counting them as not-yet-won drags every current month down and + -- makes the trend look like a collapse. + SAFE_DIVIDE( + COALESCE(x.deals_won, 0), + COALESCE(x.deals_won, 0) + COALESCE(x.deals_lost, 0) + ) AS win_rate, + SAFE_DIVIDE( + COALESCE(x.deals_won_amount, 0), + COALESCE(x.deals_won_amount, 0) + COALESCE(x.deals_lost_amount, 0) + ) AS win_rate_by_value, + -- Average won deal size, the other half of any quota conversation. + SAFE_DIVIDE(COALESCE(x.deals_won_amount, 0), NULLIF(x.deals_won, 0)) AS average_won_deal_size +FROM grid g +LEFT JOIN created c + ON c.month = g.month + AND c.zoho_org = g.zoho_org + AND c.owner_id = g.owner_id +LEFT JOIN closed x + ON x.month = g.month + AND x.zoho_org = g.zoho_org + AND x.owner_id = g.owner_id +ORDER BY g.month, g.owner_name diff --git a/zoho-crm/weld/core/deal_pipeline.sql b/zoho-crm/weld/core/deal_pipeline.sql new file mode 100644 index 0000000..0e8202c --- /dev/null +++ b/zoho-crm/weld/core/deal_pipeline.sql @@ -0,0 +1,96 @@ +-- zoho_crm_deal_pipeline - every deal, classified, with its account and owner. +-- Grain: one row per deal. Currency: org reporting currency (see scope note). +-- Depends on: staging.zoho_crm.deal, .account, .user +-- +-- This is the deal-grain model the other reports read. Classify once here rather +-- than repeating the stage logic in every downstream query. +-- +-- SCOPE: current state, not history. Zoho CRM has no history tables in Weld, and +-- this connector syncs no stage-change audit, so there is no way to ask what the +-- pipeline looked like last Tuesday or how long a deal sat in Negotiation. Every +-- deal shows only where it stands now. If you need stage velocity or a pipeline +-- snapshot over time, materialise this model daily and keep the runs. +-- +-- Joins are written with explicit ON rather than USING: both deal and account +-- carry owner_id, so a USING(owner_id) join further down would be ambiguous. + +WITH deal AS ( + SELECT + d.*, + -- Zoho ships no is_won / is_closed boolean, so won and lost have to be + -- read out of the stage string. Matched on a pattern, not an equality + -- list, because Zoho's own defaults include "Closed-Lost to Competition" + -- and most orgs add their own stages on top. + -- + -- Won is tested first so a stage that somehow contains both words is + -- counted once. tests/assert_deal_stages_are_classified.sql lists every + -- stage falling through to Open - read it before trusting the split. + CASE + WHEN LOWER(d.stage) LIKE '%won%' THEN 'Won' + WHEN LOWER(d.stage) LIKE '%lost%' THEN 'Lost' + ELSE 'Open' + END AS stage_status + FROM {{staging.zoho_crm.deal}} d +) + +SELECT + d.zoho_org, + d.deal_id, + d.deal_name, + d.stage, + d.stage_status, + d.stage_status = 'Open' AS is_open, + d.stage_status = 'Won' AS is_won, + d.stage_status = 'Lost' AS is_lost, + + d.amount, + -- Split out so BI can sum a column instead of writing the CASE again. An open + -- deal contributes to pipeline_amount only; a closed one to won or lost. + CASE WHEN d.stage_status = 'Open' THEN d.amount END AS pipeline_amount, + CASE WHEN d.stage_status = 'Won' THEN d.amount END AS won_amount, + CASE WHEN d.stage_status = 'Lost' THEN d.amount END AS lost_amount, + + DATE(d.created_time) AS created_date, + d.closing_date, + d.created_time, + d.modified_time, + + -- Age of the deal. For a closed deal this is how long it took; for an open one + -- how long it has been sitting. closing_date is Zoho's EXPECTED close date and + -- it is not cleared when a deal closes, so on a won or lost deal it is the + -- forecast that was in place, not necessarily the day money changed hands. + DATE_DIFF( + COALESCE(CASE WHEN d.stage_status <> 'Open' THEN d.closing_date END, CURRENT_DATE()), + DATE(d.created_time), + DAY + ) AS age_days, + -- Open deals whose expected close date has already passed: the cheapest + -- pipeline-hygiene number there is, and usually the first thing a sales lead + -- asks for. + d.stage_status = 'Open' AND d.closing_date < CURRENT_DATE() AS is_overdue, + CASE + WHEN d.stage_status = 'Open' + THEN DATE_DIFF(d.closing_date, CURRENT_DATE(), DAY) + END AS days_to_expected_close, + + d.account_id, + a.account_name, + a.industry, + + d.owner_id, + u.full_name AS owner_name, + u.email AS owner_email, + u.role_name AS owner_role, + -- A deal owned by a deactivated rep is unmanaged pipeline. NULL here means the + -- owner_id did not resolve at all - see tests/assert_owner_ids_resolve.sql. + u.is_active AS owner_is_active +FROM deal d +LEFT JOIN {{staging.zoho_crm.account}} a + ON a.zoho_org = d.zoho_org + AND a.account_id = d.account_id +-- LEFT, not INNER: an owner who has been deleted from Zoho no longer appears in +-- the user module, and an inner join would silently drop their deals from the +-- pipeline total. +LEFT JOIN {{staging.zoho_crm.user}} u + ON u.zoho_org = d.zoho_org + AND u.user_id = d.owner_id diff --git a/zoho-crm/weld/core/rep_activity.sql b/zoho-crm/weld/core/rep_activity.sql new file mode 100644 index 0000000..3d4b8db --- /dev/null +++ b/zoho-crm/weld/core/rep_activity.sql @@ -0,0 +1,119 @@ +-- zoho_crm_rep_activity - calls, meetings and tasks logged per rep per day. +-- Grain: day x owner. +-- Depends on: staging.zoho_crm.call, .event, .task, .user +-- +-- READ THIS BEFORE USING IT. This model counts activity per REP, never per deal, +-- account, contact or lead. That is not a design choice - Zoho's Calls, Events and +-- Tasks modules all carry What_Id / Who_Id pointing at the record the activity +-- belongs to, and the Weld connector does not sync either field. owner_id is the +-- only foreign key on all three streams. +-- +-- So this model answers "how much is each rep doing" and cannot answer "how much +-- activity did we put into the deals we won". Any dashboard promising +-- activity-to-outcome attribution from this connector alone is inventing the link. +-- Until the connector syncs the parent ids, the join has to come from somewhere +-- else - a calendar or dialer source keyed on email, or Zoho's Notes module, whose +-- parent_id_id IS synced. +-- +-- WHY THERE IS NO tasks_completed COLUMN: the connector syncs is_completed but no +-- completion timestamp. Completions can be counted as a current-state total, never +-- placed on a day. Booking them on modified_time would move a task's completion +-- every time anyone edited it afterwards. + +WITH activity AS ( + SELECT zoho_org, owner_id, DATE(call_start_time) AS activity_date, + 1 AS calls_logged, call_duration_minutes AS call_minutes, + 0 AS events_held, 0 AS event_minutes, + 0 AS tasks_created + FROM {{staging.zoho_crm.call}} + WHERE call_start_time IS NOT NULL + + UNION ALL + + SELECT zoho_org, owner_id, DATE(start_date_time) AS activity_date, + 0, 0, + 1 AS events_held, duration_minutes AS event_minutes, + 0 + FROM {{staging.zoho_crm.event}} + WHERE start_date_time IS NOT NULL + + UNION ALL + + -- Tasks land on the day they were CREATED, not their due date. Due dates get + -- pushed; creation is when the rep actually did something. + SELECT zoho_org, owner_id, DATE(created_time) AS activity_date, + 0, 0, + 0, 0, + 1 AS tasks_created + FROM {{staging.zoho_crm.task}} + WHERE created_time IS NOT NULL +), + +rolled_up AS ( + SELECT + activity_date, + zoho_org, + owner_id, + SUM(calls_logged) AS calls_logged, + SUM(call_minutes) AS call_minutes, + SUM(events_held) AS events_held, + SUM(event_minutes) AS event_minutes, + SUM(tasks_created) AS tasks_created + FROM activity + GROUP BY 1, 2, 3 +), + +date_spine AS ( + SELECT day AS activity_date + FROM UNNEST(GENERATE_DATE_ARRAY( + (SELECT MIN(activity_date) FROM rolled_up), + CURRENT_DATE(), + INTERVAL 1 DAY + )) AS day +), + +owners AS ( + -- Reps who have logged anything at all. Spining every Zoho user against every + -- day would bury the active team under admin and integration logins. + SELECT DISTINCT zoho_org, owner_id FROM rolled_up +), + +grid AS ( + SELECT s.activity_date, o.zoho_org, o.owner_id + FROM date_spine s + CROSS JOIN owners o +) + +SELECT + g.activity_date, + g.zoho_org, + g.owner_id, + u.full_name AS owner_name, + u.email AS owner_email, + u.role_name AS owner_role, + u.is_active AS owner_is_active, + + COALESCE(r.calls_logged, 0) AS calls_logged, + -- NULL rather than 0 when no call was logged: a zero here would drag down any + -- average-call-length metric computed over the column. + r.call_minutes, + COALESCE(r.events_held, 0) AS events_held, + r.event_minutes, + COALESCE(r.tasks_created, 0) AS tasks_created, + + COALESCE(r.calls_logged, 0) + + COALESCE(r.events_held, 0) + + COALESCE(r.tasks_created, 0) AS total_activities, + -- Touchpoints that involved a human on the other end, which is usually the + -- number a sales lead actually wants out of an activity report. + COALESCE(r.calls_logged, 0) + + COALESCE(r.events_held, 0) AS live_touchpoints +FROM grid g +LEFT JOIN rolled_up r + ON r.activity_date = g.activity_date + AND r.zoho_org = g.zoho_org + AND r.owner_id = g.owner_id +LEFT JOIN {{staging.zoho_crm.user}} u + ON u.zoho_org = g.zoho_org + AND u.user_id = g.owner_id +ORDER BY g.activity_date, owner_name diff --git a/zoho-crm/weld/staging/account.sql b/zoho-crm/weld/staging/account.sql new file mode 100644 index 0000000..679d9b8 --- /dev/null +++ b/zoho-crm/weld/staging/account.sql @@ -0,0 +1,21 @@ +-- staging.zoho_crm.account +-- Thin wrapper over raw `account`. Casts, renames, normalises blanks to NULL. +-- +-- Single Zoho org. To add another, UNION ALL a second block below pointing at that +-- org's connector with a different zoho_org label. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS account_id, + NULLIF(TRIM(CAST(account_name AS STRING)), '') AS account_name, + NULLIF(TRIM(CAST(industry AS STRING)), '') AS industry, + NULLIF(TRIM(CAST(phone AS STRING)), '') AS phone, + NULLIF(TRIM(CAST(website AS STRING)), '') AS website, + -- Zoho flattens its lookup fields into _id / _name / _email triples. Keep the + -- id for joining and drop the denormalised name - staging.zoho_crm.user is the + -- single source of truth for what a user is called, so a rep who is renamed in + -- Zoho does not leave stale labels scattered across every module. + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.account}} diff --git a/zoho-crm/weld/staging/call.sql b/zoho-crm/weld/staging/call.sql new file mode 100644 index 0000000..3ebb7d6 --- /dev/null +++ b/zoho-crm/weld/staging/call.sql @@ -0,0 +1,31 @@ +-- staging.zoho_crm.call +-- Thin wrapper over raw `call`. Casts, renames, parses the duration string. +-- +-- NO PARENT LINK. Same as task: Zoho's Calls module has What_Id / Who_Id, the +-- connector does not sync them. owner_id is the only key. +-- +-- DURATION IS A STRING, NOT A NUMBER. `call_duration` arrives as text with one +-- colon. Zoho's API reference documents the field as hh:mm; the CRM UI shows +-- mm:ss for short calls, and there is no Call_Duration_in_seconds column on this +-- stream to disambiguate. This model parses it as the DOCUMENTED hh:mm. +-- +-- Reconcile call_duration_minutes against one call of known length before +-- reporting on it. If your org turns out to emit mm:ss, divide by 60. +-- tests/assert_call_duration_parses.sql guards the shape, not the unit. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS call_id, + NULLIF(TRIM(CAST(subject AS STRING)), '') AS subject, + NULLIF(TRIM(CAST(call_type AS STRING)), '') AS call_type, + CAST(call_start_time AS TIMESTAMP) AS call_start_time, + CAST(call_duration AS STRING) AS call_duration_raw, + CASE + WHEN REGEXP_CONTAINS(CAST(call_duration AS STRING), r'^\s*\d+:\d{1,2}\s*$') + THEN SAFE_CAST(SPLIT(TRIM(CAST(call_duration AS STRING)), ':')[OFFSET(0)] AS INT64) * 60 + + SAFE_CAST(SPLIT(TRIM(CAST(call_duration AS STRING)), ':')[OFFSET(1)] AS INT64) + END AS call_duration_minutes, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.call}} diff --git a/zoho-crm/weld/staging/contact.sql b/zoho-crm/weld/staging/contact.sql new file mode 100644 index 0000000..ddc5eeb --- /dev/null +++ b/zoho-crm/weld/staging/contact.sql @@ -0,0 +1,23 @@ +-- staging.zoho_crm.contact +-- Thin wrapper over raw `contact`. Casts, renames, builds full_name. +-- +-- NOTE ON THE ACCOUNT KEY: Zoho's lookup field is called Account_Name, so the +-- connector emits `account_name_id` (the account's id) and `account_name_name` +-- (its label). The _id column is the real foreign key despite the name. It is +-- renamed to account_id here so core reads like a normal star schema. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS contact_id, + CAST(first_name AS STRING) AS first_name, + CAST(last_name AS STRING) AS last_name, + NULLIF(TRIM(CONCAT( + COALESCE(CAST(first_name AS STRING), ''), ' ', + COALESCE(CAST(last_name AS STRING), '') + )), '') AS full_name, + LOWER(NULLIF(TRIM(CAST(email AS STRING)), '')) AS email, + CAST(account_name_id AS STRING) AS account_id, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.contact}} diff --git a/zoho-crm/weld/staging/deal.sql b/zoho-crm/weld/staging/deal.sql new file mode 100644 index 0000000..561ce3f --- /dev/null +++ b/zoho-crm/weld/staging/deal.sql @@ -0,0 +1,26 @@ +-- staging.zoho_crm.deal +-- Thin wrapper over raw `deal`. Casts, renames, normalises blanks. Stage +-- classification is business logic and lives in core.zoho_crm.deal_pipeline. +-- +-- ACCOUNT KEY: as on contact, Zoho's lookup is called Account_Name, so the raw +-- column is `account_name_id`. It holds the account's id and is renamed here. +-- +-- WHAT IS NOT HERE, because the connector does not sync it: no is_won / is_closed +-- boolean, no probability, no expected_revenue, no currency, no lead_source, no +-- contact_id, no campaign_id and no pipeline name. Won/lost has to be derived from +-- the stage string, and amount is in whatever single currency the org reports in. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS deal_id, + NULLIF(TRIM(CAST(deal_name AS STRING)), '') AS deal_name, + NULLIF(TRIM(CAST(stage AS STRING)), '') AS stage, + -- No currency column exists on the stream, so this is org-reporting currency. + -- A multi-currency Zoho org cannot be summed correctly from this connector. + CAST(amount AS NUMERIC) AS amount, + CAST(closing_date AS DATE) AS closing_date, + CAST(account_name_id AS STRING) AS account_id, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.deal}} diff --git a/zoho-crm/weld/staging/event.sql b/zoho-crm/weld/staging/event.sql new file mode 100644 index 0000000..7193058 --- /dev/null +++ b/zoho-crm/weld/staging/event.sql @@ -0,0 +1,24 @@ +-- staging.zoho_crm.event +-- Thin wrapper over raw `event` (Zoho's Meetings module). Casts, renames, derives +-- duration from the start and end timestamps. +-- +-- NO PARENT LINK. Same as task and call: owner_id is the only foreign key. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS event_id, + NULLIF(TRIM(CAST(event_title AS STRING)), '') AS event_title, + CAST(start_date_time AS TIMESTAMP) AS start_date_time, + CAST(end_date_time AS TIMESTAMP) AS end_date_time, + -- Unlike call, event carries both ends, so duration is computed rather than + -- parsed out of a string and the unit is unambiguous. + TIMESTAMP_DIFF( + CAST(end_date_time AS TIMESTAMP), + CAST(start_date_time AS TIMESTAMP), + MINUTE + ) AS duration_minutes, + NULLIF(TRIM(CAST(location AS STRING)), '') AS location, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.event}} diff --git a/zoho-crm/weld/staging/lead.sql b/zoho-crm/weld/staging/lead.sql new file mode 100644 index 0000000..e44bb29 --- /dev/null +++ b/zoho-crm/weld/staging/lead.sql @@ -0,0 +1,24 @@ +-- staging.zoho_crm.lead +-- Thin wrapper over raw `lead`. Casts, renames, builds full_name. +-- +-- Leads are a terminal island in this schema. The connector syncs no +-- Converted_Account / Converted_Contact / Converted_Deal fields, so a lead cannot +-- be followed into the deal it became. Lead reporting here is volume and status +-- only - see the README before promising a lead-to-deal conversion rate. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS lead_id, + CAST(first_name AS STRING) AS first_name, + CAST(last_name AS STRING) AS last_name, + NULLIF(TRIM(CONCAT( + COALESCE(CAST(first_name AS STRING), ''), ' ', + COALESCE(CAST(last_name AS STRING), '') + )), '') AS full_name, + LOWER(NULLIF(TRIM(CAST(email AS STRING)), '')) AS email, + NULLIF(TRIM(CAST(company AS STRING)), '') AS company, + NULLIF(TRIM(CAST(lead_status AS STRING)), '') AS lead_status, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.lead}} diff --git a/zoho-crm/weld/staging/note.sql b/zoho-crm/weld/staging/note.sql new file mode 100644 index 0000000..0727cff --- /dev/null +++ b/zoho-crm/weld/staging/note.sql @@ -0,0 +1,27 @@ +-- staging.zoho_crm.note +-- Thin wrapper over raw `note`. Casts, renames, normalises blanks. +-- +-- THE ONE STREAM WITH A REAL PARENT LINK. Every other activity module (call, task, +-- event) loses its What_Id / Who_Id in this connector, but note keeps its parent: +-- Zoho's Parent_Id lookup arrives as `parent_id_id`, and it is the only way to tie +-- anything a rep wrote down back to the deal, account, contact or lead it was +-- about. +-- +-- WHAT MODULE IS THE PARENT? Zoho's API exposes $se_module to say which one, and +-- the connector does not sync it. Record ids are globally unique across Zoho +-- modules though, so the module can be recovered by joining parent_id to each +-- module in turn and seeing which one matches - that is what +-- core.zoho_crm.account_360 does. `parent_name` is the label Zoho denormalised +-- onto the note and is kept only as a fallback for a parent that no longer exists. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS note_id, + NULLIF(TRIM(CAST(note_title AS STRING)), '') AS note_title, + NULLIF(TRIM(CAST(note_content AS STRING)), '') AS note_content, + CAST(parent_id_id AS STRING) AS parent_id, + NULLIF(TRIM(CAST(parent_id_name AS STRING)), '') AS parent_name, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.note}} diff --git a/zoho-crm/weld/staging/task.sql b/zoho-crm/weld/staging/task.sql new file mode 100644 index 0000000..f01cc15 --- /dev/null +++ b/zoho-crm/weld/staging/task.sql @@ -0,0 +1,24 @@ +-- staging.zoho_crm.task +-- Thin wrapper over raw `task`. Casts, renames, derives is_completed. +-- +-- NO PARENT LINK. Zoho's Tasks module has What_Id / Who_Id (the deal, account, +-- contact or lead the task hangs off), but the connector does not sync them. The +-- only foreign key on this stream is owner_id, so a task can be counted per rep +-- and per day and nothing else. Do not join it to a deal - there is no key to +-- join on. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS task_id, + NULLIF(TRIM(CAST(subject AS STRING)), '') AS subject, + NULLIF(TRIM(CAST(status AS STRING)), '') AS status, + NULLIF(TRIM(CAST(priority AS STRING)), '') AS priority, + CAST(due_date AS DATE) AS due_date, + -- Zoho's default Task statuses are Not Started, Deferred, In Progress, + -- Completed and Waiting for input. Matched on a pattern rather than equality + -- so a renamed or translated status does not silently read as incomplete. + LOWER(CAST(status AS STRING)) LIKE '%complet%' AS is_completed, + CAST(owner_id AS STRING) AS owner_id, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.task}} diff --git a/zoho-crm/weld/staging/user.sql b/zoho-crm/weld/staging/user.sql new file mode 100644 index 0000000..13bf12b --- /dev/null +++ b/zoho-crm/weld/staging/user.sql @@ -0,0 +1,27 @@ +-- staging.zoho_crm.user +-- Thin wrapper over raw `user`. Casts, renames, derives is_active. No other logic. +-- +-- This is the only dimension every other stream can join to: every record-owning +-- module carries owner_id, and nothing else in the Zoho schema carries a foreign +-- key. Sync this stream even if you think you do not need it. +-- +-- Single Zoho org. To add another, UNION ALL a second block below pointing at that +-- org's connector with a different zoho_org label. Keep it in staging so the core +-- models never have to know how many orgs there are. + +SELECT + 'org_1' AS zoho_org, + CAST(id AS STRING) AS user_id, + NULLIF(TRIM(CAST(full_name AS STRING)), '') AS full_name, + CAST(first_name AS STRING) AS first_name, + CAST(last_name AS STRING) AS last_name, + LOWER(NULLIF(TRIM(CAST(email AS STRING)), '')) AS email, + CAST(status AS STRING) AS status, + -- Zoho keeps deactivated users in the module rather than deleting them, so + -- filtering on this is how you get "reps who could take a deal today". + LOWER(CAST(status AS STRING)) = 'active' AS is_active, + CAST(role_name AS STRING) AS role_name, + CAST(profile_name AS STRING) AS profile_name, + CAST(created_time AS TIMESTAMP) AS created_time, + CAST(modified_time AS TIMESTAMP) AS modified_time +FROM {{raw.zoho_crm.user}} From 8606fb4e9d0bf0547f69d525b3e159677c7a28bf Mon Sep 17 00:00:00 2001 From: ada-weld Date: Thu, 27 Aug 2026 15:10:06 +0200 Subject: [PATCH 2/2] fix(zoho-crm): keep is_overdue and is_stale strictly boolean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Copilot review on #2. closing_date is nullable, and NULL < CURRENT_DATE() is NULL rather than FALSE, so is_overdue was tri-valued: a downstream WHERE NOT is_overdue silently dropped every open deal with no expected close date. Wrapped in COALESCE(..., FALSE) — no close date means not overdue. Same defect class in account_360.is_stale_with_open_pipeline, which went NULL when an account had neither a note nor a created_time. Fixed the same way rather than left inconsistent. Also reworded the rep_activity header note that named the raw column parent_id_id while the surrounding model talks in staging names; it now says parent_id_id in raw, parent_id after staging. Verified against BigQuery: the old expression returns NULL for an open deal with a null closing_date, the new one returns FALSE, and the three other cases are unchanged. NOT changed — the six review comments about GENERATE_DATE_ARRAY erroring on a NULL start date. BigQuery returns NULL for a NULL argument rather than raising, and UNNEST(NULL) yields zero rows, so an empty source already produces an empty result set. Confirmed by query: GENERATE_DATE_ARRAY(NULL, CURRENT_DATE(), INTERVAL 1 DAY) IS NULL -> true, and the UNNEST row count is 0. The spines are correct as written. --- .../dbt/models/core/core_zoho_crm__account_360.sql | 12 +++++++++--- .../dbt/models/core/core_zoho_crm__deal_pipeline.sql | 8 +++++++- .../dbt/models/core/core_zoho_crm__rep_activity.sql | 4 ++-- zoho-crm/weld/core/account_360.sql | 12 +++++++++--- zoho-crm/weld/core/deal_pipeline.sql | 8 +++++++- zoho-crm/weld/core/rep_activity.sql | 4 ++-- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql b/zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql index 5091694..2ac1764 100644 --- a/zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql +++ b/zoho-crm/dbt/models/core/core_zoho_crm__account_360.sql @@ -114,9 +114,15 @@ SELECT -- An account with open pipeline and nothing written on it in a quarter is the -- report this model exists to produce. - COALESCE(d.deals_open, 0) > 0 - AND COALESCE(n.last_note_at, a.created_time) - < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY) AS is_stale_with_open_pipeline + -- Wrapped for the same reason as is_overdue above: if there is no note and no + -- account created_time, the comparison is NULL rather than FALSE, and the flag + -- stops being a clean boolean. Nothing to go on means not stale. + COALESCE( + COALESCE(d.deals_open, 0) > 0 + AND COALESCE(n.last_note_at, a.created_time) + < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY), + FALSE + ) AS is_stale_with_open_pipeline FROM {{ ref('stg_zoho_crm__account') }} a LEFT JOIN contacts c ON c.zoho_org = a.zoho_org AND c.account_id = a.account_id LEFT JOIN deals d ON d.zoho_org = a.zoho_org AND d.account_id = a.account_id diff --git a/zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql b/zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql index 8cff8bb..6876ab1 100644 --- a/zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql +++ b/zoho-crm/dbt/models/core/core_zoho_crm__deal_pipeline.sql @@ -69,7 +69,13 @@ SELECT -- Open deals whose expected close date has already passed: the cheapest -- pipeline-hygiene number there is, and usually the first thing a sales lead -- asks for. - d.stage_status = 'Open' AND d.closing_date < CURRENT_DATE() AS is_overdue, + -- + -- COALESCE, because closing_date can be NULL and NULL < CURRENT_DATE() is + -- NULL, not FALSE. Without it the flag is tri-valued and a downstream + -- WHERE NOT is_overdue quietly drops every deal with no expected close date. + -- No close date means not overdue. + COALESCE(d.stage_status = 'Open' AND d.closing_date < CURRENT_DATE(), FALSE) + AS is_overdue, CASE WHEN d.stage_status = 'Open' THEN DATE_DIFF(d.closing_date, CURRENT_DATE(), DAY) diff --git a/zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql b/zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql index 9a74207..282e6f5 100644 --- a/zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql +++ b/zoho-crm/dbt/models/core/core_zoho_crm__rep_activity.sql @@ -12,8 +12,8 @@ -- activity did we put into the deals we won". Any dashboard promising -- activity-to-outcome attribution from this connector alone is inventing the link. -- Until the connector syncs the parent ids, the join has to come from somewhere --- else - a calendar or dialer source keyed on email, or Zoho's Notes module, whose --- parent_id_id IS synced. +-- else - a calendar or dialer source keyed on email, or Zoho's Notes module, +-- whose parent id IS synced (parent_id_id in raw, parent_id after staging). -- -- WHY THERE IS NO tasks_completed COLUMN: the connector syncs is_completed but no -- completion timestamp. Completions can be counted as a current-state total, never diff --git a/zoho-crm/weld/core/account_360.sql b/zoho-crm/weld/core/account_360.sql index 54d7ab2..fe9b104 100644 --- a/zoho-crm/weld/core/account_360.sql +++ b/zoho-crm/weld/core/account_360.sql @@ -114,9 +114,15 @@ SELECT -- An account with open pipeline and nothing written on it in a quarter is the -- report this model exists to produce. - COALESCE(d.deals_open, 0) > 0 - AND COALESCE(n.last_note_at, a.created_time) - < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY) AS is_stale_with_open_pipeline + -- Wrapped for the same reason as is_overdue above: if there is no note and no + -- account created_time, the comparison is NULL rather than FALSE, and the flag + -- stops being a clean boolean. Nothing to go on means not stale. + COALESCE( + COALESCE(d.deals_open, 0) > 0 + AND COALESCE(n.last_note_at, a.created_time) + < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY), + FALSE + ) AS is_stale_with_open_pipeline FROM {{staging.zoho_crm.account}} a LEFT JOIN contacts c ON c.zoho_org = a.zoho_org AND c.account_id = a.account_id LEFT JOIN deals d ON d.zoho_org = a.zoho_org AND d.account_id = a.account_id diff --git a/zoho-crm/weld/core/deal_pipeline.sql b/zoho-crm/weld/core/deal_pipeline.sql index 0e8202c..88c84a4 100644 --- a/zoho-crm/weld/core/deal_pipeline.sql +++ b/zoho-crm/weld/core/deal_pipeline.sql @@ -67,7 +67,13 @@ SELECT -- Open deals whose expected close date has already passed: the cheapest -- pipeline-hygiene number there is, and usually the first thing a sales lead -- asks for. - d.stage_status = 'Open' AND d.closing_date < CURRENT_DATE() AS is_overdue, + -- + -- COALESCE, because closing_date can be NULL and NULL < CURRENT_DATE() is + -- NULL, not FALSE. Without it the flag is tri-valued and a downstream + -- WHERE NOT is_overdue quietly drops every deal with no expected close date. + -- No close date means not overdue. + COALESCE(d.stage_status = 'Open' AND d.closing_date < CURRENT_DATE(), FALSE) + AS is_overdue, CASE WHEN d.stage_status = 'Open' THEN DATE_DIFF(d.closing_date, CURRENT_DATE(), DAY) diff --git a/zoho-crm/weld/core/rep_activity.sql b/zoho-crm/weld/core/rep_activity.sql index 3d4b8db..27ef0a0 100644 --- a/zoho-crm/weld/core/rep_activity.sql +++ b/zoho-crm/weld/core/rep_activity.sql @@ -12,8 +12,8 @@ -- activity did we put into the deals we won". Any dashboard promising -- activity-to-outcome attribution from this connector alone is inventing the link. -- Until the connector syncs the parent ids, the join has to come from somewhere --- else - a calendar or dialer source keyed on email, or Zoho's Notes module, whose --- parent_id_id IS synced. +-- else - a calendar or dialer source keyed on email, or Zoho's Notes module, +-- whose parent id IS synced (parent_id_id in raw, parent_id after staging). -- -- WHY THERE IS NO tasks_completed COLUMN: the connector syncs is_completed but no -- completion timestamp. Completions can be counted as a current-state total, never