From 0205247ed09e6787a3b9fefe12329209c79e2a75 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 15:23:16 +0100 Subject: [PATCH 01/14] Add setup tools for package installation --- elt-pipelines/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elt-pipelines/pyproject.toml b/elt-pipelines/pyproject.toml index 5bdc8562..c8475923 100644 --- a/elt-pipelines/pyproject.toml +++ b/elt-pipelines/pyproject.toml @@ -9,6 +9,9 @@ dependencies = [ "pydantic-settings>=2.14.2", ] +[tool.setuptools] +packages = ["facility_ops", "fase"] + [project.optional-dependencies] proposal = [ "sqlalchemy[postgresql-psycopgbinary]>=2.0.0", From 93524eaf3eaf37bce6fdd5eb9089d421c42f97c3 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 15:51:29 +0100 Subject: [PATCH 02/14] create accelerator sources file --- .../models/staging/accelerator/_accelerator__sources.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml new file mode 100644 index 00000000..f0db35d4 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -0,0 +1,7 @@ +version: 2 + +sources: + - name: accelerator_opralogweb + database: facility_ops_landing + tables: + - name: AdditionalColumns From 90955933841003fd5a2d1b03342a41d7e495a7b1 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 15:51:53 +0100 Subject: [PATCH 03/14] migrate additional columns table to elt --- .../stg_opralogweb_additional_columns.sql | 22 +++++++++++++++++++ .../stg_opralogweb_additional_columns.yml | 2 +- .../base_opralogweb__additional_columns.sql | 22 ------------------- 3 files changed, 23 insertions(+), 23 deletions(-) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.sql rename warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.yml (78%) delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.sql new file mode 100644 index 00000000..b80c4a9a --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.sql @@ -0,0 +1,22 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'AdditionalColumns') }} + +), + +renamed as ( + + select + + AdditionalColumnId as additional_column_id, + trim(ColTitle) as column_title + +from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.yml similarity index 78% rename from warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.yml index 3463a69f..a049d580 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.yml @@ -1,5 +1,5 @@ models: - - name: base_opralogweb__additional_columns + - name: stg_opralogweb_additional_columns description: Opralogweb additional_columns table with basic cleaning columns: - name: additional_column_id diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql deleted file mode 100644 index 4a125cb2..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql +++ /dev/null @@ -1,22 +0,0 @@ -with - -source as ( - - select * from {{ source('accelerator_opralogweb', 'additional_columns') }} - -), - -renamed as ( - - select - - additional_column_id, - trim(col_title) as column_title - -from - - source - -) - -select * from renamed From 5e4e31a57427ccb9a8bcb08373dcbf53eef50261 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 16:09:57 +0100 Subject: [PATCH 04/14] migrate chapter entry table to elt --- .../accelerator/_accelerator__sources.yml | 1 + .../stg_opralogweb_chapter_entry.sql | 24 +++++++++++++++++++ .../stg_opralogweb_chapter_entry.yml | 2 +- .../base_opralogweb__chapter_entry.sql | 24 ------------------- 4 files changed, 26 insertions(+), 25 deletions(-) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.sql rename warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.yml (78%) delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml index f0db35d4..67eaf34d 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -5,3 +5,4 @@ sources: database: facility_ops_landing tables: - name: AdditionalColumns + - name: ChapterEntry diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.sql new file mode 100644 index 00000000..15c8f39e --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.sql @@ -0,0 +1,24 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'ChapterEntry') }} + +), + +renamed as ( + +select + + EntryId as entry_id, + PrincipalLogbook as principal_logbook, + LogbookChapterNo as logbook_chapter_no, + LogbookId as logbook_id + +from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.yml similarity index 78% rename from warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.yml index c57b7ff0..0c70321f 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.yml @@ -1,5 +1,5 @@ models: - - name: base_opralogweb__chapter_entry + - name: stg_opralogweb_chapter_entry description: Opralogweb chapter_entry table with basic cleaning columns: - name: entry_id diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql deleted file mode 100644 index f7ae0497..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql +++ /dev/null @@ -1,24 +0,0 @@ -with - -source as ( - - select * from {{ source('accelerator_opralogweb', 'chapter_entry') }} - -), - -renamed as ( - -select - - entry_id, - principal_logbook, - logbook_chapter_no, - logbook_id - -from - - source - -) - -select * from renamed From 63ebcbcc220dcdf27164d743de173929295cf155 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 16:14:09 +0100 Subject: [PATCH 05/14] migrate entries table to elt --- .../accelerator/_accelerator__sources.yml | 1 + .../accelerator/stg_opralogweb_entries.sql | 26 +++++++++++++++++++ .../accelerator/stg_opralogweb_entries.yml | 2 +- .../accelerator/base_opralogweb__entries.sql | 26 ------------------- 4 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.sql rename warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.yml (81%) delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml index 67eaf34d..f285a7ba 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -6,3 +6,4 @@ sources: tables: - name: AdditionalColumns - name: ChapterEntry + - name: Entries diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.sql new file mode 100644 index 00000000..ecfd6624 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.sql @@ -0,0 +1,26 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'Entries') }} + +), + +renamed as ( + + select + + EntryId as entry_id, + with_timezone(EntryTimestamp, 'UTC') as fault_occurred_at, + cast({{ adapter.quote('EntryTimestamp') }} as date) as fault_date, + trim(AdditionalComment) as fault_description, + case + when LogicallyDeleted = 'Y' then true + else false + end as logically_deleted + + from source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.yml similarity index 81% rename from warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.yml index ef22cea4..db4b771d 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.yml @@ -1,5 +1,5 @@ models: - - name: base_opralogweb__entries + - name: stg_opralogweb_entries description: > Opralogweb "entries" table with basic cleaning. columns: diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql deleted file mode 100644 index 08a66cc3..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql +++ /dev/null @@ -1,26 +0,0 @@ -with - -source as ( - - select * from {{ source('accelerator_opralogweb', 'entries') }} - -), - -renamed as ( - - select - - entry_id, - with_timezone(entry_timestamp, 'UTC') as fault_occurred_at, - cast({{ adapter.quote('entry_timestamp') }} as date) as fault_date, - trim(additional_comment) as fault_description, - case - when logically_deleted = 'Y' then true - else false - end as logically_deleted - - from source - -) - -select * from renamed From c450bd5b9fee7cf7990b4fb86d9ef083adf347eb Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 16:17:16 +0100 Subject: [PATCH 06/14] migrate logbook chapter to elt --- .../accelerator/_accelerator__sources.yml | 1 + .../stg_opralogweb_logbook_chapter.sql | 21 +++++++++++++++++++ .../stg_opralogweb_logbook_chapter.yml | 2 +- .../base_opralogweb__logbook_chapter.sql | 21 ------------------- 4 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.sql rename warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.yml (78%) delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml index f285a7ba..3f1f9fa3 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -7,3 +7,4 @@ sources: - name: AdditionalColumns - name: ChapterEntry - name: Entries + - name: LogBookChapter diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.sql new file mode 100644 index 00000000..d615464c --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.sql @@ -0,0 +1,21 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'LogBookChapter') }} + +), + +renamed as ( + + select + + LogBookChapterNo as logbook_chapter_no + + from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.yml similarity index 78% rename from warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.yml index 6401cc8c..3eadc504 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.yml @@ -1,5 +1,5 @@ models: - - name: base_opralogweb__logbook_chapter + - name: stg_opralogweb_logbook_chapter description: Opralogweb logbook_chapter table with basic cleaning columns: - name: logbook_chapter_no diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql deleted file mode 100644 index 34b5f312..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql +++ /dev/null @@ -1,21 +0,0 @@ -with - -source as ( - - select * from {{ source('accelerator_opralogweb', 'logbook_chapter') }} - -), - -renamed as ( - - select - - logbook_chapter_no - - from - - source - -) - -select * from renamed From af3a2a80345f8e6afabfc194abff7af78dc99a6d Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 16:21:01 +0100 Subject: [PATCH 07/14] migrate logbooks to elt --- .../accelerator/_accelerator__sources.yml | 1 + .../accelerator/stg_opralogweb_logbooks.sql | 21 +++++++++++++++++++ .../accelerator/stg_opralogweb_logbooks.yml | 2 +- .../accelerator/base_opralogweb__logbooks.sql | 21 ------------------- 4 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.sql rename warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.yml (80%) delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml index 3f1f9fa3..b1a21525 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -8,3 +8,4 @@ sources: - name: ChapterEntry - name: Entries - name: LogBookChapter + - name: Logbooks diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.sql new file mode 100644 index 00000000..fada42d6 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.sql @@ -0,0 +1,21 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'Logbooks') }} + +), + +renamed as ( + + select + + LogbookId as logbook_id, + LogbookName as logbook_name + + from + + source +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.yml similarity index 80% rename from warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.yml index 21346c87..9f8833ec 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.yml @@ -1,5 +1,5 @@ models: - - name: base_opralogweb__logbooks + - name: stg_opralogweb_logbooks description: Opralogweb logbooks table with basic cleaning columns: - name: logbook_id diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql deleted file mode 100644 index eb8714d3..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql +++ /dev/null @@ -1,21 +0,0 @@ -with - -source as ( - - select * from {{ source('accelerator_opralogweb', 'logbooks') }} - -), - -renamed as ( - - select - - logbook_id, - logbook_name - - from - - source -) - -select * from renamed From d5425129c4c7d59a3ea91a7b49bd09a029ade70b Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 16:24:49 +0100 Subject: [PATCH 08/14] migrate more entry columns to elt --- .../accelerator/_accelerator__sources.yml | 1 + .../stg_opralogweb_more_entry_columns.sql | 22 +++++++++++++++++++ .../stg_opralogweb_more_entry_columns.yml | 2 +- .../base_opralogweb__more_entry_columns.sql | 22 ------------------- 4 files changed, 24 insertions(+), 23 deletions(-) create mode 100644 elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.sql rename warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.yml (76%) delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml index b1a21525..28b76ead 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -9,3 +9,4 @@ sources: - name: Entries - name: LogBookChapter - name: Logbooks + - name: MoreEntryColumns diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.sql new file mode 100644 index 00000000..8c8d28a2 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.sql @@ -0,0 +1,22 @@ +with + +source as ( + select * from {{ source('accelerator_opralogweb', 'MoreEntryColumns') }} +), + +renamed as ( + + select + + EntryId as entry_id, + {{ normalize_whitespace('ColData') }} as string_data, + NumberValue as number_data, + AdditionalColumnId as additional_column_id + + from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.yml similarity index 76% rename from warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.yml index 8794b332..ae709358 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.yml @@ -1,5 +1,5 @@ models: - - name: base_opralogweb__more_entry_columns + - name: stg_opralogweb_more_entry_columns description: Opralogweb more_entry_columns table with basic cleaning columns: - name: entry_id diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql deleted file mode 100644 index 02b8a4cd..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql +++ /dev/null @@ -1,22 +0,0 @@ -with - -source as ( - select * from {{ source('accelerator_opralogweb', 'more_entry_columns') }} -), - -renamed as ( - - select - - entry_id, - {{ normalize_whitespace('col_data') }} as string_data, - number_value as number_data, - additional_column_id - - from - - source - -) - -select * from renamed From 418ada1328aa995e819925c54c18e1c885e8b1ee Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Fri, 4 Sep 2026 16:27:57 +0100 Subject: [PATCH 09/14] migrate mcr equiment downtime to elt --- .../stg_opralogweb_mcr_equipment_downtime.sql | 12 ++++++------ .../stg_opralogweb_mcr_equipment_downtime.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) rename warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql (82%) rename warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml => elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml (96%) diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql similarity index 82% rename from warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql index e0b780ba..235901f6 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql @@ -3,17 +3,17 @@ with -staging_entries as ( select * from {{ ref('base_opralogweb__entries') }} ), +staging_entries as ( select * from {{ ref('stg_opralogweb_entries') }} ), -staging_chapter_entry as ( select * from {{ ref('base_opralogweb__chapter_entry') }} ), +staging_chapter_entry as ( select * from {{ ref('stg_opralogweb_chapter_entry') }} ), -staging_logbook_chapter as ( select * from {{ ref('base_opralogweb__logbook_chapter') }} ), +staging_logbook_chapter as ( select * from {{ ref('stg_opralogweb_logbook_chapter') }} ), -staging_logbooks as ( select * from {{ ref('base_opralogweb__logbooks') }} ), +staging_logbooks as ( select * from {{ ref('stg_opralogweb_logbooks') }} ), -staging_more_entry_columns as ( select * from {{ ref('base_opralogweb__more_entry_columns') }} ), +staging_more_entry_columns as ( select * from {{ ref('stg_opralogweb_more_entry_columns') }} ), -staging_additional_columns as ( select * from {{ ref('base_opralogweb__additional_columns') }} ), +staging_additional_columns as ( select * from {{ ref('stg_opralogweb_additional_columns') }} ), denormalized as ( select diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml similarity index 96% rename from warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml index 625c4aeb..ff022ffc 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml @@ -1,5 +1,5 @@ models: - - name: stg_opralogweb__mcr_equipment_downtime + - name: stg_opralogweb_mcr_equipment_downtime description: > Equipment downtime records for the MCR assembled from the relational database of the web-based Opralog application. Entries before cycle 17/01 From af2583a881e892c42da80f57104b93f99884cc31 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Mon, 7 Sep 2026 17:40:22 +0100 Subject: [PATCH 10/14] corrected nomenclature; add base directory. References https://github.com/ISISNeutronMuon/analytics-data-platform/pull/462#discussion_r3947860058 --- .../base_opralogweb__additional_columns.sql} | 0 .../base_opralogweb__additional_columns.yml} | 0 .../base_opralogweb__chapter_entry.sql} | 0 .../base_opralogweb__chapter_entry.yml} | 0 .../base_opralogweb__entries.sql} | 0 .../base_opralogweb__entries.yml} | 0 .../base_opralogweb__logbook_chapter.sql} | 0 .../base_opralogweb__logbook_chapter.yml} | 0 .../base_opralogweb__logbooks.sql} | 0 .../base_opralogweb__logbooks.yml} | 0 .../base_opralogweb__more_entry_columns.sql} | 0 .../base_opralogweb__more_entry_columns.yml} | 0 .../stg_opralogweb_mcr_equipment_downtime.sql | 12 ++++++------ 13 files changed, 6 insertions(+), 6 deletions(-) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_additional_columns.sql => base/base_opralogweb__additional_columns.sql} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_additional_columns.yml => base/base_opralogweb__additional_columns.yml} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_chapter_entry.sql => base/base_opralogweb__chapter_entry.sql} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_chapter_entry.yml => base/base_opralogweb__chapter_entry.yml} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_entries.sql => base/base_opralogweb__entries.sql} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_entries.yml => base/base_opralogweb__entries.yml} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_logbook_chapter.sql => base/base_opralogweb__logbook_chapter.sql} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_logbook_chapter.yml => base/base_opralogweb__logbook_chapter.yml} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_logbooks.sql => base/base_opralogweb__logbooks.sql} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_logbooks.yml => base/base_opralogweb__logbooks.yml} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_more_entry_columns.sql => base/base_opralogweb__more_entry_columns.sql} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_more_entry_columns.yml => base/base_opralogweb__more_entry_columns.yml} (100%) diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.sql similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_additional_columns.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.sql similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_chapter_entry.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.sql similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_entries.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.sql similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbook_chapter.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.sql similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_logbooks.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.sql similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_more_entry_columns.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql index 235901f6..e0b780ba 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql @@ -3,17 +3,17 @@ with -staging_entries as ( select * from {{ ref('stg_opralogweb_entries') }} ), +staging_entries as ( select * from {{ ref('base_opralogweb__entries') }} ), -staging_chapter_entry as ( select * from {{ ref('stg_opralogweb_chapter_entry') }} ), +staging_chapter_entry as ( select * from {{ ref('base_opralogweb__chapter_entry') }} ), -staging_logbook_chapter as ( select * from {{ ref('stg_opralogweb_logbook_chapter') }} ), +staging_logbook_chapter as ( select * from {{ ref('base_opralogweb__logbook_chapter') }} ), -staging_logbooks as ( select * from {{ ref('stg_opralogweb_logbooks') }} ), +staging_logbooks as ( select * from {{ ref('base_opralogweb__logbooks') }} ), -staging_more_entry_columns as ( select * from {{ ref('stg_opralogweb_more_entry_columns') }} ), +staging_more_entry_columns as ( select * from {{ ref('base_opralogweb__more_entry_columns') }} ), -staging_additional_columns as ( select * from {{ ref('stg_opralogweb_additional_columns') }} ), +staging_additional_columns as ( select * from {{ ref('base_opralogweb__additional_columns') }} ), denormalized as ( select From 2ab568a033de49e99801a8c0635021865dc19e1f Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Tue, 8 Sep 2026 12:07:55 +0100 Subject: [PATCH 11/14] Corrected .yml configuration; copied files instead of moving them --- .../base_opralogweb__additional_columns.yml | 2 +- .../base/base_opralogweb__chapter_entry.yml | 2 +- .../base/base_opralogweb__entries.yml | 2 +- .../base/base_opralogweb__logbook_chapter.yml | 2 +- .../base/base_opralogweb__logbooks.yml | 2 +- .../base_opralogweb__more_entry_columns.yml | 2 +- .../base_opralogweb__additional_columns.sql | 22 +++++ .../base_opralogweb__additional_columns.yml | 7 ++ .../base_opralogweb__chapter_entry.sql | 24 +++++ .../base_opralogweb__chapter_entry.yml | 7 ++ .../accelerator/base_opralogweb__entries.sql | 26 +++++ .../accelerator/base_opralogweb__entries.yml | 8 ++ .../base_opralogweb__logbook_chapter.sql | 21 ++++ .../base_opralogweb__logbook_chapter.yml | 7 ++ .../accelerator/base_opralogweb__logbooks.sql | 21 ++++ .../accelerator/base_opralogweb__logbooks.yml | 7 ++ .../base_opralogweb__more_entry_columns.sql | 22 +++++ .../base_opralogweb__more_entry_columns.yml | 7 ++ .../stg_opralogweb_mcr_equipment_downtime.sql | 97 +++++++++++++++++++ .../stg_opralogweb_mcr_equipment_downtime.yml | 41 ++++++++ 20 files changed, 323 insertions(+), 6 deletions(-) create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml index a049d580..3463a69f 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml @@ -1,5 +1,5 @@ models: - - name: stg_opralogweb_additional_columns + - name: base_opralogweb__additional_columns description: Opralogweb additional_columns table with basic cleaning columns: - name: additional_column_id diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml index 0c70321f..c57b7ff0 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml @@ -1,5 +1,5 @@ models: - - name: stg_opralogweb_chapter_entry + - name: base_opralogweb__chapter_entry description: Opralogweb chapter_entry table with basic cleaning columns: - name: entry_id diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml index db4b771d..ef22cea4 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml @@ -1,5 +1,5 @@ models: - - name: stg_opralogweb_entries + - name: base_opralogweb__entries description: > Opralogweb "entries" table with basic cleaning. columns: diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml index 3eadc504..6401cc8c 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml @@ -1,5 +1,5 @@ models: - - name: stg_opralogweb_logbook_chapter + - name: base_opralogweb__logbook_chapter description: Opralogweb logbook_chapter table with basic cleaning columns: - name: logbook_chapter_no diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml index 9f8833ec..21346c87 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml @@ -1,5 +1,5 @@ models: - - name: stg_opralogweb_logbooks + - name: base_opralogweb__logbooks description: Opralogweb logbooks table with basic cleaning columns: - name: logbook_id diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml index ae709358..8794b332 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml @@ -1,5 +1,5 @@ models: - - name: stg_opralogweb_more_entry_columns + - name: base_opralogweb__more_entry_columns description: Opralogweb more_entry_columns table with basic cleaning columns: - name: entry_id diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql new file mode 100644 index 00000000..b80c4a9a --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql @@ -0,0 +1,22 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'AdditionalColumns') }} + +), + +renamed as ( + + select + + AdditionalColumnId as additional_column_id, + trim(ColTitle) as column_title + +from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml new file mode 100644 index 00000000..3463a69f --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__additional_columns + description: Opralogweb additional_columns table with basic cleaning + columns: + - name: additional_column_id + tests: + - not_null diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql new file mode 100644 index 00000000..15c8f39e --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql @@ -0,0 +1,24 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'ChapterEntry') }} + +), + +renamed as ( + +select + + EntryId as entry_id, + PrincipalLogbook as principal_logbook, + LogbookChapterNo as logbook_chapter_no, + LogbookId as logbook_id + +from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml new file mode 100644 index 00000000..c57b7ff0 --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__chapter_entry + description: Opralogweb chapter_entry table with basic cleaning + columns: + - name: entry_id + tests: + - not_null diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql new file mode 100644 index 00000000..ecfd6624 --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql @@ -0,0 +1,26 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'Entries') }} + +), + +renamed as ( + + select + + EntryId as entry_id, + with_timezone(EntryTimestamp, 'UTC') as fault_occurred_at, + cast({{ adapter.quote('EntryTimestamp') }} as date) as fault_date, + trim(AdditionalComment) as fault_description, + case + when LogicallyDeleted = 'Y' then true + else false + end as logically_deleted + + from source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml new file mode 100644 index 00000000..ef22cea4 --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.yml @@ -0,0 +1,8 @@ +models: + - name: base_opralogweb__entries + description: > + Opralogweb "entries" table with basic cleaning. + columns: + - name: entry_id + tests: + - not_null diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql new file mode 100644 index 00000000..d615464c --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql @@ -0,0 +1,21 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'LogBookChapter') }} + +), + +renamed as ( + + select + + LogBookChapterNo as logbook_chapter_no + + from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml new file mode 100644 index 00000000..6401cc8c --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__logbook_chapter + description: Opralogweb logbook_chapter table with basic cleaning + columns: + - name: logbook_chapter_no + tests: + - not_null diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql new file mode 100644 index 00000000..fada42d6 --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql @@ -0,0 +1,21 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'Logbooks') }} + +), + +renamed as ( + + select + + LogbookId as logbook_id, + LogbookName as logbook_name + + from + + source +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml new file mode 100644 index 00000000..21346c87 --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__logbooks + description: Opralogweb logbooks table with basic cleaning + columns: + - name: logbook_id + tests: + - not_null diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql new file mode 100644 index 00000000..8c8d28a2 --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql @@ -0,0 +1,22 @@ +with + +source as ( + select * from {{ source('accelerator_opralogweb', 'MoreEntryColumns') }} +), + +renamed as ( + + select + + EntryId as entry_id, + {{ normalize_whitespace('ColData') }} as string_data, + NumberValue as number_data, + AdditionalColumnId as additional_column_id + + from + + source + +) + +select * from renamed diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml new file mode 100644 index 00000000..8794b332 --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__more_entry_columns + description: Opralogweb more_entry_columns table with basic cleaning + columns: + - name: entry_id + tests: + - not_null diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql new file mode 100644 index 00000000..e0b780ba --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql @@ -0,0 +1,97 @@ +{%- set OPRALOG_EPOCH = dbt.string_literal("2017-04-25") -%} +{%- set MCR_LOGBOOK = dbt.string_literal("MCR Running Log") -%} + +with + +staging_entries as ( select * from {{ ref('base_opralogweb__entries') }} ), + +staging_chapter_entry as ( select * from {{ ref('base_opralogweb__chapter_entry') }} ), + +staging_logbook_chapter as ( select * from {{ ref('base_opralogweb__logbook_chapter') }} ), + +staging_logbooks as ( select * from {{ ref('base_opralogweb__logbooks') }} ), + +staging_more_entry_columns as ( select * from {{ ref('base_opralogweb__more_entry_columns') }} ), + +staging_additional_columns as ( select * from {{ ref('base_opralogweb__additional_columns') }} ), + +denormalized as ( + select + + staging_entries.entry_id, + staging_entries.fault_occurred_at, + staging_entries.fault_date, + staging_additional_columns.column_title, + staging_more_entry_columns.string_data, + staging_more_entry_columns.number_data, + staging_entries.fault_description + + from + staging_entries + join staging_chapter_entry on staging_chapter_entry.entry_id = staging_entries.entry_id + join staging_logbook_chapter on staging_logbook_chapter.logbook_chapter_no = staging_chapter_entry.logbook_chapter_no + join staging_logbooks on staging_logbooks.logbook_id = staging_chapter_entry.logbook_id + left outer join staging_more_entry_columns on staging_more_entry_columns.entry_id = staging_entries.entry_id + left outer join staging_additional_columns on staging_additional_columns.additional_column_id = staging_more_entry_columns.additional_column_id + + where + + staging_entries.logically_deleted = false + and staging_entries.fault_date >= from_iso8601_date({{ OPRALOG_EPOCH }}) + and staging_logbooks.logbook_name = {{ MCR_LOGBOOK }} + and staging_chapter_entry.logbook_id = staging_chapter_entry.principal_logbook + and staging_additional_columns.column_title in ('Equipment', 'Group', 'Lost Time', 'Group Leader comments') + and ( + staging_more_entry_columns.string_data is not null + or staging_more_entry_columns.number_data is not null + ) +), + +mcr_equipment_downtime as ( + select + * + from + ( + select + + min( + case + column_title + when 'Equipment' then string_data + end + ) as equipment, + min( + case + column_title + when 'Lost Time' then number_data + end + ) as downtime_mins, + fault_date, + fault_occurred_at, + min( + case + column_title + when 'Group' then string_data + end + ) as {{ adapter.quote('group') }}, + fault_description, + min( + case + column_title + when 'Group Leader comments' then string_data + end + ) as managers_comments + from + denormalized + group by + fault_occurred_at, + fault_date, + fault_description + ) + where + equipment is not null + and downtime_mins is not null + and {{ adapter.quote('group') }} is not null +) + +select * from mcr_equipment_downtime diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml new file mode 100644 index 00000000..ff022ffc --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml @@ -0,0 +1,41 @@ +models: + - name: stg_opralogweb_mcr_equipment_downtime + description: > + Equipment downtime records for the MCR assembled from the relational + database of the web-based Opralog application. Entries before cycle 17/01 + are discarded as Opralog was in testing mode before this. + +unit_tests: + - name: test_stg_opralogweb__mcr_equipment_downtime + description: "Check null & before epoch entries are discarded" + model: stg_opralogweb__mcr_equipment_downtime + given: + - input: ref('base_opralogweb__entries') + format: sql + fixture: base_opralogweb__entries + - input: ref('base_opralogweb__chapter_entry') + format: sql + fixture: base_opralogweb__chapter_entry + - input: ref('base_opralogweb__logbook_chapter') + format: sql + fixture: base_opralogweb__logbook_chapter + - input: ref('base_opralogweb__logbooks') + format: sql + fixture: base_opralogweb__logbooks + - input: ref('base_opralogweb__more_entry_columns') + format: sql + fixture: base_opralogweb__more_entry_columns + - input: ref('base_opralogweb__additional_columns') + format: sql + fixture: base_opralogweb__additional_columns + expect: + rows: + - { + equipment: "Equipment 101", + downtime_mins: 5.3, + fault_date: "2024-01-01", + fault_occurred_at: "2024-01-01 00:00:01 UTC", + group: "Group 101", + fault_description: "Comment 101", + managers_comments: null, + } From 512a341c61df354af064e2f5c19652134ccc144a Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Tue, 8 Sep 2026 15:45:02 +0100 Subject: [PATCH 12/14] Revert warehouse changes, referencing https://github.com/ISISNeutronMuon/analytics-data-platform/pull/462/changes#r3958187463 --- .../base_opralogweb__additional_columns.sql | 6 +- .../base_opralogweb__chapter_entry.sql | 10 +- .../accelerator/base_opralogweb__entries.sql | 12 +-- .../base_opralogweb__logbook_chapter.sql | 4 +- .../accelerator/base_opralogweb__logbooks.sql | 6 +- .../base_opralogweb__more_entry_columns.sql | 10 +- ...stg_opralogweb__mcr_equipment_downtime.sql | 97 +++++++++++++++++++ ...stg_opralogweb__mcr_equipment_downtime.yml | 41 ++++++++ 8 files changed, 162 insertions(+), 24 deletions(-) create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql create mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql index b80c4a9a..4a125cb2 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__additional_columns.sql @@ -2,7 +2,7 @@ with source as ( - select * from {{ source('accelerator_opralogweb', 'AdditionalColumns') }} + select * from {{ source('accelerator_opralogweb', 'additional_columns') }} ), @@ -10,8 +10,8 @@ renamed as ( select - AdditionalColumnId as additional_column_id, - trim(ColTitle) as column_title + additional_column_id, + trim(col_title) as column_title from diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql index 15c8f39e..f7ae0497 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__chapter_entry.sql @@ -2,7 +2,7 @@ with source as ( - select * from {{ source('accelerator_opralogweb', 'ChapterEntry') }} + select * from {{ source('accelerator_opralogweb', 'chapter_entry') }} ), @@ -10,10 +10,10 @@ renamed as ( select - EntryId as entry_id, - PrincipalLogbook as principal_logbook, - LogbookChapterNo as logbook_chapter_no, - LogbookId as logbook_id + entry_id, + principal_logbook, + logbook_chapter_no, + logbook_id from diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql index ecfd6624..08a66cc3 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__entries.sql @@ -2,7 +2,7 @@ with source as ( - select * from {{ source('accelerator_opralogweb', 'Entries') }} + select * from {{ source('accelerator_opralogweb', 'entries') }} ), @@ -10,12 +10,12 @@ renamed as ( select - EntryId as entry_id, - with_timezone(EntryTimestamp, 'UTC') as fault_occurred_at, - cast({{ adapter.quote('EntryTimestamp') }} as date) as fault_date, - trim(AdditionalComment) as fault_description, + entry_id, + with_timezone(entry_timestamp, 'UTC') as fault_occurred_at, + cast({{ adapter.quote('entry_timestamp') }} as date) as fault_date, + trim(additional_comment) as fault_description, case - when LogicallyDeleted = 'Y' then true + when logically_deleted = 'Y' then true else false end as logically_deleted diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql index d615464c..34b5f312 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbook_chapter.sql @@ -2,7 +2,7 @@ with source as ( - select * from {{ source('accelerator_opralogweb', 'LogBookChapter') }} + select * from {{ source('accelerator_opralogweb', 'logbook_chapter') }} ), @@ -10,7 +10,7 @@ renamed as ( select - LogBookChapterNo as logbook_chapter_no + logbook_chapter_no from diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql index fada42d6..eb8714d3 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__logbooks.sql @@ -2,7 +2,7 @@ with source as ( - select * from {{ source('accelerator_opralogweb', 'Logbooks') }} + select * from {{ source('accelerator_opralogweb', 'logbooks') }} ), @@ -10,8 +10,8 @@ renamed as ( select - LogbookId as logbook_id, - LogbookName as logbook_name + logbook_id, + logbook_name from diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql index 8c8d28a2..02b8a4cd 100644 --- a/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql +++ b/warehouses/facility_ops/transform/models/staging/accelerator/base_opralogweb__more_entry_columns.sql @@ -1,17 +1,17 @@ with source as ( - select * from {{ source('accelerator_opralogweb', 'MoreEntryColumns') }} + select * from {{ source('accelerator_opralogweb', 'more_entry_columns') }} ), renamed as ( select - EntryId as entry_id, - {{ normalize_whitespace('ColData') }} as string_data, - NumberValue as number_data, - AdditionalColumnId as additional_column_id + entry_id, + {{ normalize_whitespace('col_data') }} as string_data, + number_value as number_data, + additional_column_id from diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql new file mode 100644 index 00000000..e0b780ba --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql @@ -0,0 +1,97 @@ +{%- set OPRALOG_EPOCH = dbt.string_literal("2017-04-25") -%} +{%- set MCR_LOGBOOK = dbt.string_literal("MCR Running Log") -%} + +with + +staging_entries as ( select * from {{ ref('base_opralogweb__entries') }} ), + +staging_chapter_entry as ( select * from {{ ref('base_opralogweb__chapter_entry') }} ), + +staging_logbook_chapter as ( select * from {{ ref('base_opralogweb__logbook_chapter') }} ), + +staging_logbooks as ( select * from {{ ref('base_opralogweb__logbooks') }} ), + +staging_more_entry_columns as ( select * from {{ ref('base_opralogweb__more_entry_columns') }} ), + +staging_additional_columns as ( select * from {{ ref('base_opralogweb__additional_columns') }} ), + +denormalized as ( + select + + staging_entries.entry_id, + staging_entries.fault_occurred_at, + staging_entries.fault_date, + staging_additional_columns.column_title, + staging_more_entry_columns.string_data, + staging_more_entry_columns.number_data, + staging_entries.fault_description + + from + staging_entries + join staging_chapter_entry on staging_chapter_entry.entry_id = staging_entries.entry_id + join staging_logbook_chapter on staging_logbook_chapter.logbook_chapter_no = staging_chapter_entry.logbook_chapter_no + join staging_logbooks on staging_logbooks.logbook_id = staging_chapter_entry.logbook_id + left outer join staging_more_entry_columns on staging_more_entry_columns.entry_id = staging_entries.entry_id + left outer join staging_additional_columns on staging_additional_columns.additional_column_id = staging_more_entry_columns.additional_column_id + + where + + staging_entries.logically_deleted = false + and staging_entries.fault_date >= from_iso8601_date({{ OPRALOG_EPOCH }}) + and staging_logbooks.logbook_name = {{ MCR_LOGBOOK }} + and staging_chapter_entry.logbook_id = staging_chapter_entry.principal_logbook + and staging_additional_columns.column_title in ('Equipment', 'Group', 'Lost Time', 'Group Leader comments') + and ( + staging_more_entry_columns.string_data is not null + or staging_more_entry_columns.number_data is not null + ) +), + +mcr_equipment_downtime as ( + select + * + from + ( + select + + min( + case + column_title + when 'Equipment' then string_data + end + ) as equipment, + min( + case + column_title + when 'Lost Time' then number_data + end + ) as downtime_mins, + fault_date, + fault_occurred_at, + min( + case + column_title + when 'Group' then string_data + end + ) as {{ adapter.quote('group') }}, + fault_description, + min( + case + column_title + when 'Group Leader comments' then string_data + end + ) as managers_comments + from + denormalized + group by + fault_occurred_at, + fault_date, + fault_description + ) + where + equipment is not null + and downtime_mins is not null + and {{ adapter.quote('group') }} is not null +) + +select * from mcr_equipment_downtime diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml new file mode 100644 index 00000000..625c4aeb --- /dev/null +++ b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml @@ -0,0 +1,41 @@ +models: + - name: stg_opralogweb__mcr_equipment_downtime + description: > + Equipment downtime records for the MCR assembled from the relational + database of the web-based Opralog application. Entries before cycle 17/01 + are discarded as Opralog was in testing mode before this. + +unit_tests: + - name: test_stg_opralogweb__mcr_equipment_downtime + description: "Check null & before epoch entries are discarded" + model: stg_opralogweb__mcr_equipment_downtime + given: + - input: ref('base_opralogweb__entries') + format: sql + fixture: base_opralogweb__entries + - input: ref('base_opralogweb__chapter_entry') + format: sql + fixture: base_opralogweb__chapter_entry + - input: ref('base_opralogweb__logbook_chapter') + format: sql + fixture: base_opralogweb__logbook_chapter + - input: ref('base_opralogweb__logbooks') + format: sql + fixture: base_opralogweb__logbooks + - input: ref('base_opralogweb__more_entry_columns') + format: sql + fixture: base_opralogweb__more_entry_columns + - input: ref('base_opralogweb__additional_columns') + format: sql + fixture: base_opralogweb__additional_columns + expect: + rows: + - { + equipment: "Equipment 101", + downtime_mins: 5.3, + fault_date: "2024-01-01", + fault_occurred_at: "2024-01-01 00:00:01 UTC", + group: "Group 101", + fault_description: "Comment 101", + managers_comments: null, + } From 7a943b137b95b2bf0a6b1c7d03f41e5bd5c6c8a6 Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Tue, 8 Sep 2026 16:09:57 +0100 Subject: [PATCH 13/14] remove newer mcr downtime files, resolving duplication error --- .../stg_opralogweb_mcr_equipment_downtime.sql | 97 ------------------- .../stg_opralogweb_mcr_equipment_downtime.yml | 41 -------- 2 files changed, 138 deletions(-) delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql delete mode 100644 warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql deleted file mode 100644 index e0b780ba..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql +++ /dev/null @@ -1,97 +0,0 @@ -{%- set OPRALOG_EPOCH = dbt.string_literal("2017-04-25") -%} -{%- set MCR_LOGBOOK = dbt.string_literal("MCR Running Log") -%} - -with - -staging_entries as ( select * from {{ ref('base_opralogweb__entries') }} ), - -staging_chapter_entry as ( select * from {{ ref('base_opralogweb__chapter_entry') }} ), - -staging_logbook_chapter as ( select * from {{ ref('base_opralogweb__logbook_chapter') }} ), - -staging_logbooks as ( select * from {{ ref('base_opralogweb__logbooks') }} ), - -staging_more_entry_columns as ( select * from {{ ref('base_opralogweb__more_entry_columns') }} ), - -staging_additional_columns as ( select * from {{ ref('base_opralogweb__additional_columns') }} ), - -denormalized as ( - select - - staging_entries.entry_id, - staging_entries.fault_occurred_at, - staging_entries.fault_date, - staging_additional_columns.column_title, - staging_more_entry_columns.string_data, - staging_more_entry_columns.number_data, - staging_entries.fault_description - - from - staging_entries - join staging_chapter_entry on staging_chapter_entry.entry_id = staging_entries.entry_id - join staging_logbook_chapter on staging_logbook_chapter.logbook_chapter_no = staging_chapter_entry.logbook_chapter_no - join staging_logbooks on staging_logbooks.logbook_id = staging_chapter_entry.logbook_id - left outer join staging_more_entry_columns on staging_more_entry_columns.entry_id = staging_entries.entry_id - left outer join staging_additional_columns on staging_additional_columns.additional_column_id = staging_more_entry_columns.additional_column_id - - where - - staging_entries.logically_deleted = false - and staging_entries.fault_date >= from_iso8601_date({{ OPRALOG_EPOCH }}) - and staging_logbooks.logbook_name = {{ MCR_LOGBOOK }} - and staging_chapter_entry.logbook_id = staging_chapter_entry.principal_logbook - and staging_additional_columns.column_title in ('Equipment', 'Group', 'Lost Time', 'Group Leader comments') - and ( - staging_more_entry_columns.string_data is not null - or staging_more_entry_columns.number_data is not null - ) -), - -mcr_equipment_downtime as ( - select - * - from - ( - select - - min( - case - column_title - when 'Equipment' then string_data - end - ) as equipment, - min( - case - column_title - when 'Lost Time' then number_data - end - ) as downtime_mins, - fault_date, - fault_occurred_at, - min( - case - column_title - when 'Group' then string_data - end - ) as {{ adapter.quote('group') }}, - fault_description, - min( - case - column_title - when 'Group Leader comments' then string_data - end - ) as managers_comments - from - denormalized - group by - fault_occurred_at, - fault_date, - fault_description - ) - where - equipment is not null - and downtime_mins is not null - and {{ adapter.quote('group') }} is not null -) - -select * from mcr_equipment_downtime diff --git a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml b/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml deleted file mode 100644 index ff022ffc..00000000 --- a/warehouses/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml +++ /dev/null @@ -1,41 +0,0 @@ -models: - - name: stg_opralogweb_mcr_equipment_downtime - description: > - Equipment downtime records for the MCR assembled from the relational - database of the web-based Opralog application. Entries before cycle 17/01 - are discarded as Opralog was in testing mode before this. - -unit_tests: - - name: test_stg_opralogweb__mcr_equipment_downtime - description: "Check null & before epoch entries are discarded" - model: stg_opralogweb__mcr_equipment_downtime - given: - - input: ref('base_opralogweb__entries') - format: sql - fixture: base_opralogweb__entries - - input: ref('base_opralogweb__chapter_entry') - format: sql - fixture: base_opralogweb__chapter_entry - - input: ref('base_opralogweb__logbook_chapter') - format: sql - fixture: base_opralogweb__logbook_chapter - - input: ref('base_opralogweb__logbooks') - format: sql - fixture: base_opralogweb__logbooks - - input: ref('base_opralogweb__more_entry_columns') - format: sql - fixture: base_opralogweb__more_entry_columns - - input: ref('base_opralogweb__additional_columns') - format: sql - fixture: base_opralogweb__additional_columns - expect: - rows: - - { - equipment: "Equipment 101", - downtime_mins: 5.3, - fault_date: "2024-01-01", - fault_occurred_at: "2024-01-01 00:00:01 UTC", - group: "Group 101", - fault_description: "Comment 101", - managers_comments: null, - } From 3e3a3c9979461c3304418899768114b290ead1db Mon Sep 17 00:00:00 2001 From: Andrew Bolt Date: Tue, 8 Sep 2026 16:23:05 +0100 Subject: [PATCH 14/14] Corrected file naming for elt-pipeline views --- ...nt_downtime.sql => stg_opralogweb__mcr_equipment_downtime.sql} | 0 ...nt_downtime.yml => stg_opralogweb__mcr_equipment_downtime.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_mcr_equipment_downtime.sql => stg_opralogweb__mcr_equipment_downtime.sql} (100%) rename elt-pipelines/facility_ops/transform/models/staging/accelerator/{stg_opralogweb_mcr_equipment_downtime.yml => stg_opralogweb__mcr_equipment_downtime.yml} (100%) diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.sql rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml similarity index 100% rename from elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb_mcr_equipment_downtime.yml rename to elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml