diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml new file mode 100644 index 00000000..28b76ead --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml @@ -0,0 +1,12 @@ +version: 2 + +sources: + - name: accelerator_opralogweb + database: facility_ops_landing + tables: + - name: AdditionalColumns + - name: ChapterEntry + - name: Entries + - name: LogBookChapter + - name: Logbooks + - name: MoreEntryColumns diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.sql new file mode 100644 index 00000000..b80c4a9a --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.sql @@ -0,0 +1,22 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'AdditionalColumns') }} + +), + +renamed as ( + + select + + AdditionalColumnId as additional_column_id, + trim(ColTitle) as column_title + +from + + source + +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml new file mode 100644 index 00000000..3463a69f --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__additional_columns + description: Opralogweb additional_columns table with basic cleaning + columns: + - name: additional_column_id + tests: + - not_null diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.sql new file mode 100644 index 00000000..15c8f39e --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.sql @@ -0,0 +1,24 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'ChapterEntry') }} + +), + +renamed as ( + +select + + EntryId as entry_id, + PrincipalLogbook as principal_logbook, + LogbookChapterNo as logbook_chapter_no, + LogbookId as logbook_id + +from + + source + +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml new file mode 100644 index 00000000..c57b7ff0 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__chapter_entry + description: Opralogweb chapter_entry table with basic cleaning + columns: + - name: entry_id + tests: + - not_null diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.sql new file mode 100644 index 00000000..ecfd6624 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.sql @@ -0,0 +1,26 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'Entries') }} + +), + +renamed as ( + + select + + EntryId as entry_id, + with_timezone(EntryTimestamp, 'UTC') as fault_occurred_at, + cast({{ adapter.quote('EntryTimestamp') }} as date) as fault_date, + trim(AdditionalComment) as fault_description, + case + when LogicallyDeleted = 'Y' then true + else false + end as logically_deleted + + from source + +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml new file mode 100644 index 00000000..ef22cea4 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml @@ -0,0 +1,8 @@ +models: + - name: base_opralogweb__entries + description: > + Opralogweb "entries" table with basic cleaning. + columns: + - name: entry_id + tests: + - not_null diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.sql new file mode 100644 index 00000000..d615464c --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.sql @@ -0,0 +1,21 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'LogBookChapter') }} + +), + +renamed as ( + + select + + LogBookChapterNo as logbook_chapter_no + + from + + source + +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml new file mode 100644 index 00000000..6401cc8c --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__logbook_chapter + description: Opralogweb logbook_chapter table with basic cleaning + columns: + - name: logbook_chapter_no + tests: + - not_null diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.sql new file mode 100644 index 00000000..fada42d6 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.sql @@ -0,0 +1,21 @@ +with + +source as ( + + select * from {{ source('accelerator_opralogweb', 'Logbooks') }} + +), + +renamed as ( + + select + + LogbookId as logbook_id, + LogbookName as logbook_name + + from + + source +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml new file mode 100644 index 00000000..21346c87 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__logbooks + description: Opralogweb logbooks table with basic cleaning + columns: + - name: logbook_id + tests: + - not_null diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.sql new file mode 100644 index 00000000..8c8d28a2 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.sql @@ -0,0 +1,22 @@ +with + +source as ( + select * from {{ source('accelerator_opralogweb', 'MoreEntryColumns') }} +), + +renamed as ( + + select + + EntryId as entry_id, + {{ normalize_whitespace('ColData') }} as string_data, + NumberValue as number_data, + AdditionalColumnId as additional_column_id + + from + + source + +) + +select * from renamed diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml new file mode 100644 index 00000000..8794b332 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml @@ -0,0 +1,7 @@ +models: + - name: base_opralogweb__more_entry_columns + description: Opralogweb more_entry_columns table with basic cleaning + columns: + - name: entry_id + tests: + - not_null diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql new file mode 100644 index 00000000..e0b780ba --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql @@ -0,0 +1,97 @@ +{%- set OPRALOG_EPOCH = dbt.string_literal("2017-04-25") -%} +{%- set MCR_LOGBOOK = dbt.string_literal("MCR Running Log") -%} + +with + +staging_entries as ( select * from {{ ref('base_opralogweb__entries') }} ), + +staging_chapter_entry as ( select * from {{ ref('base_opralogweb__chapter_entry') }} ), + +staging_logbook_chapter as ( select * from {{ ref('base_opralogweb__logbook_chapter') }} ), + +staging_logbooks as ( select * from {{ ref('base_opralogweb__logbooks') }} ), + +staging_more_entry_columns as ( select * from {{ ref('base_opralogweb__more_entry_columns') }} ), + +staging_additional_columns as ( select * from {{ ref('base_opralogweb__additional_columns') }} ), + +denormalized as ( + select + + staging_entries.entry_id, + staging_entries.fault_occurred_at, + staging_entries.fault_date, + staging_additional_columns.column_title, + staging_more_entry_columns.string_data, + staging_more_entry_columns.number_data, + staging_entries.fault_description + + from + staging_entries + join staging_chapter_entry on staging_chapter_entry.entry_id = staging_entries.entry_id + join staging_logbook_chapter on staging_logbook_chapter.logbook_chapter_no = staging_chapter_entry.logbook_chapter_no + join staging_logbooks on staging_logbooks.logbook_id = staging_chapter_entry.logbook_id + left outer join staging_more_entry_columns on staging_more_entry_columns.entry_id = staging_entries.entry_id + left outer join staging_additional_columns on staging_additional_columns.additional_column_id = staging_more_entry_columns.additional_column_id + + where + + staging_entries.logically_deleted = false + and staging_entries.fault_date >= from_iso8601_date({{ OPRALOG_EPOCH }}) + and staging_logbooks.logbook_name = {{ MCR_LOGBOOK }} + and staging_chapter_entry.logbook_id = staging_chapter_entry.principal_logbook + and staging_additional_columns.column_title in ('Equipment', 'Group', 'Lost Time', 'Group Leader comments') + and ( + staging_more_entry_columns.string_data is not null + or staging_more_entry_columns.number_data is not null + ) +), + +mcr_equipment_downtime as ( + select + * + from + ( + select + + min( + case + column_title + when 'Equipment' then string_data + end + ) as equipment, + min( + case + column_title + when 'Lost Time' then number_data + end + ) as downtime_mins, + fault_date, + fault_occurred_at, + min( + case + column_title + when 'Group' then string_data + end + ) as {{ adapter.quote('group') }}, + fault_description, + min( + case + column_title + when 'Group Leader comments' then string_data + end + ) as managers_comments + from + denormalized + group by + fault_occurred_at, + fault_date, + fault_description + ) + where + equipment is not null + and downtime_mins is not null + and {{ adapter.quote('group') }} is not null +) + +select * from mcr_equipment_downtime diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml new file mode 100644 index 00000000..ff022ffc --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml @@ -0,0 +1,41 @@ +models: + - name: stg_opralogweb_mcr_equipment_downtime + description: > + Equipment downtime records for the MCR assembled from the relational + database of the web-based Opralog application. Entries before cycle 17/01 + are discarded as Opralog was in testing mode before this. + +unit_tests: + - name: test_stg_opralogweb__mcr_equipment_downtime + description: "Check null & before epoch entries are discarded" + model: stg_opralogweb__mcr_equipment_downtime + given: + - input: ref('base_opralogweb__entries') + format: sql + fixture: base_opralogweb__entries + - input: ref('base_opralogweb__chapter_entry') + format: sql + fixture: base_opralogweb__chapter_entry + - input: ref('base_opralogweb__logbook_chapter') + format: sql + fixture: base_opralogweb__logbook_chapter + - input: ref('base_opralogweb__logbooks') + format: sql + fixture: base_opralogweb__logbooks + - input: ref('base_opralogweb__more_entry_columns') + format: sql + fixture: base_opralogweb__more_entry_columns + - input: ref('base_opralogweb__additional_columns') + format: sql + fixture: base_opralogweb__additional_columns + expect: + rows: + - { + equipment: "Equipment 101", + downtime_mins: 5.3, + fault_date: "2024-01-01", + fault_occurred_at: "2024-01-01 00:00:01 UTC", + group: "Group 101", + fault_description: "Comment 101", + managers_comments: null, + } diff --git a/elt-pipelines/pyproject.toml b/elt-pipelines/pyproject.toml index 5bdc8562..c8475923 100644 --- a/elt-pipelines/pyproject.toml +++ b/elt-pipelines/pyproject.toml @@ -9,6 +9,9 @@ dependencies = [ "pydantic-settings>=2.14.2", ] +[tool.setuptools] +packages = ["facility_ops", "fase"] + [project.optional-dependencies] proposal = [ "sqlalchemy[postgresql-psycopgbinary]>=2.0.0",