feat(elt-pipelines): Port opralogweb to elt-pipelines

elt-pipelines/facility_ops/transform/models/staging/accelerator/_accelerator__sources.yml

@@ -0,0 +1,12 @@
+version: 2
+
+sources:
+  - name: accelerator_opralogweb
+    database: facility_ops_landing
+    tables:
+      - name: AdditionalColumns
+      - name: ChapterEntry
+      - name: Entries
+      - name: LogBookChapter
+      - name: Logbooks
+      - name: MoreEntryColumns

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.sql

@@ -0,0 +1,22 @@
+with
+
+source as (
+
+  select * from {{ source('accelerator_opralogweb', 'AdditionalColumns') }}
+
+),
+
+renamed as (
+
+  select
+
+    AdditionalColumnId as additional_column_id,
+    trim(ColTitle) as column_title
+
+from
+
+  source
+
+)
+
+select * from renamed

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__additional_columns.yml

@@ -0,0 +1,7 @@
+models:
+  - name: base_opralogweb__additional_columns
+    description: Opralogweb additional_columns table with basic cleaning
+    columns:
+      - name: additional_column_id
+        tests:
+          - not_null

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.sql

@@ -0,0 +1,24 @@
+with
+
+source as (
+
+  select * from {{ source('accelerator_opralogweb', 'ChapterEntry') }}
+
+),
+
+renamed as (
+
+select
+
+    EntryId as entry_id,
+    PrincipalLogbook as principal_logbook,
+    LogbookChapterNo as logbook_chapter_no,
+    LogbookId as logbook_id
+
+from
+
+  source
+
+)
+
+select * from renamed

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__chapter_entry.yml

@@ -0,0 +1,7 @@
+models:
+  - name: base_opralogweb__chapter_entry
+    description: Opralogweb chapter_entry table with basic cleaning
+    columns:
+      - name: entry_id
+        tests:
+          - not_null

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.sql

@@ -0,0 +1,26 @@
+with
+
+source as (
+
+  select * from {{ source('accelerator_opralogweb', 'Entries') }}
+
+),
+
+renamed as (
+
+  select
+
+      EntryId as entry_id,
+      with_timezone(EntryTimestamp, 'UTC') as fault_occurred_at,
+      cast({{ adapter.quote('EntryTimestamp') }} as date) as fault_date,
+      trim(AdditionalComment) as fault_description,
+      case
+          when LogicallyDeleted = 'Y' then true
+          else false
+      end as logically_deleted
+
+  from source
+
+)
+
+select * from renamed

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__entries.yml

@@ -0,0 +1,8 @@
+models:
+  - name: base_opralogweb__entries
+    description: >
+      Opralogweb "entries" table with basic cleaning.
+    columns:
+      - name: entry_id
+        tests:
+          - not_null

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.sql

@@ -0,0 +1,21 @@
+with
+
+source as (
+
+  select * from {{ source('accelerator_opralogweb', 'LogBookChapter') }}
+
+),
+
+renamed as (
+
+  select
+
+    LogBookChapterNo as logbook_chapter_no
+
+  from
+
+    source
+
+)
+
+select * from renamed

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbook_chapter.yml

@@ -0,0 +1,7 @@
+models:
+  - name: base_opralogweb__logbook_chapter
+    description: Opralogweb logbook_chapter table with basic cleaning
+    columns:
+      - name: logbook_chapter_no
+        tests:
+          - not_null

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.sql

@@ -0,0 +1,21 @@
+with
+
+source as (
+
+  select * from {{ source('accelerator_opralogweb', 'Logbooks') }}
+
+),
+
+renamed as (
+
+  select
+
+      LogbookId as logbook_id,
+      LogbookName as logbook_name
+
+  from
+
+    source
+)
+
+select * from renamed

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__logbooks.yml

@@ -0,0 +1,7 @@
+models:
+  - name: base_opralogweb__logbooks
+    description: Opralogweb logbooks table with basic cleaning
+    columns:
+      - name: logbook_id
+        tests:
+          - not_null

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.sql

@@ -0,0 +1,22 @@
+with
+
+source as (
+  select * from {{ source('accelerator_opralogweb', 'MoreEntryColumns') }}
+),
+
+renamed as (
+
+  select
+
+    EntryId as entry_id,
+    {{ normalize_whitespace('ColData') }} as string_data,
+    NumberValue as number_data,
+    AdditionalColumnId as additional_column_id
+
+  from
+
+    source
+
+)
+
+select * from renamed

elt-pipelines/facility_ops/transform/models/staging/accelerator/base/base_opralogweb__more_entry_columns.yml

@@ -0,0 +1,7 @@
+models:
+  - name: base_opralogweb__more_entry_columns
+    description: Opralogweb more_entry_columns table with basic cleaning
+    columns:
+      - name: entry_id
+        tests:
+          - not_null

elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.sql

@@ -0,0 +1,97 @@
+{%- set OPRALOG_EPOCH = dbt.string_literal("2017-04-25") -%}
+{%- set MCR_LOGBOOK = dbt.string_literal("MCR Running Log") -%}
+
+with
+
+staging_entries as ( select * from {{ ref('base_opralogweb__entries') }} ),
+
+staging_chapter_entry as ( select * from {{ ref('base_opralogweb__chapter_entry') }} ),
+
+staging_logbook_chapter as ( select * from {{ ref('base_opralogweb__logbook_chapter') }} ),
+
+staging_logbooks as ( select * from {{ ref('base_opralogweb__logbooks') }} ),
+
+staging_more_entry_columns as ( select * from {{ ref('base_opralogweb__more_entry_columns') }} ),
+
+staging_additional_columns as ( select * from {{ ref('base_opralogweb__additional_columns') }} ),
+
+denormalized as (
+  select
+
+    staging_entries.entry_id,
+    staging_entries.fault_occurred_at,
+    staging_entries.fault_date,
+    staging_additional_columns.column_title,
+    staging_more_entry_columns.string_data,
+    staging_more_entry_columns.number_data,
+    staging_entries.fault_description
+
+  from
+    staging_entries
+    join staging_chapter_entry on staging_chapter_entry.entry_id = staging_entries.entry_id
+    join staging_logbook_chapter on staging_logbook_chapter.logbook_chapter_no = staging_chapter_entry.logbook_chapter_no
+    join staging_logbooks on staging_logbooks.logbook_id = staging_chapter_entry.logbook_id
+    left outer join staging_more_entry_columns on staging_more_entry_columns.entry_id = staging_entries.entry_id
+    left outer join staging_additional_columns on staging_additional_columns.additional_column_id = staging_more_entry_columns.additional_column_id
+
+  where
+
+    staging_entries.logically_deleted = false
+    and staging_entries.fault_date >= from_iso8601_date({{ OPRALOG_EPOCH }})
+    and staging_logbooks.logbook_name = {{ MCR_LOGBOOK }}
+    and staging_chapter_entry.logbook_id = staging_chapter_entry.principal_logbook
+    and staging_additional_columns.column_title in ('Equipment', 'Group', 'Lost Time', 'Group Leader comments')
+    and (
+      staging_more_entry_columns.string_data is not null
+      or staging_more_entry_columns.number_data is not null
+    )
+),
+
+mcr_equipment_downtime as (
+  select
+    *
+  from
+    (
+      select
+
+        min(
+          case
+            column_title
+            when 'Equipment' then string_data
+          end
+        ) as equipment,
+        min(
+          case
+            column_title
+            when 'Lost Time' then number_data
+          end
+        ) as downtime_mins,
+        fault_date,
+        fault_occurred_at,
+        min(
+          case
+            column_title
+            when 'Group' then string_data
+          end
+        ) as {{ adapter.quote('group') }},
+        fault_description,
+        min(
+          case
+            column_title
+            when 'Group Leader comments' then string_data
+          end
+        ) as managers_comments
+      from
+        denormalized
+      group by
+        fault_occurred_at,
+        fault_date,
+        fault_description
+    )
+  where
+    equipment is not null
+    and downtime_mins is not null
+    and {{ adapter.quote('group') }} is not null
+)
+
+select * from mcr_equipment_downtime

elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_opralogweb__mcr_equipment_downtime.yml

@@ -0,0 +1,41 @@
+models:
+  - name: stg_opralogweb_mcr_equipment_downtime
+    description: >
+      Equipment downtime records for the MCR assembled from the relational
+      database of the web-based Opralog application. Entries before cycle 17/01
+      are discarded as Opralog was in testing mode before this.
+
+unit_tests:
+  - name: test_stg_opralogweb__mcr_equipment_downtime
+    description: "Check null & before epoch entries are discarded"
+    model: stg_opralogweb__mcr_equipment_downtime
+    given:
+      - input: ref('base_opralogweb__entries')
+        format: sql
+        fixture: base_opralogweb__entries
+      - input: ref('base_opralogweb__chapter_entry')
+        format: sql
+        fixture: base_opralogweb__chapter_entry
+      - input: ref('base_opralogweb__logbook_chapter')
+        format: sql
+        fixture: base_opralogweb__logbook_chapter
+      - input: ref('base_opralogweb__logbooks')
+        format: sql
+        fixture: base_opralogweb__logbooks
+      - input: ref('base_opralogweb__more_entry_columns')
+        format: sql
+        fixture: base_opralogweb__more_entry_columns
+      - input: ref('base_opralogweb__additional_columns')
+        format: sql
+        fixture: base_opralogweb__additional_columns
+    expect:
+      rows:
+        - {
+            equipment: "Equipment 101",
+            downtime_mins: 5.3,
+            fault_date: "2024-01-01",
+            fault_occurred_at: "2024-01-01 00:00:01 UTC",
+            group: "Group 101",
+            fault_description: "Comment 101",
+            managers_comments: null,
+          }

elt-pipelines/pyproject.toml

@@ -9,6 +9,9 @@ dependencies = [
     "pydantic-settings>=2.14.2",
 ]
 
+[tool.setuptools]
+packages = ["facility_ops", "fase"]
+
 [project.optional-dependencies]
 proposal = [
     "sqlalchemy[postgresql-psycopgbinary]>=2.0.0",