feat(elt-pipelines) - Add transform method to get times spent in status per issue

elt-common/pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "pydantic>=2.13.4",
   "dbt-core>=1.12.0",
   "dbt-trino>=1.10.3",
+  "python-dotenv>=1.2.2",
 ]
 
 

elt-common/src/elt_common/ingest.py

@@ -10,6 +10,7 @@
 
 import pyarrow as pa
 import pyarrow.compute as pc
+from dotenv import load_dotenv, find_dotenv
 from pyiceberg.exceptions import NoSuchTableError
 
 from elt_common.extract import (
@@ -29,6 +30,8 @@
 def run_ingest(job: ELTIngestManifest) -> dict[str, int]:
     """Import the extract function, call it, and write results to Iceberg."""
 
+    load_dotenv(find_dotenv(usecwd=True), verbose=True)
+
     # Create the object that will do the extraction.
     # Environment variables for the object's configuration must have been set
     # before reaching here.

elt-pipelines/README.md

@@ -39,10 +39,12 @@ Pipelines are run using the `elt` CLI tool. As an example, with `elt-pipelines`
   connection, so configuration follows [their approach](https://py.iceberg.apache.org/configuration/). See
   the [getting started guide](../docs-devel/getting-started.md#configure-iceberg-connection) for the local configuration
   values
-- Any pipelines that include a `config_cls` require/have optional configuration values to be set
+- Any pipelines that include a `config_cls` require/have optional configuration values
    - These use [`pydantic_settings`](https://pydantic.dev/docs/validation/latest/concepts/pydantic_settings/)
    - Environment variables are used to set the values. The name of the variable(s) must be prefixed with
      `<JOB_NAME>__`, where `<JOB_NAME>` is the name of the pipeline
+   - Values for variables can also be specified in a `.env` file in the working directory. If a value is present both
+     in the `.env` file and as an environment variable, the environment variable is preferred
 
 ## Writing a pipeline
 

elt-pipelines/facility_ops/ingest/computing/jira/jira.py

@@ -64,7 +64,7 @@ def extract_resource_properties(self) -> Iterator[tuple[str, ResourceProperties]
             ),
         )
         yield (
-            "issue_status_changelog",
+            "issue_status_changelogs",
             ResourceProperties(
                 extractor=self.extract_issue_status_changelogs,
                 write_properties=ResourceWriteProperties(write_mode="replace"),

elt-pipelines/facility_ops/transform/models/intermediate/computing/int_times_in_status.sql

@@ -0,0 +1,43 @@
+-- Get the differences between the status. No need for final status, so this query is perfectly suitable
+with status_to_from as (
+  select
+    issue_key,
+    from_status as status,
+    lag(changed_at) over (partition by issue_key order by changed_at) as status_from,
+    changed_at as status_to
+    from {{ ref('stg_jira_issue_status_changelogs') }} as changelogs
+),
+
+-- Populate null values of status from with the issue creation date. Join required
+nn_status_to_from as (
+  select
+    status_to_from.issue_key,
+    status_to_from.status,
+    COALESCE(status_to_from.status_from, issues.created) as status_from,
+    status_to_from.status_to
+    from status_to_from
+    inner join facility_ops_landing.computing_jira.isis_jira_issues as issues
+    on status_to_from.issue_key = issues.issue_key
+),
+
+-- Subtract to and from date
+status_durations as (
+  select
+    issue_key,
+    status,
+    date_diff('second', status_from, status_to) as status_duration
+    from nn_status_to_from
+),
+
+-- Aggregate similar statuses and add their durations
+times_in_status as (
+  select
+    issue_key,
+    status,
+    sum(status_duration) as time_in_status
+    from status_durations
+    group by issue_key,
+    status
+)
+
+select * from times_in_status

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.sql

@@ -0,0 +1,84 @@
+{{
+  config(
+    on_table_exists = 'drop'
+)
+}}
+
+with times_in_status as (
+  select * from {{ ref('int_times_in_status') }} where issue_key like 'CI-%'
+),
+
+time_in_status_data_driven_facility as (
+    select
+    issue_key,
+    MAX(
+        case
+        when status = 'analyzing' then time_in_status
+        else null
+        end
+    ) as time_in_analysing_secs,
+    MAX(
+        case
+        when status = 'backlog' then time_in_status
+        else null
+        end
+    ) as time_in_backlog_secs,
+    MAX(
+        case
+        when status = 'done' then time_in_status
+        else null
+        end
+    ) as time_in_done_secs,
+    MAX(
+        case
+        when status = 'funnel' then time_in_status
+        else null
+        end
+    ) as time_in_funnel_secs,
+    MAX(
+        case
+        when status = 'implementing' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_secs,
+    MAX(
+        case
+        when status = 'implementing (mvp)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_mvp_secs,
+    MAX(
+        case
+        when status = 'implementing (persevere)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_persevere_secs,
+    MAX(
+        case
+        when status = 'in progress' then time_in_status
+        else null
+        end
+    ) as time_in_in_progress_secs,
+    MAX(
+        case
+        when status = 'portfolio backlog' then time_in_status
+        else null
+        end
+    ) as time_in_portfolio_backlog_secs,
+    MAX(
+        case
+        when status = 'ready' then time_in_status
+        else null
+        end
+    ) as time_in_ready_secs,
+    MAX(
+        case
+        when status = 'reviewing' then time_in_status
+        else null
+        end
+    ) as time_in_reviewing_secs
+    from times_in_status
+    group by issue_key
+
+)
+select * from time_in_status_data_driven_facility

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.yml

@@ -0,0 +1,19 @@
+models:
+  - name: time_in_status_computing_infrastructure
+    description: >
+      Get length of time a Computing Infrastructure issue spends in each status for user software issues.
+    columns:
+      - name: issue_key
+        data_tests:
+        - not_null
+      - name: time_in_analysing_secs
+      - name: time_in_backlog_secs
+      - name: time_in_done_secs
+      - name: time_in_funnel_secs
+      - name: time_in_implementing_secs
+      - name: time_in_implementing_mvp_secs
+      - name: time_in_implementing_persevere_secs
+      - name: time_in_in_progress_secs
+      - name: time_in_portfolio_backlog_secs
+      - name: time_in_ready_secs
+      - name: time_in_reviewing_secs

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.sql

@@ -0,0 +1,90 @@
+{{
+  config(
+    on_table_exists = 'drop'
+)
+}}
+
+with times_in_status as (
+  select * from {{ ref('int_times_in_status') }} where issue_key like 'DD-%'
+),
+
+time_in_status_data_driven_facility as (
+    select
+    issue_key,
+    MAX(
+        case
+        when status = 'analyzing' then time_in_status
+        else null
+        end
+    ) as time_in_analysing_secs,
+    MAX(
+        case
+        when status = 'backlog' then time_in_status
+        else null
+        end
+    ) as time_in_backlog_secs,
+    MAX(
+        case
+        when status = 'done' then time_in_status
+        else null
+        end
+    ) as time_in_done_secs,
+    MAX(
+        case
+        when status = 'funnel' then time_in_status
+        else null
+        end
+    ) as time_in_funnel_secs,
+    MAX(
+        case
+        when status = 'implementing' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_secs,
+    MAX(
+        case
+        when status = 'implementing (mvp)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_mvp_secs,
+    MAX(
+        case
+        when status = 'implementing (persevere)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_persevere_secs,
+    MAX(
+        case
+        when status = 'in progress' then time_in_status
+        else null
+        end
+    ) as time_in_in_progress_secs,
+    MAX(
+        case
+        when status = 'portfolio backlog' then time_in_status
+        else null
+        end
+    ) as time_in_portfolio_backlog_secs,
+    MAX(
+        case
+        when status = 'ready' then time_in_status
+        else null
+        end
+    ) as time_in_ready_secs,
+    MAX(
+        case
+        when status = 'reviewing' then time_in_status
+        else null
+        end
+    ) as time_in_reviewing_secs,
+    MAX(
+        case
+        when status = 'selected for development' then time_in_status
+        else null
+        end
+    ) as time_in_selected_for_development_secs
+    from times_in_status
+    group by issue_key
+
+)
+select * from time_in_status_data_driven_facility

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.yml

@@ -0,0 +1,20 @@
+models:
+  - name: time_in_status_data_driven_facility
+    description: >
+      Get length of time a Data-Driven Facility issue spends in each status for user software issues.
+    columns:
+      - name: issue_key
+        data_tests:
+        - not_null
+      - name: time_in_analysing_secs
+      - name: time_in_backlog_secs
+      - name: time_in_done_secs
+      - name: time_in_funnel_secs
+      - name: time_in_implementing_secs
+      - name: time_in_implementing_mvp_secs
+      - name: time_in_implementing_persevere_secs
+      - name: time_in_in_progress_secs
+      - name: time_in_portfolio_backlog_secs
+      - name: time_in_ready_secs
+      - name: time_in_reviewing_secs
+      - name: time_in_selected_for_development_secs