Prefect

.dockerignore

@@ -1,26 +1,34 @@
+.git/
 .github/
-.venv/
 .vscode/
 .idea/
+
+.venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+.coverage.*
+htmlcov/
+test-results.xml
+
 ci/
 doc/
-gurobi/
-htmlcov/
 local_test/
-*.egg-info/
-unit_test/
+tests/
+gurobi/
+
 .env
-.env.local
-.env.template
-.gitignore
-.taplo.toml
+.env.*
+
 CHANGELOG.md
 CONTRIBUTING.md
-dev.Dockerfile
-docker-compose.worker-only.yml
-docker-compose.yml
 LICENSE
 run.bat
 run.sh
 run_windows.sh
-test-results.xml
+dev.Dockerfile

.env.template

@@ -1,22 +1,34 @@
-ESDL_OUTPUT_PROFILES_TYPE=POSTGRESQL
+LOG_LEVEL=INFO
 
-DB_HOSTNAME=omotes_postgres
+ESDL_OUTPUT_PROFILES_TYPE=POSTGRESQL
+DB_HOSTNAME=omotes-postgres
 DB_PORT=6432
 DB_USERNAME=omotes_timeseries_rw
 DB_PASSWORD=somepass4
 
-RABBITMQ_USERNAME=celery
-RABBITMQ_PASSWORD=somepass2
-RABBITMQ_HOSTNAME=localhost
-RABBITMQ_PORT=5672
-RABBITMQ_VIRTUALHOST=omotes_celery
+# ESDL_OUTPUT_PROFILES_TYPE=INFLUXDB
+# DB_HOSTNAME=omotes-influxdb
+# DB_PORT=8096
+# DB_USERNAME=root
+# DB_PASSWORD=9012
 
-# May be grow_optimizer_default, grow_optimizer_no_heat_losses, grow_optimizer_with_pressure or grow_simulator
-GROW_TASK_TYPE=grow_optimizer_default
-LOG_LEVEL=INFO
+MINIO_HOST=omotes-minio
+MINIO_PORT=9000
+MINIO_EXTERNAL_HOST=omotes-minio
+MINIO_ACCESS_KEY=omotesadmin
+MINIO_SECRET=password
+
+PREFECT_API_URL=http://localhost:4200/api
+PREFECT_API_URL_FOR_WORKER=http://omotes-prefect-server:4200/api
+PREFECT_WORK_POOL_NAME="docker-worker"
+PREFECT_API_AUTH_STRING=admin:password
 
-TASK_RESULT_QUEUE_NAME=omotes_task_result_events
-TASK_PROGRESS_QUEUE_NAME=omotes_task_progress_events
+PREFECT_FLOW_MAX_CONCURRENT_RUNS=4
+PREFECT_FLOW_TIMEOUT_SECONDS=172800
 
-# MAX_TASKS_BEFORE_RESTART: 0 means no scheduled restarts
-MAX_TASKS_BEFORE_RESTART=0
+# For deploying to prefect (will run inside local docker container or k8s pod)
+# When using docker desktop you can use your local code without pushing to github first:
+PREFECT_USE_LOCAL_CODE_AND_IMAGE=true
+PREFECT_USE_LOCAL_SDK_AND_MESIDO=true
+# OPTIMIZER_WORKER_VERSION not used when PREFECT_USE_LOCAL_CODE_AND_IMAGE=true:
+OPTIMIZER_WORKER_VERSION=0.0.0

.github/workflows/ci.yml

@@ -1,50 +1,64 @@
-name: Build-Test-Lint-Typecheck
+name: Build-Test-Lint-Security-Typecheck
 
 on: [push]
 
 jobs:
   lint:
     name: Lint
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ "3.10" ]
     steps:
-      - uses: actions/checkout@v3
-      - name: setup python
+      - uses: actions/checkout@v4
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: pip
-      - name: create and activate venv
-        run: |
-          ./ci/linux/create_venv.sh
-          ./ci/linux/install_dependencies.sh
-      - name: run lint
-        run: |
-          ./ci/linux/lint.sh
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run linter
+        run: just lint
+      - name: Check formatting
+        run: just format-check
+
+  security:
+    name: Security Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run security checks
+        run: just security
 
   test:
-    name: Unit Tests
+    name: Test
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ "3.10" ]
     steps:
-      - uses: actions/checkout@v3
-      - name: setup python
+      - uses: actions/checkout@v4
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: pip
-      - name: create and activate venv
-        run: |
-          ./ci/linux/create_venv.sh
-          ./ci/linux/install_dependencies.sh
-      - name: run unit tests
-        run: |
-          ./ci/linux/test_unit.sh
-      - name: surface failing tests
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run unit tests
+        run: just test
+      - name: Publish test results
         if: always()
         uses: pmeier/pytest-results-action@main
         with:
@@ -54,22 +68,19 @@ jobs:
           fail-on-empty: true
 
   typecheck:
-    name: Typecheck
+    name: Type Check
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ "3.10" ]
     steps:
-      - uses: actions/checkout@v3
-      - name: setup python
+      - uses: actions/checkout@v4
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: pip
-      - name: create and activate venv
-        run: |
-          ./ci/linux/create_venv.sh
-          ./ci/linux/install_dependencies.sh
-      - name: run typechecker
-        run: |
-          ./ci/linux/typecheck.sh
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run type checker
+        run: just typecheck

.github/workflows/publish_container_image.yml

@@ -1,28 +1,48 @@
-name: Deply to GitHub Container Registry
+name: Publish Container Image
+run-name: Releasing next version 🚀
+
 on:
   push:
     tags:
-      - '*'
+      - "*"
 
 jobs:
- publish-container-image:
-   runs-on: ubuntu-latest
-   steps:
-   - name: Checkout sources
-     uses: actions/checkout@v3
-     with:
-      fetch-depth: 0       
+  publish-container-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-   - name: Login to GitHub Container Registry
-     uses: docker/login-action@v1
-     with:
-       registry: ghcr.io
-       username: ${{ github.actor }}
-       password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/project-omotes/omotes-optimizer
+          tags: |
+            type=raw,value=latest
+            type=raw,value=${{ github.ref_name }}
 
-   - name: Build the container image
-     run: |
-      docker build . --tag ghcr.io/project-omotes/omotes-grow-optimizer-worker:latest --tag ghcr.io/project-omotes/omotes-grow-optimizer-worker:${{ github.ref_name}}
-      docker push ghcr.io/project-omotes/omotes-grow-optimizer-worker:latest
-      docker push ghcr.io/project-omotes/omotes-grow-optimizer-worker:${{ github.ref_name}}
+      - name: Build and push container image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max