SK-2954: common/v2/flowvault split + flowvault insert support

.codespellrc

@@ -3,7 +3,7 @@
 ignore-words-list = Skyflow,skyflow,skyflowapi,skyflowapis,deidentify,reidentify,detokenize,upsert,upserting,binlookup,byot,creds,fpe,devsecops,formdata,vaultid,dotenv,usecwd,runid,dateutil,Homogenous
 
 # Skip these files and folders
-skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,./skyflow/generated,setup.py
+skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,generated,setup.py
 
 # If you want to verify it is working, you can uncomment this line to see what files it checks
 # count = 

.github/workflows/beta-release.yml

@@ -4,15 +4,28 @@ on:
   push:
     tags: '*.*.*b*'
     paths-ignore:
-      - "setup.py"
+      - "*/setup.py"
       - "*.yml"
       - "*.md"
-      - "skyflow/utils/_version.py"
+      - "*/skyflow/utils/_version.py"
 
 jobs:
   build-and-deploy:
+    strategy:
+      matrix:
+        include:
+          - variant: v2
+            package-name: skyflow
+            tag-prefix: ''
+          - variant: flowvault
+            package-name: skyflow_flowvault
+            tag-prefix: 'flowvault-'
+    if: (matrix.variant == 'flowvault' && startsWith(github.ref_name, 'flowvault-')) || (matrix.variant == 'v2' && !startsWith(github.ref_name, 'flowvault-'))
     uses: ./.github/workflows/shared-build-and-deploy.yml
     with:
       ref: ${{ github.ref_name }}
       tag: 'beta'
+      variant: ${{ matrix.variant }}
+      package-name: ${{ matrix.package-name }}
+      tag-prefix: ${{ matrix.tag-prefix }}
     secrets: inherit

.github/workflows/ci.yml

@@ -18,7 +18,39 @@ jobs:
           error: 'One of your your commit messages is not matching the format with JIRA ID Ex: ( SDK-123 commit message )'
 
   test:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - variant: v2
+            package-name: skyflow
+            coverage-omit: "skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/detect/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py"
+          - variant: flowvault
+            package-name: skyflow_flowvault
+            coverage-omit: "skyflow_flowvault/generated/*"
     uses: ./.github/workflows/shared-tests.yml
     with:
       python-version: '3.9'
+      variant: ${{ matrix.variant }}
+      package-name: ${{ matrix.package-name }}
+      coverage-omit: ${{ matrix.coverage-omit }}
     secrets: inherit
+
+  test-common:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+      - run: pip install -e ./common
+      - run: pip install coverage
+      - run: python -m coverage run --source=common --omit="common/generated/*,common/tests/*" -m unittest discover -s common/tests -t .
+      - run: coverage xml -o test-coverage.xml
+      - name: Codecov
+        uses: codecov/codecov-action@v2.1.0
+        with:
+          token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
+          files: test-coverage.xml
+          name: codecov-skyflow-python-common
+          verbose: true

.github/workflows/internal-release.yml

@@ -5,19 +5,33 @@ on:
     tags-ignore:
       - '*.*'
     paths-ignore:
-      - "setup.py"
+      - "*/setup.py"
       - "*.yml"
       - "*.md"
-      - "skyflow/utils/_version.py"
+      - "*/skyflow/utils/_version.py"
       - "samples/**"
+      - "flowvault/samples/**"
     branches:
       - release/*
+      - flowvault-release/*
 
 jobs:
   build-and-deploy:
+    strategy:
+      matrix:
+        include:
+          - variant: v2
+            package-name: skyflow
+            tag-prefix: ''
+          - variant: flowvault
+            package-name: skyflow_flowvault
+            tag-prefix: 'flowvault-'
+    if: (matrix.variant == 'flowvault' && startsWith(github.ref_name, 'flowvault-')) || (matrix.variant == 'v2' && !startsWith(github.ref_name, 'flowvault-'))
     uses: ./.github/workflows/shared-build-and-deploy.yml
     with:
       ref: ${{ github.ref_name }}
       tag: 'internal'
+      variant: ${{ matrix.variant }}
+      package-name: ${{ matrix.package-name }}
+      tag-prefix: ${{ matrix.tag-prefix }}
     secrets: inherit
-

.github/workflows/main.yml

@@ -7,7 +7,39 @@ on:
 
 jobs:
   test:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - variant: v2
+            package-name: skyflow
+            coverage-omit: "skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/detect/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py"
+          - variant: flowvault
+            package-name: skyflow_flowvault
+            coverage-omit: "skyflow_flowvault/generated/*"
     uses: ./.github/workflows/shared-tests.yml
     with:
       python-version: '3.9'
+      variant: ${{ matrix.variant }}
+      package-name: ${{ matrix.package-name }}
+      coverage-omit: ${{ matrix.coverage-omit }}
     secrets: inherit
+
+  test-common:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+      - run: pip install -e ./common
+      - run: pip install coverage
+      - run: python -m coverage run --source=common --omit="common/generated/*,common/tests/*" -m unittest discover -s common/tests -t .
+      - run: coverage xml -o test-coverage.xml
+      - name: Codecov
+        uses: codecov/codecov-action@v2.1.0
+        with:
+          token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
+          files: test-coverage.xml
+          name: codecov-skyflow-python-common
+          verbose: true

.github/workflows/release.yml

@@ -4,15 +4,28 @@ on:
   push:
     tags: "*.*.*"
     paths-ignore:
-      - "setup.py"
+      - "*/setup.py"
       - "*.yml"
       - "*.md"
-      - "skyflow/utils/_version.py"
+      - "*/skyflow/utils/_version.py"
 
 jobs:
   build-and-deploy:
+    strategy:
+      matrix:
+        include:
+          - variant: v2
+            package-name: skyflow
+            tag-prefix: ''
+          - variant: flowvault
+            package-name: skyflow_flowvault
+            tag-prefix: 'flowvault-'
+    if: (matrix.variant == 'flowvault' && startsWith(github.ref_name, 'flowvault-')) || (matrix.variant == 'v2' && !startsWith(github.ref_name, 'flowvault-'))
     uses: ./.github/workflows/shared-build-and-deploy.yml
     with:
       ref: main
       tag: 'public'
+      variant: ${{ matrix.variant }}
+      package-name: ${{ matrix.package-name }}
+      tag-prefix: ${{ matrix.tag-prefix }}
     secrets: inherit

.github/workflows/semgrep.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Run Semgrep
         run: |
-          semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --sarif . > results.sarif
+          semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --exclude "**/generated/**" --sarif . > results.sarif
 
       - name: Upload SARIF file
         uses: github/codeql-action/upload-sarif@v3

.github/workflows/shared-build-and-deploy.yml

@@ -7,12 +7,29 @@ on:
         description: 'Git reference to use (e.g., main or branch name)'
         required: true
         type: string
-      
+
       tag:
         description: 'Release Tag'
         required: true
         type: string
 
+      variant:
+        description: 'Build variant directory to release (v2 or flowvault)'
+        required: true
+        type: string
+
+      package-name:
+        description: 'Importable package name for this variant (e.g. skyflow or skyflow_flowvault)'
+        required: false
+        type: string
+        default: 'skyflow'
+
+      tag-prefix:
+        description: 'Prefix distinguishing this variant''s git tags from other variants'' (e.g. "flowvault-" for flowvault, empty for v2)'
+        required: false
+        type: string
+        default: ''
+
 jobs:
   build-and-deploy:
     runs-on: ubuntu-latest
@@ -29,7 +46,7 @@ jobs:
 
       - name: Resolve Branch for the Tagged Commit
         id: resolve-branch
-        if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }} 
+        if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }}
         run: |
           TAG_COMMIT=$(git rev-list -n 1 ${{ github.ref_name }})
 
@@ -47,18 +64,31 @@ jobs:
         id: previoustag
         uses: WyriHaximus/github-action-get-previous-tag@v1
         with:
-          fallback: 1.0.0
+          fallback: ${{ inputs.tag-prefix }}1.0.0
+          pattern: ${{ inputs.tag-prefix }}[0-9]*.[0-9]*.[0-9]*
+
+      - name: Resolve version number
+        id: version
+        env:
+          PREVIOUS_TAG: ${{ steps.previoustag.outputs.tag }}
+          TAG_PREFIX: ${{ inputs.tag-prefix }}
+        run: |
+          TAG="$PREVIOUS_TAG"
+          VERSION="${TAG#$TAG_PREFIX}"
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
 
       - name: Bump Version
+        working-directory: ${{ inputs.variant }}
         run: |
-          chmod +x ./ci-scripts/bump_version.sh
+          chmod +x ../ci-scripts/bump_version.sh
           if ${{ inputs.tag == 'internal' }}; then
-            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" "$(git rev-parse --short "$GITHUB_SHA")"
+            ../ci-scripts/bump_version.sh "${{ steps.version.outputs.version }}" "$(git rev-parse --short "$GITHUB_SHA")" "${{ inputs.package-name }}"
           else
-            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}"
+            ../ci-scripts/bump_version.sh "${{ steps.version.outputs.version }}" "" "${{ inputs.package-name }}"
           fi
 
       - name: Commit changes
+        working-directory: ${{ inputs.variant }}
         run: |
           git config user.name "${{ github.actor }}"
           git config user.email "${{ github.actor }}@users.noreply.github.com"
@@ -68,27 +98,29 @@ jobs:
           fi
 
           git add setup.py
-          git add skyflow/utils/_version.py
+          git add ${{ inputs.package-name }}/utils/_version.py
 
           if [[ "${{ inputs.tag }}" == "internal" ]]; then
-            VERSION="${{ steps.previoustag.outputs.tag }}.dev0+$(git rev-parse --short $GITHUB_SHA)"
-            COMMIT_MESSAGE="[AUTOMATED] Private Release $VERSION"
+            VERSION="${{ steps.version.outputs.version }}.dev0+$(git rev-parse --short $GITHUB_SHA)"
+            COMMIT_MESSAGE="[AUTOMATED] Private Release (${{ inputs.variant }}) $VERSION"
             git commit -m "$COMMIT_MESSAGE"
             git push origin ${{ github.ref_name }} -f
           fi
           if [[ "${{ inputs.tag }}" == "beta" || "${{ inputs.tag }}" == "public" ]]; then
-            COMMIT_MESSAGE="[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}"
+            COMMIT_MESSAGE="[AUTOMATED] Public Release (${{ inputs.variant }}) - ${{ steps.previoustag.outputs.tag }}"
             git commit -m "$COMMIT_MESSAGE"
             git push origin ${{ env.branch_name }}
           fi
 
-      - name: Build and install skyflow package
+      - name: Build and install package
+        working-directory: ${{ inputs.variant }}
         run: |
           python setup.py sdist bdist_wheel
-          pip install dist/skyflow-*.whl
+          pip install dist/*.whl
 
       - name: Build and Publish Package
-        if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }} 
+        if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }}
+        working-directory: ${{ inputs.variant }}
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_TOKEN }}
@@ -98,9 +130,10 @@ jobs:
 
       - name: Build and Publish to JFrog Artifactory
         if: ${{ inputs.tag == 'internal' }}
+        working-directory: ${{ inputs.variant }}
         env:
           TWINE_USERNAME: ${{ secrets.JFROG_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.JFROG_PASSWORD }}
         run: |
           python setup.py sdist bdist_wheel
-          twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/*
+          twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/*

.github/workflows/shared-tests.yml

@@ -7,6 +7,20 @@ on:
         description: 'Python version to use'
         required: true
         type: string
+      variant:
+        description: 'Build variant directory to test (v2 or flowvault)'
+        required: true
+        type: string
+      package-name:
+        description: 'Importable package name for this variant (e.g. skyflow or skyflow_flowvault)'
+        required: false
+        type: string
+        default: 'skyflow'
+      coverage-omit:
+        description: 'Comma-separated coverage --omit patterns, relative to the variant directory'
+        required: false
+        type: string
+        default: 'skyflow/generated/*'
 
 jobs:
   run-tests:
@@ -23,32 +37,36 @@ jobs:
         with:
           name: "credentials.json"
           json: ${{ secrets.VALID_SKYFLOW_CREDS_TEST }}
+          dir: ${{ inputs.variant }}
 
-      - name: Build and install skyflow package
+      - name: Build and install package
+        working-directory: ${{ inputs.variant }}
         run: |
           pip install --upgrade pip setuptools wheel
           python setup.py sdist bdist_wheel
-          pip install dist/skyflow-*.whl
+          pip install dist/*.whl
           pip install ".[dev]"
 
       - name: Run Spell Check
         run: codespell
 
       - name: Run Linter Ruff
         run: ruff check . --output-format=github
-      
+
       - name: 'Run Tests'
+        working-directory: ${{ inputs.variant }}
         run: |
           pip install -r requirements.txt
-          python -m coverage run --source=skyflow --omit=skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/detect/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py -m unittest discover
+          python -m coverage run --source=${{ inputs.package-name }} --omit=${{ inputs.coverage-omit }} -m unittest discover
 
       - name: coverage
+        working-directory: ${{ inputs.variant }}
         run: coverage xml -o test-coverage.xml
 
       - name: Codecov
         uses: codecov/codecov-action@v2.1.0
         with:
           token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
-          files: test-coverage.xml
-          name: codecov-skyflow-python
+          files: ${{ inputs.variant }}/test-coverage.xml
+          name: codecov-skyflow-python-${{ inputs.variant }}
           verbose: true

.gitignore

@@ -104,6 +104,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
+.venv-*
 env/
 venv/
 ENV/