Upgrade/v2.17 gated 20260810

.github/configs/ascend.yml

@@ -0,0 +1,77 @@
+# Huawei Ascend NPU configuration for TransformerEngine-FL
+# This file follows the same schema as cuda.yml and metax.yml.
+
+hardware_name: ascend
+display_name: 'Huawei Ascend NPU'
+checkout_submodules: recursive
+
+# CI image for the Ascend environment
+ci_image: harbor.baai.ac.cn/flagos-dev/transformerengine-fl:85c2523-ascend-dev
+container_pull_policy: always
+
+# Runner labels for the self-hosted Ascend node
+runner_labels:
+  - hw-4g-cicd-te
+
+# Container volumes
+container_volumes:
+  - /usr/local/Ascend/driver:/usr/local/Ascend/driver
+  - /usr/local/Ascend/add-ons:/usr/local/Ascend/add-ons
+
+# Container options
+container_options: >-
+  --privileged
+  --shm-size=100g
+  --ipc=host
+  --ulimit memlock=-1
+  --ulimit stack=67108864
+  --user root
+  --device=/dev/davinci0
+  --device=/dev/davinci1
+  --device=/dev/davinci2
+  --device=/dev/davinci3
+  --device=/dev/davinci_manager
+  --device=/dev/devmm_svm
+  --device=/dev/hisi_hdc
+
+# Platform-specific environment setup script
+setup_script: .github/scripts/setup_ascend.sh
+
+coverage:
+  enabled: true
+  required: true
+  python: python3
+  sources:
+    - transformer_engine
+  include:
+    - transformer_engine/pytorch/*
+    - transformer_engine/debug/*
+    - transformer_engine/plugin/*
+  omit:
+    - '*/setup.py'
+    - '*/transformer_engine/plugin/core/_build_config.py'
+
+unit_test_matrix:
+  - name: pytorch_debug
+    runner: script
+    path: qa/L0_pytorch_debug_unittest/test_ascend.sh
+
+  - name: pytorch_unittest
+    runner: script
+    path: qa/L0_pytorch_unittest/test_ascend.sh
+
+  - name: pytorch_distributed_unittest
+    runner: script
+    path: qa/L1_pytorch_distributed_unittest/test_ascend.sh
+
+  - name: pytorch_onnx_unittest
+    runner: script
+    path: qa/L1_pytorch_onnx_unittest/test_ascend.sh
+
+integration_test_matrix:
+  - name: pytorch_mcore_integration
+    path: qa/L1_pytorch_mcore_integration/test.sh
+
+# Device types to run tests on
+device_types:
+  - ascend

.github/configs/cuda.yml

@@ -0,0 +1,92 @@
+# CUDA Hardware Configuration for TransformerEngine-FL
+# Refactored for A100 Nodes
+# This file defines environment variables, volumes, and test filters for TE tests.
+
+hardware_name: cuda
+display_name: 'NVIDIA CUDA (A100)'
+checkout_submodules: recursive
+
+# CI image for online env
+ci_image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209
+
+# Runner labels for self-hosted A100 node
+runner_labels:
+  - self-hosted
+  - Linux
+  - X64
+  - nvidia
+  - gpu-8
+
+# Runner labels for online env
+runner_labels:
+  - nv-8g-cicd-te
+
+# Container volumes
+container_volumes:
+  - /home/flagscale_cicd/flask/static:/workspace/report
+
+# Container options
+container_options: >-
+  --privileged 
+  --gpus all 
+  --shm-size=500g 
+  --ipc=host 
+  --ulimit memlock=-1 
+  --ulimit stack=67108864 
+  --user root
+
+# Platform-specific environment setup script
+setup_script: .github/scripts/setup_cuda.sh
+
+# Device types to run tests on
+device_types:
+  - a100
+
+coverage:
+  enabled: true
+  required: false
+  python: /opt/miniconda3/envs/flagscale-train/bin/python3
+  sources:
+    - transformer_engine
+  include:
+    - transformer_engine/pytorch/*
+    - transformer_engine/debug/*
+    - transformer_engine/plugin/*
+  omit:
+    - '*/setup.py'
+    - '*/transformer_engine/plugin/core/_build_config.py'
+
+unit_test_matrix:
+  - name: pytorch_debug
+    runner: script
+    path: qa/L0_pytorch_debug_unittest/test.sh
+  - name: pytorch_unittest
+    runner: script
+    path: qa/L0_pytorch_unittest/test.sh
+  - name: pytorch_distributed_unittest
+    runner: script
+    path: qa/L1_pytorch_distributed_unittest/test.sh
+  - name: pytorch_onnx_unittest
+    runner: script
+    path: qa/L1_pytorch_onnx_unittest/test.sh
+
+integration_test_matrix:
+  - name: pytorch_mcore_integration
+    path: qa/L1_pytorch_mcore_integration/test.sh
+
+# Test matrix configuration
+test_matrix:
+  l0_pytorch:
+    path: 'qa/L0_pytorch_unittest/test.sh'
+    ignored_tests:
+      - test_sanity_layernorm_mlp
+      - test_sanity_gpt
+      - test_sanity_bert
+      - test_sanity_T5
+      - test_sanity_amp_and_nvfuser
+      - test_sanity_drop_path
+      - test_layernorm_mlp_accuracy
+      - test_grouped_linear_accuracy
+      - test_gpt_accuracy
+      - test_basic_linear
+      - test_layer_norm

.github/configs/hygon.yml

@@ -0,0 +1,76 @@
+# Hygon DCU / DTK configuration for TransformerEngine-FL plugin QA.
+
+hardware_name: hygon
+display_name: 'Hygon DCU (DTK)'
+checkout_submodules: 'false'
+
+ci_image: harbor.baai.ac.cn/flagos-dev/transformerengine-fl:manual-20260717-hygon-dev
+
+runner_labels:
+  - hg-8g-cicd-te
+
+container_volumes:
+  - /opt/hyhal:/opt/hyhal
+
+container_options: >-
+  --privileged
+  --ipc=host
+  --shm-size=100g
+  --ulimit memlock=-1
+  --ulimit stack=67108864
+  --user root
+  --device=/dev/kfd
+  --device=/dev/dri
+  --group-add video
+
+setup_script: .github/scripts/setup_hygon.sh
+
+coverage:
+  enabled: true
+  required: false
+  python: python3
+  sources:
+    - transformer_engine
+  include:
+    - transformer_engine/pytorch/*
+    - transformer_engine/debug/*
+    - transformer_engine/plugin/*
+  omit:
+    - '*/setup.py'
+    - '*/transformer_engine/plugin/core/_build_config.py'
+
+unit_test_matrix:
+  - name: pytorch_debug
+    runner: script
+    path: tests/plugin/backend/hygon/run_native.sh
+    args: [debug]
+    env:
+      XML_LOG_DIR: logs/L0_pytorch_debug_unittest-hygon
+
+  - name: pytorch_unittest
+    runner: script
+    path: tests/plugin/backend/hygon/run_native.sh
+    args: [unittest]
+    env:
+      XML_LOG_DIR: logs/L0_pytorch_unittest-hygon
+
+  - name: pytorch_distributed_unittest
+    runner: script
+    path: tests/plugin/backend/hygon/run_native.sh
+    args: [distributed]
+    env:
+      XML_LOG_DIR: logs/L1_pytorch_distributed_unittest-hygon
+
+  - name: pytorch_onnx_unittest
+    runner: script
+    path: tests/plugin/backend/hygon/run_native.sh
+    args: [onnx]
+    env:
+      XML_LOG_DIR: logs/L1_pytorch_onnx_unittest-hygon
+
+integration_test_matrix:
+  - name: pytorch_mcore_integration
+    path: tests/plugin/backend/hygon/run_integration.sh
+
+device_types:
+  - bw1000

.github/configs/metax.yml

@@ -0,0 +1,94 @@
+# Metax Hardware Configuration for TE-FL
+# This file defines CI/CD settings for Metax-based testing
+# This file defines environment variables, volumes, and test filters for TE tests.
+
+hardware_name: metax
+display_name: 'Metax Tests'
+checkout_submodules: recursive
+
+# CI image for Metax dev env
+# ci_image: localhost:5000/megatron-lm-with-te:v1
+
+# CI image for online env
+ci_image: harbor.baai.ac.cn/flagscale/megatron-lm-with-te:202603231839
+
+# Runner labels for self-hosted Metax node
+# runner_labels:
+#   - self-hosted
+#   - Linux
+#   - X64
+#   - metax
+#   - dev
+
+# Runner labels for online env
+runner_labels:
+  - mx-8g-cicd-te
+
+# Container volumes
+container_volumes:
+  - /nfs/metax_fs:/nfs/metax_fs
+
+# Container options
+container_options: >-
+  --hostname=te_cicd
+  --ipc=host
+  --privileged=true
+  --shm-size=100gb
+  --ulimit memlock=-1
+  --user root
+  --ulimit nofile=65535:65535
+  --group-add video
+  -e PLATFORM=metax
+  -e TORCH_DISTRIBUTED_BACKEND=mccl
+  -e LD_LIBRARY_PATH=/opt/maca/lib:/usr/local/lib:$LD_LIBRARY_PATH
+
+# Platform-specific environment setup script
+setup_script: .github/scripts/setup_metax.sh
+
+# Device types to run tests on
+device_types:
+  - c500
+
+coverage:
+  enabled: true
+  required: false
+  python: /opt/conda/bin/python3
+  sources:
+    - transformer_engine
+  include:
+    - transformer_engine/pytorch/*
+    - transformer_engine/debug/*
+    - transformer_engine/plugin/*
+  omit:
+    - '*/setup.py'
+    - '*/transformer_engine/plugin/core/_build_config.py'
+
+unit_test_matrix:
+  - name: pytorch_debug
+    runner: script
+    path: qa/L0_pytorch_debug_unittest/test.sh
+  - name: pytorch_unittest
+    runner: script
+    path: qa/L0_pytorch_unittest/test.sh
+  - name: pytorch_distributed_unittest
+    runner: script
+    path: qa/L1_pytorch_distributed_unittest/test.sh
+  - name: pytorch_onnx_unittest
+    runner: script
+    path: qa/L1_pytorch_onnx_unittest/test.sh
+
+integration_test_matrix:
+  - name: pytorch_mcore_integration
+    path: qa/L1_pytorch_mcore_integration/test.sh
+
+# Test matrix configuration
+test_matrix:
+  unit:
+    devices:
+      - c500
+    # Ignored test files for unit tests
+    # These files will be skipped when running pytest
+    ignored_tests:
+      # example: tests/unit_tests/test_example.py
+      # - tests/unit_tests/test_inference.py
+      # - tests/unit_tests/test_rl_utils.py