1.5.0 Release Candidate

CMakeLists.txt

@@ -7,7 +7,7 @@ set(PROJECT_NAME entity)
 
 project(
   ${PROJECT_NAME}
-  VERSION 1.4.4
+  VERSION 1.5.0
   LANGUAGES CXX C)
 add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
 set(hash_cmd "git diff --quiet src/ && echo $(git rev-parse HEAD) ")
@@ -58,6 +58,28 @@ set(gpu_aware_mpi
     ${default_gpu_aware_mpi}
     CACHE BOOL "Enable GPU-aware MPI")
 
+set(team_policy
+    ${default_team_policy}
+    CACHE BOOL "Enable team_policy tile-blocked deposit/pusher kernels")
+set(team_policy_tile_size
+    ${default_team_policy_tile_size}
+    CACHE STRING "team_policy tile edge length in cells")
+set(team_policy_tile_sizes
+    "4;6;8;10;12;14;16"
+    CACHE STRING "team_policy tile-size choices")
+set(team_policy_drift
+    ${default_team_policy_drift}
+    CACHE
+      STRING
+      "team_policy tiled-deposit scratch halo drift in cells (max cells a particle may move between two sorts). Sizes the deposit scratch halo only; the sort cadence is set at runtime via spatial_sorting_interval. Default 1."
+)
+set(vendor_sort
+    ${default_vendor_sort}
+    CACHE
+      BOOL
+      "Use the vendor sort_by_key (oneDPL/Thrust/rocThrust) for the team_policy spatial sort when available. OFF forces the Kokkos::BinSort fallback, which sorts each SoA member in place (lower peak memory, no maxnpart gather buffer) at the cost of sort speed."
+)
+
 # -------------------------- Compilation settings -------------------------- #
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -136,21 +158,14 @@ else()
   set(DEVICE_ENABLED OFF)
 endif()
 
+# ------------------------------ team_policy wiring ------------------------ #
+if(${team_policy})
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/team_policy.cmake)
+endif()
+
 # MPI
 if(${mpi})
-  find_or_fetch_dependency(MPI FALSE REQUIRED)
-  include_directories(${MPI_CXX_INCLUDE_PATH})
-  add_compile_options("-D MPI_ENABLED")
-  set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX)
-  if(${DEVICE_ENABLED})
-    if(${gpu_aware_mpi})
-      add_compile_options("-D GPU_AWARE_MPI")
-    endif()
-  else()
-    set(gpu_aware_mpi
-        OFF
-        CACHE BOOL "Use explicit copy when using MPI + GPU")
-  endif()
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/mpi.cmake)
 endif()
 
 # Output

cmake/defaults.cmake

@@ -92,3 +92,38 @@ else()
 endif()
 
 set_property(CACHE default_gpu_aware_mpi PROPERTY TYPE BOOL)
+
+if(DEFINED ENV{Entity_ENABLE_TEAM_POLICY})
+  set(default_team_policy
+      $ENV{Entity_ENABLE_TEAM_POLICY}
+      CACHE INTERNAL "Default flag for team_policy tile-blocked kernels")
+else()
+  set(default_team_policy
+      OFF
+      CACHE INTERNAL "Default flag for team_policy tile-blocked kernels")
+endif()
+set_property(CACHE default_team_policy PROPERTY TYPE BOOL)
+
+if(DEFINED ENV{Entity_ENABLE_VENDOR_SORT})
+  set(default_vendor_sort
+      $ENV{Entity_ENABLE_VENDOR_SORT}
+      CACHE INTERNAL
+            "Default flag for vendor sort_by_key (oneDPL/Thrust/rocThrust)")
+else()
+  set(default_vendor_sort
+      ON
+      CACHE INTERNAL
+            "Default flag for vendor sort_by_key (oneDPL/Thrust/rocThrust)")
+endif()
+set_property(CACHE default_vendor_sort PROPERTY TYPE BOOL)
+
+set(default_team_policy_tile_size
+    8
+    CACHE INTERNAL "Default tile edge length in cells for team_policy")
+
+set(default_team_policy_drift
+    1
+    CACHE
+      INTERNAL
+      "Default tiled-deposit scratch halo drift for team_policy (cells between sorts)"
+)

cmake/dependencies.cmake

@@ -4,7 +4,7 @@ set(Kokkos_REPOSITORY
     https://github.com/kokkos/kokkos.git
     CACHE STRING "Kokkos repository")
 set(Kokkos_TAG
-    5.0.1
+    5.2.1
     CACHE STRING "Kokkos tag")
 set(adios2_REPOSITORY
     https://github.com/ornladios/ADIOS2.git

cmake/mpi.cmake

@@ -0,0 +1,71 @@
+find_or_fetch_dependency(MPI FALSE REQUIRED)
+include_directories(${MPI_CXX_INCLUDE_PATH})
+add_compile_options("-D MPI_ENABLED")
+set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX)
+if(${DEVICE_ENABLED})
+  if(${gpu_aware_mpi})
+    add_compile_options("-D GPU_AWARE_MPI")
+
+    # On Cray systems (e.g. Frontier) GPU-aware Cray MPICH can only handle
+    # device pointers if the GPU Transport Layer (GTL) library is linked. The
+    # Cray compiler wrappers (cc/CC) inject this automatically, but we build
+    # with hipcc/nvcc directly, so find_package(MPI) only finds base libmpi and
+    # the GTL is left out -> MPI_Sendrecv on a device pointer fails with "OFI
+    # ... Bad address". Add it explicitly here.
+    #
+    # Cray PE exports PE_MPICH_GTL_DIR_<accel> / PE_MPICH_GTL_LIBS_<accel> (e.g.
+    # amd_gfx90a -> -lmpi_gtl_hsa). Their absence means this is not a Cray MPICH
+    # build, in which case nothing extra is needed.
+    if("${Kokkos_DEVICES}" MATCHES "HIP")
+      set(_gtl_accels amd_gfx942 amd_gfx940 amd_gfx90a amd_gfx908 amd_gfx906)
+    elseif("${Kokkos_DEVICES}" MATCHES "CUDA")
+      set(_gtl_accels nvidia90 nvidia80 nvidia70)
+    elseif("${Kokkos_DEVICES}" MATCHES "SYCL")
+      set(_gtl_accels ponteVecchio)
+    else()
+      set(_gtl_accels "")
+    endif()
+
+    set(_gtl_dir "")
+    set(_gtl_libflag "")
+    foreach(_accel ${_gtl_accels})
+      if((NOT _gtl_dir) AND (DEFINED ENV{PE_MPICH_GTL_DIR_${_accel}}))
+        # strip the leading "-L" from the Cray-provided value
+        string(REGEX REPLACE "^-L" "" _gtl_dir
+                             "$ENV{PE_MPICH_GTL_DIR_${_accel}}")
+        string(REGEX REPLACE "^-l" "" _gtl_libflag
+                             "$ENV{PE_MPICH_GTL_LIBS_${_accel}}")
+      endif()
+    endforeach()
+
+    if(_gtl_dir AND _gtl_libflag)
+      find_library(
+        MPI_GTL_LIBRARY
+        NAMES ${_gtl_libflag}
+        HINTS "${_gtl_dir}"
+        NO_DEFAULT_PATH)
+      if(MPI_GTL_LIBRARY)
+        message(
+          STATUS "GPU-aware MPI: linking Cray GTL library ${MPI_GTL_LIBRARY}")
+        set(DEPENDENCIES ${DEPENDENCIES} ${MPI_GTL_LIBRARY})
+      else()
+        message(
+          FATAL_ERROR
+            "${Red}gpu_aware_mpi=ON: Cray MPICH detected but the GTL "
+            "library 'lib${_gtl_libflag}' was not found in '${_gtl_dir}'. "
+            "GPU-aware MPI will crash at runtime without it. Make sure the "
+            "craype-accel module is loaded, or build with gpu_aware_mpi=OFF."
+            "${ColorReset}")
+      endif()
+    else()
+      message(
+        STATUS "GPU-aware MPI: no Cray GTL environment found; assuming the MPI "
+               "implementation is GPU-aware without an extra transport library."
+      )
+    endif()
+  endif()
+else()
+  set(gpu_aware_mpi
+      OFF
+      CACHE BOOL "Use explicit copy when using MPI + GPU")
+endif()

cmake/report.cmake

@@ -83,7 +83,7 @@ printchoices(
   ${default_deposit}
   "${Blue}"
   DEPOSIT_REPORT
-  46)
+  44)
 printchoices(
   "Shape order"
   "shape_order"
@@ -110,18 +110,43 @@ printchoices(
   OFF
   "${Green}"
   MPI_REPORT
-  46)
-if(${mpi} AND ${DEVICE_ENABLED})
-  printchoices(
-    "GPU-aware MPI"
-    "gpu_aware_mpi"
-    "${ON_OFF_VALUES}"
-    ${gpu_aware_mpi}
-    OFF
-    "${Green}"
-    GPU_AWARE_MPI_REPORT
-    46)
-endif()
+  44)
+printchoices(
+  "GPU-aware MPI"
+  "gpu_aware_mpi"
+  "${ON_OFF_VALUES}"
+  ${gpu_aware_mpi}
+  OFF
+  "${Green}"
+  GPU_AWARE_MPI_REPORT
+  44)
+printchoices(
+  "Team Policy"
+  "team_policy"
+  "${ON_OFF_VALUES}"
+  ${team_policy}
+  OFF
+  "${Green}"
+  TEAM_POLICY_REPORT
+  44)
+printchoices(
+  "Tile Size"
+  "team_policy_tile_size"
+  "${team_policy_tile_sizes}"
+  ${team_policy_tile_size}
+  ${default_team_policy_tile_size}
+  "${Blue}"
+  TEAM_POLICY_TILE_SIZE_REPORT
+  44)
+printchoices(
+  "Vendor sort"
+  "vendor_sort"
+  "${ON_OFF_VALUES}"
+  ${vendor_sort}
+  ON
+  "${Green}"
+  VENDOR_SORT_REPORT
+  44)
 printchoices(
   "Debug mode"
   "DEBUG"
@@ -158,62 +183,91 @@ string(APPEND REPORT_TEXT ${DASHED_LINE_SYMBOL} "\n" "Configurations" "\n")
 
 if(${PGEN_FOUND})
   string(APPEND REPORT_TEXT "  " ${PGEN_REPORT} "\n")
+else()
+  string(
+    APPEND
+    REPORT_TEXT
+    "  - Problem generator [${Magenta}pgen${ColorReset}]:                 ${Dim}none${ColorReset}\n"
+  )
 endif()
-string(APPEND REPORT_TEXT "  " ${TESTS_REPORT} "\n")
+
+string(REPLACE ";" "+" Kokkos_ARCH "${Kokkos_ARCH}")
+string(REPLACE ";" "+" Kokkos_DEVICES "${Kokkos_DEVICES}")
 
 string(
   APPEND
   REPORT_TEXT
   "  "
-  ${PRECISION_REPORT}
+  ${TESTS_REPORT}
   "\n"
   "  "
-  ${DEPOSIT_REPORT}
+  ${OUTPUT_REPORT}
+  "\n"
+  "  - Install prefix [${Magenta}CMAKE_INSTALL_PREFIX${ColorReset}]:    "
+  "${CMAKE_INSTALL_PREFIX}"
+  "\n"
+  ${DASHED_LINE_SYMBOL}
+  "\n"
+  "Algorithmic specs"
+  "\n"
+  "  "
+  ${PRECISION_REPORT}
   "\n"
   "  "
   ${SHAPEFUNCTION_REPORT}
   "\n"
+  "  > PIC-specific specs"
+  "\n"
+  "    "
+  ${DEPOSIT_REPORT}
+  "\n"
+  ${DASHED_LINE_SYMBOL}
+  "\n"
+  "Performance specs"
+  "\n"
   "  "
-  ${OUTPUT_REPORT}
-  "\n")
-
-string(REPLACE ";" "+" Kokkos_ARCH "${Kokkos_ARCH}")
-string(REPLACE ";" "+" Kokkos_DEVICES "${Kokkos_DEVICES}")
-
-string(
-  APPEND
-  REPORT_TEXT
+  ${DEBUG_REPORT}
+  "\n"
   "  - ARCH [${Magenta}Kokkos_ARCH_***${ColorReset}]:                   "
   "${Kokkos_ARCH}"
   "\n"
   "  - DEVICES [${Magenta}Kokkos_ENABLE_***${ColorReset}]:              "
   "${Kokkos_DEVICES}"
   "\n"
-  "  "
+  "  > Multi-node specs"
+  " ${Dim}[requires mpi=ON]${ColorReset}"
+  "\n"
+  "    "
   ${MPI_REPORT}
+  "\n"
+  "    "
+  ${GPU_AWARE_MPI_REPORT}
+  "\n"
+  "  > Team-policy specs"
+  " ${Dim}[requires team_policy=ON]${ColorReset}"
+  "\n"
+  "    "
+  ${TEAM_POLICY_REPORT}
+  "\n"
+  "    "
+  ${TEAM_POLICY_TILE_SIZE_REPORT}
+  "\n"
+  "    "
+  "- Deposit drift [${Magenta}team_policy_drift${ColorReset}]:      "
+  ${team_policy_drift}
+  "\n"
+  "    "
+  ${VENDOR_SORT_REPORT}
   "\n")
 
-if(${mpi} AND ${DEVICE_ENABLED})
-  string(APPEND REPORT_TEXT "  " ${GPU_AWARE_MPI_REPORT} "\n")
-endif()
-
 string(
   APPEND
   REPORT_TEXT
   "  "
-  ${DEBUG_REPORT}
-  "\n"
-  "  - Install prefix [${Magenta}CMAKE_INSTALL_PREFIX${ColorReset}]:    "
-  "${CMAKE_INSTALL_PREFIX}"
-  "\n"
   ${DASHED_LINE_SYMBOL}
   "\n"
   "Compilers & dependencies"
-  "\n")
-
-string(
-  APPEND
-  REPORT_TEXT
+  "\n"
   "  - C compiler [${Magenta}CMAKE_C_COMPILER${ColorReset}]: v"
   ${CMAKE_C_COMPILER_VERSION}
   "\n"