Overlap serialization and permutation edit.

.github/workflows/clang-tidy.yml

@@ -63,7 +63,7 @@ jobs:
         repo-path: 'kokkos/kokkos'
         repo-ref: '4.6.01'
         cache: true
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=OFF
                   -DKokkos_ENABLE_SERIAL=ON
                   -DKokkos_ENABLE_OPENMP=OFF
@@ -78,7 +78,7 @@ jobs:
         repo-path: 'kokkos/kokkos-kernels'
         repo-ref: '4.6.01'
         cache: true
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=OFF
                   -DKokkos_DIR=${{ runner.temp }}/build-kokkos-openmpi/install/lib/cmake/Kokkos'
 
@@ -189,10 +189,10 @@ jobs:
                   -DCMAKE_CXX_COMPILER=mpicxx \
                   -DCMAKE_Fortran_COMPILER=mpifort \
                   -DPCMS_ENABLE_PETSC=ON \
+                  -DPETSC_LINK_STATIC=ON \
                   -DPETSC_DIR=${{ runner.temp }}/petsc-openmpi \
                   -DPETSC_ARCH=ubuntu-kokkos \
                   -DPCMS_TIMEOUT=10 \
-                  -DPCMS_ENABLE_SPDLOG=OFF \
                   -DCatch2_DIR=${{ runner.temp }}/build-Catch2-openmpi/install/lib/cmake/Catch2 \
                   -DOmega_h_DIR=${{ runner.temp }}/build-omega_h-openmpi/install/lib/cmake/Omega_h \
                   -Dmeshfields_DIR=${{ runner.temp }}/build-meshFields-openmpi/install/lib/cmake/meshfields \

.github/workflows/cmake-test.yml

@@ -25,6 +25,23 @@ jobs:
         compiler: [g++]
         language: ['cpp']
         python_api: [OFF, ON]
+        meshfields: [ON]
+        petsc: [ON]
+        include:
+          - build_type: Release
+            memory_test: OFF
+            compiler: g++
+            language: 'cpp'
+            python_api: OFF
+            meshfields: OFF
+            petsc: ON
+          - build_type: Release
+            memory_test: OFF
+            compiler: g++
+            language: 'cpp'
+            python_api: OFF
+            meshfields: ON
+            petsc: OFF
         exclude:
           - build_type: Release
             memory_test: ON
@@ -68,7 +85,7 @@ jobs:
         repo-ref: '4.6.01'
         cache: true
         cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                   -DKokkos_ENABLE_SERIAL=ON
                   -DKokkos_ENABLE_OPENMP=OFF
@@ -84,7 +101,7 @@ jobs:
         repo-ref: '4.6.01'
         cache: true
         cache-suffix: ${{ matrix.python_api == 'ON' && '-shared' || '' }}
-        options: '-DCMAKE_CXX_STANDARD=17
+        options: '-DCMAKE_CXX_STANDARD=20
                   -DBUILD_SHARED_LIBS=${{ matrix.python_api }}
                   -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos'
 
@@ -106,6 +123,7 @@ jobs:
                   -DKokkos_DIR=${{ runner.temp }}/build-kokkos/install/lib/cmake/Kokkos'
 
     - name: build meshFields
+      if: matrix.meshfields == 'ON'
       uses: ./.github/actions/install-repo
       with:
         repo-name: 'meshFields'
@@ -153,7 +171,17 @@ jobs:
                   -DADIOS2_DIR=${{ runner.temp }}/build-ADIOS2/install/lib/cmake/adios2
                   -Dperfstubs_DIR=${{ runner.temp }}/build-perfstubs/install/lib/cmake'
 
+    - name: Set LD_LIBRARY_PATH for shared libraries
+      if: matrix.python_api == 'ON'
+      run: |
+        meshfields_lib=""
+        if [ "${{ matrix.meshfields }}" = "ON" ]; then
+          meshfields_lib="${{ runner.temp }}/build-meshFields/install/lib:"
+        fi
+        echo "LD_LIBRARY_PATH=${{ runner.temp }}/build-kokkos/install/lib:${{ runner.temp }}/build-omega_h/install/lib:${meshfields_lib}${{ runner.temp }}/build-redev/install/lib:${{ runner.temp }}/build-ADIOS2/install/lib:${{ runner.temp }}/build-perfstubs/install/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+
     - name: clone petsc
+      if: matrix.petsc == 'ON'
       id: clone-petsc
       run: |
         cd ${{ runner.temp }}
@@ -162,25 +190,35 @@ jobs:
         echo "petsc-commit-hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
 
     - name: Cache PETSc
+      if: matrix.petsc == 'ON'
       id: cache-petsc
       uses: actions/cache@v3
       with:
         path: ${{ runner.temp }}/petsc
-        key: build-petsc-${{ steps.clone-petsc.outputs.petsc-commit-hash }}
+        key: build-petsc-${{ steps.clone-petsc.outputs.petsc-commit-hash }}-${{ matrix.python_api }}
 
     - name: build petsc
-      if: steps.cache-petsc.outputs.cache-hit != 'true'
+      if: matrix.petsc == 'ON' && steps.cache-petsc.outputs.cache-hit != 'true'
       run: |
         cd ${{ runner.temp }}/petsc
         ./configure \
           PETSC_ARCH=ubuntu-kokkos \
           --with-kokkos-dir="${{ runner.temp }}/build-kokkos/install/" \
           --with-kokkos-kernels-dir="${{ runner.temp }}/build-kokkos-kernels/install/" \
           --with-cuda=0 \
-          --with-shared-libraries=0 \
+          --with-shared-libraries=${{ matrix.python_api == 'ON' && '1' || '0' }} \
           --download-fblaslapack
         make all check
 
+    - name: Set PCMS PETSc options
+      run: |
+        echo "PCMS_ENABLE_PETSC=-DPCMS_ENABLE_PETSC=${{ matrix.petsc }}" >> $GITHUB_ENV
+        if [ "${{ matrix.petsc }}" = "ON" ]; then
+          echo "PCMS_PETSC_OPTIONS=-DPETSC_LINK_STATIC=ON -DPETSC_DIR=${{ runner.temp }}/petsc -DPETSC_ARCH=ubuntu-kokkos" >> $GITHUB_ENV
+        else
+          echo "PCMS_PETSC_OPTIONS=" >> $GITHUB_ENV
+        fi
+
     - name: checkout pcms_testcases
       uses: actions/checkout@v3
       with:
@@ -190,10 +228,14 @@ jobs:
     - name: Install fftw3
       run: sudo apt-get install -yq libfftw3-dev pkg-config
 
-    - name: Set LD_LIBRARY_PATH for shared libraries
-      if: matrix.python_api == 'ON'
+    - name: Set PCMS MeshFields options
       run: |
-        echo "LD_LIBRARY_PATH=${{ runner.temp }}/build-kokkos/install/lib:${{ runner.temp }}/build-omega_h/install/lib:${{ runner.temp }}/build-meshFields/install/lib:${{ runner.temp }}/build-redev/install/lib:${{ runner.temp }}/build-ADIOS2/install/lib:${{ runner.temp }}/build-perfstubs/install/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+        echo "PCMS_ENABLE_MESHFIELDS=-DPCMS_ENABLE_MESHFIELDS=${{ matrix.meshfields }}" >> $GITHUB_ENV
+        if [ "${{ matrix.meshfields }}" = "ON" ]; then
+          echo "PCMS_MESHFIELDS_DIR=-Dmeshfields_DIR=${{ runner.temp }}/build-meshFields/install/lib/cmake/meshfields" >> $GITHUB_ENV
+        else
+          echo "PCMS_MESHFIELDS_DIR=" >> $GITHUB_ENV
+        fi
 
     - name: build pcms
       uses: ./.github/actions/install-repo
@@ -207,15 +249,14 @@ jobs:
                   -DCMAKE_CXX_COMPILER=`which mpicxx`
                   -DCMAKE_Fortran_COMPILER=`which mpifort`
                   -DMPIEXEC_PREFLAGS="--oversubscribe"
-                  -DPCMS_ENABLE_PETSC=ON
-                  -DPETSC_DIR=${{ runner.temp }}/petsc
-                  -DPETSC_ARCH=ubuntu-kokkos
+                  ${{ env.PCMS_ENABLE_MESHFIELDS }}
+                  ${{ env.PCMS_ENABLE_PETSC }}
+                  ${{ env.PCMS_PETSC_OPTIONS }}
                   -DPCMS_TIMEOUT=10
-                  -DPCMS_ENABLE_SPDLOG=OFF
                   -DPCMS_ENABLE_Python=${{ matrix.python_api }}
                   -DCatch2_DIR=${{ runner.temp }}/build-Catch2/install/lib/cmake/Catch2
                   -DOmega_h_DIR=${{ runner.temp }}/build-omega_h/install/lib/cmake/Omega_h
-                  -Dmeshfields_DIR=${{ runner.temp }}/build-meshFields/install/lib/cmake/meshfields
+                  ${{ env.PCMS_MESHFIELDS_DIR }}
                   -Dredev_DIR=${{ runner.temp }}/build-redev/install/lib/cmake/redev
                   -DMPIEXEC_EXECUTABLE=`which mpirun`
                   -DADIOS2_DIR=${{ runner.temp }}/build-ADIOS2/install/lib/cmake/adios2
@@ -256,12 +297,13 @@ jobs:
         cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms-config.cmake
         echo "-------- PCMS TARGETS -------------"
         cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms-targets.cmake
-        echo "-------- PCMS INTERPOLATOR TARGETS -------------"
-        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms_interpolator-targets.cmake
+        echo "-------- PCMS TRANSFER TARGETS -------------"
+        cat ${{ runner.temp }}/build-pcms/install/lib/cmake/pcms/pcms_transfer-targets.cmake
         export VERBOSE=1
         cmake \
           -B ${{github.workspace}}/examples/external-usage-example/build \
           -S ${{github.workspace}}/examples/external-usage-example/ \
           -Dpcms_DIR=${{ runner.temp }}/build-pcms/install/lib/cmake/pcms \
+          ${{ matrix.petsc == 'ON' && format('-DPETSC_LINK_STATIC=ON -DPETSC_DIR={0}/petsc -DPETSC_ARCH=ubuntu-kokkos', runner.temp) || '' }} \
           --debug-output
         cmake --build ${{github.workspace}}/examples/external-usage-example/build

.github/workflows/perlmutter/install.sh

@@ -45,7 +45,7 @@ build-pcms/install
 # cmake -S kokkos-kernels -B build-kokkos-kernels \
 #   -DCMAKE_INSTALL_PREFIX=$PWD/build-kokkos-kernels/install \
 #   -DCMAKE_CXX_COMPILER=CC \
-#   -DCMAKE_CXX_STANDARD=17
+#   -DCMAKE_CXX_STANDARD=20
 # cmake --build build-kokkos-kernels -j24 --target install
 
 
@@ -131,4 +131,4 @@ cmake -S pcms -B build-pcms \
   -DPCMS_TIMEOUT=100 \
   -DCMAKE_CXX_STANDARD=20 \
   -DPCMS_TEST_DATA_DIR=$PWD/pcms_testcases
-cmake --build build-pcms -j8
+cmake --build build-pcms -j8

.github/workflows/self-hosted.yml

@@ -180,15 +180,37 @@ jobs:
 
           git clone https://github.com/jacobmerson/pcms_testcases.git ${workDir}/pcms_testcases
 
+          # pcms - petsc off
+          bdir=${workDir}/build-pcms-petsc-off
+          cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DCMAKE_C_COMPILER=mpicc \
+            -DCMAKE_CXX_COMPILER=mpicxx \
+            -DPCMS_TIMEOUT=20 \
+            -DPCMS_ENABLE_PETSC=OFF \
+            -Dredev_DIR=$rdbdir/install/lib64/cmake/redev/ \
+            -DOmega_h_DIR=$ohbdir/install/lib64/cmake/Omega_h/ \
+            -Dperfstubs_DIR=$psbdir/install/lib/cmake/ \
+            -DADIOS2_DIR=$adiosbdir/install/lib64/cmake/adios2/ \
+            -DCatch2_DIR=$c2bdir/install/lib64/cmake/Catch2/ \
+            -DKokkos_DIR=$kkbdir/install/lib64/cmake/Kokkos/ \
+            -DKokkosKernels_DIR=$kkkbdir/install/lib64/cmake/KokkosKernels/ \
+            -Dmeshfields_DIR=$mfbdir/install/lib64/cmake/meshfields/ \
+            -DPCMS_TEST_DATA_DIR=${workDir}/pcms_testcases/ \
+            -DCMAKE_CXX_EXTENSIONS=Off
+          cmake --build $bdir
+          ctest --test-dir $bdir --output-on-failure
+
           # pcms
+          export PETSC_OPTIONS="-use_gpu_aware_mpi 0"
           bdir=${workDir}/build-pcms
           cmake -S ${{github.workspace}}/pcms_${{ github.event.id }} -B $bdir \
             -DCMAKE_BUILD_TYPE=Debug \
             -DCMAKE_C_COMPILER=mpicc \
             -DCMAKE_CXX_COMPILER=mpicxx \
             -DPCMS_TIMEOUT=20 \
-            -DPCMS_ENABLE_SPDLOG=OFF \
             -DPCMS_ENABLE_PETSC=ON \
+            -DPETSC_LINK_STATIC=ON \
             -DPETSC_DIR=${workDir}/petsc \
             -DPETSC_ARCH=cuda-kokkos \
             -Dredev_DIR=$rdbdir/install/lib64/cmake/redev/ \
@@ -218,4 +240,4 @@ jobs:
         if: ${{ !cancelled() }}
         run: |
           echo "PCMS_WORK_DIR $PCMS_WORK_DIR"
-          rm -rf $PCMS_WORK_DIR
+          rm -rf $PCMS_WORK_DIR

CMakeLists.txt

@@ -44,15 +44,14 @@ option(PCMS_ENABLE_CLIENT "enable the coupling client implementation" ON)
 
 option(PCMS_ENABLE_XGC "enable xgc field adapter" ON)
 option(PCMS_ENABLE_OMEGA_H "enable Omega_h field adapter" OFF)
+option(PCMS_ENABLE_MFEM "enable MFEM field adapter" OFF)
 option(PCMS_ENABLE_C "Enable pcms C api" ON)
 option(PCMS_ENABLE_Python "Enable pcms Python api" OFF)
 
 option(PCMS_ENABLE_PRINT "PCMS print statements enabled" ON)
-option(PCMS_ENABLE_SPDLOG "use spdlog for logging" ON)
 
-if(PCMS_ENABLE_SPDLOG)
-  find_package(spdlog REQUIRED)
-endif()
+option(PETSC_LINK_STATIC "Use pkg-config --static results for PETSc"
+       ${_pcms_link_petsc_static_default})
 
 # find package before fortran enabled, so we don't require the adios2 fortran
 # interfaces this is important because adios2 build with clang/gfortran is
@@ -121,11 +120,29 @@ if(PCMS_ENABLE_OMEGA_H)
     message(FATAL_ERROR "Omega_h must be built with MPI enabled.")
   endif()
 endif()
+
+if(PCMS_ENABLE_MFEM)
+  find_package(MFEM REQUIRED)
+  message(STATUS "Found MFEM: ${MFEM_DIR} (found version ${MFEM_VERSION})")
+  if(NOT MFEM_USE_MPI)
+    message(FATAL_ERROR "MFEM must be built with MPI enabled.")
+  endif()
+endif()
 # adios2 adds C and Fortran depending on how it was built
 find_package(ADIOS2 CONFIG 2.10.2 REQUIRED)
 find_package(Kokkos CONFIG 4.5 REQUIRED)
 
-find_package(meshfields REQUIRED)
+option(PCMS_ENABLE_MESHFIELDS "Enable MeshFields support" ON)
+option(PCMS_ENABLE_PETSC "Enable PETSc support" OFF)
+
+if(PCMS_ENABLE_MESHFIELDS)
+  find_package(meshfields REQUIRED)
+  message(STATUS "Found MeshFields: ${meshfields_DIR} (found version ${meshfields_VERSION})")
+endif()
+
+if(PCMS_ENABLE_PETSC)
+  find_package(PETSc REQUIRED)
+endif()
 
 add_subdirectory(src)
 

MAINTAINER.md

@@ -43,6 +43,13 @@ see issue #<###>
 6. create the tag `git tag -a v#.#.# -m "pcms version #.#.#"`
 7. push the tag `git push origin v#.#.#`
 
+## Branch Management
+
+The `develop` branch is the main development branch where all new features and bug fixes are merged. The `master` branch is the stable release branch that reflects the latest released version of pcms. All changes should be made in the `develop` branch and then merged into `master` when ready for release. 
+
+It is generally recommended to use forks for development to keep the main repository clean and to facilitate code review. Contributors can create a fork of the repository, make their changes in a new branch, and then submit a pull request to the `develop` branch of the main repository. Ensure that the pull request includes a clear description of the changes made and references any relevant issues.
+
+When creating a new feature or fixing a bug in main repo, create a new branch from `develop` with a descriptive name (e.g., `feature/new-feature`, `bugfix/issue-123`).
 
 ## Maintain CI/CD
 

README.md

@@ -32,7 +32,7 @@ module load cuda/12.1.1-zxa4msk
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos.git
 cmake -S kokkos -B build-kokkos \
   -DCMAKE_INSTALL_PREFIX=build-kokkos/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DKokkos_ENABLE_SERIAL=ON \
   -DKokkos_ENABLE_OPENMP=OFF \
   -DKokkos_ENABLE_CUDA=OFF \
@@ -44,7 +44,7 @@ cmake --build build-kokkos --target install
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos-kernels.git
 cmake -S kokkos-kernels -B build-kokkos-kernels \
   -DCMAKE_INSTALL_PREFIX=build-kokkos-kernels/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DKokkos_ROOT=$PWD/build-kokkos/install/lib64/cmake \
   -DBUILD_SHARED_LIBS=OFF
 cmake --build build-kokkos-kernels --target install
@@ -97,7 +97,7 @@ cmake --build build-redev --target install
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos.git
 cmake -S kokkos -B build-kokkos \
   -DCMAKE_INSTALL_PREFIX=build-kokkos/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DCMAKE_BUILD_TYPE="Release" \
   -DCMAKE_CXX_COMPILER=$PWD/kokkos/bin/nvcc_wrapper \
   -DKokkos_ARCH_AMPERE80=ON \
@@ -112,7 +112,7 @@ cmake --build build-kokkos --target install
 git clone --branch 4.6.01 --depth 1 git@github.com:kokkos/kokkos-kernels.git
 cmake -S kokkos-kernels -B build-kokkos-kernels \
   -DCMAKE_INSTALL_PREFIX=build-kokkos-kernels/install \
-  -DCMAKE_CXX_STANDARD=17 \
+  -DCMAKE_CXX_STANDARD=20 \
   -DKokkos_ROOT=$PWD/build-kokkos/install/lib64/cmake \
   -DBUILD_SHARED_LIBS=off
 cmake --build build-kokkos-kernels --target install
@@ -248,9 +248,19 @@ If work in SCOREC system, you can directly use the spack-scorec.yaml which conta
    Then add the binded python module to your `PYTHONPATH` environment variable. You can find the install prefix with `find $(spack location -i pcms) -name "*.so" | grep -i pcms`.
    ```console
     $ export PYTHONPATH=$<your-path-to-module>:$PYTHONPATH
-    ```
+   ```
 
    Test the python API with `python -c "import pcms; print(pcms.__version__)"`. You should see the version of PCMS printed out without any errors. 
+
+   Point-evaluator construction in Python uses `EvaluationRequest`, for
+   example:
+   ```python
+   request = pcms.EvaluationRequest.from_coordinates(eval_coords)
+   evaluator = source_space.create_point_evaluator(request)
+
+   optimized_request = pcms.EvaluationRequest.from_function_space(target_space)
+   evaluator = source_space.create_point_evaluator(optimized_request)
+   ```
 
    PS: If you need certain config options for the python API, you can specify them in the spack spec. For example, to build the python API with Exodus support, one compatible config is `pcms+python ^omega-h+trilinos ^trilinos@15.0.0:+exodus ^netcdf-c@4.8.1+mpi`. At this moment, omega-h spack package does not have the exodus support, so you need to manually add `args.append("-Omega_h_USE_SEACASExodus:BOOL=ON")` to the `package.py` file of omega-h in spack before installing pcms with spack.
 
@@ -262,3 +272,11 @@ Assign the full path of testdatas to test_dir in the test_init.cc file; Use "mpi
 
 ## Creating Archive for Release
 `git archive --format=tar.gz -o /tmp/pcms.tar.gz --prefix=pcms/ develop`
+
+## Running clang format on SCOREC
+```console
+module use /opt/scorec/spack/rhel9/v0222_2/lmod/linux-rhel9-x86_64/Core/
+module load clang-format
+module load llvm
+find . -regex '.*\.\(cpp\|cxx\|cc\|c\|hpp\|h\)' -exec clang-format -style=file -i {} \;
+```