Speed up CI

.gitlab-ci.yml

@@ -1,113 +1,82 @@
+
+include:
+  - local: 'ci_tools/reusable_ci_jobs.yml'
+
 cmake_tests:
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      changes:
-        - vendor/sll/**/*
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      variables:
-        SLL_BUILD_TESTING: "OFF"
-  stage: test
-  needs: []
+  extends: .cmake_tests
   image: ghcr.io/gyselax/gyselalibxx_env
   timeout: 2 hours
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-    SLL_BUILD_TESTING: "ON"
   script: |
+    set -x
     git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
     rm -rf build || true
     mkdir build
     cd build
-    cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DBUILD_BENCHMARKS=ON -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 ..
+    cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DBUILD_BENCHMARKS=ON $GENERAL_CMAKE_OPTIONS ..
     make -j 4
     ctest -j 2 --output-on-failure --timeout 5 -LE test_on_Release_only --output-junit tests.xml
   artifacts:
     when: always
+    expire_in: 1 day
     reports:
       junit: ./build/tests.xml
+
 cmake_tests_Release:
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      changes:
-        - vendor/sll/**/*
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      variables:
-        SLL_BUILD_TESTING: "OFF"
-  stage: test
-  needs: []
+  extends: .cmake_tests
   image: ghcr.io/gyselax/gyselalibxx_env
-  timeout: 2 hours
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-    SLL_BUILD_TESTING: "ON"
   script: |
-    set -e
+    set -x
     git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
     rm -rf build || true
     mkdir build
     cd build
-    cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 ..
+    cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release $GENERAL_CMAKE_OPTIONS ..
     make -j 4
     ctest -j 2 --output-on-failure --timeout 5 --output-junit tests.xml
     ls ./tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
     ls ./tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
   artifacts:
     when: always
+    expire_in: 1 day
     paths:
       - ./build/tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
       - ./build/tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
     reports:
       junit: ./build/tests.xml
+
 cmake_tests_Release_GPU:
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      changes:
-        - vendor/sll/**/*
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      variables:
-        SLL_BUILD_TESTING: "OFF"
+  extends: .cmake_tests
   tags:
     - persee
-  stage: test
-  needs: []
-  timeout: 2 hours
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-    SLL_BUILD_TESTING: "ON"
   before_script: |
     . /data/gyselarunner/spack-0.20.0/share/spack/setup-env.sh
     spack load gcc@11
     spack env activate voicexx-env-omp-cuda
   script: |
-    set -e
+    set -x
+    export OMP_PROC_BIND=spread
+    export OMP_PLACES=threads
+    export OMP_NUM_THREADS=8
     git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos
     rm -rf build || true
     mkdir build
     cd build
-    cmake -DCMAKE_CXX_COMPILER=${CI_PROJECT_DIR}/vendor/kokkos/bin/nvcc_wrapper -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare -Xcudafe --diag_suppress=unsigned_compare_with_zero -Xcudafe --diag_suppress=integer_sign_change" -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=ON -DKokkos_ENABLE_CUDA=ON ../
+    cmake -DCMAKE_CXX_COMPILER=${CI_PROJECT_DIR}/vendor/kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare -Xcudafe --diag_suppress=unsigned_compare_with_zero -Xcudafe --diag_suppress=integer_sign_change" \
+          -DCMAKE_BUILD_TYPE=Release $GENERAL_CMAKE_OPTIONS -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=ON -DKokkos_ENABLE_CUDA=ON ..
     make -j 8
-    export OMP_PROC_BIND=spread
-    export OMP_PLACES=threads
-    export OMP_NUM_THREADS=8
     ctest --output-on-failure --timeout 5 --output-junit tests.xml -LE ResultsNotTested
     ls ./tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
     ls ./tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
   artifacts:
     when: always
+    expire_in: 1 day
     paths:
       - ./build/tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png
       - ./build/tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png
     reports:
       junit: ./build/tests.xml
+
 indentation:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -144,6 +113,7 @@ python_best_practices:
     for f in $(git diff origin/main --name-only); do grep $f post_process_errors.txt || true; done | tee filtered_errors.txt
     # Raise an error if post-process in filtered errors
     ! grep "post-process" filtered_errors.txt >/dev/null
+
 public_mirror:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

ci_tools/reusable_ci_jobs.yml

@@ -0,0 +1,47 @@
+.setup_env:
+  stage: .pre
+  image: ghcr.io/gyselax/gyselalibxx_env
+  needs: []
+  timeout: 10 mins
+  script:
+    - FOUND_SLL=$(grep "vendor/sll/.*" -x changed_files.txt || true)
+    - if [ -z "$FOUND_SLL" ]; then echo "SLL_BUILD_TESTING=OFF" >> build.env; else echo "SLL_BUILD_TESTING=ON" >> build.env; fi
+    - FOUND_POISSON_2D=$(grep "src/geometryRTheta/poisson/.*" -x changed_files.txt || true)
+    - if [ -z "$FOUND_POISSON_2D" ]; then echo "POISSON_2D_BUILD_TESTING=OFF" >> build.env; else echo "POISSON_2D_BUILD_TESTING=ON" >> build.env; fi
+  artifacts:
+    expire_in: 1 week
+    reports:
+      dotenv: build.env
+
+setup_env_push:
+  extends: .setup_env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  before_script:
+    - set -x
+    - git log -1 --format=%H | git diff --name-only" > changed_files.txt
+
+setup_env_mr:
+  extends: .setup_env
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  before_script:
+    - set -x
+    - ${GIT_DIFF_COMMAND} > changed_files.txt
+    - git config --global --add safe.directory $(pwd)
+    - git fetch --no-recurse-submodules origin ${CI_MERGE_REQUEST_DIFF_BASE_SHA}
+    - git diff --name-only ${CI_MERGE_REQUEST_DIFF_BASE_SHA} > changed_files.txt
+
+
+.cmake_tests:
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      needs: [ setup_env_push ]
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: manual
+      needs: [ setup_env_mr ]
+  stage: test
+  timeout: 2 hours
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+    GENERAL_CMAKE_OPTIONS: "-DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DPOISSON_2D_BUILD_TESTING=$POISSON_2D_BUILD_TESTING"

tests/geometryRTheta/2d_spline_interpolator/CMakeLists.txt

@@ -26,3 +26,4 @@ add_test(NAME Test_2d_spline_interpolator_Convergence
            "$<TARGET_FILE:${test_name}>")
 
 set_property(TEST Test_2d_spline_interpolator_Convergence PROPERTY TIMEOUT 200)
+set_property(TEST Test_2d_spline_interpolator_Convergence PROPERTY COST 100)

tests/geometryRTheta/CMakeLists.txt

@@ -1,6 +1,11 @@
 # SPDX-License-Identifier: MIT
 
-add_subdirectory(polar_poisson)
+# Allow the possibility of deactivating slow polar poisson tests
+option(POISSON_2D_BUILD_TESTING "Build polar poisson tests" ON)
+
+if("${POISSON_2D_BUILD_TESTING}")
+    add_subdirectory(polar_poisson)
+endif()
 add_subdirectory(2d_spline_interpolator)
 add_subdirectory(quadrature)
 add_subdirectory(advection_2d_rp)

tests/geometryRTheta/advection_2d_rp/CMakeLists.txt

@@ -51,6 +51,7 @@ foreach(MAPPING_TYPE "CIRCULAR_MAPPING" "CZARNY_MAPPING_PHYSICAL" "CZARNY_MAPPIN
                COMMAND "$<TARGET_FILE:Python3::Interpreter>" "${CMAKE_CURRENT_SOURCE_DIR}/display_all_errors_for_gtest.py"
                     "$<TARGET_FILE:advection_${MAPPING_TYPE}__${TIME_METHOD}__${SIMULATION}>")
         set_property(TEST TestAdvectionRPConvergence_${MAPPING_TYPE}__${TIME_METHOD}__${SIMULATION} PROPERTY TIMEOUT 20)
+        set_property(TEST TestAdvectionRPConvergence_${MAPPING_TYPE}__${TIME_METHOD}__${SIMULATION} PROPERTY COST 10)
     endforeach()
 endforeach()
 
@@ -89,6 +90,3 @@ target_link_libraries(advection_ALL
 target_compile_definitions(advection_ALL PUBLIC)
 
 find_package(Python3 REQUIRED COMPONENTS Interpreter)
-
-
-

tests/geometryRTheta/polar_poisson/CMakeLists.txt

@@ -27,6 +27,7 @@ foreach(MAPPING_TYPE "CIRCULAR_MAPPING" "CZARNY_MAPPING")
         COMMAND "$<TARGET_FILE:Python3::Interpreter>" "${CMAKE_CURRENT_SOURCE_DIR}/test_poisson.py"
             "$<TARGET_FILE:${test_name}>")
     set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY TIMEOUT 200)
+    set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY COST 100)
   endforeach()
 endforeach()
 
@@ -60,4 +61,5 @@ foreach(MAPPING_TYPE "CIRCULAR_MAPPING" "CZARNY_MAPPING")
         COMMAND "$<TARGET_FILE:Python3::Interpreter>" "${CMAKE_CURRENT_SOURCE_DIR}/test_vlasov_poisson.py"
             "$<TARGET_FILE:${test_name}>")
     set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY TIMEOUT 200)
+    set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY COST 100)
 endforeach()

tests/geometryXVx/bump_on_tail/CMakeLists.txt

@@ -11,13 +11,15 @@ add_test(NAME TestSimulationBumpontailFFT
         "$<TARGET_FILE:Python3::Interpreter>"
         "fft")
 set_property(TEST TestSimulationBumpontailFFT PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationBumpontailFFT PROPERTY COST 100)
 
 add_test(NAME TestSimulationBumpontailRestartFFT_XVx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_bumpontail_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:bumpontail_fft>"
         "restart")
 set_property(TEST TestSimulationBumpontailRestartFFT_XVx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationBumpontailRestartFFT_XVx PROPERTY COST 100)
 set_tests_properties(TestSimulationBumpontailRestartFFT_XVx PROPERTIES LABELS "ResultsNotTested" )
 
 add_test(NAME TestSimulationBumpontailFemUniform_xperiod_vx
@@ -27,11 +29,13 @@ add_test(NAME TestSimulationBumpontailFemUniform_xperiod_vx
         "$<TARGET_FILE:Python3::Interpreter>"
         "fem")
 set_property(TEST TestSimulationBumpontailFemUniform_xperiod_vx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationBumpontailFemUniform_xperiod_vx PROPERTY COST 100)
 
 add_test(NAME TestSimulationBumpontailRestartFemUniform_xperiod_vx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_bumpontail_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:bumpontail_fem_uniform_xperiod_vx>"
         "restart")
 set_property(TEST TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTY COST 100)
 set_tests_properties(TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTIES LABELS "ResultsNotTested" )

tests/geometryXVx/landau/CMakeLists.txt

@@ -11,13 +11,15 @@ add_test(NAME TestSimulationLandauFFT_XVx
         "$<TARGET_FILE:Python3::Interpreter>"
         "fft")
 set_property(TEST TestSimulationLandauFFT_XVx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationLandauFFT_XVx PROPERTY COST 100)
 
 add_test(NAME TestSimulationLandauRestartFFT_XVx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_landau_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:landau_fft>"
         "restart")
 set_property(TEST TestSimulationLandauRestartFFT_XVx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationLandauRestartFFT_XVx PROPERTY COST 100)
 set_tests_properties(TestSimulationLandauRestartFFT_XVx PROPERTIES LABELS "ResultsNotTested" )
 
 add_test(NAME TestSimulationLandauFemUniform_xperiod_vx
@@ -27,11 +29,13 @@ add_test(NAME TestSimulationLandauFemUniform_xperiod_vx
         "$<TARGET_FILE:Python3::Interpreter>"
         "fem")
 set_property(TEST TestSimulationLandauFemUniform_xperiod_vx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationLandauFemUniform_xperiod_vx PROPERTY COST 100)
 
 add_test(NAME TestSimulationLandauRestartFemUniform_xperiod_vx
     COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_landau_restart.sh"
         "${PROJECT_SOURCE_DIR}"
         "$<TARGET_FILE:landau_fem_uniform_xperiod_vx>"
         "restart")
 set_property(TEST TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTY COST 100)
 set_tests_properties(TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTIES LABELS "ResultsNotTested" )

tests/geometryXVx/sheath/CMakeLists.txt

@@ -8,4 +8,5 @@ add_test(NAME TestSimulationSheathRestart_xperiod_vx
         "$<TARGET_FILE:sheath_xperiod_vx>"
         "restart")
 set_property(TEST TestSimulationSheathRestart_xperiod_vx PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationSheathRestart_xperiod_vx PROPERTY COST 100)
 set_tests_properties(TestSimulationSheathRestart_xperiod_vx PROPERTIES LABEL "ResultsNotTested" )

tests/geometryXYVxVy/landau/CMakeLists.txt

@@ -11,3 +11,4 @@ add_test(NAME TestSimulationLandauFFT_XYVxVy
         "$<TARGET_FILE:Python3::Interpreter>"
         "fft")
 set_property(TEST TestSimulationLandauFFT_XYVxVy PROPERTY TIMEOUT 200)
+set_property(TEST TestSimulationLandauFFT_XYVxVy PROPERTY COST 100)