CI for Regression Testing & Benchmarking

.gitignore

@@ -106,3 +106,4 @@ bench_input
 bench_output
 bench_save
 bench_work
+/models

.gitlab-ci.yml

@@ -1,3 +1,5 @@
+include: ci/.gitlab-setup.yml
+
 stages:
   - sync
   - build
@@ -17,6 +19,9 @@ variables:
   CPU_CORES:
     description: "Select number of CPU cores and test workers"
     value: "32"
+  CPU_CORES_BENCH:
+    description: "Select number of CPU cores for benchmark runs"
+    value: "8"
   PARALLEL_JOBS:
     description: "Number of parallel Slurm array jobs per Benchmark job"
     value: "1"
@@ -30,15 +35,14 @@ variables:
     description: "Optional QoS option (include --qos, e.g., --qos express)"
     value: ""
   MANUAL_CFG_PATH:
-    description: "Use this config file instead of configs stored in the repo. Path must be accessible to runner"
+    description: "Name (in ci/cfg/) or path (relative to LOCAL_CFG_DIR) of benchmarking config to run"
     value: ""
 
 workflow:
   name: '$PIPELINE_NAME'
   rules:
-    # Run pipeline for GitHub PRs to dev or main (does not support PRs from forks)
+    # Run pipeline for GitHub PRs to dev (does not support PRs from forks)
     - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
-    - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "main"
     # Run pipeline for pushes to dev or main
     - if: $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH == "main"
     # Run pipeline if manually triggered via API or web GUI
@@ -67,38 +71,6 @@ Sync finn-dev:
     - git pull upstream dev
     - git push origin finn-dev
 
-.n2_setup_general: &n2_setup_general
-  - module load lang/Python/3.10.4-GCCcore-11.3.0
-  - module load devel/Autoconf/2.71-GCCcore-11.3.0
-  - module load lang/Bison/3.8.2-GCCcore-11.3.0
-  - module load lang/flex/2.6.4-GCCcore-11.3.0
-  - module load compiler/GCC/11.3.0
-  - module load lib/pybind11/2.9.2-GCCcore-11.3.0
-  - module load devel/Boost/1.79.0-GCC-11.3.0
-  - module load lib/fmt/9.1.0-GCCcore-11.3.0
-  - ulimit -s unlimited # Increase stack size limit
-
-.n2_setup_xilinx_2022_2: &n2_setup_xilinx_2022_2
-  - module load fpga
-  - module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
-  # module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
-  - export PLATFORM_REPO_PATHS=$PLATFORM_PATH
-
-.n2_setup_xilinx_2024_2: &n2_setup_xilinx_2024_2
-  - module load fpga
-  - module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
-  - module swap xilinx/vitis/24.2 # switch to Vitis/Vivado 2024.2
-  # module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
-  - export PLATFORM_REPO_PATHS=$PLATFORM_PATH
-
-.setup_venv_from_whl: &setup_venv_from_whl
-  # Move everything to working directory (e.g., RAMdisk)
-  - cp -dfR . $PATH_WORKDIR
-  - cd $PATH_WORKDIR
-  # Create fresh virtual environment and install finn-plus from .whl (artifact)
-  - python3 -m venv finn-plus-venv
-  - finn-plus-venv/bin/pip install dist/*.whl
-
 Build:
   id_tokens:
     CI_JOB_JWT:
@@ -113,8 +85,8 @@ Build:
     # Otherwise run
     - when: always
   before_script:
-    - *n2_setup_general
-    - *n2_setup_xilinx_2022_2
+    - !reference [.n2_setup_general, before_script]
+    - !reference [.n2_setup_xilinx_2022_2, before_script]
     # Install current version of Poetry
     - python3 -m venv poetry-install
     - poetry-install/bin/pip install poetry
@@ -151,6 +123,9 @@ FINN Test Suite 2022.2:
     # Do not run if test suite has been deselected
     - if: $TEST_SUITE == "none"
       when: never
+    # Do not run for PRs to dev (run only for pushes)
+    - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
+      when: never
     # Always run, as long as there was no prior failure
     - when: on_success
   cache:
@@ -159,13 +134,10 @@ FINN Test Suite 2022.2:
     paths:
       - deps
   variables:
-    GIT_STRATEGY: empty # Do not pull repository, use PyPI installation instead
+    GIT_STRATEGY: empty # Do not pull repository, install from wheel (artifact) instead
     SCHEDULER_PARAMETERS: "-A $PROJECT_ACCOUNT -p $SLURM_PARTITION -t $SLURM_TIMEOUT $SLURM_QOS --nodes 1 --ntasks 1 --cpus-per-task $CPU_CORES --exclusive"
     PYTEST_PARALLEL: "$CPU_CORES"
-  before_script:
-    - *n2_setup_general
-    - *n2_setup_xilinx_2022_2
-    - *setup_venv_from_whl
+  extends: .setup_full_2022_2
   script:
     # Launch additional monitoring
     - $JOB_MONITORING_DIR/monitor.sh $JOB_MONITORING_DIR/$CI_PIPELINE_ID/$HOSTNAME.log &
@@ -182,8 +154,71 @@ FINN Test Suite 2022.2:
       junit: reports/*.xml
 
 FINN Test Suite 2024.2:
-  extends: FINN Test Suite 2022.2
-  before_script:
-    - *n2_setup_general
-    - *n2_setup_xilinx_2024_2
-    - *setup_venv_from_whl
+  extends:
+    - FINN Test Suite 2022.2
+    - .setup_full_2024_2
+  rules:
+    # Do not run on a schedule
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    # Do not run if test suite has been deselected
+    - if: $TEST_SUITE == "none"
+      when: never
+    # Always run, as long as there was no prior failure
+    - when: on_success
+
+Bench (Manual):
+  stage: test
+  rules:
+    # Do not run on a schedule
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    - if: $MANUAL_CFG_PATH != ""
+  trigger:
+    include: ci/.gitlab-bench.yml
+    strategy: depend
+    forward:
+      pipeline_variables: true
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    BENCH_CFG: "manual"
+
+Bench (Basic):
+  stage: test
+  rules:
+    # Do not run on a schedule
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    - if: $MANUAL_CFG_PATH == ""
+  trigger:
+    include: ci/.gitlab-bench.yml
+    strategy: depend
+    forward:
+      pipeline_variables: true
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+  parallel:
+    matrix:
+      - BENCH_CFG: [regression_basic]
+
+Bench (Extended):
+  stage: test
+  rules:
+    # Do not run on a schedule
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    # Do not run for PRs to dev (run only for pushes)
+    - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
+      when: never
+    - if: $MANUAL_CFG_PATH == ""
+  trigger:
+    include: ci/.gitlab-bench.yml
+    strategy: depend
+    forward:
+      pipeline_variables: true
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    PARALLEL_JOBS: "4"
+  parallel:
+    matrix:
+      - BENCH_CFG: [regression_extended, microbenchmark_basic]

.pre-commit-config.yaml

@@ -43,6 +43,7 @@ repos:
   - id: check-merge-conflict
   - id: check-xml
   - id: check-yaml
+    args: ['--unsafe']
   - id: debug-statements
     exclude: '^src/finn/builder/build_dataflow.py$'
   - id: end-of-file-fixer

ci/.gitlab-bench.yml

@@ -0,0 +1,81 @@
+include: ci/.gitlab-setup.yml
+
+stages:
+  - build
+  - measure
+  - collect
+
+variables:
+  BENCH_CFG:
+    description: "Select config, usually provided by parent pipeline"
+    value: ""
+
+workflow:
+  name: "bench_$BENCH_CFG"
+
+FINN Build:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: build
+  needs:
+    - job: Build
+      pipeline: $PARENT_PIPELINE_ID
+  variables:
+    SCHEDULER_PARAMETERS: "-A $PROJECT_ACCOUNT -p $SLURM_PARTITION -t $SLURM_TIMEOUT $SLURM_QOS --nodes 1 --ntasks 1 --cpus-per-task $CPU_CORES_BENCH --exclusive --array 0-$( expr $PARALLEL_JOBS - 1 )"
+    NUM_DEFAULT_WORKERS: "$CPU_CORES_BENCH"
+  extends: .setup_full_2022_2
+  script:
+    # Launch additional monitoring
+    - $JOB_MONITORING_DIR/monitor.sh $JOB_MONITORING_DIR/$CI_PIPELINE_ID/$HOSTNAME.log &
+    # Launch benchmarking script via FINN CLI, includes deps update and environment preparation
+    # TODO: cache dvc pull
+    - |
+      source finn-plus-venv/bin/activate
+      dvc pull
+      finn bench --dependency-path ./deps --build-path $FINN_BUILD_DIR --num-workers $CPU_CORES_BENCH --bench_config $BENCH_CFG
+  cache:
+    key: $CI_COMMIT_SHA
+    policy: pull
+    paths:
+      - deps
+  artifacts:
+    name: "build_artifacts"
+    when: always
+    paths:
+      - build_artifacts/
+
+Measurement:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: measure
+  tags:
+    - board
+  rules:
+    # Also run on failure of previous tasks to measure partial results
+    - when: always
+  script:
+    # Run as root and activate the PYNQ venv manually to use PYNQ outside of the typical Jupyter environment
+    - sudo bash -c "source /etc/profile.d/pynq_venv.sh && export XILINX_XRT=/usr && python ci/measure.py"
+  artifacts:
+    name: "measurement_artifacts"
+    when: always
+    paths:
+      - measurement_artifacts/
+
+Result Collection:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: collect
+  tags:
+    - image_build
+  rules:
+    # Also run on failure of previous tasks to collect partial results
+    - when: always
+  script:
+    # pulling models seems to be needed for dvclive to save experiments, even though they are not used or modified
+    - dvc pull
+    - python3.10 ci/collect.py
+    - dvc exp push -f -j 4 -r push git@github.com:eki-project/finn-plus.git

ci/.gitlab-setup.yml

@@ -0,0 +1,49 @@
+# This file defines some basic scripts used to setup the FINN environment on the runner
+
+.n2_setup_general:
+  before_script:
+    - module load lang/Python/3.10.4-GCCcore-11.3.0
+    - module load devel/Autoconf/2.71-GCCcore-11.3.0
+    - module load lang/Bison/3.8.2-GCCcore-11.3.0
+    - module load lang/flex/2.6.4-GCCcore-11.3.0
+    - module load compiler/GCC/11.3.0
+    - module load lib/pybind11/2.9.2-GCCcore-11.3.0
+    - module load devel/Boost/1.79.0-GCC-11.3.0
+    - module load lib/fmt/9.1.0-GCCcore-11.3.0
+    - ulimit -s unlimited # Increase stack size limit
+
+.n2_setup_xilinx_2022_2:
+  before_script:
+    - module load fpga
+    - module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
+    # module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
+    - export PLATFORM_REPO_PATHS=$PLATFORM_PATH
+
+.n2_setup_xilinx_2024_2:
+  before_script:
+    - module load fpga
+    - module load xilinx/xrt/2.14 # includes Vitis/Vivado 2022.2
+    - module swap xilinx/vitis/24.2 # switch to Vitis/Vivado 2024.2
+    # module load will set PLATFORM_REPO_PATHS to one specific platform, revert to top-level PLATFORM_PATH
+    - export PLATFORM_REPO_PATHS=$PLATFORM_PATH
+
+.setup_venv_from_whl:
+  before_script:
+    # Move everything to working directory (e.g., RAMdisk)
+    - cp -dfR . $PATH_WORKDIR
+    - cd $PATH_WORKDIR
+    # Create fresh virtual environment and install finn-plus from .whl (artifact)
+    - python3 -m venv finn-plus-venv
+    - finn-plus-venv/bin/pip install dist/*.whl
+
+.setup_full_2022_2:
+  before_script:
+    - !reference [.n2_setup_general, before_script]
+    - !reference [.n2_setup_xilinx_2022_2, before_script]
+    - !reference [.setup_venv_from_whl, before_script]
+
+.setup_full_2024_2:
+  before_script:
+    - !reference [.n2_setup_general, before_script]
+    - !reference [.n2_setup_xilinx_2024_2, before_script]
+    - !reference [.setup_venv_from_whl, before_script]

ci/cfg/live_fifosizing.yml

@@ -0,0 +1,50 @@
+[
+    # Real models
+    {
+        "dut": ["vgg10"],
+        "live_fifo_sizing": [True],
+        "generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
+    },
+    {
+        "dut": ["mobilenetv1"],
+        "live_fifo_sizing": [True],
+        "generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
+    },
+    {
+        "dut": ["resnet50"],
+        "live_fifo_sizing": [True],
+        "generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
+    },
+
+    # Synthetic non-linear models
+    {
+        "dut": ["synthetic_nonlinear"],
+        "dim": [64],
+        "kernel_size": [5],
+        "ch": [8],
+        "simd": [8],
+        "pe": [8],
+        "parallel_window": [1],
+
+        "lb_num_layers": [1],
+        "rb_num_layers": [4, 8, 16],
+
+        "live_fifo_sizing": [True],
+        "generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
+    },
+    {
+        "dut": ["synthetic_nonlinear"],
+        "dim": [64],
+        "kernel_size": [5],
+        "ch": [8],
+        "simd": [1],
+        "pe": [1],
+        "parallel_window": [0],
+
+        "lb_num_layers": [1],
+        "rb_num_layers": [4, 8, 16],
+
+        "live_fifo_sizing": [True],
+        "generate_outputs": [["bitfile", "pynq_driver", "deployment_package"]]
+    }
+]