feat: add test_helpers module (error_utils, test_utils) behind test_utlis flag

.github/workflows/rust.yml

@@ -18,6 +18,7 @@ env:
     cairo_programs/**/*.json
     !cairo_programs/manually_compiled/*
     cairo_programs/cairo-1-programs/bitwise.sierra
+    vm/src/tests/cairo_test_suite/**/*.json
   TEST_COLLECT_COVERAGE: 1
   PROPTEST_CASES: 100
 
@@ -46,6 +47,7 @@ jobs:
         - cairo_bench_programs
         - cairo_proof_programs
         - cairo_test_programs
+        - cairo_test_suite_programs
         - cairo_1_test_contracts
         - cairo_2_test_contracts
     name: Build Cairo programs
@@ -66,7 +68,7 @@ jobs:
       id: cache-programs
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
 
     # This is not pretty, but we need `make` to see the compiled programs are
     # actually newer than the sources, otherwise it will try to rebuild them
@@ -120,37 +122,43 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: cairo_test_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: cairo_test_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
     - name: Fetch proof programs
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
     - name: Fetch bench programs
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: cairo_bench_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: cairo_bench_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        fail-on-cache-miss: true
+    - name: Fetch cairo test suite programs
+      uses: actions/cache/restore@v3
+      with:
+        path: ${{ env.CAIRO_PROGRAMS_PATH }}
+        key: cairo_test_suite_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
     - name: Fetch test contracts (Cairo 1)
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: cairo_1_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: cairo_1_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
     - name: Fetch test contracts (Cairo 2)
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: cairo_2_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: cairo_2_test_contracts-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
     - name: Merge caches
       uses: actions/cache/save@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
 
   lint:
     needs: merge-caches
@@ -179,7 +187,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     - name: Run clippy
@@ -229,7 +237,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     # NOTE: we do this separately because --workspace operates in weird ways
@@ -274,7 +282,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     - name: Check all features (workspace)
@@ -309,7 +317,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: all-programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     - name: Install testing tools
@@ -409,7 +417,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     # This is not pretty, but we need `make` to see the compiled programs are
@@ -456,7 +464,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: ${{ matrix.program-target }}-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     - name: Generate traces
@@ -675,7 +683,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     - name: Run script
@@ -715,7 +723,7 @@ jobs:
       uses: actions/cache/restore@v3
       with:
         path: ${{ env.CAIRO_PROGRAMS_PATH }}
-        key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'Makefile', 'requirements.txt') }}
+        key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'vm/src/tests/cairo_test_suite/**/*.cairo', 'Makefile', 'requirements.txt') }}
         fail-on-cache-miss: true
 
     - name: Fetch pie

.gitignore

@@ -4,6 +4,7 @@
 **/*.json
 !hint_accountant/whitelists/*.json
 !cairo_programs/manually_compiled/*.json
+!vm/src/test_helpers/dummy.json
 **/*.casm
 **/*.sierra
 **/*.trace

CHANGELOG.md

@@ -11,6 +11,9 @@ Both branches support Stwo prover opcodes (Blake2s, QM31) since v2.0.0.
 ---
 
 #### Upcoming Changes
+* feat: add `test_helpers` module (`error_utils`, `test_utils`) with `assert_mr_eq`, `load_cairo_program!` macro and `expect_*` error checkers, behind `test_utils` feature flag [#2381](https://github.com/starkware-libs/cairo-vm/pull/2381)
+
+* feat(makefile,ci): add `cairo_test_suite_programs` Makefile target and CI integration to compile Cairo test suite programs before running tests [#2380](https://github.com/starkware-libs/cairo-vm/pull/2380)
 
 * Add Stwo cairo runner API [#2351](https://github.com/starkware-libs/cairo-vm/pull/2351)
 * feat: refactor CairoRunner ctors to accept CairoLayout directly [#2363](https://github.com/starkware-libs/cairo-vm/pull/2363)

Makefile

@@ -21,7 +21,7 @@ UNAME := $(shell uname)
 	compare_trace_memory compare_trace compare_memory compare_pie compare_all_no_proof \
 	compare_trace_memory_proof  compare_all_proof compare_trace_proof compare_memory_proof compare_air_public_input  compare_air_private_input\
 	hyper-threading-benchmarks \
-	cairo_bench_programs cairo_proof_programs cairo_test_programs cairo_1_test_contracts cairo_2_test_contracts \
+	cairo_bench_programs cairo_proof_programs cairo_test_programs cairo_test_suite_programs cairo_1_test_contracts cairo_2_test_contracts \
 	cairo_trace cairo-vm_trace cairo_proof_trace cairo-vm_proof_trace python-deps python-deps-macos \
 	build-cairo-lang hint-accountant \ create-proof-programs-symlinks \
 	$(RELBIN) $(DBGBIN)
@@ -124,6 +124,10 @@ NORETROCOMPAT_DIR:=cairo_programs/noretrocompat
 NORETROCOMPAT_FILES:=$(wildcard $(NORETROCOMPAT_DIR)/*.cairo)
 COMPILED_NORETROCOMPAT_TESTS:=$(patsubst $(NORETROCOMPAT_DIR)/%.cairo, $(NORETROCOMPAT_DIR)/%.json, $(NORETROCOMPAT_FILES))
 
+CAIRO_TEST_SUITE_ROOT:=vm/src/tests/cairo_test_suite
+CAIRO_TEST_SUITE_FILES:=$(shell find $(CAIRO_TEST_SUITE_ROOT) -name "*.cairo")
+COMPILED_CAIRO_TEST_SUITE:=$(patsubst %.cairo,%.json,$(CAIRO_TEST_SUITE_FILES))
+
 $(BENCH_DIR)/%.json: $(BENCH_DIR)/%.cairo
 	cairo-compile --cairo_path="$(TEST_DIR):$(BENCH_DIR)" $< --output $@ --proof_mode
 
@@ -145,6 +149,9 @@ $(BAD_TEST_DIR)/%.json: $(BAD_TEST_DIR)/%.cairo
 $(PRINT_TEST_DIR)/%.json: $(PRINT_TEST_DIR)/%.cairo
 	cairo-compile $< --output $@
 
+$(CAIRO_TEST_SUITE_ROOT)/%.json: $(CAIRO_TEST_SUITE_ROOT)/%.cairo
+	cairo-compile $< --output $@
+
 # ======================
 # Test Cairo 1 Contracts
 # ======================
@@ -268,6 +275,8 @@ run:
 check:
 	cargo check
 
+cairo_test_suite_programs: $(COMPILED_CAIRO_TEST_SUITE)
+
 cairo_test_programs: $(COMPILED_TESTS) $(COMPILED_BAD_TESTS) $(COMPILED_NORETROCOMPAT_TESTS) $(COMPILED_PRINT_TESTS) $(COMPILED_MOD_BUILTIN_TESTS) $(COMPILED_SECP_CAIRO0_HINTS) $(COMPILED_KZG_DA_CAIRO0_HINTS) $(COMPILED_SEGMENT_ARENA_CAIRO0_HINTS)
 cairo_proof_programs: $(COMPILED_PROOF_TESTS) $(COMPILED_MOD_BUILTIN_PROOF_TESTS) $(COMPILED_STWO_EXCLUSIVE_TESTS)
 cairo_bench_programs: $(COMPILED_BENCHES)
@@ -286,7 +295,7 @@ ifdef TEST_COLLECT_COVERAGE
 	TEST_COMMAND:=cargo llvm-cov nextest --no-report
 endif
 
-test: cairo_proof_programs cairo_test_programs cairo_1_test_contracts cairo_2_test_contracts cairo_1_program
+test: cairo_proof_programs cairo_test_programs cairo_test_suite_programs cairo_1_test_contracts cairo_2_test_contracts cairo_1_program
 	$(TEST_COMMAND) --workspace --features "test_utils, cairo-1-hints"
 test-extensive_hints: cairo_proof_programs cairo_test_programs cairo_1_test_contracts cairo_1_program cairo_2_test_contracts 
 	$(TEST_COMMAND) --workspace --features "test_utils, cairo-1-hints, cairo-0-secp-hints, cairo-0-data-availability-hints, extensive_hints"

vm/src/lib.rs

@@ -26,6 +26,9 @@ pub mod types;
 pub mod utils;
 pub mod vm;
 
+#[cfg(feature = "test_utils")]
+pub mod test_helpers;
+
 // TODO: use `Felt` directly
 pub use starknet_types_core::felt::Felt as Felt252;
 

vm/src/test_helpers/dummy.json

@@ -0,0 +1,69 @@
+{
+    "attributes": [],
+    "builtins": [],
+    "compiler_version": "0.13.5",
+    "data": [
+        "0x208b7fff7fff7ffe"
+    ],
+    "debug_info": {
+        "file_contents": {},
+        "instruction_locations": {
+            "0": {
+                "accessible_scopes": [
+                    "__main__",
+                    "__main__.main"
+                ],
+                "flow_tracking_data": {
+                    "ap_tracking": {
+                        "group": 0,
+                        "offset": 0
+                    },
+                    "reference_ids": {}
+                },
+                "hints": [],
+                "inst": {
+                    "end_col": 15,
+                    "end_line": 2,
+                    "input_file": {
+                        "filename": "vm/src/test_helpers/dummy.cairo"
+                    },
+                    "start_col": 5,
+                    "start_line": 2
+                }
+            }
+        }
+    },
+    "hints": {},
+    "identifiers": {
+        "__main__.main": {
+            "decorators": [],
+            "pc": 0,
+            "type": "function"
+        },
+        "__main__.main.Args": {
+            "full_name": "__main__.main.Args",
+            "members": {},
+            "size": 0,
+            "type": "struct"
+        },
+        "__main__.main.ImplicitArgs": {
+            "full_name": "__main__.main.ImplicitArgs",
+            "members": {},
+            "size": 0,
+            "type": "struct"
+        },
+        "__main__.main.Return": {
+            "cairo_type": "()",
+            "type": "type_definition"
+        },
+        "__main__.main.SIZEOF_LOCALS": {
+            "type": "const",
+            "value": 0
+        }
+    },
+    "main_scope": "__main__",
+    "prime": "0x800000000000011000000000000000000000000000000000000000000000001",
+    "reference_manager": {
+        "references": []
+    }
+}