From c6a2d7519382e89261cdc29dcb18dffab417502b Mon Sep 17 00:00:00 2001
From: Arseny Kapoulkine <arseny.kapoulkine@gmail.com>
Date: Mon, 17 Oct 2022 10:02:21 -0700
Subject: [PATCH] Improve testing coverage with codegen/O2 in mind (#709)

This change adds codegen runs to coverage config and adds O2/codegen
testing to CI.

Note that we don't run O2 combinations in coverage - it's better that we
see gaps in O2 coverage in compiler tests, as these are valuable for
validating codegen intricacies that are difficult to see from
conformance tests passing/failing.
---
 .github/workflows/build.yml | 38 ++++++++++++++++++++++++++-----------
 Makefile                    | 13 +++++++++----
 2 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 78c3a34..dbd6a49 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -25,12 +25,21 @@ jobs:
     runs-on: ${{matrix.os}}-latest
     steps:
     - uses: actions/checkout@v1
-    - name: make test
+    - name: make tests
       run: |
-        make -j2 config=sanitize werror=1 test
-    - name: make test w/flags
+        make -j2 config=sanitize werror=1 native=1 luau-tests
+    - name: run tests
       run: |
-        make -j2 config=sanitize werror=1 flags=true test
+        ./luau-tests
+        ./luau-tests --fflags=true
+    - name: run extra conformance tests
+      run: |
+        ./luau-tests -ts=Conformance -O2
+        ./luau-tests -ts=Conformance -O2 --fflags=true
+        ./luau-tests -ts=Conformance --codegen
+        ./luau-tests -ts=Conformance --codegen --fflags=true
+        ./luau-tests -ts=Conformance --codegen -O2
+        ./luau-tests -ts=Conformance --codegen -O2 --fflags=true
     - name: make cli
       run: |
          make -j2 config=sanitize werror=1 luau luau-analyze # match config with tests to improve build time
@@ -45,18 +54,25 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: cmake configure
-      run: cmake . -A ${{matrix.arch}} -DLUAU_WERROR=ON
-    - name: cmake test
+      run: cmake . -A ${{matrix.arch}} -DLUAU_WERROR=ON -DLUAU_NATIVE=ON
+    - name: cmake build
+      run: cmake --build . --target Luau.UnitTest Luau.Conformance --config Debug
+    - name: run tests
       shell: bash # necessary for fail-fast
       run: |
-        cmake --build . --target Luau.UnitTest Luau.Conformance --config Debug
         Debug/Luau.UnitTest.exe
         Debug/Luau.Conformance.exe
-    - name: cmake test w/flags
-      shell: bash # necessary for fail-fast
-      run: |
         Debug/Luau.UnitTest.exe --fflags=true
         Debug/Luau.Conformance.exe --fflags=true
+    - name: run extra conformance tests
+      shell: bash # necessary for fail-fast
+      run: |
+        Debug/Luau.Conformance.exe -O2
+        Debug/Luau.Conformance.exe -O2 --fflags=true
+        Debug/Luau.Conformance.exe --codegen
+        Debug/Luau.Conformance.exe --codegen --fflags=true
+        Debug/Luau.Conformance.exe --codegen -O2
+        Debug/Luau.Conformance.exe --codegen -O2 --fflags=true
     - name: cmake cli
       shell: bash # necessary for fail-fast
       run: |
@@ -73,7 +89,7 @@ jobs:
         sudo apt install llvm
     - name: make coverage
       run: |
-        CXX=clang++-10 make -j2 config=coverage coverage
+        CXX=clang++-10 make -j2 config=coverage native=1 coverage
     - name: upload coverage
       uses: codecov/codecov-action@v3
       with:
diff --git a/Makefile b/Makefile
index 2ac4b33..48d399e 100644
--- a/Makefile
+++ b/Makefile
@@ -148,11 +148,16 @@ clean:
 	rm -rf $(EXECUTABLE_ALIASES)
 
 coverage: $(TESTS_TARGET)
-	$(TESTS_TARGET) --fflags=true
-	mv default.profraw default-flags.profraw
 	$(TESTS_TARGET)
-	llvm-profdata merge default.profraw default-flags.profraw -o default.profdata
-	rm default.profraw default-flags.profraw
+	mv default.profraw tests.profraw
+	$(TESTS_TARGET) --fflags=true
+	mv default.profraw tests-flags.profraw
+	$(TESTS_TARGET) -ts=Conformance --codegen
+	mv default.profraw codegen.profraw
+	$(TESTS_TARGET) -ts=Conformance --codegen --fflags=true
+	mv default.profraw codegen-flags.profraw
+	llvm-profdata merge tests.profraw tests-flags.profraw codegen.profraw codegen-flags.profraw -o default.profdata
+	rm *.profraw
 	llvm-cov show -format=html -show-instantiations=false -show-line-counts=true -show-region-summary=false -ignore-filename-regex=\(tests\|extern\|CLI\)/.* -output-dir=coverage --instr-profile default.profdata build/coverage/luau-tests
 	llvm-cov report -ignore-filename-regex=\(tests\|extern\|CLI\)/.* -show-region-summary=false --instr-profile default.profdata build/coverage/luau-tests
 	llvm-cov export -ignore-filename-regex=\(tests\|extern\|CLI\)/.* -format lcov --instr-profile default.profdata build/coverage/luau-tests >coverage.info