From e463bb906cfa86dbd5fe78a34b10551418179c0c Mon Sep 17 00:00:00 2001
From: Tab Atkins-Bittner <jackalmage@gmail.com>
Date: Fri, 6 Dec 2024 14:58:51 -0800
Subject: [PATCH] Add a lint for failing and orphaned tests.

---
 .github/workflows/lint-tests.yml     | 25 ++++++++++++++++++++
 .github/workflows/lint-tests/lint.py | 35 ++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 .github/workflows/lint-tests.yml
 create mode 100644 .github/workflows/lint-tests/lint.py

diff --git a/.github/workflows/lint-tests.yml b/.github/workflows/lint-tests.yml
new file mode 100644
index 0000000..3005211
--- /dev/null
+++ b/.github/workflows/lint-tests.yml
@@ -0,0 +1,25 @@
+name: Lint the test files
+
+on:
+  push:
+    paths:
+      - "tests/**"
+  pull_request:
+    paths:
+      - "tests/**"
+  workflow_dispatch: {}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Verify failing tests and orphaned tests
+      run: |
+        cd tests/test_cases
+        python ../../.github/workflows/lint-tests/lint.py
\ No newline at end of file
diff --git a/.github/workflows/lint-tests/lint.py b/.github/workflows/lint-tests/lint.py
new file mode 100644
index 0000000..d23f6dc
--- /dev/null
+++ b/.github/workflows/lint-tests/lint.py
@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import os
+import sys
+import typing
+
+def findTestFiles(path) -> typing.Generator[str, None, None]:
+    for root, _, filenames in os.walk(path):
+        for filename in filenames:
+            yield os.path.join(root, filename)
+
+# strip the leading folder name, so they can be directly compared
+inputFiles = set(x[len("input")+1:] for x in findTestFiles("input"))
+validFiles = set(x[len("expected_kdl")+1:] for x in findTestFiles("expected_kdl"))
+
+invalidFiles = inputFiles - validFiles
+orphanedFiles = validFiles - inputFiles
+
+SUCCESS = True
+
+if orphanedFiles:
+	SUCCESS = False
+	print("ERROR: There are outputs in /expected_kdl without corresponding tests in /input:\n" + "\n".join(["  "+x for x in orphanedFiles]))
+
+misnamedFiles: list[str] = []
+for filepath in invalidFiles:
+	basepath, ext = os.path.splitext(filepath)
+	if not basepath.endswith("_fail"):
+		misnamedFiles.append(filepath)
+if misnamedFiles:
+	SUCCESS = False
+	print("ERROR: There are tests in /input without corresponding outputs in /expected_kdl, but they don't have a _fail suffix:\n" + "\n".join(["  "+x for x in misnamedFiles]))
+
+if not SUCCESS:
+	sys.exit(1)
\ No newline at end of file