Indicate expected-fail tests with a _fail suffix. Fixes #365 (#418)

* Indicate expected-fail tests with a _fail suffix. Fixes #365

* Update the README with directions for formatting failing tests

* Add a lint for failing and orphaned tests.

.github/workflows/lint-tests.yml

@@ -0,0 +1,25 @@
+name: Lint the test files
+
+on:
+  push:
+    paths:
+      - "tests/**"
+  pull_request:
+    paths:
+      - "tests/**"
+  workflow_dispatch: {}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Verify failing tests and orphaned tests
+      run: |
+        cd tests/test_cases
+        python ../../.github/workflows/lint-tests/lint.py

.github/workflows/lint-tests/lint.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import os
+import sys
+import typing
+
+def findTestFiles(path) -> typing.Generator[str, None, None]:
+    for root, _, filenames in os.walk(path):
+        for filename in filenames:
+            yield os.path.join(root, filename)
+
+# strip the leading folder name, so they can be directly compared
+inputFiles = set(x[len("input")+1:] for x in findTestFiles("input"))
+validFiles = set(x[len("expected_kdl")+1:] for x in findTestFiles("expected_kdl"))
+
+invalidFiles = inputFiles - validFiles
+orphanedFiles = validFiles - inputFiles
+
+SUCCESS = True
+
+if orphanedFiles:
+	SUCCESS = False
+	print("ERROR: There are outputs in /expected_kdl without corresponding tests in /input:\n" + "\n".join(["  "+x for x in orphanedFiles]))
+
+misnamedFiles: list[str] = []
+for filepath in invalidFiles:
+	basepath, ext = os.path.splitext(filepath)
+	if not basepath.endswith("_fail"):
+		misnamedFiles.append(filepath)
+if misnamedFiles:
+	SUCCESS = False
+	print("ERROR: There are tests in /input without corresponding outputs in /expected_kdl, but they don't have a _fail suffix:\n" + "\n".join(["  "+x for x in misnamedFiles]))
+
+if not SUCCESS:
+	sys.exit(1)

tests/README.md

@@ -2,9 +2,11 @@
 
 The `input` folder contains test cases for KDL parsers. The `expected_kdl`
 folder contains files with the same name as those in `input` with the expected
-output after being run through the parser and printed out again. If there's no
-file in `expected_kdl` with a name corresponding to one in `input` it
-indicates that parsing for that case should fail.
+output after being run through the parser and printed out again.
+
+If a testcase is intended to fail parsing,
+the `input` file _MUST_ have a `_fail` suffix,
+and there must be no corresponding file in `expected_kdl`.
 
 ## Translation Rules