test: data-driven validation tests (#556)

Closes partially #543.
Closes partially #433.

### Summary of Changes

* Port the data-driven framework for validation tests from the old Xtext
implementation to Langium.
* Document how to write validation tests.

---------

Co-authored-by: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com>

docs/development/validation-testing.md

@@ -0,0 +1,59 @@
+# Validation Testing
+
+Validation tests are data-driven instead of being specified explicitly. This document explains how to add a new
+validation test.
+
+## Adding a validation test
+
+1. Create a new **folder** (not just a file!) in the `tests/resources/validation` directory or any subdirectory. Give
+   the folder a descriptive name, since the folder name becomes part of the test name.
+
+    !!! tip "Skipping a test"
+
+        If you want to skip a test, add the prefix `skip-` to the folder name.
+
+2. Add files with the extension `.sdstest`, `.sdspipe`, or `.sdsstub` **directly inside the folder**. All files in a
+   folder will be loaded into the same workspace, so they can reference each other. Files in different folders are
+   loaded into different workspaces, so they cannot reference each other.
+3. Add the Safe-DS code that you want to test to the file.
+4. Specify the expected validation results using test comments (see [below](#format-of-test-comments)) and test
+   markers (e.g. `fun »F«()`). The test comments are used to specify
+    * the presence or absence of an issue,
+    * the severity of the issue, and
+    * the message of the issue.
+
+    The test markers are used to specify the location of the issue. Test comments and test markers are mapped to each
+    other by their position in the file, i.e. the first test comment corresponds to the first test marker, the second
+    test comment corresponds to the second test marker, etc. There may be more test comments than test markers, but not
+    the other way around. Any additional test comments are applied to the entire file.
+
+5. Run the tests. The test runner will automatically pick up the new test.
+
+## Format of test comments
+
+1. As usual, test comments are single-line comments that start with `$TEST$`.
+2. Then, you specify whether the issue should be absent by writing `no` or present by writing nothing.
+3. Next, you specify the severity of the issue by writing `error`, `warning`, `info`, or `hint`.
+4. Finally, you can optionally specify the message of the issue enclosed in double-quotes. You can also add an `r`
+   before the opening double-quote to indicate that the expected message should be interpreted as a regular expression
+   that must match the entire actual message.
+
+Here are some examples:
+
+```ts
+// $TEST$ error "Incompatible type."
+```
+
+We expect an error with the exact message `Incompatible type.`.
+
+```ts
+// $TEST$ no warning "Name should be lowerCamelCase."
+```
+
+We expect no warning with the exact message `Name should be lowerCamelCase.`.
+
+```ts
+// $TEST$ info r".*empty.*"
+```
+
+We expect an info with a message that matches the regular expression `.*empty.*`.

mkdocs.yml

@@ -39,6 +39,7 @@ nav:
       - Grammar Testing: development/grammar-testing.md
       - Scoping Testing: development/scoping-testing.md
       - Formatting Testing: development/formatting-testing.md
+      - Validation Testing: development/validation-testing.md
       - Langium Quickstart: development/langium-quickstart.md
 
 # Configuration of MkDocs & Material for MkDocs --------------------------------

src/language/constant/fileExtensions.ts

@@ -25,6 +25,11 @@ export const STUB_FILE_EXTENSION = 'sdsstub';
  */
 export const TEST_FILE_EXTENSION = 'sdstest';
 
+/**
+ * All file extensions that are supported by the Safe-DS language.
+ */
+export const SAFE_DS_FILE_EXTENSIONS = [PIPELINE_FILE_EXTENSION, STUB_FILE_EXTENSION, TEST_FILE_EXTENSION];
+
 /**
  * All file extensions that are supported by the Safe-DS language.
  */

src/language/validation/nameConvention.ts

@@ -0,0 +1,124 @@
+import { SdsDeclaration } from '../generated/ast.js';
+import { ValidationAcceptor } from 'langium';
+
+const blockLambdaPrefix = '__block_lambda_';
+
+export const nameMustNotStartWithBlockLambdaPrefix = (node: SdsDeclaration, accept: ValidationAcceptor) => {
+    if (node.name.startsWith(blockLambdaPrefix)) {
+        accept(
+            'error',
+            "Names of declarations must not start with '__block_lambda_'. This is reserved for code generation of block lambdas.",
+            {
+                node,
+                property: 'name',
+                code: 'nameConvention/blockLambdaPrefix',
+            },
+        );
+    }
+};
+
+export const nameShouldHaveCorrectCasing = (node: SdsDeclaration, accept: ValidationAcceptor) => {
+    switch (node.$type) {
+        case 'SdsAnnotation':
+            if (!isUpperCamelCase(node.name)) {
+                acceptCasingWarning(node, 'annotations', 'UpperCamelCase', accept);
+            }
+            return;
+        case 'SdsAttribute':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'attributes', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsBlockLambdaResult':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'block lambda results', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsClass':
+            if (!isUpperCamelCase(node.name)) {
+                acceptCasingWarning(node, 'classes', 'UpperCamelCase', accept);
+            }
+            return;
+        case 'SdsEnum':
+            if (!isUpperCamelCase(node.name)) {
+                acceptCasingWarning(node, 'enums', 'UpperCamelCase', accept);
+            }
+            return;
+        case 'SdsEnumVariant':
+            if (!isUpperCamelCase(node.name)) {
+                acceptCasingWarning(node, 'enum variants', 'UpperCamelCase', accept);
+            }
+            return;
+        case 'SdsFunction':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'functions', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsModule':
+            const segments = node.name.split('.');
+            if (!segments.every(isLowerCamelCase)) {
+                accept('warning', 'All segments of the qualified name of a package should be lowerCamelCase.', {
+                    node,
+                    property: 'name',
+                    code: 'nameConvention/casing',
+                });
+            }
+            return;
+        case 'SdsParameter':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'parameters', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsPipeline':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'pipelines', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsPlaceholder':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'placeholders', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsResult':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'results', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsSchema':
+            if (!isUpperCamelCase(node.name)) {
+                acceptCasingWarning(node, 'schemas', 'UpperCamelCase', accept);
+            }
+            return;
+        case 'SdsSegment':
+            if (!isLowerCamelCase(node.name)) {
+                acceptCasingWarning(node, 'segments', 'lowerCamelCase', accept);
+            }
+            return;
+        case 'SdsTypeParameter':
+            if (!isUpperCamelCase(node.name)) {
+                acceptCasingWarning(node, 'type parameters', 'UpperCamelCase', accept);
+            }
+            return;
+    }
+};
+
+const isLowerCamelCase = (name: string): boolean => {
+    return /^[a-z][a-zA-Z0-9]*$/gu.test(name);
+};
+
+const isUpperCamelCase = (name: string): boolean => {
+    return /^[A-Z][a-zA-Z0-9]*$/gu.test(name);
+};
+
+const acceptCasingWarning = (
+    node: SdsDeclaration,
+    nodeName: string,
+    expectedCasing: string,
+    accept: ValidationAcceptor,
+) => {
+    accept('warning', `Names of ${nodeName} should be ${expectedCasing}.`, {
+        node,
+        property: 'name',
+        code: 'nameConvention/casing',
+    });
+};

src/language/validation/safe-ds-validator.ts

@@ -1,6 +1,7 @@
 import { ValidationChecks } from 'langium';
 import { SafeDsAstType } from '../generated/ast.js';
 import type { SafeDsServices } from '../safe-ds-module.js';
+import { nameMustNotStartWithBlockLambdaPrefix, nameShouldHaveCorrectCasing } from './nameConvention.js';
 
 /**
  * Register custom validation checks.
@@ -9,21 +10,12 @@ export const registerValidationChecks = function (services: SafeDsServices) {
     const registry = services.validation.ValidationRegistry;
     const validator = services.validation.SafeDsValidator;
     const checks: ValidationChecks<SafeDsAstType> = {
-        // Person: validator.checkPersonStartsWithCapital
+        SdsDeclaration: [nameMustNotStartWithBlockLambdaPrefix, nameShouldHaveCorrectCasing],
     };
     registry.register(checks, validator);
 };
 
 /**
  * Implementation of custom validations.
  */
-export class SafeDsValidator {
-    // checkPersonStartsWithCapital(person: Person, accept: ValidationAcceptor): void {
-    //     if (person.name) {
-    //         const firstChar = person.name.substring(0, 1);
-    //         if (firstChar.toUpperCase() !== firstChar) {
-    //             accept('warning', 'Person name should start with a capital.', { node: person, property: 'name' });
-    //         }
-    //     }
-    // }
-}
+export class SafeDsValidator {}

tests/helpers/diagnostics.ts

@@ -0,0 +1,30 @@
+import { validationHelper } from 'langium/test';
+import { LangiumServices } from 'langium';
+import { Diagnostic, DiagnosticSeverity } from 'vscode-languageserver-types';
+
+/**
+ * Get syntax errors from a code snippet.
+ *
+ * @param services The language services.
+ * @param code The code snippet to check.
+ * @returns The syntax errors.
+ */
+export const getSyntaxErrors = async (services: LangiumServices, code: string): Promise<Diagnostic[]> => {
+    const validationResult = await validationHelper(services)(code);
+    return validationResult.diagnostics.filter(
+        (d) =>
+            d.severity === DiagnosticSeverity.Error &&
+            (d.data?.code === 'lexing-error' || d.data?.code === 'parsing-error'),
+    );
+};
+
+/**
+ * The code contains syntax errors.
+ */
+export class SyntaxErrorsInCodeError extends Error {
+    constructor(readonly syntaxErrors: Diagnostic[]) {
+        const syntaxErrorsAsString = syntaxErrors.map((e) => `- ${e.message}`).join(`\n`);
+
+        super(`Code has syntax errors:\n${syntaxErrorsAsString}`);
+    }
+}

tests/helpers/testResources.test.ts

@@ -1,19 +1,56 @@
 import { describe, expect, it } from 'vitest';
-import { listTestResources } from './testResources.js';
+import { listTestResources, listTestsResourcesGroupedByParentDirectory } from './testResources.js';
 
 describe('listTestResources', () => {
     it('should yield all Safe-DS files in a directory that are not skipped', () => {
-        const result = listTestResources('helpers/listTestResources')
-            .map((path) => path.replace(/\\/gu, '/'))
-            .sort();
+        const result = listTestResources('helpers/listTestResources');
         const expected = [
             'pipeline file.sdspipe',
             'stub file.sdsstub',
             'test file.sdstest',
             'nested/pipeline file.sdspipe',
             'nested/stub file.sdsstub',
             'nested/test file.sdstest',
-        ].sort();
-        expect(result).toStrictEqual(expected);
+        ];
+        expect(normalizePaths(result)).toStrictEqual(normalizePaths(expected));
     });
 });
+
+describe('listTestResourcesGroupedByParentDirectory', () => {
+    it('should yield all Safe-DS files in a directory that are not skipped and group them by parent directory', () => {
+        const result = listTestsResourcesGroupedByParentDirectory('helpers/listTestResources');
+
+        const keys = Object.keys(result);
+        expect(normalizePaths(keys)).toStrictEqual(normalizePaths(['.', 'nested']));
+
+        const directlyInRoot = result['.'];
+        expect(normalizePaths(directlyInRoot)).toStrictEqual(
+            normalizePaths(['pipeline file.sdspipe', 'stub file.sdsstub', 'test file.sdstest']),
+        );
+
+        const inNested = result.nested;
+        expect(normalizePaths(inNested)).toStrictEqual(
+            normalizePaths(['nested/pipeline file.sdspipe', 'nested/stub file.sdsstub', 'nested/test file.sdstest']),
+        );
+    });
+});
+
+/**
+ * Normalizes the given paths by replacing backslashes with slashes and sorting them.
+ *
+ * @param paths The paths to normalize.
+ * @return The normalized paths.
+ */
+const normalizePaths = (paths: string[]): string[] => {
+    return paths.map(normalizePath).sort();
+};
+
+/**
+ * Normalizes the given path by replacing backslashes with slashes.
+ *
+ * @param path The path to normalize.
+ * @return The normalized path.
+ */
+const normalizePath = (path: string): string => {
+    return path.replace(/\\/gu, '/');
+};

tests/helpers/testResources.ts

@@ -1,10 +1,7 @@
 import path from 'path';
 import { globSync } from 'glob';
-import {
-    PIPELINE_FILE_EXTENSION,
-    STUB_FILE_EXTENSION,
-    TEST_FILE_EXTENSION,
-} from '../../src/language/constant/fileExtensions.js';
+import { SAFE_DS_FILE_EXTENSIONS } from '../../src/language/constant/fileExtensions.js';
+import { group } from 'radash';
 
 const resourcesPath = path.join(__dirname, '..', 'resources');
 
@@ -19,20 +16,32 @@ export const resolvePathRelativeToResources = (pathRelativeToResources: string)
 };
 
 /**
- * Lists all Safe-DS files in the given directory relative to `tests/resources/` except those that have a name starting
- * with 'skip'.
+ * Lists all Safe-DS files in the given directory relative to `tests/resources/` that are not skipped.
  *
  * @param pathRelativeToResources The root directory relative to `tests/resources/`.
  * @return Paths to the Safe-DS files relative to `pathRelativeToResources`.
  */
 export const listTestResources = (pathRelativeToResources: string): string[] => {
-    const fileExtensions = [PIPELINE_FILE_EXTENSION, STUB_FILE_EXTENSION, TEST_FILE_EXTENSION];
-    const pattern = `**/*.{${fileExtensions.join(',')}}`;
+    const pattern = `**/*.{${SAFE_DS_FILE_EXTENSIONS.join(',')}}`;
     const cwd = resolvePathRelativeToResources(pathRelativeToResources);
 
     return globSync(pattern, { cwd, nodir: true }).filter(isNotSkipped);
 };
 
+/**
+ * Lists all Safe-DS files in the given directory relative to `tests/resources/` that are not skipped. The result is
+ * grouped by the parent directory.
+ *
+ * @param pathRelativeToResources The root directory relative to `tests/resources/`.
+ * @return Paths to the Safe-DS files relative to `pathRelativeToResources` grouped by the parent directory.
+ */
+export const listTestsResourcesGroupedByParentDirectory = (
+    pathRelativeToResources: string,
+): Record<string, string[]> => {
+    const paths = listTestResources(pathRelativeToResources);
+    return group(paths, (p) => path.dirname(p)) as Record<string, string[]>;
+};
+
 const isNotSkipped = (pathRelativeToResources: string) => {
     const segments = pathRelativeToResources.split(path.sep);
     return !segments.some((segment) => segment.startsWith('skip'));