Skip to content

Commit

Permalink
tests: Add test harness
Browse files Browse the repository at this point in the history
  • Loading branch information
mohkale committed Apr 10, 2022
1 parent 530e708 commit eb8a7eb
Show file tree
Hide file tree
Showing 7 changed files with 259 additions and 0 deletions.
11 changes: 11 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,17 @@ compile: ## Check for byte-compiler errors
| grep . && exit 1 || true ;\
done

.PHONY: test
test: ## Run all defined test cases.
@echo "[test] Running all test cases"
@docker build -t flymake-collection-test ./tests/checkers
@docker run \
--rm \
--volume "$$(pwd)":/src:ro \
--volume "$$(pwd)/tests/checkers":/test:ro \
flymake-collection-test \
sh -c 'find /test/test-cases/ -iname \*.yml | parallel -I{} chronic /test/run-test-case {}'

.PHONY: clean
clean: ## Remove build artifacts
@echo "[clean]" $(subst .el,.elc,$(SRC))
Expand Down
3 changes: 3 additions & 0 deletions src/flymake-collection-define.el
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,9 @@ exit status %d\nStderr: %s"
;; Finished linting, cleanup any temp-files and then kill
;; the process buffer.
,@cleanup-form
(when (eq (plist-get flymake-collection-define--procs ',name)
,proc-symb)
(cl-remf flymake-collection-define--procs ',name))
(kill-buffer (process-buffer ,proc-symb)))))))
;; Push the new-process to the process to the process alist.
(setq flymake-collection-define--procs
Expand Down
49 changes: 49 additions & 0 deletions tests/checkers/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Ubuntu 20.04 LTS supported until April 2025
FROM ubuntu:20.04

# Suppress some interactive prompts by answering them with environment
# variables.
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=Etc/UTC

# Install emacs. Instructions adapted from [[https://www.masteringemacs.org/article/speed-up-emacs-libjansson-native-elisp-compilation][here]].
WORKDIR /build/emacs
RUN apt-get update && \
apt-get install -y \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common \
libjansson4 \
libjansson-dev \
git && \
git clone -b emacs-28 --single-branch git://git.savannah.gnu.org/emacs.git . && \
sed -i 's/# deb-src/deb-src/' /etc/apt/sources.list && \
apt-get update && \
apt-get build-dep -y emacs && \
./autogen.sh && \
./configure && \
make -j4 && \
make install && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

WORKDIR /build
COPY installers /build/installers
RUN apt-get update && \
apt-get install -y \
parallel python3.8 python3-pip moreutils git && \
python3.8 -m pip install pyyaml && \
find installers/ -type f -iname '*.bash' -exec {} \; && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

# Install the latest available version of flymake for these tests.
RUN cd "$(mktemp -d)" && \
git clone https://github.com/emacs-straight/flymake.git . && \
mkdir /tmp/emacs && \
cp flymake.el /etc/emacs/flymake.el && \
rm -rf "$(pwd)"

WORKDIR /src
Empty file added tests/checkers/README.org
Empty file.
1 change: 1 addition & 0 deletions tests/checkers/installers/pylint.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
python3.8 -m pip install pylint
166 changes: 166 additions & 0 deletions tests/checkers/run-test-case
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
#!/usr/bin/env python3.8
"""Test case runner for flymake-collection.
"""
import json
import logging
import pathlib
import subprocess
import sys
import tempfile
from dataclasses import dataclass
from typing import List, Optional, Tuple

import yaml


@dataclass
class TestLint:
"""A flymake diagnostic."""

point: Tuple[int, int] # Line and column
level: str
message: str


@dataclass
class TestCase:
"""A test case called `name`, given `file` that should give back `lints`."""

name: str
file: str
lints: List[TestLint]

def __post_init__(self):
self.lints = [TestLint(**it) for it in self.lints]

def run(self, checker: str) -> bool:
with tempfile.NamedTemporaryFile("w") as file:
file.write(self.file)
file.flush()
actual_lints = run_flymake(pathlib.Path(file.name), checker)
if actual_lints is None:
return False

failed = False
for lint in self.lints:
try:
pos = actual_lints.index(lint)
except ValueError:
logging.error("Expected to encounter lint: %s", lint)
failed = True
else:
actual_lints.pop(pos)
for lint in actual_lints:
logging.error("Encountered unexpected lint: %s", lint)
failed = True
return not failed


@dataclass
class TestConfig:
checker: str
tests: List[TestCase]

def __post_init__(self):
self.tests = [TestCase(**it) for it in self.tests]


def run_flymake(src: pathlib.Path, checker: str) -> Optional[List[TestLint]]:
with tempfile.NamedTemporaryFile("w") as script, tempfile.NamedTemporaryFile(
"r"
) as out:
script.write(
f"""
(require 'flymake)
(require 'json)
(add-to-list 'load-path "/src/src")
(add-to-list 'load-path "/src/src/checkers")
(require (intern "{checker}") "{checker}.el")
(setq src (find-file-literally "{src}")
out (find-file "{out.name}"))
(defun column-number (point)
"Returns the column number at POINT."
(interactive)
(save-excursion
(goto-char point)
(current-column)))
(with-current-buffer src
({checker}
(lambda (diags)
(with-current-buffer out
(cl-loop for diag in diags
collect
(insert
(json-encode
`((point . ,(with-current-buffer src
(let ((beg (flymake--diag-beg diag)))
(list (line-number-at-pos beg)
(column-number beg)))))
(level . ,(flymake--diag-type diag))
(message . ,(substring-no-properties (flymake--diag-text diag)))))
"\n"))
(save-buffer))))
;; Block until the checker process finishes.
(while flymake-collection-define--procs
(sleep-for 0.25)))
"""
)
script.flush()
proc = subprocess.run(
["emacs", "-Q", "--script", script.name],
capture_output=True,
encoding="utf-8",
)
if proc.returncode != 0:
logging.error("Failed to run checker using emacs")
logging.error("Emacs exited with stderr: %s", proc.stderr)
return None

lints = []
for line in out:
if line.strip() == "":
continue
lints.append(TestLint(**json.loads(line)))
return lints


def main(args, vargs, parser) -> bool:
failed = False
logging.info("Loading test config file=%s", args.test)
with args.test.open("r") as test_file:
cfg_obj = yaml.load(test_file, Loader=yaml.SafeLoader)
try:
cfg = TestConfig(**cfg_obj)
except ValueError:
logging.exception("Failed to read test configuration")
return False

logging.info("Running tests with checker=%s", cfg.checker)
for i, test in enumerate(cfg.tests):
logging.info("Running test case %d name=%s", i, test.name)
if not test.run(cfg.checker):
failed = True

return not failed


if __name__ == "__main__":
import argparse

parser = argparse.ArgumentParser()

parser.add_argument(
"test", type=pathlib.Path, help="Path to test cases config file"
)

args = parser.parse_args()
vargs = vars(args)

logging.basicConfig(level=logging.DEBUG)

sys.exit(0 if main(args, vargs, parser) else 1)
29 changes: 29 additions & 0 deletions tests/checkers/test-cases/pylint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
checker: flymake-collection-pylint
tests:
- name: no-lints
file: |
"""A test case with no output from pylint."""
print("hello world")
lints: []
- name: notes
file: |
"""A test case with a warning lint."""
print(f"hello world")
print(f"hello world")
lints:
- point: [3, 5]
level: warning
message: W1309 Using an f-string that does not have any interpolated variables (pylint)
- point: [4, 5]
level: warning
message: W1309 Using an f-string that does not have any interpolated variables (pylint)
- name: syntax-error
file: |
definitely should not work
lints:
- point: [1, 11]
level: error
message: E0001 invalid syntax (<unknown>, line 1) (pylint)

0 comments on commit eb8a7eb

Please sign in to comment.