#487 Implement new sample override

bapctools/export.py

@@ -100,10 +100,13 @@ def build_samples_zip(problems: list[Problem], output: Path, languages: list[str
         contents: dict[Path, Path] = {}  # Maps desination to source, to allow checking duplicates.
 
         # Add samples.
-        samples = problem.download_samples()
-        for i, (in_file, ans_file) in enumerate(samples):
+        samples = problem.samples()
+        for i, sample in enumerate(samples):
+            in_file, ans_file = sample.download
             base_name = outputdir / str(i + 1)
             contents[base_name.with_suffix(".in")] = in_file
+            if in_file.stat().st_size > 0:
+                contents[base_name.with_suffix(".in")] = in_file
             if ans_file.stat().st_size > 0:
                 contents[base_name.with_suffix(".ans")] = ans_file
 
@@ -211,11 +214,11 @@ def add_testcase(in_file: Path) -> None:
                 add_file(f.relative_to(problem.path), f)
 
     # Include all sample test cases and copy all related files.
-    samples = problem.download_samples()
+    samples = problem.samples()
     if len(samples) == 0:
         bar.error("No samples found.")
-    for in_file, _ in samples:
-        add_testcase(in_file)
+    for sample in samples:
+        add_testcase(sample.download[0])
 
     # Include all secret test cases and copy all related files.
     pattern = "data/secret/**/*.in"
@@ -348,10 +351,12 @@ def add_testcase(in_file: Path) -> None:
 
         # The downloadable samples should be copied to attachments/.
         if problem.interactive or problem.multi_pass:
-            samples = problem.download_samples()
-            for i, (in_file, ans_file) in enumerate(samples):
+            samples = problem.samples()
+            for i, sample in enumerate(samples):
+                in_file, ans_file = sample.download
                 base_name = export_dir / "attachments" / str(i + 1)
-                add_file(base_name.with_suffix(".in"), in_file)
+                if in_file.stat().st_size > 0:
+                    add_file(base_name.with_suffix(".in"), in_file)
                 if ans_file.stat().st_size > 0:
                     add_file(base_name.with_suffix(".ans"), ans_file)
 

bapctools/generate.py

@@ -329,6 +329,12 @@ def default_solution_path(generator_config: "GeneratorConfig") -> Path:
     *UNIQUE_TESTCASE_KEYS,
 )
 UNIQUE_DIRECTORY_KEYS: Final[Sequence[str]] = ("data", "test_group.yaml", "include")
+ALLOWED_LINK_KEYS: Final[Sequence[str]] = (
+    ".in.statement",
+    ".ans.statement",
+    ".in.download",
+    ".ans.download",
+)
 KNOWN_DIRECTORY_KEYS: Final[Sequence[str]] = (
     "type",
     "solution",
@@ -461,6 +467,8 @@ def __init__(
         self.copy = None
         # 3. Hardcoded cases where the source is in the yaml file itself.
         self.hardcoded = dict[str, str]()
+        # 4. Linked files belonging to the same testcase.
+        self.linked = dict[str, str]()
         # map of ext to list of patterns used to check the generated testcase.<ext>
         self.patterns = collections.defaultdict[str, list[re.Pattern[str]]](list)
 
@@ -598,20 +606,41 @@ def __init__(
                         if self.copy.with_suffix(ext).is_file():
                             hashes[ext] = hash_file_content(self.copy.with_suffix(ext))
 
-                # 3. hardcoded strings (or, for the Test Case Configuration, a yaml mapping)
+                # 3./4. hardcoded data or link to another file
                 for ext in config.KNOWN_TEXT_DATA_EXTENSIONS:
                     if ext[1:] in yaml:
                         value = yaml[ext[1:]]
                         if ext == ".yaml":
-                            assert_type(ext, value, dict)
-                            value = write_yaml(value)
-                            assert value is not None
+                            # yaml can only be hardcoded (convert dict -> str)
+                            if not isinstance(value, str):
+                                value = write_yaml(value)
+                                assert value is not None
+                        if isinstance(value, dict) and ext in ALLOWED_LINK_KEYS:
+                            # 4. linked
+                            if "link" not in value or len(value) != 1:
+                                raise ParseException(
+                                    f"{ext} should either be a string or a map with only the entry link."
+                                )
+                            value = value["link"]
+                            assert_type(f"{ext}.link", value, str)
+                            if value not in config.KNOWN_TEXT_DATA_EXTENSIONS:
+                                raise ParseException(f"Unknown value for for key {ext}.link.")
+                            assert isinstance(value, str)
+                            self.linked[ext] = value
                         else:
+                            # 3. hardcoded
                             assert_type(ext, value, str)
-                        assert isinstance(value, str)
-                        if len(value) > 0 and value[-1] != "\n":
-                            value += "\n"
-                        self.hardcoded[ext] = value
+                            assert isinstance(value, str)
+                            if len(value) > 0 and value[-1] != "\n":
+                                value += "\n"
+                            self.hardcoded[ext] = value
+
+                for link, target in self.linked.items():
+                    # do not allow links to other links to avoid cycles
+                    if target in self.linked:
+                        raise ParseException(
+                            f"link from {link}->{target} is forbidden, {target} is also a link!"
+                        )
 
                 if ".in" in self.hardcoded:
                     self.in_is_generated = False
@@ -712,6 +741,8 @@ def link(
     ) -> None:
         assert t.process
 
+        identical_exts = set()
+
         src_dir = problem.path / "data" / t.path.parent
         src = src_dir / (t.name + ".in")
 
@@ -721,10 +752,10 @@ def link(
 
             if source.is_file() and source in generator_config.known_files:
                 generator_config.known_files.add(target)
-                if target.is_file():
+                if target.exists() or target.is_symlink():
                     if target.is_symlink() and target.resolve() == source.resolve():
                         # identical -> skip
-                        pass
+                        identical_exts.add(ext)
                     else:
                         # different -> overwrite
                         generator_config.remove(target)
@@ -734,7 +765,19 @@ def link(
                     # new file -> copy it
                     ensure_symlink(target, source, relative=True)
                     bar.log(f"NEW: {target.name}")
-            elif target.is_file():
+            elif target.exists() or target.is_symlink():
+                if (
+                    config.args.no_visualizer
+                    and ext in config.KNOWN_VISUALIZER_EXTENSIONS
+                    and ".in" in identical_exts
+                    and ".ans" in identical_exts
+                ):
+                    # When running with --no-visualizer and .in/.ans files did not change,
+                    # do not remove output of visualizer.
+                    # This is useful for when a user/CI has a clean cache (e.g. after a reboot).
+                    # Also add target to known_files, so the cleanup step does not remove it.
+                    generator_config.known_files.add(target)
+                    continue
                 # Target exists but source wasn't generated -> remove it
                 generator_config.remove(target)
                 bar.log(f"REMOVED: {target.name}")
@@ -983,6 +1026,7 @@ def generate_from_rule() -> bool:
                         bar.warn(f"No files copied from {t.copy}.")
 
                 # Step 3: Write hardcoded files.
+                # Note: we cannot generate links yet, since files like .ans are not yet generated
                 for ext, contents in t.hardcoded.items():
                     # substitute in contents? -> No!
                     infile.with_suffix(ext).write_text(contents)
@@ -1041,7 +1085,7 @@ def check_match(testcase: Testcase, ext: str, bar: ProgressBar) -> None:
             if updated:
                 meta_yaml.write()
 
-        def generate_from_solution(testcase: Testcase, bar: ProgressBar) -> bool:
+        def generate_from_solution(testcase: Testcase) -> bool:
             nonlocal meta_yaml
 
             if testcase.root in [
@@ -1126,7 +1170,7 @@ def needed(
             assert ansfile.is_file(), f"Failed to generate ans file: {ansfile}"
             return True
 
-        def generate_visualization(testcase: Testcase, bar: ProgressBar) -> bool:
+        def generate_visualization(testcase: Testcase) -> bool:
             nonlocal meta_yaml
 
             if testcase.root in config.INVALID_CASE_DIRECTORIES:
@@ -1244,17 +1288,59 @@ def generate_empty_interactive_sample_ans() -> bool:
                     return True
             return True
 
+        def warn_override() -> None:
+            def find_override(*exts: str) -> list[str]:
+                found = [
+                    ext for ext in exts if infile.with_suffix(ext).is_file() or ext in t.linked
+                ]
+                if len(found) > 1:
+                    bar.warn(f"There should be at most one of {', '.join(found)}")
+                return found
+
+            statement_in = find_override(".in.statement", ".interaction")
+            download_in = find_override(".in.download")
+            if statement_in and not download_in:
+                bar.warn(f"found {statement_in[0]} but no override for download")
+            if not statement_in and download_in:
+                bar.warn(f"found {download_in[0]} but no override for statement")
+
+            statement_ans = find_override(".out", ".ans.statement", ".interaction")
+            download_ans = find_override(".out", ".ans.download")
+            if statement_ans and not download_ans:
+                bar.warn(f"found {statement_ans[0]} but no override for download")
+            if not statement_ans and download_ans:
+                bar.warn(f"found {download_ans[0]} but no override for statement")
+
         def copy_generated() -> None:
             identical_exts = set()
 
             for ext in config.KNOWN_DATA_EXTENSIONS:
                 source = infile.with_suffix(ext)
                 target = target_infile.with_suffix(ext)
 
-                if source.is_file():
+                if ext in t.linked:
                     generator_config.known_files.add(target)
-                    if target.is_file():
-                        if source.read_bytes() == target.read_bytes() and not target.is_symlink():
+                    dest = target_infile.with_suffix(t.linked[ext])
+                    if not dest.is_file():
+                        bar.warn(
+                            f"{target.name}->{dest.name} is broken since {dest.name} was not generated"
+                        )
+                    if target.exists() or target.is_symlink():
+                        if target.is_symlink() and target.resolve() == dest.resolve():
+                            # identical -> skip
+                            identical_exts.add(ext)
+                        else:
+                            # different -> overwrite
+                            ensure_symlink(target, dest, relative=True)
+                            bar.log(f"CHANGED: {target.name}")
+                    else:
+                        # new link -> create it
+                        ensure_symlink(target, dest, relative=True)
+                        bar.log(f"NEW: {target.name}")
+                elif source.is_file():
+                    generator_config.known_files.add(target)
+                    if target.exists() or target.is_symlink():
+                        if not target.is_symlink() and source.read_bytes() == target.read_bytes():
                             # identical -> skip
                             identical_exts.add(ext)
                         else:
@@ -1266,7 +1352,7 @@ def copy_generated() -> None:
                         # new file -> copy it
                         shutil.copy(source, target, follow_symlinks=True)
                         bar.log(f"NEW: {target.name}")
-                elif target.is_file():
+                elif target.is_file() or target.is_symlink():
                     if (
                         config.args.no_visualizer
                         and ext in config.KNOWN_VISUALIZER_EXTENSIONS
@@ -1345,7 +1431,7 @@ def add_test_case_to_cache() -> None:
             check_match(testcase, "in", bar)
 
             # Step 4: generate .ans and .interaction if needed
-            if not generate_from_solution(testcase, bar):
+            if not generate_from_solution(testcase):
                 return
 
             # Step 5: validate .ans (and .out if it exists)
@@ -1356,14 +1442,17 @@ def add_test_case_to_cache() -> None:
             check_match(testcase, "ans", bar)
 
             # Step 6: generate visualization if needed
-            if not generate_visualization(testcase, bar):
+            if not generate_visualization(testcase):
                 return
 
         # Step 7: for interactive and/or multi-pass samples, generate empty .ans if it does not exist
         if not generate_empty_interactive_sample_ans():
             return
 
-        # Step 8: copy all generated files
+        # Step 8: warn if statement/download files are inconsistent
+        warn_override()
+
+        # Step 9: copy and link all generated files
         copy_generated()
 
         # Note that we set this to true even if not all files were overwritten -- a different log/warning message will be displayed for that.
@@ -2086,7 +2175,7 @@ def generate_copies_and_includes(d: Directory) -> None:
 
         bar.finalize()
 
-    # move a file or into the trash directory
+    # move a file or directory into the trash directory
     def remove(self, src: Path) -> None:
         if self.trash_dir is None:
             self.trash_dir = self.problem.tmpdir / "trash" / secrets.token_hex(4)

bapctools/latex.py

@@ -49,7 +49,7 @@ def create_samples_file(problem: "Problem", language: str) -> None:
 
     # create the samples.tex file
     # For samples, find all .in/.ans/.interaction pairs.
-    samples = problem.statement_samples()
+    samples = problem.samples()
 
     samples_file_path = builddir / "samples.tex"
 
@@ -63,7 +63,8 @@ def build_sample_command(content: str) -> str:
 
     samples_data = []
     fallback_call = []
-    for i, sample in enumerate(samples):
+    for i, data in enumerate(samples):
+        sample = data.statement
         fallback_call.append(f"\t\\csname Sample{i + 1}\\endcsname\n")
 
         current_sample = []