! Unify package and generator repository

MatejKafka · MatejKafka · commit 038f946852bf · 2025-01-02T04:04:07.000+01:00
Previously, there was a separate repository for packages and for manifest generators, but the two had to be kept in sync. Original rationale for the split has two parts:
1) Conceptually, packages are for users, while generators are for the author - users are interested in the repository, but not necessarily in the generators. Since we now have remote repos, this is more-or-less solved.
2) A package author might want to have a public package repo, but private generators that rely on internal tooling. This is still potentially an issue, but I don't see it coming up in practice; if necessary, it can also be solved by generating a remote repo and keeping the sources private, at the cost of having the origin local package repo public.

TODO: when next release is published, push changes in `PogPackages` and remove `PogManifestGenerators`.
diff --git a/app/Pog/Pog.Format.ps1xml b/app/Pog/Pog.Format.ps1xml
@@ -177,31 +177,5 @@
                 </TableRowEntries>
             </TableControl>
         </View>
-        <View>
-            <Name>Pog.PackageGenerator</Name>
-            <ViewSelectedBy>
-                <TypeName>Pog.PackageGenerator</TypeName>
-            </ViewSelectedBy>
-            <TableControl>
-                <TableHeaders>
-                    <TableColumnHeader>
-                        <Width>20</Width>
-                    </TableColumnHeader>
-                    <TableColumnHeader/>
-                </TableHeaders>
-                <TableRowEntries>
-                    <TableRowEntry>
-                        <TableColumnItems>
-                            <TableColumnItem>
-                                <PropertyName>PackageName</PropertyName>
-                            </TableColumnItem>
-                            <TableColumnItem>
-                                <PropertyName>Path</PropertyName>
-                            </TableColumnItem>
-                        </TableColumnItems>
-                    </TableRowEntry>
-                </TableRowEntries>
-            </TableControl>
-        </View>
     </ViewDefinitions>
 </Configuration>
diff --git a/app/Pog/container/Env_ManifestGenerator.psm1 b/app/Pog/container/Env_ManifestGenerator.psm1
@@ -13,32 +13,32 @@ $PSDefaultParameterValues = @{
 
 $ContainerModule = New-ContainerModule
 
-function ListVersions($Generator, $ExistingVersionSet) {
+function ListVersions($Package, $ExistingVersionSet) {
 	try {
-		return & $ContainerModule $Generator.Manifest.ListVersionsSb $ExistingVersionSet
+		return & $ContainerModule $Package.Generator.ListVersionsSb $ExistingVersionSet
 	} catch {
-		throw [Exception]::new("Manifest generator for package '$($Generator.PackageName)' failed while listing versions: $_", $_.Exception)
+		throw [Exception]::new("Manifest generator for package '$($Package.PackageName)' failed while listing versions: $_", $_.Exception)
 	}
 }
 
 <# Retrieve all existing versions of a package by calling the package version generator script. #>
-function RetrievePackageVersions([Pog.PackageGenerator]$Generator, $ExistingVersionSet) {
-	foreach ($Obj in ListVersions $Generator $ExistingVersionSet) {
+function RetrievePackageVersions($Package, $ExistingVersionSet) {
+	foreach ($Obj in ListVersions $Package $ExistingVersionSet) {
 		# the returned object should either be the version string directly, or a map object
 		#  (hashtable/pscustomobject/psobject/ordered) that has the Version property
 		#  why not use -is? that's why: https://github.com/PowerShell/PowerShell/issues/16361
 		$IsMap = $Obj.PSTypeNames[0] -in @("System.Collections.Hashtable", "System.Management.Automation.PSCustomObject", "System.Collections.Specialized.OrderedDictionary")
 		$VersionStr = if (-not $IsMap) {$Obj} else {
 			try {$Obj.Version}
 			catch {
-				throw "Version generator for package '$($Generator.PackageName)' returned a custom object without a Version property: $Obj" +`
+				throw "Version generator for package '$($Package.PackageName)' returned a custom object without a Version property: $Obj" +`
 					"  (Version generators must return either a version string, or a" +`
 					" map container (hashtable, psobject, pscustomobject) with a Version property.)"
 			}
 		}
 
 		if ([string]::IsNullOrEmpty($VersionStr)) {
-			throw "Empty package version generated by the version generator for package '$($Generator.PackageName)' (either `$null or empty string)."
+			throw "Empty package version generated by the version generator for package '$($Package.PackageName)' (either `$null or empty string)."
 		}
 
 		[pscustomobject]@{
@@ -56,7 +56,6 @@ function __main {
 	### Internally, it invokes `ListVersions` for the generator and then invokes `Generate` for each version
 	### that should be regenerated.
 	param(
-		[Pog.PackageGenerator]$Generator,
 		[Pog.LocalRepositoryVersionedPackage]$Package,
 		[string[]]$Version,
 		[bool]$Force,
@@ -70,7 +69,7 @@ function __main {
 	# list available versions without existing manifest (unless -Force is set, then all versions are listed)
 	# only generate manifests for versions that don't already exist, unless -Force is passed
 	$ExistingVersions = [System.Collections.Generic.HashSet[string]]::new($Package.EnumerateVersionStrings())
-	$GeneratedVersions = RetrievePackageVersions $Generator $ExistingVersions `
+	$GeneratedVersions = RetrievePackageVersions $Package $ExistingVersions `
 		<# if -Force was not passed, filter out versions with already existing manifest #> `
 		| ? {$Force -or -not $ExistingVersions.Contains($_.Version)} `
 		<# if $Version was passed, filter out the versions; as the versions generated by the script
@@ -97,8 +96,8 @@ function __main {
 		$p = $Package.GetVersionPackage($v.Version, $false)
 
 		try {
-			$TemplateData = if ($Generator.Manifest.GenerateSb) {
-				$Sb = $ContainerModule.NewBoundScriptBlock($Generator.Manifest.GenerateSb)
+			$TemplateData = if ($Package.Generator.GenerateSb) {
+				$Sb = $ContainerModule.NewBoundScriptBlock($Package.Generator.GenerateSb)
 				# pass the value both as $_ and as a parameter, the scriptblock can accept whichever one is more convenient
 				Invoke-DollarUnder $Sb $v.OriginalValue $v.OriginalValue
 			} else {
diff --git a/app/Pog/lib_compiled/Pog/src/Commands/ConfirmPogRepositoryCommand.cs b/app/Pog/lib_compiled/Pog/src/Commands/ConfirmPogRepositoryCommand.cs
@@ -181,7 +181,7 @@ private void ValidateTemplatedPackage(LocalRepositoryVersionedPackage vp) {
         }
 
         // validate .template dir
-        ValidateManifestDirectory($"package '{vp.PackageName}'", templateDirPath);
+        ValidateManifestDirectory($"package '{vp.PackageName}'", templateDirPath, true);
 
         // validate that manifest template exists
         if (!File.Exists(templatePath)) {
@@ -220,7 +220,7 @@ private void ValidatePackageVersions(LocalRepositoryVersionedPackage vp, bool va
     private void ValidatePackageVersion(LocalRepositoryPackage p, bool validateManifestDir = true) {
         if (validateManifestDir) {
             var path = p is TemplatedLocalRepositoryPackage tp ? tp.TemplateDirPath : p.Path;
-            ValidateManifestDirectory(p.GetDescriptionString(), path);
+            ValidateManifestDirectory(p.GetDescriptionString(), path, false);
         }
 
         // validate the manifest
@@ -254,18 +254,25 @@ private void ValidatePackageVersion(LocalRepositoryPackage p, bool validateManif
         }
     }
 
-    private void ValidateManifestDirectory(string packageInfoStr, string templateDirPath) {
-        var extraEntries = GetFileList(Directory.EnumerateFileSystemEntries(templateDirPath)
-                .Where(p => !p.EndsWith(@"\pog.psd1") && !p.EndsWith(@"\.pog")));
-        if (extraEntries != "") {
-            AddIssue($"Manifest directory for {packageInfoStr} at '{templateDirPath}' contains extra " +
-                     $"entries: {extraEntries}. Only a 'pog.psd1' manifest file and an optional '.pog' directory for extra " +
-                     $"files is allowed.");
+    private void ValidateManifestDirectory(string packageInfoStr, string manifestDirPath, bool isTemplate) {
+        var extraEntries = Directory.EnumerateFileSystemEntries(manifestDirPath)
+                .Where(p => !p.EndsWith(@"\pog.psd1") && !p.EndsWith(@"\.pog"));
+
+        if (isTemplate) {
+            // allow the generator manifest for template dir
+            extraEntries = extraEntries.Where(p => !p.EndsWith(@"\generator.psd1"));
+        }
+
+        var extraEntriesStr = GetFileList(extraEntries);
+        if (extraEntriesStr != "") {
+            AddIssue($"Manifest directory for {packageInfoStr} at '{manifestDirPath}' contains extra entries: " +
+                     $"{extraEntriesStr}. Only a 'pog.psd1' manifest file{(isTemplate ? ", a 'generator.psd1' generator file" : "")} " +
+                     $"and an optional '.pog' directory for extra files is allowed.");
         }
 
-        if (File.Exists($"{templateDirPath}\\.pog")) {
+        if (File.Exists($"{manifestDirPath}\\.pog")) {
             AddIssue($"Extra resource directory for {packageInfoStr} must be a directory, " +
-                     $"found a file at '{templateDirPath}\\.pog'.");
+                     $"found a file at '{manifestDirPath}\\.pog'.");
         }
 
         // pog.psd1 manifest is validated separately
diff --git a/app/Pog/lib_compiled/Pog/src/Commands/UpdatePogRepositoryCommand.cs b/app/Pog/lib_compiled/Pog/src/Commands/UpdatePogRepositoryCommand.cs
@@ -1,5 +1,4 @@
-﻿using System;
-using System.Linq;
+﻿using System.Linq;
 using System.Management.Automation;
 using System.Security;
 using JetBrains.Annotations;
@@ -79,49 +78,46 @@ protected override void ProcessRecord() {
         if (PackageName == null) return;
 
         foreach (var pn in PackageName) {
-            var generator = InternalState.GeneratorRepository.GetPackage(pn, true, true);
-            // ensure manifest is loaded
-            generator.ReloadManifest();
+            var p = (LocalRepositoryVersionedPackage) _repo.GetPackage(pn, true, true);
+            if (!p.HasGenerator) {
+                WriteError(new PackageGeneratorNotFoundException($"Package '{p.PackageName}' does not have a generator."),
+                        "MissingGenerator", ErrorCategory.ObjectNotFound, pn);
+                continue;
+            }
 
             // if -Version was passed, overwrite even existing manifests
-            ProcessPackage(generator, Force || Version != null);
+            ProcessPackage(p, Force || Version != null);
         }
     }
 
     private void UpdateAll() {
-        var generators = InternalState.GeneratorRepository.Enumerate().ToArray();
-        if (generators.Length == 0) {
+        var packages = _repo.EnumerateGeneratedPackages().ToArray();
+        if (packages.Length == 0) {
             return;
         }
 
         using var progressBar = new CmdletProgressBar(this, new() {
             Activity = "Updating repository",
             // this duplicates the code below in the loop, but we have to do that due to
             //  https://github.com/PowerShell/PowerShell/issues/24728
-            Description = $"Updating '{generators[0].PackageName}'...",
+            Description = $"Updating '{packages[0].PackageName}'...",
         });
 
         var i = 0;
-        foreach (var generator in generators) {
-            progressBar.Update((double) i++ / generators.Length, $"Updating '{generator.PackageName}'...");
-            ProcessPackage(generator, Force);
+        foreach (var p in packages) {
+            progressBar.Update((double) i++ / packages.Length, $"Updating '{p.PackageName}'...");
+            ProcessPackage(p, Force);
         }
     }
 
-    private void ProcessPackage(PackageGenerator generator, bool force) {
-        var package = (LocalRepositoryVersionedPackage) _repo.GetPackage(generator.PackageName, true, true);
-        if (!package.IsTemplated) {
-            var e = new ArgumentException(
-                    $"Package '{package.PackageName}' {(package.Exists ? "is not templated" : "does not exist yet")}, " +
-                    "generators are only supported for existing templated packages.");
-            WriteError(e, "NonTemplatedPackage", ErrorCategory.InvalidArgument, generator.PackageName);
-            return;
-        }
+    private void ProcessPackage(LocalRepositoryVersionedPackage package, bool force) {
+        // ensure generator manifest is loaded
+        package.ReloadGenerator();
 
         var it = InvokePogCommand(new InvokeContainer(this) {
             Modules = [$@"{InternalState.PathConfig.ContainerDir}\Env_ManifestGenerator.psm1"],
             Run = ps => ps.AddCommand("__main").AddParameters(
-                    new object?[] {generator, package, Version, force, ListOnly, GitHubToken}),
+                    new object?[] {package, Version, force, ListOnly, GitHubToken}),
         });
 
         foreach (var o in it) {
diff --git a/app/Pog/lib_compiled/Pog/src/LocalRepositoryTypes.cs b/app/Pog/lib_compiled/Pog/src/LocalRepositoryTypes.cs
@@ -27,6 +27,14 @@ public IEnumerable<RepositoryVersionedPackage> Enumerate(string searchPattern =
         return EnumeratePackageNames(searchPattern).Select(p => new LocalRepositoryVersionedPackage(repo, p));
     }
 
+    public IEnumerable<string> EnumerateGeneratedPackageNames(string searchPattern = "*") {
+        return EnumerateGeneratedPackages(searchPattern).Select(p => p.PackageName);
+    }
+
+    public IEnumerable<LocalRepositoryVersionedPackage> EnumerateGeneratedPackages(string searchPattern = "*") {
+        return Enumerate(searchPattern).Cast<LocalRepositoryVersionedPackage>().Where(p => p.HasGenerator);
+    }
+
     public RepositoryVersionedPackage GetPackage(string packageName, bool resolveName, bool mustExist) {
         Verify.PackageName(packageName);
         if (resolveName) {
@@ -51,9 +59,15 @@ public sealed class LocalRepositoryVersionedPackage : RepositoryVersionedPackage
     public readonly string Path;
     public override bool Exists => Directory.Exists(Path);
     public bool IsTemplated => Directory.Exists(TemplateDirPath);
+    public bool HasGenerator => File.Exists(GeneratorPath);
 
     public string TemplateDirPath => $"{Path}\\{PPaths.RepositoryTemplateDirName}";
     public string TemplatePath => $"{TemplateDirPath}\\{PPaths.ManifestFileName}";
+    public string GeneratorPath => $"{TemplateDirPath}\\{PPaths.GeneratorFileName}";
+
+    private PackageGeneratorManifest? _generator;
+    public PackageGeneratorManifest Generator => _generator ?? ReloadGenerator();
+
     internal override string ExpectedPathStr => $"expected path: {Path}";
 
     internal LocalRepositoryVersionedPackage(LocalRepository repository, string packageName) : base(packageName) {
@@ -88,6 +102,12 @@ protected override RepositoryPackage GetPackageUnchecked(PackageVersion version)
                 ? new TemplatedLocalRepositoryPackage(this, version)
                 : new DirectLocalRepositoryPackage(this, version);
     }
+
+    /// <exception cref="PackageManifestNotFoundException">Thrown if the package generator does not exist.</exception>
+    /// <exception cref="PackageManifestParseException">Thrown if the package generator file is not a valid PowerShell data file (.psd1).</exception>
+    public PackageGeneratorManifest ReloadGenerator() {
+        return _generator = new PackageGeneratorManifest(GeneratorPath);
+    }
 }
 
 [PublicAPI]
diff --git a/app/Pog/lib_compiled/Pog/src/PSAttributes/Completers.cs b/app/Pog/lib_compiled/Pog/src/PSAttributes/Completers.cs
@@ -117,7 +117,9 @@ protected override IEnumerable<string> GetMatchingItems(string searchPattern, ID
 [PublicAPI]
 public sealed class RepositoryPackageGeneratorNameCompleter : DirectoryListingArgumentCompleter {
     protected override IEnumerable<string> GetMatchingItems(string searchPattern, IDictionary _) {
-        return InternalState.GeneratorRepository.EnumerateGeneratorNames(searchPattern);
+        return InternalState.Repository is LocalRepository localRepo
+                ? localRepo.EnumerateGeneratedPackageNames(searchPattern)
+                : [];
     }
 }
 
diff --git a/app/Pog/lib_compiled/Pog/src/PackageGeneratorTypes.cs b/app/Pog/lib_compiled/Pog/src/PackageGeneratorTypes.cs
diff --git a/app/Pog/lib_compiled/Pog/src/Pog.InternalState.cs b/app/Pog/lib_compiled/Pog/src/Pog.InternalState.cs
@@ -62,10 +62,6 @@ public static IRepository Repository {
         set => Interlocked.Exchange(ref _repository, value);
     }
 
-    private static GeneratorRepository? _generatorRepository;
-    public static GeneratorRepository GeneratorRepository => LazyInitializer.EnsureInitialized(ref _generatorRepository,
-            () => new GeneratorRepository(PathConfig.ManifestGeneratorDir))!;
-
     private static ImportedPackageManager? _importedPackageManager;
     public static ImportedPackageManager ImportedPackageManager => LazyInitializer.EnsureInitialized(
             ref _importedPackageManager,
diff --git a/app/Pog/lib_compiled/Pog/src/Pog.PackageManifestGenerator.cs b/app/Pog/lib_compiled/Pog/src/Pog.PackageManifestGenerator.cs
@@ -1,12 +1,15 @@
 ﻿using System;
 using System.Collections;
+using System.IO;
 using System.Management.Automation;
 using JetBrains.Annotations;
 
 namespace Pog;
 
 public class InvalidGeneratorManifestException(string message) : Exception(message);
 
+public class PackageGeneratorNotFoundException(string message) : FileNotFoundException(message);
+
 [PublicAPI]
 public class PackageGeneratorManifest {
     public readonly Hashtable Raw;
diff --git a/app/Pog/lib_compiled/Pog/src/Pog.PathConfig.cs b/app/Pog/lib_compiled/Pog/src/Pog.PathConfig.cs
@@ -38,6 +38,7 @@ public static class PackagePaths {
         internal const string ManifestFileName = "pog.psd1";
         internal const string ManifestResourceDirName = ".pog";
         internal const string RepositoryTemplateDirName = ".template";
+        internal const string GeneratorFileName = "generator.psd1";
 
         // suffixes, not relative paths (do not add a slash when joining these)
         internal const string ShortcutDirRelSuffix = "";
@@ -71,7 +72,6 @@ public static class PackagePaths {
     /// Directory where exported shortcuts from packages are copied (per-user).
     public readonly string ExportedShortcutDir;
     public readonly string ExportedCommandDir;
-    public readonly string ManifestGeneratorDir;
 
     /// Directory where package files with known hash are cached.
     public readonly string DownloadCacheDir;
@@ -102,7 +102,6 @@ public PathConfig(string pogRootPath, string? dataRootPath = null, string? short
 
         var dataPath = $"{dataRootPath}\\data";
         ExportedCommandDir = $"{dataPath}\\package_bin";
-        ManifestGeneratorDir = $"{dataPath}\\manifest_generators";
         PackageRoots = new PackageRootConfig($"{dataPath}\\package_roots.txt");
         Path7Zip = $"{ExportedCommandDir}\\7z.exe";
         PathOpenedFilesView = $"{ExportedCommandDir}\\OpenedFilesView.exe";
diff --git a/app/Pog/tests/TestEnvironmentSetup.ps1 b/app/Pog/tests/TestEnvironmentSetup.ps1
diff --git a/setup.ps1 b/setup.ps1

Original file line number	Diff line number	Diff line change
`@@ -117,7 +117,9 @@ protected override IEnumerable<string> GetMatchingItems(string searchPattern, ID`
`117`	`117`	`[PublicAPI]`
`118`	`118`	`public sealed class RepositoryPackageGeneratorNameCompleter : DirectoryListingArgumentCompleter {`
`119`	`119`	`protected override IEnumerable<string> GetMatchingItems(string searchPattern, IDictionary _) {`
`120`		`- return InternalState.GeneratorRepository.EnumerateGeneratorNames(searchPattern);`
	`120`	`+ return InternalState.Repository is LocalRepository localRepo`
	`121`	`+ ? localRepo.EnumerateGeneratedPackageNames(searchPattern)`
	`122`	`+ : [];`
`121`	`123`	`}`
`122`	`124`	`}`
`123`	`125`