Split in two commands: githubGenerateSnapshot and githubPublishSnapshot

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
         working-directory: sbt-plugin
     steps:
       - uses: actions/checkout@v3
-      - uses: coursier/setup-action@v1.2.0-M3
+      - uses: coursier/setup-action@v1.3.5
         with:
           apps: scalafmt sbt
       - run: scalafmt --test
@@ -43,7 +43,7 @@ jobs:
         jvm: ${{ matrix.jvm }}
         apps: sbt
     - run: sbt test
-    - run: sbt "scripted dependency-manifest/*"
+    - run: sbt "scripted dependency-manifest/* generate-snapshot/*"
     - run: sbt "scripted submit-snapshot/*"
       if: github.event_name == 'push' || github.event.pull_request.head.repo.owner.login == 'scalacenter'
 

sbt-plugin/.scalafmt.conf

@@ -1,4 +1,4 @@
-version = "3.4.3"
+version = "3.8.1"
 runner.dialect = scala212source3
 maxColumn = 120
 align.preset = some

sbt-plugin/src/main/contraband/input.contra

@@ -8,8 +8,8 @@ enum OnFailure {
   warning
 }
 
-## Input of the githubSubmitDependencyGraph command
-type SubmitInput {
+## Input of the githubGenerateSnapshot command
+type DependencySnapshotInput {
   onResolveFailure: ch.epfl.scala.OnFailure
 
   ## A set of modules to ignore.

sbt-plugin/src/main/scala/ch/epfl/scala/GithubDependencyGraphPlugin.scala

@@ -27,10 +27,12 @@ object GithubDependencyGraphPlugin extends AutoPlugin {
       .map(_.toConfigRef)
 
   object autoImport {
-    val githubSubmitInputKey: AttributeKey[SubmitInput] = AttributeKey("githubSubmitInput")
+    val githubSnapshotInputKey: AttributeKey[DependencySnapshotInput] = AttributeKey("githubSnapshotInput")
     val githubBuildFile: AttributeKey[githubapi.FileInfo] = AttributeKey("githubBuildFile")
     val githubManifestsKey: AttributeKey[Map[String, githubapi.Manifest]] = AttributeKey("githubDependencyManifests")
     val githubProjectsKey: AttributeKey[Seq[ProjectRef]] = AttributeKey("githubProjectRefs")
+    val githubSnapshotFileKey: AttributeKey[File] = AttributeKey("githubSnapshotFile")
+
     val githubDependencyManifest: TaskKey[Option[githubapi.Manifest]] = taskKey(
       "The dependency manifest of the project"
     )
@@ -90,7 +92,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin {
   }
 
   private def includeProject(projectRef: ProjectRef, state: State, logger: Logger): Boolean = {
-    val ignoredModules = state.attributes(githubSubmitInputKey).ignoredModules
+    val ignoredModules = state.attributes(githubSnapshotInputKey).ignoredModules
     val moduleName = getModuleName(projectRef, state)
     val ignored = ignoredModules.contains(moduleName)
     if (!ignored) logger.info(s"Including dependency graph of $moduleName")
@@ -120,7 +122,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin {
     val thisProject = Keys.thisProject.value
     val internalConfigurationMap = Keys.internalConfigurationMap.value
 
-    val inputOpt = state.get(githubSubmitInputKey)
+    val inputOpt = state.get(githubSnapshotInputKey)
     val buildFileOpt = state.get(githubBuildFile)
 
     val onResolveFailure = inputOpt.flatMap(_.onResolveFailure)
@@ -198,7 +200,7 @@ object GithubDependencyGraphPlugin extends AutoPlugin {
                 else DependencyScope.development
               val metadata = Map("config" -> JString(configRef.name))
               val node = DependencyNode(packageUrl, metadata, Some(relationship), Some(scope), dependencies)
-              (moduleRef -> node)
+              moduleRef -> node
             }
 
         val projectModuleRef = getReference(projectID)

sbt-plugin/src/main/scala/ch/epfl/scala/SubmitDependencyGraph.scala

@@ -1,6 +1,5 @@
 package ch.epfl.scala
 
-import java.nio.charset.StandardCharsets
 import java.nio.file.Paths
 import java.time.Instant
 
@@ -22,31 +21,35 @@ import sjsonnew.shaded.scalajson.ast.unsafe.JValue
 import sjsonnew.support.scalajson.unsafe.{Parser => JsonParser, _}
 
 object SubmitDependencyGraph {
-  val Submit = "githubSubmitDependencyGraph"
-  val usage: String = s"""$Submit {"projects":[], "scalaVersions":[]}"""
-  val brief = "Submit the dependency graph to Github Dependency API."
-  val detail = "Submit the dependency graph of a set of projects and scala versions to Github Dependency API"
+  val Generate = "githubGenerateSnapshot"
+  private val GenerateUsage = s"""$Generate {"projects":[], "scalaVersions":[]}"""
+  private val GenerateDetail = "Generate the dependency graph of a set of projects and scala versions"
 
-  val SubmitInternal: String = s"${Submit}Internal"
-  val internalOnly = "internal usage only"
+  private val GenerateInternal = s"${Generate}Internal"
+  private val InternalOnly = "internal usage only"
+
+  val Submit = "githubSubmitSnapshot"
+  private val SubmitDetail = "Submit the dependency graph to Github Dependency API."
+
+  def usage(command: String): String = s"""$command {"projects":[], "scalaVersions":[]}"""
 
   val commands: Seq[Command] = Seq(
-    Command(Submit, (usage, brief), detail)(inputParser)(submit),
-    Command.command(SubmitInternal, internalOnly, internalOnly)(submitInternal)
+    Command(Generate, (GenerateUsage, GenerateDetail), GenerateDetail)(inputParser)(generate),
+    Command.command(GenerateInternal, InternalOnly, InternalOnly)(generateInternal),
+    Command.command(Submit, SubmitDetail, SubmitDetail)(submit)
   )
 
   private lazy val http: HttpClient = Gigahorse.http(Gigahorse.config)
 
-  private def inputParser(state: State): Parser[SubmitInput] =
+  private def inputParser(state: State): Parser[DependencySnapshotInput] =
     Parsers.any.*.map { raw =>
       JsonParser
         .parseFromString(raw.mkString)
-        .flatMap(Converter.fromJson[SubmitInput])
+        .flatMap(Converter.fromJson[DependencySnapshotInput])
         .get
     }.failOnException
 
-  private def submit(state: State, input: SubmitInput): State = {
-    checkGithubEnv() // fail fast if the Github CI environment is incomplete
+  private def generate(state: State, input: DependencySnapshotInput): State = {
     val loadedBuild = state.setting(Keys.loadedBuild)
     // all project refs that have a Scala version
     val projectRefs = loadedBuild.allProjectRefs
@@ -65,48 +68,58 @@ object SubmitDependencyGraph {
     state.log.info(s"Resolving snapshot of $buildFile")
 
     val initState = state
-      .put(githubSubmitInputKey, input)
+      .put(githubSnapshotInputKey, input)
       .put(githubBuildFile, githubapi.FileInfo(buildFile.toString))
       .put(githubManifestsKey, Map.empty[String, Manifest])
       .put(githubProjectsKey, projectRefs)
 
     val storeAllManifests = scalaVersions.flatMap { scalaVersion =>
       Seq(s"++$scalaVersion", s"Global/${githubStoreDependencyManifests.key} $scalaVersion")
     }
-    val commands = storeAllManifests :+ SubmitInternal
+    val commands = storeAllManifests :+ GenerateInternal
     commands.toList ::: initState
   }
 
-  private def submitInternal(state: State): State = {
+  private def generateInternal(state: State): State = {
     val snapshot = githubDependencySnapshot(state)
-    val snapshotUrl = s"${githubApiUrl()}/repos/${githubRepository()}/dependency-graph/snapshots"
-
     val snapshotJson = CompactPrinter(Converter.toJsonUnsafe(snapshot))
-
     val snapshotJsonFile = IO.withTemporaryFile("dependency-snapshot-", ".json", keepFile = true) { file =>
       IO.write(file, snapshotJson)
       state.log.info(s"Dependency snapshot written to ${file.getAbsolutePath}")
       file
     }
+    setGithubOutputs("snapshot-json-path" -> snapshotJsonFile.getAbsolutePath)
+    state.put(githubSnapshotFileKey, snapshotJsonFile)
+  }
 
+  def submit(state: State): State = {
+    checkGithubEnv() // fail if the Github CI environment
+    val snapshotJsonFile = state
+      .get(githubSnapshotFileKey)
+      .getOrElse(
+        throw new MessageOnlyException(
+          "Missing snapshot file. This command must execute after the githubGenerateSnapshot command"
+        )
+      )
+    val snapshotUrl = s"${githubApiUrl()}/repos/${githubRepository()}/dependency-graph/snapshots"
+    val job = githubJob()
     val request = Gigahorse
       .url(snapshotUrl)
-      .post(snapshotJson, StandardCharsets.UTF_8)
+      .post(snapshotJsonFile)
       .addHeaders(
         "Content-Type" -> "application/json",
         "Authorization" -> s"token ${githubToken()}"
       )
 
-    state.log.info(s"Submiting dependency snapshot of job ${snapshot.job} to $snapshotUrl")
+    state.log.info(s"Submitting dependency snapshot of job $job to $snapshotUrl")
     val result = for {
       httpResp <- Try(Await.result(http.processFull(request), Duration.Inf))
       snapshot <- getSnapshot(httpResp)
     } yield {
       state.log.info(s"Submitted successfully as $snapshotUrl/${snapshot.id}")
       setGithubOutputs(
         "submission-id" -> s"${snapshot.id}",
-        "submission-api-url" -> s"${snapshotUrl}/${snapshot.id}",
-        "snapshot-json-path" -> snapshotJsonFile.getAbsolutePath
+        "submission-api-url" -> s"${snapshotUrl}/${snapshot.id}"
       )
       state
     }
@@ -115,11 +128,9 @@ object SubmitDependencyGraph {
   }
 
   // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
-  private def setGithubOutputs(outputs: (String, String)*): Unit = IO.writeLines(
-    file(githubOutput),
-    outputs.toSeq.map { case (name, value) => s"${name}=${value}" },
-    append = true
-  )
+  private def setGithubOutputs(outputs: (String, String)*): Unit =
+    for (output <- githubOutput())
+      IO.writeLines(output, outputs.map { case (name, value) => s"${name}=${value}" }, append = true)
 
   private def getSnapshot(httpResp: FullResponse): Try[SnapshotResponse] =
     httpResp.status match {
@@ -165,32 +176,32 @@ object SubmitDependencyGraph {
   }
 
   private def checkGithubEnv(): Unit = {
-    githubWorkspace()
-    githubWorkflow()
-    githubJobName()
-    githubAction()
-    githubRunId()
-    githubSha()
-    githubRef()
-    githubApiUrl()
-    githubRepository()
-    githubToken()
-  }
-
-  private def githubWorkspace(): String = githubCIEnv("GITHUB_WORKSPACE")
-  private def githubWorkflow(): String = githubCIEnv("GITHUB_WORKFLOW")
-  private def githubJobName(): String = githubCIEnv("GITHUB_JOB")
-  private def githubAction(): String = githubCIEnv("GITHUB_ACTION")
-  private def githubRunId(): String = githubCIEnv("GITHUB_RUN_ID")
-  private def githubSha(): String = githubCIEnv("GITHUB_SHA")
-  private def githubRef(): String = githubCIEnv("GITHUB_REF")
-  private def githubApiUrl(): String = githubCIEnv("GITHUB_API_URL")
-  private def githubRepository(): String = githubCIEnv("GITHUB_REPOSITORY")
-  private def githubToken(): String = githubCIEnv("GITHUB_TOKEN")
-  private def githubOutput(): String = githubCIEnv("GITHUB_OUTPUT")
-
-  private def githubCIEnv(name: String): String =
-    Properties.envOrNone(name).getOrElse {
+    def check(name: String): Unit = Properties.envOrNone(name).orElse {
       throw new MessageOnlyException(s"Missing environment variable $name. This task must run in a Github Action.")
     }
+    check("GITHUB_WORKSPACE")
+    check("GITHUB_WORKFLOW")
+    check("GITHUB_JOB")
+    check("GITHUB_ACTION")
+    check("GITHUB_RUN_ID")
+    check("GITHUB_SHA")
+    check("GITHUB_REF")
+    check("GITHUB_API_URL")
+    check("GITHUB_REPOSITORY")
+    check("GITHUB_TOKEN")
+    check("GITHUB_OUTPUT")
+  }
+
+  private def githubWorkspace(): String = Properties.envOrElse("GITHUB_WORKSPACE", "")
+  private def githubWorkflow(): String = Properties.envOrElse("GITHUB_WORKFLOW", "")
+  private def githubJobName(): String = Properties.envOrElse("GITHUB_JOB", "")
+  private def githubAction(): String = Properties.envOrElse("GITHUB_ACTION", "")
+  private def githubRunId(): String = Properties.envOrElse("GITHUB_RUN_ID", "")
+  private def githubSha(): String = Properties.envOrElse("GITHUB_SHA", "")
+  private def githubRef(): String = Properties.envOrElse("GITHUB_REF", "")
+
+  private def githubApiUrl(): String = Properties.envOrElse("GITHUB_API_URL", "")
+  private def githubRepository(): String = Properties.envOrElse("GITHUB_REPOSITORY", "")
+  private def githubToken(): String = Properties.envOrElse("GITHUB_TOKEN", "")
+  private def githubOutput(): Option[File] = Properties.envOrNone("GITHUB_OUTPUT").map(file)
 }

sbt-plugin/src/sbt-test/dependency-manifest/ignore-scaladoc/build.sbt

@@ -1,7 +1,7 @@
 import ch.epfl.scala.githubapi.DependencyRelationship
 import ch.epfl.scala.githubapi.DependencyScope
 import ch.epfl.scala.githubapi.Manifest
-import ch.epfl.scala.SubmitInput
+import ch.epfl.scala.DependencySnapshotInput
 import sjsonnew.shaded.scalajson.ast.unsafe.JString
 
 val checkScaladoc = taskKey[Unit]("Check scaladoc_3 is in the manifest ")
@@ -17,8 +17,8 @@ inThisBuild(
 )
 
 Global / ignoreScaladoc := {
-  val input = SubmitInput(None, Vector.empty, ignoredConfigs = Vector("scala-doc-tool"))
-  StateTransform(state => state.put(githubSubmitInputKey, input))
+  val input = DependencySnapshotInput(None, Vector.empty, ignoredConfigs = Vector("scala-doc-tool"))
+  StateTransform(state => state.put(githubSnapshotInputKey, input))
 }
 
 lazy val p1 = project

sbt-plugin/src/sbt-test/dependency-manifest/ignore-test/build.sbt

@@ -1,7 +1,7 @@
 import ch.epfl.scala.githubapi.DependencyRelationship
 import ch.epfl.scala.githubapi.DependencyScope
 import ch.epfl.scala.githubapi.Manifest
-import ch.epfl.scala.SubmitInput
+import ch.epfl.scala.DependencySnapshotInput
 import sjsonnew.shaded.scalajson.ast.unsafe.JString
 
 val checkTest = taskKey[Unit]("Check munit_3 is in the manifest ")
@@ -17,8 +17,8 @@ inThisBuild(
 )
 
 Global / ignoreTestConfig := {
-  val input = SubmitInput(None, Vector.empty, ignoredConfigs = Vector("test"))
-  StateTransform(state => state.put(githubSubmitInputKey, input))
+  val input = DependencySnapshotInput(None, Vector.empty, ignoredConfigs = Vector("test"))
+  StateTransform(state => state.put(githubSnapshotInputKey, input))
 }
 
 lazy val p1 = project

sbt-plugin/src/sbt-test/generate-snapshot/generate-snapshot/test

@@ -0,0 +1,11 @@
+> 'githubGenerateSnapshot {}'
+> Global / checkManifests 4
+
+> 'githubGenerateSnapshot {"ignoredModules":["a_2.13", "b_2.13"]}'
+> Global / checkManifests 2
+
+> 'githubGenerateSnapshot {"ignoredModules":["a_2.12", "a_2.13", "b_2.13"]}'
+> Global / checkManifests 1
+
+> 'githubGenerateSnapshot {"ignoredModules":[]}'
+> Global / checkManifests 4

sbt-plugin/src/sbt-test/generate-snapshot/resolve-failure/test

@@ -0,0 +1,5 @@
+-> 'githubGenerateSnapshot {}'
+-> 'Global / githubGenerateSnapshot {"onResolveFailure": "error"}'
+
+> 'githubGenerateSnapshot {"onResolveFailure": "warning"}'
+> Global / checkManifests 1