Skip to content

Parse using Circe #215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mergify merged 1 commit into from
May 15, 2021
Merged

Parse using Circe #215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 19 additions & 20 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
val ScalaVersion = "2.12.13"
val AkkaStreamsJson = "0.7.0"
val Circe = "0.13.0"

lazy val authors = project
.in(file("."))
Expand All @@ -12,27 +14,24 @@ lazy val core = project
val Akka = "2.6.14"
val AkkaHttp = "10.2.4"
libraryDependencies ++= Seq(
"com.typesafe.akka" %% "akka-actor" % Akka,
"com.typesafe.akka" %% "akka-stream" % Akka,
"com.typesafe.akka" %% "akka-slf4j" % Akka,
"com.typesafe.akka" %% "akka-http" % AkkaHttp,
"com.tradeshift" %% "ts-reaktive-marshal-akka" % "0.16.3" exclude ("org.slf4j", "slf4j-log4j12"),
"com.madgag.scala-git" %% "scala-git" % "4.2",
"ch.qos.logback" % "logback-classic" % "1.2.3",
"org.scalatest" %% "scalatest" % "3.2.8" % "test",
"com.typesafe.akka" %% "akka-testkit" % Akka % "test",
// these come from ts-reaktive-marshal-akka
"com.typesafe.akka" %% "akka-persistence" % Akka,
"com.typesafe.akka" %% "akka-remote" % Akka,
"com.typesafe.akka" %% "akka-cluster" % Akka,
"com.typesafe.akka" %% "akka-cluster-tools" % Akka,
"com.typesafe.akka" %% "akka-distributed-data" % Akka,
"com.typesafe.akka" %% "akka-persistence-query" % Akka,
"com.typesafe.akka" %% "akka-cluster-sharding" % Akka,
"com.typesafe.akka" %% "akka-protobuf" % Akka,
"com.typesafe.akka" %% "akka-http-jackson" % AkkaHttp
"com.typesafe.akka" %% "akka-actor" % Akka,
"com.typesafe.akka" %% "akka-stream" % Akka,
"com.typesafe.akka" %% "akka-slf4j" % Akka,
"com.typesafe.akka" %% "akka-http" % AkkaHttp,
"com.madgag.scala-git" %% "scala-git" % "4.2",
"ch.qos.logback" % "logback-classic" % "1.2.3",
"org.mdedetrich" %% "akka-stream-circe" % AkkaStreamsJson,
"org.mdedetrich" %% "akka-http-circe" % AkkaStreamsJson,
"com.lightbend.akka" %% "akka-stream-alpakka-json-streaming" % "3.0.0-M1",
"io.circe" %% "circe-generic" % Circe,
"io.circe" %% "circe-generic-extras" % Circe,
"org.scalatest" %% "scalatest" % "3.2.8" % "test",
"com.typesafe.akka" %% "akka-testkit" % Akka % "test"
)
}
},
addCompilerPlugin(
"org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full
)
)
.enablePlugins(AutomateHeaderPlugin)

Expand Down
21 changes: 13 additions & 8 deletions core/src/main/scala/Authors.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,27 @@ import akka.http.scaladsl.Http
import akka.http.scaladsl.marshalling.Marshal
import akka.http.scaladsl.marshalling.PredefinedToRequestMarshallers._
import akka.http.scaladsl.model.{HttpRequest, Uri}
import akka.stream.alpakka.json.scaladsl.JsonReader
import akka.stream.scaladsl.{Flow, Source}
import akka.util.ByteString

import com.madgag.git._
import com.tradeshift.reaktive.marshal.stream.{ActsonReader, ProtocolReader}
import org.eclipse.jgit.diff.DiffFormatter
import org.eclipse.jgit.internal.storage.file.FileRepository
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.util.io.DisabledOutputStream
import org.mdedetrich.akka.stream.support.CirceStreamSupport

import lt.dvim.authors.GithubProtocol.{AuthorStats, Commit, Stats}
import lt.dvim.authors.Authors.{AuthorStats, Stats}
import lt.dvim.authors.GithubProtocol.{Commit, GitAuthor, GithubAuthor}

object Authors {
final val MaxAuthors = 1024
final val GithubApiUrl = "api.github.com"

final case class Stats(additions: Int, deletions: Int, commits: Int)
final case class AuthorStats(gitAuthor: GitAuthor, githubAuthor: Option[GithubAuthor], stats: Stats)

def summary(repo: Option[String], from: String, to: String, path: String): Future[String] = {
val cld = classOf[ActorSystem].getClassLoader
implicit val sys = ActorSystem("Authors", classLoader = Some(cld))
Expand All @@ -55,8 +60,8 @@ object Authors {
parseRepo(gitRepository.getConfig().getString("remote", "origin", "url"))

DiffSource(repo.getOrElse(parsedRepo), from, to)
.via(ActsonReader.instance)
.via(ProtocolReader.of(GithubProtocol.compareProto))
.via(JsonReader.select("$.commits[*]"))
.via(CirceStreamSupport.decode[Commit])
.via(StatsAggregator())
.via(SortingMachine())
.via(MarkdownConverter())
Expand Down Expand Up @@ -120,10 +125,10 @@ object SortingMachine {
object StatsAggregator {
def apply()(implicit repo: FileRepository, log: LoggingAdapter): Flow[Commit, AuthorStats, NotUsed] =
Flow[Commit]
.filterNot(_.message.startsWith("Merge pull request"))
.groupBy(Authors.MaxAuthors, commit => commit.githubAuthor.map(_.login).getOrElse(commit.gitAuthor.email))
.filterNot(_.commit.message.startsWith("Merge pull request"))
.groupBy(Authors.MaxAuthors, commit => commit.author.map(_.login).getOrElse(commit.commit.author.email))
.log("Commit")
.map(commit => AuthorStats(commit.gitAuthor, commit.githubAuthor, Authors.shaToStats(commit.sha)))
.map(commit => AuthorStats(commit.commit.author, commit.author, Authors.shaToStats(commit.sha)))
.reduce((aggr, elem) =>
aggr.copy(
stats = Stats(
Expand All @@ -143,7 +148,7 @@ object MarkdownConverter {
val authorId = author.githubAuthor.map { gh =>
// using html instead of markdown, because default
// avatars come from github not resized
s"""[<img width="20" alt="${gh.login}" src="${gh.avatar}&amp;s=40"/> **${gh.login}**](${gh.url})"""
s"""[<img width="20" alt="${gh.login}" src="${gh.avatarUrl}&amp;s=40"/> **${gh.login}**](${gh.htmlUrl})"""
} getOrElse {
author.gitAuthor.name
}
Expand Down
55 changes: 6 additions & 49 deletions core/src/main/scala/GithubProtocol.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,56 +16,13 @@

package lt.dvim.authors

import com.tradeshift.reaktive.json.JSONProtocol._
import com.tradeshift.reaktive.marshal.Protocol._
import io.vavr.control.{Option => VavrOption}
import lt.dvim.scala.compat.vavr.OptionConverters._
import io.circe.generic.extras.{Configuration, ConfiguredJsonCodec}

object GithubProtocol {
final case class GithubAuthor(login: String, url: String, avatar: String)
final case class GitAuthor(name: String, email: String)
final case class Stats(additions: Int, deletions: Int, commits: Int)
final case class Commit(sha: String, message: String, gitAuthor: GitAuthor, githubAuthor: Option[GithubAuthor])
final case class AuthorStats(gitAuthor: GitAuthor, githubAuthor: Option[GithubAuthor], stats: Stats)
implicit val config: Configuration = Configuration.default.withSnakeCaseMemberNames.withSnakeCaseConstructorNames

val compareProto =
`object`(
field(
"commits",
array(commitProto)
)
)

lazy val commitProto =
`object`(
option(
field("author", githubAuthorProto)
),
field(
"commit",
`object`(
field("author", gitAuthorProto),
field("message", stringValue),
(gitAuthor: GitAuthor, message: String) => (gitAuthor, message)
)
),
field("sha", stringValue),
(githubAuthor: VavrOption[GithubAuthor], gitAuthorAndMessage: (GitAuthor, String), sha: String) =>
Commit(sha, gitAuthorAndMessage._2, gitAuthorAndMessage._1, githubAuthor)
)

lazy val gitAuthorProto =
`object`(
field("name", stringValue),
field("email", stringValue),
GitAuthor.apply(_, _)
)

lazy val githubAuthorProto =
`object`(
field("login", stringValue),
field("html_url", stringValue),
field("avatar_url", stringValue),
GithubAuthor.apply(_, _, _)
)
@ConfiguredJsonCodec final case class GithubAuthor(login: String, htmlUrl: String, avatarUrl: String)
@ConfiguredJsonCodec final case class GitAuthor(name: String, email: String)
@ConfiguredJsonCodec final case class GitCommit(message: String, author: GitAuthor)
@ConfiguredJsonCodec final case class Commit(sha: String, commit: GitCommit, author: Option[GithubAuthor])
}
27 changes: 0 additions & 27 deletions core/src/main/scala/ScalaVavrCompat.scala
View file
Open in desktop

This file was deleted.

36 changes: 22 additions & 14 deletions core/src/test/scala/AuthorsSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,18 @@ import scala.concurrent.duration._

import akka.actor.ActorSystem
import akka.event.Logging
import akka.stream.alpakka.json.scaladsl.JsonReader
import akka.stream.scaladsl.{FileIO, Sink, Source}
import akka.testkit.TestKit

import com.tradeshift.reaktive.marshal.stream.{ActsonReader, ProtocolReader}
import com.typesafe.config.ConfigFactory
import org.mdedetrich.akka.stream.support.CirceStreamSupport
import org.scalatest.{BeforeAndAfterAll, Inside}
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike

import lt.dvim.authors.Authors._
import lt.dvim.authors.GithubProtocol._

object AuthorsSpec {
Expand All @@ -54,11 +56,22 @@ class AuthorsSpec
"parse compare json" in {
val res = FileIO
.fromPath(Paths.get(getClass.getResource("/compare.json").toURI), 64)
.via(ActsonReader.instance)
.via(ProtocolReader.of(GithubProtocol.compareProto))
.via(JsonReader.select("$.commits[*]"))
.via(CirceStreamSupport.decode[Commit])
.runFold(Seq.empty[Commit])(_ :+ _)

res.futureValue should have length 2
res.futureValue should contain theSameElementsInOrderAs Seq(
Commit(
"03ac2f41efff9cdfac9419ba7e6e34b30f9111e0",
GitCommit("Add some contents", GitAuthor("Martynas Mickevičius", "martynas@2m.lt")),
Some(GithubAuthor("2m", "https://github.com/2m", "https://avatars0.githubusercontent.com/u/422086?v=3"))
),
Commit(
"e5fee6fbc982cea605a820c82a8ae8f14ead26e0",
GitCommit("Add some other contents", GitAuthor("Test User", "test.user@2m.lt")),
None
)
)
}

"get commit stats from sha" in {
Expand All @@ -72,14 +85,12 @@ class AuthorsSpec
List(
Commit(
"f576a45",
"message",
GitAuthor("test", "test@test.lt"),
GitCommit("message", GitAuthor("test", "test@test.lt")),
Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
),
Commit(
"bce0e63",
"message",
GitAuthor("test", "test@test.lt"),
GitCommit("message", GitAuthor("test", "test@test.lt")),
Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
)
)
Expand All @@ -98,14 +109,12 @@ class AuthorsSpec
List(
Commit(
"f576a45",
"message",
GitAuthor("test", "test1@test.lt"),
GitCommit("message", GitAuthor("test", "test1@test.lt")),
Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
),
Commit(
"bce0e63",
"message",
GitAuthor("test", "test2@test.lt"),
GitCommit("message", GitAuthor("test", "test2@test.lt")),
Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
)
)
Expand All @@ -124,8 +133,7 @@ class AuthorsSpec
List(
Commit(
"901392a",
"message",
GitAuthor("test", "test1@test.lt"),
GitCommit("message", GitAuthor("test", "test1@test.lt")),
Some(GithubAuthor("test", "http://users/test", "http://avatars/test"))
)
)
Expand Down