Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement PubSub api #714

Merged
merged 58 commits into from
Jul 27, 2023
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
a280504
Add redis pubsub layer
0pg Dec 28, 2022
cd4a3f3
Fix PubSub api
0pg Feb 7, 2023
a01213d
Apply changes to t/c
0pg Feb 7, 2023
9541519
Set private accessor
0pg Feb 7, 2023
ba89a59
Merge remote-tracking branch 'upstream/master' into impl/pubsub
0pg Feb 7, 2023
9723182
Fix cleanup code
0pg Feb 7, 2023
65b524b
Modify callback and unsubscribe logic in SingleNodeRedisPubSub
0pg Feb 7, 2023
892ba20
Apply changes to t/c
0pg Feb 7, 2023
43edb47
Modify PubSub api
0pg Feb 7, 2023
a6f74fa
Apply changes to t/c
0pg Feb 7, 2023
c7decb7
Fix typo
0pg Feb 7, 2023
b6438f0
Modify accessor
0pg Feb 7, 2023
5fb75e4
Update redis/src/main/scala/zio/redis/Output.scala
0pg Feb 9, 2023
ce97f95
Update redis/src/main/scala/zio/redis/Output.scala
0pg Feb 9, 2023
a25805b
Modify responsibility to transform streams
0pg Feb 9, 2023
fd0b303
Apply changes to t/c
0pg Feb 9, 2023
7cb1491
Move SubscriptionKey into only used place
0pg Feb 10, 2023
2ab2db1
Merge remote-tracking branch 'upstream/master' into impl/pubsub
0pg Feb 10, 2023
87eee71
Fix onSubscribe callback race condition bug
0pg Feb 11, 2023
f532e4b
Add pacakge private accessor to PubSubCommand
0pg Feb 11, 2023
e9e2a3c
Modify unsubscrption release logic
0pg Feb 11, 2023
014d2e4
Merge remote-tracking branch 'upstream/master' into impl/pubsub
0pg Feb 14, 2023
49c141b
Modify RedisPubSubCommand interface
0pg Feb 14, 2023
8b6ce73
Apply changes to test
0pg Feb 14, 2023
f44c7fc
Add pubsub layer in Main
0pg Feb 14, 2023
758ea9b
Remove BinaryCodec layer to RedisPubSub interface
0pg Feb 20, 2023
9febcf8
Modify SingleNodeRedisPubSub failover step
0pg Feb 20, 2023
92aea5e
Merge remote-tracking branch 'upstream/master' into impl/pubsub
0pg Mar 26, 2023
3fc77c0
Separate Publish and Subscribe
0pg Mar 30, 2023
3c8a307
Rename classes
0pg Mar 30, 2023
0ea75a1
Reduce duplication
0pg Mar 31, 2023
ef3b9f1
Merge remote-tracking branch 'upstream/master' into impl/pubsub
0pg Mar 31, 2023
478577b
Apply upstream changes
0pg Mar 31, 2023
c4105ff
Replace Hub to Queue
0pg Apr 4, 2023
f432815
Merge remote-tracking branch 'upstream/master' into impl/pubsub
0pg Apr 25, 2023
149ae4b
Remove unused code
0pg Apr 27, 2023
67652e6
Refine subscription api
0pg Apr 27, 2023
8606409
Rename fields
0pg May 2, 2023
6a3f68f
Update modules/redis/src/main/scala/zio/redis/RedisSubscription.scala
0pg May 2, 2023
1a0862f
Add newline
0pg May 5, 2023
92457c4
Fix broken compile
0pg May 7, 2023
bf52e26
Formatting
0pg May 7, 2023
c027295
Fix lint
0pg May 7, 2023
a154683
Reduce duplications and refactor package layout
0pg May 10, 2023
592d327
Commit suggestion
0pg May 10, 2023
386fefd
Fix broken compile
0pg May 23, 2023
415f75e
Merge remote-tracking branch 'upstream/master' into impl/pubsub
0pg May 23, 2023
1699211
Use hub instead of chunks of queue
0pg May 30, 2023
be11d67
Add release on error
0pg May 30, 2023
552e002
Extraact common logic
0pg May 30, 2023
e218b08
Ensure order of subs/unsubs
0pg May 31, 2023
ac85cb9
Update modules/redis/src/main/scala/zio/redis/internal/SingleNodeSubs…
0pg Jun 7, 2023
dadb5e7
Use ConcurrentMap instead of Map of Ref
0pg Jun 7, 2023
88c3f4e
Add private accessor
0pg Jun 7, 2023
6e50d4d
Fix accessor
0pg Jun 7, 2023
5ca98e7
Add doc
0pg Jun 26, 2023
539ef3f
Add missing params
0pg Jun 26, 2023
63cc263
Fix test codes
0pg Jul 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ object BenchmarkRuntime {
private final val Layer =
ZLayer.make[Redis](
RedisExecutor.local,
RedisPubSub.local,
ZLayer.succeed[BinaryCodec](ProtobufCodec),
RedisLive.layer
)
Expand Down
2 changes: 1 addition & 1 deletion example/src/main/scala/example/Main.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import example.config.AppConfig
import sttp.client3.httpclient.zio.HttpClientZioBackend
import zhttp.service.Server
import zio._
import zio.redis.{RedisExecutor, RedisLive}
import zio.redis.{RedisExecutor, RedisLive, RedisPubSub}
import zio.schema.codec.{BinaryCodec, ProtobufCodec}

object Main extends ZIOAppDefault {
Expand Down
8 changes: 5 additions & 3 deletions redis/src/main/scala/zio/redis/ClusterExecutor.scala
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,11 @@ object ClusterExecutor {
} yield ExecutorScope(executor, closableScope)

private def redis(address: RedisUri) = {
val executorLayer = ZLayer.succeed(RedisConfig(address.host, address.port)) >>> RedisExecutor.layer
val codecLayer = ZLayer.succeed[BinaryCodec](StringUtf8Codec)
val redisLayer = executorLayer ++ codecLayer >>> RedisLive.layer
val redisConfigLayer = ZLayer.succeed(RedisConfig(address.host, address.port))
val codecLayer = ZLayer.succeed[BinaryCodec](StringUtf8Codec)
val executorLayer = redisConfigLayer >>> RedisExecutor.layer
val pubSubLayer = redisConfigLayer ++ codecLayer >>> RedisPubSub.layer
val redisLayer = executorLayer ++ pubSubLayer ++ codecLayer >>> RedisLive.layer
for {
closableScope <- Scope.make
layer <- closableScope.extend[Any](redisLayer.memoize)
Expand Down
46 changes: 46 additions & 0 deletions redis/src/main/scala/zio/redis/Output.scala
Original file line number Diff line number Diff line change
Expand Up @@ -827,4 +827,50 @@ object Output {
case other => throw ProtocolError(s"$other isn't an array")
}
}

case object PushProtocolOutput extends Output[PushProtocol] {
protected def tryDecode(respValue: RespValue)(implicit codec: BinaryCodec): PushProtocol =
respValue match {
case RespValue.NullArray => throw ProtocolError(s"Array must not be empty")
case RespValue.Array(values) =>
val name = MultiStringOutput.unsafeDecode(values(0))
val key = MultiStringOutput.unsafeDecode(values(1))
name match {
case "subscribe" =>
val num = LongOutput.unsafeDecode(values(2))
PushProtocol.Subscribe(key, num)
case "psubscribe" =>
val num = LongOutput.unsafeDecode(values(2))
PushProtocol.PSubscribe(key, num)
case "unsubscribe" =>
val num = LongOutput.unsafeDecode(values(2))
PushProtocol.Unsubscribe(key, num)
case "punsubscribe" =>
val num = LongOutput.unsafeDecode(values(2))
PushProtocol.PUnsubscribe(key, num)
case "message" =>
val message = values(2)
PushProtocol.Message(key, message)
0pg marked this conversation as resolved.
Show resolved Hide resolved
case "pmessage" =>
val channel = MultiStringOutput.unsafeDecode(values(2))
val message = values(3)
PushProtocol.PMessage(key, channel, message)
0pg marked this conversation as resolved.
Show resolved Hide resolved
case other => throw ProtocolError(s"$other isn't a pushed message")
}
case other => throw ProtocolError(s"$other isn't an array")
}
}

case object NumSubResponseOutput extends Output[Chunk[NumSubResponse]] {
protected def tryDecode(respValue: RespValue)(implicit codec: BinaryCodec): Chunk[NumSubResponse] =
respValue match {
case RespValue.Array(values) =>
Chunk.fromIterator(values.grouped(2).map { chunk =>
val channel = MultiStringOutput.unsafeDecode(chunk(0))
val numOfSubscription = LongOutput.unsafeDecode(chunk(1))
NumSubResponse(channel, numOfSubscription)
})
case other => throw ProtocolError(s"$other isn't an array")
}
}
}
29 changes: 29 additions & 0 deletions redis/src/main/scala/zio/redis/PubSubCommand.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package zio.redis

import zio.schema.codec.BinaryCodec
import zio.stream._
import zio.{IO, ZLayer}

final case class RedisPubSubCommand(command: PubSubCommand, codec: BinaryCodec, executor: RedisPubSub) {
def run: IO[RedisError, List[Stream[RedisError, PushProtocol]]] = {
val codecLayer = ZLayer.succeed(codec)
executor.execute(command).provideLayer(codecLayer)
}
}

sealed trait PubSubCommand

object PubSubCommand {
case class Subscribe(
channel: String,
channels: List[String],
onSubscribe: PubSubCallback
) extends PubSubCommand
case class PSubscribe(
pattern: String,
patterns: List[String],
onSubscribe: PubSubCallback
) extends PubSubCommand
case class Unsubscribe(channels: List[String]) extends PubSubCommand
case class PUnsubscribe(patterns: List[String]) extends PubSubCommand
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should hide these from the end user. These should be in a dedicated PubSubCommand file.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I moved it to its own file but do we need to add package private access modifier?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that we do if it's not part of the api interface.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added package private accessor

16 changes: 12 additions & 4 deletions redis/src/main/scala/zio/redis/Redis.scala
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,22 @@ trait Redis
with api.SortedSets
with api.Streams
with api.Scripting
with api.Cluster {
with api.Cluster
with api.PubSub {
def codec: BinaryCodec
def executor: RedisExecutor
def pubSub: RedisPubSub
}

final case class RedisLive(codec: BinaryCodec, executor: RedisExecutor) extends Redis
final case class RedisLive(codec: BinaryCodec, executor: RedisExecutor, pubSub: RedisPubSub) extends Redis

object RedisLive {
lazy val layer: URLayer[RedisExecutor with BinaryCodec, Redis] =
ZLayer.fromFunction(RedisLive.apply _)
lazy val layer: URLayer[RedisPubSub with RedisExecutor with BinaryCodec, Redis] =
ZLayer.fromZIO(
for {
codec <- ZIO.service[BinaryCodec]
executor <- ZIO.service[RedisExecutor]
pubSub <- ZIO.service[RedisPubSub]
} yield RedisLive(codec, executor, pubSub)
)
}
1 change: 1 addition & 0 deletions redis/src/main/scala/zio/redis/RedisEnvironment.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ import zio.schema.codec.BinaryCodec
private[redis] trait RedisEnvironment {
def codec: BinaryCodec
def executor: RedisExecutor
def pubSub: RedisPubSub
}
4 changes: 3 additions & 1 deletion redis/src/main/scala/zio/redis/RedisError.scala
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,7 @@ object RedisError {
object Moved {
def apply(slotAndAddress: (Slot, RedisUri)): Moved = Moved(slotAndAddress._1, slotAndAddress._2)
}
final case class IOError(exception: IOException) extends RedisError
final case class NoPubSubStream(key: String) extends RedisError
final case class NoUnsubscribeRequest(key: String) extends RedisError
final case class IOError(exception: IOException) extends RedisError
}
4 changes: 2 additions & 2 deletions redis/src/main/scala/zio/redis/RedisExecutor.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ trait RedisExecutor {

object RedisExecutor {
lazy val layer: ZLayer[RedisConfig, RedisError.IOError, RedisExecutor] =
RedisConnectionLive.layer >>> SingleNodeExecutor.layer
RedisConnectionLive.layer.fresh >>> SingleNodeExecutor.layer

lazy val local: ZLayer[Any, RedisError.IOError, RedisExecutor] =
RedisConnectionLive.default >>> SingleNodeExecutor.layer
RedisConnectionLive.default.fresh >>> SingleNodeExecutor.layer
}
22 changes: 22 additions & 0 deletions redis/src/main/scala/zio/redis/RedisPubSub.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package zio.redis

import zio.schema.codec.BinaryCodec
import zio.stream._
import zio.{ZIO, ZLayer}

trait RedisPubSub {
def execute(command: PubSubCommand): ZIO[BinaryCodec, RedisError, List[Stream[RedisError, PushProtocol]]]
}

object RedisPubSub {
lazy val layer: ZLayer[RedisConfig with BinaryCodec, RedisError.IOError, RedisPubSub] =
RedisConnectionLive.layer.fresh >>> pubSublayer

lazy val local: ZLayer[BinaryCodec, RedisError.IOError, RedisPubSub] =
RedisConnectionLive.default.fresh >>> pubSublayer

private lazy val pubSublayer: ZLayer[RedisConnection with BinaryCodec, RedisError.IOError, RedisPubSub] =
ZLayer.scoped(
ZIO.service[RedisConnection].flatMap(SingleNodeRedisPubSub.create(_))
)
}
5 changes: 5 additions & 0 deletions redis/src/main/scala/zio/redis/ResultBuilder.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package zio.redis
import zio.IO
import zio.redis.ResultBuilder.NeedsReturnType
import zio.schema.Schema
import zio.stream.Stream

sealed trait ResultBuilder {
final def map(f: Nothing => Any)(implicit nrt: NeedsReturnType): IO[Nothing, Nothing] = ???
Expand Down Expand Up @@ -46,4 +47,8 @@ object ResultBuilder {
trait ResultOutputBuilder extends ResultBuilder {
def returning[R: Output]: IO[RedisError, R]
}

trait ResultStreamBuilder[+F[_]] {
def returning[R: Schema]: IO[RedisError, F[Stream[RedisError, R]]]
}
}
Loading