scalaz-stream

 

Gleb Kanterov

 

gleb@kanterov.ru

@kanterov

 
  • Push-like Streams
    • JavaRx
    • AkkaStream

 

  • Pull-like Streams
    • Haskell Machines
    • scalaz-streams

 

1

Pull vs. Push

 

scalaz-stream

 

Types

 
sealed trait Process[+F[_], +A]

class Task[+A]

val digits: Process[Nothing, Int] = 
  Process.emitAll(List(0, 1, 2, 3, 4, 5, 6, 7, 8, 9))

val tweets: Process[Task, Tweet]

Task vs. Future

 
val hello = Task.delay { println("hello"); 10 }
val world = Task.delay { println("world"); 20 }

// output is fully deterministic

val task = for {
  x <- hello
  y <- world
} yield x + y

task.run

> hello
> world
> 30

task.run

> hello
> world
> 30
val hello = Future { println("hello"); 10 }
val world = Future { println("world"); 20 }

// output is non-deterministic and happens once

> world

val future = for {
  x <- hello
  y <- world
} yield x + y

future.get
> hello
> 30

future.get
> 30

Non-determinism in Task

 
val hello = Task.delay { println("hello"); 10 }
val world = Task.delay { println("world"); 20 }

val task = Nondeterminism[Task].both(hello, world)

task.run
> hello
> world

task.run
> world
> hello

Task

 
  • Fully lazy

 

  • Deterministic by default

 

  • Easy to combine

 

  • Declarative, can be executed multiple times

 

  • Easy to move between thread pools

 

 

Process

 
  • Ordered sequence of actions

 

  • Deterministic by default

 

  • Easy to combine

 

  • Declarative, can be executed multiple times

 

 

Process[Task, T]

 
case class Tweet(username: String, text: String)
case class User(username: String, following: Boolean)

def lastTweet(username: String): Task[Tweet]

val users: Process[Nothing, User] = 
  Process.emitAll(List(
    User("@scala",  following = true),
    User("@github", following = true),
    User("@ruby",   following = false)
  ))

val tweets: Process[Task, Tweet] = 
  users.filter(_.following).flatMap { user =>
    Process.eval(lastTweet(user))
  }

val printTweets = (tweets.map(_.text) to io.stdOutLines).run

printTweets.run

Process#append

 
val xs = Process.emitAll(List(1, 2, 3))
val ys = Process.emitAll(List(4, 5, 6))

val zs = xs.append(ys)

zs.runLog.run
> Vector(1, 2, 3, 4, 5, 6)

Process#zip

 
val xs = Process.emitAll(List(1, 2, 3))
val ys = Process.emitAll(List("a", "b"))

xs.zip(ys).runLog.run
> Vector((1, "a"), (2, "b"))

Process#tee

 
def awaitL[T]: Tee[T, Any, T]
def awaitR[T]: Tee[Any, T, T]

def zip[T]: Tee[T, T, (T, T)] = {
  for {
    left  <- awaitL[T]
    right <- awaitR[T]
    r     <- Process.emit((left, right))
  }
}

xs.tee(ys).runLog.run
> Vector((1, "a"), (2, "b"))

Process#tee

 
type Tee[L, R, A] = Process[ReadTee[L, R, ?], A]

sealed trait ReadTee[L, R, A]

object ReadTee {
  case class Left[L]() extends ReadTee[L, Any, L]
  case class Right[L]() extends ReadTee[Any, R, R]
}

def awaitL[T]: Tee[T, Any, T]
def awaitR[T]: Tee[Any, T, T]

def zip[T]: Tee[T, T, (T, T)] = {
  for {
    left  <- awaitL[T]
    right <- awaitR[T]
    r     <- Process.emit((left, right))
  }
}

xs.tee(ys)(zip).runLog.run
> Vector((1, "a"), (2, "b'))

Process#merge

 
val xs = Process.emitAll(List(2, 4, 6))
val ys = Process.emitAll(List(1, 3, 5))

xs.merge(ys).runLog.run
> Vector(1, 2, 3, 4, 5, 6)

xs.merge(ys).runLog.run
> Vector(1, 3, 2, 4, 6, 5)

Process#wye

 
type Wye[L, R, A] = Process[ReadWye[L, R, ?], A]

sealed trait ReadWye[+L, +R, +A]

object ReadWye {
  case class Left[L]() extends ReadWye[L, Any, L]
  case class Right[R]() extends ReadWye[Any, R, R]
  case class Both[L, R]() extends ReadWye[L, R, Receive[L, R]]
}

sealed trait Receive[+L, +R]

object Receive {
  case class ReceiveL[+L](get: L) extends Receive[L, Nothing]
  case class ReceiveR[+R](get: R) extends Receive[Nothing, R]

  case class HaltL(cause: Cause) extends Receive[Nothing, Nothing]
  case class HaltR(cause: Cause) extends Receive[Nothing, Nothing]
}

def receiveBoth[I, I2, O](rcv: ReceiveY[I, I2] => Wye[I, I2, O]): Wye[I,I2,O] = {
  Process.eval(ReadWye.Both()).flatMap(rcv)
}

def merge[I]: Wye[I,I,I] =
  receiveBoth {
    case ReceiveL(i)  => emit(i) ++ merge
    case ReceiveR(i)  => emit(i) ++ merge
    case HaltL(End)   => awaitR.repeat
    case HaltR(End)   => awaitL.repeat
    case HaltOne(rsn) => Halt(rsn)
  }

Process#merge

 
/**
 * Non-deterministic interleave of both inputs. Emits values whenever either
 * of the inputs is available.
 *
 * Will terminate once both sides terminate.
 */
def merge[I]: Wye[I,I,I] =
  receiveBoth {
    case ReceiveL(i)  => emit(i) ++ merge
    case ReceiveR(i)  => emit(i) ++ merge
    case HaltL(End)   => awaitR.repeat
    case HaltR(End)   => awaitL.repeat
    case HaltOne(rsn) => Halt(rsn)
  }

/**
 * Like `merge`, but terminates whenever one side terminate.
 */
def mergeHaltBoth[I]: Wye[I,I,I] =
  receiveBoth {
    case ReceiveL(i)  => emit(i) ++ mergeHaltBoth
    case ReceiveR(i)  => emit(i) ++ mergeHaltBoth
    case HaltOne(rsn) => Halt(rsn)
  }

Process#wye

 
val xs = Process.emitAll(List(2, 4, 6))
val ys = Process.emitAll(List(1, 3, 5))

def merge[I]: Wye[I,I,I] =
  receiveBoth {
    case ReceiveL(i)  => emit(i) ++ merge
    case ReceiveR(i)  => emit(i) ++ merge
    case HaltL(End)   => awaitR.repeat
    case HaltR(End)   => awaitL.repeat
    case HaltOne(rsn) => Halt(rsn)
  }

xs.wye(ys)(merge).runLog.run
> Vector(1, 2, 3, 4, 5, 6)

Process#resource

 
def resource[A, O](
  acquire: Task[A])(
  release: A => Task[Unit])(
  step: A => Task[O]
): Process[Task, O]

val acquire: Task[Source] = Task.delay { Source.fromFile("filename") }

def release(source: Source): Task[Unit] = Task.delay { source.close() }

def step(source: Source): Task[String] = {
  val lines: Iterator[String] = source.getLines
  Task.delay { it.next }
}

val lines: Process[Task, String] = resource(acquire)(release)(step)

Sink

 
def to(
  source: Process[Task, A], 
  sink: Sink[Task, A]
): Process[Task, Unit]

val lines: Process[Task, String] = 
  Process.emitAll(List("Hello", "World"))

val stdOutLines: Sink[Task, String] = sink.lift { line =>
  Task.delay { println(line) }
}

lines.to.stdOutLines.run.run

> Hello
> World

Sink

 
type Sink[F[_], A] = Process[F, A => F[Unit]]

def to(
  source: Process[Task, A], 
  sink: Process[Task, A => Task[Unit]]
): Process[Task, Unit] = 
  source.zipWith(sink).flatMap {
    case (el, f) => Process.eval(f(el))
  }

val lines: Process[Task, String] = 
  Process.emitAll(List("Hello", "World"))

val stdOutLines: Sink[Task, String] = sink.lift { line =>
  Task.delay { println(line) }
}

lines.to.stdOutLines.run.run

> Hello
> World

Channel

 
type Channel[F[_], -I, +O] = Process[F, I => F[O]]
type Sink[F[_], -I] = Channel[F, I, Unit]

def through[I, O](
  source: Process[Task, I], 
  channel: Process[Task, I => Task[O]]
): Process[Task, O] = 
  source.zipWith(sink).flatMap {
    case (el, f) => Process.eval(f(el))
  }

val tweets: Channel[Task, String, Tweet]

val usernames: Process[Task, String] =
  Process.emitAll(List("@scala", "@github"))

val tweets: Process[Task, Tweet] = usernames.through(tweets)

Tcp Echo Server

 
val server = merge.mergeN(Netty.server(address).map { incoming =>
  for {
    exchange <- incoming
    _ <- Process.eval(log("New connection"))
    _ <- exchange.read.to(exchange.write)
  } yield ()
})

// incoming
Process[Task, Exchange[ByteVector, ByteVector]]

case class Exchange[I, W](
  read: Process[Task, I], 
  write: Sink[Task, W]
)

// Netty.server(address)
Process[Task, Process[Task, Exchange[ByteVector, ByteVector]]]

// exchange.read.to(exchange.write)
Process[Task, Unit]

// mergeN: Process[Task, Process[Task, Unit]] => Process[Task, Unit]

Tcp Chat

 
val relay = async.topic[ByteVector]()

val server = merge.mergeN(Netty.server(address).map { incoming =>
  for {
    exchange <- incoming

    in  = exchange.read.to(relay.publish)
    out = relay.subscribe.to(exchange.write)

    _ <- in.merge(out)
  } yield ()
})

server.run.run

// relay.publish
Sink[Task, ByteVector]

// relay.subscribe
Process[Task, ByteVector]

HTTP Chat

 
val relay = async.topic[String]()

val server = BlazeBuilder.bindHttp(host = "localhost", port = 9002)
  .mountService(HttpService {
    case GET -> Root / "chat" =>
      Ok(relay.subscribe.map(_ + "\n"), 
        Headers(`Transfer-Encoding`(TransferCoding.chunked)))

    case req @ POST -> Root / "chat" =>
      req.decode[String] { message =>
        for {
          _ <- relay.publishOne(message)
          response <- Ok(s"Sent: $message\n")
        } yield response
      }
  })

server.run.awaitShutdown()

Ecosystem

 
  • http4s
  • doobie (jdbc)
  • scodec-stream
  • scalaz-stream-netty
  • others
1

FS2

 
  • scalaz-stream 0.9 -> FS2
  • New Algebra
  • Abstracted away from Task
  • Better performance
1

scalaz-stream

 
  • Declarative
  • Deterministic
  • Composable
  • Resource-safe
  • IO-friendly
 

Thanks! Q&A

 

scalaz-stream

By Gleb Kanterov