JavaScript Iteration Protocols

Luciano Mammino (@loige)

loige.link/iter-summit

2023-09-20

loige

A new adventure awaits you!

Grab the slides!

loige.link/iter-summit

loige

You just joined a new STARTUP.
This is DAY 1!

Grab the slides!

loige.link/iter-summit

loige

CTO: Can you help me with troubleshooting something?

Grab the slides!

loige.link/iter-summit

loige

CTO: I am debugging a PRODUCTION issue.

Grab the slides!

loige.link/iter-summit

loige

CTO: We need to count how many "ERR_SYS_FCKD" there are per customer 

Grab the slides!

loige.link/iter-summit

loige

{"requestId":"d2328daa...","level":"ERROR","timestamp":"2023-05-26T23:18:24.834Z","customerId":"01H1D5NGA5D689HD57CN5BZSG3","error":"ERR_SYS_FCKD"}

{"requestId":"c4fe5fdd...","level":"INFO","timestamp":"2023-05-26T23:18:29.966Z","customerId":"01H1D5MZAVPSXSXPTYADF4BDND","message":"transaction in progress"}

{"requestId":"69733d2e...","level":"ERROR","timestamp":"2023-05-26T23:18:33.570Z","customerId":"01H1D5MZAVPSXSXPTYADF4BDND","error":"ERR_SYS_FCKD"}

{"requestId":"2845fc48...","level":"ERROR","timestamp":"2023-05-26T23:18:40.489Z","customerId":"01H1D5KGHRV3SNJ71C4E920872","error":"ERR_SYS_FCKD"}

{"requestId":"3e3e49b5...","level":"ERROR","timestamp":"2023-05-26T23:18:50.277Z","customerId":"01H1D5KSG0K81CCAGTSTBXHFA6","error":"ERR_CLIENT_TIMEOUT"}

{"requestId":"f28abf98...","level":"ERROR","timestamp":"2023-05-26T23:18:56.575Z","customerId":"01H1D5K2JDVD941R6H67YEY0G8","error":"ERR_BUSY"}

{"requestId":"608e579f...","level":"ERROR","timestamp":"2023-05-26T23:19:04.529Z","customerId":"01H1D5KSG0K81CCAGTSTBXHFA6","error":"ERR_NOT_FOUND"}

{"requestId":"3125588e...","level":"ERROR","timestamp":"2023-05-26T23:19:11.514Z","customerId":"01H1D5MJ4XNJ580DMM8Y7T1HRW","error":"ERR_SERVER_TIMEOUT"}

{"requestId":"40fb0329...","level":"INFO","timestamp":"2023-05-26T23:19:13.536Z","customerId":"01H1D5N6HS2EMA900A2V6NQQ2Q","message":"user logged in"}

{"requestId":"564afe31...","level":"ERROR","timestamp":"2023-05-26T23:19:21.708Z","customerId":"01H1D5K2JDVD941R6H67YEY0G8","error":"ERR_CLIENT_TIMEOUT"}

loige

Let's do it!

Grab the slides!

loige.link/iter-summit

loige

import { readFile } from 'node:fs/promises'

const rawData = await readFile('logs.jsonl', 'utf-8')
const lines = rawData.split(/\n+/)
const messages = lines.map(line => line === '' ? {} : JSON.parse(line))
const errors = messages.filter(message => message.error === 'ERR_SYS_FCKD')
const errorsByCustomer = errors.reduce((acc, error) => {
  if (!acc[error.customerId]) {
    acc[error.customerId] = 0
  }
  acc[error.customerId]++
  return acc
}, {})

console.log('Errors by customer:')
console.table(errorsByCustomer)

loige

Errors by customer:

{
  '01H1D5NGA5D689HD57CN5BZSG3': 109,
  '01H1D5MZAVPSXSXPTYADF4BDND': 118,
  '01H1D5KGHRV3SNJ71C4E920872': 118,
  '01H1D5KSG0K81CCAGTSTBXHFA6': 95,
  '01H1D5MJ4XNJ580DMM8Y7T1HRW': 103,
  '01H1D5N6HS2EMA900A2V6NQQ2Q': 96,
  '01H1D5M5RTKKK5SC4ADWAPK7Q0': 130,
  '01H1D5K2JDVD941R6H67YEY0G8': 115,
  '01H1D5KZ7GV7HK213XE3WV5GQX': 113
}

loige

CTO: Great job! Exactly what I needed! 

Grab the slides!

loige.link/iter-summit

loige

DAY 2

Grab the slides!

loige.link/iter-summit

loige

CTO: that script it's now giving "RangeError: Invalid string length"!

Grab the slides!

loige.link/iter-summit

loige

import { readFile } from 'node:fs/promises'

const rawData = await readFile('logs.jsonl', 'utf-8')
const lines = rawData.split(/\n+/)
const messages = lines.map(line => line === '' ? {} : JSON.parse(line))
const errors = messages.filter(message => message.error === 'ERR_SYS_FCKD')
const errorsByCustomer = errors.reduce((acc, error) => {
  if (!acc[error.customerId]) {
    acc[error.customerId] = 0
  }
  acc[error.customerId]++
  return acc
}, {})

console.log('Errors by customer:')
console.table(errorsByCustomer)

loige

import { readFile } from 'node:fs/promises'

const rawData = await readFile('logs.jsonl', 'utf-8')
const lines = rawData.split(/\n+/)
const messages = lines.map(line => line === '' ? {} : JSON.parse(line))
const errors = messages.filter(message => message.error === 'ERR_SYS_FCKD')
const errorsByCustomer = errors.reduce((acc, error) => {
  if (!acc[error.customerId]) {
    acc[error.customerId] = 0
  }
  acc[error.customerId]++
  return acc
}, {})

console.log('Errors by customer:')
console.table(errorsByCustomer)

We are loading everything into memory...

If the file is too big this will fail!

loige

Ideal approach

Lazy processing, line by line

Current line:

Current state:

Parse
Filter
Aggregate

//
{
  
}

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"d2328daa...","level":"ERROR","timestamp":"2023-05-26T23:18:24.834Z","customerId":"01H1D5NGA5D689HD57CN5BZSG3","error":"ERR_SYS_FCKD"}
{
  
}

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"d2328daa...","level":"ERROR","timestamp":"2023-05-26T23:18:24.834Z","customerId":"01H1D5NGA5D689HD57CN5BZSG3","error":"ERR_SYS_FCKD"}
{
  
}

OK

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"d2328daa...","level":"ERROR","timestamp":"2023-05-26T23:18:24.834Z","customerId":"01H1D5NGA5D689HD57CN5BZSG3","error":"ERR_SYS_FCKD"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1
}

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"c4fe5fdd...","level":"INFO","timestamp":"2023-05-26T23:18:29.966Z","customerId":"01H1D5MZAVPSXSXPTYADF4BDND","message":"transaction in progress"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1
}

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"c4fe5fdd...","level":"INFO","timestamp":"2023-05-26T23:18:29.966Z","customerId":"01H1D5MZAVPSXSXPTYADF4BDND","message":"transaction in progress"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1
}

NOPE

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"69733d2e...","level":"ERROR","timestamp":"2023-05-26T23:18:33.570Z","customerId":"01H1D5MZAVPSXSXPTYADF4BDND","error":"ERR_SYS_FCKD"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1
}

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"69733d2e...","level":"ERROR","timestamp":"2023-05-26T23:18:33.570Z","customerId":"01H1D5MZAVPSXSXPTYADF4BDND","error":"ERR_SYS_FCKD"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1
}

OK

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"69733d2e...","level":"ERROR","timestamp":"2023-05-26T23:18:33.570Z","customerId":"01H1D5MZAVPSXSXPTYADF4BDND","error":"ERR_SYS_FCKD"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1,
  "01H1D5MZAVPSXSXPTYADF4BDND": 1
}

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"2845fc48...","level":"ERROR","timestamp":"2023-05-26T23:18:40.489Z","customerId":"01H1D5NGA5D689HD57CN5BZSG3","error":"ERR_SYS_FCKD"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1,
  "01H1D5MZAVPSXSXPTYADF4BDND": 1
}

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"2845fc48...","level":"ERROR","timestamp":"2023-05-26T23:18:40.489Z","customerId":"01H1D5NGA5D689HD57CN5BZSG3","error":"ERR_SYS_FCKD"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 1,
  "01H1D5MZAVPSXSXPTYADF4BDND": 1
}

OK

Ideal approach

Lazy processing, line by line

loige

Current line:

Current state:

Parse
Filter
Aggregate

{"requestId":"2845fc48...","level":"ERROR","timestamp":"2023-05-26T23:18:40.489Z","customerId":"01H1D5NGA5D689HD57CN5BZSG3","error":"ERR_SYS_FCKD"}
{
  "01H1D5NGA5D689HD57CN5BZSG3": 2,
  "01H1D5MZAVPSXSXPTYADF4BDND": 1
}

Ideal approach

Lazy processing, line by line

loige

We can do this using Iterators!

Player 1

I'm Luciano

Senior Architect @ fourTheorem (Dublin)

nodejsdp.link

Co-Author of Node.js Design Patterns

Let's connect!

  loige.co (blog)

  @loige (twitter)

  loige (twitch)

  lmammino (github)

Always re-imagining

We are a pioneering technology consultancy focused on Cloud, AWS & serverless

loige

fth.link/careers

We can help with:

Cloud Migrations

Training & Cloud enablement

Building serverless applications

Cutting cloud costs

loige

Why Iterators?

  • Lazy abstraction to represent repetition

  • You can consume the collection 1 item at the time

  • Great for large (or endless) datasets

loige

The concept of Iterators exists already in JavaScript...

Good news...

... since ES2015!

loige

const array = ['foo', 'bar', 'baz']

for (const item of array) {
  console.log(item)
}

 

Prints all the items in the array!

Does it need to be an array? 🤔

Output:
foo
bar
baz

loige

const str = 'foo'

for (const item of str) {
  console.log(item)
}
Output:
f
o
o

loige

const obj = {
  foo: 'bar',
  baz: 'qux'
}

for (const item of obj) {
  console.log(item)
}
Output:
⛔️ Uncaught TypeError: obj is not iterable

OMG `for ... of`

does not work with plain objects! 😱

loige

loige

const array = ['foo', 'bar', 'baz']

console.log(...array)
Output:
foo bar baz

spread syntax!

loige

loige

Generators

loige

Generator fn & obj

loige

function * myGenerator () {
  // generator body
  yield 'someValue'
  // ... do more stuff
}
const genObj = myGenerator()
genObj.next() // -> { done: false, value: 'someValue' }

loige

function * fruitGen () {
  yield '🍑'
  yield '🍉'
  yield '🍋'
  yield '🥭'
}

const fruitGenObj = fruitGen()
console.log(fruitGenObj.next()) // { value: '🍑', done: false }
console.log(fruitGenObj.next()) // { value: '🍉', done: false }
console.log(fruitGenObj.next()) // { value: '🍋', done: false }
console.log(fruitGenObj.next()) // { value: '🥭', done: false }
console.log(fruitGenObj.next()) // { value: undefined, done: true }

loige

function * fruitGen () {
  yield '🍑'
  yield '🍉'
  yield '🍋'
  yield '🥭'
}

const fruitGenObj = fruitGen()
// generator objects are iterable!
for (const fruit of fruitGenObj) {
  console.log(fruit)
}

// 🍑
// 🍉
// 🍋
// 🥭

Iterators & Iterables

loige

Iterator Obj

loige

An object that acts like a cursor to iterate over blocks of data sequentially

Iterable Obj

loige

An object that provides data that can be iterated over sequentially

The iterator protocol

Iterator object:

 

next() method returns:

  • done (boolean)

  • value (any)

loige

const iterator = {
  next () {
    return { 
      done: false,
      value: "someValue"
    }
  }
}
function createCountdown (from) {
  let nextVal = from
  return {
    next () {
      if (nextVal < 0) {
        return { done: true }
      }

      return { 
        done: false,
        value: nextVal--
      }
    }
  }
}

loige

A factory function that creates an iterator

returns an object

... which has a next() method

... which returns an object with

done & value

const countdown = createCountdown(3)
console.log(countdown.next())
// { done: false, value: 3 }

console.log(countdown.next())
// { done: false, value: 2 }

console.log(countdown.next())
// { done: false, value: 1 }

console.log(countdown.next())
// { done: false, value: 0 }

console.log(countdown.next())
// { done: true }

loige

function createCountdown (from) {
  let nextVal = from
  return {
    next () {
      if (nextVal < 0) {
        return { done: true }
      }

      return { 
        done: false,
        value: nextVal--
      }
    }
  }
}

loige

Generator objects implement the iterator protocol

function * createCountdown (from) {
  for (let i = from; i >= 0; i--) {
    yield i
  }
}

Equivalent code using generators

The iterable protocol

loige

Iterable object:

 

Symbol.iterator() method returns:

  • an iterator

const iterable = {
  [Symbol.iterator] () {
    return { 
      // iterator
      next () {
        return { 
          done: false,
          value: "someValue"
        }
      }
    }
  }
}
function createCountdown (from) {
  let nextVal = from
  return {
    [Symbol.iterator]: () => ({
      next () {
        if (nextVal < 0) {
          return { done: true }
        }

        return { done: false, value: nextVal-- }
      }
    })
  }
}

loige

[...createCountdown(3)] // [3,2,1,0]

for (const value of createCountdown(3)) {
  console.log(value)
}

// 3
// 2
// 1
// 0

loige

We can use spread and for ... of with iterable objects!

loige

function createCountdown (from) {
  let nextVal = from
  return {
    [Symbol.iterator]: () => ({
      next () {
        if (nextVal < 0) {
          return { done: true }
        }

        return {
          done: false, 
          value: nextVal--
        }
      }
    })
  }
}

Generator objects implement the iterable protocol

function * createCountdown (from) {
  for (let i = from; i >= 0; i--) {
    yield i
  }
}

Equivalent code using generators

YES, it's the same code as the iterator example!

const iterableIterator = {
  next () {
    return { done: false, value: 'hello' }
  },
  [Symbol.iterator] () {
    return this
  }
}

loige

An object can be both an iterator and an iterable!

Iterator protocol

Iterable protocol

loige

ASYNC
Iterators & Iterables

loige

The async iterator protocol

Async Iterator object:

 

next() method returns a

Promise that resolves to:

  • done (boolean)

  • value (any)

const asyncIterator = {
  async next () {
    return { 
      done: false,
      value: "someValue"
    }
  }
}

loige

The async iterable protocol

Async Iterable object:

 

Symbol.asyncIterator() method returns:

  • an async iterator

const asyncIterable = {
  [Symbol.asyncIterator] () {
    return { 
      // iterator
      async next () {
        return { 
          done: false,
          value: "someValue"
        }
      }
    }
  }
}
import { setTimeout } from 'node:timers/promises'

async function * createAsyncCountdown (from, delay = 1000) {
  for (let i = from; i >= 0; i--) {
    await setTimeout(delay)
    yield i
  }
}

loige

Async Generator objects implement the async iterator & async iterable protocols

const countdown = createAsyncCountdown(3)

for await (const value of countdown) {
  console.log(value)
}

loige

We can use for await ... of with async iterable objects!

loige

Let's rewrite our log analyser to use iterators!

import { createReadStream } from 'node:fs'

const readable = createReadStream(
  'logs.jsonl',
  { encoding: 'utf-8' }
)

// a readable stream is an Async Iterable!
for await (const chunk of readable) {
  // do something with a chunk of data
}

loige

We can read a file incrementally using Node.js streams!

import { createReadStream } from 'node:fs'

const readable = createReadStream(
  'logs.jsonl',
  { encoding: 'utf-8' }
)

// a readable stream is an Async Iterable!
for await (const chunk of readable) {
  // do something with a chunk of data
}

loige

We can read a file incrementally using Node.js streams!

A chunk is an arbitrary amount of data (not necessarily a line)

// utils/byline.js

export async function * byLine (asyncIterable) {
  let remainder = ''
  for await (const chunk of asyncIterable) {
    const lines = (remainder + chunk).split(/\n+/)
    remainder = lines.pop()
    yield * lines
  }
  if (remainder.length > 0) {
    yield remainder
  }
}

loige

import { createReadStream } from 'node:fs'
import { byLine } from './utils/byline.js'

const readable = createReadStream('logs.jsonl', { encoding: 'utf-8' })

const errorsByCustomer = {}

for await (const line of byLine(readable)) {
  const message = line === '' ? {} : JSON.parse(line)
  if (message.error === 'ERR_SYS_FCKD') {
    if (!errorsByCustomer[message.customerId]) {
      errorsByCustomer[message.customerId] = 0
    }
    errorsByCustomer[message.customerId]++
  }
}

console.log('Errors by customer:')
console.log(errorsByCustomer)

loige

import { createReadStream } from 'node:fs'
import { byLine } from './utils/byline.js'

const readable = createReadStream('logs.jsonl', { encoding: 'utf-8' })

const errorsByCustomer = {}

for await (const line of byLine(readable)) {
  const message = line === '' ? {} : JSON.parse(line)
  if (message.error === 'ERR_SYS_FCKD') {
    if (!errorsByCustomer[message.customerId]) {
      errorsByCustomer[message.customerId] = 0
    }
    errorsByCustomer[message.customerId]++
  }
}

console.log('Errors by customer:')
console.log(errorsByCustomer)

loige

CTO: It works with

any file now! 

Can't we use .map(), .filter(), .reduce()?

loige

BONUS Material

loige

loige.link/streams-wrk

loige.link/iter-wrk

SLIDES: loige.link/iter-summit

Code: loige.link/iter-summit-code

Original Front cover photo by Jörg Angeli on Unsplash

Original Background photo by Michael Behrens on Unsplash

Thanks!

loige

nodejsdp.link

loige.link/iter-summit