Data Transformations with Lodash

Kyle Coberly

kylecoberly.com

"Compose a data transformation pipeline with `lodash/fp`"

What's a data transformation?

{
  users: {
    1: {
      name: "Amelia Bedelia",
    },
    2: {
      name: "Bob Barker",
    }
  },
  availabilities: [{
    date: "2020-09-01",
    startTime: 12,
    duration: 1,
    user_id: 1,
  },{
    date: "2020-09-01",
    startTime: 14,
    duration: 2,
    user_id: 1,
  },{
    date: "2020-09-03",
    startTime: 18,
    duration: 2,
    user_id: 2,
  }],
}
{
  users: [{
    id: 1,
    name: "Amelia Bedelia",
    days: {
      "2020-09-01": [{
        startTime: 12,
        duration: 1,
      },{
        startTime: 14,
        duration: 2,
      }],
    },
  },{
    id: 2,
    name: "Bob Barker",
    days: {
      "2020-09-03": [{
        startTime: 18,
        duration: 2,
      }],
    },
  }],
}

Common Transformations

  • map/flatMap/mapValues - Same count, but changed
  • filter/reject/where - Same or fewer count
  • find - First match
  • groupBy - Turn values into keys
  • assign/merge/concat - Combine two things

What is Lodash?

Stop Writing Functions That...

  • Get the difference between two arrays
  • Map over the values of an object
  • Debounce a function
  • Get random values from an array
  • Sort a collection by a property
  • Clamp numbers

Functional Functions

// Plain ole JS
[1, 2, 3].map(number => number * 2)
// Sexy Lodash
const { map } = require("lodash/fp")

map(number => number *2 )([1, 2, 3])

Not a method

Curried

Data Last

What is composing?

Custom Functions!

const { map } = require("lodash/fp")

const double = number => number * 2
const doubleAll = map(double)

doubleAll([1, 2, 3])

What is a pipeline?

Transformation Pipelines!!

const { flow } = require("lodash/fp")

const someMath = flow([
  doubleAll,
  doubleAll,
  onlyOver10,
  halfAll
  sum,  
])

someMath(numbers)
{
  users: {
    1: {
      name: "Amelia Bedelia",
    },
    2: {
      name: "Bob Barker",
    }
  },
  availabilities: [{
    date: "2020-09-01",
    startTime: 12,
    duration: 1,
    user_id: 1,
  },{
    date: "2020-09-01",
    startTime: 14,
    duration: 2,
    user_id: 1,
  },{
    date: "2020-09-03",
    startTime: 18,
    duration: 2,
    user_id: 2,
  }],
}
{
  users: [{
    id: 1,
    name: "Amelia Bedelia",
    days: {
      "2020-09-01": [{
        startTime: 12,
        duration: 1,
      },{
        startTime: 14,
        duration: 2,
      }],
    },
  },{
    id: 2,
    name: "Bob Barker",
    days: {
      "2020-09-03": [{
        startTime: 18,
        duration: 2,
      }],
    },
  }],
}

The Goal

function transformImperative({ users, availabilities }){
  const transformedUsers = []

  for (let id in users){
    const user = users[id]
    user.id = +id
    user.days = {}

    for (let availability of availabilities){
      if (availability.user_id != id){
        continue;
      }
      const date = availability.date

      if (!user.days[date]){
        user.days[date] = []
      }

      delete availability.date
      delete availability.user_id

      user.days[date].push(availability)
    }

    transformedUsers.push(user)
  }

  return { users: transformedUsers }
}

function transformImperative({ users, availabilities }){
  const transformedUsers = []

  for (let id in users){
    const user = users[id]
    user.id = +id // Mutation
    user.days = {} // Mutation

    for (let availability of availabilities){
      if (availability.user_id != id){
        continue;
      }
      const date = availability.date

      if (!user.days[date]){
        user.days[date] = [] // Mutation
      }

      delete availability.date // Mutation
      delete availability.user_id // Mutation

      user.days[date].push(availability) // Mutation
    }

    transformedUsers.push(user) // Mutation
  }

  return { users: transformedUsers }
}

function transformImperative({ users, availabilities }){
  const transformedUsers = []

  for (let id in users){
    const user = users[id]
    user.id = +id // Mutation
    user.days = {} // Mutation

    for (let availability of availabilities){
      if (availability.user_id != id){
        continue;
      }
      const date = availability.date

      if (!user.days[date]){
        user.days[date] = [] // Mutation
      }

      delete availability.date // Mutation
      delete availability.user_id // Mutation
      // vvv ALSO RELIES ON MUTABLE STATE!! vvv
      user.days[date].push(availability) // Mutation
    }

    transformedUsers.push(user) // Mutation
  }

  return { users: transformedUsers }
}

Imperative Solution

Imperative Solution

  • Hard to decompose
  • Hard to test
  • Hard to debug
  • Hard to reuse
  • Hard to read

Step 1: Group Availabilities by User

[{
  date: "2020-09-01",
  startTime: 12,
  duration: 1,
  user_id: 1,
},{
  date: "2020-09-01",
  startTime: 14,
  duration: 2,
  user_id: 1,
},{
  date: "2020-09-03",
  startTime: 18,
  duration: 2,
  user_id: 2,
}]
{
  1: [{
    date: "2020-09-01",
    startTime: 12,
    duration: 1,
  },{
    date: "2020-09-01",
    startTime: 14,
    duration: 2,
  }],
  2: [{
    date: "2020-09-03",
    startTime: 18,
    duration: 2,
  }]
}

Step 2: Group Availabilities by Date

[{
  date: "2020-09-01",
  startTime: 12,
  duration: 1,
},{
  date: "2020-09-01",
  startTime: 14,
  duration: 2,
},{
  date: "2020-09-03",
  startTime: 18,
  duration: 2,
}]
{
  "2020-09-01": [{
    startTime: 12,
    duration: 1,
  },{
    startTime: 14,
    duration: 2,
  }],
  "2020-09-03": [{
    startTime: 18,
    duration: 2,
  }],
}

Step 3: Wrap Availabilities

{
  1: {
    "2020-09-01": [{
      startTime: 12,
      duration: 1,
    },{
      startTime: 14,
      duration: 2,
    }],
  },
  2: {
    "2020-09-03": [{
      startTime: 14,
      duration: 2,
    }],
  }
}
{
  1: {
    days: {
      "2020-09-01": [{
        startTime: 12,
        duration: 1,
      },{
        startTime: 14,
        duration: 2,
      }],
    },
  },
  2: {
    days: {
      "2020-09-03": [{
        startTime: 14,
        duration: 2,
      }],
    },
  },
}

Step 4: Merge Users and Availabilities

{
  1: {
    days: {
      "2020-09-01": [{
        startTime: 12,
        duration: 1,
      },{
        startTime: 14,
        duration: 2,
      }],
    },
  },
}

{
  1: {
    name: "Amelia Bedelia",
  },
}
{
  1: {
    name: "Amelia Bedelia",
    days: {
      "2020-09-01": [{
        startTime: 12,
        duration: 1,
      },{
        startTime: 14,
        duration: 2,
      }],
    },
  },
}

Step 5: Hash to Array

{
  1: {
    name: "Amelia Bedelia",
    days: {
      "2020-09-01": [{
        startTime: 12,
        duration: 1,
      },{
        startTime: 14,
        duration: 2,
      }],
    },
  },
}
[{
  name: "Amelia Bedelia",
  id: 1,
  days: {
    "2020-09-01": [{
      startTime: 12,
      duration: 1,
    },{
      startTime: 14,
      duration: 2,
    }],
  },
}]
{
  users: {
    1: {
      name: "Amelia Bedelia",
    },
    2: {
      name: "Bob Barker",
    }
  },
  availabilities: [{
    date: "2020-09-01",
    startTime: 12,
    duration: 1,
    user_id: 1,
  },{
    date: "2020-09-01",
    startTime: 14,
    duration: 2,
    user_id: 1,
  },{
    date: "2020-09-03",
    startTime: 18,
    duration: 2,
    user_id: 2,
  }],
}
{
  users: [{
    id: 1,
    name: "Amelia Bedelia",
    days: {
      "2020-09-01": [{
        startTime: 12,
        duration: 1,
      },{
        startTime: 14,
        duration: 2,
      }],
    },
  },{
    id: 2,
    name: "Bob Barker",
    days: {
      "2020-09-03": [{
        startTime: 18,
        duration: 2,
      }],
    },
  }],
}

Booyah Achieved

Questions?

kylecoberly.com

Data Transformations with Lodash

By Kyle Coberly

Made with Slides.com

Data Transformations with Lodash

Meetup talk for DenverScript, August 2020

  • 1,143

More from Kyle Coberly