CLU

We believe it is helpful to associate two structures with a program: its logical structure and its physical structure.

- Barbara Liskov, Programming with Abstract Data Types (1974)

Abstract Data Type: a class of abstract objects which are entirely characterized by the operations available on those objects.

Figure 1: Erlang Maps, Under the Surface

Atlas: Flexible Software Composition Using Protocols

Quinn Wilton / @wilton_quinn / quinnwilton.com / SYNOPSYS

Let's go back in time a year...

Introducing Atlas!

  • A unified reporting dashboard for a disparate set of security scanners

  • Each scanner...

    • Was deployed as a separate web service

    • Operated in different domains (web apps, APIs, etc)

    • Required new APIs to expose the data we needed

Development Goals

  • Close the feedback loop between building new remote APIs and frontend code that depends on them

  • Quickly iterate on frontend development with extensive generation of mock data

  • Simplify local development by building fully-functional mocks of the remote services

defmodule Twitter do
  @callback get_by_username(username :: String.t) :: %Twitter.User{}
  @callback followers_for(username :: String.t) :: [%Twitter.User{}]
end

defmodule TwitterMock do
  @behaviour Twitter

  def get_by_username("josevalim") do
    %Twitter.User{
      username: "josevalim"
    }
  end
  
  def followers_for("josevalim") do
    # return literally everyone
  end
end

Mocks and explicit contracts

defmodule Twitter do
  @callback register(username :: String.t()) :: :ok | :already_taken
  @callback get_by_username(username :: String.t) :: %Twitter.User{}
end

assert nil == TwitterMock.get_by_username("quinn")
assert :ok == TwitterMock.register("quinn")
assert %Twitter.User{username: "quinn"} = TwitterMock.get_by_username("quinn")

** (MatchError) no match of right hand side value: nil

Stateful behaviours?

  1. Modify the behaviour to take a reference to some sort of non-global mutable state, like a PID
  2. Access shared global state, using a named process, ETS table, or persistent term
defprotocol Twitter do
  def register(twitter, username)
  def get_by_username(twitter, username)
end

defmodule Twitter.InMemory do
  defstruct [:pid]
  
  def new(opts \\ []) do
    {:ok, pid} = start_link(opts)
    %__MODULE__{pid: pid}
  end
  
  def start_link(opts \\ []) do
    Agent.start_link(fn -> %{} end)
  end
  
  defimpl Twitter do
    def register(%{pid: pid}, username) do
      Agent.get_and_update(pid, fn
        %{^username => _user} = state ->
          {:already_taken, state}
        
        state ->
          {:ok, Map.put(state, username, %Twitter.User{username: username})}
      end)
    end
    
    def get_by_username(%{pid: pid}, username) do
      Agent.get(pid, &Map.get(&1, username))
    end
  end
end

Once more, with ad-hoc polymorphism

iex(1)> twitter = Twitter.InMemory.new
%Twitter.InMemory{pid: #PID<0.282.0>}
iex(2)> Twitter.get_by_username(twitter, "quinn")
nil
iex(3)> Twitter.register(twitter, "quinn")
:ok
iex(4)> Twitter.get_by_username(twitter, "quinn")
%Twitter.User{username: "quinn"}

Once more, with ad-hoc polymorphism

  • Can build frontends that depend on APIs that don't exist yet

  • simplify development by not requiring those services to be running

  • enable exploration of the domain model without leaving Elixir

  • take advantage of extensive mock data generation

Norm

  • Library for specifying, validating, and generating data

  • Given a data specification, Norm automatically infers:

    • Conformers, for validation

    • Generators, for property testing

Data Specification

id = spec(is_integer() and (&(&1 > 0)))

name =
  with_gen(
    spec(is_binary()),
    StreamData.string(:alphanumeric, min_length: 5, max_length: 15)
  )

product_schema =
  schema(%{
    id: id,
    name: name
  })

Data Validation

iex> conform!(%{id: 5, name: "GRiSP 2"}, selection(product_schema))
%{id: 5, name: "GRiSP 2"}

iex> conform!(%{id: 5, name: nil}, selection(product_schema))
** (Norm.MismatchError) Could not conform input:
val: nil in: :name fails: is_binary() 
    (norm 0.12.0) lib/norm.ex:64: Norm.conform!/2

Data Generation

product_generator = gen(product_schema)

iex> Enum.take(gen(product_schema), 5)
[
  %{id: 1, name: "gbHa8"},
  %{id: 1, name: "IuI6D"},
  %{id: 3, name: "3q49g"},
  %{id: 3, name: "gZRjG"},
  %{id: 1, name: "1Bg3M"}
]

demo time

Accounts service

defprotocol EasyWire.Accounts.Service do
  def get_account_for_profile(service, profile_id)
  def deposit_money(service, profile_id, amount)
end

defimpl EasyWire.Accounts.Service, for: EasyWire.Accounts.InMemory do
  alias EasyWire.Accounts.InMemory

  defdelegate get_account_for_profile(service, profile_id), to: InMemory
  defdelegate deposit_money(service, profile_id, amount), to: InMemory
end

Accounts service

defmodule EasyWire.Accounts.InMemory do
  alias EasyWire.Accounts.Account

  defstruct [:pid]

  def new(opts) do
    {:ok, pid} = start_link(opts)

    %__MODULE__{pid: pid}
  end

  def start_link(opts) do
    Agent.start_link(fn ->
      profile_ids = Keyword.get(opts, :profile_ids, [])
      generation_size = Keyword.get(opts, :generation_size, 1)

      model = %{
        generation_size: generation_size,
        accounts: %{}
      }

      Enum.reduce(profile_ids, model, fn profile_id, model ->
        account =
          Account.schema()
          |> Norm.gen()
          |> StreamData.resize(model.generation_size)
          |> Enum.at(0)
          |> Map.put(:profile_id, profile_id)

        Map.update!(model, :accounts, &Map.put(&1, account.profile_id, account))
      end)
    end)
  end

  ...
end

Profile Schema

defmodule EasyWire.Profiles.Profile do
  import Norm

  alias EasyWire.Types

  defstruct [
    :id,
    :name,
    :trust,
    :company
  ]

  def schema,
    do:
      schema(%__MODULE__{
        id: Types.id(),
        name: Types.person_name(),
        trust: trust(),
        company: company()
      })

  defp trust() do
    alt(
      verified: :verified,
      unverified: :unverified
    )
  end

  defp company() do
    generator =
      StreamData.sized(fn _ ->
        StreamData.constant(Faker.Company.name())
      end)

    with_gen(spec(is_binary), generator)
  end
end

🚨Here be dragons🚨

Service mesh

defmodule EasyWire.ServiceRouter do
  use ServiceMesh.Router, otp_app: :easy_wire

  middleware ServiceMesh.Middleware.Telemetry
  middleware ServiceMesh.Middleware.SimulateLatency, latency_ms: 100
  middleware ServiceMesh.Middleware.SimulateNetworkFailure, failure_rate: 0.01

  register :accounts, EasyWire.Accounts.Service
  register :profiles, EasyWire.Profiles.Service
  register :transactions, EasyWire.Transactions.Service
end

# in application.ex

def start(_type, _args) do
  children = [
    {ServiceMesh, EasyWire.ServiceRouter},
  ]

  Supervisor.start_link(children, ...)
end

Service router

# in config/config.exs

config :easy_wire, EasyWire.ServiceRouter, fn ->
  profile_ids =
    EasyWire.Types.id()
    |> Norm.gen()
    |> Enum.take(10)

  %{
    profiles: EasyWire.Profiles.InMemory.new(profile_ids: profile_ids),
    accounts: EasyWire.Accounts.InMemory.new(profile_ids: profile_ids),
    transactions: EasyWire.Transactions.InMemory.new(profile_ids: profile_ids)
  }
end

Service Configuration

{:ok, account} =
  ServiceMesh.call(
    :accounts,
    :get_account_for_profile,
    [profile_id]
  )

Service Usage

"program one decision at a time"

defmodule EasyWire.Search.Parallel do
  defstruct [
    :services
  ]

  defimpl EasyWire.Search.Service do
    alias EasyWire.Search

    def search(%{services: services}, query, opts) do
      services
      |> Enum.map(fn service -> Task.async(&Search.Service.search(query, opts)) end)
      |> Task.await_many()
      |> List.flatten()
      |> Enum.sort_by(&(&1.score))
    end
  end
end

Transparent concurrency

defmodule Analytics.Experiment do
  defstruct [
    :old,
    :new
  ]

  defimpl Analytics.Service do
    require Logger

    def some_complicated_query(service) do
      [old, new] =
        Task.await_many([
          Task.async(fn -> Analytics.Service.some_complicated_query(service.old) end),
          Task.async(fn -> Analytics.Service.some_complicated_query(service.new) end)
        ])

      if old != new do
        Logger.warning(fn -> "Expected: #{old}, got: #{new}" end)
      end

      old
    end
  end
end

Refactoring Experiments

defmodule AuthService.Migration do
  defstruct [
    :map_service_fn
  ]

  defimpl AuthService do
    def authenticate(service, credentials) do
      inner_service = service.map_service_fn.(credentials)
      
      AuthService.authenticate(inner_service, credentials)
    end
  end
end

# in config

%AuthService.Migration{
  map_service_fn: fn scan ->
    case credentials.version do
      :v0 -> %AuthService.LegacyService{...}
      :v1 -> %AuthService.NewService{...}
    end
  end
}

migrations

stateful Property Based testing

  • Variant of property based testing

    • A model of a complex system is defined

    • Sequences of commands are generated

    • Those commands are run against the real system

    • The real system is compared against the model

  • Requires PropEr

    • PropCheck is an Elixir wrapper

  • See Mentat: https://github.com/keathley/mentat

Summary

  1. Protocols allow you to separate the logical structure of your program from the physical
  2. Protocols + Norm allow you to quickly prototype by modeling remote services
  3. Protocol composition enables you to layer new behavior onto an existing system

Thank you!

Quinn Wilton / @wilton_quinn / quinnwilton.com

Atlas: Flexible Software Composition using Protocols

By quinnwilton

Made with Slides.com

Atlas: Flexible Software Composition using Protocols

  • 494

More from quinnwilton