LiveData: LiveView for JSON endpoints

Hans Elias B. Josephsen

@hansihe

  • Original creator of Rustler
  • Worked on Lumen

You write the HTML

LiveView takes care of sending minimal updates when things change

You write the data structure

LiveData takes care of sending minimal updates when things change

Diffing engine

Demo #1

Simple LiveData + Svelte

defmodule PdvBackendWeb.SimpleData do
  use LiveData

  def mount(_params, socket) do
    {:ok, _tref} = :timer.send_after(1000, :tick)
    socket = assign(socket, :counter, 0)
    {:ok, socket}
  end

  def handle_info(:tick, socket) do
    {:ok, _tref} = :timer.send_after(1000, :tick)
    socket = assign(socket, :counter, socket.assigns.counter + 1)
    {:ok, socket}
  end
  
  deft render(assigns) do
    %{
      "counter" => assigns.counter,
    }
  end
  
end

<script>
    import LiveDataSocket from './live_data/LiveDataSocket.svelte';
    import LiveData from './live_data/LiveData.svelte';

    export let dataViewUrl;
</script>

<main>
    <LiveDataSocket url={dataViewUrl}>
        <LiveData path="/simple_data" let:data={data}>
            The counter is: {data.counter}
        </LiveData>
    </LiveDataSocket>
</main>

Objective:

Send the least amount of data to the client

Avoid duplication!

Avoid duplication

<div>
  <h1>Hello <%= @name %></h1>
  <p>Status: <%= @status %></p>
</div>
[
  "<div><h1>Hello ", 
  "!</h1><p>Status: ", 
  "</p></div>"
]
[name, status]

+

Avoid duplication

{
  "id": 33421,
  "name": "Adam Johnson",
  "status": "Relaxing at home"
}
{
  "id": <slot1>,
  "name": <slot2>,
  "status": <slot3>
}
[id, name, status]

+

Objective:

Send the least amount of data to the client

  • Avoid duplication
    • Send static data only once
{
  "id": <slot1>,
  "name": <slot2>,
  "status": <slot3>
}
[id, name, status]

+

[
  {"id": 1, "name": "Paul"},
  {"id": 2, "name": "Andrew"},
  {"id": 3, "name": "Helly"},
  {"id": 4, "name": "Stian"},
  {"id": 5, "name": "Kristine"}
]
deft render(assigns) do
  for user <- assigns[:users] do
    %{
      id: user.id,
      name: user.id
    }
  end
end
[
  {"id": 1, "name": "Paul"},
  {"id": 2, "name": "Andrew"},
  {"id": 4, "name": "Stian"},
  {"id": 3, "name": "Helly"},
  {"id": 5, "name": "Kristine"}
]
deft render(assigns) do
  for user <- assigns[:users] do
    %{
      id: user.id,
      name: user.id
    }
  end
end

State of the art in general tree diffing algorithms is O(n^3)

Keys!

[
  {"id": 1, "name": "Paul"},
  {"id": 2, "name": "Andrew"},
  {"id": 4, "name": "Stian"},
  {"id": 3, "name": "Helly"},
  {"id": 5, "name": "Kristine"}
]
deft render(assigns) do
  for user <- assigns[:users] do
    %{
      id: user.id,
      name: user.name
    }
  end
end
[
  {"id": 1, "name": "Paul"},
  {"id": 2, "name": "Andrew"},
  {"id": 4, "name": "Stian"},
  {"id": 3, "name": "Helly"},
  {"id": 5, "name": "Kristine"}
]
deft render(assigns) do
  for user <- assigns[:users] do
    keyed user.id, %{
      id: user.id,
      name: user.name
    }
  end
end

1

2

3

4

5

Objective:

Send the least amount of data to the client

  • Avoid duplication
    • Send static data only once
    • Give subtrees identity
      • Ability to diff dynamic parts
deft render(assigns) do
  for user <- assigns[:users] do
    track(render_user(user))
  end
end

deft render_user(user) do
  keyed user.id, %{
    id: user.id,
    name: user.id
  }
end
deft render(assigns) do
  ...
end

deft -> define tracked

deft render(assigns) do
  ...
end
def render(assigns) do
  ...
end

def __tracked_meta__render__1__(...) do
  ...
end
def __tracked__render__(assigns) do
  ...
end
def __tracked_meta__render__1__(:statics) do
  %{
    {:expr, 8} =>
      {:make_map, nil,
       [
        {
         {:literal, "counter"}, 
         %{__struct__: LiveData.Tracked.Tree.Slot, num: 0}
        }
       ]}
  }
end
deft render(assigns) do
  %{
    "counter" => assigns.counter,
  }
end
def __tracked__render__(assigns) do
  gen_var_13 = assigns.counter

  %LiveData.Tracked.RenderTree.Static{
    id: {{PdvBackendWeb.SimpleData, :render, 1}, {:expr, 8}},
    slots: [gen_var_13]
  }
end
deft render(assigns) do
  %{
    "counter" => assigns.counter,
  }
end

Downside:

deft is not magic

deft
  • The deft compiler needs to be able to introspect the AST of the function
    • Higher order functions need to be explicitly supported
    • Can still be called, less efficient on changes

make map

return

"counter"

.counter

arg: assigns

Ask me afterwards if you are curious about the deft compiler, I think it's quite neat!

Demo #2

LiveData + DSL + Native Mobile App (Flutter)

defmodule PdvBackendWeb.FlutterViewDemoData do
  use LiveData
  use FlutterView

  alias PdvBackendWeb.LoginChangeset

  def mount(_params, socket) do
    changeset = LoginChangeset.changeset(%LoginChangeset{})
    
    socket = assign(socket, :changeset, changeset)
    {:ok, socket}
  end

  def handle_event(%{"e" => "do_validate"} = event, socket) do
    data = event["data"]
    changeset = LoginChangeset.changeset(%LoginChangeset{}, data)

    socket = assign(socket, :changeset, changeset)
    {:ok, socket}
  end

  deft render(assigns) do
    scaffold(
      app_bar: app_bar(
        title: text("My Form")
      ),
      body: form(assigns.changeset, fn f ->
        column(
          children: [
            form_text_input(f, :name, hint: "Name"),
            form_text_input(f, :email, hint: "Email"),
            form_text_input(f, :password, hint: "Password")
          ]
        )
      end)
    )
  end
end
defmodule PdvBackendWeb.FlutterViewDemoData do
  use LiveData
  use FlutterView

  alias PdvBackendWeb.LoginChangeset

  def mount(_params, socket) do
    changeset = LoginChangeset.changeset(%LoginChangeset{})
    
    socket = assign(socket, :changeset, changeset)
    {:ok, socket}
  end

  def handle_event(%{"e" => "do_validate"} = event, socket) do
    data = event["data"]
    changeset = LoginChangeset.changeset(%LoginChangeset{}, data)

    socket = assign(socket, :changeset, changeset)
    {:ok, socket}
  end

  deft render(assigns) do
    scaffold(
      app_bar: app_bar(
        title: text("My Form")
      ),
      body: form(assigns.changeset, fn f ->
        column(
          children: [
            form_text_input(f, :name, hint: "Name"),
            form_text_input(f, :email, hint: "Email"),
            form_text_input(f, :password, hint: "Password")
          ]
        )
      end)
    )
  end
end
defmodule PdvBackendWeb.LoginChangeset do
  use Ecto.Schema
  import Ecto.Changeset

  embedded_schema do
    field :name, :string
    field :email, :string
    field :password, :string
  end

  def changeset(%__MODULE__{} = struct, params \\ %{}) do
    struct
    |> cast(params, [:name, :email, :password])
    |> validate_required([:name, :email, :password])
  end
end

import 'package:flutter/material.dart';
import 'package:live_data_client/live_data_flutter.dart';

WidgetRegistry makeWidgetRegistry() {
  var registry = new WidgetRegistry();
  registerBasicWidgets(registry);
  registerMaterialWidgets(registry);
  return registry;
}

WidgetRegistry widgetRegistry = makeWidgetRegistry();

void main() {
  var service = DataViewSocketService(
      "ws://localhost:4000/data/websocket");
  runApp(DataViewSocket(service: service, child: MyApp()));
}

class MyApp extends StatelessWidget {
  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      title: 'Flutter Demo',
      theme: ThemeData.light(),
      home: LiveNativeMount(widgetRegistry, "/flutter_view_demo"),
    );
  }
}

Wire protocol

Two tables:

  • Table of Templates
  • Table of Fragments

Templates

{
  "id": ["$s",0],
  "name": ["$s",1],
  "links": {
    "profile": ["$s",2] 
  }
}
  • Templates have an arbitrary amount of slots
  • Can be any JSON

Fragments

["$t", 1, ["$f", 5]]
  • Can instantiate templates
  • Can embed other fragments
  • Can contain parts of data structure too

Each message from server -> client consists of a series of OPs

  • SetFragment
  • PatchFragment
  • SetTemplate
  • ...
  • Render

Each message from server -> client consists of a series of OPs

Render:

Takes ID of root fragment, renders it out to JSON

  • LiveView-like development of mobile apps
  • Easy integration with Javascript apps
  • Redux pattern over the network

Future work

  • Getting it production ready
  • Data dependency tracking
  • Support binary/string construction
  • Better diffing
  • Other "data structures"
  • Other clients

github.com/hansihe/live_data