Peer to Peer Video Streaming With WebRTC

Brian Mau

@basscord on all social medias

Brian Mau

Long time Apple developer

Brian Mau

CTO @ UenMe

Show of hands:

How many of you have

  • Already seen a WebRTC demo before?

Show of hands:

How many of you have

  • Already seen a WebRTC demo before?
  • Built something with WebRTC before?

Show of hands:

How many of you have

  • Already seen a WebRTC demo before?
  • Built something with WebRTC before?
  • Have something in production using WebRTC?

What is WebRTC?

What is WebRTC?

  • Access to Camera, Audio, Mic, and Screen

What is WebRTC?

  • Access to Camera, Audio, Mic, and Screen
  • Peer to Peer Communication

What is WebRTC?

  • Access to Camera, Audio, Mic, and Screen
  • Peer to Peer Communication
  • Techniques for traversing NAT

What is WebRTC?

  • Access to Camera, Audio, Mic, and Screen
  • Peer to Peer Communication
  • Techniques for traversing NAT
  • High Quality Royalty Free Codecs (Opus & VP8)

What is WebRTC?

  • Access to Camera, Audio, Mic, and Screen
  • Peer to Peer Communication
  • Techniques for traversing NAT
  • High Quality Royalty Free Codecs (Opus & VP8) 
  • A Data channel

Use Cases?

  • VoIP
  • Video Chat

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging
  • PeerCDN

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging
  • PeerCDN
  • WebTorrent

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging
  • PeerCDN
  • WebTorrent
  • Music Visualization

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging
  • PeerCDN
  • WebTorrent
  • Music Visualization
  • IoT interconnectivity

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging
  • PeerCDN
  • WebTorrent
  • Music Visualization
  • IoT interconnectivity
  • Security Cameras

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging
  • PeerCDN
  • WebTorrent
  • Music Visualization
  • IoT interconnectivity
  • Security Cameras
  • Telepresence Robots

Use Cases?

  • VoIP
  • Video Chat
  • Multiplayer gaming
  • Screen Capture
  • Instant Messaging
  • PeerCDN
  • WebTorrent
  • Music Visualization
  • IoT interconnectivity
  • Security Cameras
  • Telepresence Robots
  • ???

RTC?

What does Real Time Communication even mean?

  • UDP connections
  • Not AJAX or TCP/IP
  • Video and Audio Streams
  • Peer to Peer

A Naive Example?

let socket = io.connect(window.location.origin);

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');
  navigator.mediaDevices.getUserMedia({ video : true })
  .then(function(stream) {
    video.srcObject = stream;
    setInterval(function() {
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
      const imageData = canvas.toDataURL("image/jpeg", 0.4);
      socket.emit('sendImage', { image : imageData });
    }, 100);
  }).catch(error => console.error(error));
}

function watch() {
  const img = document.querySelector('img');
  socket.on('getImage', data => img.src = data.image);
}

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

  /** @type {MediaStreamConstraints} */
  const constraints = { video : true };

Media Stream Constraints

/** @type {MediaStreamConstraints} */
const constraints = {
  audio: true,
  video: true
};

Overconstrained Error:

/** @type {MediaStreamConstraints} */
const constraints = {
  audio: true,
  video: { 
    facingMode: "user",
    width: {
      min: 1024,
      ideal: 1280,
      max: 1920
    },
    height: {
      min: 576,
      ideal: 720,
      max: 1080
    },
    frameRate: { ideal: 10, max: 15 }
  }
};

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

  /** @type {MediaStreamConstraints} */
  const constraints = { video : true };

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

  /** @type {MediaStreamConstraints} */
  const constraints = { video : true };

  // getUserMedia()
  navigator.mediaDevices.getUserMedia(constraints)
    .then(function(stream) {
      video.srcObject = stream;
      // ...

getUserMedia()

// Promise based:
navigator.mediaDevices.getUserMedia({video: true})
.then(function(stream) {
  video.srcObject = stream;
  // ...
})

gUM() circa 2013

// Callbacks, and directly on the navigator global:
navigator.getUserMedia({video: true}, success, error)

Media Streams

navigator.mediaDevices.getUserMedia({video: true})
.then(function(stream) {

  // Streams contain 0 or more tracks
  stream.getVideoTracks() // get all video tracks
  stream.getAudioTracks() // get all audio tracks
  stream.getTracks() // // get all the tracks
})

srcObject

/** @type {MediaStream} */
video.srcObject = stream; // srcObject

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;
    // ...

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    // Draw the video to the canvas
    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    // Draw the video to the canvas
    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

    // Take a JPEG snapshot of the canvas (potato quality)
    const imageData = canvas.toDataURL("image/jpeg", 0.4);

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    // Draw the video to the canvas
    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

    // Take a JPEG snapshot of the canvas (potato quality)
    const imageData = canvas.toDataURL("image/jpeg", 0.4);

    // emit the image to our Socket.io server
    socket.emit('sendImage', {
      image: imageData
    });

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;
    setInterval(function() {
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;

      // Draw the video to the canvas
      ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

      // Take a JPEG snapshot of the canvas (potato quality)
      const imageData = canvas.toDataURL("image/jpeg", 0.4);

      // emit the image to our Socket.io server
      socket.emit('sendImage', {
        image: imageData
      });
    }, 100); // Do it 10x per second

And on the other client:


let socket = io.connect(window.location.origin);

function watch() {
  const img = document.querySelector('img');

  // Every time we receive an image, display it.
  socket.on('getImage', data => img.src = data.image);
}

Demo:

https://video-chat.basscord.co:7777

Repo:

https://github.com/basscord/video-images-broadcast

Did that work?

What if we could send that video stream directly to the other user's browser, without any web servers or AJAX?

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
})

createOffer()

  • createOffer creates SDP
  • It's promise returns the SDP

SDP

  • SDP is short for Session Description Protocol
  • SDP contains everything the peer needs to connect

createOffer()

  • createOffer creates SDP
  • It's promise returns the SDP

setLocalDescription()

  • setLocalDescription sets the peerConnection's SDP

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
  .then(sdp => peerConnection.setLocalDescription(sdp))
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    console.log(peerConnection.localDescription);
  });
})

{"type":"offer","sdp":"v=0\r\no=- 8845556371065191854 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE video\r\na=msid-semantic: WMS d9MzOhvRYFeiKQSG96u54GFQpqkd6coBerZP\r\nm=video 61283 UDP/TLS/RTP/SAVPF 96 98 100 102 127 97 99 101 125\r\nc=IN IP4 192.168.1.159\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:105744546 1 udp 2122260223 192.168.1.159 61283 typ host generation 0 network-id 1 network-cost 10\r\na=candidate:1221489746 1 tcp 1518280447 192.168.1.159 9 typ host tcptype active generation 0 network-id 1 network-cost 10\r\na=ice-ufrag:qfBb\r\na=ice-pwd:GdnVgPdtMJCliTbFUdYW4UYb\r\na=ice-options:trickle\r\na=fingerprint:sha-256 95:C1:36:BE:CE:62:F3:D2:22:BF:4D:19:52:2B:55:8E:3F:47:7B:6F:DF:99:D2:47:E1:27:F9:1C:71:68:A5:B4\r\na=setup:actpass\r\na=mid:video\r\na=extmap:2 urn:ietf:params:rtp-hdrext:toffset\r\na=extmap:3 http://www.webrtc.org/experiments/rtp-hdrext/abs-...

Local Description & SDP

  • SDP is short for Session Description Protocol
  • SDP contains everything the peer needs to connect
  • SDP contains a lot of information!

Local Description & SDP

Local Description & SDP

  • SDP is short for Session Description Protocol
  • SDP contains everything the peer needs to connect
  • SDP contains a lot of information!
  • https://webrtchacks.com/sdp-anatomy/
  • It's not important that we know whats in there
  • It's important that we get that info over to the peer's browser.

Signaling

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:
  • SIP over Websockets

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:
  • SIP over Websockets
  • XMPP/Jabber message

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:
  • SIP over Websockets
  • XMPP/Jabber message
  • Facebook Messenger

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:
  • SIP over Websockets
  • XMPP/Jabber message
  • Facebook Messenger
  • Email

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:
  • SIP over Websockets
  • XMPP/Jabber message
  • Facebook Messenger
  • Email
  • Snail Mail

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:
  • SIP over Websockets
  • XMPP/Jabber message
  • Facebook Messenger
  • Email
  • Snail Mail
  • Carrier Pigeon

Signaling

  • Signaling is whatever mechanism you use to transport the Local Description & SDP.
  • WebRTC intentionally does not specify what signaling method to use.
  • Valid methods include:
  • SIP over Websockets
  • XMPP/Jabber message
  • Facebook Messenger
  • Email
  • Snail Mail
  • Carrier Pigeon
  • etc.

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  video.srcObject = stream;
  peerConnection.addStream(stream);
  peerConnection.createOffer()
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    console.log(peerConnection.localDescription);
  });
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  video.srcObject = stream;
  peerConnection.addStream(stream);
  peerConnection.createOffer();
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('offer', peerConnection.localDescription);
  });
})

I thought WebRTC is P2P?

I thought WebRTC is P2P?

  • The media streaming will be completely peer-to-peer

I thought WebRTC is P2P?

  • The media streaming will be completely peer-to-peer
  • How the two peers discover each other is not defined

I thought WebRTC is P2P?

  • The media streaming will be completely peer-to-peer
  • How the two peers discover each other is not defined
  • Possible you have a novel way for transmitting SDP?

Browser B:

const peerConnection = new RTCPeerConnection();

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  
});

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
});

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
});

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
  .then(sdp => peerConnection.setLocalDescription(sdp))
});

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('answer', peerConnection.localDescription);
  });
});

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('answer', peerConnection.localDescription);
  });
});

peerConnection.onaddstream = function (event) {
  video.srcObject = event.stream;  
};

Back to Browser A:

const peerConnection = new RTCPeerConnection();

navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer();
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('offer', peerConnection.localDescription);
  });
})

Back to Browser A:

const peerConnection = new RTCPeerConnection();

navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer();
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('offer', peerConnection.localDescription);
  });
})

socket.on('answer', function (message) {
  peerConnection.setRemoteDescription(message);
});

Demo:

https://video-chat.basscord.co:8888

Repo:

https://github.com/basscord/webrtc-video-broadcast

Did that work?

WebRTC in the wild:

WebRTC in the wild:

  • Your device may not have a public IP Address

WebRTC in the wild:

  • Your device may not have a public IP Address
  • STUN: Public Services to ask "What IP and Port am I"
  • "Session Traversal Utilities for NAT"

STUN Configuration

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

WebRTC in the wild:

  • Your device may not have a public IP Address
  • STUN: Public Services to ask "What IP and Port am I"
  • "Session Traversal Utilities for NAT"
  • TURN: STUN server + Media Relay (see RFC5766)

WebRTC in the wild:

  • Your device may not have a public IP Address
  • STUN: Public Services to ask "What IP and Port am I"
  • "Session Traversal Utilities for NAT"
  • TURN: STUN server + Media Relay (see RFC5766)
  • "Traversal Using Relay around NAT"

TURN Configuration

/** @type {RTCConfiguration} */
const config = {
    'iceServers': [{
      'urls': ['turn:54.149.135.227:3478'],
      'username': 'basscord',
      'credential': 'limpbizkitrulez1998',
      'credentialType': 'password'
    }]
  };

TURN Configuration

/** @type {RTCConfiguration} */
const config = {
    'iceServers': [{
      'urls': ['turn:54.149.135.227:3478'],
      'username': 'basscord',
      'credential': 'limpbizkitrulez1998',
      'credentialType': 'password'
    }]
  };

Don't use long term passwords!

https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00

WebRTC in the wild:

  • Your device may not have a public IP Address
  • STUN: Public Services to ask "What IP and Port am I"
  • "Session Traversal Utilities for NAT"
  • TURN: STUN server + Media Relay (see RFC5766)
  • "Traversal Using Relay around NAT"

WebRTC in the wild:

  • Your device may not have a public IP Address
  • STUN: Public Services to ask "What IP and Port am I"
  • "Session Traversal Utilities for NAT"
  • TURN: STUN server + Media Relay (see RFC5766)
  • "Traversal Using Relay around NAT"
  • ICE Candidate: Description of connectable IP and Port
  • "Interactive Connection Establishment"

WebRTC in the wild:

  • Your device may not have a public IP Address
  • STUN: Public Services to ask "What IP and Port am I"
  • "Session Traversal Utilities for NAT"
  • TURN: STUN server + Media Relay (see RFC5766)
  • "Traversal Using Relay around NAT"
  • ICE Candidate: Description of connectable IP and Port
  • "Interactive Connection Establishment"
  • WAE: "Worst Acronyms Ever"

RTC Configuration

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

ICE Candidates

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {

};

ICE Candidates

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {
  if (event.candidate) {
    socket.emit('candidate', event.candidate);
  }
};

ICE Candidates

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {
  if (event.candidate) {
    socket.emit('candidate', event.candidate);
  }
};

socket.on('candidate', function (candidate) {

});

ICE Candidates

 

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {
  if (event.candidate) {
    socket.emit('candidate', event.candidate);
  }
};

socket.on('candidate', function (candidate) {
  const c = new RTCIceCandidate(candidate);
  peerConnection.addIceCandidate(c);
});

That's it!?

So many acronyms.

Ok Enough. CanIUse?

Ok Enough. CanIUse?

  • Safari 11 now supports WebRTC (macOS and iOS)

Ok Enough. CanIUse?

  • Safari 11 now supports WebRTC (macOS and iOS)
  • Microsoft Edge, Safari, Chrome, Firefox all support

Ok Enough. CanIUse?

  • Safari 11 now supports WebRTC (macOS and iOS)
  • Microsoft Edge, Safari, Chrome, Firefox all support
  • Huge empowerment win for browsers

Ok Enough. CanIUse?

  • Safari 11 now supports WebRTC (macOS and iOS)
  • Microsoft Edge, Safari, Chrome, Firefox all support
  • Huge empowerment win for browsers
  • Future of decentralized applications?

Video Chat Demo:

https://video-chat.basscord.co

Repo:

https://github.com/basscord/video-chat

Questions?

Privacy & Security?

Should I pay for WebRTC?

  • Many companies selling WebRTC PaaS
  • PaaS provide fallbacks for edge cases
  • PaaS provide media relay and storage
  • Not necessarily easier than WebRTC API
Made with Slides.com