Peer to Peer Video Streaming With WebRTC

Brian Mau

@basscord on all social medias

Brian Mau

Long time Apple developer

Brian Mau

CTO @ UenMe

Show of hands:

How many of you have

• Already seen a WebRTC demo before?

Show of hands:

How many of you have

• Already seen a WebRTC demo before?
• Built something with WebRTC before?

Show of hands:

How many of you have

• Already seen a WebRTC demo before?
• Built something with WebRTC before?
• Have something in production using WebRTC?

What is WebRTC?

What is WebRTC?

• Access to Camera, Audio, Mic, and Screen

What is WebRTC?

• Access to Camera, Audio, Mic, and Screen
• Peer to Peer Communication

What is WebRTC?

• Access to Camera, Audio, Mic, and Screen
• Peer to Peer Communication
• Techniques for traversing NAT

What is WebRTC?

• Access to Camera, Audio, Mic, and Screen
• Peer to Peer Communication
• Techniques for traversing NAT
• High Quality Royalty Free Codecs (Opus & VP8)

What is WebRTC?

• Access to Camera, Audio, Mic, and Screen
• Peer to Peer Communication
• Techniques for traversing NAT
• High Quality Royalty Free Codecs (Opus & VP8) 
• A Data channel

Use Cases?

• VoIP
• Video Chat

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging
• PeerCDN

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging
• PeerCDN
• WebTorrent

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging
• PeerCDN
• WebTorrent
• Music Visualization

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging
• PeerCDN
• WebTorrent
• Music Visualization
• IoT interconnectivity

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging
• PeerCDN
• WebTorrent
• Music Visualization
• IoT interconnectivity
• Security Cameras

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging
• PeerCDN
• WebTorrent
• Music Visualization
• IoT interconnectivity
• Security Cameras
• Telepresence Robots

Use Cases?

• VoIP
• Video Chat
• Multiplayer gaming
• Screen Capture
• Instant Messaging
• PeerCDN
• WebTorrent
• Music Visualization
• IoT interconnectivity
• Security Cameras
• Telepresence Robots
• ???

RTC?

What does Real Time Communication even mean?

• UDP connections
• Not AJAX or TCP/IP
• Video and Audio Streams
• Peer to Peer

A Naive Example?

let socket = io.connect(window.location.origin);

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');
  navigator.mediaDevices.getUserMedia({ video : true })
  .then(function(stream) {
    video.srcObject = stream;
    setInterval(function() {
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
      const imageData = canvas.toDataURL("image/jpeg", 0.4);
      socket.emit('sendImage', { image : imageData });
    }, 100);
  }).catch(error => console.error(error));
}

function watch() {
  const img = document.querySelector('img');
  socket.on('getImage', data => img.src = data.image);
}

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

  /** @type {MediaStreamConstraints} */
  const constraints = { video : true };

Media Stream Constraints

/** @type {MediaStreamConstraints} */
const constraints = {
  audio: true,
  video: true
};

Overconstrained Error:

/** @type {MediaStreamConstraints} */
const constraints = {
  audio: true,
  video: { 
    facingMode: "user",
    width: {
      min: 1024,
      ideal: 1280,
      max: 1920
    },
    height: {
      min: 576,
      ideal: 720,
      max: 1080
    },
    frameRate: { ideal: 10, max: 15 }
  }
};

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

  /** @type {MediaStreamConstraints} */
  const constraints = { video : true };

broadcast()

function broadcast() {
  const video = document.querySelector('video');
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');

  /** @type {MediaStreamConstraints} */
  const constraints = { video : true };

  // getUserMedia()
  navigator.mediaDevices.getUserMedia(constraints)
    .then(function(stream) {
      video.srcObject = stream;
      // ...

getUserMedia()

// Promise based:
navigator.mediaDevices.getUserMedia({video: true})
.then(function(stream) {
  video.srcObject = stream;
  // ...
})

gUM() circa 2013

// Callbacks, and directly on the navigator global:
navigator.getUserMedia({video: true}, success, error)

Media Streams

navigator.mediaDevices.getUserMedia({video: true})
.then(function(stream) {

  // Streams contain 0 or more tracks
  stream.getVideoTracks() // get all video tracks
  stream.getAudioTracks() // get all audio tracks
  stream.getTracks() // // get all the tracks
})

srcObject

/** @type {MediaStream} */
video.srcObject = stream; // srcObject

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;
    // ...

  

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    

  

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    // Draw the video to the canvas
    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

  

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    // Draw the video to the canvas
    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

    // Take a JPEG snapshot of the canvas (potato quality)
    const imageData = canvas.toDataURL("image/jpeg", 0.4);

  

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;

    canvas.width = video.videoWidth;
    canvas.height = video.videoHeight;

    // Draw the video to the canvas
    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

    // Take a JPEG snapshot of the canvas (potato quality)
    const imageData = canvas.toDataURL("image/jpeg", 0.4);

    // emit the image to our Socket.io server
    socket.emit('sendImage', {
      image: imageData
    });


  

Camera to Canvas:

navigator.mediaDevices.getUserMedia(constraints)
  .then(function(stream) {
    video.srcObject = stream;
    setInterval(function() {
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;

      // Draw the video to the canvas
      ctx.drawImage(video, 0, 0, canvas.width, canvas.height);

      // Take a JPEG snapshot of the canvas (potato quality)
      const imageData = canvas.toDataURL("image/jpeg", 0.4);

      // emit the image to our Socket.io server
      socket.emit('sendImage', {
        image: imageData
      });
    }, 100); // Do it 10x per second

And on the other client:

let socket = io.connect(window.location.origin);

function watch() {
  const img = document.querySelector('img');

  // Every time we receive an image, display it.
  socket.on('getImage', data => img.src = data.image);
}

Demo:

https://video-chat.basscord.co:7777

Repo:

https://github.com/basscord/video-images-broadcast

Did that work?

What if we could send that video stream directly to the other user's browser, without any web servers or AJAX?

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
})

createOffer()

• createOffer creates SDP
• It's promise returns the SDP

SDP

• SDP is short for Session Description Protocol
• SDP contains everything the peer needs to connect

createOffer()

• createOffer creates SDP
• It's promise returns the SDP

setLocalDescription()

• setLocalDescription sets the peerConnection's SDP

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
  .then(sdp => peerConnection.setLocalDescription(sdp))
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer()
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    console.log(peerConnection.localDescription);
  });
})

Local Description & SDP

• SDP is short for Session Description Protocol
• SDP contains everything the peer needs to connect
• SDP contains a lot of information!

Local Description & SDP

• SDP is short for Session Description Protocol
• SDP contains everything the peer needs to connect
• SDP contains a lot of information!
• https://webrtchacks.com/sdp-anatomy/

Local Description & SDP

• SDP is short for Session Description Protocol
• SDP contains everything the peer needs to connect
• SDP contains a lot of information!
• https://webrtchacks.com/sdp-anatomy/
• It's not important that we know whats in there
• It's important that we get that info over to the peer's browser.

Signaling

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:
• SIP over Websockets

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:
• SIP over Websockets
• XMPP/Jabber message

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:
• SIP over Websockets
• XMPP/Jabber message
• Facebook Messenger

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:
• SIP over Websockets
• XMPP/Jabber message
• Facebook Messenger
• Email

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:
• SIP over Websockets
• XMPP/Jabber message
• Facebook Messenger
• Email
• Snail Mail

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:
• SIP over Websockets
• XMPP/Jabber message
• Facebook Messenger
• Email
• Snail Mail
• Carrier Pigeon

Signaling

• Signaling is whatever mechanism you use to transport the Local Description & SDP.
• WebRTC intentionally does not specify what signaling method to use.
• Valid methods include:
• SIP over Websockets
• XMPP/Jabber message
• Facebook Messenger
• Email
• Snail Mail
• Carrier Pigeon
• etc.

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  video.srcObject = stream;
  peerConnection.addStream(stream);
  peerConnection.createOffer()
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    console.log(peerConnection.localDescription);
  });
})

RTC Peer Connection

const peerConnection = new RTCPeerConnection();

// Combine RTCPeerConnection with getUserMedia:
navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  video.srcObject = stream;
  peerConnection.addStream(stream);
  peerConnection.createOffer();
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('offer', peerConnection.localDescription);
  });
})

I thought WebRTC is P2P?

I thought WebRTC is P2P?

• The media streaming will be completely peer-to-peer

I thought WebRTC is P2P?

• The media streaming will be completely peer-to-peer
• How the two peers discover each other is not defined

I thought WebRTC is P2P?

• The media streaming will be completely peer-to-peer
• How the two peers discover each other is not defined
• Possible you have a novel way for transmitting SDP?

Browser B:

const peerConnection = new RTCPeerConnection();


    

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  
});

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
});
    

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
});
    

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
  .then(sdp => peerConnection.setLocalDescription(sdp))
});
    

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('answer', peerConnection.localDescription);
  });
});
    

Browser B:

const peerConnection = new RTCPeerConnection();

socket.on('offer', function (message) {
  peerConnection.setRemoteDescription(message)
  .then(() => peerConnection.createAnswer())
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('answer', peerConnection.localDescription);
  });
});

peerConnection.onaddstream = function (event) {
  video.srcObject = event.stream;  
};

Back to Browser A:

const peerConnection = new RTCPeerConnection();

navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer();
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('offer', peerConnection.localDescription);
  });
})

    

Back to Browser A:

const peerConnection = new RTCPeerConnection();

navigator.mediaDevices.getUserMedia({
  audio: true,
  video: true
})
.then(function(stream) {
  peerConnection.addStream(stream);
  peerConnection.createOffer();
  .then(sdp => peerConnection.setLocalDescription(sdp))
  .then(function() {
    socket.emit('offer', peerConnection.localDescription);
  });
})

socket.on('answer', function (message) {
  peerConnection.setRemoteDescription(message);
});
    

Demo:

https://video-chat.basscord.co:8888

Repo:

https://github.com/basscord/webrtc-video-broadcast

Did that work?

WebRTC in the wild:

WebRTC in the wild:

• Your device may not have a public IP Address

WebRTC in the wild:

• Your device may not have a public IP Address
• STUN: Public Services to ask "What IP and Port am I"
• "Session Traversal Utilities for NAT"

STUN Configuration

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};




    

WebRTC in the wild:

• Your device may not have a public IP Address
• STUN: Public Services to ask "What IP and Port am I"
• "Session Traversal Utilities for NAT"
• TURN: STUN server + Media Relay (see RFC5766)

WebRTC in the wild:

• Your device may not have a public IP Address
• STUN: Public Services to ask "What IP and Port am I"
• "Session Traversal Utilities for NAT"
• TURN: STUN server + Media Relay (see RFC5766)
• "Traversal Using Relay around NAT"

TURN Configuration

/** @type {RTCConfiguration} */
const config = {
    'iceServers': [{
      'urls': ['turn:54.149.135.227:3478'],
      'username': 'basscord',
      'credential': 'limpbizkitrulez1998',
      'credentialType': 'password'
    }]
  };



    

TURN Configuration

/** @type {RTCConfiguration} */
const config = {
    'iceServers': [{
      'urls': ['turn:54.149.135.227:3478'],
      'username': 'basscord',
      'credential': 'limpbizkitrulez1998',
      'credentialType': 'password'
    }]
  };



    

Don't use long term passwords!

https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00

WebRTC in the wild:

• Your device may not have a public IP Address
• STUN: Public Services to ask "What IP and Port am I"
• "Session Traversal Utilities for NAT"
• TURN: STUN server + Media Relay (see RFC5766)
• "Traversal Using Relay around NAT"

WebRTC in the wild:

• Your device may not have a public IP Address
• STUN: Public Services to ask "What IP and Port am I"
• "Session Traversal Utilities for NAT"
• TURN: STUN server + Media Relay (see RFC5766)
• "Traversal Using Relay around NAT"
• ICE Candidate: Description of connectable IP and Port
• "Interactive Connection Establishment"

WebRTC in the wild:

• Your device may not have a public IP Address
• STUN: Public Services to ask "What IP and Port am I"
• "Session Traversal Utilities for NAT"
• TURN: STUN server + Media Relay (see RFC5766)
• "Traversal Using Relay around NAT"
• ICE Candidate: Description of connectable IP and Port
• "Interactive Connection Establishment"
• WAE: "Worst Acronyms Ever"

RTC Configuration

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);




    

ICE Candidates

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {

};




    

ICE Candidates

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {
  if (event.candidate) {
    socket.emit('candidate', event.candidate);
  }
};




    

ICE Candidates

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {
  if (event.candidate) {
    socket.emit('candidate', event.candidate);
  }
};

socket.on('candidate', function (candidate) {

});


    

ICE Candidates

/** @type {RTCConfiguration} */
const config = {
  'iceServers': [{
    'urls': ['stun:stun.l.google.com:19302']
  }]
};

const peerConnection = new RTCPeerConnection(config);

peerConnection.onicecandidate = function(event) {
  if (event.candidate) {
    socket.emit('candidate', event.candidate);
  }
};

socket.on('candidate', function (candidate) {
  const c = new RTCIceCandidate(candidate);
  peerConnection.addIceCandidate(c);
});


    

That's it!?

So many acronyms.

Ok Enough. CanIUse?

Ok Enough. CanIUse?

• Safari 11 now supports WebRTC (macOS and iOS)

Ok Enough. CanIUse?

• Safari 11 now supports WebRTC (macOS and iOS)
• Microsoft Edge, Safari, Chrome, Firefox all support

Ok Enough. CanIUse?

• Safari 11 now supports WebRTC (macOS and iOS)
• Microsoft Edge, Safari, Chrome, Firefox all support
• Huge empowerment win for browsers

Ok Enough. CanIUse?

• Safari 11 now supports WebRTC (macOS and iOS)
• Microsoft Edge, Safari, Chrome, Firefox all support
• Huge empowerment win for browsers
• Future of decentralized applications?

Video Chat Demo:

https://video-chat.basscord.co

Repo:

https://github.com/basscord/video-chat

Questions?

Privacy & Security?

• Early on, Real IP was leaked without any indication
• https://tools.ietf.org/html/draft-ietf-rtcweb-ip-handling-04
• No longer true
• Peer to Peer transactions more private than Big Internet
• WebRTC only allowed over HTTPS

Should I pay for WebRTC?

• Many companies selling WebRTC PaaS
• PaaS provide fallbacks for edge cases
• PaaS provide media relay and storage
• Not necessarily easier than WebRTC API