ARCore - A big step towards universal mobile Mixed Reality

Codecamp Iași, Spring 2019

Dragoș Silion

What AR(MR) mostly meant until one year ago

Smart glasses had their go

Vuzix Blade
Epson Moverio
Google Glass
ODG

Most SDKs where Marker Based

ARToolkit
AR.js
Vuforia
Wikitude

Google Tango

Actual Mixed Reality
Dense scans
Anchor persistence

MR Headsets

Hololens
Daqri
Meta

MR Headsets

Spatial Mapping
Spatial Understanding
Persistent Anchors
Hand Gestures
Voice Assistant

Not cheap
Not so comfortable
Small FoV
Volatile SDKs with low regard to backwards compatibility
No support for mobile SDKs / Web

ARKit & ARCore

(Q1 of 2018)

Common Features

Spatial Mapping - Feature Points
Spatial Understanding - Vertical/Horizontal Planes
Shared Anchors
Light Estimation
Facial Mapping
Image detection and Tracking

ARCore

Object Tracking
Anchors Persistence

ARCore

Cross-platform (iOS)

Code example &

live demo

Also works on Android, iOS, React Native and (soon?) on web

Feature Points

IMU

Camera Feed

Feature Points

Camera Pose

Env. Mapping

SLAM - Simulatenously Location and Mapping

AR Session

AR Frames

Trackables

Camera feed

Camera Pose

Light Estimation

etc.

Dynamic Data

Planar Tracking

TrackableHit hit;
TrackableHitFlags raycastFilter = TrackableHitFlags.PlaneWithinPolygon |
    TrackableHitFlags.FeaturePointWithSurfaceNormal;

if (Frame.Raycast(touch.position.x, touch.position.y, raycastFilter, out hit))
{
    if ((hit.Trackable is DetectedPlane) &&
        Vector3.Dot(FirstPersonCamera.transform.position - hit.Pose.position,
            hit.Pose.rotation * Vector3.up) < 0)
    {
        Debug.Log("Hit at back of the current DetectedPlane");
    }
    else
    {
        GameObject prefab;
        if (hit.Trackable is FeaturePoint)
        {
            prefab = AndyPointPrefab;
        }
        else
        {
            prefab = AndyPlanePrefab;
        }

        var andyObject = Instantiate(prefab, hit.Pose.position, hit.Pose.rotation);
        andyObject.transform.Rotate(0, k_ModelRotation, 0, Space.Self);

        var anchor = hit.Trackable.CreateAnchor(hit.Pose);

        andyObject.transform.parent = anchor.transform;
    }
}

TrackableHit hit;
TrackableHitFlags raycastFilter = TrackableHitFlags.PlaneWithinPolygon |
    TrackableHitFlags.FeaturePointWithSurfaceNormal;

if (Frame.Raycast(touch.position.x, touch.position.y, raycastFilter, out hit))
{
    if ((hit.Trackable is DetectedPlane) &&
        Vector3.Dot(FirstPersonCamera.transform.position - hit.Pose.position,
            hit.Pose.rotation * Vector3.up) < 0)
    {
        Debug.Log("Hit at back of the current DetectedPlane");
    }
    else
    {
        GameObject prefab;
        if (hit.Trackable is FeaturePoint)
        {
            prefab = AndyPointPrefab;
        }
        else
        {
            prefab = AndyPlanePrefab;
        }

        var andyObject = Instantiate(prefab, hit.Pose.position, hit.Pose.rotation);
        andyObject.transform.Rotate(0, k_ModelRotation, 0, Space.Self);

        var anchor = hit.Trackable.CreateAnchor(hit.Pose);

        andyObject.transform.parent = anchor.transform;
    }
}

TrackableHit hit;
TrackableHitFlags raycastFilter = TrackableHitFlags.PlaneWithinPolygon |
    TrackableHitFlags.FeaturePointWithSurfaceNormal;

if (Frame.Raycast(touch.position.x, touch.position.y, raycastFilter, out hit))
{
    if ((hit.Trackable is DetectedPlane) &&
        Vector3.Dot(FirstPersonCamera.transform.position - hit.Pose.position,
            hit.Pose.rotation * Vector3.up) < 0)
    {
        Debug.Log("Hit at back of the current DetectedPlane");
    }
    else
    {
        GameObject prefab;
        if (hit.Trackable is FeaturePoint)
        {
            prefab = AndyPointPrefab;
        }
        else
        {
            prefab = AndyPlanePrefab;
        }

        var andyObject = Instantiate(prefab, hit.Pose.position, hit.Pose.rotation);
        andyObject.transform.Rotate(0, k_ModelRotation, 0, Space.Self);

        var anchor = hit.Trackable.CreateAnchor(hit.Pose);

        andyObject.transform.parent = anchor.transform;
    }
}

Image Targets

// Get updated augmented images for this frame.
Session.GetTrackables<AugmentedImage>(
    m_TempAugmentedImages, TrackableQueryFilter.Updated);

// Create visualizers and anchors for updated augmented images that are tracking and do
// not previously have a visualizer. Remove visualizers for stopped images.
foreach (var image in m_TempAugmentedImages)
{
    AugmentedImageVisualizer visualizer = null;
    m_Visualizers.TryGetValue(image.DatabaseIndex, out visualizer);
    if (image.TrackingState == TrackingState.Tracking && visualizer == null)
    {
        // Create an anchor to ensure that ARCore keeps tracking this augmented image.
        Anchor anchor = image.CreateAnchor(image.CenterPose);
        visualizer = (AugmentedImageVisualizer)Instantiate(
            AugmentedImageVisualizerPrefab, anchor.transform);
        visualizer.Image = image;
        m_Visualizers.Add(image.DatabaseIndex, visualizer);
    }
    else if (image.TrackingState == TrackingState.Stopped && visualizer != null)
    {
        m_Visualizers.Remove(image.DatabaseIndex);
        GameObject.Destroy(visualizer.gameObject);
    }
}

Image Targets

if (Image == null || Image.TrackingState != TrackingState.Tracking)
{
    FrameLowerLeft.SetActive(false);
    FrameLowerRight.SetActive(false);
    FrameUpperLeft.SetActive(false);
    FrameUpperRight.SetActive(false);
    return;
}

float halfWidth = Image.ExtentX / 2;
float halfHeight = Image.ExtentZ / 2;
FrameLowerLeft.transform.localPosition =
(halfWidth * Vector3.left) + (halfHeight * Vector3.back);
FrameLowerRight.transform.localPosition =
(halfWidth * Vector3.right) + (halfHeight * Vector3.back);
FrameUpperLeft.transform.localPosition =
(halfWidth * Vector3.left) + (halfHeight * Vector3.forward);
FrameUpperRight.transform.localPosition =
(halfWidth * Vector3.right) + (halfHeight * Vector3.forward);

FrameLowerLeft.SetActive(true);
FrameLowerRight.SetActive(true);
FrameUpperLeft.SetActive(true);
FrameUpperRight.SetActive(true);