Andrey Mikhaylov (lolmaus)

frotnend developer at Perforce

https://lolma.us

https://github.com/lolmaus

https://perforce.com/products/helix-teamhub

from developer's perspective

Plan

Not gonna tell you why automated tests are important
Acceptance vs unit tests
BDD
Our current test setup
What Cucumber does
Why we should adopt Cucumber
This talk is frontend-focused

Hierarchy of tests

Acceptance

Integration

Unit

$$$

Correct hierarchy of tests

Unit

Integration

Acceptance

Controller.extend({
  spaceHorn: service(),
  society:   service(),

  caramelize () {
    const confusion = this.spaceHorn.honk()
    return this.society.riot(confusion)
  }
})

Is a unit test needed here?

Test-driven development

Test

Code

Refactor

User story

Behavior-driven development

Page

Component

Model

Service

Network
adapter

Start by writing a full feature spec in form of user stories

Our typical acceptance test

test("apples should be sorted by id", function (assert) {
    const user = server.create("user")
    server.createList("apples", 20, {ownerId: user.id})

    authenticateSession(user)
    visit("/apples")
    
    const apples = findAll(".apples .apple")

    apples.forEach((currentApple, i) => {
        if (i === 0) return

        const currentValue = currentApple.querySelector(".apple-id").textContent
         
        const previousApple = apples[i - 1]
        const previousValue = previousApple.querySelector(".apple-id").textContent

        assert.ok(
            currentValue > previousValue,
            `Apple ${i} id should be greater than apple ${i -1} id`
        )
    })
})

Applying
the page object technique

test("apples should be sorted by id", function (assert) {
    const user = server.create("user")
    server.createList("apples", 20, {ownerId: user.id})

    authenticateSession(user)
    page.visit()

    page.apples.forEach((currentApple, i) => {
        if (i === 0) return
         
        const previousApple = page.apples[i - 1]

        assert.ok(
            currentApple.id > previousApple.id,
            `Apple ${i} id should be greater than apple ${i -1} id`
        )
    })
})

export default create({
    visitable: "/apples",
    
    apples: collection({
        scope: ".apples",
        itemScope: ".apple",
        item: {
            id: text('.apple-id')
        }
    })
})

Extracting test logic into reusable functions

test("apples should be sorted by id", function (assert) {

    const user = server.create("user")


    server.createList("apples", 20, {ownerId: user.id})


    authenticateSession(user)


    page.visit()


    page.apples.forEach((currentApple, i) => {
        if (i === 0) return
         
        const previousApple = page.apples[i - 1]

        assert.ok(
            currentApple.id > previousApple.id,
            `Apple ${i} id should be greater than apple ${i -1} id`
        )
    })
})



createUser()


createApples(amount)


login()


visit()


applesShouldBeSortedById(assert)

let user, apples

function createUser () {
    user = server.create("user")
}

function createApples () {
    apples = server.createList("apples", 20, {ownerId: user.id})
}

function login () {
    authenticateSession(user)
}

function visit () {
    page.visit()
}

function applesShouldBeSortedById (assert) {
    page.apples.forEach((currentApple, i) => {
        if (i === 0) return
         
        const previousApple = page.apples[i - 1]

        assert.ok(
            currentApple.id > previousApple.id,
            `Apple ${i} id should be greater than apple ${i -1} id`
        )
    })
}

test("apples should be sorted by id", function (assert) {
    createUser()
    createApples()

    login()
    visit()

    applesShouldBeSortedById(assert)
})

Extracting test logic into reusable functions

Reuse the test logic

test("apples should be sorted by id (asc) by default", function (assert) {
    createUser()
    createApples()

    login()
    visit()

    applesShouldBeSortedBy({column: "id", order: "asc", assert})
})



test("apples should be sorted by id (desc) when URL contains order=desc", function (assert) {
    createUser()
    createApples()

    login()
    visit({order: "desc"})

    applesShouldBeSortedBy({column: "id", order: "desc", assert})
})



test("apples should be sorted by title (asc) when URL contains column=title", function (assert) {
    createUser()
    createApples()

    login()
    visit({column: "title"})

    applesShouldBeSortedBy({column: "title", order: "asc", assert})
})

DRY

const cases = [
  {queryParams: {                              }, column: "id",    order: "asc" },
  {queryParams: {                 order: "asc" }, column: "id",    order: "asc" },
  {queryParams: {                 order: "desc"}, column: "id",    order: "desc"},

  {queryParams: {column: "title"               }, column: "title", order: "asc" },
  {queryParams: {column: "title", order: "asc" }, column: "title", order: "asc" },
  {queryParams: {column: "title", order: "desc"}, column: "title", order: "desc"},

  {queryParams: {column: "color"               }, column: "color", order: "asc" },
  {queryParams: {column: "color", order: "asc" }, column: "color", order: "asc" },
  {queryParams: {column: "color", order: "desc"}, column: "color", order: "desc"},
]


cases.forEach(({queryParams, column, order}) => {
    const params = JSON.stringify(queryParams)

    test(`apples should be sorted by ${column} (${order}) when URL conains query params ${params}`, function (assert) {
        createUser()
        createApples()
    
        login()
        visit(queryParams)
    
        applesShouldBeSortedBy({column, order, assert})
    })
})

Enter Cucumber

Scenario: Apples should be sorted by id
    Given a user
    And 20 apples

    When user is authenticated
    And user visits the apples page

    Then apples should be sorted by id

test("apples should be sorted by id", function (assert) {
    createUser()
    createApples()

    login()
    visit()

    applesShouldBeSortedById(assert)
})

Cucumber feature
with a test matrix

Feature: Sorting apples

Scenario: Apples should be sorted by [Column] ([Order]) when URL contains QPs [QPs]
    Given a user
    And 20 apples

    When user is authenticated
    And user visits the apples page using QPs [QPs]

    Then apples should be sorted by [Column] column in [Order] order

Where:
    
    -----------------------------------------------------
    | QPs                              | Column | Order |
    | {}                               | id     | asc   |
    | {order: "asc"}                   | id     | asc   |
    | {order: "desc"}                  | id     | asc   |
    | {column: "title"}                | title  | asc   |
    | {column: "title", order: "asc"}  | title  | asc   |
    | {column: "title", order: "desc"} | title  | desc  |
    | {column: "color"}                | color  | asc   |
    | {column: "color", order: "asc"}  | color  | asc   |
    | {column: "color", order: "desc"} | color  | desc  |
    -----------------------------------------------------


{
    "given a user" () {
        this.ctx.user = server.create("user")
    },

    "given $count apples" (countStr) {
        const count = parseInt(countStr, 10)
        const onwerId = this.ctx.user.id
        this.ctx.apples = server.createList("apple", count, {ownerId})
    },

    "when user is authenticated" () {
        authenticateSession(this.ctx.user)
    },

    "when user visits the apples page using QPs $qps" (qpsStr) {
        const qps = JSON.parse(qpsStr)
        page.visit(qps)
    },

    "then apples should be sorted by $column column in $order order" (column, order, assert) {
        page.apples.forEach((currentApple, i) => {
            if (i === 0) return
             
            const currentValue  = currentApple[column].text
            const previousApple = page.apples[i - 1]
            const previousValue = previousApple[column].text

            const result =
                order === "asc"  ? currentValue >= previousValue :
                order === "desc" ? currentValue <= previousValue :
                null
    
            assert.ok(result, `Apple ${i} id should be ${order} than apple ${i - 1} id`)
        })
    }
}

let user, apples

function createUser () {
    user = server.create("user")
}

function createApples (count) {
    apples = server.createList("apples", count , {ownerId: user.id})
}



function login () {
    authenticateSession(user)
}

function visit (queryParams = {}) {
    page.visit(qps)
}


function applesShouldBeSortedBy (column, order , assert) {
    page.apples.forEach((currentApple, i) => {
        if (i === 0) return
         
        const currentValue  = currentApple[column].text
        const previousApple = page.apples[i - 1]
        const previousValue = previousApple[column].text

        const result =
            order === "asc"  ? currentValue >= previousValue :
            order === "desc" ? currentValue <= previousValue :
            null

        assert.ok(result, `Apple ${i} id should be ${order} than apple ${i - 1} id`)
    })
}

Defining test logic

1. Complexity

Current setup

With Cucumber

Mirage API layer
Mirage factories and models
Mirage scenarios (reusable seeding logic)
Page objects
Page object components
Test suite scaffolding
Assertion library
Technical implementation of tests
Test matrix scaffolding (sometimes)

Mirage API layer
Mirage factories and models
Mirage scenarios (reusable seeding logic)
Page objects
Page object components
Test suite scaffolding
Assertion library
Technical implementation of user story steps
Cucumber features, composed of human-readable steps

2. Separation of concerns

Current setup

With Cucumber

Three different concerns:
- test case descriptions,
- their technical implementation,
- glue code
... are all cluttered together.

Test cases are separated from technical implementation.
Technical implementation is broken into small, atomic steps.
Steps are composed naturally, so glue code almost doesn’t exist.

3. Code Reusability

Current setup

With Cucumber

We still keep writing duplicate test code every month

Steps are reusable.

4. Code Readability

Current setup

With Cucumber

Tests are mostly “write only code”.
It takes minutes to red into each test in order to validate it.
Technical implementation is visual noise.

Test cases are readable.
Easy to read through & validate.

5. Code Maitainability

Current setup

With Cucumber

Tests are effectively unmaintained.
We don't have test maintenance on schedule.
Maintaining long sheets of code is hard.

Little maintenance needed.
Maintenance is easy.

6. Ease of validation

Current setup

With Cucumber

Difficult to validate.
Tests become dead code over time.
Sit and pray.

Easy to validate.
Pure English.

7. Workflow integration

Current setup

With Cucumber

Only developers can read tests.
Feature creation workflow is interrupted.

Testers, designers, product owners can participate in writing features.
...and in validating features.
Feature creation workflow is continuous.

8. Discipline

Current setup

With Cucumber

No intrinsic structure that could impose discipline.

Strict methodology and code style greatly contributes to code quality.

9. Coverage

Current setup

With Cucumber

We only cover very basic cases, even when attempting BDD.
BDD isn’t enforced by anything but dev’s conscience.

Cucumber methodology imposes maximum coverage.
Developers are less likely to cut corners.

10. Code unification

Current setup

With Cucumber

Code style still varies greatly across our test codebase.
No reference implementation.

Cucumber imposes a lot of structure.
It's harder to stray.

10. Speed of writing tests

Current setup

With Cucumber

Initially faster.
Hidden costs.

Generally slower.
Pays off greatly later.

Cucumber doesn't add much to the complexity of our test codebase.
In our attempt to make tests better we have essentially reinvented Cucumber in an amateur, imperfect way.
Using a well-thought out methodology is obviously better than its imperfect custom equivalent.
The debate boils down to whether we should go all the way with BDD or not.
I strongly believe that we should, since it is provides numerous benefits.
It does not prevent us from doing it the old way when we feel like it. We should simply agree to avoid the old way as much as possible.

TL/DL

That's all! Thx everyone!

Andrey Mikhaylov (lolmaus)

from developer's perspective

Plan

Hierarchy of tests

Correct hierarchy of tests

Our typical acceptance test

Applying the page object technique

Extracting test logic into reusable functions

Extracting test logic into reusable functions

Reuse the test logic

DRY

Enter Cucumber

Cucumber feature with a test matrix

Defining test logic

Applying
the page object technique

Cucumber feature
with a test matrix