Project Testing

Reasons

  • Team is growing
  • Current unit test coverage is very low
  • Almost no integration tests for now
  • New people don't have product knowledge
  • No automated regression for typical behavior scenarios
  • Very long feedback loop as mostly testing covered by separate QA team which receive updates once in two weeks

Unit Tests

  • Build a bridge between code and task with unit tests
  • Test tasks close to code
  • Code coverage

coverage-blamer

Enzyme

import ProcessSelection from 'components/LayerConfiguration/processSelection';
import { testProcessSelection, testProcessSelectionNoRoot } from '../resources/testProcessSelection';

describe('ProcessSelection tests', () => {

    it('should select root', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2J", testProcessSelection,
            ["F:2S", "F:2T"]);

        expect(selectedIds).to.eql(["F:2I"]);
    });

    it('should select leaf on empty tree', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:55", testProcessSelection,
            []);

        expect(selectedIds).to.eql(["F:55"]);
    });

    it('should unselect leaf on empty tree', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:55", testProcessSelection,
            ["F:55"]);

        expect(selectedIds).to.eql([]);
    });

    it('should unselect leaf', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:55", testProcessSelection,
            ["F:55", "F:2K", "F:2L"]);

        expect(selectedIds).to.eql(["F:2K", "F:2L"]);
    });

    it('should select parent when all children are selected', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:57", testProcessSelection,
            ["F:56"]);

        expect(selectedIds).to.eql(["F:2S"]);
    });

    it('should select root by propagation', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:57", testProcessSelection,
            ["F:56", "F:2T", "F:2J"]);

        expect(selectedIds).to.eql(["F:2I"]);
    });

    it('should select brothers when the parent has been selected', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2U", testProcessSelection,
            ["F:2T"]);

        expect(selectedIds).to.eql(["F:58", "F:2V", "F:59"]);
    });

    it('should select brothers and uncles when the root has been selected', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2U", testProcessSelection,
            ["F:2I"]);

        expect(selectedIds).to.eql(["F:2J", "F:2S", "F:58", "F:2V", "F:59"]);
    });

    it('should deselect children', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2T", testProcessSelection,
            ["F:2J", "F:2S", "F:58", "F:2V", "F:59"]);

        expect(selectedIds).to.eql(["F:2J", "F:2S"]);
    });

    it('should unselect root unselect all children', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2I", testProcessSelection,
            ["F:2J", "F:2S", "F:58", "F:2V", "F:59"]);

        expect(selectedIds).to.eql([]);
    });

    it('should select parent', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2T", testProcessSelection,
            []);

        expect(selectedIds).to.eql(["F:2T"]);
    });

    it('should unselect parent', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2T", testProcessSelection,
            ["F:2S", "F:2T"]);

        expect(selectedIds).to.eql(["F:2S"]);
    });

    it('should add selection', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2V", testProcessSelection,
            ["F:2J", "F:2S", "F:58"]);

        expect(selectedIds).to.eql(["F:2J", "F:2S", "F:58", "F:2V"]);
    });

    it('should remove selection', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2V", testProcessSelection,
            ["F:2J", "F:2S", "F:58", "F:2V"]);

        expect(selectedIds).to.eql(["F:2J", "F:2S", "F:58"]);
    });

    it('should unselect root unselect parents', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2I", testProcessSelection,
            ["F:2J", "F:2S"]);

        expect(selectedIds).to.eql([]);
    });

    it('should unselect root unselect leafs', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2I", testProcessSelection,
            ["F:2R", "F:57", "F:59"]);

        expect(selectedIds).to.eql([]);
    });

    it('should select leaf with uncles', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:55", testProcessSelection,
            ["F:2S", "F:2T"]);

        expect(selectedIds).to.eql(["F:2S", "F:2T", "F:55"]);
    });

    it('should unselect leaf with uncles', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:55", testProcessSelection,
            ["F:2S", "F:2T", "F:55"]);

        expect(selectedIds).to.eql(["F:2S", "F:2T"]);
    });

    it('should select leaf with cousins', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:55", testProcessSelection,
            ["F:57", "F:59"]);

        expect(selectedIds).to.eql(["F:57", "F:59", "F:55"]);
    });

    it('should unselect leaf with cousins', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:55", testProcessSelection,
            ["F:55", "F:57", "F:59"]);

        expect(selectedIds).to.eql(["F:57", "F:59"]);
    });

    it('should select all on level 0', () => {
        let selectedIds = ProcessSelection.getSelectedRcsCodes("F:2J", testProcessSelectionNoRoot,
            ["F:2S", "F:2T"]);

        expect(selectedIds).to.eql(["F:2J", "F:2S", "F:2T"]);
    });

});


TDD

  • Navigation map

  • Test first

  • Assert first

  • Fail first

Mutation testing (or mutation analysis or program mutation) is used to design new software tests and evaluate the quality of existing software tests. Mutation testing involves modifying a program in small ways.[1] Each mutated version is called a mutant and tests detect and reject mutants by causing the behavior of the original version to differ from the mutant. This is called killingthe mutant. Test suites are measured by the percentage of mutants that they kill. New tests can be designed to kill additional mutants. Mutants are based on well-defined mutation operators that either mimic typical programming errors (such as using the wrong operator or variable name) or force the creation of valuable tests (such as dividing each expression by zero). The purpose is to help the tester develop effective tests or locate weaknesses in the test data used for the program or in sections of the code that are seldom or never accessed during execution. Mutation testing is a form of white-box testing.

Coverage Blamer

┌──────────────────────────────────────────────────┬───────────────┬───────────────┬───────────────┐
│ Author                                           │ Lines         │ Uncovered Li… │ Coverage      │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Alexander Jeng                                   │ 2             │ 1             │ 50%           │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Tim Wood                                         │ 203           │ 80            │ 60.591133004… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Brian Wyant                                      │ 39            │ 13            │ 66.666666666… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Isaac Cambron                                    │ 25            │ 5             │ 80%           │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Iskren Chernev                                   │ 550           │ 59            │ 89.272727272… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ paladox                                          │ 27            │ 0             │ 100%          │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Igor Lima                                        │ 14            │ 0             │ 100%          │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Dan Dascalescu                                   │ 25            │ 0             │ 100%          │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Mehul Patel                                      │ 1             │ 0             │ 100%          │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Lachèze Alexandre                                │ 39            │ 0             │ 100%          │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Dennis                                           │ 9             │ 0             │ 100%          │
└──────────────────────────────────────────────────┴───────────────┴───────────────┴───────────────┘
┌──────────────────────────────────────────────────┬───────────────┬───────────────┬───────────────┐
│ File                                             │ Lines         │ Uncovered Li… │ Coverage      │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/history.js                               │ 123           │ 56            │ 54.471544715… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/size.js                                  │ 59            │ 24            │ 59.322033898… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/component.js                             │ 10            │ 4             │ 60%           │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/embed_locales.js                         │ 47            │ 17            │ 63.829787234… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/zones.js                                 │ 70            │ 24            │ 65.714285714… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/check_sauce_creds.js                     │ 13            │ 4             │ 69.230769230… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/bump_version.js                          │ 65            │ 7             │ 89.230769230… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ ./tasks/transpile.js                             │ 288           │ 22            │ 92.361111111… │
└──────────────────────────────────────────────────┴───────────────┴───────────────┴───────────────┘
┌──────────────────────────────────────────────────┬───────────────┬───────────────┬───────────────┐
│ Date                                             │ Lines         │ Uncovered Li… │ Coverage      │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Mon Aug 20 2012                                  │ 178           │ 77            │ 56.497175141… │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Tue Aug 21 2012                                  │ 6             │ 2             │ 60%           │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Fri Aug 31 2012                                  │ 9             │ 0             │ 100%          │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤
│ Wed Oct 03 2012                                  │ 1             │ 1             │ 0%            │
├──────────────────────────────────────────────────┼───────────────┼───────────────┼───────────────┤

Time to adoption

  • CI for branches - 1 week
  • Mutation tests - 2-5days
  • Coverage blamer - 1-2 days
  • Reformat/refactor tests - 1 week 
  • TDD - continuously 
  • Tech depts managing practices 

Integration

Tests

  • Test communication between components 
  • Test communication between server and client
  • Test messages

Enzyme

import fetchMock from 'fetch-mock';
import WmtsCapabilities from 'layers/WMTS/wmtsCapabilities';
import { testWmtsCapabilities } from '../../resources/testWmtsCapabilities';

const successUrl = 'http://localhost:8080/GetCapabilities/wmts/success';
const failUrl = 'http://localhost:8080/GetCapabilities/wmts/fail';

describe('WmtsCapabilities tests', () => {
    let layers;
    it('should successfully parse valid capabilities document', (done) => {
        let caps = new WmtsCapabilities(successUrl);
        fetchMock.mock(new RegExp('^' + successUrl), testWmtsCapabilities, null);
        caps.get().then(l => {
            layers = l;
            expect(layers).to.not.be.null;

            done();
        }, error => {
            done(error);
        });
    });

    fetchMock.restore();

    it('should find 10 layers', () => {
        expect(layers).to.have.lengthOf(10);
    });

    it('should extract layer title', () => {
        expect(layers[0].title).to.equal('VegoilTerminals');
    });

    it('should extract layer name', () => {
        expect(layers[0].name).to.equal('tank_terminals:VegoilTerminals');
    });

    it('should extract layer identifier', () => {
        expect(layers[0].identifier).to.equal('tank_terminals:VegoilTerminals');
    });

    it('should extract map formats', () => {
        expect(layers[0].formats).to.deep.equal(['image/png']);
    });

    it('should extract featureInfo formats', () => {
        expect(layers[0].infoFormats).to.deep.equal([
            'text/plain',
            'application/vnd.ogc.gml',
            'text/xml',
            'application/vnd.ogc.gml/3.1.1',
            'text/xml',
            'text/html',
            'application/json']);
    });

    it('should extract layers bbox', () => {
        expect(layers[0].bbox).to.deep.equal({
            minx: -123.05457830429106,
            miny: -46.59404355158038,
            maxx: 176.91429823637043,
            maxy: 63.8635901900421
        });
    });

    it('should extract queryable attribute', () => {
        expect(layers[0].isQueryable).to.equal(true);
    });

    it('should fail with invalid capabilities document', (done) => {
        let caps = new WmtsCapabilities(failUrl);
        fetchMock.mock(new RegExp('^' + failUrl), "this isn't a valid capabilities document", null);
        caps.get().then(layers => {
            // nice browsers should detect invalid xml and reject, but PhantomJS won't so check the result
            if (layers && layers.length)
                done("This shouldn't succeed!" + JSON.stringify(layers));
            else
                done();
        }, error => {
            done();
        });
    });

    fetchMock.restore();
    //afterEach(fetchMock.restore);
});
import { expect } from 'chai';
import { shallow } from 'enzyme';
import MyComponent from './path/to/MyComponent';
 
describe('<MyComponent />', () => {
  it('should match snapshot', () => {
    const wrapper = shallow(<MyComponent />)
    
    // You can match Enzyme wrappers
    expect(wrapper).to.matchSnapshot();
    
    // Strings
    expect('you can match strings').to.matchSnapshot();
    
    // Numbers
    expect(123).to.matchSnapshot();
    
    // Or any object
    expect({ a: 1, b: { c: 1 } }).to.matchSnapshot();
   
  });
});
const ajv = new Ajv({$data: true});

const schema = {
  "properties": {
    "smaller": {
      "type": "number",
      "maximum": { "$data": "1/larger" }
    },
    "larger": { "type": "number" }
  }
};

const validData = {
  smaller: 5,
  larger: 7
};

ajv.validate(schema, validData); // true

UI Tests

  • Bridge between tasks and product
  • E2e tests
Feature: App

  Scenario: As a user I can open the app and see the UI
    Given App has been opened
    Then Title contains "Hello World!"
    And Content contains "Hello World!"
const { Given, Then } = require('cucumber')
const expect = require('chai').expect
const { checkPartialTitle, checkElementPartialText } = require('../support/helpers.js')

Given(/^App has been opened$/, () => {
  return app.client
    .waitUntilWindowLoaded()
    .getWindowCount().then(count => {
      expect(count).to.be.equal(1)
    })
    .browserWindow.isMinimized().then(x => {
      expect(x).to.be.false
    })
    .browserWindow.isVisible().then(x => {
      expect(x).to.be.true
    })
    .browserWindow.isFocused().then(x => {
      expect(x).to.be.true
    })
    .browserWindow.getBounds().then(({width, height}) => {
      expect(width).to.be.above(0)
      expect(height).to.be.above(0)
    })
})

Then(/^Title contains "([^"]*)"$/, (text) => {
  return checkPartialTitle(text, true)
})

Then(/^Content contains "([^"]*)"$/, (text) => {
  return checkElementPartialText(text, 'body', true)
})
const Application = require('spectron').Application
const electronPath = require('electron')
const path = require('path')
const { setWorldConstructor, BeforeAll, AfterAll } = require('cucumber')

const app = new Application({
  path: electronPath,
  args: [path.join(__dirname, '../../')]
})

BeforeAll(() => {
  return app.start().then(() => {
    setWorldConstructor(function() {
      global.app = app
    })
  })
})

AfterAll(() => {
  return app.stop()
})

Improve reporting

Roles

  • Unit Tests - Developers
  • Integration tests - Developers
  • UI/Cypress tests - Automation QAs
  • Features - BAs ???

Other

  • Performance
  • Accessibility
  • Security
  • Lisence checks

TODO Plan

  • Add unit tests coverage thresholds on CI (1 day)
  • Add mutation testing or/and coverage-blamer to CI (3-7 days) 
  • Add tests for contracts and integration tests (10-15 days) 
  • Add 2-3 automatisation QA's
  • Add e2e tests with Spectrum & Cucumber (10-20 days)
  • Cover important cases with cypress records (10-20 days)
  • Add tools for improve CI/CD and reporting - (5-10 days) (https://reportportal.io/)

Thank you!