Unveiling the Power of Approval Testing

Janina Nemec

I am…

👩‍💻 a Software Developer/Crafter

a co-organizer for a security conference

I like…

💡Learning & Improving ⇒ Software Teaming

🔄 Automatization

🎨 Drawing

👩‍🌾 Gardening

🎮 Video Games

🐶 Dogs 💜

class ShopOrderTest {

  @Test
  void assertionTest() throws JsonProcessingException {
  	// given
    String orderId = "someOrderId";
    ShopOrder order = aDefaultOrder(orderId);
        
    // when
    anOrderWasProcessed(order);
	
    // then
    OrderResult orderResult = jsonMapper.readValue(callRestEndpoint(orderId), OrderResult.class);

    assertThat(orderResult.getId()).isEqualTo("someOrderId");
    assertThat(orderResult.getVersion()).isEqualTo(1);

    ItemResult item = orderResult.getItems().getFirst();
    assertThat(item.getId()).isEqualTo("someItemId");
    assertThat(item.getName()).isEqualTo("ATD 3 Conf. Days");
    assertThat(item.getAmount()).isEqualTo(2);

    PriceResult itemPrice = item.getPrice();
    assertThat(itemPrice.getValue()).isEqualTo(225000);
    assertThat(itemPrice.getMonetaryUnit()).isEqualTo("cent");
    assertThat(itemPrice.getCurrency()).isEqualTo("EUR");

    CouponResult coupon = orderResult.getCoupons().getFirst();
    assertThat(coupon.getId()).isEqualTo("someCouponId");
    assertThat(coupon.getDescription()).isEqualTo("Speaker Coupon");
    assertThat(coupon.getReducedRateInPercentage()).isEqualTo(100);

    assertThat(orderResult.getOrderTimeStamp()).isEqualTo(LocalDateTime.of(2024, 7, 19, 11, 45));
    assertThat(orderResult.getDeliveryDate()).isEqualTo(LocalDate.of(2024, 11, 22));

    PriceResult shippingCost = orderResult.getShippingCost().getFirst();
    assertThat(shippingCost.getValue()).isEqualTo(500);
    assertThat(shippingCost.getMonetaryUnit()).isEqualTo("cent");
    assertThat(shippingCost.getCurrency()).isEqualTo("EUR");

    CustomerResult customer = orderResult.getCustomer();
    assertThat(customer.getId()).isEqualTo("someCustomerId");
    assertThat(customer.getFirstName()).isEqualTo("REWE");
    assertThat(customer.getLastName()).isEqualTo("Digital");

    AddressResult shippingAddress = orderResult.getShippingAddress();
    assertThat(shippingAddress.getId()).isEqualTo("someShippingAddressId");
    assertThat(shippingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(shippingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(shippingAddress.getStreetName()).isEqualTo("Schanzenstr.");
    assertThat(shippingAddress.getHouseNumber()).isEqualTo("6-20");
    assertThat(shippingAddress.getPostalCode()).isEqualTo("51063");
    assertThat(shippingAddress.getCity()).isEqualTo("Köln");
    assertThat(shippingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(shippingAddress.getPhone()).isEqualTo("0221 9758420");
    assertThat(shippingAddress.getLatitude()).isEqualTo("50.96490882194811");
    assertThat(shippingAddress.getLongitude()).isEqualTo("7.014472855463499");
    assertThat(shippingAddress.getEmail()).isEqualTo("kontakt@rewe-digital.com");

    AddressResult billingAddress = orderResult.getBillingAddress();
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Micha");
    assertThat(billingAddress.getLastName()).isEqualTo("Kutz");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

class ShopOrderTest {

  @Test
  void assertionTest() throws JsonProcessingException {
  	// given
    String orderId = "someOrderId";
    ShopOrder order = aDefaultOrder(orderId);
        
    // when
    anOrderWasProcessed(order);
	
    // then
    JsonApprovals.verifyJson(callRestEndpoint(orderId));
  }
}

record Address(
    String id,
    String firstName,
    String lastName,
    String streetName,
    String houseNumber,
    String city,
    String country,
    String phone,
    String latitude,
    String longitude,
    String email,
    String postalCode
) {}

record Order(
    String id,
    int version,
    List<Item> items,
    List<Coupon> coupons,
    LocalDateTime orderTimeStamp,
    LocalDate deliveryDate,
    List<Price> shippingCost,
    Customer customer,
    Address shippingAddress,
    Address billingAddress
) {}

record Item(
    String id,
    String name,
    int amount,
    Price price
) {}

record Coupon(
    String id,
    String description,
    int reducedRateInPercentage
) {}

record Price(
    int value,
    String monetaryUnit,
    String currency
) {}

record Customer(
    String id,
    String firstName,
    String lastName
) {}

record Address(
    String id,
    String firstName,
    String lastName,
    String streetName,
    String houseNumber,
    String city,
    String country,
    String phone,
    String latitude,
    String longitude,
    String email,
    String postalCode
) {}

Green Assertion Tests

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

v ✅ Test Results

v ✅ AddressAssertionsTest

✅ assertionTest

v ✅ Test Results

v ✅ AddressAssertionsTest

✅ assertionTest

Problem

Are the tests good?
Do they test all fields?
Do they test the correct thing?

class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

A Red Assertion

class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstra.").houseNumber("21")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Expected :"Domstr."
Actual :"Domstra."

at AddressAssertionTest.java:19

Problem

Only first failing assertion is visible
Are there multiple errors?
Is there an invisible pattern?

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Expected :"Domstr."
Actual :"Domstra."

at AddressAssertionTest.java:19

Soft Assertions

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    
    SoftAssertions.assertSoftly( softly -> {
      softly.assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
      softly.assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
      softly.assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
      softly.assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
      softly.assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
      softly.assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
      softly.assertThat(billingAddress.getCity()).isEqualTo("Köln");
      softly.assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
      softly.assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
      softly.assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
      softly.assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
      softly.assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
      softly.assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
    });
  }
}

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Multiple Failures (2 failures)
-- failure 1 --

expected :"Domstr."
but was :"Domstra."

at AddressAssertionTest.java:16
-- failure 2 --

expected :"20"
but was :"21"

at AddressAssertionTest.java:17

Improvements

All failing assertions visible
Easy to read

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Multiple Failures (2 failures)
-- failure 1 --

expected :"Domstr."
but was :"Domstra."

at AddressAssertionTest.java:16
-- failure 2 --

expected :"20"
but was :"21"

at AddressAssertionTest.java:17

Problem

Missing Assertions
A lot boilerplate code
Not used much
Still not testing the right thing

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Multiple Failures (2 failures)
-- failure 1 --

expected :"Domstr."
but was :"Domstra."

at AddressAssertionTest.java:16
-- failure 2 --

expected :"20"
but was :"21"

at AddressAssertionTest.java:17

Object Assertion

class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstra.").houseNumber("21")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstra.").houseNumber("21")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    AddressResult expectedBillingAddress = anAddressResult().id("someBillingAddressId")
        .firstName("Janina").lastName("Nemec")
        .streetName("Domstr.").houseNumber("20")
        .postalCode("50668").city("Köln").country("Deutschland")
        .phone("+49 221 1490").email("info@rewe-group.com")
        .latitude("50.94603935915518").longitude("6.959302840118697")
        .status(CustomerStatus.NEW_CUSTOMER)
        .build();

    assertThat(billingAddress).isEqualTo(expectedBillingAddress);
  }
}

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Expected :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstr., houseNumber=20, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, posta ...
Actual :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstra., houseNumber=21, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, post ...

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Expected :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstr., houseNumber=20, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, posta ...
Actual :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstra., houseNumber=21, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, post ...

Problem

Failure message hard to read
Are there multiple errors?
Diff not assessable

Problem

Diff is unpleasent to read
Differences can be overlooked
Not always feasible
Still not testing the right thing

Improvements

All failures are visible
All data is tested

One Text Assertion

class AddressAssertionsTest {

  @Test
  void assertionTest() {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();
        
    anOrderWasProcessed(shopOrder);
    
    assertThat(callRestEndpointForBillingAddress(orderId)).isEqualToIgnoringWhitespace("""
    {
      "id": "someBillingAddressId",
      "firstName": "Janina",
      "lastName": "Nemec",
      "streetName": "Domstr.",
      "houseNumber": "20",
      "city": "Köln",
      "country": "Deutschland",
      "phone": "+49 221 1490",
      "latitude": "50.94603935915518",
      "longitude": "6.959302840118697",
      "email": "info@rewe-group.com",
      "postalCode": "50668",
      "status":"new_customer"
    }""");
  }
}

class AddressAssertionsTest {

  private String TEST_DIR = "src/test/resources/json/FileComparisonTest/";

  @Test
  void assertionTest() {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();

    anOrderWasProcessed(shopOrder);

    assertThat(callRestEndpointForBillingAddress(orderId)).isEqualToIgnoringWhitespace(
    	new String(Files.readAllBytes(Paths.get(TEST_DIR + "expectedBillingAddress.json")))
    );
  }
}

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Expecting actual:
"{"id":"someBillingAddressId", "firstName":"Janina", "lastName":"Nemec", "streetName":"Domstr.", "houseNumber":"20", "city":"Köln", "country":"Deutschland", "phone":"+49 221 1490", "status":"new_customer", "latitude":"50.94603935915518", "longitude":"6.959302840118697", "email":"info@rewe-group.com", "postalCode":"50668"}"
to be equal to:

"{
"id": "someBillingAddressId",
"firstName": "Janina",
"lastName": "Nemec",
"streetName": "Domstr.",
"houseNumber": "20",
"city": "Köln",
"country": "Deutschland",
"phone": "+49 221 1490",
"latitude": "50.94603935915518",
"longitude": "6.959302840118697",
"email": "info@rewe-group.com",
"postalCode": "50668"
}
"

when ignoring whitespace differences

v ❌ Test Results

v ❌ AddressAssertionsTest

❌ assertionTest

Expecting actual:
"{"id":"someBillingAddressId", "firstName":"Janina", "lastName":"Nemec", "streetName":"Domstr.", "houseNumber":"20", "city":"Köln", "country":"Deutschland", "phone":"+49 221 1490", "status":"new_customer", "latitude":"50.94603935915518", "longitude":"6.959302840118697", "email":"info@rewe-group.com", "postalCode":"50668"}"
to be equal to:

"{
"id": "someBillingAddressId",
"firstName": "Janina",
"lastName": "Nemec",
"streetName": "Domstr.",
"houseNumber": "20",
"city": "Köln",
"country": "Deutschland",
"phone": "+49 221 1490",
"latitude": "50.94603935915518",
"longitude": "6.959302840118697",
"email": "info@rewe-group.com",
"postalCode": "50668"
}
"

when ignoring whitespace differences

Problem

Failure messages not readable
Hard to know what is going on
These tests are hard to maintain

Problem

Diff is unpleasent to read
Differences can be overlooked
False positives can be highlighted

Improvements

All failures are visible
All data is tested
We test the right thing

THE INCIDENT

> ✅ All Tests

✔️ Code Review

Merged

THE INCIDENT

{
  "id": "someBillingAddressId",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "firstName": "Micha",
  "lastName": "Kutz",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "status": "new_customer",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "postalCode": "50668",
  "email": "info@rewe-group.com"
}

{
  "id": "anotherBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Schanzenstr.",
  "houseNumber": "6-20",
  "city": "Köln-Mülheim",
  "country": "Deutschland",
  "phone": "0221 9758420",
  "latitude": "50.96490882194811",
  "longitude": "7.014472855463499",
  "status": "KNOWN_CUSTOMER",
  "email": "kontakt@rewe-digital.com",
  "postalCode": "51063"
}

Before

After

public enum OldStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;

  @JsonValue
  String value() { return this.name().toLowerCase(); }
}

public enum NewStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;
}

> ✅ All Tests

All the tests were green

But how?

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

public enum CustomerStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;

  @JsonValue
  String value() { return this.name().toLowerCase(); }
}

public enum NewStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;
}

public enum NewStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;

  @JsonValue
  String value() { return this.name().toLowerCase(); }
}

{
  "id": "someBillingAddressId",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "firstName": "Micha",
  "lastName": "Kutz",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "status": "new_customer",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "postalCode": "50668",
  "email": "info@rewe-group.com"
}

{
  "id": "anotherBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Schanzenstr.",
  "houseNumber": "6-20",
  "city": "Köln-Mülheim",
  "country": "Deutschland",
  "phone": "0221 9758420",
  "latitude": "50.96490882194811",
  "longitude": "7.014472855463499",
  "status": "KNOWN_CUSTOMER",
  "email": "kontakt@rewe-digital.com",
  "postalCode": "51063"
}

{
  "id": "someBillingAddressId",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "firstName": "Micha",
  "lastName": "Kutz",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "status": "new_customer",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "postalCode": "50668",
  "email": "info@rewe-group.com"
}

{
  "id": "anotherBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Schanzenstr.",
  "houseNumber": "6-20",
  "city": "Köln-Mülheim",
  "country": "Deutschland",
  "phone": "0221 9758420",
  "latitude": "50.96490882194811",
  "longitude": "7.014472855463499",
  "status": "known_customer",
  "email": "kontakt@rewe-digital.com",
  "postalCode": "51063"
}

How to prevent this in the future?

More file comparison Tests?

A lot of work to Maintain

Hard to read when they fail

Pact Tests?

A lot of setup necessary

Other teams are not interested

Cost to Benefit?

Approval Testing?

?

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    JsonApprovals.verifyJson(callRestEndpointForBillingAddress(orderId));
  }
}

Green Approval Tests

v ✅ Test Results

v ✅ AddressAssertionsTest

✅ assertionTest

v ✅ Test Results

v ✅ AddressAssertionsTest

✅ assertionTest

Benefits

Easy to write
Easy to maintain
Less test code
Everything is tested
The right level is tested
No handling of whitespaces

Red Approval Tests

Benefits

Opens a diff tool already installed
All deviations are visible
Good overview
No issues with whitespaces
Test is easy to fix - if change is wanted

Problems?

Remember to commit approved.json
Test easy is to "fix" - if change is unintentional
Is easily overused

Approval Tests

Usefull for:

Testing Interfaces
Legacy Code Refactoring

Be carefull of:

Approving to fast
Over-use

Outlook:

Visual Approval Tests

Unveiling the Power of Approval Testing

Unveiling the Power of Approval Testing

Janina Nemec

I am…

I like…

Green Assertion Tests

Problem

A Red Assertion

Problem

Soft Assertions

Improvements

Problem

Object Assertion

Problem

Problem

Improvements

One Text Assertion

Problem

Problem

Improvements

THE INCIDENT

THE INCIDENT

THE INCIDENT

THE INCIDENT

Before

After

All the tests were green

But how?

How to prevent this in the future?

More file comparison Tests?

A lot of work to Maintain

Hard to read when they fail

Pact Tests?

A lot of setup necessary

Other teams are not interested

Cost to Benefit?

Approval Testing?

?

Green Approval Tests

Benefits

Red Approval Tests

Benefits

Problems?

Approval Tests

Usefull for:

Testing Interfaces

Legacy Code Refactoring

Be carefull of:

Approving to fast

Over-use

Outlook:

Visual Approval Tests

Live Coding

Workshop

https://github.com/IsItArtOrTrash/approval-testing-power

https://github.com/emilybache/GildedRose-Refactoring-Kata

Unveiling the Power of Approval Testing

More from Janina Nemec