Unveiling the Power of Approval Testing

Unveiling the Power of Approval Testing

Janina Nemec

I am…

👩‍💻 a Software Developer/Crafter

     a co-organizer for a security conference


I like…

💡Learning & Improving ⇒ Software Teaming

🔄 Automatization

🎨 Drawing

👩‍🌾 Gardening

🎮 Video Games

🐶 Dogs 💜

class ShopOrderTest {

  @Test
  void assertionTest() throws JsonProcessingException {
  	// given
    String orderId = "someOrderId";
    ShopOrder order = aDefaultOrder(orderId);
        
    // when
    anOrderWasProcessed(order);
	
    // then
    OrderResult orderResult = jsonMapper.readValue(callRestEndpoint(orderId), OrderResult.class);

    assertThat(orderResult.getId()).isEqualTo("someOrderId");
    assertThat(orderResult.getVersion()).isEqualTo(1);

    ItemResult item = orderResult.getItems().getFirst();
    assertThat(item.getId()).isEqualTo("someItemId");
    assertThat(item.getName()).isEqualTo("ATD 3 Conf. Days");
    assertThat(item.getAmount()).isEqualTo(2);

    PriceResult itemPrice = item.getPrice();
    assertThat(itemPrice.getValue()).isEqualTo(225000);
    assertThat(itemPrice.getMonetaryUnit()).isEqualTo("cent");
    assertThat(itemPrice.getCurrency()).isEqualTo("EUR");

    CouponResult coupon = orderResult.getCoupons().getFirst();
    assertThat(coupon.getId()).isEqualTo("someCouponId");
    assertThat(coupon.getDescription()).isEqualTo("Speaker Coupon");
    assertThat(coupon.getReducedRateInPercentage()).isEqualTo(100);

    assertThat(orderResult.getOrderTimeStamp()).isEqualTo(LocalDateTime.of(2024, 7, 19, 11, 45));
    assertThat(orderResult.getDeliveryDate()).isEqualTo(LocalDate.of(2024, 11, 22));

    PriceResult shippingCost = orderResult.getShippingCost().getFirst();
    assertThat(shippingCost.getValue()).isEqualTo(500);
    assertThat(shippingCost.getMonetaryUnit()).isEqualTo("cent");
    assertThat(shippingCost.getCurrency()).isEqualTo("EUR");

    CustomerResult customer = orderResult.getCustomer();
    assertThat(customer.getId()).isEqualTo("someCustomerId");
    assertThat(customer.getFirstName()).isEqualTo("REWE");
    assertThat(customer.getLastName()).isEqualTo("Digital");

    AddressResult shippingAddress = orderResult.getShippingAddress();
    assertThat(shippingAddress.getId()).isEqualTo("someShippingAddressId");
    assertThat(shippingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(shippingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(shippingAddress.getStreetName()).isEqualTo("Schanzenstr.");
    assertThat(shippingAddress.getHouseNumber()).isEqualTo("6-20");
    assertThat(shippingAddress.getPostalCode()).isEqualTo("51063");
    assertThat(shippingAddress.getCity()).isEqualTo("Köln");
    assertThat(shippingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(shippingAddress.getPhone()).isEqualTo("0221 9758420");
    assertThat(shippingAddress.getLatitude()).isEqualTo("50.96490882194811");
    assertThat(shippingAddress.getLongitude()).isEqualTo("7.014472855463499");
    assertThat(shippingAddress.getEmail()).isEqualTo("kontakt@rewe-digital.com");

    AddressResult billingAddress = orderResult.getBillingAddress();
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Micha");
    assertThat(billingAddress.getLastName()).isEqualTo("Kutz");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

class ShopOrderTest {

  @Test
  void assertionTest() throws JsonProcessingException {
  	// given
    String orderId = "someOrderId";
    ShopOrder order = aDefaultOrder(orderId);
        
    // when
    anOrderWasProcessed(order);
	
    // then
    JsonApprovals.verifyJson(callRestEndpoint(orderId));
  }
}


record Address(
    String id,
    String firstName,
    String lastName,
    String streetName,
    String houseNumber,
    String city,
    String country,
    String phone,
    String latitude,
    String longitude,
    String email,
    String postalCode
) {}

record Order(
    String id,
    int version,
    List<Item> items,
    List<Coupon> coupons,
    LocalDateTime orderTimeStamp,
    LocalDate deliveryDate,
    List<Price> shippingCost,
    Customer customer,
    Address shippingAddress,
    Address billingAddress
) {}

record Item(
    String id,
    String name,
    int amount,
    Price price
) {}

record Coupon(
    String id,
    String description,
    int reducedRateInPercentage
) {}

record Price(
    int value,
    String monetaryUnit,
    String currency
) {}

record Customer(
    String id,
    String firstName,
    String lastName
) {}

record Address(
    String id,
    String firstName,
    String lastName,
    String streetName,
    String houseNumber,
    String city,
    String country,
    String phone,
    String latitude,
    String longitude,
    String email,
    String postalCode
) {}

Green Assertion Tests

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

 v  ✅  Test Results

      v  ✅  AddressAssertionsTest

                ✅  assertionTest

 v  ✅  Test Results

      v  ✅  AddressAssertionsTest

                ✅  assertionTest

Problem

  • Are the tests good?
  • Do they test all fields?
  • Do they test the correct thing?
class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}
class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

A Red Assertion

class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstra.").houseNumber("21")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

                       Expected :"Domstr."
                       Actual       :"Domstra."

                           at AddressAssertionTest.java:19

Problem

  • Only first failing assertion is visible
  • Are there multiple errors?
  • Is there an invisible pattern?

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

                       Expected :"Domstr."
                       Actual       :"Domstra."

                           at AddressAssertionTest.java:19

Soft Assertions

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}


class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    
    SoftAssertions.assertSoftly( softly -> {
      softly.assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
      softly.assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
      softly.assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
      softly.assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
      softly.assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
      softly.assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
      softly.assertThat(billingAddress.getCity()).isEqualTo("Köln");
      softly.assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
      softly.assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
      softly.assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
      softly.assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
      softly.assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
      softly.assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
    });
  }
}

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

                       Multiple Failures (2 failures)
                       -- failure 1 --

                       expected :"Domstr."
                          but was :"Domstra."

                       at AddressAssertionTest.java:16
                       -- failure 2 --

                       expected :"20"
                          but was :"21"

                       at AddressAssertionTest.java:17

Improvements

  • All failing assertions visible
  • Easy to read

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

                       Multiple Failures (2 failures)
                       -- failure 1 --

                       expected :"Domstr."
                          but was :"Domstra."

                       at AddressAssertionTest.java:16
                       -- failure 2 --

                       expected :"20"
                          but was :"21"

                       at AddressAssertionTest.java:17

Problem

  • Missing Assertions
  • A lot boilerplate code
  • Not used much
  • Still not testing the right thing

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

                       Multiple Failures (2 failures)
                       -- failure 1 --

                       expected :"Domstr."
                          but was :"Domstra."

                       at AddressAssertionTest.java:16
                       -- failure 2 --

                       expected :"20"
                          but was :"21"

                       at AddressAssertionTest.java:17

Object Assertion

class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstra.").houseNumber("21")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}


class AddressAssertionTest {
  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstra.").houseNumber("21")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    AddressResult expectedBillingAddress = anAddressResult().id("someBillingAddressId")
        .firstName("Janina").lastName("Nemec")
        .streetName("Domstr.").houseNumber("20")
        .postalCode("50668").city("Köln").country("Deutschland")
        .phone("+49 221 1490").email("info@rewe-group.com")
        .latitude("50.94603935915518").longitude("6.959302840118697")
        .status(CustomerStatus.NEW_CUSTOMER)
        .build();

    assertThat(billingAddress).isEqualTo(expectedBillingAddress);
  }
}

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

                       Expected :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstr., houseNumber=20, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, posta ...
                       Actual       :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstra., houseNumber=21, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, post ...

<Click to see difference>

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

                       Expected :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstr., houseNumber=20, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, posta ...
                       Actual       :AddressResult(id=someBillingAddressId, firstName=Janina, lastName=Nemec, streetName=Domstra., houseNumber=21, city=Köln, country=Deutschland, phone=+49 221 1490, latitude=50.94603935915518, longitude=6.959302840118697, email=info@rewe-group.com, post ...

<Click to see difference>

Problem

  • Failure message hard to read
  • Are there multiple errors?
  • Diff not assessable

Problem

  • Diff is unpleasent to read
  • Differences can be overlooked
  • Not always feasible
  • Still not testing the right thing

Improvements

  • All failures are visible
  • All data is tested

One Text Assertion

class AddressAssertionsTest {

  @Test
  void assertionTest() {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();
        
    anOrderWasProcessed(shopOrder);
    
    assertThat(callRestEndpointForBillingAddress(orderId)).isEqualToIgnoringWhitespace("""
    {
      "id": "someBillingAddressId",
      "firstName": "Janina",
      "lastName": "Nemec",
      "streetName": "Domstr.",
      "houseNumber": "20",
      "city": "Köln",
      "country": "Deutschland",
      "phone": "+49 221 1490",
      "latitude": "50.94603935915518",
      "longitude": "6.959302840118697",
      "email": "info@rewe-group.com",
      "postalCode": "50668",
      "status":"new_customer"
    }""");
  }
}

class AddressAssertionsTest {

  private String TEST_DIR = "src/test/resources/json/FileComparisonTest/";

  @Test
  void assertionTest() {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
    	.billingAddress(...)
        .build();

    anOrderWasProcessed(shopOrder);

    assertThat(callRestEndpointForBillingAddress(orderId)).isEqualToIgnoringWhitespace(
    	new String(Files.readAllBytes(Paths.get(TEST_DIR + "expectedBillingAddress.json")))
    );
  }
}

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

Expecting actual:
"{"id":"someBillingAddressId", "firstName":"Janina", "lastName":"Nemec", "streetName":"Domstr.", "houseNumber":"20", "city":"Köln", "country":"Deutschland", "phone":"+49 221 1490", "status":"new_customer", "latitude":"50.94603935915518", "longitude":"6.959302840118697", "email":"info@rewe-group.com", "postalCode":"50668"}"
to be equal to:

"{
  "id": "someBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "email": "info@rewe-group.com",
  "postalCode": "50668"
}
"

when ignoring whitespace differences

<Click to see difference>

 v  ❌  Test Results

      v  ❌ AddressAssertionsTest

                ❌  assertionTest

Expecting actual:
"{"id":"someBillingAddressId", "firstName":"Janina", "lastName":"Nemec", "streetName":"Domstr.", "houseNumber":"20", "city":"Köln", "country":"Deutschland", "phone":"+49 221 1490", "status":"new_customer", "latitude":"50.94603935915518", "longitude":"6.959302840118697", "email":"info@rewe-group.com", "postalCode":"50668"}"
to be equal to:

"{
  "id": "someBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "email": "info@rewe-group.com",
  "postalCode": "50668"
}
"

when ignoring whitespace differences

<Click to see difference>

Problem

  • Failure messages not readable
  • Hard to know what is going on
  • These tests are hard to maintain

Problem

  • Diff is unpleasent to read
  • Differences can be overlooked
  • False positives can be highlighted

Improvements

  • All failures are visible
  • All data is tested
  • We test the right thing

THE INCIDENT

THE INCIDENT

THE INCIDENT

 >  ✅  All Tests

 ✔️ Code Review

Merged

THE INCIDENT

{
  "id": "someBillingAddressId",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "firstName": "Micha",
  "lastName": "Kutz",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "status": "new_customer",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "postalCode": "50668",
  "email": "info@rewe-group.com"
}
{
  "id": "anotherBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Schanzenstr.",
  "houseNumber": "6-20",
  "city": "Köln-Mülheim",
  "country": "Deutschland",
  "phone": "0221 9758420",
  "latitude": "50.96490882194811",
  "longitude": "7.014472855463499",
  "status": "KNOWN_CUSTOMER",
  "email": "kontakt@rewe-digital.com",
  "postalCode": "51063"
}

Before

After

public enum OldStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;

  @JsonValue
  String value() { return this.name().toLowerCase(); }
}
public enum NewStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;
}

 >  ✅  All Tests

All the tests were green

But how?

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}


class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}


public enum CustomerStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;

  @JsonValue
  String value() { return this.name().toLowerCase(); }
}
public enum NewStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;
}
public enum NewStatus {
  NEW_CUSTOMER,
  KNOWN_CUSTOMER;

  @JsonValue
  String value() { return this.name().toLowerCase(); }
}
{
  "id": "someBillingAddressId",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "firstName": "Micha",
  "lastName": "Kutz",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "status": "new_customer",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "postalCode": "50668",
  "email": "info@rewe-group.com"
}
{
  "id": "anotherBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Schanzenstr.",
  "houseNumber": "6-20",
  "city": "Köln-Mülheim",
  "country": "Deutschland",
  "phone": "0221 9758420",
  "latitude": "50.96490882194811",
  "longitude": "7.014472855463499",
  "status": "KNOWN_CUSTOMER",
  "email": "kontakt@rewe-digital.com",
  "postalCode": "51063"
}
{
  "id": "someBillingAddressId",
  "streetName": "Domstr.",
  "houseNumber": "20",
  "firstName": "Micha",
  "lastName": "Kutz",
  "city": "Köln",
  "country": "Deutschland",
  "phone": "+49 221 1490",
  "status": "new_customer",
  "latitude": "50.94603935915518",
  "longitude": "6.959302840118697",
  "postalCode": "50668",
  "email": "info@rewe-group.com"
}
{
  "id": "anotherBillingAddressId",
  "firstName": "Janina",
  "lastName": "Nemec",
  "streetName": "Schanzenstr.",
  "houseNumber": "6-20",
  "city": "Köln-Mülheim",
  "country": "Deutschland",
  "phone": "0221 9758420",
  "latitude": "50.96490882194811",
  "longitude": "7.014472855463499",
  "status": "known_customer",
  "email": "kontakt@rewe-digital.com",
  "postalCode": "51063"
}

How to prevent this in the future?

  • More file comparison Tests?

A lot of work to Maintain

Hard to read when they fail

  • Pact Tests?

A lot of setup necessary

Other teams are not interested

Cost to Benefit?

  • Approval Testing?

?

class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    AddressResult billingAddress = jsonMapper.readValue(callRestEndpointForBillingAddress(orderId));
    assertThat(billingAddress.getId()).isEqualTo("someBillingAddressId");
    assertThat(billingAddress.getFirstName()).isEqualTo("Janina");
    assertThat(billingAddress.getLastName()).isEqualTo("Nemec");
    assertThat(billingAddress.getStreetName()).isEqualTo("Domstr.");
    assertThat(billingAddress.getHouseNumber()).isEqualTo("20");
    assertThat(billingAddress.getPostalCode()).isEqualTo("50668");
    assertThat(billingAddress.getCity()).isEqualTo("Köln");
    assertThat(billingAddress.getCountry()).isEqualTo("Deutschland");
    assertThat(billingAddress.getPhone()).isEqualTo("+49 221 1490");
    assertThat(billingAddress.getLatitude()).isEqualTo("50.94603935915518");
    assertThat(billingAddress.getLongitude()).isEqualTo("6.959302840118697");
    assertThat(billingAddress.getStatus()).isEqualTo(CustomerStatus.NEW_CUSTOMER);
    assertThat(billingAddress.getEmail()).isEqualTo("info@rewe-group.com");
  }
}


class AddressAssertionTest {

  @Test
  void assertionTest() throws JsonProcessingException {
    String orderId = "someOrderId";
    ShopOrder shopOrder = anyOrder(orderId)
        .billingAddress(anAddress().id("someBillingAddressId")
            .firstName("Janina").lastName("Nemec")
            .streetName("Domstr.").houseNumber("20")
            .postalCode("50668").city("Köln").country("Deutschland")
            .phone("+49 221 1490").email("info@rewe-group.com").build()
        ).build();

    anOrderWasProcessed(shopOrder);

    JsonApprovals.verifyJson(callRestEndpointForBillingAddress(orderId));
  }
}

Green Approval Tests

 v  ✅  Test Results

      v  ✅  AddressAssertionsTest

                ✅  assertionTest

 v  ✅  Test Results

      v  ✅  AddressAssertionsTest

                ✅  assertionTest

Benefits

  • Easy to write
  • Easy to maintain
  • Less test code
  • Everything is tested
  • The right level is tested
  • No handling of whitespaces

Red Approval Tests

Benefits

  • Opens a diff tool already installed
  • All deviations are visible
  • Good overview
  • No issues with whitespaces
  • Test is easy to fix - if change is wanted

Problems?

  • Remember to commit approved.json
  • Test easy is to "fix" - if change is unintentional
  • Is easily overused

Approval Tests

Usefull for:

  • Testing Interfaces

  • Legacy Code Refactoring

Be carefull of:

  • Approving to fast

  • Over-use

Outlook:

  • Visual Approval Tests

Live Coding

Workshop

https://github.com/IsItArtOrTrash/approval-testing-power

https://github.com/emilybache/GildedRose-Refactoring-Kata