Building Resilient

API Dependency

Dolganov Sergey

@ Evil Martians

No Magic Involved

Title Text

What's my problem?

eBay for Business (eBaymag)

Distributed system built upon lots of API calls to external systems, e.g.

eBay, UPS, Google Translate

Contract is:

— Pre-conditions (request)

— Post-conditions (response)

— Invariants (state)

Input

Output

function (input) { ... } # =>

State

Pre-conditions

Post-conditions

Invariants

Contract

Evolution

— Input/Output Control

— State Control

— Homogeneous Design

Problem solving

API Client Problem

Step One

Standalone

Policy

eBay Orders,

eBaymag Parcels

Layout (for DHL)

Lots of validations

HTTP Client

Confusing

content

module DHL
  module API
    def get_tariff(mapped_request)
      sampler = TariffSampler.new
      sampler.register_request(mapped_request)
      response = http_client.get("/GetQuote", mapped_request.to_xml)
      sampler.register_response(response)
      sampler.tag!
      response
    end

    class TariffSampler < BaseSampler
      def tag!
        tag_success!
        tag_dhl_address_ambiguity_warning!
        tag_dhl_composite_call_warning!
        tag_data_invalid_input_error!
        tag_destination_invalid_for_dhl!
        tag_unexpected_behavior! if tags.empty?
      end

      # ...
    end
  end
end

"Contract"

        module DHL
          class Calculator < ::BaseService
            attr_reader :errors
            param :parcel

            def call
              policy = DHL::RegionalPolicy[parcel]
              return if policy.invalid?

              policy = DHL::Requests::TariffPolicy[parcel]
              return if policy.invalid?

              response = DHL::API.get_tariff(Requests::TariffMapper[parcel])
              mapped_response = Responses::TariffMapper[response]

              policy = DHL::Responses::TariffPolicy[response, mapped_response]
              return if policy.invalid?

              parcel.merge_update!(dhl: mapped_response)
            ensure
              @errors = policy&.errors 
            end
          end
        end

        if (service = DHL::Calculator.new(parcel)).call
          render json: { tariff: parcel.reload.dhl["tariff"] }
        else
          render json: { errors: service.errors.to_h }
        end

module DHL
  module Responses
    class TariffPolicy < Tram::Policy
      param :response
      param :mapped_response

      validate :no_response_errors, stop_on_failure: true
      validate :delivery_cost_presence
      validate :delivery_currency_presence
      validate :delivery_date_presence

      def no_response_errors
        return if response.errors.empty?

        errors.add :dhl_responded_with_error, 
                   errors: response.errors
      end

      # implementation of other methods
  end
end

        module DHL
          class Calculator < ::BaseService
            attr_reader :errors
            param :parcel

            def call
              policy = DHL::RegionalPolicy[parcel]
              return if policy.invalid?

              policy = DHL::Requests::TariffPolicy[parcel]
              return if policy.invalid?

              response = DHL::API.get_tariff(Requests::TariffMapper[parcel])
              mapped_response = Responses::TariffMapper[response]

              policy = DHL::Responses::TariffPolicy[response, mapped_response]
              return if policy.invalid?

              parcel.merge_update!(dhl: mapped_response)
            ensure
              @errors = policy&.errors 
            end
          end
        end

        if (service = DHL::Calculator.new(parcel)).call
          render json: { tariff: parcel.reload.dhl["tariff"] }
        else
          render json: { errors: service.errors.to_h }
        end

    class DHL::Requests::TariffMapper 
      extend Dry::Initializer

      option :parcel, optional: true

      option :origin,        default: -> { default_origin }
      option :destination,   default: -> { default_destination }
      option :reference_id,  default: -> { default_reference_id }
      option :value,         default: -> { default_value }
      option :weight,        default: -> { default_weight }

      # other parameters

      def call
        [
          route_attributes,
          size_weight_attributes,
          package_attributes,
        ].reduce(:merge)
      end
    end

        module DHL
          class Calculator < ::BaseService
            attr_reader :errors
            param :parcel

            def call
              policy = DHL::RegionalPolicy[parcel]
              return if policy.invalid?

              policy = DHL::Requests::TariffPolicy[parcel]
              return if policy.invalid?

              response = DHL::API.get_tariff(Requests::TariffMapper[parcel])
              mapped_response = Responses::TariffMapper[response]

              policy = DHL::Responses::TariffPolicy[response, mapped_response]
              return if policy.invalid?

              parcel.merge_update!(dhl: mapped_response)
            ensure
              @errors = policy&.errors 
            end
          end
        end

        if (service = DHL::Calculator.new(parcel)).call
          render json: { tariff: parcel.reload.dhl["tariff"] }
        else
          render json: { errors: service.errors.to_h }
        end

Dry::Initializer

+

Tram::Policy

+

BloodContracts

Order

Step Two

Sagas

Transactions without Transactions

Sagas (with Orchestration):

1. Wrap each mutation in a Command

(each could be done and undone, e.g. #up and #down in AR migration)

2. Chain Command

3. Execute chain

4. If one fails run `undo`

...

Failure

Validate

...

Success

Validate

Parcel

Parcel*

Saga Orchestrator

...

Success

Validate

PrintLabel

Validate#call

PrintLabel#call

...

Failure

Validate

PrintLabel

Validate#undo

PrintLabel#undo

Aaand that's it?!

DirtyPipeline

(Sagas)

Real World

What if?

Orchestrator runtime error
Application shuts down in the middle
Another pipeline runs in parallel

RussianPost Pipeline

def create
  account = Account.find(params[:account_id])
  parcel = account.parcels.find(params[:id]) 
  status = 
    RussianPost::Pipeline.chain(:Validate)
                         .chain(:GetTransportationCost)
                         .chain(:Register)
                         .chain(:PayOnline)
                         .chain(:PrintLabel)
                         .new(parcel)
                         .call

  render json: status.payload
end

...

Success

Action 1

Action N

Pipeline

(Action 1 Attempt)

(Action N Attempt)

Success

Failure

(Done)

Subject

call

Events

(Action 1 Attempt)

(Action N Attempt)

(Done)

Cleaner

Success

...

Success

Action 1

Action N

applied?

Events

...

Success

Action 1

Action N

Pipeline

(Action 1 Attempt)

(Action N Attempt)

Success

Failure

(Done)

Subject

call

Events

(Action 1 Attempt)

(Action N Attempt)

(Done)

Cleaner

Failure

...

Success

Action N

Action 1

undo

(reversed)

undo

(Action 1 Attempt)

(Action N Attempt)

(Done)

Failure

...

Success

Action N

Action 1

undo

Failure

(reversed)

lock Subject

with error

Cleaner

Events

Solution from eBaymag

— Pipeline

— Lock (interface)

— Action

— Events (interface)

— Clean

— Status

Check out in more details

https://slides.com/sergeydolganov/sagas-wizards-ruby

Step Three

Homogeneous Design

Functional Way to Validation

Compose

+

Errors Processing

Algebraic Data Types

module UPS
  module Rates
    # types product, 
    # Request passes both RegionalParcel and Schema contracts
    RequestContract = 
      RegionalParcelContract.and_then(SchemaContract)
    
    # types sum, 
    # Error passes either RecoverableInputError or 
    # InvalidRequest or BasicError contract
    ErrorContract = 
      RecoverableInputErrorContract
        .or(InvalidRequestContract)
        .or(BasicErrorContract) 

    # combine
    ResponseContract = 
      Contracts::JSONResponseContract.and_then(
        TariffContract.or(ErrorContract)
      )
  end
end

Refinement Types

— Type (ADT)

— Predicate

{ n :: Integer | n > 5 }

Context:

"Why did validation fail?"

module Contracts
  class JSONResponseContract < BaseContract
    alias :response :value

    def match
      context[:body] = response.body
      context[:parsed] = ::JSON.parse(response.body)
      self
    rescue StandardError => error
      ContractFailure.new(error, context) 
    end

    def unpack
      match.context[:parsed]
    end
  end
end

Refinement Type

module UPS
  module Shipping
    class ParcelContract < BaseContract
      extract :origin
      extract :destination
      extract :weight

      def match
        extract!
        data = context.slice(*self.class.extractables)
        return self if ParcelPolicy[**data].valid?
        PolicyFailure.new(policy.errors, context)
      end

      def unpack
        match.context.slice(*self.class.extractables)
      end
    end
  end
end

Refinement Type

Advanced

        class UPS::RatesContract < BaseContract # sampling by types
          def match(*input)
             input_match = RequestContract.match(*input) 
             return input_match if input_match.invalid?
           
             result = yield(input_match)
             ResponseContract.match(result, input_match.context)
          end
        end

        class UPS::API
          def rates_request(parcel)
            RatesContract.match(parcel) do |mapped_parcel|
              http_client.get('/1.0/Rates', mapped_parcel.unpack)
            end # returns a contract object
          end
        end

        case (match = UPS::API.rate_request(parcel))
        when TariffContract
           render json: { tariff: match.unpack }
        when PolicyFailure
          render json: { errors: match.errors }
        when InvalidRequest
          Honeybadger.notify "Invalid UPS request", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        when ContractFailure
          Honeybadger.notify "Unexpected UPS behavior", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        else
          raise "Invalid contract processing"
        end

Contract Definition

module UPS
  module Rates
    # types product, 
    # Request passes both RegionalParcel and Schema contracts
    RequestContract = 
      RegionalParcelContract.and_then(SchemaContract)
    
    # types sum, 
    # Error passes either RecoverableInputError or 
    # InvalidRequest or BasicError contract
    ErrorContract = 
      RecoverableInputErrorContract
        .or(InvalidRequestContract)
        .or(BasicErrorContract) 

    # combine
    ResponseContract = 
      Contracts::JSONResponseContract.and_then(
        TariffContract.or(ErrorContract)
      )
  end
end

        class UPS::RatesContract < BaseContract # sampling by types
          def match(*input)
             input_match = RequestContract.match(*input) 
             return input_match if input_match.invalid?
           
             result = yield(input_match)
             ResponseContract.match(result, input_match.context)
          end
        end

        class UPS::API
          def rates_request(parcel)
            RatesContract.match(parcel) do |mapped_parcel|
              http_client.get('/1.0/Rates', mapped_parcel.unpack)
            end # returns a contract object
          end
        end

        case (match = UPS::API.rate_request(parcel))
        when TariffContract
           render json: { tariff: match.unpack }
        when PolicyFailure
          render json: { errors: match.errors }
        when InvalidRequest
          Honeybadger.notify "Invalid UPS request", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        when ContractFailure
          Honeybadger.notify "Unexpected UPS behavior", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        else
          raise "Invalid contract processing"
        end

        class UPS::RatesContract < BaseContract # sampling by types
          def match(*input)
             input_match = RequestContract.match(*input) 
             return input_match if input_match.invalid?
           
             result = yield(input_match)
             ResponseContract.match(result, input_match.context)
          end
        end

        class UPS::API
          def rates_request(parcel)
            RatesContract.match(parcel) do |mapped_parcel|
              http_client.get('/1.0/Rates', mapped_parcel.unpack)
            end # returns a contract object
          end
        end

        case (match = UPS::API.rate_request(parcel))
        when TariffContract
           render json: { tariff: match.unpack }
        when PolicyFailure
          render json: { errors: match.errors }
        when InvalidRequest
          Honeybadger.notify "Invalid UPS request", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        when ContractFailure
          Honeybadger.notify "Unexpected UPS behavior", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        else
          raise "Invalid contract processing"
        end

        class UPS::RatesContract < BaseContract # sampling by types
          def match(*input)
             input_match = RequestContract.match(*input) 
             return input_match if input_match.invalid?
           
             result = yield(input_match)
             ResponseContract.match(result, input_match.context)
          end
        end

        class UPS::API
          def rates_request(parcel)
            RatesContract.match(parcel) do |mapped_parcel|
              http_client.get('/1.0/Rates', mapped_parcel.unpack)
            end # returns a contract object
          end
        end

        case (match = UPS::API.rate_request(parcel))
        when TariffContract
           render json: { tariff: match.unpack }
        when PolicyFailure
          render json: { errors: match.errors }
        when InvalidRequest
          Honeybadger.notify "Invalid UPS request", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        when ContractFailure
          Honeybadger.notify "Unexpected UPS behavior", context: match.context
          render json: { errors: "Unprocessable now, try later" }
        else
          raise "Invalid contract processing"
        end