Triton meets RxJava

CAIS report

Credit
Account
Information
Sharing

CAIS report

Credit
Account
Information
Sharing

report

database

convert

database

format

map

fetch

write

report

convert

format

map

fetch

write

report

triton-enterprise

batch

cais

database

report

triton-enterprise

database

convert

format

map

fetch

write

batch

cais

Module dependency

batch

cais

triton-enterprise

Module dependency

batch

cais

triton-enterprise

Generate the report

public static void generateCaisReport() {

  try (final Writer writer = new FileWriter("CAIS-report.txt")) {

    final List<CaisDTO> caisDTOs = CaisDAO.fetchAllData();

    CaisReport.writeReport(caisDTOs, writer);

  } catch (IOException e) {
    throw new ApplicationException("Error writing CAIS report file.", e);
  }
}

public static List<CaisDTO> fetchAllData() {
  return new QueryExecutor(new Query()
        .select(
            "Product.number AS accountNumber",
            "Person.relationNumber",
            "PP1.isMainProductHolder",
            "PP2.personID AS otherPersonID",
            "BaProduct.iban",
            "Product.activeBeginDate",
            "Product.activeEndDate",
            "Private.prefixTitle",
            "Private.forenames",
            "Private.initials",
            "Person.fullSurname",
            "Private.birthDate",
            "Address.coLine",
            "Address.subBuilding",
            "Address.houseName",
            "Address.houseNumber",
            "Address.street",
            "Address.place",
            "Address.locality",
            "Address.district",
            "Address.region",
            "Address.postcode",
            "BaProductOverdraftTerms.dtLimitAmount",
            "BaProductMgtInfo.balance",
            "BaProductMgtInfo.lastDebitTxDate",
            "BaProductMgtInfo.lastCreditTxDate"
        .from("Product")
        .join("ProductType ON ProductType.productTypeID = Product.productTypeID")
        .join("BaProduct ON BaProduct.productID = Product.productID")
        .join("PersonProduct PP1 ON PP1.productID = Product.productID")
        .leftJoin("PersonProduct PP2 ON PP2.productID = PP1.productID AND PP2.productRoleID = PP1.productRoleID AND PP2.personID <> PP1.personID")
        .join("Person ON Person.personID = PP1.personID")
        .join("Private ON Private.personID = Person.personID")
        .join("Address ON Address.personID = Person.personID")
        .leftJoin("BaProductOverdraftTerms ON BaProductOverdraftTerms.productID = Product.productID")
        .join("BaProductMgtInfo ON BaProductMgtInfo.productID = Product.productID")
        .where("Product.activeBeginDate <= ?", getToDate())
        .and("Product.activeEndDate IS NULL OR (Product.activeEndDate >= ? AND Product.activeEndDate <= ?)", getFromDate(), getToDate())
        .and("ProductType.marketID = " + ID_PERSONAL)
        .and("ProductType.productGroupID = " + ID_CURRENTACCOUNT)
        .and("ProductType.administrationID = " + ID_TRIODOS_BANK_UK)
        .and("BaProduct.signatureCard = " + RECEIVED.getKey())
        .and("PP1.productRoleID = " + ID_ACCOUNTHOLDER)
        .and("Address.addressTypeID = " + ID_OFFICIAL)
        .and("Address.isActive = 'T'")
        .and("BaProductMgtInfo.monthRunYearNumber = ?", getFromDate().getYear())
        .and("BaProductMgtInfo.monthRunMonthNumber = ?", getFromDate().getMonth())
        .getObjects(CaisDTO.class);
}

Fetch the data

public static List<CaisDTO> fetchAllData() {

  return new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...
      .getObjects(CaisDTO.class);
}

Fetch the data

public static void writeReport(final List<CaisDTO> caisDTOs, final Writer writer) {

  writeHeaderRecord(writer);

  caisDTOs.stream()
      .filter(new CaisValidator()::validateDTO)
      .map(CaisMapper::mapToRecord)
      .forEach(recordData -> writeAccountRecord(recordData, writer));

  writeTrailerRecord(caisDTOs.size(), writer);
}

WRITE THE REPORT

public static void writeReport(final List<CaisDTO> caisDTOs, final Writer writer) {

  writeHeaderRecord(writer);

  caisDTOs.stream()
      .filter(new CaisValidator()::validateDTO)
      .map(CaisMapper::mapToRecord)
      .forEach(recordData -> writeAccountRecord(recordData, writer));

  writeTrailerRecord(caisDTOs.size(), writer);
}

WRITE THE REPORT

Gold-plate the generator

public static void generateCaisReport() {

  try (final Writer writer = new FileWriter("CAIS-report.txt")) {

    final List<CaisDTO> caisDTOs = CaisDAO.fetchAllData();

    CaisReport.writeReport(caisDTOs, writer);

  } catch (IOException e) {
    throw new ApplicationException("Error writing CAIS report file.", e);
  }
}

public static void generateCaisReport() {

  try (final Writer writer = new FileWriter("CAIS-report.txt")) {

    final Stream<CaisDTO> dtoStream = CaisDAO.fetchDataAsStream();

    CaisReport.writeReport(dtoStream, writer);

  } catch (IOException e) {
    throw new ApplicationException("Error writing CAIS report file.", e);
  }
}

Gold-plate the generator

public static void writeReport(final Stream<CaisDTO> caisDTOs, final Writer writer) {

  writeHeaderRecord(writer);

  int numRecords = 0;
  caisDTOs
      .filter(new CaisValidator()::validateDTO)
      .map(CaisMapper::mapToRecord)
      .forEach(recordData -> {
        writeAccountRecord(recordData, writer);
        numRecords++;
      });

  writeTrailerRecord(numRecords, writer);
}

Gold-plate the writer

Gold-plate Fetching data

public static Stream<CaisDTO> fetchDataAsStream() {

  return new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...
      .getObjects(CaisDTO.class)
      .stream();
}

Gold-plate Fetching data

public static Stream<CaisDTO> fetchDataAsStream() {

  return new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...
      .getObjects(CaisDTO.class)
      .stream();
}

public static Stream<CaisDTO> fetchDataAsStream() {

  return new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...
      .getObjects(CaisDTO.class)
      .stream();
}

Gold-plate Fetching data

// class QueryExecutor
public <T> List<T> getObjects(@NotNull final Class<T> clazz) {
  List<Object[]> rows = getResults(Page.ALL_ROWS_ON_ONE_PAGE);
  RowConverter<T> converter = ReflectionBasedRowConverter.create(clazz,
      resultSetColumnNames);
  return convert(rows, converter);
}

Gold-plate Fetching data

// class MyStreamingQueryExecutor
public <T> List<T> getObjects(final Class<T> clazz) {
  List<T> fetchedDTOs = new ArrayList<>();

    ...
  String sql = "SELECT * FROM ...";
  Statement statement = jdbcConnection.createStatement(sql);
  ResultSet resultSet = statement.executeQuery();

  while (resultSet.next()) {
    T dto = clazz.newInstance();
    doSomeReflectionMagic(resultSet, dto);

    fetchedDTOs.add(dto);
  }

  return fetchedDTOs;
}

Gold-plate Fetching data

// class MyStreamingQueryExecutor
public <T> void writeObjects(Class<T> clazz, CaisValidator validator, Writer writer) {


    ...
  String sql = "SELECT * FROM ...";
  Statement statement = jdbcConnection.createStatement(sql);
  ResultSet resultSet = statement.executeQuery();

  while (resultSet.next()) {
    T dto = clazz.newInstance();
    doSomeReflectionMagic(resultSet, dto);

    if (dto instanceof CaisDTO) {
      CaisDTO caisDTO = (CaisDTO) dto;
      if (validator.validateDTO(caisDTO)) {
        final RecordData recordData = CaisMapper.mapToRecord(caisDTO);
        CaisReport.writeAccountRecord(recordData, writer);
      }
    }
  }
}

Gold-plate Fetching data

// class MyStreamingQueryExecutor
public <T> void processObjects(Class<T> clazz, Consumer<T> processFunc) {


    ...
  String sql = "SELECT * FROM ...";
  Statement statement = jdbcConnection.createStatement(sql);
  ResultSet resultSet = statement.executeQuery();

  while (resultSet.next()) {
    T dto = clazz.newInstance();
    doSomeReflectionMagic(resultSet, dto);

    processFunc.accept(dto);
  }
}


// (in cais module)
public void processDTO(CaisDTO dto) {
  if (validator.validateDTO(dto)) { // validator?
    final RecordData recordData = CaisMapper.mapToRecord(dto);
    CaisReport.writeAccountRecord(recordData, writer); // writer?
  }
}

Gold-plate Fetching data

public static void fetchAndProcess() {

  new MyStreamingQueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...
      .processObjects(CaisDTO.class, CaisReport::processDTO);
}

Gold-plate Fetching data

public static void generateCaisReport() {

  try (final Writer writer = new FileWriter("CAIS-report.txt")) {

    writeHeaderRecord(writer);

    fetchAndProcess();

    writeTrailerRecord(writer);

  } catch (IOException e) {
    throw new ApplicationException("Error writing CAIS report file.", e);
  }
}

Gold-plate the Report

public static void generateCaisReport() {

  try (final Writer writer = new FileWriter("CAIS-report.txt")) {

    writeHeaderRecord(writer);

    fetchAndProcess(); // TODO - count records and use validator and writer

    writeTrailerRecord(writer); // TODO - use numAccountRecords

  } catch (IOException e) {
    throw new ApplicationException("Error writing CAIS report file.", e);
  }
}

Gold-plate the Report

Javadoc of JDBC Driver for DB2:

By default the driver collects all the results for the query at once. This can be inconvenient for large data sets so the JDBC driver provides a means of basing a ResultSet on a database cursor and only fetching a small number of rows. The fetchSize property differs from the queryDataSize property. fetchSize affects the number of rows that are returned, and queryDataSize affects the number of bytes that are returned.
 

fetchSize property

Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is ignored. The default value is zero.

RxJava

to the rescue!

Observable pattern

public interface Observer<T> {

  void onNext(T item);

  void onCompleted();

  void onError(Throwable t);
}

public interface Observable<T> {

  void subscribe(Observer<T> observer);






}

Observable pattern

public interface Observable<T> {

  void subscribe(Observer<T> observer);

  Observable<T> filter(Predicate<T> filterFunc);
  <R> Observable<R> map(Function<T, R> mapFunc);
  Observable<T> delay(long delay, TimeUnit unit);
  Observable<Integer> count();
    ...
}

public interface Observer<T> {

  void onNext(T item);

  void onCompleted();

  void onError(Throwable t);
}

// class CaisDAO
public static List<CaisDTO> fetchAllData() {

  final List<CaisDTO> fetchedDTOs = new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...


      .getObjects(CaisDTO.class);






  return fetchedDTOs;
}

Gold-plating (continued)

// class CaisDAO
public static List<CaisDTO> fetchNextPage() {

  final List<CaisDTO> fetchedDTOs = new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...


      .getObjects(CaisDTO.class, new Page(FIRST_PAGE, MAX_ROWS));






  return fetchedDTOs;
}

Gold-plating (continued)

// class CaisDAO
public static List<CaisDTO> fetchNextPage(final FetchContext fetchContext) {

  final List<CaisDTO> fetchedDTOs = new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...
      .and("Product.number > ?", fetchContext.getLastFetchedAccountNumber())
      .orderBy("Product.number ASC")
      .getObjects(CaisDTO.class, new Page(FIRST_PAGE, MAX_ROWS));






  return fetchedDTOs;
}

Gold-plating (continued)

// class CaisDAO
public static List<CaisDTO> fetchNextPage(final FetchContext fetchContext) {

  final List<CaisDTO> fetchedDTOs = new QueryExecutor(new Query()
      .select(
          "Product.number AS accountNumber",
          "Person.relationNumber",
           ...
      .from("Product")
      .join("PersonProduct ON PersonProduct.productID = Product.productID")
      .join("Person ON Person.personID = PersonProduct.personID")
        ...
      .where("PersonProduct.productRoleID = " + ID_ACCOUNTHOLDER)
        ...
      .and("Product.number > ?", fetchContext.getLastFetchedAccountNumber())
      .orderBy("Product.number ASC")
      .getObjects(CaisDTO.class, new Page(FIRST_PAGE, MAX_ROWS));

  if (!fetchedDTOs.isEmpty()) {
    final CaisDTO lastDTO = fetchedDTOs.get(fetchedDTOs.size() - 1);
    fetchContext.setLastFetchedAccountNumber(lastDTO.getAccountNumber());
  }

  return fetchedDTOs;
}

Gold-plating (continued)

Create an Observable

public static Observable<CaisDTO> toObservable(
    final FetchContext fetchContext,
    final Function<FetchContext, List<CaisDTO>> fetchNextPageFunc) {

  return Observable.create(









      );
}

Create an Observable

public static Observable<CaisDTO> toObservable(
    final FetchContext fetchContext,
    final Function<FetchContext, List<CaisDTO>> fetchNextPageFunc) {

  return Observable.create(
      observer -> {








      });
}

Create an Observable

public static Observable<CaisDTO> toObservable(
    final FetchContext fetchContext,
    final Function<FetchContext, List<CaisDTO>> fetchNextPageFunc) {

  return Observable.create(
      observer -> {


            fetchNextPageFunc.apply(fetchContext) ... // page of List<CaisDTO>





      });
}

Create an Observable

public static Observable<CaisDTO> toObservable(
    final FetchContext fetchContext,
    final Function<FetchContext, List<CaisDTO>> fetchNextPageFunc) {

  return Observable.create(
      observer -> {


            fetchNextPageFunc.apply(fetchContext).forEach(observer::onNext);





      });
}

Create an Observable

public static Observable<CaisDTO> toObservable(
    final FetchContext fetchContext,
    final Function<FetchContext, List<CaisDTO>> fetchNextPageFunc) {

  return Observable.create(
      observer -> {

          while (fetchContext.isDataAvailable()) {
            fetchNextPageFunc.apply(fetchContext).forEach(observer::onNext);
          }




      });
}

Create an Observable

public static Observable<CaisDTO> toObservable(
    final FetchContext fetchContext,
    final Function<FetchContext, List<CaisDTO>> fetchNextPageFunc) {

  return Observable.create(
      observer -> {

          while (fetchContext.isDataAvailable()) {
            fetchNextPageFunc.apply(fetchContext).forEach(observer::onNext);
          }
          observer.onCompleted();



      });
}

Create an Observable

public static Observable<CaisDTO> toObservable(
    final FetchContext fetchContext,
    final Function<FetchContext, List<CaisDTO>> fetchNextPageFunc) {

  return Observable.create(
      observer -> {
        try {
          while (fetchContext.isDataAvailable()) {
            fetchNextPageFunc.apply(fetchContext).forEach(observer::onNext);
          }
          observer.onCompleted();
        } catch (Exception e) {
          observer.onError(e);
        }
      });
}

Create an Observable

public static <F extends FetchContext, R> Observable<R> toObservable(
    final F fetchContext,
    final Function<F, List<R>> fetchNextPageFunc) {

  return Observable.create(
      observer -> {
        try {
          while (fetchContext.isDataAvailable()) {
            fetchNextPageFunc.apply(fetchContext).forEach(observer::onNext);
          }
          observer.onCompleted();
        } catch (Exception e) {
          observer.onError(e);
        }
      });
}

Create an Observable

public static generateCaisReport() {

  try (final Writer writer = new FileWriter("CAIS-report.txt")) {

    final List<CaisDTO> dtoList
        = CaisDAO.fetchAllData();

    CaisReport.writeReport(dtoList, writer);

  } catch (IOException e) {
    throw new ApplicationException("Error writing CAIS report file.", e);
  }
}

All-at-once report

public static generateCaisReport() {

  try (final Writer writer = new FileWriter("CAIS-report.txt")) {

    final Observable<CaisDTO> dtoObservable
        = toObservable(new FetchContext(), CaisDAO::fetchNextPage));

    CaisReport.writeReport(dtoObservable, writer);

  } catch (IOException e) {
    throw new ApplicationException("Error writing CAIS report file.", e);
  }
}

Streaming report

public static void writeReport(
    final List<CaisDTO> caisDTOs,
    final Writer writer) {

  writeHeaderRecord(writer);
  caisDTOs.stream()
      .filter(new CaisValidator()::validateDTO)
      .map(CaisMapper::mapToRecord)
      .forEach(recordData -> writeAccountRecord(recordData, writer));
  writeTrailerRecord(caisDTOs.size(), writer); // bug!
}

All-at-once report

public static void writeReport(
    final Observable<CaisDTO> caisDTOs,
    final Writer writer) {

  caisDTOs
      .doOnSubscribe(()     -> writeHeaderRecord(writer))
      .filter(                 new CaisValidator()::validateDTOs)
      .map(                    CaisMapper::mapToRecordData)
      .doOnNext(recordData  -> writeAccountRecord(recordData, writer))
      .count()
      .subscribe(numRecords -> writeTrailerRecord(numRecords, writer),
                               LOG::error);
}

Streaming report