Boto3
collections
CSV
gzip
multiprocessing
NumPy
pandas 
pickle
PyGreSQL
re
SciPy
sklearn
sklearn.feature_extraction
sklearn.preprocessing
xml.etree.ElementTree
zipfile

import boto3
import numpy
import pandas


print('boto3', boto3.__version__)
print('numpy', numpy.__version__)
print('pandas', pandas.__version__)

Traceback (most recent call last):
  File "/tmp/runscript.py", line 117, in <module>
    download_and_install(args.extra_py_files)
  File "/tmp/runscript.py", line 56, in download_and_install
    download_from_s3(s3_file_path, local_file_path)
  File "/tmp/runscript.py", line 81, in download_from_s3
    s3.download_file(bucket_name, s3_key, new_file_path)
  File "/usr/local/lib/python3.6/site-packages/boto3/s3/inject.py", line 172, in download_file
    extra_args=ExtraArgs, callback=Callback)
  File "/usr/local/lib/python3.6/site-packages/boto3/s3/transfer.py", line 307, in download_file
    future.result()
  File "/usr/local/lib/python3.6/site-packages/s3transfer/futures.py", line 106, in result
    return self._coordinator.result()
  File "/usr/local/lib/python3.6/site-packages/s3transfer/futures.py", line 265, in result
    raise self._exception
  File "/usr/local/lib/python3.6/site-packages/s3transfer/tasks.py", line 255, in _main
    self._submit(transfer_future=transfer_future, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/s3transfer/download.py", line 345, in _submit
    **transfer_future.meta.call_args.extra_args
  File "/usr/local/lib/python3.6/site-packages/botocore/client.py", line 357, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.6/site-packages/botocore/client.py", line 661, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (404) when calling the HeadObject operation: Not Found

numpy-1.19.0-cp36-cp36m-manylinux2010_x86_64.whl
pandas-1.0.5-cp36-cp36m-manylinux1_x86_64.whl

import subprocess
import runpy
import sys
import tempfile
import boto3
import argparse
import os
from urllib.parse import urlparse
from importlib import reload
from setuptools.command import easy_install
import site
import logging as log
from pathlib import Path

##TODO: add basic unittest  

raw_to_refined/
├── Makefile
├── config.py
├── raw_to_refined.py
└── requirements.txt

raw_to_refined/
├── Makefile
├── config.py
├── raw_to_refined.py
└── requirements.txt

raw_to_refined/
├── Makefile
├── config.py
├── raw_to_refined.py
└── requirements.txt

import logging.config
import sys

from glue_shared import parse_args
from glue_shared.defaults import default_logging_config

arguments = parse_args(sys.argv, ["APP_SETTINGS_ENVIRONMENT", "LOG_LEVEL"])

LOGGING_CONFIG = default_logging_config(arguments["LOG_LEVEL"])
logging.config.dictConfig(LOGGING_CONFIG)

JOB_CONFIG = dict(arguments)
# must be hard-coded because glue does not provide this in PyShell jobs
JOB_CONFIG["JOB_NAME"] = JOB_CONFIG.get("JOB_NAME") or "ds1-raw-to-refined"
JOB_CONFIG["JOB_ID"] = JOB_CONFIG.get("JOB_ID") or "ds1-raw-to-refined"
JOB_CONFIG["JOB_RUN_ID"] = JOB_CONFIG.get("JOB_RUN_ID") or "ds1-raw-to-refined"

JOB_CONFIG["WORKFLOW_NAME"] = JOB_CONFIG.get("WORKFLOW_NAME")
JOB_CONFIG["WORKFLOW_RUN_ID"] = JOB_CONFIG.get("WORKFLOW_RUN_ID")

raw_to_refined/
├── Makefile
├── config.py
├── raw_to_refined.py
└── requirements.txt

import logging
import pandas as pd
from glue_shared.pandas_helpers import write_parquet

LOGGER = logging.getLogger("job")


def main():
    LOGGER.info("JOB_NAME: %s", JOB_CONFIG["JOB_NAME"])
    LOGGER.info("JOB_ID: %s", JOB_CONFIG["JOB_ID"])
    LOGGER.info("JOB_RUN_ID %s", JOB_CONFIG["JOB_RUN_ID"])

    LOGGER.info("WORKFLOW_NAME: %s", JOB_CONFIG["WORKFLOW_NAME"])
    LOGGER.info("WORKFLOW_RUN_ID %s", JOB_CONFIG["WORKFLOW_RUN_ID"])

    LOGGER.info("Reading raw data from %s", JOB_CONFIG["dataset_url"])
    df = pd.read_csv(JOB_CONFIG["dataset_url"], sep=";")
    write_parquet(df, f"s3://{JOB_CONFIG['s3_bucket']}/{JOB_CONFIG['s3_prefix']}")


if __name__ == "__main__":
    from config import JOB_CONFIG

    main()

raw_to_refined/
├── Makefile
├── config.py
├── raw_to_refined.py
└── requirements.txt

import datetime
import logging

import pyspark
from glue_shared import get_spark_session_and_glue_job
from glue_shared.spark_helpers import read_parquet

LOGGER = logging.getLogger("job")


def run_etl(cfg, spark: pyspark.sql.SQLContext):
    df = read_parquet(spark, f"s3://{cfg['s3_bucket']}/{cfg['s3_prefix']}")
    LOGGER.debug("Count in: %s", df.count())

def main():
    spark, job = get_spark_session_and_glue_job(JOB_CONFIG)
    run_etl(JOB_CONFIG, spark)

if __name__ == "__main__":
    from config import JOB_CONFIG
    main()

glue_shared_lib
├── Makefile
├── requirements-dev.txt
├── setup.py
├── src
│   └── glue_shared
│       └── __init__.py
│       ├── argument_handlers.py
│       ├── boto3_helpers.py
│       ├── defaults.py
│       ├── glue_interface.py
│       ├── pandas_helpers.py
│       ├── spark_helpers.py
└── tests
    └── __init__.py

PySpark - .py, .zip, Python Shell - .egg, .whl

--WORKFLOW_NAME=my-workflow
--WORKFLOW_RUN_ID=wr_1e927676cc826831704712dc067b464800....

glue_client = boto3.client("glue")
args = getResolvedOptions(sys.argv, ['JOB_NAME','WORKFLOW_NAME', 'WORKFLOW_RUN_ID'])
workflow_name = args['WORKFLOW_NAME']
workflow_run_id = args['WORKFLOW_RUN_ID']
workflow_params = glue_client.get_workflow_run_properties(Name=workflow_name,
                                        RunId=workflow_run_id)["RunProperties"]

---
Resources:
  StartRawToRefinedTrigger:
    Type: AWS::Glue::Trigger
    Properties:
      Actions:
      - JobName: "job1"
      Description: Start raw to refined jobs
      Name: start-raw-to-refined
      Type: ON_DEMAND
      StartOnCreation: 'false'
      WorkflowName: etl-workflow
      Tags:
        terraform: 'true'
  StartRefinedToCuratedTrigger:
    Type: AWS::Glue::Trigger
    Properties:
      Actions:
      - JobName: "job2"
      Description: Start refined to curated jobs
      Name: start-refined-to-curated
      Type: CONDITIONAL
      StartOnCreation: 'true'
      Predicate:
        Conditions:
        - LogicalOperator: EQUALS
          JobName: "job1"
          State: SUCCEEDED
        Logical: AND
      WorkflowName: etl-workflow
      Tags:
        terraform: 'true'

resource "aws_glue_trigger" "start" {
  name = "start"
  type = "ON_DEMAND"
  workflow_name = module.glue.workflow_name
  actions {
    job_name = module.ds1_raw_to_refined_job.job_name
  }
  actions {
    job_name = module.third_job.job_name
  }
}

resource "aws_glue_trigger" "wait_for_first" {
  name = "wait_for_first_two"
  type = "CONDITIONAL"
  workflow_name = module.glue.workflow_name
  actions {
    job_name = module.ds1_refined_to_curated_job.job_name
  }
  predicate {
    conditions {
      job_name = module.ds1_raw_to_refined_job.job_name
      state    = "SUCCEEDED"
    }
    conditions {
      job_name = module.third_job.job_name
      state    = "SUCCEEDED"
    }
  }
}

resource "aws_glue_trigger" "wait_for_second" {
  name = "wait_for_second"
  type = "CONDITIONAL"
  workflow_name = module.glue.workflow_name
  actions {
    job_name = module.fifth_job.job_name
  }

  predicate {
    conditions {
      job_name = module.fourth_job.job_name
      state    = "SUCCEEDED"
    }
  }
}