James Alexander
jamesralexander.com
QR Link to Slides:
By JeremyA at English Wikipedia - Transferred from en.wikipedia to Commons by Gavin.perch., CC BY 3.0, https://commons.wikimedia.org/w/index.php?curid=13488176
Leaf is a Consultant
Leaf's Client is a Software Vendor: e.g. "CapTech"
"CapTech" has clients that use the Software
S3
cfntemplate-to-ruby [EXISTING_CFN] > [NEW_NAME.rb]
template.rb create --stack-name my_stack \
--parameters "BucketName=bucket-s3-static;SnsQueue=mysnsqueue"
AWS configuration provided by Cloud Formation Templates:
[assembly : LambdaSerializer (typeof (Amazon.Lambda.Serialization.Json.JsonSerializer))]
public class LambdaParameters {
public string Name { get; set; }
public string Comment { get; set; }
public bool ShouldRun { get; set; }
public int QueueDepth { get; set; }
public int ActiveRunningLoaders { get; set; }
public int LoadersToRun { get; set; }
public int MaxLoaders { get; set; }
public string Bucket { get; set; }
public string Key { get; set; }
public override string ToString()
{
return $"Parameters: Comment: {Comment} - QueueDepth: {QueueDepth}";
}
}
public class LoaderLambda {
public int Trigger (SNSEvent evnt, ILambdaContext context) { ... }
public LambdaParameters ShouldContinueRunning(LambdaParameters p, ILambdaContext context) { ... }
public LambdaParameters GetQueueDepth(LambdaParameters p, ILambdaContext context) { ... }
public LambdaParameters GetActiveRunningLoaders(LambdaParameters p, ILambdaContext context) { ... }
public LambdaParameters GetMaxLoaders(LambdaParameters p, ILambdaContext context) { ... }
public LambdaParameters StartLoaders(LambdaParameters p, ILambdaContext context) { ... }
}
public LambdaParameters StartLoaders(LambdaParameters p, ILambdaContext context) {
var runningLoaders = GetActiveRunningLoaders(p, context).ActiveRunningLoaders;
var tasksToRun = Math.Min(p.MaxLoaders - runningLoaders, p.QueueDepth);
for (int i = 0; i < tasksToRun; i++)
{
var runParameters = ReceiveMessageOnQueue(context);
StartLoaderTask(runParameters, context);
}
return p;
}
public void StartLoaderTask(LambdaParameters p, ILambdaContext context) {
using (var client = new AmazonECSClient ()) {
var taskName = GetSsmParameter("LOADER_TASK_NAME");
var clusterName = GetSsmParameter("ECS_CLUSTER");
var taskRequest = new RunTaskRequest {
TaskDefinition = taskName,
Count = 1,
LaunchType = new LaunchType ("FARGATE")
};
// ... Populate Network Config
taskRequest.Overrides = new TaskOverride {
ContainerOverrides = new List<ContainerOverride> {
new ContainerOverride {
Name = "file-loader",
Command = new List<string> (
new string[] {
"/app/startloader.sh",
p.Bucket,
p.Key
}
)
}
}
};
context.Logger.LogLine($"Triggering Task: {taskName} on cluster: {clusterName}");
var response = client.RunTaskAsync (taskRequest, new CancellationToken ()).Result;
// ... Logging for Errors / Failures
}
}
# /app/startloader.sh
#!/bin/sh
env
cd CommandLine
/app/chamber exec $ENVIRONMENT -- dotnet MyNamespace.Interface.CommandLine.dll -j \
import -b $1 -k $2
dotnet lambda deploy-function $ENVIRONMENT-ldr-trigger -fh \
AwsLambda::AwsLambda.LoaderLambda::Trigger
dotnet lambda deploy-function $ENVIRONMENT-ldr-should-continue-running -fh \
AwsLambda::AwsLambda.LoaderLambda::ShouldContinueRunning
dotnet lambda deploy-function $ENVIRONMENT-ldr-get-queue-depth -fh \
AwsLambda::AwsLambda.LoaderLambda::GetQueueDepth
dotnet lambda deploy-function $ENVIRONMENT-ldr-get-max-loaders -fh \
AwsLambda::AwsLambda.LoaderLambda::GetMaxLoaders
dotnet lambda deploy-function $ENVIRONMENT-ldr-start-loaders -fh \
AwsLambda::AwsLambda.LoaderLambda::StartLoaders
awsume client1-source-profile
Exports "client1-source-profile" credentials into current shell, will ask for MFA if needed
aws s3 ls
~/.aws/config
[default]
region = us-east-1
[profile internal-admin]
role_arn = arn:aws:iam::<your aws account id>:role/admin-role
source_profile = joel
region = us-east-1
[profile client1-source-profile]
role_arn = arn:aws:iam::<client #1 account id>:role/admin-role
mfa_serial = arn:aws:iam::<your aws account id>:mfa/joel
source_profile = joel
region = us-west-2
[profile client2-admin]
role_arn = arn:aws:iam::<client #2 account id>:role/admin-role
mfa_serial = arn:aws:iam::<your aws account id>:mfa/joel
source_profile = joel
region = us-east-1
chamber write <service> <key> <value|->
chamber write dev myVariable myValue
chamber list dev
chamber export dev --format json --output-file myValues.json
chamber import dev myValues.json
# For those that read comments, here's a Pun:
# Boss: How good are you at Power Point?
# Me: I Excel at it
# Boss: Was that a Microsoft Office pun?
# Me: Word
# build.sh - more from our build later
case "$BITBUCKET_BRANCH" in
develop)
echo Build a Dev build for devolop branch;
RELEASE_TAG="development";
ENVIRONMENT="dev";;
test/*)
echo Build a QA build for the test branch
export RELEASE_TAG=${BITBUCKET_BRANCH#test/};
ENVIRONMENT="test";;
master)
echo Build a prod build for master branch;
RELEASE_TAG="production"
ENVIRONMENT="prd"
DEPLOY_DR="true"
;;
*)
echo Invalid Branch, cannot proceed with Build and deployment.
exit 1;;
esac
/chamber exec $ENVIRONMENT -- AWS/build-app.sh;
/chamber exec $ENVIRONMENT -- AWS/build-container.sh $RELEASE_TAG;
/chamber exec $ENVIRONMENT -- AWS/deploy-app.sh $RELEASE_TAG $ENVIRONMENT
#!/bin/bash
# web-task-definition.sh
cat <<JSON
{
"family": "capnet-web",
"networkMode": "awsvpc",
"taskRoleArn": "${WEB_TASK_ROLE_ARN}",
"containerDefinitions": [
{
"name": "capnet-web",
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "${WEB_TASK_LOGS_GROUP}",
"awslogs-region": "${WEB_TASK_LOGS_REGION}",
"awslogs-stream-prefix": "${WEB_TASK_LOGS_PREFIX}"
}
},
"image": "${IMAGE}",
"portMappings": [
{
"containerPort": 80,
"hostPort": 80,
"protocol": "tcp"
}
],
"essential": true,
......
#!/bin/bash (continued)
# web-task-definition.sh
.....
"command": [
"/app/startweb.sh"
],
"environment": [
{
"name": "ENVIRONMENT",
"value": "${ENVIRONMENT}"
}
]
}
],
"requiresCompatibilities": [
"FARGATE"
],
"cpu": "${WEB_TASK_CPU}",
"memory": "${WEB_TASK_MEMORY}",
"executionRoleArn": "arn:aws:iam::123456789912:role/ecsTaskExecutionRole"
}
JSON
#!/bin/bash
# web-service-definition.sh
cat <<JSON
{
"cluster": "${ECS_CLUSTER}",
"serviceName": "capnet-webserver",
"taskDefinition": "capnet-web:135",
"loadBalancers": [
{
"targetGroupArn": "${WEB_TARGETGROUP_ARN}",
"containerName": "capnet-web",
"containerPort": 80
}
],
"desiredCount": 0,
"deploymentConfiguration": {
"maximumPercent": 200,
"minimumHealthyPercent": 50
},
"launchType": "FARGATE",
"networkConfiguration": {
"awsvpcConfiguration": {
"subnets": [
"subnet-1qaz2wsx",
"subnet-0okmnji9"
],
"securityGroups": [
"${WEB_SECURITY_GROUP_ID}"
]
}
}
}
JSON
pipelines:
branches:
develop:
- step:
name: Build and Deploy to Dev
deployment: staging
services:
- docker
caches:
- dotnetcore
- bowerlib
script:
- AWS/build.sh
test/*:
- step:
name: Build and Deploy to QA
deployment: test
... (same as develop for test and prod)
# build.sh
case "$BITBUCKET_BRANCH" in
develop)
echo Build a Dev build for devolop branch;
RELEASE_TAG="development";
ENVIRONMENT="dev";;
test/*)
echo Build a QA build for the test branch
export RELEASE_TAG=${BITBUCKET_BRANCH#test/};
ENVIRONMENT="test";;
master)
echo Build a prod build for master branch;
RELEASE_TAG="production"
ENVIRONMENT="prd"
DEPLOY_DR="true"
;;
*)
echo Invalid Branch, cannot proceed with Build and deployment.
exit 1;;
esac
/chamber exec $ENVIRONMENT -- AWS/build-app.sh;
/chamber exec $ENVIRONMENT -- AWS/build-container.sh $RELEASE_TAG;
/chamber exec $ENVIRONMENT -- AWS/deploy-app.sh $RELEASE_TAG $ENVIRONMENT
# build-app.sh
...
echo "register new web task definition"
AWS/web-task-definition.sh > AWS/web-task-definition.json
cat AWS/web-task-definition.json
task_definition_arn=$(
aws ecs register-task-definition \
--cli-input-json file://AWS/web-task-definition.json \
| jq -r '.taskDefinition.taskDefinitionArn'
)
# Update the existing service to use the new task definition
aws ecs update-service \
--cluster $ECS_CLUSTER \
--service capnet-webserver \
--task-definition $task_definition_arn \
> /dev/null
...
# build-container.sh
...
mkdir -p docker
mkdir -p docker/Web
cp -a Capnet.Interface.WebServer/* docker/Web
cp -f AWS/startweb.sh docker
chmod 755 docker/startweb.sh
...
cp -f AWS/Dockerfile docker/Dockerfile
docker build -t ${AWS_APP_REPOSITORY}:$BUILD_ID ./docker
docker push ${AWS_APP_REPOSITORY}:$BUILD_ID
docker tag ${AWS_APP_REPOSITORY}:$BUILD_ID ${AWS_APP_REPOSITORY}:${IMAGE_TAG}
docker push ${AWS_APP_REPOSITORY}:${IMAGE_TAG}
# JA: We're almost done, promise, have another pun:
# I remember the first time I saw a universal remote control.
# I thought to myself: "Well, this changes everything."
1vCPU @ $0.04048 / hr * 24 * 30 = $29.1456
4GB @ 0.004445 / hr * 24 * 30 = $12.8016
Find me at: jamesralexander.com