From 3de9693808f23e0a94e8e2a60360d6bad3d64096 Mon Sep 17 00:00:00 2001 From: Baptiste Bouchereau Date: Mon, 10 Feb 2020 23:06:08 +0100 Subject: [PATCH] Add files for the localstack-part-3 tutorial --- .gitignore | 1 - localstack-part-1/docker-compose.yml | 2 +- localstack-part-2/docker-compose.yml | 2 +- localstack-part-2/localstack.tf | 10 +--- localstack-part-3/README.md | 27 ++++++++++ localstack-part-3/docker-compose.yml | 46 ++++++++++++++++++ .../docker-events-listener-build/Dockerfile | 30 ++++++++++++ .../aws_config.txt | 3 ++ .../aws_credentials.txt | 3 ++ .../listen-docker-events.sh | 26 ++++++++++ localstack-part-3/localstack/Dockerfile | 5 ++ localstack-part-3/localstack/localstack.patch | 13 +++++ localstack-part-3/terraform/lambda.zip | Bin 0 -> 741 bytes localstack-part-3/terraform/localstack.tf | 40 +++++++++++++++ 14 files changed, 196 insertions(+), 12 deletions(-) create mode 100644 localstack-part-3/README.md create mode 100644 localstack-part-3/docker-compose.yml create mode 100644 localstack-part-3/docker-events-listener-build/Dockerfile create mode 100644 localstack-part-3/docker-events-listener-build/aws_config.txt create mode 100644 localstack-part-3/docker-events-listener-build/aws_credentials.txt create mode 100644 localstack-part-3/docker-events-listener-build/listen-docker-events.sh create mode 100644 localstack-part-3/localstack/Dockerfile create mode 100644 localstack-part-3/localstack/localstack.patch create mode 100644 localstack-part-3/terraform/lambda.zip create mode 100644 localstack-part-3/terraform/localstack.tf diff --git a/.gitignore b/.gitignore index e6de9b1..cb94f88 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .terraform/ -*.zip output.txt terraform.tfstate terraform.tfstate.backup diff --git a/localstack-part-1/docker-compose.yml b/localstack-part-1/docker-compose.yml index 85df016..7493833 100644 --- a/localstack-part-1/docker-compose.yml +++ b/localstack-part-1/docker-compose.yml @@ -13,7 +13,7 @@ volumes: services: localstack: - image: localstack/localstack:0.9.0 + image: localstack/localstack:0.10.7 ports: - 8080:8080 # webui - 4569:4569 # dynamodb diff --git a/localstack-part-2/docker-compose.yml b/localstack-part-2/docker-compose.yml index 85df016..7493833 100644 --- a/localstack-part-2/docker-compose.yml +++ b/localstack-part-2/docker-compose.yml @@ -13,7 +13,7 @@ volumes: services: localstack: - image: localstack/localstack:0.9.0 + image: localstack/localstack:0.10.7 ports: - 8080:8080 # webui - 4569:4569 # dynamodb diff --git a/localstack-part-2/localstack.tf b/localstack-part-2/localstack.tf index 7c3c23e..e5704e9 100644 --- a/localstack-part-2/localstack.tf +++ b/localstack-part-2/localstack.tf @@ -31,12 +31,4 @@ resource "aws_lambda_function" "counter" { handler = "main.handler" runtime = "nodejs8.10" timeout = 30 - - lifecycle { - ignore_changes = [ - "environment", - "memory_size", - "role", - ] - } -} \ No newline at end of file +} diff --git a/localstack-part-3/README.md b/localstack-part-3/README.md new file mode 100644 index 0000000..1766669 --- /dev/null +++ b/localstack-part-3/README.md @@ -0,0 +1,27 @@ +Deploy AWS resources in localstack with Terraform +================================================= + +An example on how to use docker events to automatically deploy localstack resources that mock AWS services. The following instructions focus on how to deploy: +* a dynamodb table +* a lambda reading data and putting data to this table + +Usage +----- + +Run + +```bash +docker network create localstack-tutorial +docker-compose up -d +docker-compose logs -f localstack +``` + +Wait for the resources to be deployed, then invoke the lambda multiple times and scan the table to see new items and their counters being incremented: + +```bash +aws lambda invoke --function-name counter --endpoint-url=http://localhost:4574 --payload '{"id": "test"}' output.txt +aws dynamodb scan --endpoint-url http://localhost:4569 --table-name table_1 + +aws lambda invoke --function-name counter --endpoint-url=http://localhost:4574 --payload '{"id": "test2"}' output.txt +aws dynamodb scan --endpoint-url http://localhost:4569 --table-name table_1 +``` \ No newline at end of file diff --git a/localstack-part-3/docker-compose.yml b/localstack-part-3/docker-compose.yml new file mode 100644 index 0000000..823c6ae --- /dev/null +++ b/localstack-part-3/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.3' + +networks: + + default: + external: + name: localstack-tutorial + +volumes: + + localstack: + +services: + + localstack: + build: + context: localstack + container_name: localstack + ports: + - 8080:8080 # webui + - 4569:4569 # dynamodb + - 4574:4574 # lamba + environment: + - DATA_DIR=/tmp/localstack/data + - DEBUG=1 + - DEFAULT_REGION=ap-southeast-2 + - DOCKER_HOST=unix:///var/run/docker.sock + - LAMBDA_EXECUTOR=docker-reuse + - PORT_WEB_UI=8080 + - SERVICES=lambda,dynamodb + - LAMBDA_DOCKER_NETWORK=localstack-tutorial + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - localstack:/tmp/localstack/data + depends_on: + - docker-events-listener + + docker-events-listener: + build: + context: docker-events-listener-build # 3 + volumes: + - /var/run/docker.sock:/var/run/docker.sock # 4 + - ./terraform:/opt/terraform/ + environment: + APPLY_TERRAFORM_ON_START: "true" + INVOKE_LAMBDAS_ON_START: counter diff --git a/localstack-part-3/docker-events-listener-build/Dockerfile b/localstack-part-3/docker-events-listener-build/Dockerfile new file mode 100644 index 0000000..adbb919 --- /dev/null +++ b/localstack-part-3/docker-events-listener-build/Dockerfile @@ -0,0 +1,30 @@ +FROM docker:19.03.5 + +RUN apk update && \ + apk upgrade && \ + apk add --no-cache bash wget unzip + +# Install AWS CLI +RUN echo -e 'http://dl-cdn.alpinelinux.org/alpine/edge/main\nhttp://dl-cdn.alpinelinux.org/alpine/edge/community\nhttp://dl-cdn.alpinelinux.org/alpine/edge/testing' > /etc/apk/repositories && \ + wget "s3.amazonaws.com/aws-cli/awscli-bundle.zip" -O "awscli-bundle.zip" && \ + unzip awscli-bundle.zip && \ + apk add --update groff less python curl && \ + rm /var/cache/apk/* && \ + ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws && \ + rm awscli-bundle.zip && \ + rm -rf awscli-bundle +COPY aws_credentials.txt /root/.aws/credentials +COPY aws_config.txt /root/.aws/config + +# Install terraform +RUN wget https://releases.hashicorp.com/terraform/0.12.20/terraform_0.12.20_linux_amd64.zip \ + && unzip terraform_0.12.20_linux_amd64 \ + && mv terraform /usr/local/bin/terraform \ + && chmod +x /usr/local/bin/terraform + +RUN mkdir -p /opt/terraform +WORKDIR /opt/terraform + +COPY listen-docker-events.sh /var/listen-docker-events.sh + +CMD ["/bin/bash", "/var/listen-docker-events.sh"] diff --git a/localstack-part-3/docker-events-listener-build/aws_config.txt b/localstack-part-3/docker-events-listener-build/aws_config.txt new file mode 100644 index 0000000..9d43032 --- /dev/null +++ b/localstack-part-3/docker-events-listener-build/aws_config.txt @@ -0,0 +1,3 @@ +[default] +output = json +region = ap-southeast-2 \ No newline at end of file diff --git a/localstack-part-3/docker-events-listener-build/aws_credentials.txt b/localstack-part-3/docker-events-listener-build/aws_credentials.txt new file mode 100644 index 0000000..e947722 --- /dev/null +++ b/localstack-part-3/docker-events-listener-build/aws_credentials.txt @@ -0,0 +1,3 @@ +[default] +aws_secret_access_key = fake +aws_access_key_id = fake \ No newline at end of file diff --git a/localstack-part-3/docker-events-listener-build/listen-docker-events.sh b/localstack-part-3/docker-events-listener-build/listen-docker-events.sh new file mode 100644 index 0000000..3ea1e13 --- /dev/null +++ b/localstack-part-3/docker-events-listener-build/listen-docker-events.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +docker events --filter 'event=create' --filter 'event=start' --filter 'type=container' --format '{{.Actor.Attributes.name}} {{.Status}}' | while read event_info + +do + event_infos=($event_info) + container_name=${event_infos[0]} + event=${event_infos[1]} + + echo "$container_name: status = ${event}" + + if [[ $APPLY_TERRAFORM_ON_START == "true" ]] && [[ $container_name = "localstack" ]] && [[ $event == "start" ]]; then + terraform init + terraform apply --auto-approve + echo "The terraform configuration has been applied." + if [[ -n $INVOKE_LAMBDAS_ON_START ]]; then + echo "Invoking the lambda functions specified in the INVOKE_LAMBDAS_ON_START env variable" + while IFS=' ' read -ra lambdas; do + for lambda in "${lambdas[@]}"; do + echo "Invoking ${lambda}" + aws lambda invoke --function-name ${lambda} --endpoint-url=http://localstack:4574 output.txt & + done + done <<< "$INVOKE_LAMBDAS_ON_START" + fi + fi +done diff --git a/localstack-part-3/localstack/Dockerfile b/localstack-part-3/localstack/Dockerfile new file mode 100644 index 0000000..502da04 --- /dev/null +++ b/localstack-part-3/localstack/Dockerfile @@ -0,0 +1,5 @@ +FROM localstack/localstack:0.10.7 + +# Update localstack code to prevent the destruction of the lambda containers every 10 minutes +COPY localstack.patch /opt/code/localstack/localstack.patch +RUN apk add git && git apply localstack.patch && apk del --purge git diff --git a/localstack-part-3/localstack/localstack.patch b/localstack-part-3/localstack/localstack.patch new file mode 100644 index 0000000..12170fe --- /dev/null +++ b/localstack-part-3/localstack/localstack.patch @@ -0,0 +1,13 @@ +diff --git a/localstack/services/awslambda/lambda_executors.py b/localstack/services/awslambda/lambda_executors.py +index 19f1e07..85d4781 100644 +--- a/localstack/services/awslambda/lambda_executors.py ++++ b/localstack/services/awslambda/lambda_executors.py +@@ -301,7 +301,7 @@ class LambdaExecutorReuseContainers(LambdaExecutorContainers): + def startup(self): + self.cleanup() + # start a process to remove idle containers +- self.start_idle_container_destroyer_interval() ++ # self.start_idle_container_destroyer_interval() + + def cleanup(self, arn=None): + if arn: diff --git a/localstack-part-3/terraform/lambda.zip b/localstack-part-3/terraform/lambda.zip new file mode 100644 index 0000000000000000000000000000000000000000..ba7d7e585c2e2ff0e4df217dadcfd41c257c6ca9 GIT binary patch literal 741 zcmWIWW@Zs#U|`^2ur8AHS5!XQCrPK#n^oQ&0?EQQwN@`gNBy!JUb?_oots6f6|okBe|7r)&^^1 zqocV-C7gOwb}rc&k+Mxog6S?_hcPer!W2KDu-=|cJ6O^ROnfg1uttV>d6axqO`1A? z?$oZ$mlPgv|9Lum{e89Ox$n<;r}50pRf%4C^SJhf?1PO*gqcioEwy~x`|a{~h^T3c z?AKxSDA}QU=O^c;d-c*UOWusYS^-Rdyy#D-%`s}8Ox;$7+elUJtw$)nh{dJWfRPfNF7^m|(( zU)gb%V;{vDrr&!lIH&y7HJ`~{ncIxkt-5{nws~3bA@OZY<_5o>$4_?8y7i4Saq9Py zPKmY84V!8l)sFA35O#fM=YDv1U%B}4S3hRINt@~_x`SPOg?x{P1=O*Y^Km2ycW{cgbhr1g7O>UVV zc=x{MJL8-=H(V~(82j;@y?sffQTmy@>xUi&ze)9zeptP{-nrlUr2nURSN_MTKU`-~ z?|1C@FLU?xsWLN4Vh+_VHJO!jG9|F-+iyi9*6ij*1Gn*d0x*D;LXS+ v$BZitN&wR)0|O%vFKGm^;3<+7k|NPkV}Lg+FzJFcFf#Z8=_X)eW?%pSX8JdA literal 0 HcmV?d00001 diff --git a/localstack-part-3/terraform/localstack.tf b/localstack-part-3/terraform/localstack.tf new file mode 100644 index 0000000..ccdd81d --- /dev/null +++ b/localstack-part-3/terraform/localstack.tf @@ -0,0 +1,40 @@ +terraform { + required_providers { + aws = "~> 2.39.0" + } +} + +provider "aws" { + region = "ap-southeast-2" + access_key = "fake" + secret_key = "fake" + + skip_credentials_validation = true + skip_requesting_account_id = true + + endpoints { + dynamodb = "http://localstack:4569" + lambda = "http://localstack:4574" + } +} + +resource "aws_dynamodb_table" "table_1" { + name = "table_1" + read_capacity = "20" + write_capacity = "20" + hash_key = "id" + + attribute { + name = "id" + type = "S" + } +} + +resource "aws_lambda_function" "counter" { + function_name = "counter" + filename = "lambda.zip" + role = "fake_role" + handler = "main.handler" + runtime = "nodejs8.10" + timeout = 30 +}