Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions sfn-glue-terraform/README.md
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
# AWS Step Functions to start a AWS Glue Job Through a Cloudwatch event rule
# AWS Step Functions to start a AWS Glue Job Through a CloudWatch event rule
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🙅‍♂️ Cloudwatch
🙆‍♂️ CloudWatch


The Terraform template deploys a AWS Step Function, a AWS Glue Job, a Cloudwatch Event Rule, a Amazon S3 bucket and the minimum IAM resources required to run the application.
The Terraform template deploys a AWS Step Functions, a AWS Glue Job, a CloudWatch Event Rule, a Amazon S3 bucket and the minimum IAM resources required to run the application.
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🙅‍♂️ Step Function
🙆‍♂️ Step Functions


## Architecture
![Alt](./resources/architecture.png)

This pattern demonstrates the use of Terraform modules and deploys the below resources:
* Amazon S3 bucket and load the sample Python script as an object
* Sample AWS Glue Job which executes the script in the S3 bucket
* AWS Step Function to invoke the AWS Glue Job synchronously. The Function will wait until the Job is completed
* Cloudwatch Event Rule which is configured to start the AWS Step Function evey 10 minutes
* AWS Step Functions to invoke the AWS Glue Job synchronously. The Function will wait until the Job is completed
* CloudWatch Event Rule which is configured to start the AWS Step Functions evey 10 minutes


## How it works

The AWS Cloudwatch rule is configured to start a Step Function execution every 10 minutes. The Step function then invokes a AWS Glue Job with some default arguments and a test message.
The AWS CloudWatch rule is configured to start a Step Functions execution every 10 minutes. The Step function then invokes a AWS Glue Job with some default arguments and a test message.
The Arguments to the AWS Glue Job, the Python script and the CloudWatch event rule can be modified as per requirement.


Expand Down Expand Up @@ -71,7 +71,7 @@ After deployment, go to the cloudwatch logs to check the event details.
```bash
terraform show
```
```

----
Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.

Expand Down
2 changes: 1 addition & 1 deletion sfn-glue-terraform/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 3.27"
version = "~> 6.0"
}
}

Expand Down
12 changes: 6 additions & 6 deletions sfn-glue-terraform/modules/terraform-amazon-s3/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@ resource "aws_s3_bucket" "s3_sample_glue_bucket" {
resource "aws_s3_bucket_public_access_block" "s3_glue_bucket_block_public_access" {
bucket = aws_s3_bucket.s3_sample_glue_bucket.id

block_public_acls = true
block_public_policy = true
ignore_public_acls = true
block_public_acls = true
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note: I applied terraform fmt command for all HCL code😀

block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}

# upload the AWS Glue script to the bucket
resource "aws_s3_bucket_object" "glue_script_object" {
resource "aws_s3_object" "glue_script_object" {
bucket = aws_s3_bucket.s3_sample_glue_bucket.bucket
key = "glue_script.py"
source = "./resources/glue_script.py"
acl = "private"
acl = "private"
}

## IAM Resources
Expand All @@ -39,5 +39,5 @@ output "bucket_arn" {
}

output "glue_script_name" {
value = aws_s3_bucket_object.glue_script_object.key
value = aws_s3_object.glue_script_object.key
}
16 changes: 8 additions & 8 deletions sfn-glue-terraform/modules/terraform-aws-cloudwatch/main.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
locals{
schedule_expression= "rate(10 minutes)"
locals {
schedule_expression = "rate(10 minutes)"
}

# Variables
Expand Down Expand Up @@ -30,8 +30,8 @@ resource "aws_iam_role" "allow_cloudwatch_to_execute_role" {
}

resource "aws_iam_role_policy" "state_execution" {
name = "state_execution_policy"
role = aws_iam_role.allow_cloudwatch_to_execute_role.id
name = "state_execution_policy"
role = aws_iam_role.allow_cloudwatch_to_execute_role.id

policy = <<EOF
{
Expand All @@ -51,14 +51,14 @@ EOF
}

resource "aws_cloudwatch_event_rule" "stf_trigger_rule" {
name = "stf_trigger_rule"
name = "stf_trigger_rule"
schedule_expression = local.schedule_expression
description = "Sample Event for Glue terraform example"
description = "Sample Event for Glue terraform example"
}

resource "aws_cloudwatch_event_target" "cloudwatch_event_target" {
rule = aws_cloudwatch_event_rule.stf_trigger_rule.name
arn = var.stf_function_arn
rule = aws_cloudwatch_event_rule.stf_trigger_rule.name
arn = var.stf_function_arn
role_arn = aws_iam_role.allow_cloudwatch_to_execute_role.arn
}

Expand Down
14 changes: 7 additions & 7 deletions sfn-glue-terraform/modules/terraform-aws-glue/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@ data "aws_iam_policy_document" "policy_document" {
}

resource "aws_iam_policy" "s3_access_iam_policy" {
name = "sample-glue-s3-access-policy"
name = "sample-glue-s3-access-policy"
policy = data.aws_iam_policy_document.policy_document.json
}

# Glue IAM roles and Policies
resource "aws_iam_role" "sample_glue_role" {
name = "sample-glue-role"
name = "sample-glue-role"
assume_role_policy = <<EOF
{
"Version":"2012-10-17",
Expand All @@ -58,7 +58,7 @@ EOF
}

resource "aws_iam_role_policy_attachment" "glue_service_policy" {
role = aws_iam_role.sample_glue_role.name
role = aws_iam_role.sample_glue_role.name
policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
}

Expand All @@ -70,8 +70,8 @@ resource "aws_iam_role_policy_attachment" "platform_metrics_glue_iam_policy" {
resource "aws_glue_job" "glue_job" {
count = var.create ? 1 : 0

name = "sample-glue-job-terraform"
description = "AWS Glue Job terraform example"
name = "sample-glue-job-terraform"
description = "AWS Glue Job terraform example"
role_arn = aws_iam_role.sample_glue_role.arn
max_capacity = var.dpu
glue_version = "3.0"
Expand All @@ -81,8 +81,8 @@ resource "aws_glue_job" "glue_job" {
}

default_arguments = merge(local.default_arguments, var.arguments)
max_retries = var.max_retries
timeout = var.timeout
max_retries = var.max_retries
timeout = var.timeout

execution_property {
max_concurrent_runs = var.max_concurrent
Expand Down
2 changes: 1 addition & 1 deletion sfn-glue-terraform/modules/terraform-aws-glue/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ variable "create" {


variable "role_arn" {
default =""
default = ""
}

variable "connections" {
Expand Down
12 changes: 6 additions & 6 deletions sfn-glue-terraform/modules/terraform-aws-step-function/main.tf
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
# variables
variable "glue_job_arn" {}
variable "glue_job_name" {}
variable "glue_message" {default = "This is a message passed by the AWS Step Function"}
variable "glue_message" { default = "This is a message passed by the AWS Step Function" }

# AWS Step Functions IAM roles and Policies
resource "aws_iam_role" "aws_stf_role" {
name = "aws-stf-role"
name = "aws-stf-role"
assume_role_policy = <<EOF
{
"Version":"2012-10-17",
Expand All @@ -27,9 +27,9 @@ EOF

resource "aws_iam_role_policy" "step_function_policy" {
name = "aws-stf-policy"
role = aws_iam_role.aws_stf_role.id
role = aws_iam_role.aws_stf_role.id

policy = <<EOF
policy = <<EOF
{
"Version":"2012-10-17",
"Statement":[
Expand All @@ -51,8 +51,8 @@ EOF

# AWS Step function definition
resource "aws_sfn_state_machine" "aws_step_function_workflow" {
name = "aws-step-function-workflow"
role_arn = aws_iam_role.aws_stf_role.arn
name = "aws-step-function-workflow"
role_arn = aws_iam_role.aws_stf_role.arn
definition = <<EOF
{
"Comment":"A description of the sample glue job state machine using Terraform",
Expand Down