This exercise applies all four Module 1 concepts by building a production-patterned Terraform configuration for the IPL data engineering platform in Python. The exercise uses the `moto` library — an AWS mock that intercepts boto3 API calls and simulates the AWS API locally without incurring any charges. You will validate every resource definition, simulate provisioning with moto, and verify the created resources have correct configuration: VPC with four subnets, S3 data lake with versioning and encryption enabled, and an IAM role for Glue with a correctly scoped trust policy.
The exercise is structured in three steps. Step 1 validates the Terraform HCL configuration without provisioning — CIDR overlap detection, required tag presence, lifecycle block on critical resources, and Spot configuration on worker node fleets. Step 2 uses moto to provision the VPC, subnets, S3 bucket, and IAM role and asserts the resource attributes match the configuration. Step 3 simulates cost estimation by computing the monthly S3 storage cost under the lifecycle tiering rules and verifying that the Spot cluster configuration produces the correct cost reduction versus On-Demand.
Analogy🏏Cricket
🏏 Think of it like cricket: This exercise is the IPL official statistics team building their daily automated processing pipeline — the complete workflow that takes raw ball-by-ball records from every ground and produces the certified statistics published on the official website by midnight. Stage 1 is the data catalogue check: verify that the incoming scorecards match the expected format before any processing begins. Stage 2 is the statistics calculation: joins with match metadata, derivation of over-level stats, economy rate computation. Stage 3 is the official record update: load the new statistics into the production database using the certified upsert protocol — delete the old version of today's record and insert the freshly computed one — so no match ever has two records in the official database.
Step 1 — Terraform Configuration Validation
Define the complete Terraform configuration as Python dicts (representing the parsed HCL structure), validate CIDR non-overlap for all subnet definitions, verify that all resource definitions include the required tags (Project, Environment, ManagedBy), check that the S3 bucket resource includes a `prevent_destroy` lifecycle block, and confirm the EMR worker fleet uses Spot capacity with an On-Demand fallback. Assert all validations pass before proceeding to the moto provisioning step.
Analogy🏏Cricket
🏏 Think of it like cricket: validating the Terraform configuration before any moto call is exactly like the ground curator and match referee inspecting the stadium blueprint the morning before a match, long before a single player walks out. Just as the referee checks that no two boundary ropes overlap so fielders never end up in an impossible position, `validate_no_cidr_overlap` uses `IPv4Network.overlaps()` to prove no two subnet CIDRs collide. Just as every piece of equipment must carry its official asset label before it enters the ground, the required-tag check confirms Project, Environment and ManagedBy are stamped on every resource. Just as the referee confirms the immovable sight-screen is bolted down, the `prevent_destroy` lifecycle check protects the S3 bucket, and confirming the EMR worker fleet uses Spot with an On-Demand fallback is like naming both a first-choice bowler and a backup in case of injury. The payoff: every structural rule is proven true on paper, so provisioning proceeds only after the blueprint is certified match-ready.
python
# exercise_terraform_vpc.py — Step 1: Configuration validation
# pip install moto[all] boto3
import ipaddress
import json
from typing import Any
# ── Infrastructure configuration (Terraform HCL represented as Python dicts) ──
INFRA_CONFIG = {
"vpc": {
"cidr": "10.0.0.0/16",
"region": "ap-south-1",
"tags": {"Project": "IPL-DataPlatform", "Environment": "dev",
"ManagedBy": "Terraform"},
},
"public_subnets": [
{"cidr": "10.0.1.0/24", "az": "ap-south-1a", "name": "pub-1a",
"tags": {"Project": "IPL-DataPlatform", "Environment": "dev", "ManagedBy": "Terraform"}},
{"cidr": "10.0.2.0/24", "az": "ap-south-1b", "name": "pub-1b",
"tags": {"Project": "IPL-DataPlatform", "Environment": "dev", "ManagedBy": "Terraform"}},
],
"private_subnets": [
{"cidr": "10.0.10.0/24", "az": "ap-south-1a", "name": "priv-1a",
"tags": {"Project": "IPL-DataPlatform", "Environment": "dev", "ManagedBy": "Terraform"}},
{"cidr": "10.0.11.0/24", "az": "ap-south-1b", "name": "priv-1b",
"tags": {"Project": "IPL-DataPlatform", "Environment": "dev", "ManagedBy": "Terraform"}},
],
"s3_data_lake": {
"bucket": "ipl-data-lake-dev",
"versioning": True,
"encryption": "AES256",
"block_public": True,
"prevent_destroy": True, # lifecycle block
"lifecycle_rules": [
{"prefix": "raw/", "ia_days": 30, "glacier_days": 90},
],
"tags": {"Project": "IPL-DataPlatform", "Environment": "dev", "ManagedBy": "Terraform"},
},
"glue_execution_role": {
"name": "ipl-glue-execution-dev",
"trust_service": "glue.amazonaws.com",
"s3_permissions": ["s3:GetObject", "s3:PutObject", "s3:ListBucket"],
"prevent_destroy": False,
"tags": {"Project": "IPL-DataPlatform", "Environment": "dev", "ManagedBy": "Terraform"},
},
"security_groups": [
{
"name": "redshift-sg",
"rules": [
{"type": "ingress", "port": 5439, "source": "10.0.0.0/16",
"protocol": "tcp"}, # allow from within VPC only
{"type": "egress", "port": -1, "source": "0.0.0.0/0",
"protocol": "-1"}, # allow all outbound
],
},
],
}
# ── Validation functions ───────────────────────────────────────────────────────
REQUIRED_TAGS = {"Project", "Environment", "ManagedBy"}
def validate_no_cidr_overlap(config: dict) -> list[str]:
"""Assert no two subnet CIDRs overlap."""
subnets = (
[s["cidr"] for s in config["public_subnets"]] +
[s["cidr"] for s in config["private_subnets"]]
)
errors = []
networks = [ipaddress.IPv4Network(c) for c in subnets]
for i, n1 in enumerate(networks):
for n2 in networks[i+1:]:
if n1.overlaps(n2):
errors.append(f"CIDR overlap: {n1} and {n2}")
return errors
def validate_required_tags(config: dict) -> list[str]:
"""All resources must have the required tag keys."""
errors = []
resources = (
[config["vpc"]] +
config["public_subnets"] + config["private_subnets"] +
[config["s3_data_lake"], config["glue_execution_role"]]
)
for res in resources:
missing = REQUIRED_TAGS - set(res.get("tags", {}).keys())
if missing:
name = res.get("name", res.get("bucket", res.get("cidr", "?")))
errors.append(f"Resource '{name}' missing tags: {missing}")
return errors
def validate_s3_security(config: dict) -> list[str]:
"""S3 bucket must have encryption, versioning, public access block, and prevent_destroy."""
errors = []
s3 = config["s3_data_lake"]
if not s3.get("versioning"): errors.append("S3: versioning not enabled")
if not s3.get("encryption"): errors.append("S3: encryption not configured")
if not s3.get("block_public"): errors.append("S3: public access block not set")
if not s3.get("prevent_destroy"): errors.append("S3: prevent_destroy lifecycle missing")
return errors
def validate_sg_no_public_db(config: dict) -> list[str]:
"""No security group should allow database ports from 0.0.0.0/0."""
DB_PORTS = {5439, 5432, 3306, 1521, 27017}
errors = []
for sg in config.get("security_groups", []):
for rule in sg["rules"]:
if (rule["type"] == "ingress" and
rule.get("port") in DB_PORTS and
rule.get("source") == "0.0.0.0/0"):
errors.append(
f"SG '{sg['name']}': port {rule['port']} open to 0.0.0.0/0"
)
return errors
# ── Run all validations ───────────────────────────────────────────────────────
all_errors = (
validate_no_cidr_overlap(INFRA_CONFIG) +
validate_required_tags(INFRA_CONFIG) +
validate_s3_security(INFRA_CONFIG) +
validate_sg_no_public_db(INFRA_CONFIG)
)
if all_errors:
for e in all_errors:
print(f" ✗ {e}")
raise AssertionError(f"{len(all_errors)} validation error(s)")
else:
print(f" ✓ CIDR overlap validation: no overlaps")
print(f" ✓ Required tags: all {len(REQUIRED_TAGS)} tags present on all resources")
print(f" ✓ S3 security: versioning, encryption, public-block, prevent_destroy")
print(f" ✓ Security groups: no database ports open to 0.0.0.0/0")
print("Step 1 ✓: all configuration validations passed")Step 2 — Moto Provisioning and Cost Estimation
Use moto to simulate provisioning the VPC, subnets, S3 bucket with lifecycle rule, and IAM role. Assert that the provisioned resources have the correct attributes: VPC CIDR matches configuration, subnets are in the correct AZs, S3 bucket has versioning status `Enabled` and public access block set to True on all four settings, and the IAM role's trust policy references `glue.amazonaws.com`. Then compute the monthly storage cost under the lifecycle tiering model for a 10TB dataset and verify that Spot cluster costs are 70% below On-Demand.
Analogy🏏Cricket
🏏 Think of it like cricket: the moto provisioning step is the dress-rehearsal match played in an empty stadium the day before the real fixture — everything is built and checked exactly as it will be, but no tickets are sold and no money changes hands. Just as the groundstaff lay out the full ground and then walk it to confirm the pitch is where the plan says, the boundary is the right size and both dugouts are in the correct zones, the code creates the VPC, four subnets and S3 bucket via moto and then asserts the CIDR matches, the subnets sit in the right availability zones, versioning reads `Enabled` and all four public-access blocks are True. Just as the accreditation desk verifies each staff pass names the right role, the IAM assertion confirms the trust policy references `glue.amazonaws.com`. Just as the franchise CFO then projects the season's cost from last year's published rate card, the cost step prices a 10TB dataset across storage tiers and confirms Spot runs ~70% below On-Demand. The payoff: the whole ground is proven correct and affordable before a single billable resource exists.
python
# exercise_terraform_vpc.py — Step 2: Moto provisioning and cost estimation
import boto3
import json
from moto import mock_aws
@mock_aws
def provision_and_verify():
ec2 = boto3.client("ec2", region_name="ap-south-1")
s3 = boto3.client("s3", region_name="ap-south-1")
iam = boto3.client("iam", region_name="ap-south-1")
# ── Create VPC ────────────────────────────────────────────────────────────
vpc_resp = ec2.create_vpc(CidrBlock="10.0.0.0/16")
vpc_id = vpc_resp["Vpc"]["VpcId"]
ec2.create_tags(Resources=[vpc_id], Tags=[
{"Key": k, "Value": v}
for k, v in INFRA_CONFIG["vpc"]["tags"].items()
])
# ── Create subnets ────────────────────────────────────────────────────────
subnet_ids = []
for sn in INFRA_CONFIG["public_subnets"] + INFRA_CONFIG["private_subnets"]:
resp = ec2.create_subnet(
VpcId = vpc_id,
CidrBlock = sn["cidr"],
AvailabilityZone = sn["az"],
)
subnet_ids.append(resp["Subnet"]["SubnetId"])
# ── Create S3 data lake bucket ────────────────────────────────────────────
bucket = INFRA_CONFIG["s3_data_lake"]["bucket"]
s3.create_bucket(
Bucket = bucket,
CreateBucketConfiguration = {"LocationConstraint": "ap-south-1"},
)
s3.put_bucket_versioning(
Bucket = bucket,
VersioningConfiguration = {"Status": "Enabled"},
)
s3.put_public_access_block(
Bucket = bucket,
PublicAccessBlockConfiguration = {
"BlockPublicAcls": True,
"IgnorePublicAcls": True,
"BlockPublicPolicy": True,
"RestrictPublicBuckets": True,
},
)
s3.put_bucket_lifecycle_configuration(
Bucket = bucket,
LifecycleConfiguration = S3_LIFECYCLE_CONFIG,
)
# ── Create IAM role for Glue ─────────────────────────────────────────────
trust_policy = {
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {"Service": "glue.amazonaws.com"},
"Action": "sts:AssumeRole",
}],
}
iam.create_role(
RoleName = INFRA_CONFIG["glue_execution_role"]["name"],
AssumeRolePolicyDocument = json.dumps(trust_policy),
)
# ── Assertions ────────────────────────────────────────────────────────────
# VPC CIDR
vpc_info = ec2.describe_vpcs(VpcIds=[vpc_id])["Vpcs"][0]
assert vpc_info["CidrBlock"] == "10.0.0.0/16"
# Subnet count and AZs
subnets = ec2.describe_subnets(
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
)["Subnets"]
assert len(subnets) == 4, f"Expected 4 subnets, got {len(subnets)}"
azs = {s["AvailabilityZone"] for s in subnets}
assert "ap-south-1a" in azs and "ap-south-1b" in azs
# S3 versioning enabled
ver = s3.get_bucket_versioning(Bucket=bucket)
assert ver.get("Status") == "Enabled", f"Versioning: {ver}"
# S3 public access block
pab = s3.get_public_access_block(Bucket=bucket)["PublicAccessBlockConfiguration"]
assert all(pab[k] for k in
["BlockPublicAcls","IgnorePublicAcls",
"BlockPublicPolicy","RestrictPublicBuckets"])
# IAM role trust policy
role = iam.get_role(RoleName=INFRA_CONFIG["glue_execution_role"]["name"])["Role"]
trust = json.loads(role["AssumeRolePolicyDocument"])
principal = trust["Statement"][0]["Principal"]["Service"]
assert principal == "glue.amazonaws.com", f"Trust: {principal}"
print(f" VPC {vpc_id}: CIDR 10.0.0.0/16 ✓")
print(f" Subnets: {len(subnets)} across {sorted(azs)} ✓")
print(f" S3 {bucket}: versioning=Enabled, public_access_blocked ✓")
print(f" IAM role: trust=glue.amazonaws.com ✓")
return {"vpc_id": vpc_id, "subnets": len(subnets), "bucket": bucket}
result = provision_and_verify()
print(f" Provisioned resources: {result}")
# ── Cost estimation ────────────────────────────────────────────────────────────
TB = 1024 # GB
# 10TB dataset: 2TB written in last 30 days, 3TB between 30-90 days, 5TB > 90 days
data_distribution = {
"Standard (0-30 days, 2TB)": 2 * TB * 0.023,
"Standard-IA (30-90 days, 3TB)": 3 * TB * 0.0125,
"Glacier (>90 days, 5TB)": 5 * TB * 0.004,
}
total_cost = sum(data_distribution.values())
no_tiering_cost = 10 * TB * 0.023
print("\nMonthly S3 cost for 10TB dataset with lifecycle tiering:")
for tier, cost in data_distribution.items():
print(f" {tier:<45}: ${cost:.2f}")
print(f" {'Total with tiering':<45}: ${total_cost:.2f}")
print(f" {'Without tiering (all Standard)':<45}: ${no_tiering_cost:.2f}")
print(f" {'Saving':<45}: ${no_tiering_cost - total_cost:.2f} ({(1-total_cost/no_tiering_cost)*100:.0f}%)")
# Spot vs On-Demand cost verification
assert (cost_on_demand - cost_spot) / cost_on_demand >= 0.65, \
"Expected >= 65% Spot discount"
print(f"\n Spot discount verified: {(1-cost_spot/cost_on_demand)*100:.0f}% ✓")
print("Step 2 ✓: provisioning and cost assertions complete")