TerraformでマルチリージョンGPU推論基盤を一括コード化！起動テンプレートとエフェメラルIaCで運用する

# terraform/ecr.tf
# 東京（プライマリ）リージョンの ECR リポジトリ
resource "aws_ecr_repository" "vllm" {
  name                 = var.ecr_repository_name
  image_tag_mutability = "MUTABLE"

  tags = { Project = "vllm-ephemeral" }
}

# 古いタグの自動削除ポリシー（無駄なストレージ課金を防止）
resource "aws_ecr_lifecycle_policy" "vllm_lifecycle" {
  repository = aws_ecr_repository.vllm.name
  policy = jsonencode({
    rules = [{
      rulePriority = 1
      description  = "最新イメージのみ保持し、旧世代は30日で失効"
      selection    = { tagStatus = "any", countType = "sinceImagePushed", countUnit = "days", countNumber = 30 }
      action       = { type = "expire" }
    }]
  })
}

# ★マルチリージョン・クロスリージョン自動レプリケーション
resource "aws_ecr_replication_configuration" "vllm_replication" {
  replication_configuration {
    rule {
      dynamic "destination" {
        for_each = [for r in var.target_regions : r if r != var.primary_region]
        content {
          region      = destination.value
          registry_id = data.aws_caller_identity.current.account_id
        }
      }
    }
  }
}

# 各リージョン向けのセキュリティグループ（インバウンドルールなし、アウトバウンド全開放）
resource "aws_security_group" "vllm_sg" {
  name        = "vllm-ssm-isolated-sg"
  description = "Zero-inbound security group for vLLM SSM port forwarding"
  vpc_id      = data.aws_vpc.default.id

  # インバウンドルールは定義しない（完全閉塞）

  egress {
    description = "Allow all outbound traffic for SSM Agent and S3/ECR pull"
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = { Name = "vllm-ssm-isolated-sg" }
}

# EC2インスタンス用 IAMロール
resource "aws_iam_role" "ec2_role" {
  name = "vllm-ssm-instance-role"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action    = "sts:AssumeRole"
      Effect    = "Allow"
      Principal = { Service = "ec2.amazonaws.com" }
    }]
  })
}

# SSMエージェント通信用ポリシー
resource "aws_iam_role_policy_attachment" "ssm_core" {
  role       = aws_iam_role.ec2_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
}

# S3 & ECR 読み取りポリシー
resource "aws_iam_role_policy_attachment" "s3_read" {
  role       = aws_iam_role.ec2_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess"
}
resource "aws_iam_role_policy_attachment" "ecr_read" {
  role       = aws_iam_role.ec2_role.name
  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
}

# EC2インスタンスプロファイル
resource "aws_iam_instance_profile" "ec2_profile" {
  name = "vllm-ssm-instance-profile"
  role = aws_iam_role.ec2_role.name
}

# Lambda GPUブローカー用 IAMロール
resource "aws_iam_role" "lambda_role" {
  name = "vllm-spot-broker-lambda-role"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action    = "sts:AssumeRole"
      Effect    = "Allow"
      Principal = { Service = "lambda.amazonaws.com" }
    }]
  })
}

resource "aws_iam_role_policy_attachment" "lambda_basic" {
  role       = aws_iam_role.lambda_role.name
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

# マルチリージョンでのスポット探索・EC2起動権限
resource "aws_iam_role_policy" "lambda_ec2_broker" {
  name = "VLLMBrokerEC2Policy"
  role = aws_iam_role.lambda_role.id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "ec2:DescribeSpotPriceHistory",
          "ec2:DescribeSecurityGroups",
          "ec2:DescribeImages",
          "ec2:RunInstances",
          "ec2:CreateTags",
          "ssm:GetParameter",
          "iam:PassRole"
        ]
        Resource = "*"
      }
    ]
  })
}

#!/bin/bash
set -euo pipefail
LOG_FILE="/var/log/userdata-bootstrap.log"
exec > >(tee -a "$LOG_FILE" | logger -t user-data -s 2>/dev/console) 2>&1

AWS_REGION="${AWS_REGION}"
S3_BUCKET="${S3_BUCKET_NAME}"
DOCKER_IMAGE="${DOCKER_IMAGE}"

echo "=== S3から起動スクリプト (startup.sh) を取得 ==="
aws s3 cp "s3://$S3_BUCKET/scripts/startup.sh" /tmp/startup.sh --region "$AWS_REGION"
chmod +x /tmp/startup.sh

echo "=== startup.sh 実行 ==="
/tmp/startup.sh "$AWS_REGION" "$S3_BUCKET" "$DOCKER_IMAGE"

# terraform/launch_template.tf
# ==============================================================================
# EC2 起動テンプレート（Launch Template）全候補リージョン一括プロビジョニング
# EBS(40GB gp3)、IAMプロファイル、セキュリティグループ、UserData(S3ブートストラップ)
# を全リージョンで共通のテンプレート名「vllm-spot-template」としてコード化する
# ==============================================================================

# 1. 東京リージョン (ap-northeast-1)
resource "aws_launch_template" "lt_ap_northeast_1" {
  name          = "vllm-spot-template"
  description   = "vLLM Ephemeral Spot Node (g6.xlarge, NVIDIA L4 24GB) [ap-northeast-1]"
  instance_type = var.instance_type

  iam_instance_profile {
    name = aws_iam_instance_profile.ec2_profile.name
  }

  vpc_security_group_ids = [
    aws_security_group.sg_ap_northeast_1.id
  ]

  block_device_mappings {
    device_name = "/dev/sda1"
    ebs {
      volume_size           = 40
      volume_type           = "gp3"
      delete_on_termination = true
    }
  }

  instance_market_options {
    market_type = "spot"
    spot_options {
      spot_instance_type             = "one-time"
      instance_interruption_behavior = "terminate"
    }
  }

  user_data = base64encode(templatefile("${path.module}/templates/userdata_bootstrap.sh", {
    AWS_REGION     = "ap-northeast-1"
    S3_BUCKET_NAME = "my-vllm-models-hackathon-2026-${data.aws_caller_identity.current.account_id}-ap-northeast-1-an"
    DOCKER_IMAGE   = "${data.aws_caller_identity.current.account_id}.dkr.ecr.ap-northeast-1.amazonaws.com/${var.ecr_repository_name}:latest"
  }))

  tag_specifications {
    resource_type = "instance"
    tags = {
      Name       = "vllm-ssm-spot-node"
      Project    = "vllm-ephemeral"
      MarketType = "spot"
    }
  }
}

# 2. バージニア北部リージョン (us-east-1)
resource "aws_launch_template" "lt_us_east_1" {
  provider      = aws.us_east_1
  name          = "vllm-spot-template"
  description   = "vLLM Ephemeral Spot Node (g6.xlarge, NVIDIA L4 24GB) [us-east-1]"
  instance_type = var.instance_type

  iam_instance_profile {
    name = aws_iam_instance_profile.ec2_profile.name
  }

  vpc_security_group_ids = [
    aws_security_group.sg_us_east_1.id
  ]

  block_device_mappings {
    device_name = "/dev/sda1"
    ebs {
      volume_size           = 40
      volume_type           = "gp3"
      delete_on_termination = true
    }
  }

  instance_market_options {
    market_type = "spot"
    spot_options {
      spot_instance_type             = "one-time"
      instance_interruption_behavior = "terminate"
    }
  }

  user_data = base64encode(templatefile("${path.module}/templates/userdata_bootstrap.sh", {
    AWS_REGION     = "us-east-1"
    S3_BUCKET_NAME = "my-vllm-models-hackathon-2026-${data.aws_caller_identity.current.account_id}-us-east-1-an"
    DOCKER_IMAGE   = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-east-1.amazonaws.com/${var.ecr_repository_name}:latest"
  }))

  tag_specifications {
    resource_type = "instance"
    tags = {
      Name       = "vllm-ssm-spot-node"
      Project    = "vllm-ephemeral"
      MarketType = "spot"
    }
  }
}

# 3. オハイオリージョン (us-east-2)
resource "aws_launch_template" "lt_us_east_2" {
  provider      = aws.us_east_2
  name          = "vllm-spot-template"
  description   = "vLLM Ephemeral Spot Node (g6.xlarge, NVIDIA L4 24GB) [us-east-2]"
  instance_type = var.instance_type

  iam_instance_profile {
    name = aws_iam_instance_profile.ec2_profile.name
  }

  vpc_security_group_ids = [
    aws_security_group.sg_us_east_2.id
  ]

  block_device_mappings {
    device_name = "/dev/sda1"
    ebs {
      volume_size           = 40
      volume_type           = "gp3"
      delete_on_termination = true
    }
  }

  instance_market_options {
    market_type = "spot"
    spot_options {
      spot_instance_type             = "one-time"
      instance_interruption_behavior = "terminate"
    }
  }

  user_data = base64encode(templatefile("${path.module}/templates/userdata_bootstrap.sh", {
    AWS_REGION     = "us-east-2"
    S3_BUCKET_NAME = "my-vllm-models-hackathon-2026-${data.aws_caller_identity.current.account_id}-us-east-2-an"
    DOCKER_IMAGE   = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-east-2.amazonaws.com/${var.ecr_repository_name}:latest"
  }))

  tag_specifications {
    resource_type = "instance"
    tags = {
      Name       = "vllm-ssm-spot-node"
      Project    = "vllm-ephemeral"
      MarketType = "spot"
    }
  }
}

# 4. オレゴンリージョン (us-west-2)
resource "aws_launch_template" "lt_us_west_2" {
  provider      = aws.us_west_2
  name          = "vllm-spot-template"
  description   = "vLLM Ephemeral Spot Node (g6.xlarge, NVIDIA L4 24GB) [us-west-2]"
  instance_type = var.instance_type

  iam_instance_profile {
    name = aws_iam_instance_profile.ec2_profile.name
  }

  vpc_security_group_ids = [
    aws_security_group.sg_us_west_2.id
  ]

  block_device_mappings {
    device_name = "/dev/sda1"
    ebs {
      volume_size           = 40
      volume_type           = "gp3"
      delete_on_termination = true
    }
  }

  instance_market_options {
    market_type = "spot"
    spot_options {
      spot_instance_type             = "one-time"
      instance_interruption_behavior = "terminate"
    }
  }

  user_data = base64encode(templatefile("${path.module}/templates/userdata_bootstrap.sh", {
    AWS_REGION     = "us-west-2"
    S3_BUCKET_NAME = "my-vllm-models-hackathon-2026-${data.aws_caller_identity.current.account_id}-us-west-2-an"
    DOCKER_IMAGE   = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-west-2.amazonaws.com/${var.ecr_repository_name}:latest"
  }))

  tag_specifications {
    resource_type = "instance"
    tags = {
      Name       = "vllm-ssm-spot-node"
      Project    = "vllm-ephemeral"
      MarketType = "spot"
    }
  }
}

# 5. フランクフルトリージョン (eu-central-1)
resource "aws_launch_template" "lt_eu_central_1" {
  provider      = aws.eu_central_1
  name          = "vllm-spot-template"
  description   = "vLLM Ephemeral Spot Node (g6.xlarge, NVIDIA L4 24GB) [eu-central-1]"
  instance_type = var.instance_type

  iam_instance_profile {
    name = aws_iam_instance_profile.ec2_profile.name
  }

  vpc_security_group_ids = [
    aws_security_group.sg_eu_central_1.id
  ]

  block_device_mappings {
    device_name = "/dev/sda1"
    ebs {
      volume_size           = 40
      volume_type           = "gp3"
      delete_on_termination = true
    }
  }

  instance_market_options {
    market_type = "spot"
    spot_options {
      spot_instance_type             = "one-time"
      instance_interruption_behavior = "terminate"
    }
  }

  user_data = base64encode(templatefile("${path.module}/templates/userdata_bootstrap.sh", {
    AWS_REGION     = "eu-central-1"
    S3_BUCKET_NAME = "my-vllm-models-hackathon-2026-${data.aws_caller_identity.current.account_id}-eu-central-1-an"
    DOCKER_IMAGE   = "${data.aws_caller_identity.current.account_id}.dkr.ecr.eu-central-1.amazonaws.com/${var.ecr_repository_name}:latest"
  }))

  tag_specifications {
    resource_type = "instance"
    tags = {
      Name       = "vllm-ssm-spot-node"
      Project    = "vllm-ephemeral"
      MarketType = "spot"
    }
  }
}

output "launch_template_ids" {
  value = {
    "ap-northeast-1" = aws_launch_template.lt_ap_northeast_1.id
    "us-east-1"      = aws_launch_template.lt_us_east_1.id
    "us-east-2"      = aws_launch_template.lt_us_east_2.id
    "us-west-2"      = aws_launch_template.lt_us_west_2.id
    "eu-central-1"   = aws_launch_template.lt_eu_central_1.id
  }
  description = "EC2 Launch Template IDs across all target regions"
}

output "launch_template_name" {
  value       = "vllm-spot-template"
  description = "Common EC2 Launch Template Name across all target regions"
}

# Pythonコードの自動ZIPパッケージング
data "archive_file" "lambda_zip" {
  type        = "zip"
  source_dir  = "${path.module}/../lambda"
  output_path = "${path.module}/dist/lambda_function.zip"
}

# GPU起動ブローカー Lambda関数
resource "aws_lambda_function" "broker" {
  function_name    = "vllm-spot-broker"
  runtime          = "python3.12"
  handler          = "handler.lambda_handler"
  role             = aws_iam_role.lambda_role.arn
  timeout          = 120
  memory_size      = 256

  filename         = data.archive_file.lambda_zip.output_path
  source_code_hash = data.archive_file.lambda_zip.output_base64sha256

  environment {
    variables = {
      TARGET_REGIONS        = join(",", var.target_regions)
      S3_BUCKET_NAME        = "my-vllm-models-hackathon-2026-{account_id}-{region}-an"
      INSTANCE_PROFILE_NAME = aws_iam_instance_profile.ec2_profile.name
      ECR_REPOSITORY_NAME   = aws_ecr_repository.vllm.name
      TAG_SERVER_NAME       = "vllm-ssm-spot-node"
      SG_NAME               = "vllm-ssm-isolated-sg"
      LAUNCH_TEMPLATE_NAME  = aws_launch_template.vllm_spot_template.name
    }
  }

  tags = { Project = "vllm-ephemeral" }
}