EC2スポットインスタンス×vLLMでGPUコストを大幅削減する

#!/bin/bash
set -euo pipefail

# ==============================================================================
# setup_multiregion_assets.sh
# マルチリージョンスポット運用に必要なAWSアセットを一括反映・同期するスクリプト
# ==============================================================================

# 候補リージョン一覧 (デフォルト: 東京, オレゴン, バージニア)
# ※社内規定でデータ国外転送が禁止されている場合は国内限定に設定:
#   CANDIDATE_REGIONS=("ap-northeast-1" "ap-northeast-3")
CANDIDATE_REGIONS=("ap-northeast-1" "us-west-2" "us-east-1")
SRC_REGION="ap-northeast-1"

KEY_NAME="${KEY_NAME:-my-vllm-models-hackathon-2026}"
KEY_PATH="${KEY_PATH:-${HOME}/.ssh/${KEY_NAME}.pem}"
SG_NAME="vllm-ssh-tunnel-sg"
ECR_REPO_NAME="vllm-openai"
MODEL_PREFIX="models/Qwen/Qwen2.5-Coder-7B-Instruct"

echo "=== マルチリージョン・アセット一括同期セットアップ ==="
AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)

# 1. EC2 キーペアの反映
TMP_PUBKEY=$(mktemp)
trap 'rm -f "${TMP_PUBKEY}"' EXIT
ssh-keygen -y -f "${KEY_PATH}" > "${TMP_PUBKEY}"

for REGION in "${CANDIDATE_REGIONS[@]}"; do
    if ! aws ec2 describe-key-pairs --region "${REGION}" --key-names "${KEY_NAME}" >/dev/null 2>&1; then
        echo "[${REGION}] キーペア '${KEY_NAME}' をインポート中..."
        aws ec2 import-key-pair \
            --region "${REGION}" \
            --key-name "${KEY_NAME}" \
            --public-key-material "fileb://${TMP_PUBKEY}"
    fi
done

# 2. SSH専用セキュリティグループの事前作成 (デフォルトSG誤適用の防止)
for REGION in "${CANDIDATE_REGIONS[@]}"; do
    SG_ID=$(aws ec2 describe-security-groups \
        --region "${REGION}" \
        --filters "Name=group-name,Values=${SG_NAME}" \
        --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || true)
    
    if [ -z "${SG_ID}" ] || [ "${SG_ID}" = "None" ]; then
        DEFAULT_VPC=$(aws ec2 describe-vpcs --region "${REGION}" --filters "Name=isDefault,Values=true" --query 'Vpcs[0].VpcId' --output text)
        SG_ID=$(aws ec2 create-security-group \
            --region "${REGION}" \
            --group-name "${SG_NAME}" \
            --description "Allow SSH port 22 only for vLLM tunnel" \
            --vpc-id "${DEFAULT_VPC}" \
            --query 'GroupId' --output text)
        aws ec2 authorize-security-group-ingress \
            --region "${REGION}" --group-id "${SG_ID}" \
            --protocol tcp --port 22 --cidr "0.0.0.0/0"
        echo "[${REGION}] 作成完了: ${SG_ID} (ポート22のみ許可)"
    fi
done

# 3. S3 バケット作成 & モデルデータの同期
SRC_BUCKET="my-vllm-models-hackathon-2026-${AWS_ACCOUNT_ID}-${SRC_REGION}-an"

for REGION in "${CANDIDATE_REGIONS[@]}"; do
    DEST_BUCKET="my-vllm-models-hackathon-2026-${AWS_ACCOUNT_ID}-${REGION}-an"
    if ! aws s3 ls "s3://${DEST_BUCKET}" >/dev/null 2>&1; then
        echo "[${REGION}] S3バケット '${DEST_BUCKET}' を作成中..."
        aws s3 mb "s3://${DEST_BUCKET}" --region "${REGION}"
    fi

    if [ "${REGION}" != "${SRC_REGION}" ]; then
        echo "[${REGION}] モデル重みデータの同期中 (s3://${SRC_BUCKET} -> s3://${DEST_BUCKET})..."
        aws s3 sync "s3://${SRC_BUCKET}/${MODEL_PREFIX}/" "s3://${DEST_BUCKET}/${MODEL_PREFIX}/" --no-progress
    fi
done

# 4. ECR リポジトリ作成 & クロスリージョン自動レプリケーション設定
for REGION in "${CANDIDATE_REGIONS[@]}"; do
    if ! aws ecr describe-repositories --region "${REGION}" --repository-names "${ECR_REPO_NAME}" >/dev/null 2>&1; then
        aws ecr create-repository --region "${REGION}" --repository-name "${ECR_REPO_NAME}" \
            --image-scanning-configuration scanOnPush=true --image-tag-mutability MUTABLE >/dev/null
    fi
done

# ECR レプリケーション設定とトリガー
REPL_DESTS=()
for REGION in "${CANDIDATE_REGIONS[@]}"; do
    if [ "${REGION}" != "${SRC_REGION}" ]; then
        REPL_DESTS+=("{\"region\":\"${REGION}\",\"registryId\":\"${AWS_ACCOUNT_ID}\"}")
    fi
done
REPL_DESTS_JSON=$(IFS=,; echo "${REPL_DESTS[*]}")
aws ecr put-replication-configuration --region "${SRC_REGION}" \
    --replication-configuration "{\"rules\":[{\"destinations\":[${REPL_DESTS_JSON}]}]}" >/dev/null 2>&1 || true

MANIFEST=$(aws ecr batch-get-image --region "${SRC_REGION}" --repository-name "${ECR_REPO_NAME}" --image-ids imageTag=latest --query 'images[0].imageManifest' --output text 2>/dev/null || echo "")
if [ -n "${MANIFEST}" ] && [ "${MANIFEST}" != "None" ]; then
    TMP_MANIFEST=$(mktemp)
    echo "${MANIFEST}" > "${TMP_MANIFEST}"
    aws ecr put-image --region "${SRC_REGION}" --repository-name "${ECR_REPO_NAME}" --image-tag "latest" --image-manifest "file://${TMP_MANIFEST}" >/dev/null 2>&1 || true
    rm -f "${TMP_MANIFEST}"
fi
echo "マルチリージョン・アセット反映が完了しました！"

#!/bin/bash
set -euo pipefail

# ==============================================================================
# 02_ec2_userdata.sh
# EC2起動時に実行されるUserDataスクリプト (S3・ECR並列ダウンロード最適化版)
#
# 前提:
#   - AMI: Ubuntu 22.04 Deep Learning AMI (NVIDIA Driver & Docker導入済み)
#   - インスタンスタイプ: g6.xlarge (NVIDIA L4 GPU: 24GB VRAM, 250GB NVMe SSD付属)
#   - IAMロール: S3(ReadOnly/FullAccess) 及び ECR(ReadOnly) 権限アタッチ済み
# ==============================================================================

LOG_FILE="/var/log/userdata-vllm.log"
exec > >(tee -a "${LOG_FILE}") 2>&1
echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] UserData 実行開始 ==="

# 設定パラメータ
AWS_REGION="${AWS_REGION:-ap-northeast-1}"
S3_BUCKET_NAME="${S3_BUCKET_NAME:-my-llm-models-tokyo}"
HF_MODEL_ID="${HF_MODEL_ID:-Qwen/Qwen2.5-Coder-7B-Instruct}"
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-Qwen/Qwen2.5-Coder-7B-Instruct}"
VLLM_PORT="8000"
GPU_MEMORY_UTILIZATION="0.90"
MAX_MODEL_LEN="16384"
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"

echo "取得モデル(HF) : ${HF_MODEL_ID}"
echo "公開モデル名   : ${SERVED_MODEL_NAME}"
echo "S3バケット     : s3://${S3_BUCKET_NAME}"
echo "Dockerイメージ : ${DOCKER_IMAGE}"

# 1. ローカル NVMe インスタンスストア（250GB）の検出と活用
NVME_DIR="/opt/dlami/nvme"
if mountpoint -q "${NVME_DIR}" || [ -d "${NVME_DIR}" ]; then
    echo "DLAMI既定の NVMe マウント (${NVME_DIR}) を検出しました。モデル＆Docker領域として活用します..."
    mkdir -p "${NVME_DIR}/models" "${NVME_DIR}/docker"
    mkdir -p /data
    ln -sfn "${NVME_DIR}/models" /data/models
else
    NVME_DEV=$(lsblk -d -n -o NAME,SIZE | grep -E '250G|232G' | head -n1 | awk '{print $1}')
    if [ -n "${NVME_DEV}" ]; then
        echo "ローカル NVMe SSD (/dev/${NVME_DEV}) を検出しました。/data にマウントします..."
        mkfs.ext4 -F "/dev/${NVME_DEV}" || true
        mkdir -p /data
        mount -o noatime "/dev/${NVME_DEV}" /data || true
    else
        mkdir -p /data
    fi
    mkdir -p /data/models /data/docker
    NVME_DIR="/data"
fi

LOCAL_MODEL_ROOT="/data/models"
LOCAL_MODEL_DIR="${LOCAL_MODEL_ROOT}/${HF_MODEL_ID}"
mkdir -p "${LOCAL_MODEL_DIR}"
chmod 777 "${LOCAL_MODEL_ROOT}"

# 2. Docker & containerd のデータ領域を NVMe に配置し、EBS枯渇防止＆レイヤー展開を高速化
echo "Docker/containerd を停止して NVMe 領域へのバインドマウントを設定します..."
systemctl stop docker containerd || true

mkdir -p "${NVME_DIR}/docker" "${NVME_DIR}/containerd"
mkdir -p /var/lib/docker /var/lib/containerd

cp -a /var/lib/docker/* "${NVME_DIR}/docker/" 2>/dev/null || true
cp -a /var/lib/containerd/* "${NVME_DIR}/containerd/" 2>/dev/null || true

mount --bind "${NVME_DIR}/docker" /var/lib/docker
mount --bind "${NVME_DIR}/containerd" /var/lib/containerd

if ! grep -q "/var/lib/docker" /etc/fstab; then
    echo "${NVME_DIR}/docker /var/lib/docker none defaults,bind 0 0" >> /etc/fstab
fi
if ! grep -q "/var/lib/containerd" /etc/fstab; then
    echo "${NVME_DIR}/containerd /var/lib/containerd none defaults,bind 0 0" >> /etc/fstab
fi

mkdir -p /etc/docker
cat <<EOF > /etc/docker/daemon.json
{
  "data-root": "/var/lib/docker"
}
EOF

systemctl daemon-reload
systemctl start containerd
systemctl start docker

# ==============================================================================
# 3. S3モデル同期 と ECR Dockerイメージpull の完全並列実行 (高速化の肝)
# ==============================================================================
echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] S3モデル同期とDockerイメージ取得を並列開始します ==="
S3_SRC="s3://${S3_BUCKET_NAME}/models/${HF_MODEL_ID}"
aws configure set default.s3.max_concurrent_requests 20

# [並列タスクA] Dockerイメージの取得 (ECRログイン & pull)
(
    echo "--- [Task A] Dockerイメージ取得開始: ${DOCKER_IMAGE} ---"
    if [[ "${DOCKER_IMAGE}" == *".dkr.ecr."* ]]; then
        ECR_REGISTRY=$(echo "${DOCKER_IMAGE}" | cut -d'/' -f1)
        ECR_REGION=$(echo "${ECR_REGISTRY}" | awk -F. '{print $4}')
        echo "ECR ログイン認証中 (${ECR_REGION:-${AWS_REGION}})..."
        aws ecr get-login-password --region "${ECR_REGION:-${AWS_REGION}}" | docker login --username AWS --password-stdin "${ECR_REGISTRY}" || true
    fi
    echo "docker pull 実行中..."
    docker pull "${DOCKER_IMAGE}"
    echo "--- [Task A] Dockerイメージ取得完了 ---"
) > /var/log/docker-pull.log 2>&1 &
PID_DOCKER_PULL=$!

# [並列タスクB] S3モデルデータの高速同期
rm -f /tmp/.has_s3_model
(
    echo "--- [Task B] S3モデル同期開始: ${S3_SRC} ---"
    for i in {1..15}; do
        if aws s3 ls "s3://${S3_BUCKET_NAME}" --region "${AWS_REGION}" >/dev/null 2>&1; then
            break
        fi
        sleep 2
    done

    if aws s3 ls "${S3_SRC}/" --region "${AWS_REGION}" 2>&1 | grep -E '(\.safetensors|\.bin|\.json)' >/dev/null; then
        echo "S3からモデルを高速同期中..."
        aws s3 sync "${S3_SRC}" "${LOCAL_MODEL_DIR}" --region "${AWS_REGION}" --no-progress
        touch /tmp/.has_s3_model
    else
        echo "S3にモデルがないため、Hugging Faceから直接取得中..."
        python3 -m pip install -U "huggingface_hub[cli]" || pip3 install -U "huggingface_hub[cli]" || true
        python3 -c "
import sys
from huggingface_hub import snapshot_download
try:
    snapshot_download(repo_id='${HF_MODEL_ID}', local_dir='${LOCAL_MODEL_DIR}', local_dir_use_symlinks=False)
except Exception as e:
    print(f'ダウンロードエラー: {e}', file=sys.stderr)
    sys.exit(1)
"
    fi
    echo "--- [Task B] モデルダウンロード完了 ---"
) > /var/log/model-sync.log 2>&1 &
PID_MODEL_SYNC=$!

echo "並列ダウンロード待機中 (Docker Pull: PID ${PID_DOCKER_PULL} / S3 Sync: PID ${PID_MODEL_SYNC})..."
wait "${PID_DOCKER_PULL}"
wait "${PID_MODEL_SYNC}"
echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] S3モデル同期・Docker pull 両方の完了を確認しました！ ==="

[ -f /tmp/.has_s3_model ] && HAS_S3_MODEL=true || HAS_S3_MODEL=false

echo "ローカルモデル容量確認:"
du -sh "${LOCAL_MODEL_DIR}"

# 4. vLLMコンテナの起動 (ローカルにイメージもモデルも揃っているため即時起動)
echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] vLLMコンテナ起動 ==="
CONTAINER_NAME="vllm-server"
docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true

docker run -d \
    --name "${CONTAINER_NAME}" \
    --restart unless-stopped \
    --gpus all \
    --ipc=host \
    -p "${VLLM_PORT}:8000" \
    -v "${LOCAL_MODEL_ROOT}:/models" \
    "${DOCKER_IMAGE}" \
    --model "/models/${HF_MODEL_ID}" \
    --served-model-name "${SERVED_MODEL_NAME}" \
    --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
    --max-model-len "${MAX_MODEL_LEN}" \
    --trust-remote-code \
    --enable-auto-tool-choice \
    --tool-call-parser hermes

# 5. ヘルスチェック (起動待機)
echo "vLLM サーバーの起動ヘルスチェックを開始します (ポート ${VLLM_PORT})..."
MAX_RETRIES=120
RETRY_COUNT=0

while [ ${RETRY_COUNT} -lt ${MAX_RETRIES} ]; do
    if curl -s "http://127.0.0.1:${VLLM_PORT}/health" > /dev/null 2>&1; then
        echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] vLLMサーバーが正常に起動しました！ ==="
        break
    fi
    echo "起動待機中... (${RETRY_COUNT}/${MAX_RETRIES})"
    sleep 5
    RETRY_COUNT=$((RETRY_COUNT + 1))
done

if [ ${RETRY_COUNT} -eq ${MAX_RETRIES} ]; then
    echo "警告: vLLMヘルスチェックがタイムアウトしました。'docker logs ${CONTAINER_NAME}' を確認してください。"
else
    if [ "${HAS_S3_MODEL}" = "false" ]; then
        echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] 初回取得モデルを裏でS3へバックアップ開始 ==="
        (
            if ! aws s3 ls "s3://${S3_BUCKET_NAME}" --region "${AWS_REGION}" 2>/dev/null; then
                if [ "${AWS_REGION}" = "us-east-1" ]; then
                    aws s3api create-bucket --bucket "${S3_BUCKET_NAME}" 2>/dev/null || true
                else
                    aws s3api create-bucket --bucket "${S3_BUCKET_NAME}" --region "${AWS_REGION}" --create-bucket-configuration LocationConstraint="${AWS_REGION}" 2>/dev/null || true
                fi
            fi
            ionice -c 3 aws s3 sync "${LOCAL_MODEL_DIR}" "${S3_SRC}" --region "${AWS_REGION}" --no-progress 2>/dev/null || \
            aws s3 sync "${LOCAL_MODEL_DIR}" "${S3_SRC}" --region "${AWS_REGION}" --no-progress 2>/dev/null || true
            echo "[$(date '+%Y-%m-%d %H:%M:%S')] S3モデルバックアップ完了！"
        ) > /var/log/s3-backup.log 2>&1 &
    fi
fi

# 6. 1時間アイドル時の自動シャットダウン（自爆Terminate）デーモン起動
echo "=== アイドル自動終了デーモンを設定・起動します ==="
cat <<'EOF' > /usr/local/bin/auto-idle-shutdown.sh
#!/bin/bash
IDLE_LIMIT_SEC=3600  # 1時間 (3600秒)
IDLE_COUNT=0
CHECK_INTERVAL=300   # 5分おきにチェック

while true; do
    sleep "${CHECK_INTERVAL}"
    REQ_COUNT=$(docker logs --since 5m vllm-server 2>&1 | grep -c "POST /v1" || true)
    
    if [ "${REQ_COUNT}" -eq 0 ]; then
        IDLE_COUNT=$((IDLE_COUNT + CHECK_INTERVAL))
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] アイドル継続中: ${IDLE_COUNT}s / ${IDLE_LIMIT_SEC}s"
        
        if [ "${IDLE_COUNT}" -ge "${IDLE_LIMIT_SEC}" ]; then
            echo "[$(date '+%Y-%m-%d %H:%M:%S')] 1時間アイドル状態が継続したため、自動終了(Terminate)を実行します。"
            shutdown -h now
            exit 0
        fi
    else
        IDLE_COUNT=0
    fi
done
EOF

chmod +x /usr/local/bin/auto-idle-shutdown.sh
nohup /usr/local/bin/auto-idle-shutdown.sh > /var/log/auto-idle-shutdown.log 2>&1 &
echo "=== [$(date '+%Y-%m-%d %H:%M:%S')] UserData 全処理完了 (自動アイドル監視稼働中) ==="

#!/bin/bash
set -euo pipefail

# ==============================================================================
# ec2_launch_spot_and_sync_openwebui.sh
# 最安リージョンのスポットインスタンスを自動選定・起動し、
# SSHトンネルを確立してOpen WebUIへ直結するスクリプト
# ==============================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
USER_DATA_FILE="${USER_DATA_FILE:-${SCRIPT_DIR}/02_ec2_userdata.sh}"
INSTANCE_STATE_FILE="${SCRIPT_DIR}/.current_instance_id"
TUNNEL_PID_FILE="${SCRIPT_DIR}/.current_tunnel_pid"
REGION_STATE_FILE="${SCRIPT_DIR}/.current_region"

# 候補リージョン一覧 (デフォルト: 東京, オレゴン, バージニア, フランクフルト)
CANDIDATE_REGIONS=("ap-northeast-1" "us-west-2" "us-east-1" "eu-central-1")
CANDIDATE_INSTANCE_TYPES=("g6.xlarge")
IAM_ROLE_NAME="${IAM_ROLE_NAME:-EC2-S3-FullAccess-Profile}"
EBS_SIZE_GB="${EBS_SIZE_GB:-40}"
KEY_NAME="${KEY_NAME:-my-vllm-models-hackathon-2026}"
KEY_PATH="${KEY_PATH:-${HOME}/.ssh/${KEY_NAME}.pem}"

echo "=== 1. マルチリージョン スポット総合優先度探索 (g6.xlarge) ==="
ITYPE="${CANDIDATE_INSTANCE_TYPES[0]}"
START_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

# Spot Advisor 中断頻度を一括取得
declare -A REGION_INTR_TIERS
PY_BIN=""
command -v python3 >/dev/null 2>&1 && PY_BIN="python3" || (command -v python >/dev/null 2>&1 && PY_BIN="python")
if [ -n "${PY_BIN}" ]; then
    ADVISOR_DATA=$("${PY_BIN}" -c "
import urllib.request, json
try:
    with urllib.request.urlopen('https://spot-bid-advisor.s3.amazonaws.com/spot-advisor-data.json', timeout=4) as res:
        adv = json.loads(res.read()).get('spot_advisor', {})
        for r, info in adv.items():
            print(f'{r}:{info.get(\"Linux\", {}).get(\"${ITYPE}\", {}).get(\"r\", 4)}')
except Exception: pass
" 2>/dev/null || true)
    while IFS=':' read -r r_id r_tier; do
        r_id=$(echo "${r_id}" | tr -d '[:space:]')
        r_tier=$(echo "${r_tier}" | tr -d '[:space:]')
        [ -n "${r_id}" ] && REGION_INTR_TIERS["${r_id}"]="${r_tier}"
    done <<< "${ADVISOR_DATA}"
fi

declare -A REG_STATUS REG_MIN_PRICES REG_AZ_COUNTS REG_SCORES REG_RANKS
BEST_PRICE="999.0"

for REGION in "${CANDIDATE_REGIONS[@]}"; do
    RAW_PRICES=$(aws ec2 describe-spot-price-history \
        --region "${REGION}" \
        --instance-types "${ITYPE}" \
        --product-descriptions "Linux/UNIX" \
        --start-time "${START_TIME}" \
        --no-paginate \
        --query 'SpotPriceHistory[*].[AvailabilityZone, SpotPrice]' \
        --output text 2>/dev/null | tr -d '\r' || true)

    [ -z "${RAW_PRICES}" ] && { REG_STATUS["${REGION}"]="unavailable"; continue; }

    REG_MIN_PRICE="999.0"
    AZ_COUNT=0
    while read -r AZ PRICE; do
        AZ=$(echo "${AZ}" | tr -d '[:space:]')
        PRICE=$(echo "${PRICE}" | tr -d '[:space:]')
        [ -z "${AZ}" ] || [ -z "${PRICE}" ] && continue
        AZ_COUNT=$((AZ_COUNT + 1))
        if awk -v p="${PRICE}" -v m="${REG_MIN_PRICE}" 'BEGIN {exit !(p < m)}' 2>/dev/null; then
            REG_MIN_PRICE="${PRICE}"
        fi
    done <<< "${RAW_PRICES}"

    [ "${REG_MIN_PRICE}" = "999.0" ] && { REG_STATUS["${REGION}"]="unavailable"; continue; }
    REG_STATUS["${REGION}"]="ok"
    REG_MIN_PRICES["${REGION}"]="${REG_MIN_PRICE}"
    REG_AZ_COUNTS["${REGION}"]="${AZ_COUNT}"

    if awk -v p="${REG_MIN_PRICE}" -v m="${BEST_PRICE}" 'BEGIN {exit !(p < m)}' 2>/dev/null; then
        BEST_PRICE="${REG_MIN_PRICE}"
    fi
done

# スコア計算 (価格40点 + 余剰AZ数40点 + 中断頻度20点 = 100点満点)
BEST_REGION=""
BEST_SCORE=-1

for REGION in "${CANDIDATE_REGIONS[@]}"; do
    [ "${REG_STATUS[${REGION}]:-}" != "ok" ] && continue
    PRICE="${REG_MIN_PRICES[${REGION}]}"
    TIER="${REGION_INTR_TIERS[${REGION}]:-4}"
    AZ_COUNT="${REG_AZ_COUNTS[${REGION}]}"

    P_SCORE=$(awk -v min="${BEST_PRICE}" -v cur="${PRICE}" 'BEGIN {printf "%.0f", 40.0 * (min / cur)}')
    case "${TIER}" in
        0) I_SCORE=20; I_TEXT="< 5% (極低)" ;;
        1) I_SCORE=15; I_TEXT="5-10% (低)" ;;
        2) I_SCORE=10; I_TEXT="10-15% (中)" ;;
        3) I_SCORE=5;  I_TEXT="15-20% (中高)" ;;
        *) I_SCORE=0;  I_TEXT="> 20% (高)" ;;
    esac

    if [ "${AZ_COUNT}" -ge 5 ]; then C_SCORE=40
    elif [ "${AZ_COUNT}" -eq 4 ]; then C_SCORE=32
    elif [ "${AZ_COUNT}" -eq 3 ]; then C_SCORE=24
    elif [ "${AZ_COUNT}" -eq 2 ]; then C_SCORE=16
    else C_SCORE=8
    fi

    TOTAL_SCORE=$(( P_SCORE + I_SCORE + C_SCORE ))
    REG_SCORES["${REGION}"]="${TOTAL_SCORE}"

    if [ "${TOTAL_SCORE}" -ge 75 ]; then RANK="Rank S"
    elif [ "${TOTAL_SCORE}" -ge 65 ]; then RANK="Rank A"
    elif [ "${TOTAL_SCORE}" -ge 50 ]; then RANK="Rank B"
    else RANK="Rank C"
    fi
    REG_RANKS["${REGION}"]="${RANK}"

    PRICE_FMT=$(awk -v p="${PRICE}" 'BEGIN {printf "%.4f", p}')
    echo "  - [${REGION}] 価格=\$${PRICE_FMT}/h, 中断率=${I_TEXT}, 余剰AZ=${AZ_COUNT} -> 総合スコア: ${TOTAL_SCORE}点 (${RANK})"

    if [ "${TOTAL_SCORE}" -gt "${BEST_SCORE}" ]; then
        BEST_SCORE="${TOTAL_SCORE}"
        BEST_REGION="${REGION}"
    fi
done

AWS_REGION="${BEST_REGION:-ap-northeast-1}"
INSTANCE_TYPE="${ITYPE}"
SELECTED_PRICE="${REG_MIN_PRICES[${AWS_REGION}]:-0.5672}"
SELECTED_PRICE_FMT=$(awk -v p="${SELECTED_PRICE}" 'BEGIN {printf "%.4f", p}')
SELECTED_SCORE="${REG_SCORES[${AWS_REGION}]:-40}"
SELECTED_RANK="${REG_RANKS[${AWS_REGION}]:-Rank C}"

echo "=========================================================="
echo " 🏆 総合選定結果: ${AWS_REGION} (スコア: ${SELECTED_SCORE}点 / ${SELECTED_RANK})"
echo "    インスタンスタイプ: ${INSTANCE_TYPE}, 最安スポット価格: \$${SELECTED_PRICE_FMT}/h"
echo "=========================================================="

# Deep Learning AMI 検索
AMI_ID=$(aws ec2 describe-images \
    --region "${AWS_REGION}" \
    --owners amazon \
    --filters "Name=name,Values=Deep Learning OSS Nvidia Driver AMI GPU PyTorch * (Ubuntu 22.04)*" "Name=state,Values=available" \
    --query 'sort_by(Images, &CreationDate)[-1].ImageId' \
    --output text)

# SSH専用SGの確認 (ポート22のみ)
SG_NAME="vllm-ssh-tunnel-sg"
SG_ID=$(aws ec2 describe-security-groups \
    --region "${AWS_REGION}" \
    --filters "Name=group-name,Values=${SG_NAME}" \
    --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || true)

# セキュリティグループIDの厳格な検証 (デフォルトSGでの起動事故を完全防止)
if [ -z "${SG_ID}" ] || [ "${SG_ID}" = "None" ] || [[ ! "${SG_ID}" =~ ^sg-[0-9a-f]+$ ]]; then
    echo "エラー: セキュリティグループの取得・作成に失敗しました (SG_ID: '${SG_ID}')"
    echo "デフォルトセキュリティグループでの誤起動を防止するため、処理を中止します。"
    exit 1
fi

SPOT_OPTIONS='{
  "MarketType": "spot",
  "SpotOptions": {
    "SpotInstanceType": "one-time",
    "InstanceInterruptionBehavior": "terminate"
  }
}'

BLOCK_DEVICE_MAPPINGS="[
  {
    \"DeviceName\": \"/dev/sda1\",
    \"Ebs\": {
      \"VolumeSize\": ${EBS_SIZE_GB},
      \"VolumeType\": \"gp3\",
      \"DeleteOnTermination\": true
    }
  }
]"

AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
export S3_BUCKET_NAME="my-vllm-models-hackathon-2026-${AWS_ACCOUNT_ID}-${AWS_REGION}-an"

ECR_IMAGE="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/vllm-openai:latest"
if aws ecr describe-images --region "${AWS_REGION}" --repository-name "vllm-openai" --image-ids imageTag=latest >/dev/null 2>&1; then
    DOCKER_IMAGE="${ECR_IMAGE}"
else
    DOCKER_IMAGE="vllm/vllm-openai:latest"
fi

TMP_USERDATA=$(mktemp)
sed -e "s|^AWS_REGION=.*|AWS_REGION=\"${AWS_REGION}\"|" \
    -e "s|^S3_BUCKET_NAME=.*|S3_BUCKET_NAME=\"${S3_BUCKET_NAME}\"|" \
    -e "s|^DOCKER_IMAGE=.*|DOCKER_IMAGE=\"${DOCKER_IMAGE}\"|" \
    "${USER_DATA_FILE}" > "${TMP_USERDATA}"

echo "インスタンスタイプ '${INSTANCE_TYPE}' でスポット起動をリクエスト中..."
RUN_OUTPUT=$(aws ec2 run-instances \
    --region "${AWS_REGION}" \
    --image-id "${AMI_ID}" \
    --instance-type "${INSTANCE_TYPE}" \
    --key-name "${KEY_NAME}" \
    --iam-instance-profile "Name=${IAM_ROLE_NAME}" \
    --security-group-ids "${SG_ID}" \
    --instance-market-options "${SPOT_OPTIONS}" \
    --instance-initiated-shutdown-behavior terminate \
    --block-device-mappings "${BLOCK_DEVICE_MAPPINGS}" \
    --user-data "file://${TMP_USERDATA}" \
    --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=vllm-spot-${INSTANCE_TYPE}}]" \
    --output json)

rm -f "${TMP_USERDATA}"

INSTANCE_ID=$(echo "${RUN_OUTPUT}" | jq -r '.Instances[0].InstanceId')
echo "スポットインスタンス起動成功: ${INSTANCE_ID} (Type: ${INSTANCE_TYPE})"
echo "${INSTANCE_ID}" > "${INSTANCE_STATE_FILE}"
echo "${AWS_REGION}" > "${REGION_STATE_FILE}"

echo "インスタンスの実行状態（running）を待機中..."
aws ec2 wait instance-running --region "${AWS_REGION}" --instance-ids "${INSTANCE_ID}"

PUBLIC_IP=$(aws ec2 describe-instances \
    --region "${AWS_REGION}" \
    --instance-ids "${INSTANCE_ID}" \
    --query 'Reservations[0].Instances[0].PublicIpAddress' \
    --output text)

echo "インスタンス起動完了！ パブリックIP: ${PUBLIC_IP}"

# 既存SSHトンネルのクリーンアップ＆新規確立
if [ -f "${TUNNEL_PID_FILE}" ]; then
    OLD_PID=$(cat "${TUNNEL_PID_FILE}" | tr -d '[:space:]')
    if ps -p "${OLD_PID}" > /dev/null 2>&1; then
        kill -9 "${OLD_PID}" 2>/dev/null || true
    fi
    rm -f "${TUNNEL_PID_FILE}"
fi

# SSH接続待機
while ! nc -z -w 3 "${PUBLIC_IP}" 22 2>/dev/null; do
    sleep 2
done

# バックグラウンドSSHトンネル確立 (localhost:8000 -> EC2内部の8000)
ssh -i "${KEY_PATH}" \
    -o StrictHostKeyChecking=no \
    -o UserKnownHostsFile=/dev/null \
    -o ServerAliveInterval=15 \
    -o ServerAliveCountMax=3 \
    -N -f -L 8000:localhost:8000 \
    ubuntu@"${PUBLIC_IP}"

TUNNEL_PID=$(pgrep -f "ssh.*-L 8000:localhost:8000.*ubuntu@${PUBLIC_IP}" | head -n 1 || true)
if [ -n "${TUNNEL_PID}" ]; then
    echo "${TUNNEL_PID}" > "${TUNNEL_PID_FILE}"
fi

echo "=== vLLM サーバーの起動待機 (http://localhost:8000/health) ==="
while ! curl -s "http://localhost:8000/health" > /dev/null 2>&1; do
    sleep 5
done

echo "=========================================================="
echo " スポット環境の全工程が完了しました！"
echo " EC2 Instance ID: ${INSTANCE_ID} (Spot: ${INSTANCE_TYPE}, Region: ${AWS_REGION})"
echo " SSH Tunnel     : localhost:8000 -> EC2:8000 (暗号化中)"
echo " Web UI URL     : http://localhost:3000"
echo "=========================================================="

#!/bin/bash
set -euo pipefail

# ==============================================================================
# ec2_terminate.sh
# スポットインスタンスおよびSSHトンネルプロセスを完全破棄するスクリプト
# ==============================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
INSTANCE_STATE_FILE="${SCRIPT_DIR}/.current_instance_id"
TUNNEL_PID_FILE="${SCRIPT_DIR}/.current_tunnel_pid"
REGION_STATE_FILE="${SCRIPT_DIR}/.current_region"

AWS_REGION=$(cat "${REGION_STATE_FILE}" 2>/dev/null | tr -d '[:space:]' || echo "ap-northeast-1")
INSTANCE_ID=$(cat "${INSTANCE_STATE_FILE}" 2>/dev/null | tr -d '[:space:]' || echo "${1:-}")

# 1. SSHトンネルプロセスの終了
if [ -f "${TUNNEL_PID_FILE}" ]; then
    TUNNEL_PID=$(cat "${TUNNEL_PID_FILE}" | tr -d '[:space:]')
    if [ -n "${TUNNEL_PID}" ] && ps -p "${TUNNEL_PID}" > /dev/null 2>&1; then
        echo "SSHトンネルプロセス (PID: ${TUNNEL_PID}) を終了中..."
        kill -9 "${TUNNEL_PID}" 2>/dev/null || true
    fi
    rm -f "${TUNNEL_PID_FILE}"
fi

# 2. EC2インスタンスの終了
if [ -n "${INSTANCE_ID}" ]; then
    echo "=== スポットインスタンスの完全終了 (Terminate) ==="
    aws ec2 terminate-instances --region "${AWS_REGION}" --instance-ids "${INSTANCE_ID}" --output table
    aws ec2 wait instance-terminated --region "${AWS_REGION}" --instance-ids "${INSTANCE_ID}"
    rm -f "${INSTANCE_STATE_FILE}" "${REGION_STATE_FILE}"
    echo "スポットインスタンス、EBSボリューム、SSHトンネルの完全破棄が完了しました！"
fi