From 914a018fcd1a5b7dd57a2d4410aafb4fd309760c Mon Sep 17 00:00:00 2001 From: mason Date: Tue, 7 Apr 2026 02:44:00 +0900 Subject: [PATCH 1/2] feat: setup gitops infra with karpenter and argo rollouts --- .github/workflows/.cicd.yml.swp | Bin 0 -> 12288 bytes .../workflows/build-and-update-manifest.yaml | 47 ++++++++++++ .github/workflows/cd.yaml | 67 ------------------ .github/workflows/ci.yaml | 37 ---------- .github/workflows/cicd.yaml | 62 ++++++++++++++++ .gitignore | 14 ++++ k8s/argocd/app-prod.yaml | 4 +- k8s/base/cronjob.yaml | 4 +- k8s/base/deployment.yaml | 27 ------- k8s/base/hpa.yaml | 26 +------ k8s/base/ingress.yaml | 12 ++-- k8s/base/kustomization.yaml | 9 +-- k8s/base/rollout.yaml | 36 ++++++++++ k8s/base/service-canary.yaml | 7 ++ k8s/base/service-stable.yaml | 8 +++ k8s/base/service.yaml | 11 --- k8s/overlays/{prod => v1}/kustomization.yaml | 3 +- k8s/overlays/v2/kustomization.yaml | 9 +++ terraform/addons.tf | 58 +++++++++++++++ terraform/karpenter.tf | 39 ++++++++++ terraform/network.tf | 10 ++- terraform/providers.tf | 34 ++++++--- terraform/security.tf | 1 - 23 files changed, 331 insertions(+), 194 deletions(-) create mode 100644 .github/workflows/.cicd.yml.swp create mode 100644 .github/workflows/build-and-update-manifest.yaml delete mode 100644 .github/workflows/cd.yaml delete mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/cicd.yaml create mode 100644 .gitignore delete mode 100644 k8s/base/deployment.yaml create mode 100644 k8s/base/rollout.yaml create mode 100644 k8s/base/service-canary.yaml create mode 100644 k8s/base/service-stable.yaml delete mode 100644 k8s/base/service.yaml rename k8s/overlays/{prod => v1}/kustomization.yaml (74%) create mode 100644 k8s/overlays/v2/kustomization.yaml create mode 100644 terraform/addons.tf create mode 100644 terraform/karpenter.tf diff --git a/.github/workflows/.cicd.yml.swp b/.github/workflows/.cicd.yml.swp new file mode 100644 index 0000000000000000000000000000000000000000..d33ac77c05f7be69ce39a5b4bd62f81ff11868fa GIT binary patch literal 12288 zcmeI%Jr05}7=Yp8ZZtZZpw9g1M=bb9oRgsG}r|$lIiC+J0^N zJzLX_y_zrdndt~sx_R?1_kkt?nS`hBL`h%7t+O!p-I6FFu0@~~TntyH(ujPS?!q{* z%)OT$&KpaeaNPM}mRMVF>zbFqFFlS$1Q4hs&=%8)W9M7OqoLmKEi3IN7y$$jKmY** z5I_I{1nMo2Mw+;(6k4A&v>$!Fv*h*&Abgn? g|M&f0jGIkE_6Q(=00IagfB*srAbgw}3m$hQIsgCw literal 0 HcmV?d00001 diff --git a/.github/workflows/build-and-update-manifest.yaml b/.github/workflows/build-and-update-manifest.yaml new file mode 100644 index 0000000..df6b9f2 --- /dev/null +++ b/.github/workflows/build-and-update-manifest.yaml @@ -0,0 +1,47 @@ +name: Build and Update Manifest + +on: + push: + branches: [ "main" ] + paths: [ "app/**", "Dockerfile", "k8s/**" ] + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + + steps: + - uses: actions/checkout@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: eu-west-1 + + - name: Login to ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build and Push Image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/data-pipeline-app:$IMAGE_TAG . + docker push $ECR_REGISTRY/data-pipeline-app:$IMAGE_TAG + + - name: Update Kustomize Tag + run: | + cd k8s/overlays/prod + kustomize edit set image data-pipeline-app=${{ steps.login-ecr.outputs.registry }}/data-pipeline-app:${{ github.sha }} + + - name: Commit and Push Manifest Change + run: | + git config --global user.email "github-actions@github.com" + git config --global user.name "github-actions" + git add k8s/overlays/prod/kustomization.yaml + git commit -m "Update image tag to ${{ github.sha }} [skip ci]" + git push diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml deleted file mode 100644 index a882a2c..0000000 --- a/.github/workflows/cd.yaml +++ /dev/null @@ -1,67 +0,0 @@ -name: CD - -on: - push: - branches: [main] - paths: - - "app/**" - - "k8s/**" - - ".github/workflows/cd.yaml" - -permissions: - id-token: write - contents: write - -concurrency: - group: prod-deploy - cancel-in-progress: true - -env: - AWS_REGION: eu-west-1 - ECR_REPOSITORY: data-pipeline-app - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Configure AWS credentials with OIDC - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }} - - - name: Login to ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build and push image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ github.sha }} - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG ./app - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - - - name: Install kustomize - run: | - curl -sL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.4.1/kustomize_v5.4.1_linux_amd64.tar.gz | tar xz - sudo mv kustomize /usr/local/bin/kustomize - - - name: Update prod image tag - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ github.sha }} - run: | - cd k8s/overlays/prod - kustomize edit set image data-pipeline-app=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - - - name: Commit manifest change - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add k8s/overlays/prod/kustomization.yaml - git commit -m "deploy: image ${{ github.sha }}" || echo "No changes" - git push diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 2087bd5..0000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: CI - -on: - pull_request: - branches: [main] - paths: - - "app/**" - - ".github/workflows/ci.yaml" - push: - branches: [main] - paths: - - "app/**" - - ".github/workflows/ci.yaml" - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r app/requirements.txt - pip install pytest httpx - - - name: Run tests - working-directory: app - run: pytest -q - - - name: Docker build check - run: docker build -t data-pipeline-app-ci ./app diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml new file mode 100644 index 0000000..fb42d8c --- /dev/null +++ b/.github/workflows/cicd.yaml @@ -0,0 +1,62 @@ +name: CI/CD Pipeline with Canary Deployment + +on: + push: + branches: [ "main" ] + +env: + AWS_REGION: eu-west-1 + ECR_REPOSITORY: data-pipeline-app + KUBE_NAMESPACE: default + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::123456789012:role/GitHubActionsEKSRole # 생성한 IAM Role ARN 입력 + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + + - name: Setup Kubeconfig + run: | + aws eks update-kubeconfig --region ${{ env.AWS_REGION }} --name my-eks-cluster # 실제 클러스터 명으로 변경 + + - name: Install Kustomize + uses: imranismail/setup-kustomize@v2 + + - name: Deploy v2 App to EKS + env: + IMAGE_TAG: ${{ github.sha }} + run: | + cd k8s/overlays/v2 + kustomize edit set image data-pipeline-app=123456789012.dkr.ecr.eu-west-1.amazonaws.com/data-pipeline-app:$IMAGE_TAG + kustomize build . | kubectl apply -f - + kubectl rollout status deployment/data-pipeline-app-v2 -n ${{ env.KUBE_NAMESPACE }} --timeout=120s + + - name: Apply Canary Ingress (v1: 90%, v2: 10%) + env: + WEIGHT_V1: 90 + WEIGHT_V2: 10 + run: | + envsubst < k8s/ingress/ingress.yaml | kubectl apply -f - diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c996cac --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Terraform +.terraform/ +*.tfstate +*.tfstate.backup +*.tfvars +.terraform.lock.hcl + +# IDE +.idea/ +.vscode/ + +# Python +__pycache__/ +*.pyc diff --git a/k8s/argocd/app-prod.yaml b/k8s/argocd/app-prod.yaml index 634a02e..14f79da 100644 --- a/k8s/argocd/app-prod.yaml +++ b/k8s/argocd/app-prod.yaml @@ -6,8 +6,8 @@ metadata: spec: project: default source: - repoURL: https://github.com/yooseongjin527/asac_de2_infra_1st - targetRevision: cicd + repoURL: https://github.com/masondev1024/my-data-platform + targetRevision: main path: k8s/overlays/prod destination: server: https://kubernetes.default.svc diff --git a/k8s/base/cronjob.yaml b/k8s/base/cronjob.yaml index 2de1bdc..d6b13f8 100644 --- a/k8s/base/cronjob.yaml +++ b/k8s/base/cronjob.yaml @@ -10,11 +10,11 @@ spec: spec: containers: - name: draw-worker - image: 486053612615.dkr.ecr.eu-west-2.amazonaws.com/my-raffle-app:v1 + image: 486053612615.dkr.ecr.eu-west-1.amazonaws.com/my-raffle-app:v1 command: ["python", "draw_winner.py"] envFrom: - configMapRef: name: raffle-config - secretRef: name: raffle-secret - restartPolicy: OnFailure \ No newline at end of file + restartPolicy: OnFailure diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml deleted file mode 100644 index 20a9c19..0000000 --- a/k8s/base/deployment.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: data-pipeline-app -spec: - replicas: 2 - selector: - matchLabels: - app: data-pipeline-app - template: - metadata: - labels: - app: data-pipeline-app - spec: - containers: - - name: app-pipeline-app - image: data-pipeline-app - imagePullPolicy: Always - ports: - - containerPort: 8080 - resources: - requests: - cpu: "200m" - memory: "256Mi" - limits: - cpu: "500m" - memory: "512Mi" diff --git a/k8s/base/hpa.yaml b/k8s/base/hpa.yaml index 07d4137..6275ebe 100644 --- a/k8s/base/hpa.yaml +++ b/k8s/base/hpa.yaml @@ -2,32 +2,12 @@ apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: - name: raffle-hpa-v1 + name: data-pipeline-hpa spec: scaleTargetRef: apiVersion: apps/v1 - kind: Deployment - name: raffle-app-v1 - minReplicas: 2 - maxReplicas: 20 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 50 ---- -# V2 오토스케일러 -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: raffle-hpa-v2 -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: raffle-app-v2 + kind: Rollout + name: data-pipeline-app minReplicas: 2 maxReplicas: 20 metrics: diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml index f4d7302..a06ae12 100644 --- a/k8s/base/ingress.yaml +++ b/k8s/base/ingress.yaml @@ -1,16 +1,13 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: raffle-ingress + name: data-pipeline-ingress annotations: kubernetes.io/ingress.class: alb alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/target-type: ip - # 카나리아 가중치 설정 (현재 v1: 100%, v2: 0%) - alb.ingress.kubernetes.io/actions.canary-routing: > - {"type":"forward","forwardConfig":{"targetGroups":[{"serviceName":"raffle-svc-v1","servicePort":"80","weight":100},{"serviceName":"raffle-svc-v2","servicePort":"80","weight":0}]}} -spec: ingressClassName: alb +spec: rules: - http: paths: @@ -18,6 +15,5 @@ spec: pathType: Prefix backend: service: - name: canary-routing - port: - name: use-annotation + name: data-pipeline-svc-stable + port: { number: 80 } diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index d0941d0..2de705f 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -3,7 +3,8 @@ kind: Kustomization resources: - deployment.yaml - service.yaml -images: - - name: data-pipeline-app - newName: 123456789012.dkr.ecr.eu-west-1.amazonaws.com/data-pipeline-app - newTag: latest + - hpa.yaml +# images: +# - name: data-pipeline-app +# newName: 123456789012.dkr.ecr.eu-west-1.amazonaws.com/data-pipeline-app +# newTag: latest diff --git a/k8s/base/rollout.yaml b/k8s/base/rollout.yaml new file mode 100644 index 0000000..632c9a1 --- /dev/null +++ b/k8s/base/rollout.yaml @@ -0,0 +1,36 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: data-pipeline-rollout +spec: + replicas: 2 + strategy: + canary: + canaryService: data-pipeline-svc-canary + stableService: data-pipeline-svc-stable + trafficRouting: + alb: + ingress: data-pipeline-ingress + servicePort: 80 + steps: + - setWeight: 10 + - pause: { duration: 2m } + - setWeight: 50 + - pause: { duration: 5m } + selector: + matchLabels: + app: data-pipeline-app + template: + metadata: + labels: + app: data-pipeline-app + spec: + containers: + - name: app-container + image: 123456789012.dkr.ecr.eu-west-1.amazonaws.com/data-pipeline-app:latest + ports: + - containerPort: 8080 + resources: + requests: + cpu: "200m" + memory: "256Mi" diff --git a/k8s/base/service-canary.yaml b/k8s/base/service-canary.yaml new file mode 100644 index 0000000..8ff2eb3 --- /dev/null +++ b/k8s/base/service-canary.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +kind: Service +metadata: + name: data-pipeline-svc-canary +spec: + ports: [{ port: 80, targetPort: 8080 }] + selector: { app: data-pipeline-app } diff --git a/k8s/base/service-stable.yaml b/k8s/base/service-stable.yaml new file mode 100644 index 0000000..2d2ddfc --- /dev/null +++ b/k8s/base/service-stable.yaml @@ -0,0 +1,8 @@ +# service-stable.yaml +apiVersion: v1 +kind: Service +metadata: + name: data-pipeline-svc-stable +spec: + ports: [{ port: 80, targetPort: 8080 }] + selector: { app: data-pipeline-app } diff --git a/k8s/base/service.yaml b/k8s/base/service.yaml deleted file mode 100644 index 0f5baa8..0000000 --- a/k8s/base/service.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: data-pipeline-app -spec: - selector: - app: data-pipeline-app - ports: - - port: 80 - targetPort: 8080 - protocol: TCP diff --git a/k8s/overlays/prod/kustomization.yaml b/k8s/overlays/v1/kustomization.yaml similarity index 74% rename from k8s/overlays/prod/kustomization.yaml rename to k8s/overlays/v1/kustomization.yaml index a03caa0..b503bc5 100644 --- a/k8s/overlays/prod/kustomization.yaml +++ b/k8s/overlays/v1/kustomization.yaml @@ -2,7 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../base +nameSuffix: -v1 images: - name: data-pipeline-app newName: 123456789012.dkr.ecr.eu-west-1.amazonaws.com/data-pipeline-app - newTag: latest + newTag: v1 # 초기 배포 시 사용할 태그 diff --git a/k8s/overlays/v2/kustomization.yaml b/k8s/overlays/v2/kustomization.yaml new file mode 100644 index 0000000..2da0ee4 --- /dev/null +++ b/k8s/overlays/v2/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../base +nameSuffix: -v2 +images: + - name: data-pipeline-app + newName: 123456789012.dkr.ecr.eu-west-1.amazonaws.com/data-pipeline-app + newTag: v2 # CI/CD 파이프라인에 의해 덮어씌워짐 diff --git a/terraform/addons.tf b/terraform/addons.tf new file mode 100644 index 0000000..38af200 --- /dev/null +++ b/terraform/addons.tf @@ -0,0 +1,58 @@ +# 1. AWS Load Balancer Controller용 IAM Role (IRSA) +module "lb_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + role_name = "eks-alb-controller-role" + attach_load_balancer_controller_policy = true + oidc_providers = { + main = { + provider_arn = aws_iam_openid_connect_provider.github_actions.arn # 기존 OIDC 활용 + namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] + } + } +} + +# 2. ALB Controller 설치 (Helm) +resource "helm_release" "alb_controller" { + name = "aws-load-balancer-controller" + repository = "https://aws.github.io/eks-charts" + chart = "aws-load-balancer-controller" + namespace = "kube-system" + + set { + name = "clusterName" + value = aws_eks_cluster.main.name + } + + set { + name = "serviceAccount.create" + value = "true" + } + + set { + name = "serviceAccount.name" + value = "aws-load-balancer-controller" + } + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.lb_role.iam_role_arn + } +} + +# 3. Argo Rollouts 설치 +resource "helm_release" "argo_rollouts" { + name = "argo-rollouts" + repository = "https://argoproj.github.io/argo-helm" + chart = "argo-rollouts" + namespace = "argo-rollouts" + create_namespace = true +} + +# 4. Metrics Server (HPA 작동 필수) +resource "helm_release" "metrics_server" { + name = "metrics-server" + repository = "https://kubernetes-sigs.github.io/metrics-server/" + chart = "metrics-server" + namespace = "kube-system" +} diff --git a/terraform/karpenter.tf b/terraform/karpenter.tf new file mode 100644 index 0000000..b2551c6 --- /dev/null +++ b/terraform/karpenter.tf @@ -0,0 +1,39 @@ +# Karpenter Controller용 IAM Role +module "karpenter_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + role_name = "karpenter-controller-role" + attach_karpenter_controller_policy = true + karpenter_controller_cluster_name = aws_eks_cluster.main.name + oidc_providers = { + main = { + provider_arn = aws_iam_openid_connect_provider.github_actions.arn + namespace_service_accounts = ["karpenter:karpenter"] + } + } +} + +# Karpenter 설치 +resource "helm_release" "karpenter" { + namespace = "karpenter" + create_namespace = true + name = "karpenter" + repository = "https://charts.karpenter.sh" + chart = "karpenter" + version = "v0.32.1" + + set { + name = "settings.aws.clusterName" + value = aws_eks_cluster.main.name + } + + set { + name = "settings.aws.defaultInstanceProfile" + value = aws_iam_instance_profile.bastion.name + } + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.karpenter_role.iam_role_arn + } +} diff --git a/terraform/network.tf b/terraform/network.tf index 2da8000..fed60a9 100644 --- a/terraform/network.tf +++ b/terraform/network.tf @@ -38,7 +38,10 @@ resource "aws_subnet" "app_private_a" { cidr_block = cidrsubnet(var.vpc_cidr, 6, 1) availability_zone = data.aws_availability_zones.available.names[0] tags = { Name = "sbn-app-a" - "kubernetes.io/role/internal-elb" = "1" } + "kubernetes.io/role/internal-elb" = "1" + "karpenter.sh/discovery" = "data-engineer-cluster" + "kubernetes.io/cluster/data-engineer-cluster" = "shared" # ALB 및 기타 AWS 컨트롤러 연동을 위한 명시적 태그 + } } resource "aws_subnet" "app_private_b" { @@ -46,7 +49,10 @@ resource "aws_subnet" "app_private_b" { cidr_block = cidrsubnet(var.vpc_cidr, 6, 2) availability_zone = data.aws_availability_zones.available.names[1] tags = { Name = "sbn-app-b" - "kubernetes.io/role/internal-elb" = "1" } + "kubernetes.io/role/internal-elb" = "1" + "karpenter.sh/discovery" = "data-engineer-cluster" + "kubernetes.io/cluster/data-engineer-cluster" = "shared"} + } resource "aws_subnet" "db_private_a" { diff --git a/terraform/providers.tf b/terraform/providers.tf index 994f5dd..b467b30 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -1,20 +1,36 @@ terraform { required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" + aws = { source = "hashicorp/aws", version = "~> 5.0" } + helm = { source = "hashicorp/helm", version = "~> 2.0" } + kubernetes = { source = "hashicorp/kubernetes", version = "~> 2.0" } } +} +provider "aws" { + region = var.aws_region +} + +# EKS 생성 후 인증 정보를 가져오기 위한 데이터 소스 +data "aws_eks_cluster_auth" "cluster" { name = aws_eks_cluster.main.name } + +provider "kubernetes" { + host = aws_eks_cluster.main.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.main.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +provider "helm" { + kubernetes { + host = aws_eks_cluster.main.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.main.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token } - # [참고] backend_resources.tf를 먼저 apply 한 후, 생성된 S3 버킷 이름을 확인하고 주석을 해제하세요. + # [주의] 이 블록은 2단계에서 주석을 해제합니다. # backend "s3" { - # bucket = "data-engineer-tf-state-gx5de6" + # bucket = "data-engineer-tf-state-xxxxx" # backend.resource.tf에서 생성될 버킷명 # key = "global/s3/terraform.tfstate" - # region = "eu-west-1" # 단일 리전 + # region = "eu-west-1" # dynamodb_table = "data-engineer-tf-locks" # encrypt = true # } -} -provider "aws" { - region = var.aws_region } diff --git a/terraform/security.tf b/terraform/security.tf index b6ef52d..166dd1a 100644 --- a/terraform/security.tf +++ b/terraform/security.tf @@ -77,7 +77,6 @@ resource "aws_security_group" "rds" { } } -data "aws_region" "current" {} resource "aws_vpc_endpoint" "s3" { vpc_id = aws_vpc.main.id From 57c8195da58c7a20ceea3198379c5cb249e4c851 Mon Sep 17 00:00:00 2001 From: mason Date: Fri, 10 Apr 2026 22:23:43 +0900 Subject: [PATCH 2/2] add CI/CD function --- .github/workflows/.cicd.yml.swp | Bin 12288 -> 0 bytes .../workflows/build-and-update-manifest.yaml | 4 +- k8s/argocd/app-prod.yaml | 2 +- k8s/base/karpenter-config.yaml | 40 ++++++++++++++++++ k8s/base/kustomization.yaml | 1 + terraform/bastion_and_rds.tf | 40 +++++++++++++++--- terraform/eks_and_iam.tf | 36 +++++++++++++++- terraform/karpenter.tf | 2 +- terraform/providers.tf | 40 +++++++++--------- terraform/variables.tf | 2 +- 10 files changed, 134 insertions(+), 33 deletions(-) delete mode 100644 .github/workflows/.cicd.yml.swp create mode 100644 k8s/base/karpenter-config.yaml diff --git a/.github/workflows/.cicd.yml.swp b/.github/workflows/.cicd.yml.swp deleted file mode 100644 index d33ac77c05f7be69ce39a5b4bd62f81ff11868fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI%Jr05}7=Yp8ZZtZZpw9g1M=bb9oRgsG}r|$lIiC+J0^N zJzLX_y_zrdndt~sx_R?1_kkt?nS`hBL`h%7t+O!p-I6FFu0@~~TntyH(ujPS?!q{* z%)OT$&KpaeaNPM}mRMVF>zbFqFFlS$1Q4hs&=%8)W9M7OqoLmKEi3IN7y$$jKmY** z5I_I{1nMo2Mw+;(6k4A&v>$!Fv*h*&Abgn? g|M&f0jGIkE_6Q(=00IagfB*srAbgw}3m$hQIsgCw diff --git a/.github/workflows/build-and-update-manifest.yaml b/.github/workflows/build-and-update-manifest.yaml index df6b9f2..ef59c46 100644 --- a/.github/workflows/build-and-update-manifest.yaml +++ b/.github/workflows/build-and-update-manifest.yaml @@ -2,7 +2,7 @@ name: Build and Update Manifest on: push: - branches: [ "main" ] + branches: [ "cicd_test" ] paths: [ "app/**", "Dockerfile", "k8s/**" ] jobs: @@ -44,4 +44,4 @@ jobs: git config --global user.name "github-actions" git add k8s/overlays/prod/kustomization.yaml git commit -m "Update image tag to ${{ github.sha }} [skip ci]" - git push + git push origin cicd_test diff --git a/k8s/argocd/app-prod.yaml b/k8s/argocd/app-prod.yaml index 14f79da..8043282 100644 --- a/k8s/argocd/app-prod.yaml +++ b/k8s/argocd/app-prod.yaml @@ -7,7 +7,7 @@ spec: project: default source: repoURL: https://github.com/masondev1024/my-data-platform - targetRevision: main + targetRevision: cicd_test path: k8s/overlays/prod destination: server: https://kubernetes.default.svc diff --git a/k8s/base/karpenter-config.yaml b/k8s/base/karpenter-config.yaml new file mode 100644 index 0000000..1d55a63 --- /dev/null +++ b/k8s/base/karpenter-config.yaml @@ -0,0 +1,40 @@ +# 1. 어떤 사양의 EC2를 띄울 것인가? (인프라 설정) +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2023 # 최신 Amazon Linux 2023 사용 + role: EKSNodeRole # 테라폼에서 생성한 노드 역할 이름과 일치해야 함 + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "data-engineer-cluster" + securityGroupSelectorTerms: + - tags: + kubernetes.io/cluster/data-engineer-cluster: "owned" +--- +# 2. 어떤 조건에서 노드를 늘릴 것인가? (스케일링 정책) +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] # 비용 절감을 원하면 spot 포함 + - key: node.kubernetes.io/instance-type + operator: In + values: ["t3.medium", "c5.large"] # 필요한 인스턴스 사양 정의 + nodeClassRef: + name: default + limits: + cpu: 100 # 전체 클러스터의 최대 CPU 제한 + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h # 30일 후 노드 교체 (보안 업데이트 등 방어용) diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index 2de705f..d311bc3 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -4,6 +4,7 @@ resources: - deployment.yaml - service.yaml - hpa.yaml + - karpenter-config.yaml # images: # - name: data-pipeline-app # newName: 123456789012.dkr.ecr.eu-west-1.amazonaws.com/data-pipeline-app diff --git a/terraform/bastion_and_rds.tf b/terraform/bastion_and_rds.tf index 1d31032..31d12e1 100644 --- a/terraform/bastion_and_rds.tf +++ b/terraform/bastion_and_rds.tf @@ -1,23 +1,47 @@ resource "aws_iam_role" "bastion" { - name = "CommandServerRole" + name = "CommandServerRole-v2" assume_role_policy = jsonencode({ Version = "2012-10-17" Statement = [{ Action = "sts:AssumeRole", Effect = "Allow", Principal = { Service = "ec2.amazonaws.com" } }] }) } - -resource "aws_iam_role_policy_attachment" "bastion_admin" { +# [수정] AdministratorAccess 삭제 및 최소 권한 정책 연결 +# 1. SSM을 통한 접속 권한 (필수) +resource "aws_iam_role_policy_attachment" "bastion_ssm" { + role = aws_iam_role.bastion.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} +# 2. EKS 관리 및 리소스 조회 권한 (Bastion에서 kubectl 사용을 위해 필요) +resource "aws_iam_role_policy_attachment" "bastion_eks_read" { role = aws_iam_role.bastion.name - policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess" + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" } -resource "aws_iam_role_policy_attachment" "bastion_ssm" { +# 3. EC2 리소스 조회 권한 (Karpenter 노드 확인 등) +resource "aws_iam_role_policy_attachment" "bastion_ec2_read" { role = aws_iam_role.bastion.name - policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess" } +resource "aws_iam_role_policy" "bastion_eks_describe" { + name = "BastionEKSDescribePolicy" + role = aws_iam_role.bastion.id + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "eks:DescribeCluster", + "eks:ListClusters" + ] + Resource = "*" # 특정 클러스터 ARN으로 제한하는 것이 더 안전합니다. + } + ] + }) +} resource "aws_iam_instance_profile" "bastion" { - name = "CommandServerInstanceProfile" + name = "CommandServerInstanceProfile-v2" role = aws_iam_role.bastion.name } @@ -31,6 +55,8 @@ resource "aws_instance" "bastion" { subnet_id = aws_subnet.public_a.id vpc_security_group_ids = [aws_security_group.command_server.id] iam_instance_profile = aws_iam_instance_profile.bastion.name + # 퍼블릭 IP 할당 명시 (이미 서브넷 설정에 되어있지만 가독성을 위해 추가) + associate_public_ip_address = true tags = { Name = "bastion-host" } } diff --git a/terraform/eks_and_iam.tf b/terraform/eks_and_iam.tf index 31cdef2..fae0192 100644 --- a/terraform/eks_and_iam.tf +++ b/terraform/eks_and_iam.tf @@ -1,5 +1,5 @@ resource "aws_iam_role" "eks_cluster" { - name = "EKSClusterRole" + name = "EKSClusterRole-v2" assume_role_policy = jsonencode({ Version = "2012-10-17" Statement = [{ @@ -16,7 +16,7 @@ resource "aws_iam_role_policy_attachment" "eks_cluster_policy" { } resource "aws_iam_role" "eks_node" { - name = "EKSNodeRole" + name = "EKSNodeRole-v2" assume_role_policy = jsonencode({ Version = "2012-10-17" Statement = [{ @@ -71,3 +71,35 @@ resource "aws_eks_node_group" "main" { aws_iam_role_policy_attachment.ecr_policy ] } + +# 1. Bastion의 IAM 역할에 DescribeCluster 권한 부여 +resource "aws_iam_role_policy" "bastion_eks_access" { + name = "BastionEKSAccess" + role = aws_iam_role.bastion.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["eks:DescribeCluster", "eks:ListClusters"] + Resource = "*" + }] + }) +} + +# 2. EKS 클러스터 내부에서 Bastion 역할을 관리자로 등록 (Access Entry) +resource "aws_eks_access_entry" "bastion" { + cluster_name = aws_eks_cluster.main.name + principal_arn = aws_iam_role.bastion.arn + type = "STANDARD" +} + +resource "aws_eks_access_policy_association" "bastion_admin" { + cluster_name = aws_eks_cluster.main.name + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + principal_arn = aws_iam_role.bastion.arn + + access_scope { + type = "cluster" + } +} diff --git a/terraform/karpenter.tf b/terraform/karpenter.tf index b2551c6..ab70ece 100644 --- a/terraform/karpenter.tf +++ b/terraform/karpenter.tf @@ -18,7 +18,7 @@ resource "helm_release" "karpenter" { namespace = "karpenter" create_namespace = true name = "karpenter" - repository = "https://charts.karpenter.sh" + repository = "oci://public.ecr.aws/karpenter" chart = "karpenter" version = "v0.32.1" diff --git a/terraform/providers.tf b/terraform/providers.tf index b467b30..4d28d00 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -1,36 +1,38 @@ terraform { required_providers { - aws = { source = "hashicorp/aws", version = "~> 5.0" } + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } helm = { source = "hashicorp/helm", version = "~> 2.0" } kubernetes = { source = "hashicorp/kubernetes", version = "~> 2.0" } - } + } + + # backend 블록은 반드시 terraform 블록 '안'에 있어야 합니다. + # 현재는 apply -target 중이므로 주석 처리가 되어 있어야 합니다. + # backend "s3" { + # bucket = "data-engineer-tf-state-xxxxx" + # key = "global/s3/terraform.tfstate" + # region = "eu-west-1" + # dynamodb_table = "data-engineer-tf-locks" + # encrypt = true + # } } + provider "aws" { region = var.aws_region } - -# EKS 생성 후 인증 정보를 가져오기 위한 데이터 소스 -data "aws_eks_cluster_auth" "cluster" { name = aws_eks_cluster.main.name } - -provider "kubernetes" { - host = aws_eks_cluster.main.endpoint - cluster_ca_certificate = base64decode(aws_eks_cluster.main.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.cluster.token +# EKS 클러스터 인증 정보를 가져오기 위한 데이터 소스 선언 +data "aws_eks_cluster_auth" "cluster" { + name = aws_eks_cluster.main.name } +# helm provider 내부에는 backend가 들어갈 수 없습니다. provider "helm" { kubernetes { host = aws_eks_cluster.main.endpoint cluster_ca_certificate = base64decode(aws_eks_cluster.main.certificate_authority[0].data) token = data.aws_eks_cluster_auth.cluster.token } - # [주의] 이 블록은 2단계에서 주석을 해제합니다. - # backend "s3" { - # bucket = "data-engineer-tf-state-xxxxx" # backend.resource.tf에서 생성될 버킷명 - # key = "global/s3/terraform.tfstate" - # region = "eu-west-1" - # dynamodb_table = "data-engineer-tf-locks" - # encrypt = true - # } - } + diff --git a/terraform/variables.tf b/terraform/variables.tf index c68014b..f278d03 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -30,5 +30,5 @@ variable "github_repo" { variable "github_branch" { description = "GitHub branch allowed to assume the OIDC role" type = string - default = "main" + default = "cicd_test" }