EKS + Terraform + Argo CD + Helm: deployment architecture

module "vpc" {
  source  = "terraform-aws-modules/vpc/aws"
  name    = "eks-gitops-demo"
  cidr    = "10.0.0.0/16"
  azs             = ["ap-southeast-2a", "ap-southeast-2b"]
  public_subnets  = ["10.0.0.0/20", "10.0.16.0/20"]
  private_subnets = ["10.0.32.0/20", "10.0.48.0/20"]
  enable_nat_gateway      = false           # demo: nodes in public subnets, saves ~US$45/month
  map_public_ip_on_launch = true
  public_subnet_tags      = { "kubernetes.io/role/elb" = 1 }   # where the ALB may go
}

module "eks" {
  source  = "terraform-aws-modules/eks/aws"
  version = "~> 21.0"

  name               = "eks-gitops-demo"
  kubernetes_version = "1.37"
  endpoint_public_access       = true
  endpoint_public_access_cidrs = [var.admin_cidr]          # only your IP can reach the API
  enable_cluster_creator_admin_permissions = true          # access entry for the admin running Terraform

  vpc_id     = module.vpc.vpc_id
  subnet_ids = module.vpc.public_subnets

  addons = {                                              # EKS managed add-ons (AWS API objects)
    vpc-cni                = { before_compute = true }
    eks-pod-identity-agent = { before_compute = true }
    coredns                = {}
    kube-proxy             = {}
  }

  eks_managed_node_groups = {
    default = {
      ami_type       = "AL2023_ARM_64_STANDARD"
      instance_types = ["t4g.large"]
      capacity_type  = "ON_DEMAND"                         # idea: SPOT
      min_size       = 2
      max_size       = 3
      desired_size   = 2
    }
  }
}

module "lbc_pod_identity" {
  source = "terraform-aws-modules/eks-pod-identity/aws"
  name   = "aws-lbc"
  attach_aws_lb_controller_policy = true
  associations = {
    main = {
      cluster_name    = module.eks.cluster_name
      namespace       = "kube-system"
      service_account = "aws-load-balancer-controller"
    }
  }
}
# Same pattern for external-dns (scoped to the hosted zone)
# and external-secrets (scoped to the demo/* secrets).

With Pod Identity, the service account needs no role annotation: AWS maps namespace + service account to the role. That is why the Helm values for the controllers stay free of ARNs.

resource "helm_release" "argocd" {
  name             = "argocd"
  namespace        = "argocd"
  create_namespace = true
  repository       = "https://argoproj.github.io/argo-helm"
  chart            = "argo-cd"
  version          = var.argocd_chart_version
}

# The bridge: Terraform outputs handed to Argo CD as a cluster Secret.
resource "kubernetes_secret_v1" "in_cluster" {
  metadata {
    name      = "in-cluster"
    namespace = "argocd"
    labels = {
      "argocd.argoproj.io/secret-type"     = "cluster"
      enable_aws_load_balancer_controller = "true"
      enable_external_dns                 = "true"
      enable_external_secrets             = "true"
    }
    annotations = {
      cluster_name = module.eks.cluster_name
      vpc_id       = module.vpc.vpc_id
      aws_region   = var.region
      domain       = var.domain
    }
  }
  data       = { name = "in-cluster", server = "https://kubernetes.default.svc" }
  depends_on = [helm_release.argocd]
}

# Root Application via the argocd-apps chart: no Argo CRDs needed at plan time.
resource "helm_release" "root" {
  name       = "root"
  namespace  = "argocd"
  repository = "https://argoproj.github.io/argo-helm"
  chart      = "argocd-apps"
  values     = [file("${path.module}/root-app.yaml")]
  depends_on = [helm_release.argocd]
}

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata: { name: aws-load-balancer-controller, namespace: argocd }
spec:
  goTemplate: true
  generators:
    - clusters:
        selector: { matchLabels: { enable_aws_load_balancer_controller: "true" } }
  template:
    metadata: { name: "aws-lbc-{{.name}}", annotations: { argocd.argoproj.io/sync-wave: "0" } }
    spec:
      project: default
      source:
        repoURL: https://aws.github.io/eks-charts
        chart: aws-load-balancer-controller
        targetRevision: "<pinned version>"
        helm:
          valuesObject:
            clusterName: "{{.metadata.annotations.cluster_name}}"
            vpcId: "{{.metadata.annotations.vpc_id}}"
            region: "{{.metadata.annotations.aws_region}}"
            serviceAccount: { name: aws-load-balancer-controller }   # matches the Pod Identity association
      destination: { server: "{{.server}}", namespace: kube-system }
      syncPolicy:
        automated: { prune: true, selfHeal: true }

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata: { name: demo-api-dev, namespace: argocd }
spec:
  project: default
  sources:
    - repoURL: <account>.dkr.ecr.ap-southeast-2.amazonaws.com/charts   # Helm repo = ECR (OCI)
      chart: demo-api
      targetRevision: 0.1.42                                           # bumped by the CI bot PR
      helm:
        valueFiles: [$values/values/dev/demo-api.yaml]
    - repoURL: https://github.com/<you>/eks-gitops-demo-gitops          # values come from Git
      targetRevision: main
      ref: values
  destination: { server: https://kubernetes.default.svc, namespace: demo }
  syncPolicy:
    automated: { prune: true, selfHeal: true }
    syncOptions: [CreateNamespace=true]

image:
  repository: <account>.dkr.ecr.ap-southeast-2.amazonaws.com/demo-api
  tag: 3f9c2e1                         # git SHA, written by the CI bot; ECR tags are immutable
config:
  oidcIssuer: https://auth.frankbluemind.com/realms/demo
  oidcAudience: demo-api
  env: dev
ingress:
  enabled: true
  className: alb                       # handled by the AWS Load Balancer Controller
  host: app.frankbluemind.com                # external-dns creates the record, ACM cert matched by host
  annotations:
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/target-type: ip                 # ALB → pod IPs directly (VPC CNI)
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
    alb.ingress.kubernetes.io/group.name: demo                # app + auth share one ALB

name: release
on:
  push:
    branches: [main]
permissions:
  id-token: write          # GitHub OIDC → AWS role, no stored keys
  contents: read
jobs:
  release:
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@v4
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ vars.ECR_PUSH_ROLE_ARN }}
          aws-region: ap-southeast-2
      - uses: aws-actions/amazon-ecr-login@v2
        id: ecr
      - run: make test
      - run: |
          IMG=${{ steps.ecr.outputs.registry }}/demo-api:${GITHUB_SHA::7}
          docker build -t "$IMG" . && docker push "$IMG"
      - run: |
          helm package deploy/helm/demo-api --version "0.1.${GITHUB_RUN_NUMBER}" --app-version "${GITHUB_SHA::7}"
          helm push demo-api-*.tgz oci://${{ steps.ecr.outputs.registry }}/charts
      - run: scripts/promote.sh dev "0.1.${GITHUB_RUN_NUMBER}" "${GITHUB_SHA::7}"   # opens the gitops PR
        env:
          GH_TOKEN: ${{ secrets.GITOPS_BOT_TOKEN }}                            # write access to the gitops repo only

# gitops charts/demo-secrets: two Secrets from the same source, refreshed at different rates
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata: { name: demo-api-partner-file, namespace: demo }
spec:
  refreshInterval: 30s
  secretStoreRef: { kind: ClusterSecretStore, name: aws-secrets-manager }
  target: { name: demo-api-partner-file }        # K8s Secret created and owned by ESO
  data:
    - secretKey: api_key
      remoteRef: { key: demo/dev/demo-api/partner-api, property: api_key }
# demo-api-partner-env: the same, refreshInterval 3m
---
# demo-api Deployment (excerpt, rendered from the Helm chart, values partnerKey.*)
metadata:
  annotations:
    secret.reloader.stakater.com/reload: "demo-api-db,demo-api-payments,demo-api-partner-env"   # NOT the file Secret
spec:
  template:
    spec:
      securityContext: { fsGroup: 65532 }      # mounted files belong to the app's group
      containers:
        - name: api
          env:
            - name: PARTNER_API_KEY              # fixed until the pod is replaced
              valueFrom: { secretKeyRef: { name: demo-api-partner-env, key: api_key } }
            - name: PARTNER_KEY_FILE             # re-read in place
              value: /var/run/secrets/partner/api_key
          volumeMounts:                          # whole volume, never subPath (subPath never updates)
            - { name: partner-key, mountPath: /var/run/secrets/partner, readOnly: true }
      volumes:
        - name: partner-key
          secret: { secretName: demo-api-partner-file, defaultMode: 0440 }

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata: { name: demo-api }
spec:
  scaleTargetRef: { apiVersion: apps/v1, kind: Deployment, name: demo-api }
  minReplicas: 2
  maxReplicas: 6
  metrics:
    - type: Resource
      resource: { name: cpu, target: { type: Utilization, averageUtilization: 60 } }
  behavior:
    scaleDown: { stabilizationWindowSeconds: 120 }     # default 300 s; shortened for the demo
---
# in apps/dev/demo-api.yaml (Argo Application)
spec:
  ignoreDifferences:
    - { group: apps, kind: Deployment, jsonPointers: [/spec/replicas] }