Merge pull request #5063 from pkprzekwas/prow-build-canary-cluster

prow-build-canary-cluster: provisioning scripts

infra/aws/terraform/prow-build-cluster/Makefile

@@ -0,0 +1,58 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TF ?= terraform
+ASSUME_ROLE ?= true
+
+# Valid values are: canary, prod
+WORKSPACE_NAME ?= canary
+
+.PHONY: workspace-select
+workspace-select:
+	$(TF) workspace select $(WORKSPACE_NAME)
+
+.PHONY: init
+init:
+	$(TF) $@
+
+.PHONY: plan
+plan: workspace-select
+	$(TF) $@ \
+		-var-file=./terraform.$(WORKSPACE_NAME).tfvars \
+		-var="assume_role=$(ASSUME_ROLE)"
+
+.PHONY: apply
+apply: workspace-select
+	$(TF) $@ \
+		-var-file=./terraform.$(WORKSPACE_NAME).tfvars \
+		-var="assume_role=$(ASSUME_ROLE)"
+
+.PHONY: destroy
+destroy: workspace-select
+	$(TF) $@ \
+		-var-file=./terraform.$(WORKSPACE_NAME).tfvars \
+		-var="assume_role=$(ASSUME_ROLE)"
+
+.PHONY: fmt
+fmt:
+	$(TF) $@
+
+.PHONY: output
+output:
+	$(TF) $@
+
+.PHONY: clean
+clean:
+	rm -rf ./.terraform
+

infra/aws/terraform/prow-build-cluster/README.md

@@ -0,0 +1,109 @@
+# Provisioninig EKS clusters
+
+## Prod vs Canary
+
+These scripts support provisioning two types of EKS clusters. One is meant for hosting prow jobs
+on production and the other one is for testing infrastructure changes before promoting them to
+production.
+
+Here are some differences between canary and production setups:
+* cluster name,
+* cluster admin IAM role name,
+* secrets-manager IAM policy name,
+* canary is missing k8s prow OIDC provider and corresponding role,
+* subnet setup is different,
+* instance type and autoscaling paramethers (mainly for saving),
+
+## Provisioning Cluster
+
+Running installation from scratch is different than consecutive invocations of Terraform.
+First run creates a role that can be later assumed by other users. Becasue of that additional
+variable has to be set:
+
+```bash
+# For provisioning Prod:
+export WORKSPACE_NAME=prod
+# For provisioning Canary:
+export WORKSPACE_NAME=canary
+
+# Just making sure we don't have state cached locally.
+ASSUME_ROLE=false make init
+ASSUME_ROLE=false make apply
+```
+
+Once the infrastructure is provisioned, next step is RBAC setup:
+
+```bash
+# Fetch & update kubeconfig.
+# For Prod:
+aws eks update-kubeconfig --region us-east-2 --name prow-build-cluster
+# For Canary:
+aws eks update-kubeconfig --region us-east-2 --name prow-build-canary-cluster
+
+# create cluster role bindings
+kubectl apply -f ./resources/rbac
+```
+
+Lastly, run Terraform script again without additinal variable. This time, it will implicitly assume
+previously created role and provision resources on top of EKS cluster.
+
+```bash
+make apply
+```
+
+From here, all consecutive runs should be possible with command from above.
+
+## Using cluster
+
+### Fetch kubeconfig
+
+```bash
+# Prod:
+aws eks update-kubeconfig --region us-east-2 --name prow-build-cluster
+# Canary:
+aws eks update-kubeconfig --region us-east-2 --name prow-build-canary-cluster
+```
+
+### Open kubeconfig and add assume role argument
+
+For Prod:
+```yaml
+args:
+  - --region
+  - us-east-2
+  - eks
+  - get-token
+  - --cluster-name
+  - prow-build-cluster
+  - --role-arn
+  - arn:aws:iam::468814281478:role/Prow-Cluster-Admin
+```
+
+For Canary:
+```yaml
+args:
+  - --region
+  - us-east-2
+  - eks
+  - get-token
+  - --cluster-name
+  - prow-build-canary-cluster
+  - --role-arn
+  - arn:aws:iam::468814281478:role/canary-Prow-Cluster-Admin
+```
+
+## Removing cluster
+
+Same as for installation, cluster removal requires running Terraform twice.
+**IMPORTANT**: It's possible only for users with assigned `AdministratorAccess` policy.
+
+```bash
+export WORKSPACE_NAME= # choose between canary/prod
+
+# First remove resources running on the cluster and IAM role. This fails once assumed role gets deleted.
+make destroy
+
+# Clean up the rest. 
+ASSUME_ROLE=false make destroy
+```
+

infra/aws/terraform/prow-build-cluster/eks.tf

@@ -18,6 +18,31 @@ limitations under the License.
 # EKS Cluster
 ###############################################
 
+locals {
+  aws_auth_roles = concat(
+    terraform.workspace == "prod" ? [
+      # Allow access to the Prow-EKS-Admin IAM role (used by Prow directly).
+      {
+        "rolearn"  = aws_iam_role.eks_admin[0].arn
+        "username" = "eks-admin"
+        "groups" = [
+          "eks-prow-cluster-admin"
+        ]
+      }
+    ] : [],
+    [
+      # Allow access to the Prow-Cluster-Admin IAM role (used with assume role with other IAM accounts).
+      {
+        "rolearn"  = aws_iam_role.iam_cluster_admin.arn
+        "username" = "eks-cluster-admin"
+        "groups" = [
+          "eks-cluster-admin"
+        ]
+      }
+    ]
+  )
+}
+
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
   version = "19.10.0"
@@ -31,24 +56,8 @@ module "eks" {
   manage_aws_auth_configmap = true
 
   # Configure aws-auth
-  aws_auth_roles = [
-    # Allow access to the Prow-EKS-Admin IAM role (used by Prow directly).
-    {
-      "rolearn"  = aws_iam_role.eks_admin.arn
-      "username" = "eks-admin"
-      "groups" = [
-        "eks-prow-cluster-admin"
-      ]
-    },
-    # Allow access to the Prow-Cluster-Admin IAM role (used with assume role with other IAM accounts).
-    {
-      "rolearn"  = aws_iam_role.iam_cluster_admin.arn
-      "username" = "eks-cluster-admin"
-      "groups" = [
-        "eks-cluster-admin"
-      ]
-    },
-  ]
+  aws_auth_roles = local.aws_auth_roles
+
   # Allow EKS access to the root account.
   aws_auth_users = [
     {

infra/aws/terraform/prow-build-cluster/iam.tf

@@ -26,8 +26,8 @@ data "aws_iam_user" "user_pprzekwa" {
 }
 
 resource "aws_iam_role" "iam_cluster_admin" {
-  name        = "Prow-Cluster-Admin"
-  description = "IAM role used to delegate access to prow-build-cluster"
+  name        = "${local.canary_prefix}Prow-Cluster-Admin"
+  description = "IAM role used to delegate access to ${local.canary_prefix}prow-build-cluster"
 
   assume_role_policy = jsonencode({
     Version = "2012-10-17"