SmoothGlue Cluster IAC

Overview

This project contains the IaC to deploy an RKE2 Based K8s cluster. It is intended to be executed by the GitLab-CI pipeline.

Requirements

Name	Version
terraform	>= 0.14
archive	~> 2.4.0
aws	<= 5.22.0
local	~> 2.4.0
null	~> 3.2.2
postgresql	~> 1.21.0
random	~> 3.6.0
sops	~> 1.0.0
template	~> 2.2.0
time	~> 0.9.0
tls	~> 4.0.5

Providers

Name	Version
aws	5.22.0
local	2.4.1
null	3.2.2
random	3.6.1
tls	4.0.5

Modules

Name	Source	Version
app_clb	../modules/loadbalancer	n/a
app_nlb	terraform-aws-modules/alb/aws	~> 9.0.0
log_bucket_protected	../modules/structsure_s3_protected	n/a
log_bucket_unprotected	../modules/structsure_s3	n/a
rke2_agents	../modules/rke2-aws-tf/modules/agent-nodepool	n/a
rke2_cp	../modules/rke2-aws-tf	n/a
sso_clb	../modules/loadbalancer	n/a
sso_nlb	terraform-aws-modules/alb/aws	~> 9.0.0
userdata_s3	../modules/structsure_s3	n/a
zarf_registry_s3	terraform-aws-modules/s3-bucket/aws	~> 3.0

Resources

Name	Type
aws_autoscaling_attachment.app-http	resource
aws_autoscaling_attachment.app-https	resource
aws_autoscaling_attachment.sso-http	resource
aws_autoscaling_attachment.sso-https	resource
aws_iam_policy.allow_ebs	resource
aws_iam_policy.allow_kms	resource
aws_iam_policy.allow_s3_etcd_snapshots	resource
aws_iam_policy.allow_s3_userdata	resource
aws_iam_policy.allow_s3_zarf	resource
aws_iam_role_policy_attachment.allow_s3_etcd_snapshots	resource
aws_iam_role_policy_attachment.cp_allow_ebs	resource
aws_iam_role_policy_attachment.cp_allow_kms	resource
aws_iam_role_policy_attachment.cp_allow_s3_userdata	resource
aws_iam_role_policy_attachment.cp_allow_s3_zarf	resource
aws_iam_role_policy_attachment.cp_allow_ssm_cloudwatch	resource
aws_iam_role_policy_attachment.cp_allow_ssm_core	resource
aws_iam_role_policy_attachment.cp_collab_allow_efs	resource
aws_iam_role_policy_attachment.va_allow_ebs	resource
aws_iam_role_policy_attachment.va_allow_kms	resource
aws_iam_role_policy_attachment.va_allow_s3_userdata	resource
aws_iam_role_policy_attachment.va_allow_s3_zarf	resource
aws_iam_role_policy_attachment.va_allow_ssm_cloudwatch	resource
aws_iam_role_policy_attachment.va_allow_ssm_core	resource
aws_iam_role_policy_attachment.va_collab_allow_efs	resource
aws_key_pair.ssh_public_key	resource
aws_s3_bucket_logging.rke2-s3-logging	resource
aws_s3_bucket_versioning.rke2-s3-versioning	resource
aws_s3_object.userdata_files	resource
aws_security_group_rule.loadbalancer_ingress	resource
aws_security_group_rule.rke2_agent_ssh_rule	resource
aws_security_group_rule.rke2_cp_ssh_rule	resource
local_file.bigbang_values	resource
local_file.zarf_ca_bundle	resource
local_file.zarf_init_config	resource
null_resource.agent_instance_name_length_checker	resource
null_resource.cp_instance_name_length_checker	resource
random_password.zarf_registry_pull_password	resource
random_shuffle.cp_lb_subnets	resource
random_shuffle.va_lb_subnets	resource
random_shuffle.va_nlb_subnets	resource
tls_private_key.ssh	resource
aws_ami.rke2	data source
aws_iam_policy_document.allow_ebs	data source
aws_iam_policy_document.allow_kms	data source
aws_iam_policy_document.allow_s3_etcd_snapshots	data source
aws_iam_policy_document.allow_s3_userdata	data source
aws_iam_policy_document.allow_s3_zarf	data source
aws_partition.current	data source
aws_region.current	data source
aws_s3_object.kubeconfig	data source
aws_service.s3	data source
aws_vpc.vpc	data source

Inputs

Name	Description	Type	Default	Required
additional_images	A list of S3 URIs. Enables downloading of image artifacts to nodes	list(string)	[]	no
alb_enabled	Set to true if using an ALB to expose the public ingress gateway. Sets the hosts to *	bool	false	no
allowed_ingress_cidr_blocks	Allows ingress from these cidr blocks.	list(string)	[]	no
ami_filters	List of AMI filters used to select the AMI	list(object({ name = string values = list(string) }))	[]	no
ami_id	AMI ID to be used for both control plane and agent nodes, unless overridden	string	""	no
ami_most_recent	Select the most recent version of the AMI	bool	true	no
ami_owners	List of AWS account IDs used in AMI lookup filter	list(string)	[ "241559654725", "171179903432" ]	no
application_nlb_enable	Toggles the creation of the Application Network Load Balancer off/on	bool	true	no
aws_region	AWS Region used to configure the AWS provider	string	n/a	yes
bigbang_values_filename	Filename of the locally-created big bang values include	string	"bigbang-values-rke2.yaml"	no
blackhole_github	Configure CoreDNS to blackhole api.github.com	bool	false	no
block_public_access	If true, a public access block will be created which disallows public access to the bucket.	bool	true	no
bucket_force_destroy	Bool that allows S3 buckets to be destroyed when not empty	bool	false	no
clamav_notification_email_address	Name of the email address to receive clamav detections	string	n/a	yes
cloud_config	Object that enables and determines the contents of the cloud-config file	object({ enabled = string value = string })	{ "enabled": false, "value": "" }	no
cluster_agent_iam_role	IAM role for the agents to use	string	"InstanceOpsRole"	no
cluster_cp_iam_role	IAM role for the Control Plane to use	string	"InstanceOpsRole"	no
cluster_extra_tags	Additional tags to add to all cluster nodes	map(string)	{}	no
common_userdata_variables	Extra environmental variables that need to be passed between userdata scripts.	list(object({ name = string value = string }))	[]	no
compatibility_mode	If enabled, this flag disables some AWS features which are not available in all AWS partitions/regions.	bool	true	no
config_output_dir	Path to directory where local config files should be output	string	"."	no
cp_allow_ssh	Toggles SSH security group rule for structsure Control Plane nodes on or off	bool	true	no
cp_allowed_cidrs	Server pool security group allowed cidr ranges	list(string)	[ "0.0.0.0/0" ]	no
cp_ami	Control plane AMI ID, overrides the ami_id variable	string	null	no
cp_bdm_delete_on_term	Control Plane block device mapping delete on termination setting	bool	true	no
cp_bdm_device_name	Name of root block device mapping	string	"/dev/xvda"	no
cp_bdm_encrypted	Control Plane block device mapping encryption setting	bool	true	no
cp_bdm_size	Control Plane block device mapping size	number	40	no
cp_bdm_type	Control Plane block device mapping type	string	"gp2"	no
cp_cz_elb	Toggle between controlplane cross zone load balancing	bool	true	no
cp_download	Toggle best effort download of rke2 dependencies (rke2 and aws cli), if disabled, dependencies are assumed to exist in $PATH	bool	false	no
cp_ebs_kms_key_id	AWS KMS key ID for ebs related operations.	string	n/a	yes
cp_extra_block_device_mapping	List of objects to configure addional disks	list(map(string))	[]	no
cp_extra_security_group_ids	List of additional security group IDs to attach to control plane nodes	list(string)	[]	no
cp_extra_tags	Additional tags to add to control plane nodes	map(string)	{}	no
cp_instance_type	Server pool instance type	string	"t3a.large"	no
cp_internal_elb	Toggle between public or private control plane load balancer	bool	true	no
cp_lb_ha	Need control plane load balancers to have high availability (multi-az)	bool	true	no
cp_rke2_config	Server pool additional configuration passed as rke2 config file, see here for full list of options	string	"disable:\n - \"rke2-ingress-nginx\"\nkubelet-arg:\n - \"cloud-provider=external\"\n - \"protect-kernel-defaults=true\"\n - \"streaming-connection-idle-timeout=5m\"\n - \"tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\"\nkube-apiserver-arg:\n - \"enable-admission-plugins=NodeRestriction\"\n - \"request-timeout=60s\"\n - \"audit-log-mode=blocking-strict\"\n - \"tls-min-version=VersionTLS12\"\n - \"tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\"\n - \"anonymous-auth=false\"\n - \"authorization-mode=RBAC,Node\"\nkube-scheduler-arg:\n - \"tls-min-version=VersionTLS12\"\n - \"tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\"\n - \"bind-address=0.0.0.0\"\nkube-controller-manager-arg:\n - \"cloud-provider=external\"\n - \"configure-cloud-routes=false\"\n - \"tls-min-version=VersionTLS12\"\n - \"tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\"\n - \"bind-address=0.0.0.0\"\nkube-proxy-arg:\n - \"metrics-bind-address=0.0.0.0\"\npod-security-admission-config-file: /etc/rancher/rke2/rke2-pss-custom.yaml\nprofile: cis-1.23\nnode-taint:\n # work around https://github.com/rancher/rke2/issues/508n - \"node-role.kubernetes.io/master:PreferNoSchedule\"\nwrite-kubeconfig-mode: \"0640\"\nsecrets-encryption: true\nselinux: true\netcd-expose-metrics: true\ndisable-cloud-controller: true\n"	no
cp_servers	Number of servers to create	number	n/a	yes
cp_spot	Toggle spot requests for server pool	bool	false	no
cp_ssh_cidr_blocks	CIDR block allowed ssh access to structsure Control Plane nodes	list(string)	[ "10.0.0.0/8" ]	no
cp_subnets	List of subnet IDs to create resources in	list(string)	n/a	yes
cp_userdata_variables	Extra environmental variables specific to control plane that need to be passed between userdata scripts.	list(object({ name = string value = string }))	[]	no
cp_wait_for_capacity_timeout	How long Terraform should wait for ASG instances to be healthy before timing out.	string	"20m"	no
create_app_elb	Bool that determines if a classic elb is created	bool	false	no
create_sso_elb	Bool that determines if a classic elb is created	bool	false	no
domain_name	Name of the domain.	string	"build.smoothglue.io"	no
enable_ccm	Toggle enabling the cluster as aws aware, this will ensure the appropriate IAM policies are present	bool	false	no
etcd_backup	Automatically back up etcd snapshots to the rke2 token bucket	bool	true	no
helm_charts	List of objects describing helm charts to be deployed	list(map(any))	[]	no
log_bucket_prevent_destroy	Bool that allows the S3 log bucket to be destroyed	bool	false	no
mail_relay_from_address	Name of the from email address required to use the ac2sp relay	string	n/a	yes
name_prefix	Name to prefix resources with.	string	"build"	no
name_prefix_include_workspace	Toogle to include workspace in name prefix	bool	false	no
notification_alert_email	Name of the email address.	string	n/a	yes
persistent	Flag to set the deployment to persistent or ephemeral	bool	false	no
registries_config	RKE2 registries.yaml configuration	any	{}	no
resize_disks	Enables automatic resizing of disks in userdata script	bool	true	no
root_cas	List containing root certificate authorities (optionally base64 encoded)	list( object( { name = string cert = string } ) )	[]	no
sso_passthrough_enable	Toggles SSO (Keycloak) to use the passthrough Network Load Balancer	bool	false	no
userdata_command	A string containing shell commands which will be executed during the provisioning process for nodes. These commands will run on both control plane and agent nodes, and will execute at the end of the pre_userdata section of the provisioning, prior to the starting of the RKE2 service.	string	""	no
userdata_command_files	List of relative paths (relative to the userdata_file_dir) to shell (sh) scripts which should be executed as part of the instance provisioning process (after being downloaded from the S3 userdata bucket). These scripts will run on both control plane and agent nodes, and will execute at the end of the pre_userdata section of the provisioning, prior to the starting of the RKE2 service.	list(string)	[]	no
userdata_files_dir	The path to a directory containing files to upload to the userdata files S3 bucket. If these conflict with files in the "${path.module}/files" directory, the files in the userdata_files_dir will overwrite the defaults in the "${path.module}/files" bucket.	string	"fakeemptydir1234"	no
userdata_files_s3_prefix	This prefix will be added to the object keys for all userdata files uploaded to S3.	string	""	no
va_allow_ssh	Toggles SSH security group rule for structsure Agent nodes on or off	bool	false	no
va_ami	Agent node AMI ID, overrides the ami_id variable	string	null	no
va_asg	Node pool AutoScalingGroup scaling definition	object({ min = number max = number desired = number })	n/a	yes
va_bdm_delete_on_term	structsure Agents block device mapping delete on termination setting	bool	true	no
va_bdm_device_name	Name of root block device mapping	string	"/dev/xvda"	no
va_bdm_encrypted	structsure Agents block device mapping encryption setting	bool	true	no
va_bdm_size	structsure Agents block device mapping size	number	80	no
va_bdm_type	structsure Agents block device mapping type	string	"gp2"	no
va_download	Toggle best effort download of rke2 dependencies (rke2 and aws cli), if disabled, dependencies are assumed to exist in $PATH	bool	false	no
va_ebs_kms_key_id	AWS KMS key ID for ebs related operations.	string	n/a	yes
va_enable_autoscaler	Toggle configure the nodepool for cluster autoscaler, this will ensure the appropriate IAM policies are present, you are still responsible for ensuring cluster autoscaler is installed	bool	true	no
va_extra_block_device_mapping	List of objects to configure addional disks	list(map(string))	[]	no
va_extra_security_group_ids	List of additional security group IDs to attach to structsure agent nodes	list(string)	[]	no
va_extra_tags	Additional tags to add to structsure agent nodes	map(string)	{}	no
va_instance_type	Server pool instance type	string	"t3a.large"	no
va_lb_ha	Need application and sso load balancers to have high availability (multi-az)	bool	true	no
va_name	Nodepool name	string	"cbc2_structsure-agents"	no
va_nlb_enable_deletion_protection	Prevent the NLB(s) from being deleted	bool	true	no
va_nlb_preserve_client_ip	Toggles the preserve_client_ip setting on/off. Will always be false if compatibility_mode is set	bool	true	no
va_nlb_stickiness_config	Application NLB Stickiness settings. Will always be an empty set if compatibility_mode is set	object({ enabled = bool duration = number type = string })	{ "duration": 3600, "enabled": true, "type": "source_ip" }	no
va_rke2_config	Agent pool additional configuration passed as rke2 config file, see here for full list of options	string	"kubelet-arg:\n - \"cloud-provider=external\"\n - \"protect-kernel-defaults=true\"\n - \"streaming-connection-idle-timeout=5m\"\n - \"tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\"\nnode-label:\n - \"genericAgent=true\"\nkube-proxy-arg:\n - \"metrics-bind-address=0.0.0.0\"\nselinux: true\n"	no
va_spot	Toggle spot requests for node pool	bool	false	no
va_ssh_cidr_blocks	CIDR block allowed ssh access to structsure Agent nodes	list(string)	[ "10.0.0.0/8" ]	no
va_subnets	List of subnet IDs to create resources in	list(string)	n/a	yes
va_userdata_variables	Extra environmental variables specific to agents that need to be passed between userdata scripts.	list(object({ name = string value = string }))	[]	no
va_wait_for_capacity_timeout	How long Terraform should wait for ASG instances to be healthy before timing out.	string	"30m"	no
vault_passthrough_enable	Toggles Vault to use the passthrough Network Load Balancer	bool	false	no
vpc_id	VPC ID to create resources in	string	n/a	yes
zarf_init_ca_bundle_filename	Filename of the locally-created zarf init config yaml file	string	"zarf-ca-bundle.pem"	no
zarf_init_config_filename	Filename of the locally-created zarf init config yaml file	string	"zarf-init-config.yaml"	no
zarf_registry_enabled	Flag to enable the creation of zarf registry bucket and configuration	bool	true	no
zarf_registry_ironbank_mirror_enabled	toggle for configuring a containerd mirror to point registry1.dso.mil to zarf's registry	bool	true	no
zarf_registry_nodeport	zarf registry node port; must be between 30000-32767	number	31999	no
zarf_registry_pull_password	zarf registry pull password; if not supplied, a random one will be generated	string	""	no
zarf_registry_pull_username	zarf registry pull username	string	"zarf-pull"	no
zarf_registry_redirect_disable	Disable the registry redirect	string	"true"	no
zarf_registry_shared_bucket_id	Name of an existing shared zarf registry bucket	string	"shared-zarf-registry"	no
zarf_registry_shared_enabled	Flag to enable using an existing shared zarf registry bucket	bool	false	no

Outputs

Name	Description
app_elb_dns_name	DNS name for the application ELB
app_nlb_dns_name	DNS name for the app NLB
aws_dns_suffix	Output the DNS suffix of the current aws partition
aws_partition	Output the name of the current aws partition
aws_region	AWS region used
bucket_prefix	Prefix used to create gitlab buckets
cluster_name	Name of the created cluster
cluster_security_group	Security group shared by cluster nodes, this is different than nodepool security groups
create_agent_role	Bool that determines if an IAM role is created
desired_node_count	Total desired node count across all node groups
domain	DNS name of the cluster that was created
kubeconfig	The content of the kubeconfig file
kubeconfig_data	Map of the connection information contained within the kubeconfig
kubeconfig_url	S3 URL to the rke2.yaml kubeconfig
log_bucket_id	S3 log bucket name for storing S3 server logs
minimum_node_count	Total minimum node count across all node groups
node_group_names	n/a
rke2_agents_iam_role	IAM role of rke2_agents node pool
rke2_bucket	S3 bucket for rke2 storage
ssh_private_key	The SSH private key configured as an authorized key on nodes.
sso_elb_dns_name	DNS name for the SSO ELB
sso_nlb_dns_name	DNS name for the sso NLB
target_group_arns	n/a
zarf_registry_bucket_id	Name of the bucket created for use with the Zarf registry