Skip to content

Commit 6132cd5

Browse files
ethanndicksondeansheather
ethanndickson
and
deansheather
authored
refactor(scaletest): use vpc for networking infrastructure (#19464)
This PR refactors the scaletest infrastructure to use a dedicated VPC for each deployment (i.e. alpha, bravo, charlie). It then peers that VPC with the observability VPC, and the Cloud SQL database. It also sets up subnetting for and within each deployment. With this deployed, I was able to get the scaletest running with metrics flowing into `scaletest.cdr.dev`. Co-authored-by: Dean Sheather <dean@deansheather.com>
1 parent 5145cd0 commit 6132cd5

File tree

4 files changed

+143
-13
lines changed

4 files changed

+143
-13
lines changed

‎scaletest/terraform/action/gcp_clusters.tf

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,12 +78,13 @@ resource "google_container_cluster" "cluster" {
7878
name = "${var.name}-${each.key}"
7979
location = each.value.zone
8080
project = var.project_id
81-
network = local.vpc_name
82-
subnetwork = local.subnet_name
81+
network = google_compute_network.network.name
82+
subnetwork = google_compute_subnetwork.subnetwork[each.key].name
8383
networking_mode = "VPC_NATIVE"
8484
default_max_pods_per_node = 256
8585
ip_allocation_policy { # Required with networking_mode=VPC_NATIVE
86-
86+
cluster_secondary_range_name = local.secondary_ip_range_k8s_pods
87+
services_secondary_range_name = local.secondary_ip_range_k8s_services
8788
}
8889
release_channel {
8990
# Setting release channel as STABLE can cause unexpected cluster upgrades.
@@ -108,7 +109,6 @@ resource "google_container_cluster" "cluster" {
108109
workload_pool = "${data.google_project.project.project_id}.svc.id.goog"
109110
}
110111

111-
112112
lifecycle {
113113
ignore_changes = [
114114
maintenance_policy,

‎scaletest/terraform/action/gcp_db.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ resource "google_sql_database_instance" "db" {
2323

2424
ip_configuration {
2525
ipv4_enabled = false
26-
private_network = local.vpc_id
26+
private_network = google_compute_network.network.id
2727
}
2828

2929
insights_config {

‎scaletest/terraform/action/gcp_vpc.tf

Lines changed: 133 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,91 @@
11
locals {
2-
vpc_name = "scaletest"
3-
vpc_id = "projects/${var.project_id}/global/networks/${local.vpc_name}"
4-
subnet_name = "scaletest"
2+
# Generate a /14 for each deployment.
3+
cidr_networks = cidrsubnets(
4+
"172.16.0.0/12",
5+
2,
6+
2,
7+
2,
8+
)
9+
10+
networks = {
11+
alpha = local.cidr_networks[0]
12+
bravo = local.cidr_networks[1]
13+
charlie = local.cidr_networks[2]
14+
}
15+
16+
# Generate a bunch of /18s within the subnet we're using from the above map.
17+
cidr_subnetworks = cidrsubnets(
18+
local.networks[var.name],
19+
4, # PSA
20+
4, # primary subnetwork
21+
4, # primary k8s pod network
22+
4, # primary k8s services network
23+
4, # europe subnetwork
24+
4, # europe k8s pod network
25+
4, # europe k8s services network
26+
4, # asia subnetwork
27+
4, # asia k8s pod network
28+
4, # asia k8s services network
29+
)
30+
31+
psa_range_address = split("/", local.cidr_subnetworks[0])[0]
32+
psa_range_prefix_length = tonumber(split("/", local.cidr_subnetworks[0])[1])
33+
34+
subnetworks = {
35+
primary = local.cidr_subnetworks[1]
36+
europe = local.cidr_subnetworks[4]
37+
asia = local.cidr_subnetworks[7]
38+
}
39+
cluster_ranges = {
40+
primary = {
41+
pods = local.cidr_subnetworks[2]
42+
services = local.cidr_subnetworks[3]
43+
}
44+
europe = {
45+
pods = local.cidr_subnetworks[5]
46+
services = local.cidr_subnetworks[6]
47+
}
48+
asia = {
49+
pods = local.cidr_subnetworks[8]
50+
services = local.cidr_subnetworks[9]
51+
}
52+
}
53+
54+
secondary_ip_range_k8s_pods = "k8s-pods"
55+
secondary_ip_range_k8s_services = "k8s-services"
56+
}
57+
58+
# Create a VPC for the deployment
59+
resource "google_compute_network" "network" {
60+
project = var.project_id
61+
name = "${var.name}-scaletest"
62+
description = "scaletest network for ${var.name}"
63+
auto_create_subnetworks = false
64+
}
65+
66+
# Create a subnetwork with a unique range for each region
67+
resource "google_compute_subnetwork" "subnetwork" {
68+
for_each = local.subnetworks
69+
name = "${var.name}-${each.key}"
70+
# Use the deployment region
71+
region = local.deployments[each.key].region
72+
network = google_compute_network.network.id
73+
project = var.project_id
74+
ip_cidr_range = each.value
75+
private_ip_google_access = true
76+
77+
secondary_ip_range {
78+
range_name = local.secondary_ip_range_k8s_pods
79+
ip_cidr_range = local.cluster_ranges[each.key].pods
80+
}
81+
82+
secondary_ip_range {
83+
range_name = local.secondary_ip_range_k8s_services
84+
ip_cidr_range = local.cluster_ranges[each.key].services
85+
}
586
}
687

88+
# Create a public IP for each region
789
resource "google_compute_address" "coder" {
890
for_each = local.deployments
991
project = var.project_id
@@ -13,17 +95,60 @@ resource "google_compute_address" "coder" {
1395
network_tier = "PREMIUM"
1496
}
1597

16-
resource "google_compute_global_address" "sql_peering" {
98+
# Reserve an internal range for Google-managed services (PSA), used for Cloud
99+
# SQL
100+
resource "google_compute_global_address" "psa_peering" {
17101
project = var.project_id
18102
name = "${var.name}-sql-peering"
19103
purpose = "VPC_PEERING"
20104
address_type = "INTERNAL"
21-
prefix_length = 16
22-
network = local.vpc_name
105+
address = local.psa_range_address
106+
prefix_length = local.psa_range_prefix_length
107+
network = google_compute_network.network.self_link
23108
}
24109

25110
resource "google_service_networking_connection" "private_vpc_connection" {
26-
network = local.vpc_id
111+
network = google_compute_network.network.id
27112
service = "servicenetworking.googleapis.com"
28-
reserved_peering_ranges = [google_compute_global_address.sql_peering.name]
113+
reserved_peering_ranges = [google_compute_global_address.psa_peering.name]
114+
}
115+
116+
# Join the new network to the observability network so we can talk to the
117+
# Prometheus instance
118+
data "google_compute_network" "observability" {
119+
project = var.project_id
120+
name = var.observability_cluster_vpc
121+
}
122+
123+
resource "google_compute_network_peering" "scaletest_to_observability" {
124+
name = "peer-${google_compute_network.network.name}-to-${data.google_compute_network.observability.name}"
125+
network = google_compute_network.network.self_link
126+
peer_network = data.google_compute_network.observability.self_link
127+
import_custom_routes = true
128+
export_custom_routes = true
129+
}
130+
131+
resource "google_compute_network_peering" "observability_to_scaletest" {
132+
name = "peer-${data.google_compute_network.observability.name}-to-${google_compute_network.network.name}"
133+
network = data.google_compute_network.observability.self_link
134+
peer_network = google_compute_network.network.self_link
135+
import_custom_routes = true
136+
export_custom_routes = true
137+
}
138+
139+
# Allow traffic from the scaletest network into the observability network so we
140+
# can connect to Prometheus
141+
resource "google_compute_firewall" "observability_allow_from_scaletest" {
142+
project = var.project_id
143+
name = "allow-from-scaletest-${var.name}"
144+
network = data.google_compute_network.observability.self_link
145+
direction = "INGRESS"
146+
source_ranges = [local.networks[var.name]]
147+
allow {
148+
protocol = "icmp"
149+
}
150+
allow {
151+
protocol = "tcp"
152+
ports = ["0-65535"]
153+
}
29154
}

‎scaletest/terraform/action/vars.tf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,11 @@ variable "observability_cluster_location" {
9696
default = "us-east1-b"
9797
}
9898

99+
variable "observability_cluster_vpc" {
100+
description = "Name of the observability cluster VPC network to peer with."
101+
default = "default"
102+
}
103+
99104
variable "cloudflare_api_token_secret" {
100105
description = "Name of the Google Secret Manager secret containing the Cloudflare API token."
101106
default = "cloudflare-api-token-dns"

0 commit comments

Comments
 (0)