This example shows how you can use the Terraform NSX-T Provider to quickly configure an Active-Active Tier-0 with a Tier-1 and a 3-tier app topology.
Also this Terraform NSX-T Provider example can be useful to quickly deploy a typical 3-tier application topology on a NSX-T environment to test and configures the following NSX-T objects:
- Active-Active Tier-0 Gateway
- Uplink interfaces on the Tier-0 Gateway
- BGP Configuration on the Tier-0 Gateway,
- BGP Neighbor Configuration with ECMP
- Tier-1 Gateway
- Route Advertisement rule on the Tier-1
- Overlay Segments for the Web, App and DB tiers.
- VLAN101 Segment for north-south traffic and BGP peering with ToR-A
- VLAN102 Segment for north-south traffic and BGP peering with ToR-B
- Local NSX-T DHCP Server for the Segments
- Security Groups for the Web, App and DB tiers based on Tags
- Custom Service (TCP 8443) for Web to App communication.
- Distributed Firewall Rules for accessing the Application and communication between the tiers, including a default deny rule and using Applied To.
Requirements:
- NSX Manager(s) ready and configured
- Compute Manager configured
- Overlay and VLAN Transport Zones configured
- Two Edge Nodes configured
- Edge Cluster configured with the two Edge Nodes as members.
Logical diagram of the topology:
You can find the latest up-to-date code on my Github repository here:
https://github.com/raymonddejong/terraform-nsxt/tree/master/Active-Active
Below is the “`variables.tf“` file:
# Variables # NSX Manager IP variable "nsx_manager" { default = "10.29.12.206" } # Username & Password for NSX-T Manager variable "username" { default = "admin" } variable "password" { default = "password" } # Transport Zones & MTU variable "vlan_tz" { default = "VLAN-TZ" } variable "overlay_tz" { default = "Overlay-TZ" } variable "tier0_uplink_mtu" { default = "1500" } # Enter Edge Nodes Display Name. Required for external interfaces. variable "edge_node_1" { default = "edge-01" } variable "edge_node_2" { default = "edge-02" } variable "edge_cluster" { default = "edge-cluster-01" } # Tier0 Gateway Configuration variable "tier0_local_as" { default = 65005 } variable "uplink_en1_fa_ip" { default = "192.168.101.254/24" } variable "uplink_en1_fb_ip" { default = "192.168.102.254/24" } variable "uplink_en2_fa_ip" { default = "192.168.101.253/24" } variable "uplink_en2_fb_ip" { default = "192.168.102.253/24" } variable "router_a_ip" { default = "192.168.101.1" } variable "router_b_ip" { default = "192.168.102.1" } variable "router_a_remote_as" { default = "65001" } variable "router_b_remote_as" { default = "65002" } variable "hold_down_time" { default = "180" } variable "keep_alive_time" { default = "60" } # Segment Names and Details variable "segment_web" { default = "TF-Segment-Web" } variable "segment_app" { default = "TF-Segment-App" } variable "segment_db" { default = "TF-Segment-DB" } variable "segment_web_cidr" { default = "172.16.10.1/24" } variable "segment_app_cidr" { default = "172.16.20.1/24" } variable "segment_db_cidr" { default = "172.16.30.1/24" } # Security Group names variable "nsx_group_web" { default = "Web Servers" } variable "nsx_group_app" { default = "App Servers" } variable "nsx_group_db" { default = "DB Servers" } variable "nsx_group_blue" { default = "Blue Application" }
Below is the “`main.tf“` file:
# NSX-T Manager Credentials provider "nsxt" { host = var.nsx_manager username = var.username password = var.password allow_unverified_ssl = true max_retries = 10 retry_min_delay = 500 retry_max_delay = 5000 retry_on_status_codes = [429] } # Data Sources we need for reference later data "nsxt_policy_transport_zone" "overlay_tz" { display_name = var.overlay_tz } data "nsxt_policy_transport_zone" "vlan_tz" { display_name = var.vlan_tz } data "nsxt_policy_edge_cluster" "edge_cluster" { display_name = var.edge_cluster } data "nsxt_policy_service" "ssh" { display_name = "SSH" } data "nsxt_policy_service" "http" { display_name = "HTTP" } data "nsxt_policy_service" "https" { display_name = "HTTPS" } data "nsxt_policy_edge_node" "edge_node_1" { edge_cluster_path = data.nsxt_policy_edge_cluster.edge_cluster.path display_name = var.edge_node_1 } data "nsxt_policy_edge_node" "edge_node_2" { edge_cluster_path = data.nsxt_policy_edge_cluster.edge_cluster.path display_name = var.edge_node_2 } # Create NSX-T VLAN Segments resource "nsxt_policy_vlan_segment" "vlan101" { display_name = "VLAN101" description = "VLAN Segment created by Terraform" transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path vlan_ids = ["101"] } resource "nsxt_policy_vlan_segment" "vlan102" { display_name = "VLAN102" description = "VLAN Segment created by Terraform" transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path vlan_ids = ["102"] } # Create Tier-0 Gateway resource "nsxt_policy_tier0_gateway" "tier0_gw" { display_name = "TF_Tier_0" description = "Tier-0 provisioned by Terraform" failover_mode = "NON_PREEMPTIVE" default_rule_logging = false enable_firewall = false # force_whitelisting = true ha_mode = "ACTIVE_ACTIVE" edge_cluster_path = data.nsxt_policy_edge_cluster.edge_cluster.path bgp_config { ecmp = true local_as_num = var.tier0_local_as inter_sr_ibgp = true multipath_relax = true } redistribution_config { enabled = true rule { name = "t0-route-redistribution" types = ["TIER1_LB_VIP", "TIER1_CONNECTED", "TIER1_SERVICE_INTERFACE", "TIER1_NAT", "TIER1_LB_SNAT"] } } tag { scope = "color" tag = "blue" } } # Create Tier-0 Gateway Uplink Interfaces # Edge Node 1 - Fabric A - Router Port Configuration resource "nsxt_policy_tier0_gateway_interface" "uplink_en1_fa" { display_name = "uplink-en1-fa" description = "Uplink Edge Node 1 - Fabric A" type = "EXTERNAL" edge_node_path = data.nsxt_policy_edge_node.edge_node_1.path gateway_path = nsxt_policy_tier0_gateway.tier0_gw.path segment_path = nsxt_policy_vlan_segment.vlan101.path subnets = [var.uplink_en1_fa_ip] mtu = var.tier0_uplink_mtu } # Edge Node 1 - Fabric B - Router Port Configuration resource "nsxt_policy_tier0_gateway_interface" "uplink_en1_fb" { display_name = "uplink-en1-fb" description = "Uplink Edge Node 1 - Fabric B" type = "EXTERNAL" edge_node_path = data.nsxt_policy_edge_node.edge_node_1.path gateway_path = nsxt_policy_tier0_gateway.tier0_gw.path segment_path = nsxt_policy_vlan_segment.vlan102.path subnets = [var.uplink_en1_fb_ip] mtu = var.tier0_uplink_mtu } # Edge Node 2 - Fabric A - Router Port Configuration resource "nsxt_policy_tier0_gateway_interface" "uplink_en2_fa" { display_name = "uplink-en1-fa" description = "Uplink Edge Node 2 - Fabric A" type = "EXTERNAL" edge_node_path = data.nsxt_policy_edge_node.edge_node_2.path gateway_path = nsxt_policy_tier0_gateway.tier0_gw.path segment_path = nsxt_policy_vlan_segment.vlan101.path subnets = [var.uplink_en2_fa_ip] mtu = var.tier0_uplink_mtu } # Edge Node 2 - Fabric B - Router Port Configuration resource "nsxt_policy_tier0_gateway_interface" "uplink_en2_fb" { display_name = "uplink-en2-fb" description = "Uplink Edge Node 2 - Fabric B" type = "EXTERNAL" edge_node_path = data.nsxt_policy_edge_node.edge_node_2.path gateway_path = nsxt_policy_tier0_gateway.tier0_gw.path segment_path = nsxt_policy_vlan_segment.vlan102.path subnets = [var.uplink_en2_fb_ip] mtu = var.tier0_uplink_mtu } # Local definitions locals { # Concatinate Uplink Source IP's for ToR-A Peering peer_a_source_addresses = concat( nsxt_policy_tier0_gateway_interface.uplink_en1_fa.ip_addresses, nsxt_policy_tier0_gateway_interface.uplink_en2_fa.ip_addresses ) # Concatinate Uplink Source IP's for ToR-B Peering peer_b_source_addresses = concat( nsxt_policy_tier0_gateway_interface.uplink_en1_fb.ip_addresses, nsxt_policy_tier0_gateway_interface.uplink_en2_fb.ip_addresses ) } # BGP Neighbor Configuration ToR-A resource "nsxt_policy_bgp_neighbor" "router_a" { display_name = "ToR-A" description = "Terraform provisioned BGP Neighbor Configuration" bgp_path = nsxt_policy_tier0_gateway.tier0_gw.bgp_config.0.path neighbor_address = var.router_a_ip remote_as_num = var.router_a_remote_as hold_down_time = var.hold_down_time keep_alive_time = var.keep_alive_time source_addresses = local.peer_a_source_addresses } resource "nsxt_policy_bgp_neighbor" "router_b" { display_name = "ToR-B" description = "Terraform provisioned BGP Neighbor Configuration" bgp_path = nsxt_policy_tier0_gateway.tier0_gw.bgp_config.0.path neighbor_address = var.router_b_ip remote_as_num = var.router_b_remote_as hold_down_time = var.hold_down_time keep_alive_time = var.keep_alive_time source_addresses = local.peer_b_source_addresses } # Create Tier-1 Gateway resource "nsxt_policy_tier1_gateway" "tier1_gw" { description = "Tier-1 provisioned by Terraform" display_name = "TF-Tier-1-01" nsx_id = "predefined_id" edge_cluster_path = data.nsxt_policy_edge_cluster.edge_cluster.path failover_mode = "NON_PREEMPTIVE" default_rule_logging = "false" enable_firewall = "true" enable_standby_relocation = "false" # force_whitelisting = "false" tier0_path = nsxt_policy_tier0_gateway.tier0_gw.path route_advertisement_types = ["TIER1_STATIC_ROUTES", "TIER1_CONNECTED"] tag { scope = "color" tag = "blue" } route_advertisement_rule { name = "Tier 1 Networks" action = "PERMIT" subnets = ["172.16.10.0/24", "172.16.20.0/24", "172.16.30.0/24"] prefix_operator = "GE" route_advertisement_types = ["TIER1_CONNECTED"] } } # DHCP Server resource "nsxt_policy_dhcp_server" "dhcp_server" { display_name = "DHCP Server" description = "Terraform provisioned DHCP Server Config" edge_cluster_path = data.nsxt_policy_edge_cluster.edge_cluster.path lease_time = 86400 } # Create NSX-T Overlay Segments resource "nsxt_policy_segment" "tf_segment_web" { display_name = var.segment_web description = "Segment created by Terraform" transport_zone_path = data.nsxt_policy_transport_zone.overlay_tz.path connectivity_path = nsxt_policy_tier1_gateway.tier1_gw.path dhcp_config_path = nsxt_policy_dhcp_server.dhcp_server.path subnet { cidr = var.segment_web_cidr dhcp_ranges = ["172.16.10.10-172.16.10.100"] dhcp_v4_config { server_address = "172.16.10.254/24" lease_time = 86400 dns_servers = ["10.29.12.197"] } } } resource "nsxt_policy_segment" "tf_segment_app" { display_name = var.segment_app description = "Segment created by Terraform" transport_zone_path = data.nsxt_policy_transport_zone.overlay_tz.path connectivity_path = nsxt_policy_tier1_gateway.tier1_gw.path dhcp_config_path = nsxt_policy_dhcp_server.dhcp_server.path subnet { cidr = var.segment_app_cidr dhcp_ranges = ["172.16.20.10-172.16.20.100"] dhcp_v4_config { server_address = "172.16.20.254/24" lease_time = 86400 dns_servers = ["10.29.12.197"] } } } resource "nsxt_policy_segment" "tf_segment_db" { display_name = var.segment_db description = "Segment created by Terraform" transport_zone_path = data.nsxt_policy_transport_zone.overlay_tz.path connectivity_path = nsxt_policy_tier1_gateway.tier1_gw.path dhcp_config_path = nsxt_policy_dhcp_server.dhcp_server.path subnet { cidr = var.segment_db_cidr dhcp_ranges = ["172.16.30.50-172.16.30.100"] dhcp_v4_config { server_address = "172.16.30.254/24" lease_time = 86400 dns_servers = ["10.29.12.197"] } } } # Create Security Groups resource "nsxt_policy_group" "web_servers" { display_name = var.nsx_group_web description = "Terraform provisioned Group" criteria { condition { key = "Tag" member_type = "VirtualMachine" operator = "CONTAINS" value = "Web" } } } resource "nsxt_policy_group" "app_servers" { display_name = var.nsx_group_app description = "Terraform provisioned Group" criteria { condition { key = "Tag" member_type = "VirtualMachine" operator = "CONTAINS" value = "App" } } } resource "nsxt_policy_group" "db_servers" { display_name = var.nsx_group_db description = "Terraform provisioned Group" criteria { condition { key = "Tag" member_type = "VirtualMachine" operator = "CONTAINS" value = "DB" } } } resource "nsxt_policy_group" "blue_servers" { display_name = var.nsx_group_blue description = "Terraform provisioned Group" criteria { condition { key = "Tag" member_type = "VirtualMachine" operator = "CONTAINS" value = "Blue" } } } # Create Custom Services resource "nsxt_policy_service" "service_tcp8443" { description = "HTTPS service provisioned by Terraform" display_name = "TCP 8443" l4_port_set_entry { display_name = "TCP8443" description = "TCP port 8443 entry" protocol = "TCP" destination_ports = ["8443"] } tag { scope = "color" tag = "blue" } } # Create Security Policies # DFW Infrastructure Category Rules resource "nsxt_policy_security_policy" "Infrastructure" { display_name = "Infrastructure" description = "Terraform provisioned Security Policy" category = "Infrastructure" locked = false stateful = true tcp_strict = false rule { display_name = "Allow DHCP" action = "ALLOW" services = ["/infra/services/DHCP-Server", "/infra/services/DHCP-Client"] logged = false notes = "Allow access to DHCP Server" } } # DFW Application Category Rules resource "nsxt_policy_security_policy" "allow_blue" { display_name = "Allow Blue Application" description = "Terraform provisioned Security Policy" category = "Application" locked = false stateful = true tcp_strict = false scope = [nsxt_policy_group.web_servers.path] rule { display_name = "Allow SSH to Blue Servers" destination_groups = [nsxt_policy_group.blue_servers.path] action = "ALLOW" services = [data.nsxt_policy_service.ssh.path] logged = true scope = [nsxt_policy_group.blue_servers.path] } rule { display_name = "Allow HTTPS to Web Servers" destination_groups = [nsxt_policy_group.web_servers.path] action = "ALLOW" services = [data.nsxt_policy_service.https.path] logged = true scope = [nsxt_policy_group.web_servers.path] } rule { display_name = "Allow TCP 8443 to App Servers" source_groups = [nsxt_policy_group.web_servers.path] destination_groups = [nsxt_policy_group.app_servers.path] action = "ALLOW" services = [nsxt_policy_service.service_tcp8443.path] logged = true scope = [nsxt_policy_group.web_servers.path, nsxt_policy_group.app_servers.path] } rule { display_name = "Allow HTTP to DB Servers" source_groups = [nsxt_policy_group.app_servers.path] destination_groups = [nsxt_policy_group.db_servers.path] action = "ALLOW" services = [data.nsxt_policy_service.http.path] logged = true scope = [nsxt_policy_group.app_servers.path, nsxt_policy_group.db_servers.path] } rule { display_name = "Any Deny" action = "REJECT" logged = false scope = [nsxt_policy_group.blue_servers.path] } }