This blog post and demo shows how you can use the Terraform NSX-T Provider to quickly configure an Active-Active Tier-0 with two VRFs and attached Tier-1 Gateways.

Supported with Terraform on a NSX-T Provider 3.0.1 https://github.com/vmware/terraform-provider-nsxt

This Terraform NSX-T Provider example can be useful to quickly deploy a typical VRF multi-tenant topology on a NSX-T environment to test and configures the following NSX-T objects:

• Active-Active Tier-0 Gateway
• Uplink interfaces on the Tier-0 Gateway and VRFs
• Two Tier-0 VRFs
• Two Tier-1 Gateways
• BGP Configuration on the Tier-0 Gateway
• BGP Neighbor Configuration on VRF
• Tier-1 Gateways, one per Tenant VRF.
• Route Advertisement rule on the Tier-1
• Segments connected to Tier-1
• VLAN101 and Trunk Segment for north-south traffic and BGP peering with ToR-A
• VLAN102 and Trunk Segment for north-south traffic and BGP peering with ToR-B
• Security Groups for the Web, App and DB tiers based on Tags
• Security Groups for Red and Blue Tenant Virtual Machines
• Custom Service (TCP 8443) for Web to App communication.
• Some example Distributed Firewall Rules for accessing the Application and communication between the tiers, including a default deny rule and using Applied To.

Logical diagram of the topology:

Tier-0 Gateway with VRF

Requirements:

• NSX Manager(s) ready and configured
• Overlay and VLAN Transport Zones configured
• Two Edge Nodes configured
• Edge Cluster configured with the two Edge Nodes as members.

Below is the “`variables.tf“` file:

# Variables
# NSX Manager
variable "nsx_manager" {
    default = "10.29.12.206"
}

# Username & Password for NSX-T Manager
  variable "username" {
  default = "admin"
}

variable "password" {
    default = "VMware1!rdejong"
}

# Enter Edge Nodes Display Name. Required for external interfaces.
variable "edge_node_1" {
   default = "edge-01"
}
variable "edge_node_2" {
   default = "edge-02"
}

# Segment Names
variable "nsx_segment_web" {
    default = "VM-Segment-Web"
}
variable "nsx_segment_app" {
    default = "VM-Segment-App"
}

variable "nsx_segment_db" {
    default = "VM-Segment-DB"
}

# Security Group names. 
variable "nsx_group_web" {
    default = "Web Servers"
}

variable "nsx_group_app" {
    default = "App Servers"
}

variable "nsx_group_db" {
    default = "DB Servers"
}

variable "nsx_group_blue" {
    default = "Blue Application"
}

variable "nsx_group_red" {
    default = "Red Application"
}

Below is the “`main.tf“` file:

# Terraform Provider
terraform {
  required_providers {
    nsxt = {
      source = "vmware/nsxt"
      version = "3.0.1"
    }
  }
}

# NSX-T Manager Credentials
provider "nsxt" {
    host                     = var.nsx_manager
    username                 = var.username
    password                 = var.password
    allow_unverified_ssl     = true
    max_retries              = 10
    retry_min_delay          = 500
    retry_max_delay          = 5000
    retry_on_status_codes    = [429]
}

# Data Sources we need for reference later
data "nsxt_policy_transport_zone" "overlay_tz" {
    display_name = "Overlay-TZ"
}

data "nsxt_policy_transport_zone" "vlan_tz" {
    display_name = "VLAN-TZ"
}

data "nsxt_policy_edge_cluster" "edge_cluster_vrf" {
    display_name = "edge-cluster-vrf"
}

# DFW Services
data "nsxt_policy_service" "ssh" {
    display_name = "SSH"
}

data "nsxt_policy_service" "http" {
    display_name = "HTTP"
}

data "nsxt_policy_service" "https" {
    display_name = "HTTPS"
}

# Edge Nodes
data "nsxt_policy_edge_node" "edge_node_1" {
   edge_cluster_path   = data.nsxt_policy_edge_cluster.edge_cluster_vrf.path
   display_name        = var.edge_node_1
}

data "nsxt_policy_edge_node" "edge_node_2" {
   edge_cluster_path   = data.nsxt_policy_edge_cluster.edge_cluster_vrf.path
   display_name        = var.edge_node_2
}

# Create VLAN Segments
resource "nsxt_policy_vlan_segment" "vlan101" {
    display_name = "VLAN101"
    description = "VLAN Segment created by Terraform"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["101"]
}

resource "nsxt_policy_vlan_segment" "vlan102" {
    display_name = "VLAN102"
    description = "VLAN Segment created by Terraform"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["102"]
}

# Create Red Tenant Access VLAN Segments for Tenant VRF
resource "nsxt_policy_vlan_segment" "vlan103" {
    display_name = "VLAN103"
    description = "VRF Red Tenant Access VLAN to ToR-A"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["103"]
}

resource "nsxt_policy_vlan_segment" "vlan104" {
    display_name = "VLAN104"
    description = "VRF Red Tenant Access VLAN to ToR-B"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["104"]
}

# Create Blue Tenant Access VLAN Segments for Tenant VRF
resource "nsxt_policy_vlan_segment" "vlan105" {
    display_name = "VLAN105"
    description = "VRF Blue Tenant Access VLAN to ToR-A"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["105"]
}

resource "nsxt_policy_vlan_segment" "vlan106" {
    display_name = "VLAN106"
    description = "VRF Blue Tenant Access VLAN to ToR-B"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["106"]
}

# Create Tenant Access VLAN Trunk Segments for Tenant VRF
resource "nsxt_policy_vlan_segment" "vrf_trunk_a" {
    display_name = "VRF-Trunk-A"
    description = "Tenant VRF Trunk"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["103-110"]
}

resource "nsxt_policy_vlan_segment" "vrf_trunk_b" {
    display_name = "VRF-Trunk-B"
    description = "Tenant VRF Trunk"
    transport_zone_path = data.nsxt_policy_transport_zone.vlan_tz.path
    vlan_ids = ["103-110"]
}

# Create Default Tier-0 Gateway for Tenants Tier-0 VRFs to be runnning on. 
resource "nsxt_policy_tier0_gateway" "vrf_tier0_gw" {
    display_name              = "VRF-Tier-0"
    description               = "VRF Tier 0"
    failover_mode             = "NON_PREEMPTIVE"
    default_rule_logging      = false
    enable_firewall           = false
    force_whitelisting        = false
    ha_mode                   = "ACTIVE_ACTIVE"
    edge_cluster_path         = data.nsxt_policy_edge_cluster.edge_cluster_vrf.path

    bgp_config {
        ecmp            = true               
        local_as_num    = "65005"
        inter_sr_ibgp   = true
        multipath_relax = true
    }
}

# Create VRF for Red Tenant 
resource "nsxt_policy_tier0_gateway" "red_vrf" {
    display_name              = "VRF-Red-Tenant"
    description               = "VRF Red Tenant"
    failover_mode             = "NON_PREEMPTIVE"
    default_rule_logging      = false
    enable_firewall           = false
    force_whitelisting        = false
    ha_mode                   = "ACTIVE_ACTIVE"
    edge_cluster_path         = data.nsxt_policy_edge_cluster.edge_cluster_vrf.path

    vrf_config {
        gateway_path    = nsxt_policy_tier0_gateway.vrf_tier0_gw.path
    }

    bgp_config {
        ecmp            = true
        inter_sr_ibgp   = true
        multipath_relax = true               
    }

    tag {
        scope = "tenant"
        tag   = "red"
    }
}

# Create VRF for Blue Tenant 
resource "nsxt_policy_tier0_gateway" "blue_vrf" {
    depends_on                = [nsxt_policy_tier0_gateway.vrf_tier0_gw]
    display_name              = "VRF-Blue-Tenant"
    description               = "VRF Blue Tenant"
    failover_mode             = "NON_PREEMPTIVE"
    default_rule_logging      = false
    enable_firewall           = false
    force_whitelisting        = false
    ha_mode                   = "ACTIVE_ACTIVE"
    edge_cluster_path         = data.nsxt_policy_edge_cluster.edge_cluster_vrf.path

    vrf_config {
        gateway_path    = nsxt_policy_tier0_gateway.vrf_tier0_gw.path
    }

    bgp_config {
        ecmp            = true
        inter_sr_ibgp   = true
        multipath_relax = true       
    }

    tag {
        scope = "tenant"
        tag   = "blue"
    }
}

# Create VRF Tier-0 Gateway Uplink Interfaces
resource "nsxt_policy_tier0_gateway_interface" "vrf_uplink1" {
    display_name        = "Uplink-01"
    description         = "Default Uplink to VRF to Provider Transit"
    type                = "EXTERNAL"
    edge_node_path      = data.nsxt_policy_edge_node.edge_node_1.path
    gateway_path        = nsxt_policy_tier0_gateway.vrf_tier0_gw.path
    segment_path        = nsxt_policy_vlan_segment.vlan101.path
    subnets             = ["192.168.101.254/24"]
    mtu                 = 1500
}

resource "nsxt_policy_tier0_gateway_interface" "vrf_uplink2" {
    display_name        = "Uplink-02"
    description         = "Default Uplink to VRF to Provider Transit"
    type                = "EXTERNAL"
    edge_node_path      = data.nsxt_policy_edge_node.edge_node_2.path
    gateway_path        = nsxt_policy_tier0_gateway.vrf_tier0_gw.path
    segment_path        = nsxt_policy_vlan_segment.vlan102.path
    subnets             = ["192.168.102.254/24"]
    mtu                 = 1500
}

# Create Red Tenant VRF Uplink Interfaces
resource "nsxt_policy_tier0_gateway_interface" "red_vrf_uplink1" {
    depends_on          = [nsxt_policy_tier0_gateway_interface.vrf_uplink1]
    display_name        = "Uplink-01"
    description         = "Uplink to VRF to Provider Transit"
    type                = "EXTERNAL"
    edge_node_path      = data.nsxt_policy_edge_node.edge_node_1.path
    gateway_path        = nsxt_policy_tier0_gateway.red_vrf.path
    segment_path        = nsxt_policy_vlan_segment.vrf_trunk_a.path
    access_vlan_id      = 103
    subnets             = ["192.168.103.254/24"]
    mtu                 = 1500
}

resource "nsxt_policy_tier0_gateway_interface" "red_vrf_uplink2" {
    depends_on          = [nsxt_policy_tier0_gateway_interface.vrf_uplink2]
    display_name        = "Uplink-02"
    description         = "Uplink to VRF to Provider Transit"
    type                = "EXTERNAL"
    edge_node_path      = data.nsxt_policy_edge_node.edge_node_2.path
    gateway_path        = nsxt_policy_tier0_gateway.red_vrf.path
    segment_path        = nsxt_policy_vlan_segment.vrf_trunk_b.path
    access_vlan_id      = 104
    subnets             = ["192.168.104.254/24"]
    mtu                 = 1500
}

# Create Blue Tenant VRF Uplink Interfaces
resource "nsxt_policy_tier0_gateway_interface" "blue_vrf_uplink1" {
    depends_on          = [nsxt_policy_tier0_gateway_interface.vrf_uplink1]
    display_name        = "Uplink-01"
    description         = "VRF Uplink to ToR"
    type                = "EXTERNAL"
    edge_node_path      = data.nsxt_policy_edge_node.edge_node_1.path
    gateway_path        = nsxt_policy_tier0_gateway.blue_vrf.path
    segment_path        = nsxt_policy_vlan_segment.vrf_trunk_a.path
    access_vlan_id      = 105
    subnets             = ["192.168.105.254/24"]
    mtu                 = 1500
}

resource "nsxt_policy_tier0_gateway_interface" "blue_vrf_uplink2" {
    depends_on          = [nsxt_policy_tier0_gateway_interface.vrf_uplink2]
    display_name        = "Uplink-02"
    description         = "VRF Uplink to ToR"
    type                = "EXTERNAL"
    edge_node_path      = data.nsxt_policy_edge_node.edge_node_2.path
    gateway_path        = nsxt_policy_tier0_gateway.blue_vrf.path
    segment_path        = nsxt_policy_vlan_segment.vrf_trunk_b.path
    access_vlan_id      = 106
    subnets             = ["192.168.106.254/24"]
    mtu                 = 1500
}

# Provider Tier-0 to ToR BGP Neighbor Configuration
resource "nsxt_policy_bgp_neighbor" "router_a" {
    display_name        = "ToR-A"
    description         = "Terraform provisioned BGP Neighbor Configuration"
    bgp_path            = nsxt_policy_tier0_gateway.vrf_tier0_gw.bgp_config.0.path
    neighbor_address    = "192.168.101.1"
    remote_as_num       = "65001"
}

resource "nsxt_policy_bgp_neighbor" "router_b" {
    display_name        = "ToR-B"
    description         = "Terraform provisioned BGP Neighbor Configuration"
    bgp_path            = nsxt_policy_tier0_gateway.vrf_tier0_gw.bgp_config.0.path
    neighbor_address    = "192.168.102.1"
    remote_as_num       = "65002"
}

# VRF Red to ToR BGP Neighbor Configuration
resource "nsxt_policy_bgp_neighbor" "vrf_red_router_a" {
    depends_on          = [nsxt_policy_bgp_neighbor.router_a]
    display_name        = "ToR-A"
    description         = "VRF Red to ToR-A"
    bgp_path            = nsxt_policy_tier0_gateway.red_vrf.bgp_config.0.path
    neighbor_address    = "192.168.103.1"
    remote_as_num       = "65001"
}

resource "nsxt_policy_bgp_neighbor" "vrf_red_router_b" {
    depends_on          = [nsxt_policy_bgp_neighbor.router_b]
    display_name        = "ToR-B"
    description         = "VRF Red to ToR-B"
    bgp_path            = nsxt_policy_tier0_gateway.red_vrf.bgp_config.0.path
    neighbor_address    = "192.168.104.1"
    remote_as_num       = "65002"
}

# VRF Blue to ToR BGP Neighbor Configuration
resource "nsxt_policy_bgp_neighbor" "vrf_blue_router_a" {
    depends_on          = [nsxt_policy_bgp_neighbor.router_a]
    display_name        = "ToR-A"
    description         = "VRF Blue to ToR-A"
    bgp_path            = nsxt_policy_tier0_gateway.blue_vrf.bgp_config.0.path
    neighbor_address    = "192.168.105.1"
    remote_as_num       = "65001"
}

resource "nsxt_policy_bgp_neighbor" "vrf_blue_router_b" {
    depends_on          = [nsxt_policy_bgp_neighbor.router_b]
    display_name        = "ToR-B"
    description         = "VRF Blue to ToR-B"
    bgp_path            = nsxt_policy_tier0_gateway.blue_vrf.bgp_config.0.path
    neighbor_address    = "192.168.106.1"
    remote_as_num       = "65002"
}

# Create Red Tier-1 Gateway
resource "nsxt_policy_tier1_gateway" "tier1_gw_red" {
    description               = "Red Tenant Tier-1"
    display_name              = "Red-Tier-1"
    edge_cluster_path         = data.nsxt_policy_edge_cluster.edge_cluster_vrf.path
    failover_mode             = "NON_PREEMPTIVE"
    default_rule_logging      = "false"
    enable_firewall           = "true"
    enable_standby_relocation = "false"
    force_whitelisting        = "true"
    tier0_path                = nsxt_policy_tier0_gateway.red_vrf.path
    route_advertisement_types = ["TIER1_STATIC_ROUTES", "TIER1_CONNECTED"]

    tag {
        scope = "tenant"
        tag   = "red"
    }

    route_advertisement_rule {
        name                      = "Tier 1 Networks"
        action                    = "PERMIT"
        subnets                   = ["172.16.10.0/24","172.16.11.0/24"]
        prefix_operator           = "GE"
        route_advertisement_types = ["TIER1_CONNECTED"]
    }
}

# Create Blue Tier-1 Gateway
resource "nsxt_policy_tier1_gateway" "tier1_gw_blue" {
    description               = "Blue Tenant Tier-1"
    display_name              = "Blue-Tier-1"
    edge_cluster_path         = data.nsxt_policy_edge_cluster.edge_cluster_vrf.path
    failover_mode             = "NON_PREEMPTIVE"
    default_rule_logging      = "false"
    enable_firewall           = "false"
    enable_standby_relocation = "false"
    force_whitelisting        = "true"
    tier0_path                = nsxt_policy_tier0_gateway.blue_vrf.path
    route_advertisement_types = ["TIER1_STATIC_ROUTES", "TIER1_CONNECTED"]

    tag {
        scope = "tenant"
        tag   = "blue"
    }

    route_advertisement_rule {
        name                      = "Tier 1 Networks"
        action                    = "PERMIT"
        subnets                   = ["172.16.20.0/24"]
        prefix_operator           = "GE"
        route_advertisement_types = ["TIER1_CONNECTED"]
    }
}

# Create Tenant Segments
resource "nsxt_policy_segment" "red_segment_1" {
    display_name = "Red-Segment-1"
    description = "Red Tenant Segment 1"
    connectivity_path   = nsxt_policy_tier1_gateway.tier1_gw_red.path
    transport_zone_path = data.nsxt_policy_transport_zone.overlay_tz.path
    
    subnet {
      cidr        = "172.16.10.1/24"
    }
}

resource "nsxt_policy_segment" "red_segment_2" {
    display_name = "Red-Segment-2"
    description = "Red Tenant Segment 2"
    connectivity_path   = nsxt_policy_tier1_gateway.tier1_gw_red.path
    transport_zone_path = data.nsxt_policy_transport_zone.overlay_tz.path

    subnet {
      cidr        = "172.16.11.1/24"
    }
}

resource "nsxt_policy_segment" "blue_segment_1" {
    display_name = "Blue-Segment-1"
    description = "Blue Tenant Segment 1"
    connectivity_path   = nsxt_policy_tier1_gateway.tier1_gw_blue.path
    transport_zone_path = data.nsxt_policy_transport_zone.overlay_tz.path
    
    subnet {
      cidr        = "172.16.20.1/24"
    }
}

# Create Security Groups
resource "nsxt_policy_group" "web_servers" {
    display_name = var.nsx_group_web
    description  = "Terraform provisioned Group"

    criteria {
        condition {
            key         = "Tag"
            member_type = "VirtualMachine"
            operator    = "CONTAINS"
            value       = "Web"
        } 
    }
}

resource "nsxt_policy_group" "app_servers" {
    display_name = var.nsx_group_app
    description  = "Terraform provisioned Group"

    criteria {
        condition {
            key         = "Tag"
            member_type = "VirtualMachine"
            operator    = "CONTAINS"
            value       = "App"
        } 
    }
}

resource "nsxt_policy_group" "db_servers" {
    display_name = var.nsx_group_db
    description  = "Terraform provisioned Group"

    criteria {
        condition {
            key         = "Tag"
            member_type = "VirtualMachine"
            operator    = "CONTAINS"
            value       = "DB"
        } 
    }
}

resource "nsxt_policy_group" "red_servers" {
    display_name = var.nsx_group_red
    description  = "Terraform provisioned Group"

    criteria {
        condition {
            key         = "Tag"
            member_type = "VirtualMachine"
            operator    = "CONTAINS"
            value       = "Red"
        } 
    }
}

resource "nsxt_policy_group" "blue_servers" {
    display_name = var.nsx_group_blue
    description  = "Terraform provisioned Group"

    criteria {
        condition {
            key         = "Tag"
            member_type = "VirtualMachine"
            operator    = "CONTAINS"
            value       = "Blue"
        } 
    }
}

# Create Custom Services
resource "nsxt_policy_service" "service_tcp8443" {
    description  = "HTTPS service provisioned by Terraform"
    display_name = "TCP 8443"

    l4_port_set_entry {
        display_name      = "TCP8443"
        description       = "TCP port 8443 entry"
        protocol          = "TCP"
        destination_ports = [ "8443" ]
    }

    tag {
        scope = "color"
        tag   = "blue"
    }
}

# Create Security Policies
resource "nsxt_policy_security_policy" "allow_blue" {
    display_name = "Allow Blue Application"
    description  = "Terraform provisioned Security Policy"
    category     = "Application"
    locked       = false
    stateful     = true
    tcp_strict   = false
    scope        = [nsxt_policy_group.web_servers.path]

    rule {
        display_name        = "Allow SSH to Blue Servers"
        destination_groups  = [nsxt_policy_group.blue_servers.path]
        action              = "ALLOW"
        services            = [data.nsxt_policy_service.ssh.path]
        logged              = true
        scope               = [nsxt_policy_group.blue_servers.path]
    }    

    rule {
        display_name        = "Allow HTTPS to Web Servers"
        destination_groups  = [nsxt_policy_group.web_servers.path]
        action              = "ALLOW"
        services            = [data.nsxt_policy_service.https.path]
        logged              = true
        scope               = [nsxt_policy_group.web_servers.path]
    }

    rule {
        display_name        = "Allow TCP 8443 to App Servers"
        source_groups       = [nsxt_policy_group.web_servers.path]
        destination_groups  = [nsxt_policy_group.app_servers.path]
        action              = "ALLOW"
        services            = [nsxt_policy_service.service_tcp8443.path]
        logged              = true
        scope               = [nsxt_policy_group.web_servers.path,nsxt_policy_group.app_servers.path]
    }

    rule {
        display_name        = "Allow HTTP to DB Servers"
        source_groups       = [nsxt_policy_group.app_servers.path]
        destination_groups  = [nsxt_policy_group.db_servers.path]
        action              = "ALLOW"
        services            = [data.nsxt_policy_service.http.path]
        logged              = true
        scope               = [nsxt_policy_group.app_servers.path,nsxt_policy_group.db_servers.path]
    }

    rule {
        display_name        = "Any Deny"
        action              = "REJECT"
        logged              = false
        scope               = [nsxt_policy_group.blue_servers.path]
    }
}