terraform-provider-utils

what && why

• Re-releasing 1.3.0 as 1.4.1 while fixing some remote state issues in 1.4.0

what

• [stack_processor]: Add process_stack_deps input var to the stack_config data source

why

• Make the settings configurable (false by default)
• Not all provider invocations need to process all stack dependencies for the components (e.g. Spacelift module needs it, remote-state does not)
• Makes invocations without processing stack dependencies 2-3 times faster

what

• [stack_processor]: Read and parse only YAML files

why

• Non-YAML files in the stacks folder cause the YAML parser to panic

what

• Add imports to stack output
• Add stacks to each component output

why

• imports attribute shows all imported stacks for the current stack - can be used in CI/CD pipelines to determine stack dependencies
• stacks attribute shows all the stacks the component (and its base component, if present) is declared in - can be used in CI/CD pipelines (e.g. Spacelift) to determine all stacks that the component depends on,
  and to provision triggers for all the dependencies (once any of the stack config files changes, the component's job will be triggered)

test

imports:
  - eks/eks-defaults
  - globals
  - uw2-globals

terraform:
  aurora-postgres:
    backend:
      acl: bucket-owner-full-control
      bucket: eg-uw2-root-tfstate
      dynamodb_table: eg-uw2-root-tfstate-lock
      encrypt: true
      key: terraform.tfstate
      region: us-west-2
      role_arn: arn:aws:iam::XXXXXXXXXXXX:role/eg-gbl-root-terraform
      workspace_key_prefix: aurora-postgres
    backend_type: s3
    stacks:
    - globals
    - uw2-dev
    - uw2-globals
    - uw2-prod
    - uw2-staging
    - uw2-uat
    env:
      ENV_TEST_1: test1
      ENV_TEST_2: test2
      ENV_TEST_3: test3
      ENV_TEST_4: test4
      ENV_TEST_5: test5
      ENV_TEST_6: test6
      ENV_TEST_7: test7
    settings:
      spacelift:
        autodeploy: false
        workspace_enabled: false
      version: 0
    vars:
      cluster_size: 1
      environment: uw2
      instance_type: db.r4.large
      namespace: eg
      region: us-west-2
      stage: dev
  aurora-postgres-2:
    backend:
      acl: bucket-owner-full-control
      bucket: eg-uw2-root-tfstate
      dynamodb_table: eg-uw2-root-tfstate-lock
      encrypt: true
      key: terraform.tfstate
      region: us-west-2
      role_arn: arn:aws:iam::XXXXXXXXXXXX:role/eg-gbl-root-terraform
      workspace_key_prefix: aurora-postgres
    backend_type: s3
    component: aurora-postgres
    stacks:
    - globals
    - uw2-dev
    - uw2-globals
    - uw2-prod
    - uw2-staging
    - uw2-uat
    env:
      ENV_TEST_1: test1_override2
      ENV_TEST_2: test2_override2
      ENV_TEST_3: test3
      ENV_TEST_4: test4
      ENV_TEST_5: test5
      ENV_TEST_6: test6
      ENV_TEST_7: test7
      ENV_TEST_8: test8
    settings:
      spacelift:
        autodeploy: true
        branch: dev
        triggers: []
        workspace_enabled: true
      version: 0
    vars:
      cluster_size: 1
      environment: uw2
      instance_type: db.r4.xlarge
      namespace: eg
      region: us-west-2
      stage: dev

helmfile:
  alb-controller:
    stacks:
    - eks/eks-defaults
    - uw2-dev
    - uw2-prod
    - uw2-staging
    - uw2-uat
    env: {}
    settings: {}
    vars:
      account_number: "1234567890"
      chart_values:
        enableCertManager: true
      environment: uw2
      installed: true
      namespace: eg
      region: us-west-2
      ssm_region: us-west-2
      stage: dev

what

• Add settings sections to data_source_stack_config_yaml data source to provide settings for Terraform and helmfile components

• Add env sections to data_source_stack_config_yaml data source to provide ENV vars for Terraform and helmfile components

• Workaround for a deep-merge bug in mergo.Merge()

• Fix failing tests

• Add stack_processor_test.go

why

• settings sections are deep-merged and used for consumption by external services (e.g. for Spacelift and Terraform Cloud Terraform modules)
• env sections are deep-merged and used to specify ENV vars for consumption by external services (e.g. for Spacelift and Terraform Cloud)
• Workaround for a deep-merge bug in mergo.Merge(). When deep-merging slice of maps in a for loop,
  mergo modifies the source of the previous loop iteration if it's a complex map and mergo gets a pointer to it,
  not only the destination of the current loop iteration.
• stack_processor_test.go to test the provider outputs using a YAML stack config

test

Given this config:

terraform:
  vars: {}
  settings:
    spacelift:
      workspace_enabled: false
      autodeploy: false
  env:
    ENV_TEST_1: test1
    ENV_TEST_2: test2
    ENV_TEST_3: test3

    aurora-postgres:
      vars:
        instance_type: db.r4.large
        cluster_size: 1
      env:
        ENV_TEST_4: test4
        ENV_TEST_5: test5
        ENV_TEST_6: test6
        ENV_TEST_7: test7

    aurora-postgres-2:
      component: aurora-postgres
      vars:
        instance_type: db.r4.xlarge
      settings:
        spacelift:
          workspace_enabled: true
          autodeploy: true
          branch: "dev"
          triggers: []
      env:
        ENV_TEST_1: test1_override2
        ENV_TEST_2: test2_override2
        ENV_TEST_8: test8

    eks:
      vars:
        spotinst_instance_profile: eg-gbl-dev-spotinst-worker
        spotinst_oceans:
          main:
            desired_group_size: 1
            max_group_size: 3
            min_group_size: 1
            kubernetes_version: null
            ami_release_version: null
            attributes: null
            disk_size: 100
            instance_types: null
            ami_type: "AL2_x86_64"
            tags: null
      settings:
        spacelift:
          workspace_enabled: true
          autodeploy: true
          branch: "test"
          triggers: []
      env:
        ENV_TEST_1: test1_override
        ENV_TEST_2: test2_override
        ENV_TEST_4: test4

it produces the following outputs:

uw2_uat_aurora_postgres_2_settings = {
  "spacelift" = {
    "autodeploy" = true
    "branch" = "dev"
     "triggers" =  []
    "workspace_enabled" = true
  }
}

uw2_dev_aurora_postgres_2_env = {
  "ENV_TEST_1" = "test1_override2"
  "ENV_TEST_2" = "test2_override2"
  "ENV_TEST_3" = "test3"
  "ENV_TEST_4" = "test4"
  "ENV_TEST_5" = "test5"
  "ENV_TEST_6" = "test6"
  "ENV_TEST_7" = "test7"
  "ENV_TEST_8" = "test8"
}

uw2_uat_aurora_postgres_settings = {
  "spacelift" = {
    "autodeploy" = false
    "workspace_enabled" = false
  }
}

uw2_dev_aurora_postgres_env = {
  "ENV_TEST_1" = "test1"
  "ENV_TEST_2" = "test2"
  "ENV_TEST_3" = "test3"
  "ENV_TEST_4" = "test4"
  "ENV_TEST_5" = "test5"
  "ENV_TEST_6" = "test6"
  "ENV_TEST_7" = "test7"
}

uw2_uat_eks_settings = {
  "spacelift" = {
    "autodeploy" = false
    "branch" = "test"
    "triggers" = []
    "workspace_enabled" = true
  }
}

uw2_dev_eks_env = {
  "ENV_TEST_1" = "test1_override"
  "ENV_TEST_2" = "test2_override"
  "ENV_TEST_3" = "test3"
  "ENV_TEST_4" = "test4"
}

what

• Add component attribute to the outputs of Terraform components that inherit from a base component

why

• Useful for Terraform modules that use the provider to know if a component has inherited vars from a base component
• Used in remote backends to decide whether or not to add the component name to the Terraform workspace name

test

    aurora-postgres:
      vars:
        instance_type: db.r4.large
        cluster_size: 2

    aurora-postgres-2:
      component: aurora-postgres
      vars:
        cluster_size: 3
        instance_type: db.r4.xlarge

       "aurora-postgres" = {
          "backend" = {
            "acl" = "bucket-owner-full-control"
            "bucket" = "eg-uw2-root-tfstate"
            "dynamodb_table" = "eg-uw2-root-tfstate-lock"
            "encrypt" = true
            "key" = "terraform.tfstate"
            "region" = "us-west-2"
            "role_arn" = "arn:aws:iam::XXXXXXXXXXXX:role/eg-gbl-root-terraform"
            "workspace_key_prefix" = "aurora-postgres"
          }
          "backend_type" = "s3"
          "vars" = {
            "cluster_size" = 2
            "environment" = "uw2"
            "instance_type" = "db.r4.large"
            "namespace" = "eg"
            "region" = "us-west-2"
            "stage" = "uat"
          }
        }
        "aurora-postgres-2" = {
          "backend" = {
            "acl" = "bucket-owner-full-control"
            "bucket" = "eg-uw2-root-tfstate"
            "dynamodb_table" = "eg-uw2-root-tfstate-lock"
            "encrypt" = true
            "key" = "terraform.tfstate"
            "region" = "us-west-2"
            "role_arn" = "arn:aws:iam::XXXXXXXXXXXX:role/eg-gbl-root-terraform"
            "workspace_key_prefix" = "aurora-postgres"
          }
          "backend_type" = "s3"
          "component" = "aurora-postgres"
          "vars" = {
            "cluster_size" = 3
            "environment" = "uw2"
            "instance_type" = "db.r4.xlarge"
            "namespace" = "eg"
            "region" = "us-west-2"
            "stage" = "uat"
          }
        }

what

• Add data_source_stack_config_yaml data source
• Fix deep-merging for data_source_deep_merge_yaml and data_source_deep_merge_json data sources

why

• data_source_stack_config_yaml data source is used to process YAML stack configurations for Terraform and helmfile components
• Speed up the stack processing (especially for SaaS as Spacelift where all components from all stacks are processed at once)
• Support unlimited imports and unlimited levels of imports in YAML configs
• Support component inheritance (aliases) using component attribute (inherit vars and backend attributes from the base component)
• deep-merging for data_source_deep_merge_yaml and data_source_deep_merge_json data sources was missing

related

• Closes #13

test

terraform config

NOTE: terraform apply finishes in less than a second processing the 4 stacks in the example below (including processing/deep-merging of all imports for all stacks, and deep-merging of vars and backend from different sections)

data "utils_stack_config_yaml" "example" {
  input = [
    "${path.module}/stacks/uw2-dev.yaml",
    "${path.module}/stacks/uw2-prod.yaml",
    "${path.module}/stacks/uw2-staging.yaml",
    "${path.module}/stacks/uw2-uat.yaml"
  ]
}

locals {
  result = [for i in data.utils_stack_config_yaml.example.output : yamldecode(i)]
}

output "output" {
  value = local.result
}

output "uw2_dev_datadog_vars" {
  value = local.result[0]["components"]["helmfile"]["datadog"]["vars"]
}

output "uw2_dev_eks_config" {
  value = local.result[0]["components"]["terraform"]["eks"]
}

output "uw2_prod_vpc_vars" {
  value = local.result[1]["components"]["terraform"]["vpc"]["vars"]
}

output "uw2_staging_aurora_postgres_backend" {
  value = local.result[2]["components"]["terraform"]["aurora-postgres"]["backend"]
}

output "uw2_staging_aurora_postgres_2_backend" {
  value = local.result[2]["components"]["terraform"]["aurora-postgres-2"]["backend"]
}

output "uw2_uat_aurora_postgres_vars" {
  value = local.result[3]["components"]["terraform"]["aurora-postgres"]["vars"]
}

output "uw2_uat_aurora_postgres_2_vars" {
  value = local.result[3]["components"]["terraform"]["aurora-postgres-2"]["vars"]
}

outputs

uw2_dev_datadog_vars = {
  "account_number" = "1234567890"
  "apm" = {
    "enabled" = true
  }
  "clusterAgent" = {
    "enabled" = true
  }
  "datadogTags" = [
    "env:uw2-dev",
    "region:us-west-2",
    "stage:dev",
  ]
  "environment" = "uw2"
  "installed" = true
  "namespace" = "eg"
  "processAgent" = {
    "enabled" = true
  }
  "region" = "us-west-2"
  "ssm_region" = "us-west-2"
  "stage" = "dev"
  "systemProbe" = {
    "enabled" = true
  }
}
uw2_dev_eks_config = {
  "backend" = {
    "acl" = "bucket-owner-full-control"
    "bucket" = "eg-uw2-root-tfstate"
    "dynamodb_table" = "eg-uw2-root-tfstate-lock"
    "encrypt" = true
    "key" = "terraform.tfstate"
    "region" = "us-west-2"
    "role_arn" = "arn:aws:iam::XXXXXXXXXXXX:role/eg-gbl-root-terraform"
    "workspace_key_prefix" = "eks"
  }
  "backend_type" = "s3"
  "vars" = {
    "environment" = "uw2"
    "namespace" = "eg"
    "region" = "us-west-2"
    "region_availability_zones" = [
      "us-west-2b",
      "us-west-2c",
      "us-west-2d",
    ]
    "spotinst_instance_profile" = "eg-gbl-dev-spotinst-worker"
    "spotinst_oceans" = {
      "main" = {
        "ami_release_version" = null
        "ami_type" = "AL2_x86_64"
        "attributes" = null
        "desired_group_size" = 1
        "disk_size" = 100
        "instance_types" = null
        "kubernetes_version" = null
        "max_group_size" = 3
        "min_group_size" = 1
        "tags" = null
      }
    }
    "stage" = "dev"
  }
}
uw2_prod_vpc_vars = {
  "availability_zones" = [
    "us-west-2b",
    "us-west-2c",
    "us-west-2d",
  ]
  "cidr_block" = "10.116.0.0/18"
  "environment" = "uw2"
  "namespace" = "eg"
  "region" = "us-west-2"
  "stage" = "prod"
  "subnet_type_tag_key" = "eg.com/subnet/type"
  "vpc_flow_logs_bucket_environment_name" = "uw2"
  "vpc_flow_logs_bucket_stage_name" = "audit"
  "vpc_flow_logs_enabled" = true
  "vpc_flow_logs_traffic_type" = "ALL"
}
uw2_staging_aurora_postgres_2_backend = {
  "acl" = "bucket-owner-full-control"
  "bucket" = "eg-uw2-root-tfstate"
  "dynamodb_table" = "eg-uw2-root-tfstate-lock"
  "encrypt" = true
  "key" = "terraform.tfstate"
  "region" = "us-west-2"
  "role_arn" = "arn:aws:iam::XXXXXXXXXXXX:role/eg-gbl-root-terraform"
  "workspace_key_prefix" = "aurora-postgres"
}
uw2_staging_aurora_postgres_backend = {
  "acl" = "bucket-owner-full-control"
  "bucket" = "eg-uw2-root-tfstate"
  "dynamodb_table" = "eg-uw2-root-tfstate-lock"
  "encrypt" = true
  "key" = "terraform.tfstate"
  "region" = "us-west-2"
  "role_arn" = "arn:aws:iam::XXXXXXXXXXXX:role/eg-gbl-root-terraform"
  "workspace_key_prefix" = "aurora-postgres"
}
uw2_uat_aurora_postgres_2_vars = {
  "cluster_size" = 3
  "environment" = "uw2"
  "instance_type" = "db.r4.xlarge"
  "namespace" = "eg"
  "region" = "us-west-2"
  "stage" = "uat"
}
uw2_uat_aurora_postgres_vars = {
  "cluster_size" = 2
  "environment" = "uw2"
  "instance_type" = "db.r4.large"
  "namespace" = "eg"
  "region" = "us-west-2"
  "stage" = "uat"
}

Initial provider release