diff --git a/.github/workflows/ci-pipeline.yaml b/.github/workflows/ci-pipeline.yaml index e1d309b..e459ad8 100644 --- a/.github/workflows/ci-pipeline.yaml +++ b/.github/workflows/ci-pipeline.yaml @@ -5,9 +5,14 @@ on: branches: - main - 'feature/**' - paths-ignore: - - 'images/**' - - '**/*.md' + paths: + - '.github/workflows/ci-pipeline.yaml' + - 'src/**/*' + - 'tests/**/*' + - 'terraform/**/*.tf' + - 'terraform/**/*.tfvars' + - 'Dockerfile' + - 'pyproject.toml' env: PYTHON_IMAGE: 'python:3.12-slim' @@ -15,6 +20,31 @@ env: DOCKER_BUILD_SUMMARY: false # Deactivate build summary generation jobs: + check-infra-code: + name: Check Infrastructure Code + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./terraform + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Init Terraform CLI + uses: hashicorp/setup-terraform@v3 + + - name: Run Format Check + run: | + terraform fmt -check -diff + + - name: Terraform Init + run: | + terraform init -input=false -backend=false + + - name: Run Validation + run: | + terraform validate + test-and-check: name: Test and Check Python Code runs-on: ubuntu-latest diff --git a/.github/workflows/create-infra-pipeline.yaml b/.github/workflows/create-infra-pipeline.yaml new file mode 100644 index 0000000..f769978 --- /dev/null +++ b/.github/workflows/create-infra-pipeline.yaml @@ -0,0 +1,64 @@ +name: Create Infrastructure Pipeline + +on: + push: + branches: + - main + - 'feature/**' + paths: + - '.github/workflows/create-infra-pipeline.yaml' + - 'terraform/**/*.tf' + - 'terraform/**/*.tfvars' + +permissions: + id-token: write # Needed for Azure CLI Login + +env: + TF_ENV_VARS_FILE_PATH: 'environments/dev.tfvars' + TF_PLAN_FILE_PATH: '/tmp/tf_plan' + +jobs: + run-tf: + name: "Run Terraform Code" + runs-on: ubuntu-latest + environment: development # This would need to be environment-specific + defaults: + run: + working-directory: ./terraform + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.INFRA_ARM_CLIENT_ID }} + tenant-id: ${{ secrets.ARM_TENANT_ID }} + subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }} + + - name: Init Terraform CLI + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + run: | + terraform init \ + --backend-config=client_id=${{ secrets.INFRA_ARM_CLIENT_ID }} \ + --backend-config=tenant_id=${{ secrets.ARM_TENANT_ID }} + + - name: Terraform Plan + run: | + terraform plan \ + -var-file=$TF_ENV_VARS_FILE_PATH \ + -var 'dockerhub_username=${{ secrets.DOCKERHUB_USERNAME }}' \ + -var 'dockerhub_password=${{ secrets.DOCKERHUB_TOKEN }}' \ + -out=$TF_PLAN_FILE_PATH + + - name: Terraform Apply + if: github.ref_name == 'main' + run: | + terraform apply \ + -var-file=$TF_ENV_VARS_FILE_PATH \ + -var 'dockerhub_username=${{ secrets.DOCKERHUB_USERNAME }}' \ + -var 'dockerhub_password=${{ secrets.DOCKERHUB_TOKEN }}' \ + -auto-approve \ + $TF_PLAN_FILE_PATH diff --git a/.github/workflows/destroy-infra-pipeline.yaml b/.github/workflows/destroy-infra-pipeline.yaml new file mode 100644 index 0000000..eed5a00 --- /dev/null +++ b/.github/workflows/destroy-infra-pipeline.yaml @@ -0,0 +1,45 @@ +name: Destroy Infrastructure Pipeline + +on: + workflow_dispatch: + +permissions: + id-token: write # Needed for Azure CLI Login + +env: + TF_ENV_VARS_FILE_PATH: 'environments/dev.tfvars' + +jobs: + run-tf: + name: "Run Terraform Code" + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./terraform + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.INFRA_ARM_CLIENT_ID }} + tenant-id: ${{ secrets.ARM_TENANT_ID }} + subscription-id: ${{ secrets.ARM_SUBSCRIPTION_ID }} + + - name: Init Terraform CLI + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + run: | + terraform init \ + --backend-config=client_id=${{ secrets.INFRA_ARM_CLIENT_ID }} \ + --backend-config=tenant_id=${{ secrets.ARM_TENANT_ID }} + + - name: Terraform Destroy + run: | + terraform destroy \ + -var-file=$TF_ENV_VARS_FILE_PATH \ + -var 'dockerhub_username=${{ secrets.DOCKERHUB_USERNAME }}' \ + -var 'dockerhub_password=${{ secrets.DOCKERHUB_TOKEN }}' \ + -auto-approve diff --git a/.gitignore b/.gitignore index 0a19790..73cd0c9 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,6 @@ cython_debug/ # PyPI configuration file .pypirc + +# Terraform +terraform/.terraform \ No newline at end of file diff --git a/README.md b/README.md index 05ca98e..cea8784 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,12 @@ A learning experience for setting up a CI/CD pipeline for Python ## ToDo -- Create Terraform scripts to deploy Docker image on Azure -- Create pipeline to setup infrastructure using Terraform scripts - Create (separate?) pipeline to run continuous delivery on newly setup infrastructure - Check remaining TODO comments - Add project documentation + +## Infrastructure + +TODO: Infrastructure Sketch + +For more detailed info see [Infrastructure README](terraform/README.md). diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..eb0286e --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.117.1" + constraints = "~> 3.0" + hashes = [ + "h1:3c9iOEtBMnHrpJLlhbQ0sCZPWhE/2dvEPcL8KkXAh7w=", + "zh:0c513676836e3c50d004ece7d2624a8aff6faac14b833b96feeac2e4bc2c1c12", + "zh:50ea01ada95bae2f187db9e926e463f45d860767a85ebc59160414e00e76c35d", + "zh:52c2a9edacc06b3f72153f5ef6daca0761c6292158815961fe37f60bc576a3d7", + "zh:618eed2a06b19b1a025b45b05891846d570a6a1cca4d23f4942f5a99e1f747ae", + "zh:61cde5d3165d7e5ec311d5d89486819cd605c1b2d54611b5c97bd4e97dba2762", + "zh:6a873358d5031fc222f5e05f029d1237f3dce8345c767665f393283dfa2627f6", + "zh:afdd80064b2a04da311856feb4ed45f77ff4df6c356e8c2b10afb51fe7e61c70", + "zh:b09113df7e0e8c8959539bd22bae6c39faeb269ba3c4cd948e742f5cf58c35fb", + "zh:d340db7973109761cfc27d52aa02560363337c908b2c99b3628adc5a70a99d5b", + "zh:d5a577226ebc8c65e8f19384878a86acc4b51ede4b4a82d37c3b331b0efcd4a7", + "zh:e2962b147f9e71732df8dbc74940c10d20906f3c003cbfaa1eb9fabbf601a9f0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..0421ef5 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,35 @@ +## Infrastructure + +*In a real-world use case, one would seperate at least three environments of infrastructure: development, staging and production. This separation was out of the scope of this sample project. However, I showed how a possible separation could look like, e.g. with different .tfvar files.* + +**Requirements** + +For simplicity, I assumed that some resources are already present and did not automate the creation of them. I decided that this was out of scope of this sample project. Here are the (manual) steps you need to take in order to setup the project in Azure. This can be done either in the Azure Portal or via the Azure CLI: + +- Create Accounts + - Create a DockerHub Account and an access token for it + - Create an Azure Account +- Create Resources + - Create a resource group (To contain all Azure resources associated with this repository) + - Create a storage account and container inside this resource group (To store Terraform state file) + - Create a user-managed identity +- Configure Resources + - Give the user-managed identity the following privileges: + - `Storage Blob Contributor` on the Storage Account Container (To read and write the Terraform state file) + - `Contributor` on the resource group (To add, change, and delete resources in it via Terraform) + - Create a federated credential for the main branch of your forked repository + - E.g. I created one with the following subject for the original repository: `repo:matrop/python-cicd:ref:refs/heads/main` +- Add GitHub Actions Secrets + - Add the client id of the newly created identity as GitHub Actions secret `INFRA_ARM_CLIENT_ID` + - Add the tenant id of your Azure account as GitHub Actions secret `ARM_TENANT_ID` + - Add the DockerHub Account name as GitHub Actions secret `DOCKERHUB_USERNAME` + - Add the DockerHub Account token as GitHub Actions secret `DOCKERHUB_TOKEN` + +**Open ID Connect** + +We use Open ID Connect (OIDC) in order to authenticate Terraform to Azure and provision resources. The usage of OIDC instead of client-secret-based authentication has several advantages: +- We do not need to store credentials in Azure or the CI/CD pipelines. They are requested automatically +- Credentials are short-lived and we do not need to rotate them +- Credentials are granular and enable a detailed access concept + +One downside with this is the lack of wildcards in the subject identifier, e.g. for branch names. I read about it [here](https://learn.microsoft.com/en-us/answers/questions/2073829/azure-github-action-federated-identity-login-issue). In a real-world use case where many different tags or branches, e.g. feature branches, would make and apply infrastructure changes, this would be an issue. \ No newline at end of file diff --git a/terraform/backend.tf b/terraform/backend.tf new file mode 100644 index 0000000..b0fec71 --- /dev/null +++ b/terraform/backend.tf @@ -0,0 +1,9 @@ +terraform { + backend "azurerm" { + use_oidc = true + use_azuread_auth = true + storage_account_name = "samauriceatropsdev" # This would need to be environment-specific + container_name = "tfstate" + key = "terraform.tfstate" + } +} diff --git a/terraform/data.tf b/terraform/data.tf new file mode 100644 index 0000000..641213c --- /dev/null +++ b/terraform/data.tf @@ -0,0 +1,3 @@ +data "azurerm_resource_group" "rg" { + name = var.resource_group_name +} \ No newline at end of file diff --git a/terraform/environments/dev.tfvars b/terraform/environments/dev.tfvars new file mode 100644 index 0000000..5685340 --- /dev/null +++ b/terraform/environments/dev.tfvars @@ -0,0 +1,11 @@ +resource_group_name = "mauriceatrops-dev" +resource_group_location = "westeurope" + +container_group_name = "maurice-sample-container-group-dev" +container_group_restart_policy = "Always" + +container_name = "python-cicd-container" +container_image = "python-cicd:latest" +container_cpu_cores = 1 +container_memory_in_gb = 1 +container_port = 8080 diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..b5e19c6 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,26 @@ +resource "azurerm_container_group" "container" { + name = var.container_group_name + location = data.azurerm_resource_group.rg.location + resource_group_name = data.azurerm_resource_group.rg.name + ip_address_type = "Public" + os_type = "Linux" + restart_policy = var.container_group_restart_policy + + container { + name = var.container_name + image = "${var.dockerhub_username}/${var.container_image}" + cpu = var.container_cpu_cores + memory = var.container_memory_in_gb + + ports { + port = var.container_port + protocol = "TCP" + } + } + + image_registry_credential { + server = "index.docker.io" + username = var.dockerhub_username + password = var.dockerhub_password + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..5a07d7b --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,3 @@ +output "container_ipv4_address" { + value = azurerm_container_group.container.ip_address +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..e43dab4 --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">=1.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~>3.0" + } + } +} + +provider "azurerm" { + features {} +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..0785f87 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,60 @@ +variable "resource_group_name" { + type = string + description = "Name of the resource group in which the project is deployed" +} + +variable "resource_group_location" { + type = string + description = "Location of the resource group" +} + +variable "container_group_name" { + type = string + description = "Name of the container group which runs the project image" +} + +variable "container_group_restart_policy" { + type = string + description = "The behavior of Azure runtime if container has stopped" + validation { + condition = contains(["Always", "Never", "OnFailure"], var.container_group_restart_policy) + error_message = "The restart_policy must be one of the following: Always, Never, OnFailure." + } +} + +variable "container_name" { + type = string + description = "Name of the container inside the container group which runs the project image" +} + +variable "container_image" { + type = string + description = "Name of the image that should be run" +} + +variable "container_cpu_cores" { + type = number + description = "Number of cpu cores the container will have available" +} + +variable "container_memory_in_gb" { + type = number + description = "Storage size in GB the container will have available" +} + +variable "container_port" { + type = number + description = "Port that the container exposes" +} + +variable "dockerhub_username" { + type = string + sensitive = true + description = "Username (and repo name) for the image repository on Dockerhub" +} + +variable "dockerhub_password" { + type = string + sensitive = true + description = "Pasword for the image repository on Dockerhub" +}