diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..eb8ea46 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Report a reproducible problem in the Cloud Security Suite +title: "[Bug]: " +labels: bug +assignees: "" +--- + +## Summary + +## Steps to reproduce + +1. +2. +3. + +## Expected behavior + +## Actual behavior + +## Environment + +- OS: +- Python version: +- Terraform version: +- AWS region: + +## Logs or screenshots + +## Additional context diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..9acdcbc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest a new check, detection, integration, or workflow +title: "[Feature]: " +labels: enhancement +assignees: "" +--- + +## Summary + +## Problem this solves + +## Proposed solution + +## Alternatives considered + +## Additional context diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..e400de1 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## Summary + +## What changed + +## Why + +## Testing performed + +## Screenshots (if applicable) + +## Checklist + +- [ ] Tests added or updated +- [ ] Documentation updated +- [ ] Terraform formatted with `terraform fmt -recursive` +- [ ] Local tests and linting pass diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..23ed143 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,41 @@ +name: lint + +on: + pull_request: + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run Ruff + run: ruff check . + + - name: Run mypy on shared modules + run: mypy shared/ + + - name: Run mypy on tools with AWS typing leniency + run: > + mypy tools/ + --ignore-missing-imports + --allow-untyped-defs + --allow-incomplete-defs + --disable-error-code attr-defined + --disable-error-code no-any-return + --disable-error-code type-arg + --disable-error-code arg-type + --disable-error-code misc diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..a80ada1 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,34 @@ +name: security + +on: + pull_request: + schedule: + - cron: "0 9 * * 1" + +jobs: + security: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Install scanners + run: | + python -m pip install --upgrade pip + pip install bandit pip-audit checkov + + - name: Run Bandit + run: bandit -r shared tools -x "*/tests/*,*/sample_data/*" + + - name: Run pip-audit + run: pip-audit -r requirements.txt + + - name: Run Checkov + run: checkov -d infrastructure --quiet --soft-fail diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..39e51cf --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,41 @@ +name: terraform + +on: + pull_request: + paths: + - "infrastructure/**" + - ".github/workflows/terraform.yml" + +jobs: + terraform: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.6.6" + + - name: Check Terraform formatting + working-directory: infrastructure + run: terraform fmt -check -recursive + + - name: Initialize Terraform without backend + working-directory: infrastructure + run: terraform init -backend=false + + - name: Validate Terraform + working-directory: infrastructure + run: terraform validate + + - name: Set up tflint + uses: terraform-linters/setup-tflint@v4 + + - name: Run tflint best-effort + working-directory: infrastructure + run: | + tflint --init + tflint --recursive || true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1264d45 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,36 @@ +name: test + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests with coverage + run: | + pytest --cov=shared --cov=tools --cov-report=term --cov-report=xml --cov-fail-under=80 -v + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: ./coverage.xml + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index 6dfcaee..35f83f4 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ dist/ build/ .pytest_cache/ .coverage +coverage.xml htmlcov/ .mypy_cache/ .ruff_cache/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a9046cb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +# Contributing + +Thanks for improving the Cloud Security Suite. This project uses small, reviewable pull requests and CI checks for tests, linting, Terraform validation, and security scanning. + +## Development Workflow + +1. Fork the repository. +2. Create a branch from `main`. +3. Make focused changes with tests and documentation when behavior changes. +4. Open a pull request and wait for CI to pass before requesting review. + +## Branch Names + +Use short branch names with one of these prefixes: + +- `feat/` for new features +- `fix/` for bug fixes +- `docs/` for documentation-only changes +- `ci/` for workflow and automation changes +- `chore/` for maintenance + +## Commit Messages + +Use Conventional Commits: + +```text +feat(iam-auditor): add access key rotation check +fix(cloudtrail): handle gzipped sample logs +docs(infra): document remote state setup +ci: add terraform validation workflow +``` + +## Run Tests Locally + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +pytest --cov=shared --cov=tools --cov-report=term --cov-fail-under=80 -v +ruff check . +mypy shared/ +mypy tools/ \ + --ignore-missing-imports \ + --allow-untyped-defs \ + --allow-incomplete-defs \ + --disable-error-code attr-defined \ + --disable-error-code no-any-return \ + --disable-error-code type-arg \ + --disable-error-code arg-type \ + --disable-error-code misc +``` + +## Terraform Validation + +Package Lambda artifacts before validation because Terraform references the generated zip files: + +```bash +cd infrastructure +./modules/iam_auditor/package.sh +./modules/guardduty_processor/package.sh +./modules/cloudtrail_analyzer/package.sh +terraform fmt -recursive +terraform init -backend=false +terraform validate +``` + +The real `terraform.tfvars` file is gitignored. Use `terraform.tfvars.example` as the starting point for local testing. diff --git a/README.md b/README.md index 4ceea4a..7214923 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,19 @@ # Cloud Security Suite +[![test](https://img.shields.io/github/actions/workflow/status/huslayer826/cloud-security-suite/test.yml?branch=main&label=tests)](https://github.com/huslayer826/cloud-security-suite/actions/workflows/test.yml) +[![lint](https://img.shields.io/github/actions/workflow/status/huslayer826/cloud-security-suite/lint.yml?branch=main&label=lint)](https://github.com/huslayer826/cloud-security-suite/actions/workflows/lint.yml) +[![coverage](https://img.shields.io/codecov/c/github/huslayer826/cloud-security-suite?label=coverage)](https://codecov.io/gh/huslayer826/cloud-security-suite) +![Python](https://img.shields.io/badge/python-3.11-blue) +![Terraform](https://img.shields.io/badge/terraform-%3E%3D1.6-844FBA) +[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) + ## Project Pitch A Python and Terraform suite for automated AWS security auditing, threat detection, and incident response. ## Badges -Placeholder for project badges. +CI badges track tests, linting, coverage, supported runtime versions, and license. ## Architecture Diagram diff --git a/infrastructure/.terraform.lock.hcl b/infrastructure/.terraform.lock.hcl new file mode 100644 index 0000000..f4078c4 --- /dev/null +++ b/infrastructure/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.42.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:B00CO2gJ6fSyfUGhi+siRqNoUG9jI7PD+3r1dHWv3OI=", + "zh:0dd774a97eaa4371a60e13b5dc56800d4fb1c48d50e79049f75fc4fe26705ff5", + "zh:237d652d8ec028f7bedce1ce056ffe42e2e120d2a4a47fe45b97263cc5948e7e", + "zh:367d9e4b816e5f857956887b8f0770aaa5bec47c4b1e2c7bf924e39192d580d6", + "zh:4194addb5b34bb803fab031a80d84e9d16cac2308df9a498d51e2214c7893900", + "zh:5bcc36226fa5d8a3c37e41ac8cfd2b1eb73e7ae96f8418f6776dcaa16a987198", + "zh:61d0632d4cc7973b779b90c1d3bb2d05a1cd4d7030bb4645831e67cf735ecaa0", + "zh:7c7efaf9e4bb662ba3e8a714abafe7107bdcd1bf2cd0867510ea32762debc11d", + "zh:7d7f8ffe00d4a90184efa454a107bcc46d81f21245baea9678dcfa2fa77ac1be", + "zh:7e534c454cdeafe9cc225bea53397883bb96c54da6ea3421fd53dbc9dfc1bb90", + "zh:99564c99a0672b2a8b666ab2040f36a7c6f372fa79ac43a6657a54734e46fff8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a6b1bb6178798882508f5512222a9433fc21e63cdae83a755650c938e6141d32", + "zh:a87592d6ff3d46ee83756f4f84503b2fe4ffc9c1d5dc9f8d4afccb5e126ae538", + "zh:daa56fb74c5c9d26b327c40b486beac71cdb9a932c7c4e4561f4401cd0d16a4a", + "zh:f35ab043d7121f3a194f42f5b0e0d59fe62d94488acea07e3783405fa5838785", + ] +} diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 75b2268..704b9b4 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -21,14 +21,14 @@ terraform { module "iam_auditor" { source = "./modules/iam_auditor" - project_name = var.project_name - environment = var.environment - aws_region = var.aws_region - notification_email = var.notification_email - schedule_expression = var.schedule_expression - lambda_package_path = "${path.module}/modules/iam_auditor/iam_auditor_lambda.zip" - lambda_source_hash = try(filebase64sha256("${path.module}/modules/iam_auditor/iam_auditor_lambda.zip"), null) - permissions_boundary = var.permissions_boundary + project_name = var.project_name + environment = var.environment + aws_region = var.aws_region + notification_email = var.notification_email + schedule_expression = var.schedule_expression + lambda_package_path = "${path.module}/modules/iam_auditor/iam_auditor_lambda.zip" + lambda_source_hash = try(filebase64sha256("${path.module}/modules/iam_auditor/iam_auditor_lambda.zip"), null) + permissions_boundary = var.permissions_boundary } module "guardduty_processor" { diff --git a/infrastructure/modules/cloudtrail_analyzer/main.tf b/infrastructure/modules/cloudtrail_analyzer/main.tf index 49b6483..8041ea5 100644 --- a/infrastructure/modules/cloudtrail_analyzer/main.tf +++ b/infrastructure/modules/cloudtrail_analyzer/main.tf @@ -68,22 +68,22 @@ resource "aws_glue_catalog_table" "cloudtrail_logs" { table_type = "EXTERNAL_TABLE" parameters = { - EXTERNAL = "TRUE" - "projection.enabled" = "true" - "projection.region.type" = "enum" - "projection.region.values" = var.aws_region - "projection.year.type" = "integer" - "projection.year.range" = "2020,2035" - "projection.month.type" = "integer" - "projection.month.range" = "1,12" - "projection.month.digits" = "2" - "projection.day.type" = "integer" - "projection.day.range" = "1,31" - "projection.day.digits" = "2" - "storage.location.template" = "${local.cloudtrail_location}/AWSLogs/${data.aws_caller_identity.current.account_id}/CloudTrail/$${region}/$${year}/$${month}/$${day}/" - "classification" = "json" - "compressionType" = "gzip" - "typeOfData" = "file" + EXTERNAL = "TRUE" + "projection.enabled" = "true" + "projection.region.type" = "enum" + "projection.region.values" = var.aws_region + "projection.year.type" = "integer" + "projection.year.range" = "2020,2035" + "projection.month.type" = "integer" + "projection.month.range" = "1,12" + "projection.month.digits" = "2" + "projection.day.type" = "integer" + "projection.day.range" = "1,31" + "projection.day.digits" = "2" + "storage.location.template" = "${local.cloudtrail_location}/AWSLogs/${data.aws_caller_identity.current.account_id}/CloudTrail/$${region}/$${year}/$${month}/$${day}/" + "classification" = "json" + "compressionType" = "gzip" + "typeOfData" = "file" } partition_keys { diff --git a/pyproject.toml b/pyproject.toml index 5e550e7..2155e29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,12 @@ select = ["E", "F", "I", "N", "UP", "B", "SIM"] [tool.mypy] python_version = "3.11" -strict = true +warn_unused_configs = true [[tool.mypy.overrides]] module = ["boto3", "boto3.*"] ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = ["shared", "shared.*"] +strict = true diff --git a/shared/aws_client.py b/shared/aws_client.py index 6cfa72f..e8bc9cb 100644 --- a/shared/aws_client.py +++ b/shared/aws_client.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import Any, cast + import boto3 from botocore.client import BaseClient @@ -18,11 +20,11 @@ def get_client( profile: str | None = None, ) -> BaseClient: """Return a boto3 client using default credentials or a named profile.""" - return _session(profile).client(service, region_name=region) + return cast(BaseClient, _session(profile).client(service, region_name=region)) # type: ignore[call-overload] def get_account_id(profile: str | None = None) -> str: """Return the AWS account ID for the active credentials.""" - sts_client = get_client("sts", profile=profile) + sts_client = cast(Any, get_client("sts", profile=profile)) identity = sts_client.get_caller_identity() return str(identity["Account"]) diff --git a/shared/tests/test_reporters.py b/shared/tests/test_reporters.py index fdc9e32..69044f7 100644 --- a/shared/tests/test_reporters.py +++ b/shared/tests/test_reporters.py @@ -1,4 +1,5 @@ import json +from pathlib import Path from rich.console import Console @@ -20,7 +21,7 @@ def make_finding(severity: Severity = Severity.MEDIUM) -> Finding: ) -def test_json_reporter_writes_findings_and_summary(tmp_path) -> None: +def test_json_reporter_writes_findings_and_summary(tmp_path: Path) -> None: output_path = tmp_path / "report.json" JSONReporter().write([make_finding(Severity.HIGH)], output_path) @@ -31,7 +32,7 @@ def test_json_reporter_writes_findings_and_summary(tmp_path) -> None: assert payload["findings"][0]["severity"] == "HIGH" -def test_html_reporter_renders_dark_report(tmp_path) -> None: +def test_html_reporter_renders_dark_report(tmp_path: Path) -> None: output_path = tmp_path / "report.html" HTMLReporter().write([make_finding(Severity.CRITICAL)], output_path) diff --git a/tools/cloudtrail_analyzer/main.py b/tools/cloudtrail_analyzer/main.py index 34eeb73..d9c9259 100644 --- a/tools/cloudtrail_analyzer/main.py +++ b/tools/cloudtrail_analyzer/main.py @@ -103,7 +103,7 @@ def build_athena_query(start_time: str | None, end_time: str | None) -> str: if end_time: filters.append(f"eventtime < '{end_time}'") where = f"WHERE {' AND '.join(filters)}" if filters else "" - return f"SELECT * FROM cloudtrail_logs {where}" + return f"SELECT * FROM cloudtrail_logs {where}" # nosec B608 def write_reports(findings, output: str, output_dir: str, metadata: dict[str, object]) -> None: diff --git a/tools/guardduty_processor/notifier.py b/tools/guardduty_processor/notifier.py index 8515a7a..bb9aa44 100644 --- a/tools/guardduty_processor/notifier.py +++ b/tools/guardduty_processor/notifier.py @@ -7,6 +7,7 @@ import os from typing import Any from urllib import request +from urllib.parse import urlparse import boto3 from botocore.exceptions import BotoCoreError, ClientError @@ -43,7 +44,7 @@ def notify( try: post_slack(webhook_url, finding, detail, remediation_findings) result["slack"] = "published" - except OSError as error: + except (OSError, ValueError) as error: LOGGER.warning("Unable to publish Slack notification: %s", error) result["slack"] = "failed" @@ -78,6 +79,10 @@ def post_slack( detail: dict[str, Any], remediation_findings: list[Finding], ) -> None: + parsed_url = urlparse(webhook_url) + if parsed_url.scheme != "https": + raise ValueError("Slack webhook URL must use HTTPS") + payload = { "blocks": [ { @@ -122,7 +127,7 @@ def post_slack( headers={"Content-Type": "application/json"}, method="POST", ) - with request.urlopen(req, timeout=5) as response: + with request.urlopen(req, timeout=5) as response: # nosec B310 response.read()