diff --git a/.buildkite/job-version-bump.json.py b/.buildkite/job-version-bump.json.py index a979d30be..980acc6c9 100644 --- a/.buildkite/job-version-bump.json.py +++ b/.buildkite/job-version-bump.json.py @@ -10,100 +10,158 @@ # # This script generates JSON for the ml-cpp version bump pipeline. # It is intended to be triggered by the centralized release-eng pipeline. -# It can be integrated into existing or new workflows and includes a plugin -# that polls artifact URLs until the expected version is available. +# +# Supports two workflows via the WORKFLOW env var: +# patch (default) — bump version on BRANCH, wait for 2 artifact sets +# minor — create minor branch + bump BRANCH, wait for 3 artifact sets import contextlib import json +import os + + +WOLFI_IMAGE = "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest" +STAGING_URL = "https://artifacts-staging.elastic.co/ml-cpp/latest" +SNAPSHOT_URL = "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest" + + +def json_watcher_plugin(url, expected_value): + return { + "elastic/json-watcher#v1.0.0": { + "url": url, + "field": ".version", + "expected_value": expected_value, + "polling_interval": "30", + } + } + + +def dra_step(label, key, depends_on, plugins): + return { + "label": label, + "key": key, + "depends_on": depends_on, + "agents": { + "image": WOLFI_IMAGE, + "cpu": "250m", + "memory": "512Mi", + "ephemeralStorage": "1Gi", + }, + "command": [ + 'echo "Waiting for DRA artifacts..."', + ], + "timeout_in_minutes": 240, + "retry": { + "automatic": [{"exit_status": "*", "limit": 2}], + "manual": {"permit_on_passed": True}, + }, + "plugins": plugins, + } def main(): - pipeline = {} - # TODO: replace the block step with version bump logic + workflow = os.environ.get("WORKFLOW", "patch") + pipeline_steps = [ { - "block": "Ready to fetch for DRA artifacts?", - "prompt": ( - "Unblock when your team is ready to proceed.\n\n" - "Trigger parameters:\n" - "- NEW_VERSION: ${NEW_VERSION}\n" - "- BRANCH: ${BRANCH}\n" - "- WORKFLOW: ${WORKFLOW}\n" - ), - "key": "block-get-dra-artifacts", - "blocked_state": "running", - }, - { - "label": "Fetch DRA Artifacts", - "key": "fetch-dra-artifacts", - "depends_on": "block-get-dra-artifacts", + "label": "Bump version to ${NEW_VERSION}", + "key": "bump-version", "agents": { - "image": "docker.elastic.co/release-eng/wolfi-build-essential-release-eng:latest", + "image": WOLFI_IMAGE, "cpu": "250m", "memory": "512Mi", - "ephemeralStorage": "1Gi", }, "command": [ - 'echo "Starting DRA artifacts retrieval..."', - ], - "timeout_in_minutes": 240, - "retry": { - "automatic": [ - { - "exit_status": "*", - "limit": 2, - } - ], - "manual": {"permit_on_passed": True}, - }, - "plugins": [ - { - "elastic/json-watcher#v1.0.0": { - "url": "https://artifacts-staging.elastic.co/ml-cpp/latest/${BRANCH}.json", - "field": ".version", - "expected_value": "${NEW_VERSION}", - "polling_interval": "30", - } - }, - { - "elastic/json-watcher#v1.0.0": { - "url": "https://storage.googleapis.com/elastic-artifacts-snapshot/ml-cpp/latest/${BRANCH}.json", - "field": ".version", - "expected_value": "${NEW_VERSION}-SNAPSHOT", - "polling_interval": "30", - } - }, + "dev-tools/bump_version.sh", ], }, ] - pipeline["steps"] = pipeline_steps - pipeline["notify"] = [ - { - "slack": {"channels": ["#machine-learn-build"]}, - "if": ( - "(build.branch == 'main' || " - "build.branch =~ /^[0-9]+\\.[0-9x]+$/) && " - "(build.state == 'passed' || build.state == 'failed')" - ), - }, - { - "slack": { - "channels": ["#machine-learn-build"], - "message": ( - "🚦 Pipeline waiting for approval 🚦\n" - "Repo: `${REPO}`\n\n" - "Ready to fetch DRA artifacts - please unblock when ready.\n" - "New version: `${NEW_VERSION}`\n" - "Branch: `${BRANCH}`\n" - "Workflow: `${WORKFLOW}`\n" - "${BUILDKITE_BUILD_URL}\n" + if workflow == "minor": + # Minor workflow: artifact checks for both the upstream branch and the + # new minor branch, running in parallel after the bump step. + # + # Derive the minor branch from NEW_VERSION: if NEW_VERSION=9.5.0 + # then the previous minor (the new branch) is 9.4 with version 9.4.0. + new_version = os.environ.get("NEW_VERSION", "0.0.0") + parts = new_version.split(".") + if len(parts) >= 2: + major, minor_num = parts[0], int(parts[1]) + minor_branch = f"{major}.{minor_num - 1}" + minor_version = f"{major}.{minor_num - 1}.0" + else: + minor_branch = "unknown" + minor_version = "unknown" + + pipeline_steps.append( + dra_step( + label=f"Fetch DRA Artifacts (${{BRANCH}})", + key="fetch-dra-upstream", + depends_on="bump-version", + plugins=[ + json_watcher_plugin( + f"{STAGING_URL}/${{BRANCH}}.json", + "${NEW_VERSION}", + ), + json_watcher_plugin( + f"{SNAPSHOT_URL}/${{BRANCH}}.json", + "${NEW_VERSION}-SNAPSHOT", + ), + ], + ) + ) + + pipeline_steps.append( + dra_step( + label=f"Fetch DRA Artifacts ({minor_branch})", + key="fetch-dra-minor", + depends_on="bump-version", + plugins=[ + json_watcher_plugin( + f"{STAGING_URL}/{minor_branch}.json", + minor_version, + ), + json_watcher_plugin( + f"{SNAPSHOT_URL}/{minor_branch}.json", + f"{minor_version}-SNAPSHOT", + ), + ], + ) + ) + else: + # Patch workflow: staging + snapshot for BRANCH + pipeline_steps.append( + dra_step( + label="Fetch DRA Artifacts", + key="fetch-dra-artifacts", + depends_on="bump-version", + plugins=[ + json_watcher_plugin( + f"{STAGING_URL}/${{BRANCH}}.json", + "${NEW_VERSION}", + ), + json_watcher_plugin( + f"{SNAPSHOT_URL}/${{BRANCH}}.json", + "${NEW_VERSION}-SNAPSHOT", + ), + ], + ) + ) + + pipeline = { + "steps": pipeline_steps, + "notify": [ + { + "slack": {"channels": ["#machine-learn-build"]}, + "if": ( + "(build.branch == 'main' || " + "build.branch =~ /^[0-9]+\\.[0-9x]+$/) && " + "(build.state == 'passed' || build.state == 'failed')" ), }, - "if": 'build.state == "blocked"', - }, - ] + ], + } print(json.dumps(pipeline, indent=2)) diff --git a/dev-tools/bump_version.sh b/dev-tools/bump_version.sh new file mode 100755 index 000000000..481e6a06d --- /dev/null +++ b/dev-tools/bump_version.sh @@ -0,0 +1,177 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Automated version bump script for the release-eng pipeline. +# +# Supports two workflows: +# WORKFLOW=patch (default) +# Updates elasticsearchVersion in gradle.properties to $NEW_VERSION +# on $BRANCH, commits, and pushes. +# +# WORKFLOW=minor +# 1. Creates a new minor branch from $BRANCH (e.g., 9.4 from main) +# inheriting the current version. +# 2. Bumps $BRANCH to $NEW_VERSION (the next minor). +# Both branches are pushed. +# +# Set DRY_RUN=true to perform all steps except git push. +# +# Follows the same pattern as the Elasticsearch repo's automated +# Lucene snapshot updates (.buildkite/scripts/lucene-snapshot/). + +set -euo pipefail + +: "${NEW_VERSION:?NEW_VERSION must be set}" +: "${BRANCH:?BRANCH must be set}" +WORKFLOW="${WORKFLOW:-patch}" +DRY_RUN="${DRY_RUN:-false}" + +GRADLE_PROPS="gradle.properties" + +if [ "$DRY_RUN" = "true" ]; then + echo "=== DRY RUN MODE — will not push ===" +fi + +git_push() { + local target_branch="$1" + if [ "$DRY_RUN" = "true" ]; then + echo " [DRY RUN] Would push $target_branch" + else + git push origin "$target_branch" + echo " Pushed $target_branch" + fi +} + +sed_inplace() { + if sed --version >/dev/null 2>&1; then + sed -i "$@" + else + sed -i '' "$@" + fi +} + +configure_git() { + git config user.name elasticsearchmachine + git config user.email 'infra-root+elasticsearchmachine@elastic.co' +} + +bump_version_on_branch() { + local target_branch="$1" + local target_version="$2" + + git checkout "$target_branch" + git pull --ff-only origin "$target_branch" + + local current_version + current_version=$(grep '^elasticsearchVersion=' "$GRADLE_PROPS" | cut -d= -f2) + if [ "$current_version" = "$target_version" ]; then + echo "Version on $target_branch is already $target_version — nothing to do" + return 0 + fi + + echo "Bumping version on $target_branch: $current_version → $target_version" + sed_inplace "s/^elasticsearchVersion=.*/elasticsearchVersion=${target_version}/" "$GRADLE_PROPS" + + if ! grep -q "^elasticsearchVersion=${target_version}$" "$GRADLE_PROPS"; then + echo "ERROR: version update verification failed on $target_branch" + grep 'elasticsearchVersion' "$GRADLE_PROPS" + exit 1 + fi + + if git diff-index --quiet HEAD --; then + echo "No changes to commit on $target_branch (file unchanged after sed)" + return 0 + fi + + configure_git + git add "$GRADLE_PROPS" + git commit -m "[ML] Bump version to ${target_version}" + git_push "$target_branch" +} + +# --------------------------------------------------------------------------- +# Patch workflow: bump version on the target branch +# --------------------------------------------------------------------------- +do_patch() { + echo "=== Patch workflow: bump $BRANCH to $NEW_VERSION ===" + bump_version_on_branch "$BRANCH" "$NEW_VERSION" +} + +# --------------------------------------------------------------------------- +# Minor workflow: create minor branch, then bump upstream to next minor +# --------------------------------------------------------------------------- +do_minor() { + echo "=== Minor workflow: create minor branch from $BRANCH, then bump to $NEW_VERSION ===" + + git checkout "$BRANCH" + git pull --ff-only origin "$BRANCH" + + local current_version + current_version=$(grep '^elasticsearchVersion=' "$GRADLE_PROPS" | cut -d= -f2) + + # Derive the minor branch name from current version (e.g., 9.4.0 → 9.4) + local major minor + major=$(echo "$current_version" | cut -d. -f1) + minor=$(echo "$current_version" | cut -d. -f2) + local minor_branch="${major}.${minor}" + + echo "Current version on $BRANCH: $current_version" + echo "Minor branch to create: $minor_branch" + echo "New version for $BRANCH: $NEW_VERSION" + + # Export minor branch info for downstream Buildkite steps + if [ "${BUILDKITE:-false}" = "true" ]; then + buildkite-agent meta-data set "MINOR_BRANCH" "$minor_branch" + buildkite-agent meta-data set "MINOR_VERSION" "$current_version" + fi + export MINOR_BRANCH="$minor_branch" + export MINOR_VERSION="$current_version" + + # Check if the minor branch already exists on the remote + if git ls-remote --exit-code --heads origin "$minor_branch" >/dev/null 2>&1; then + echo "Branch $minor_branch already exists on origin — skipping creation" + else + echo "Creating branch $minor_branch from $BRANCH..." + git checkout -b "$minor_branch" + configure_git + git_push "$minor_branch" + echo "Branch $minor_branch created with version $current_version" + fi + + # Now bump the upstream branch to the new version + bump_version_on_branch "$BRANCH" "$NEW_VERSION" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +case "$WORKFLOW" in + patch) + do_patch + ;; + minor) + do_minor + ;; + *) + echo "ERROR: unknown WORKFLOW '$WORKFLOW' (expected 'patch' or 'minor')" + exit 1 + ;; +esac + +if [ "$DRY_RUN" = "true" ]; then + echo "" + echo "=== DRY RUN SUMMARY ===" + echo "Workflow: $WORKFLOW" + echo "Branch: $BRANCH" + echo "Version: $NEW_VERSION" + echo "Recent commits:" + git log --oneline -3 +fi