Do not merge test: wpb-22439 test for 5.14 backend version

ansible/helm_external.yml

@@ -49,6 +49,7 @@
 
 - hosts: "postgresql"
   become: false
+  tags: postgresql-external
   tasks:
   - name: Create external IP directory for postgresql
     file:

ansible/inventory/offline/staging.yml

@@ -82,31 +82,4 @@ rmq-cluster:
     # host name here must match each node's actual hostname
     rabbitmq_cluster_master: datanode1
 
-postgresql:
-  children:
-    datanodes: {}
-  vars:
-    wire_dbname: wire-server
-    postgresql_network_interface: enp1s0
-    repmgr_node_config:
-      datanode1:  # Maps to postgresql_rw group
-        node_id: 1
-        priority: 150
-        role: primary
-      datanode2:  # Maps to first postgresql_ro
-        node_id: 2
-        priority: 100
-        role: standby
-      datanode3:  # Maps to second postgresql_ro
-        node_id: 3
-        priority: 50
-        role: standby
-
-postgresql_rw:
-  hosts:
-    datanode1: {}
-
-postgresql_ro:
-  hosts:
-    datanode2: {}
-    datanode3: {}
+postgresql: {}

ansible/wiab-staging-provision.yml

@@ -257,7 +257,10 @@
         yq -i ".minio.vars.minio_network_interface |= \"$GATEWAY_NAME\"" "{{ inventory_file }}"
         yq -i ".rmq-cluster.vars.rabbitmq_network_interface |= \"$GATEWAY_NAME\"" "{{ inventory_file }}"
         yq -i ".rmq-cluster.vars.rabbitmq_cluster_master |= \"datanode1\"" "{{ inventory_file }}"
-        yq -i ".postgresql.vars.postgresql_network_interface |= \"$GATEWAY_NAME\"" "{{ inventory_file }}"
+
+        # setting up ansible_ssh_private_key_file to access VMs for secondary ansible operations
+        yq -i ".all.vars.ansible_ssh_private_key_file |= \"ssh/id_ed25519\"" "{{ inventory_file }}"
+
       args:
         executable: /bin/bash
 

bin/helm-operations.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2087
+set -Eeo pipefail
+
+# Read values from environment variables with defaults
+BASE_DIR="/wire-server-deploy"
+TARGET_SYSTEM="example.dev"
+CERT_MASTER_EMAIL="certmaster@${TARGET_SYSTEM}"
+
+# this IP should match the DNS A record value for TARGET_SYSTEM
+# assuming it to be the public address used by clients to reach public Address 
+HOST_IP=""
+if [ -z "$HOST_IP" ]; then 
+HOST_IP=$(wget -qO- https://api.ipify.org)
+fi
+
+# picking a node for calling traffic (3rd kube worker node)
+CALLING_NODE=$(kubectl get nodes --no-headers | tail -n 1 | awk '{print $1}')
+if [[ -z "$CALLING_NODE" ]]; then
+  echo "Error: could not determine the last kube worker node via kubectl"
+  exit 1
+fi
+
+# Creates values.yaml from prod-values.example.yaml and secrets.yaml from prod-secrets.example.yaml
+# Works on all chart directories in $BASE_DIR/values/
+process_values() {
+
+  ENV=$1
+  TYPE=$2
+  charts=(fake-aws demo-smtp databases-ephemeral reaper wire-server webapp account-pages team-settings smallstep-accomp ingress-nginx-controller nginx-ingress-services coturn sftd cert-manager)
+
+  if [[ "$ENV" != "prod" ]] || [[ -z "$TYPE" ]] ; then
+    echo "Error: This function only supports prod deployments with TYPE as values or secrets. ENV must be 'prod', got: '$ENV' and '$TYPE'"
+    exit 1
+  fi
+  timestp=$(date +"%Y%m%d_%H%M%S")
+
+  for chart in "${charts[@]}"; do
+    chart_dir="$BASE_DIR/values/$chart"
+    if [[ -d "$chart_dir" ]]; then
+      if [[ -f "$chart_dir/${ENV}-${TYPE}.example.yaml" ]]; then
+        if [[ ! -f "$chart_dir/${TYPE}.yaml" ]]; then
+          cp "$chart_dir/${ENV}-${TYPE}.example.yaml" "$chart_dir/${TYPE}.yaml"
+          echo "Used template ${ENV}-${TYPE}.example.yaml to create $chart_dir/${TYPE}.yaml"
+        else
+          echo "$chart_dir/${TYPE}.yaml already exists, archiving it and creating a new one."
+          mv "$chart_dir/${TYPE}.yaml" "$chart_dir/${TYPE}.yaml.bak.$timestp"
+          cp "$chart_dir/${ENV}-${TYPE}.example.yaml" "$chart_dir/${TYPE}.yaml"
+        fi
+      fi
+    fi
+  done
+}
+
+# selectively setting values of following charts which requires additional values
+# wire-server, webapp, team-settings, account-pages, nginx-ingress-services, sftd and coturn
+configure_values() {
+
+  TEMP_DIR=$(mktemp -d)
+  trap 'rm -rf $TEMP_DIR' EXIT
+
+  # to find IP address of calling NODE
+  CALLING_NODE_IP=$(kubectl get node "$CALLING_NODE" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+
+  # Fixing the hosts with TARGET_SYSTEM and setting the turn server
+  sed -e "s/example.com/$TARGET_SYSTEM/g" \
+      "$BASE_DIR/values/wire-server/values.yaml" > "$TEMP_DIR/wire-server-values.yaml"
+
+  # fixing the turnStatic values
+  yq eval -i ".brig.turnStatic.v2 = [\"turn:$HOST_IP:3478\", \"turn:$HOST_IP:3478?transport=tcp\"]" "$TEMP_DIR/wire-server-values.yaml"
+
+  # Fixing the hosts in webapp team-settings and account-pages charts
+  for chart in webapp team-settings account-pages; do
+    sed "s/example.com/$TARGET_SYSTEM/g" "$BASE_DIR/values/$chart/values.yaml" > "$TEMP_DIR/$chart-values.yaml"
+  done
+
+  # Setting certManager and DNS records
+  sed -e 's/useCertManager: false/useCertManager: true/g' \
+    -e "/certmasterEmail:$/s/certmasterEmail:/certmasterEmail: $CERT_MASTER_EMAIL/" \
+    -e "s/example.com/$TARGET_SYSTEM/" \
+    "$BASE_DIR/values/nginx-ingress-services/values.yaml" > "$TEMP_DIR/nginx-ingress-services-values.yaml"
+
+  # Fixing SFTD hosts and setting the cert-manager to http01
+  sed -e "s/webapp.example.com/webapp.$TARGET_SYSTEM/" \
+      -e "s/sftd.example.com/sftd.$TARGET_SYSTEM/" \
+      -e 's/name: letsencrypt-prod/name: letsencrypt-http01/' \
+      "$BASE_DIR/values/sftd/values.yaml" > "$TEMP_DIR/sftd-values.yaml"
+
+  # Setting coturn node IP values
+  yq eval -i ".coturnTurnListenIP = \"$CALLING_NODE_IP\"" "$BASE_DIR/values/coturn/values.yaml"
+  yq eval -i ".coturnTurnRelayIP = \"$CALLING_NODE_IP\"" "$BASE_DIR/values/coturn/values.yaml"
+  yq eval -i ".coturnTurnExternalIP = \"$HOST_IP\"" "$BASE_DIR/values/coturn/values.yaml"
+
+  # Compare and copy files if different
+  for file in wire-server-values.yaml webapp-values.yaml team-settings-values.yaml account-pages-values.yaml \
+              nginx-ingress-services-values.yaml sftd-values.yaml; do
+    if ! cmp -s "$TEMP_DIR/$file" "$BASE_DIR/values/${file%-values.yaml}/values.yaml"; then
+      cp "$TEMP_DIR/$file" "$BASE_DIR/values/${file%-values.yaml}/values.yaml"
+      echo "Updating  $BASE_DIR/values/${file%-values.yaml}/values.yaml"
+    fi
+  done
+
+}
+
+deploy_charts() {
+
+  local charts=("$@")
+  echo "Following charts will be deployed: ${charts[*]}"
+
+  for chart in "${charts[@]}"; do
+    chart_dir="$BASE_DIR/charts/$chart"
+    values_file="$BASE_DIR/values/$chart/values.yaml"
+    secrets_file="$BASE_DIR/values/$chart/secrets.yaml"
+
+    if [[ ! -d "$chart_dir" ]]; then
+      echo "Error: Chart directory $chart_dir does not exist. Exiting fix the charts"
+      exit 1
+    fi
+
+    if [[ ! -f "$values_file" ]]; then
+      echo "Warning: Values file $values_file does not exist. Deploying without values."
+      values_file=""
+    fi
+
+    if [[ ! -f "$secrets_file" ]]; then
+      secrets_file=""
+    fi
+
+    helm_command="helm upgrade --install --wait --timeout=15m0s $chart $chart_dir"
+
+    if [[ -n "$values_file" ]]; then
+      helm_command+=" --values $values_file"
+    fi
+
+    if [[ -n "$secrets_file" ]]; then
+      helm_command+=" --values $secrets_file"
+    fi
+
+    echo "Deploying $chart as $helm_command"
+    eval "$helm_command"
+  done
+
+  # display running pods post deploying all helm charts in default namespace
+  kubectl get pods --sort-by=.metadata.creationTimestamp
+}
+
+deploy_cert_manager() {
+
+  kubectl get namespace cert-manager-ns || kubectl create namespace cert-manager-ns
+  helm upgrade --install -n cert-manager-ns cert-manager  "$BASE_DIR/charts/cert-manager" --values "$BASE_DIR/values/cert-manager/values.yaml"
+
+  # display running pods
+  kubectl get pods --sort-by=.metadata.creationTimestamp -n cert-manager-ns
+}
+
+deploy_calling_services() {
+
+  echo "Deploying sftd and coturn"
+  # select the node to deploy sftd
+  kubectl annotate node "$CALLING_NODE" wire.com/external-ip="$HOST_IP" --overwrite
+  helm upgrade --install sftd "$BASE_DIR/charts/sftd" --set "nodeSelector.kubernetes\\.io/hostname=$CALLING_NODE" --values  "$BASE_DIR/values/sftd/values.yaml"
+
+  kubectl annotate node "$CALLING_NODE" wire.com/external-ip="$HOST_IP" --overwrite
+  helm upgrade --install coturn "$BASE_DIR/charts/coturn" --set "nodeSelector.kubernetes\\.io/hostname=$CALLING_NODE" --values "$BASE_DIR/values/coturn/values.yaml" --values "$BASE_DIR/values/coturn/secrets.yaml"
+}
+
+main() {
+# Create prod-values.example.yaml to values.yaml and take backup
+process_values "prod" "values"
+# Create prod-secrets.example.yaml to secrets.yaml and take backup
+process_values "prod" "secrets"
+
+# configure chart specific variables for each chart in values.yaml file
+configure_values
+
+# deploying with external datastores, useful for prod setup
+deploy_charts cassandra-external elasticsearch-external minio-external rabbitmq-external fake-aws demo-smtp databases-ephemeral reaper wire-server webapp account-pages team-settings smallstep-accomp ingress-nginx-controller
+
+# deploying cert manager to issue certs, by default letsencrypt-http01 issuer is configured
+deploy_cert_manager
+
+# nginx-ingress-services chart needs cert-manager to be deployed
+deploy_charts nginx-ingress-services
+
+# deploying sft and coturn services
+# not implemented yet
+deploy_calling_services
+
+# print status of certs
+kubectl get certificate
+}
+
+main

bin/offline-cluster.sh

@@ -50,10 +50,11 @@ ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags boot
 ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/cassandra.yml
 ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/elasticsearch.yml
 ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/minio.yml
-ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/postgresql-deploy.yml
+
+#ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/postgresql-deploy.yml
 
 ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/roles/rabbitmq-cluster/tasks/configure_dns.yml
 ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/rabbitmq.yml
 
 # create helm values that tell our helm charts what the IP addresses of cassandra, elasticsearch and minio are:
-ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/helm_external.yml
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/helm_external.yml --skip-tags=postgresql-external

bin/offline-deploy.sh

@@ -41,16 +41,4 @@ fi
 
 $DOCKER_RUN_BASE $SSH_MOUNT $WSD_CONTAINER ./bin/offline-cluster.sh
 
-# Sync PostgreSQL password from K8s secret to secrets.yaml
-echo "Syncing PostgreSQL password from Kubernetes secret..."
-sudo docker run --network=host -v $PWD:/wire-server-deploy $WSD_CONTAINER ./bin/sync-k8s-secret-to-wire-secrets.sh \
-  wire-postgresql-external-secret \
-  password \
-  values/wire-server/secrets.yaml \
-  .brig.secrets.pgPassword \
-  .galley.secrets.pgPassword \
-  .spar.secrets.pgPassword \
-  .gundeck.secrets.pgPassword
-
-
-sudo docker run --network=host -v $PWD:/wire-server-deploy $WSD_CONTAINER ./bin/offline-helm.sh
+sudo docker run --network=host -v $PWD:/wire-server-deploy $WSD_CONTAINER ./bin/helm-operations.sh

bin/offline-helm.sh

@@ -3,37 +3,37 @@
 set -euo pipefail
 set -x
 
-sync_pg_secrets() {
-  # Sync postgresql secret
-  ./bin/sync-k8s-secret-to-wire-secrets.sh \
-    wire-postgresql-external-secret \
-    password \
-    values/wire-server/secrets.yaml \
-    .brig.secrets.pgPassword \
-    .galley.secrets.pgPassword \
-    .spar.secrets.pgPassword \
-    .gundeck.secrets.pgPassword
-}
+#sync_pg_secrets() {
+#  # Sync postgresql secret
+#  ./bin/sync-k8s-secret-to-wire-secrets.sh \
+#    wire-postgresql-external-secret \
+#    password \
+#    values/wire-server/secrets.yaml \
+#    .brig.secrets.pgPassword \
+#    .galley.secrets.pgPassword \
+#    .spar.secrets.pgPassword \
+#    .gundeck.secrets.pgPassword
+#}
 
 helm upgrade --install --wait cassandra-external ./charts/cassandra-external --values ./values/cassandra-external/values.yaml
-helm upgrade --install --wait postgresql-external ./charts/postgresql-external --values ./values/postgresql-external/values.yaml
+#helm upgrade --install --wait postgresql-external ./charts/postgresql-external --values ./values/postgresql-external/values.yaml
 helm upgrade --install --wait elasticsearch-external ./charts/elasticsearch-external --values ./values/elasticsearch-external/values.yaml
 helm upgrade --install --wait minio-external ./charts/minio-external --values ./values/minio-external/values.yaml
 helm upgrade --install --wait rabbitmq-external ./charts/rabbitmq-external --values ./values/rabbitmq-external/values.yaml
 helm upgrade --install --wait fake-aws ./charts/fake-aws --values ./values/fake-aws/prod-values.example.yaml
 
-sync_pg_secrets
+#sync_pg_secrets
 
 # ensure that the RELAY_NETWORKS value is set to the podCIDR
-SMTP_VALUES_FILE="./values/smtp/prod-values.example.yaml"
+SMTP_VALUES_FILE="./values/demo-smtp/prod-values.example.yaml"
 podCIDR=$(kubectl get configmap -n kube-system kubeadm-config -o yaml | grep -i 'podSubnet' | awk '{print $2}' 2>/dev/null)
 
 if [[ $? -eq 0 && -n "$podCIDR" ]]; then
     sed -i "s|RELAY_NETWORKS: \".*\"|RELAY_NETWORKS: \":${podCIDR}\"|" $SMTP_VALUES_FILE
 else
     echo "Failed to fetch podSubnet. Attention using the default value: $(grep -i RELAY_NETWORKS $SMTP_VALUES_FILE)"
 fi
-helm upgrade --install --wait smtp ./charts/smtp --values $SMTP_VALUES_FILE
+helm upgrade --install --wait demo-smtp ./charts/demo-smtp --values $SMTP_VALUES_FILE
 
 
 # helm upgrade --install --wait rabbitmq ./charts/rabbitmq --values ./values/rabbitmq/prod-values.example.yaml --values ./values/rabbitmq/prod-secrets.example.yaml