Replace a storage node

export WORK_DIR=/path/to/work/dir
mkdir -p "${WORK_DIR}"
pushd "${WORK_DIR}"

export EXOSCALE_API_KEY=<exoscale-key> (1)
export EXOSCALE_API_SECRET=<exoscale-secret>
export EXOSCALE_ZONE=<exoscale-zone> (2)
export EXOSCALE_S3_ENDPOINT="sos-${EXOSCALE_ZONE}.exo.io"

# From https://git.vshn.net/-/user_settings/personal_access_tokens, "api" scope is sufficient
export GITLAB_TOKEN=<gitlab-api-token>
export GITLAB_USER=<gitlab-user-name>

export VAULT_ADDR=https://vault-prod.syn.vshn.net
vault login -method=oidc

export HIERADATA_REPO_SECRET=$(vault kv get \
  -format=json "clusters/kv/lbaas/hieradata_repo_token" | jq '.data.data')
export HIERADATA_REPO_USER=$(echo "${HIERADATA_REPO_SECRET}" | jq -r '.user')
export HIERADATA_REPO_TOKEN=$(echo "${HIERADATA_REPO_SECRET}" | jq -r '.token')

commodore catalog compile "${CLUSTER_ID}"

cat <<EOF > ./terraform.env
EXOSCALE_API_KEY
EXOSCALE_API_SECRET
TF_VAR_control_vshn_net_token
GIT_AUTHOR_NAME
GIT_AUTHOR_EMAIL
HIERADATA_REPO_TOKEN
EOF

# Set terraform image and tag to be used
tf_image=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.image" \
  dependencies/openshift4-terraform/class/defaults.yml)
tf_tag=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.tag" \
  dependencies/openshift4-terraform/class/defaults.yml)

# Generate the terraform alias
base_dir=$(pwd)
alias terraform='touch .terraformrc; docker run -it --rm \
  -e REAL_UID=$(id -u) \
  -e TF_CLI_CONFIG_FILE=/tf/.terraformrc \
  --env-file ${base_dir}/terraform.env \
  -w /tf \
  -v $(pwd):/tf \
  --ulimit memlock=-1 \
  "${tf_image}:${tf_tag}" /tf/terraform.sh'

export GITLAB_REPOSITORY_URL=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r '.gitRepo.url' | sed 's|ssh://||; s|/|:|')
export GITLAB_REPOSITORY_NAME=${GITLAB_REPOSITORY_URL##*/}
export GITLAB_CATALOG_PROJECT_ID=$(curl -sH "Authorization: Bearer ${GITLAB_TOKEN}" "https://git.vshn.net/api/v4/projects?simple=true&search=${GITLAB_REPOSITORY_NAME/.git}" | jq -r ".[] | select(.ssh_url_to_repo == \"${GITLAB_REPOSITORY_URL}\") | .id")
export GITLAB_STATE_URL="https://git.vshn.net/api/v4/projects/${GITLAB_CATALOG_PROJECT_ID}/terraform/state/cluster"

pushd catalog/manifests/openshift4-terraform/

terraform init \
  "-backend-config=address=${GITLAB_STATE_URL}" \
  "-backend-config=lock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=unlock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=username=${GITLAB_USER}" \
  "-backend-config=password=${GITLAB_TOKEN}" \
  "-backend-config=lock_method=POST" \
  "-backend-config=unlock_method=DELETE" \
  "-backend-config=retry_wait_min=5"

if [[ "$OSTYPE" == "darwin"* ]]; then alias date=gdate; fi
job_name=$(printf "POST-silence-rook-ceph-alerts-$(date +%s)" | tr '[:upper:]' '[:lower:]')
silence_duration='+60 minutes' (1)
kubectl --as=cluster-admin -n openshift-monitoring create -f- <<EOJ
apiVersion: batch/v1
kind: Job
metadata:
  name: ${job_name}
  labels:
    app: silence-rook-ceph-alerts
spec:
 backoffLimit: 0
 template:
  spec:
    restartPolicy: Never
    containers:
      - name: silence
        image: quay.io/appuio/oc:v4.13
        command:
        - bash
        - -c
        - |
          curl_opts=( --cacert /etc/ssl/certs/serving-certs/service-ca.crt --header "Content-Type: application/json" --header "Authorization: Bearer \$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" --resolve alertmanager-main.openshift-monitoring.svc.cluster.local:9095:\$(getent hosts alertmanager-operated.openshift-monitoring.svc.cluster.local | awk '{print \$1}' | head -n 1) --silent )
          read -d "" body << EOF
          {
            "matchers": [
              {
                "name": "syn_component",
                "value": "rook-ceph",
                "isRegex": false
              }
            ],
            "startsAt": "$(date -u +'%Y-%m-%dT%H:%M:%S')",
            "endsAt": "$(date -u +'%Y-%m-%dT%H:%M:%S' --date "${silence_duration}")",
            "createdBy": "$(kubectl config current-context | cut -d/ -f3)",
            "comment": "Silence rook-ceph alerts"
          }
          EOF

          curl "\${curl_opts[@]}" \
            "https://alertmanager-main.openshift-monitoring.svc.cluster.local:9095/api/v2/silences" \
            -XPOST -d "\${body}"

        volumeMounts:
        - mountPath: /etc/ssl/certs/serving-certs/
          name: ca-bundle
          readOnly: true
        - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
          name: kube-api-access
          readOnly: true
    serviceAccountName: prometheus-k8s
    volumes:
    - name: ca-bundle
      configMap:
        defaultMode: 288
        name: serving-certs-ca-bundle
    - name: kube-api-access
      projected:
        defaultMode: 420
        sources:
          - serviceAccountToken:
              expirationSeconds: 3607
              path: 'token'
EOJ

silence_id=$(kubectl --as=cluster-admin -n openshift-monitoring logs jobs/${job_name} | \
  jq -r '.silenceID')

kubectl --as=cluster-admin -n syn patch apps root --type=json \
  -p '[{"op":"replace", "path":"/spec/syncPolicy", "value": {}}]'
kubectl --as=cluster-admin -n syn patch apps rook-ceph --type=json \
  -p '[{"op":"replace", "path":"/spec/syncPolicy", "value": {}}]'

export NODE_TO_REPLACE=storage-XXXX

if [[ "$OSTYPE" == "darwin"* ]]; then alias date=gdate; fi
job_name=$(printf "DELETE-silence-rook-ceph-alerts-$(date +%s)" | tr '[:upper:]' '[:lower:]')
kubectl --as=cluster-admin -n openshift-monitoring create -f- <<EOJ
apiVersion: batch/v1
kind: Job
metadata:
  name: ${job_name}
  labels:
    app: silence-rook-ceph-alerts
spec:
 backoffLimit: 0
 template:
  spec:
    restartPolicy: Never
    containers:
      - name: silence
        image: quay.io/appuio/oc:v4.13
        command:
        - bash
        - -c
        - |
          curl_opts=( --cacert /etc/ssl/certs/serving-certs/service-ca.crt --header "Content-Type: application/json" --header "Authorization: Bearer \$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" --resolve alertmanager-main.openshift-monitoring.svc.cluster.local:9095:\$(getent hosts alertmanager-operated.openshift-monitoring.svc.cluster.local | awk '{print \$1}' | head -n 1) --silent )

          curl "\${curl_opts[@]}" \
            "https://alertmanager-main.openshift-monitoring.svc.cluster.local:9095/api/v2/silence/${silence_id}" \
            -XDELETE

        volumeMounts:
        - mountPath: /etc/ssl/certs/serving-certs/
          name: ca-bundle
          readOnly: true
        - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
          name: kube-api-access
          readOnly: true
    serviceAccountName: prometheus-k8s
    volumes:
    - name: ca-bundle
      configMap:
        defaultMode: 288
        name: serving-certs-ca-bundle
    - name: kube-api-access
      projected:
        defaultMode: 420
        sources:
          - serviceAccountToken:
              expirationSeconds: 3607
              path: 'token'
EOJ

kubectl --as=cluster-admin -n openshift-monitoring delete jobs -l app=silence-rook-ceph-alerts

kubectl --as=cluster-admin -n syn patch apps root --type=json \
  -p '[{
    "op":"replace",
    "path":"/spec/syncPolicy",
    "value": {"automated": {"prune": true, "selfHeal": true}}
  }]'