Etcd backup and restore

export RESTIC_PASSWORD=$(vault kv get -field=password clusters/kv/<tenant>/${CLUSTER_ID}/cluster-backup)
restic -r s3:<s3-endpoint>/${CLUSTER_ID}-cluster-backup --no-lock \
  dump latest /syn-cluster-backup-object-dumper.tar > dump.tar
tar xf dump.tar ./split/syn-cluster-api/Secret.json -O > capi-secrets.jsonl

jq -s '{apiVersion:"v1", kind:"List", items:[ .[]
  | (.items? // [.])[]?
  | select(.kind == "Secret")
  | select(.metadata.name | IN(env.CLUSTER_ID+"-ca", env.CLUSTER_ID+"-talos",
      env.CLUSTER_ID+"-talosconfig", env.CLUSTER_ID+"-kubeconfig", "capcs-cloudscale"))
  | del(.metadata.ownerReferences, .metadata.uid, .metadata.resourceVersion,
        .metadata.creationTimestamp, .metadata.managedFields, .metadata.generation,
        .metadata.annotations, .status) ]}' capi-secrets.jsonl > preseed.yaml

jq -r '.items[]|select(.metadata.name==env.CLUSTER_ID+"-talosconfig").data.talosconfig' preseed.yaml | base64 -d > talosconfig-restored
jq -r '.items[]|select(.metadata.name==env.CLUSTER_ID+"-kubeconfig").data.value' preseed.yaml | base64 -d > restored.kubeconfig

export KUBECONFIG=./seed.kubeconfig
clusterctl init -n syn-cluster-api \
  --core cluster-api:<version> --bootstrap talos:<version> \
  --control-plane talos:<version> --infrastructure cloudscale-ch-cloudscale:<version>

# patched cacppt CRDs + image (from the guided setup repo), then verify the image
kubectl apply --server-side --force-conflicts -f cloudscale/manifests/cacppt-crds.yaml \
  -f cloudscale/manifests/cacppt-clusterrole.yaml -f cloudscale/manifests/cacppt-webhooks.yaml
kubectl -n syn-cluster-api set image deploy/cacppt-controller-manager manager=<patched-cacppt-image>

# let the bootstrap provider adopt the original secrets
kubectl -n syn-cluster-api apply -f preseed.yaml

export TALOSCONFIG=./talosconfig-restored

# verify that the old talosconfig authenticates against a new node
talosctl -e <new-cp1> -n <new-cp1> service etcd     # expect STATE=Preparing

# restore the snapshot on one node only
talosctl -e <new-cp1> -n <new-cp1> bootstrap --recover-from=./snap.db
talosctl -e <new-cp1> -n <new-cp1>,<new-cp2>,<new-cp3> etcd members   # expect 3 members and no learners

export KUBECONFIG=./restored.kubeconfig
kubectl -n syn-argocd-operator scale deploy --all --replicas=0
kubectl -n syn scale deploy,sts --all --replicas=0
kubectl -n syn-cluster-api scale deploy --all --replicas=0

kubectl get nodes        # new control plane nodes and workers should be Ready
kubectl get namespaces   # existing ages confirm that this is the restored state
kubectl get pods -A      # running workloads confirm that service account tokens are valid
kubectl get csr --field-selector spec.signerName=kubernetes.io/kubelet-serving   # auto-approved by the restored kubelet-serving-cert-approver

export RESTORED_KUBECONFIG=./restored.kubeconfig
export SEED_KUBECONFIG=./seed.kubeconfig

for webhook in $(kubectl --kubeconfig "${RESTORED_KUBECONFIG}" \
  get validatingwebhookconfiguration,mutatingwebhookconfiguration -o name \
  | grep -iE 'cabpt|cacppt|capcs|capi'); do
  kubectl --kubeconfig "${RESTORED_KUBECONFIG}" get "${webhook}" -o json \
    | jq '(.webhooks[].failurePolicy)="Ignore"' \
    | kubectl --kubeconfig "${RESTORED_KUBECONFIG}" replace -f -
done

for resource in machine cloudscalemachine taloscontrolplane machineset \
  machinedeployment cluster cloudscalecluster cloudscalemachinetemplate \
  talosconfig talosconfigtemplate clusterresourceset clusterresourcesetbinding; do
  for object in $(kubectl --kubeconfig "${RESTORED_KUBECONFIG}" \
    -n syn-cluster-api get "${resource}" -o name 2>/dev/null); do
    kubectl --kubeconfig "${RESTORED_KUBECONFIG}" -n syn-cluster-api \
      patch "${object}" --type merge -p '{"metadata":{"finalizers":null}}' || true
    kubectl --kubeconfig "${RESTORED_KUBECONFIG}" -n syn-cluster-api \
      delete "${object}" --wait=false || true
  done
done

# expect no resources
kubectl --kubeconfig "${RESTORED_KUBECONFIG}" -n syn-cluster-api \
  get cluster,taloscontrolplane,machine,cloudscalemachine

# verify that the SOCKS tunnel is running
# restart it through the guided setup if needed
pgrep -af 'ssh.*-D.*19000'

KIND_NODE_IPS=$(kubectl --kubeconfig "${SEED_KUBECONFIG}" get nodes \
  -ojsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}' \
  | tr ' ' ',')
kubectl --kubeconfig "${SEED_KUBECONFIG}" -n syn-cluster-api \
  set env deploy/cacppt-controller-manager \
  http_proxy=socks5://<kind-gateway-ip>:19000 \
  https_proxy=socks5://<kind-gateway-ip>:19000 \
  no_proxy=10.244.0.0/16,10.96.0.0/16,${KIND_NODE_IPS}

kubectl --kubeconfig "${SEED_KUBECONFIG}" -n syn-cluster-api get machine -o wide

clusterctl move -n syn-cluster-api --kubeconfig "${SEED_KUBECONFIG}" \
  --to-kubeconfig "${RESTORED_KUBECONFIG}"

for webhook in $(kubectl --kubeconfig "${RESTORED_KUBECONFIG}" \
  get validatingwebhookconfiguration,mutatingwebhookconfiguration -o name \
  | grep -iE 'cabpt|cacppt|capcs|capi'); do
  kubectl --kubeconfig "${RESTORED_KUBECONFIG}" get "${webhook}" -o json \
    | jq '(.webhooks[].failurePolicy)="Fail"' \
    | kubectl --kubeconfig "${RESTORED_KUBECONFIG}" replace -f -
done

kubectl --kubeconfig "${RESTORED_KUBECONFIG}" -n syn-cluster-api \
  scale deploy --all --replicas=1

kubectl --kubeconfig "${RESTORED_KUBECONFIG}" -n syn-cluster-api \
  get machine -o wide

kubectl --kubeconfig "${RESTORED_KUBECONFIG}" -n syn-argocd-operator \
  scale deploy --all --replicas=1
kubectl --kubeconfig "${RESTORED_KUBECONFIG}" -n syn \
  scale deploy,sts --all --replicas=1

kind delete cluster -n capi-seed-${CLUSTER_ID}