Installation on Exoscale

OUTPUT=$(mktemp)


set -euo pipefail
echo "Checking prerequisites..."

if which yq >/dev/null 2>&1 ; then { echo "✅ yq is installed."; } ; else { echo "❌ yq is not installed. Please install yq to proceed."; exit 1; } ; fi
if yq --version | grep -E 'version v[4-9]\.' | grep 'mikefarah' >/dev/null 2>&1 ; then { echo "✅ yq by mikefarah version 4 or higher is installed."; } ; else { echo "❌ yq version 4 or higher is required. Please upgrade yq to proceed."; exit 1; } ; fi

if which jq >/dev/null 2>&1 ; then { echo "✅ jq is installed."; } ; else { echo "❌ jq is not installed. Please install jq to proceed."; exit 1; } ; fi

if which oc >/dev/null 2>&1 ; then { echo "✅ oc (OpenShift CLI) is installed."; } ; else { echo "❌ oc (OpenShift CLI) is not installed. Please install oc to proceed."; exit 1; } ; fi

if which vault >/dev/null 2>&1 ; then { echo "✅ vault (HashiCorp Vault) is installed."; } ; else { echo "❌ vault (HashiCorp Vault) is not installed. Please install vault to proceed."; exit 1; } ; fi

if which curl >/dev/null 2>&1 ; then { echo "✅ curl is installed."; } ; else { echo "❌ curl is not installed. Please install curl to proceed."; exit 1; } ; fi

if which docker >/dev/null 2>&1 ; then { echo "✅ docker is installed."; } ; else { echo "❌ docker is not installed. Please install docker to proceed."; exit 1; } ; fi

if which glab >/dev/null 2>&1 ; then { echo "✅ glab (GitLab CLI) is installed."; } ; else { echo "❌ glab (GitLab CLI) is not installed. Please install glab to proceed."; exit 1; } ; fi

if which host >/dev/null 2>&1 ; then { echo "✅ host (DNS lookup utility) is installed."; } ; else { echo "❌ host (DNS lookup utility) is not installed. Please install host to proceed."; exit 1; } ; fi

if which mc >/dev/null 2>&1 ; then { echo "✅ mc (MinIO Client) is installed."; } ; else { echo "❌ mc (MinIO Client) is not installed. Please install mc >= RELEASE.2024-01-18T07-03-39Z to proceed."; exit 1; } ; fi
mc_version=$(mc --version | grep -Eo 'RELEASE[^ ]+')
if echo "$mc_version" | grep -E 'RELEASE\.202[4-9]-' >/dev/null 2>&1 ; then { echo "✅ mc version ${mc_version} is sufficient."; } ; else { echo "❌ mc version ${mc_version} is insufficient. Please upgrade mc to >= RELEASE.2024-01-18T07-03-39Z to proceed."; exit 1; } ; fi

if which aws >/dev/null 2>&1 ; then { echo "✅ aws (AWS CLI) is installed."; } ; else { echo "❌ aws (AWS CLI) is not installed. Please install aws to proceed. Our recommended installer is uv: 'uv tool install awscli'"; exit 1; } ; fi

if which exo >/dev/null 2>&1 ; then { echo "✅ exo (Exoscale CLI) is installed."; } ; else { echo "❌ exo (Exoscale CLI) is not installed. Please install exo to proceed. See: https://community.exoscale.com/tools/command-line-interface/"; exit 1; } ; fi

if which restic >/dev/null 2>&1 ; then { echo "✅ restic (Backup CLI) is installed."; } ; else { echo "❌ restic (Backup CLI) is not installed. Please install restic to proceed."; exit 1; } ; fi

if which gzip >/dev/null 2>&1 ; then { echo "✅ gzip (file compression) is installed."; } ; else { echo "❌ gzip (file compression) is not installed. Please install gzip to proceed."; exit 1; } ; fi

if which md5sum >/dev/null 2>&1 ; then { echo "✅ md5sum (file checksums) is installed."; } ; else { echo "❌ md5sum (file checksums) is not installed. Please install md5sum to proceed."; exit 1; } ; fi

if which virt-edit >/dev/null 2>&1 ; then { echo "✅ virt-edit (VM image editing) is installed."; } ; else { echo "❌ virt-edit (VM image editing) is not installed. Please install virt-edit to proceed."; exit 1; } ; fi

if which cpio >/dev/null 2>&1 ; then { echo "✅ cpio (file archiving) is installed."; } ; else { echo "❌ cpio (file archiving) is not installed. Please install cpio to proceed."; exit 1; } ; fi

if which emergency-credentials-receive >/dev/null 2>&1 ; then { echo "✅ emergency-credentials-receive (Cluster emergency access helper) is installed."; } ; else { echo "❌ emergency-credentials-receive is not installed. Please install it from https://github.com/vshn/emergency-credentials-receive ."; exit 1; } ; fi

echo "✅ All prerequisites are met."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)


set -euo pipefail

openshift-install() {
  ./openshift-install "${@}"
}

if [[ -f openshift-install ]]
then
  echo "Found existing openshift-install binary, checking version ..."
  INSTALLED_VERSION=$(openshift-install version | sed -E -n '/openshift-install/s/^[^ ]+ v?//;s/\.[0-9]{1,2}$//p')
  if [ "$INSTALLED_VERSION" = "$MATCH_ocp_version" ]; then
    echo "✅ openshift-install version ${MATCH_ocp_version}.XX is present."
    env -i "openshift_install_bin=$( pwd )/openshift-install" >> "$OUTPUT"
    exit 0
  else
    echo "⚠️ openshift-install version $INSTALLED_VERSION is present, but version $MATCH_ocp_version is required. Deleting local binary and installing $MATCH_ocp_version"
    rm openshift-install
  fi
fi

rm -f openshift-install-linux.tar.gz
echo "Downloading openshift-install for version ${MATCH_ocp_version} ..."
curl -LO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable-${MATCH_ocp_version}/openshift-install-linux.tar.gz
tar -xf openshift-install-linux.tar.gz openshift-install

env -i "openshift_install_bin=$( pwd )/openshift-install" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_api_url=
# export INPUT_commodore_cluster_id=

set -euo pipefail
export COMMODORE_API_URL="${INPUT_commodore_api_url}"

echo "Retrieving Commodore tenant ID for cluster ID '$INPUT_commodore_cluster_id' from API at '$INPUT_commodore_api_url'..."
tenant_id=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${INPUT_commodore_cluster_id} | jq -r .tenant)
if echo "$tenant_id" | grep 't-' >/dev/null 2>&1 ; then { echo "✅ Retrieved tenant ID '$tenant_id' for cluster ID '$INPUT_commodore_cluster_id'."; } else { echo "❌ Failed to retrieve valid tenant ID for cluster ID '$INPUT_commodore_cluster_id'. Got '$tenant_id'. Please check your Commodore API access and cluster ID."; exit 1; } ; fi
env -i "commodore_tenant_id=$tenant_id" >> "$OUTPUT"

region=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${INPUT_commodore_cluster_id} | jq -r .facts.region)
if test -z "$region" && test "$region" != "null" ; then { echo "❌ Failed to retrieve CSP region for cluster ID '$INPUT_commodore_cluster_id'."; exit 1; } ; else { echo "✅ Retrieved CSP region '$region' for cluster ID '$INPUT_commodore_cluster_id'."; } ; fi
env -i "csp_region=$region" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_cluster_id=

echo '#########################################################'
echo '#                                                       #'
echo "#  Please create a Keycloak service with the cluster's  #"
echo '#  ID as Service Name via the VSHN Control Web UI.      #'
echo '#                                                       #'
echo '#########################################################'
echo
echo "The name and ID of the service should be ${INPUT_commodore_cluster_id}."
echo "You can go to https://control.vshn.net/vshn/services/_create"
sleep 2


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_csp_region=

set -euo pipefail
export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"
exo limits > /dev/null || { echo "Invalid token!" ; exit 1 ; }
echo "Token is valid"
env -i "exoscale_s3_endpoint=sos-${INPUT_csp_region}.exo.io" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_gitlab_api_token=

set -euo pipefail
user="$( curl -sH "Authorization: Bearer ${INPUT_gitlab_api_token}" "https://git.vshn.net/api/v4/user" | jq -r .username )"
if [[ "$user" == "null" ]]
then
  echo "Error validating GitLab token. Are you sure it is valid?"
  exit 1
fi
env -i "gitlab_user_name=$user" >> "$OUTPUT"
echo "Token is valid."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_control_vshn_api_token=

set -euo pipefail

AUTH="X-AccessToken: ${INPUT_control_vshn_api_token}"

code="$( curl -H"$AUTH" https://control.vshn.net/api/servers/1/appuio/ -o /dev/null -w"%{http_code}" )"

if [[ "$code" != 200 ]]
then
  echo "ERROR: could not access Server API (Status $code)"
  echo "Please ensure your token is valid and your IP is on the allowlist."
  exit 1
fi


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_base_domain=
# export INPUT_redhat_pull_secret=

set -euo pipefail
echo "Base domain set to ${INPUT_base_domain}".
echo "${INPUT_redhat_pull_secret}" | jq > /dev/null
echo "Pull secret appears legit."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=

set -euo pipefail
export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"

instances_used="$( exo limits -Ojson | jq '.[] | select(.resource == "Compute instances").used'  )"
instances_max="$( exo limits -Ojson | jq '.[] | select(.resource == "Compute instances").max'  )"

if (( instances_max - instances_used < 15 ))
then
  echo "Instance quota is insufficient! Please increase the instance quota."
  echo "Visit https://portal.exoscale.com/u/$( exo x get-organization | jq -r .id )/organization/quotas "
  exit 1
fi
echo "Instance quota is sufficient"

echo '###################################################################################'
echo '#                                                                                 #'
echo '#  Please check the DNS subscription and upgrade it if no DNS Domains are left.   #'
echo '#                                                                                 #'
echo '###################################################################################'
echo
echo "See https://portal.exoscale.com/u/$( exo x get-organization | jq -r .id )/dns"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_commodore_cluster_id=

set -euo pipefail
export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"

echo "Create SOS IAM role, if it doesn't exist yet in the organization"
sos_iam_role_id=$(exo iam role list -O json | \
  jq -r '.[] | select(.name=="sos-full-access") | .key')
if [ -z "${sos_iam_role_id}" ]; then
echo '{
  "default-service-strategy": "deny",
  "services": {
    "sos": {"type": "allow"}
  }
}' | \
exo iam role create sos-full-access \
  --description "Full access to object storage service" \
  --policy -
fi
echo "TODO: Check whether the key with that name already exists"
# Create access key
exoscale_s3_credentials=$(exo iam api-key create -O json \
  "${INPUT_commodore_cluster_id}_object_storage" sos-full-access)
EXOSCALE_S3_ACCESSKEY=$(echo "${exoscale_s3_credentials}" | jq -r '.key')
EXOSCALE_S3_SECRETKEY=$(echo "${exoscale_s3_credentials}" | jq -r '.secret')

echo
echo "Create Exoscale CSI driver Exoscale IAM role, if it doesn't exist yet in the organization"
csidriver_role_id=$(exo iam role list -O json | \
  jq -r '.[] | select(.name=="csi-driver-exoscale") | .key')
if [ -z "${csidriver_role_id}" ]; then
cat << EOF | exo iam role create csi-driver-exoscale \
  --description "Exoscale CSI Driver: Access to storage operations and zone list" \
  --policy -
{
  "default-service-strategy": "deny",
  "services": {
    "compute": {
      "type": "rules",
      "rules": [
        {
          "expression": "operation in ['list-zones', 'get-block-storage-volume', 'list-block-storage-volumes', 'create-block-storage-volume', 'delete-block-storage-volume', 'attach-block-storage-volume-to-instance', 'detach-block-storage-volume', 'update-block-storage-volume-labels', 'resize-block-storage-volume', 'get-block-storage-snapshot', 'list-block-storage-snapshots', 'create-block-storage-snapshot', 'delete-block-storage-snapshot']",
          "action": "allow"
        }
      ]
    }
  }
}
EOF
fi
echo "TODO: Check whether the key with that name already exists"
# Create access key
csi_credentials=$(exo iam api-key create -O json \
  "${INPUT_commodore_cluster_id}_csi-driver-exoscale" csi-driver-exoscale)
CSI_ACCESSKEY=$(echo "${csi_credentials}" | jq -r '.key')
CSI_SECRETKEY=$(echo "${csi_credentials}" | jq -r '.secret')

echo
echo "Create Exoscale CCM Exoscale IAM role, if it doesn't exist yet in the organization"
ccm_role_id=$(exo iam role list -O json | \
  jq -r '.[] | select(.name=="ccm-exoscale") | .key')
if [ -z "${ccm_role_id}" ]; then
cat <<EOF | exo iam role create ccm-exoscale \
  --description "Exoscale CCM: Allow managing NLBs and reading instances/instance pools" \
  --policy -
{
  "default-service-strategy": "deny",
  "services": {
    "compute": {
      "type": "rules",
      "rules": [
        {
          "expression": "operation in ['add-service-to-load-balancer', 'create-load-balancer', 'delete-load-balancer', 'delete-load-balancer-service', 'get-load-balancer', 'get-load-balancer-service', 'get-operation', 'list-load-balancers', 'reset-load-balancer-field', 'reset-load-balancer-service-field', 'update-load-balancer', 'update-load-balancer-service']",
          "action": "allow"
        },
        {
          "expression": "operation in ['get-instance', 'get-instance-pool', 'get-instance-type', 'list-instances', 'list-instance-pools', 'list-zones']",
          "action": "allow"
        }
      ]
    }
  }
}
EOF
fi
echo "TODO: Check whether the key with that name already exists"
# Create access key
ccm_credentials=$(exo iam api-key create -O json \
  "${INPUT_commodore_cluster_id}_ccm-exoscale" ccm-exoscale)
CCM_ACCESSKEY=$(echo "${ccm_credentials}" | jq -r '.key')
CCM_SECRETKEY=$(echo "${ccm_credentials}" | jq -r '.secret')

env -i "s3_key=${EXOSCALE_S3_ACCESSKEY}" >> "$OUTPUT"
env -i "s3_secret=${EXOSCALE_S3_SECRETKEY}" >> "$OUTPUT"
env -i "csi_key=${CSI_ACCESSKEY}" >> "$OUTPUT"
env -i "csi_secret=${CSI_SECRETKEY}" >> "$OUTPUT"
env -i "ccm_key=${CCM_ACCESSKEY}" >> "$OUTPUT"
env -i "ccm_secret=${CCM_SECRETKEY}" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_api_url=
# export INPUT_csp_region=

set -euo pipefail
export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"
export COMMODORE_API_URL="${INPUT_commodore_api_url}"

exo storage create "sos://${INPUT_commodore_cluster_id}-bootstrap" --zone "${INPUT_csp_region}"

distribution="$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${INPUT_commodore_cluster_id} | jq -r .facts.distribution)"
if [[ "$distribution" != "oke" ]]
then
  exo storage create "sos://${INPUT_commodore_cluster_id}-logstore" --zone "${INPUT_csp_region}"
fi


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)


set -euo pipefail

# TODO(aa): this is not generic; it makes assumptions on the location of the script on the host.
# We will need a solution where gandalf provides the workflow base dir, e.g. in an env var
# shellcheck disable=1091
. /workflows/shared/scripts/semver.sh

MAJOR=0
MINOR=0
PATCH=0
# shellcheck disable=2034
SPECIAL=""
semverParseInto "$MATCH_image_name" MAJOR MINOR PATCH SPECIAL

image_path="rhcos-$MAJOR.$MINOR.qcow2"

env -i "image_major=$MAJOR" >> "$OUTPUT"
env -i "image_minor=$MINOR" >> "$OUTPUT"
env -i "image_patch=$PATCH" >> "$OUTPUT"

echo "Image is $image_path"

if [ -f "$image_path" ]; then
  echo "Image $image_path already exists, skipping download."
  env -i "image_path=$image_path" >> "$OUTPUT"
  exit 0
fi

echo Downloading OpenShift image "$MATCH_image_name" to "$image_path"

curl -L "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MAJOR}.${MINOR}/${MATCH_image_name}/rhcos-${MATCH_image_name}-x86_64-openstack.x86_64.qcow2.gz" | gzip -d > "$image_path"
env -i "image_path=$image_path" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_image_path=

set -euo pipefail

patched_path="${INPUT_image_path%.*}-patched.qcow2"
cp "${INPUT_image_path}" "${patched_path}"
env -i "patched_image_path=$patched_path" >> "$OUTPUT"

echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
echo "@                                                                                       @"
echo "@  Please run the following command manually. (This step is not yet automated)          @"
echo "@                                                                                       @"
echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
echo
echo "sudo virt-edit -a ${patched_path} -m /dev/sda3:/ /loader/entries/ostree-1.conf -e 's/openstack/exoscale/'"
echo
echo '(If you have some time, feel free to implement automation for this step)'
sleep 3


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_patched_image_path=
# export INPUT_commodore_cluster_id=
# export INPUT_csp_region=
# export INPUT_image_major=
# export INPUT_image_minor=
# export INPUT_image_patch=
# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_exoscale_s3_endpoint=

set -euo pipefail

export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"

RHCOS_VERSION="${INPUT_image_major}.${INPUT_image_minor}.${INPUT_image_patch}"

slug="$( exo compute instance-template list -v private -z "${INPUT_csp_region}" -Ojson | jq -r '.[] | select(.name | startswith("rhcos-'${INPUT_image_major}.${INPUT_image_minor}'")) | .name' )"

if [ -n "$slug" ] && [ "$slug" != "null" ]; then
  echo "Image '$slug' already exists in Exoscale, skipping upload."
  env -i "rhcos_template=$slug" >> "$OUTPUT"
  exit 0
fi


exo storage upload "${INPUT_patched_image_path}" "sos://${INPUT_commodore_cluster_id}-bootstrap" --acl public-read

sleep 3

exo compute instance-template register "rhcos-${RHCOS_VERSION}" \
  "https://${INPUT_exoscale_s3_endpoint}/${INPUT_commodore_cluster_id}-bootstrap/rhcos-${RHCOS_VERSION}.qcow2" \
  "$(md5sum ${INPUT_patched_image_path} | awk '{ print $1 }')" \
  --zone "${INPUT_csp_region}" \
  --boot-mode uefi \
  --disable-password \
  --username core \
  --description "Red Hat Enterprise Linux CoreOS (RHCOS) ${RHCOS_VERSION}"

exo storage delete -f "sos://${CLUSTER_ID}-bootstrap/rhcos-${RHCOS_VERSION}.qcow2"

export RHCOS_TEMPLATE="rhcos-${RHCOS_VERSION}"

env -i "rhcos_template=$RHCOS_TEMPLATE" >> "$OUTPUT"

echo "Image import initiated. ⚠️ TODO: Poll for completion."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_vault_address=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_tenant_id=
# export INPUT_s3_key=
# export INPUT_s3_secret=
# export INPUT_csi_key=
# export INPUT_csi_secret=
# export INPUT_ccm_key=
# export INPUT_ccm_secret=

set -euo pipefail

export VAULT_ADDR=${INPUT_vault_address}
vault login -method=oidc

# Set the Exoscale object storage API key
vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/exoscale/storage_iam \
  s3_access_key=${INPUT_s3_key} \
  s3_secret_key=${INPUT_s3_secret}

# Generate an HTTP secret for the registry
vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/registry \
  httpSecret="$(LC_ALL=C tr -cd "A-Za-z0-9" </dev/urandom | head -c 128)"

# Generate a master password for K8up backups
vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/global-backup \
  password="$(LC_ALL=C tr -cd "A-Za-z0-9" </dev/urandom | head -c 32)"

# Generate a password for the cluster object backups
vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/cluster-backup \
  password="$(LC_ALL=C tr -cd "A-Za-z0-9" </dev/urandom | head -c 32)"

# Set the CSI Driver Exoscale Credentials
vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/exoscale/csi_driver \
  access_key=${INPUT_csi_key} \
  secret_key=${INPUT_csi_secret}

# Set the CCM Exoscale Credentials
vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/exoscale/ccm \
  access_key=${INPUT_ccm_key} \
  secret_key=${INPUT_ccm_secret}


hieradata_repo_secret=$(vault kv get \
  -format=json "clusters/kv/lbaas/hieradata_repo_token" | jq '.data.data')
env -i "hieradata_repo_user=$(echo "${hieradata_repo_secret}" | jq -r '.user')" >> "$OUTPUT"
env -i "hieradata_repo_token=$(echo "${hieradata_repo_secret}" | jq -r '.token')" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_cluster_id=
# export INPUT_base_domain=

set -euo pipefail

cluster_domain="${INPUT_commodore_cluster_id}.${INPUT_base_domain}"
echo "Cluster domain is set to '$cluster_domain'"
echo "cluster_domain=$cluster_domain" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_api_url=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_tenant_id=
# export INPUT_cluster_domain=
# export INPUT_image_major=
# export INPUT_image_minor=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"

rm -rf inventory/classes/
mkdir -p inventory/classes/
git clone "$(curl -sH"Authorization: Bearer $(commodore fetch-token)" "${INPUT_commodore_api_url}/tenants/${INPUT_commodore_tenant_id}" | jq -r '.gitRepo.url')" inventory/classes/${INPUT_commodore_tenant_id}

pushd "inventory/classes/${INPUT_commodore_tenant_id}/"

yq eval -i ".parameters.openshift.baseDomain = \"${INPUT_cluster_domain}\"" \
  ${INPUT_commodore_cluster_id}.yml

git diff --exit-code --quiet || git commit -a -m "Configure cluster domain for ${INPUT_commodore_cluster_id}"

if ls openshift4.y*ml 1>/dev/null 2>&1; then
  yq eval -i '.classes += ".openshift4"' ${INPUT_commodore_cluster_id}.yml;
  git diff --exit-code --quiet || git commit -a -m "Include openshift4 class for ${INPUT_commodore_cluster_id}"
fi

yq eval -i '.applications += ["exoscale-cloud-controller-manager"]' ${INPUT_commodore_cluster_id}.yml
yq eval -i '.applications = (.applications | unique)' ${INPUT_commodore_cluster_id}.yml
git diff --exit-code --quiet || git commit -a -m "Deploy Exoscale cloud-controller-manager on ${INPUT_commodore_cluster_id}"


yq eval -i '.applications += ["cilium"]' ${INPUT_commodore_cluster_id}.yml
yq eval -i '.applications = (.applications | unique)' ${INPUT_commodore_cluster_id}.yml

yq eval -i '.parameters.openshift.infraID = "TO_BE_DEFINED"' ${INPUT_commodore_cluster_id}.yml
yq eval -i '.parameters.openshift.clusterID = "TO_BE_DEFINED"' ${INPUT_commodore_cluster_id}.yml

yq eval -i '.parameters.cilium.olm.generate_olm_deployment = true' ${INPUT_commodore_cluster_id}.yml

git diff --exit-code --quiet || git commit -a -m "Add Cilium addon to ${INPUT_commodore_cluster_id}"

git push

popd

commodore catalog compile ${INPUT_commodore_cluster_id} --push \
  --dynamic-fact kubernetesVersion.major=1 \
  --dynamic-fact kubernetesVersion.minor="$((INPUT_image_minor+13))" \
  --dynamic-fact openshiftVersion.Major=${INPUT_image_major} \
  --dynamic-fact openshiftVersion.Minor=${INPUT_image_minor}


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_tenant_id=
# export INPUT_base_domain=
# export INPUT_cluster_domain=
# export INPUT_vault_address=
# export INPUT_redhat_pull_secret=
# export INPUT_ccm_key=
# export INPUT_ccm_secret=
# export INPUT_openshift_install_bin=

set -euo pipefail
export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"

openshift-install() {
  "${INPUT_openshift_install_bin}" "${@}"
}

export VAULT_ADDR="${INPUT_vault_address}"
vault login -method=oidc

ssh_private_key="$(pwd)/ssh_${INPUT_commodore_cluster_id}"
ssh_public_key="${ssh_private_key}.pub"

env -i "ssh_public_key_path=$ssh_public_key" >> "$OUTPUT"

if vault kv get -format=json clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/exoscale/ssh >/dev/null 2>&1; then
  echo "SSH keypair for cluster ${INPUT_commodore_cluster_id} already exists in Vault, skipping generation."

  vault kv get -format=json clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/exoscale/ssh | \
    jq -r '.data.data.private_key|@base64d' > "${ssh_private_key}"

  chmod 600 "${ssh_private_key}"
  ssh-keygen -f "${ssh_private_key}" -y > "${ssh_public_key}"

else
  echo "Generating new SSH keypair for cluster ${INPUT_commodore_cluster_id}."

  ssh-keygen -C "vault@${INPUT_commodore_cluster_id}" -t ed25519 -f "$ssh_private_key" -N ''

  base64_no_wrap='base64'
  if [[ "$OSTYPE" == "linux"* ]]; then
    base64_no_wrap='base64 --wrap 0'
  fi

  vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/exoscale/ssh \
    private_key="$(cat "$ssh_private_key" | eval "$base64_no_wrap")"
fi

echo Adding SSH private key to ssh-agent...
echo You might need to start the ssh-agent first using: eval "\$(ssh-agent)"
echo ssh-add "$ssh_private_key"
ssh-add "$ssh_private_key"

installer_dir="$(pwd)/target"
rm -rf "${installer_dir}"
mkdir -p "${installer_dir}"

cat > "${installer_dir}/install-config.yaml" <<EOF
apiVersion: v1
metadata:
  name: ${INPUT_commodore_cluster_id}
baseDomain: ${INPUT_base_domain}
platform:
  external:
    platformName: exoscale
    cloudControllerManager: External
networking:
  networkType: Cilium
pullSecret: |
  ${INPUT_redhat_pull_secret}
sshKey: "$(cat "$ssh_public_key")"
EOF

echo Running OpenShift installer to create manifests...
openshift-install --dir "${installer_dir}" create manifests

echo Copying machineconfigs...
machineconfigs=catalog/manifests/openshift4-nodes/10_machineconfigs.yaml
if [ -f $machineconfigs ];  then
  yq --no-doc -s \
    "\"${installer_dir}/openshift/99x_openshift-machineconfig_\" + .metadata.name" \
    $machineconfigs
fi

echo Copying Exoscale CCM manifests...
for f in catalog/manifests/exoscale-cloud-controller-manager/manager/*; do
  cp "$f" "${installer_dir}/manifests/exoscale_ccm_$(basename "$f")"
done

yq -i e ".stringData.api-key=\"${INPUT_ccm_key}\",.stringData.api-secret=\"${INPUT_ccm_secret}\"" \
  "${installer_dir}/manifests/exoscale_ccm_01_secret.yaml"

echo Copying Cilium OLM manifests...
for f in catalog/manifests/cilium/olm/[a-z]*; do
  cp "$f" "${installer_dir}/manifests/cilium_$(basename "$f")"
done

# shellcheck disable=2016
# We don't want the shell to execute network.operator.openshift.io as a
# command, so we need single quotes here.
echo 'Generating initial `network.operator.openshift.io` resource...'
yq '{
"apiVersion": "operator.openshift.io/v1",
"kind": "Network",
"metadata": {
  "name": "cluster"
},
"spec": {
  "deployKubeProxy": false,
  "clusterNetwork": .spec.clusterNetwork,
  "externalIP": {
    "policy": {}
  },
  "networkType": "Cilium",
  "serviceNetwork": .spec.serviceNetwork
}}' "${installer_dir}/manifests/cluster-network-02-config.yml" \
> "${installer_dir}/manifests/cilium_cluster-network-operator.yaml"

gen_cluster_domain=$(yq e '.spec.baseDomain' \
  "${installer_dir}/manifests/cluster-dns-02-config.yml")
if [ "$gen_cluster_domain" != "$INPUT_cluster_domain" ]; then
  echo -e "\033[0;31mGenerated cluster domain doesn't match expected cluster domain: Got '$gen_cluster_domain', want '$INPUT_cluster_domain'\033[0;0m"
  exit 1
else
  echo -e "\033[0;32mGenerated cluster domain matches expected cluster domain.\033[0;0m"
fi

echo Running OpenShift installer to create ignition configs...
openshift-install --dir "${installer_dir}" \
  create ignition-configs

exo storage upload "${installer_dir}/bootstrap.ign" "sos://${INPUT_commodore_cluster_id}-bootstrap" --acl public-read

echo "✅ OpenShift installer configured successfully."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_api_url=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_tenant_id=
# export INPUT_ssh_public_key_path=
# export INPUT_hieradata_repo_user=
# export INPUT_base_domain=
# export INPUT_image_major=
# export INPUT_image_minor=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"

installer_dir="$(pwd)/target"

pushd "inventory/classes/${INPUT_commodore_tenant_id}/"

yq eval -i '.classes += ["global.distribution.openshift4.no-opsgenie"]' ${INPUT_commodore_cluster_id}.yml;
yq eval -i '.classes = (.classes | unique)' ${INPUT_commodore_cluster_id}.yml

yq eval -i ".parameters.openshift.infraID = \"$(jq -r .infraID "${installer_dir}/metadata.json")\"" \
  ${INPUT_commodore_cluster_id}.yml

yq eval -i ".parameters.openshift.clusterID = \"$(jq -r .clusterID "${installer_dir}/metadata.json")\"" \
  ${INPUT_commodore_cluster_id}.yml

yq eval -i 'del(.parameters.cilium.olm.generate_olm_deployment)' \
  ${INPUT_commodore_cluster_id}.yml

yq eval -i ".parameters.openshift.ssh_key = \"$(cat ${INPUT_ssh_public_key_path})\"" \
  ${INPUT_commodore_cluster_id}.yml

ca_cert=$(jq -r '.ignition.security.tls.certificateAuthorities[0].source' \
  "${installer_dir}/master.ign" | \
  awk -F ',' '{ print $2 }' | \
  base64 --decode)

yq eval -i ".parameters.openshift4_terraform.terraform_variables.base_domain = \"${INPUT_base_domain}\"" \
  ${INPUT_commodore_cluster_id}.yml

yq eval -i ".parameters.openshift4_terraform.terraform_variables.ignition_ca = \"${ca_cert}\"" \
  ${INPUT_commodore_cluster_id}.yml

yq eval -i ".parameters.openshift4_terraform.terraform_variables.team = \"${MATCH_team_name}\"" \
  ${INPUT_commodore_cluster_id}.yml

yq eval -i ".parameters.openshift4_terraform.terraform_variables.hieradata_repo_user = \"${INPUT_hieradata_repo_user}\"" \
  ${INPUT_commodore_cluster_id}.yml

git commit -a -m "Setup cluster ${INPUT_commodore_cluster_id}"
git push

popd

commodore catalog compile ${INPUT_commodore_cluster_id} --push \
  --dynamic-fact kubernetesVersion.major=1 \
  --dynamic-fact kubernetesVersion.minor="$((INPUT_image_minor+13))" \
  --dynamic-fact openshiftVersion.Major=${INPUT_image_major} \
  --dynamic-fact openshiftVersion.Minor=${INPUT_image_minor}


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_api_url=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_tenant_id=
# export INPUT_image_major=
# export INPUT_image_minor=
# export INPUT_rhcos_template=
# export INPUT_ssh_public_key_path=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export CLUSTER_ID="${INPUT_commodore_cluster_id}"
export TENANT_ID="${INPUT_commodore_tenant_id}"

pushd "inventory/classes/${TENANT_ID}/"

yq eval -i ".parameters.openshift4_terraform.terraform_variables.ssh_key = \"$(cat ${INPUT_ssh_public_key_path})\"" \
  ${INPUT_commodore_cluster_id}.yml

yq eval -i ".parameters.openshift4_terraform.terraform_variables.rhcos_template = \"${INPUT_rhcos_template}\"" ${CLUSTER_ID}.yml

echo "Commit changes"
git commit -a -m "Set Exoscale specific values in cluster ${CLUSTER_ID}"
git push

popd

echo "Compile and push cluster catalog"
commodore catalog compile ${CLUSTER_ID} --push \
  --dynamic-fact kubernetesVersion.major=1 \
  --dynamic-fact kubernetesVersion.minor="$((INPUT_image_minor+13))" \
  --dynamic-fact openshiftVersion.Major=${INPUT_image_major} \
  --dynamic-fact openshiftVersion.Minor=${INPUT_image_minor}

echo "✅ Changes committed and cluster catalog compiled successfully."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_commodore_cluster_id=
# export INPUT_control_vshn_api_token=
# export INPUT_gitlab_user_name=
# export INPUT_gitlab_api_token=
# export INPUT_commodore_api_url=
# export INPUT_cluster_domain=
# export INPUT_rhcos_template=
# export INPUT_csp_region=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export CLUSTER_ID="${INPUT_commodore_cluster_id}"
export GITLAB_TOKEN="${INPUT_gitlab_api_token}"
export GITLAB_USER="${INPUT_gitlab_user_name}"

cat <<EOF > ./terraform.env
EXOSCALE_API_KEY=${INPUT_exoscale_key}
EXOSCALE_API_SECRET=${INPUT_exoscale_secret}
TF_VAR_control_vshn_net_token=${INPUT_control_vshn_api_token}
TF_VAR_region=${INPUT_csp_region}
TF_VAR_rhcos_template=${INPUT_rhcos_template}
GIT_AUTHOR_NAME=$(git config --global user.name)
GIT_AUTHOR_EMAIL=$(git config --global user.email)
HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token}
EOF

# Set terraform image and tag to be used
tf_image=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.image" \
  dependencies/openshift4-terraform/class/defaults.yml)
tf_tag=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.tag" \
  dependencies/openshift4-terraform/class/defaults.yml)

# Generate the terraform alias
base_dir=$(pwd)
terraform() {
  touch .terraformrc
  docker run --rm -e REAL_UID="$(id -u)" -e TF_CLI_CONFIG_FILE=/tf/.terraformrc --env-file "${base_dir}/terraform.env" -w /tf -v "$(pwd):/tf" --ulimit memlock=-1 "${tf_image}:${tf_tag}" /tf/terraform.sh "${@}"
}

GITLAB_REPOSITORY_URL=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r '.gitRepo.url' | sed 's|ssh://||; s|/|:|')
GITLAB_REPOSITORY_NAME=${GITLAB_REPOSITORY_URL##*/}
GITLAB_CATALOG_PROJECT_ID=$(curl -sH "Authorization: Bearer ${GITLAB_TOKEN}" "https://git.vshn.net/api/v4/projects?simple=true&search=${GITLAB_REPOSITORY_NAME/.git}" | jq -r ".[] | select(.ssh_url_to_repo == \"${GITLAB_REPOSITORY_URL}\") | .id")
GITLAB_STATE_URL="https://git.vshn.net/api/v4/projects/${GITLAB_CATALOG_PROJECT_ID}/terraform/state/cluster"

pushd catalog/manifests/openshift4-terraform/

terraform init \
  "-backend-config=address=${GITLAB_STATE_URL}" \
  "-backend-config=lock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=unlock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=username=${GITLAB_USER}" \
  "-backend-config=password=${GITLAB_TOKEN}" \
  "-backend-config=lock_method=POST" \
  "-backend-config=unlock_method=DELETE" \
  "-backend-config=retry_wait_min=5"

if [ -f override.tf ]; then
  echo "override.tf already exists, not changing it"
else
  cat > override.tf <<EOF
  module "cluster" {
    bootstrap_count          = 0
    lb_count                 = 0
    master_count             = 0
    infra_count              = 0
    storage_count            = 0
    worker_count             = 0
    additional_worker_groups = {}
  }
EOF
fi

terraform apply -auto-approve

cat > override.tf <<EOF
module "cluster" {
  bootstrap_count          = 0
  master_count             = 0
  infra_count              = 0
  storage_count            = 0
  worker_count             = 0
  additional_worker_groups = {}
}
EOF
terraform apply -auto-approve -target "module.cluster.module.lb.module.hiera"

echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
echo "@                                                                                       @"
echo "@  Please review and merge the LB hieradata MR listed in Terraform output hieradata_mr. @"
echo "@                                                                                       @"
echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"

sleep 3
while (GITLAB_HOST=git.vshn.net GITLAB_TOKEN="${INPUT_gitlab_api_token}" glab mr list -R=appuio/appuio_hieradata | grep "${INPUT_commodore_cluster_id}")
do
  sleep 10
done
echo PR merged, waiting for CI to finish...
sleep 10
while (GITLAB_HOST=git.vshn.net GITLAB_TOKEN="${INPUT_gitlab_api_token}" glab mr list -R=appuio/appuio_hieradata | grep "running")
do
  sleep 10
done

terraform apply -auto-approve
terraform output -raw cluster_dns > ../../../dns.txt
echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
echo "@                                                                                       @"
echo "@  Please add the DNS records shown in the Terraform output to your DNS provider.       @"
echo "@  Most probably in https://git.vshn.net/vshn/vshn_zonefiles                            @"
echo "@                                                                                       @"
echo "@  If terminal selection does not work the entries can also be copied from              @"
echo "@    dns.txt                                                                            @"
echo "@                                                                                       @"
echo "@  Waiting for record to propagate...                                                   @"
echo "@                                                                                       @"
echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
while ! (host "api.${INPUT_cluster_domain}")
do
  sleep 15
done
rm ../../../dns.txt

lb1=$(terraform state show "module.cluster.module.lb.exoscale_domain_record.lb[0]" | grep hostname | awk '{print $3}' | tr -d ' "\r\n')
lb2=$(terraform state show "module.cluster.module.lb.exoscale_domain_record.lb[1]" | grep hostname | awk '{print $3}' | tr -d ' "\r\n')

echo "Loadbalancer FQDNs: $lb1 , $lb2"

echo "Waiting for HAproxy ..."
while true; do
  curl --connect-timeout 1 "http://api.${INPUT_cluster_domain}:6443" &>/dev/null || exit_code=$?
  if [ "$exit_code" -eq 52 ]; then
    echo "  HAproxy up!"
    break
  else
    echo -n "."
    sleep 5
  fi
done

echo "updating ssh config..."
ssh management2.corp.vshn.net "sshop --output-archive /dev/stdout" | tar -C ~ --overwrite -xzf -
echo "done"

echo "waiting for ssh access ..."
ssh "${lb1}" hostname -f
ssh "${lb2}" hostname -f

env -i "lb_fqdn_1=$lb1" >> "$OUTPUT"
env -i "lb_fqdn_2=$lb2" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_control_vshn_api_token=
# export INPUT_gitlab_user_name=
# export INPUT_gitlab_api_token=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_api_url=
# export INPUT_lb_fqdn_1=
# export INPUT_lb_fqdn_2=

set -euo pipefail

installer_dir="$(pwd)/target"
export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export CLUSTER_ID="${INPUT_commodore_cluster_id}"
export GITLAB_TOKEN="${INPUT_gitlab_api_token}"
export GITLAB_USER="${INPUT_gitlab_user_name}"

cat <<EOF > ./terraform.env
EXOSCALE_API_KEY=${INPUT_exoscale_key}
EXOSCALE_API_SECRET=${INPUT_exoscale_secret}
TF_VAR_control_vshn_net_token=${INPUT_control_vshn_api_token}
GIT_AUTHOR_NAME=$(git config --global user.name)
GIT_AUTHOR_EMAIL=$(git config --global user.email)
HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token}
EOF

# Set terraform image and tag to be used
tf_image=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.image" \
  dependencies/openshift4-terraform/class/defaults.yml)
tf_tag=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.tag" \
  dependencies/openshift4-terraform/class/defaults.yml)

# Generate the terraform alias
base_dir=$(pwd)
terraform() {
  touch .terraformrc
  docker run --rm -e REAL_UID="$(id -u)" -e TF_CLI_CONFIG_FILE=/tf/.terraformrc --env-file "${base_dir}/terraform.env" -w /tf -v "$(pwd):/tf" --ulimit memlock=-1 "${tf_image}:${tf_tag}" /tf/terraform.sh "${@}"
}


GITLAB_REPOSITORY_URL=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r '.gitRepo.url' | sed 's|ssh://||; s|/|:|')
GITLAB_REPOSITORY_NAME=${GITLAB_REPOSITORY_URL##*/}
GITLAB_CATALOG_PROJECT_ID=$(curl -sH "Authorization: Bearer ${GITLAB_TOKEN}" "https://git.vshn.net/api/v4/projects?simple=true&search=${GITLAB_REPOSITORY_NAME/.git}" | jq -r ".[] | select(.ssh_url_to_repo == \"${GITLAB_REPOSITORY_URL}\") | .id")
GITLAB_STATE_URL="https://git.vshn.net/api/v4/projects/${GITLAB_CATALOG_PROJECT_ID}/terraform/state/cluster"

pushd catalog/manifests/openshift4-terraform/

terraform init \
  "-backend-config=address=${GITLAB_STATE_URL}" \
  "-backend-config=lock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=unlock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=username=${GITLAB_USER}" \
  "-backend-config=password=${GITLAB_TOKEN}" \
  "-backend-config=lock_method=POST" \
  "-backend-config=unlock_method=DELETE" \
  "-backend-config=retry_wait_min=5"

cat > override.tf <<EOF
module "cluster" {
  bootstrap_count          = 1
  master_count             = 0
  infra_count              = 0
  storage_count            = 0
  worker_count             = 0
  additional_worker_groups = {}
}
EOF
terraform apply -auto-approve

echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
echo "@                                                                                       @"
echo "@  Please review and merge the LB hieradata MR listed in Terraform output hieradata_mr. @"
echo "@                                                                                       @"
echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
sleep 3
while (GITLAB_HOST=git.vshn.net GITLAB_TOKEN="${INPUT_gitlab_api_token}" glab mr list -R=appuio/appuio_hieradata | grep "${INPUT_commodore_cluster_id}")
do
  sleep 10
done
echo PR merged, waiting for CI to finish...
sleep 10
while (GITLAB_HOST=git.vshn.net GITLAB_TOKEN="${INPUT_gitlab_api_token}" glab mr list -R=appuio/appuio_hieradata | grep "running")
do
  sleep 10
done

ssh "${INPUT_lb_fqdn_1}" sudo puppetctl run
ssh "${INPUT_lb_fqdn_2}" sudo puppetctl run

echo -n "Waiting for Bootstrap API to become available .."
API_URL=$(yq e '.clusters[0].cluster.server' "${installer_dir}/auth/kubeconfig")
while ! curl --connect-timeout 1 "${API_URL}/healthz" -k &>/dev/null; do
  echo -n "."
  sleep 5
done && echo "✅ API is up"

env -i "kubeconfig_path=${installer_dir}/auth/kubeconfig" >> "$OUTPUT"


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_control_vshn_api_token=
# export INPUT_gitlab_user_name=
# export INPUT_gitlab_api_token=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_api_url=
# export INPUT_kubeconfig_path=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export CLUSTER_ID="${INPUT_commodore_cluster_id}"
export GITLAB_TOKEN="${INPUT_gitlab_api_token}"
export GITLAB_USER="${INPUT_gitlab_user_name}"

cat <<EOF > ./terraform.env
EXOSCALE_API_KEY=${INPUT_exoscale_key}
EXOSCALE_API_SECRET=${INPUT_exoscale_secret}
TF_VAR_control_vshn_net_token=${INPUT_control_vshn_api_token}
GIT_AUTHOR_NAME=$(git config --global user.name)
GIT_AUTHOR_EMAIL=$(git config --global user.email)
HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token}
EOF

# Set terraform image and tag to be used
tf_image=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.image" \
  dependencies/openshift4-terraform/class/defaults.yml)
tf_tag=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.tag" \
  dependencies/openshift4-terraform/class/defaults.yml)

# Generate the terraform alias
base_dir=$(pwd)
terraform() {
  touch .terraformrc
  docker run --rm -e REAL_UID="$(id -u)" -e TF_CLI_CONFIG_FILE=/tf/.terraformrc --env-file "${base_dir}/terraform.env" -w /tf -v "$(pwd):/tf" --ulimit memlock=-1 "${tf_image}:${tf_tag}" /tf/terraform.sh "${@}"
}

GITLAB_REPOSITORY_URL=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r '.gitRepo.url' | sed 's|ssh://||; s|/|:|')
GITLAB_REPOSITORY_NAME=${GITLAB_REPOSITORY_URL##*/}
GITLAB_CATALOG_PROJECT_ID=$(curl -sH "Authorization: Bearer ${GITLAB_TOKEN}" "https://git.vshn.net/api/v4/projects?simple=true&search=${GITLAB_REPOSITORY_NAME/.git}" | jq -r ".[] | select(.ssh_url_to_repo == \"${GITLAB_REPOSITORY_URL}\") | .id")
GITLAB_STATE_URL="https://git.vshn.net/api/v4/projects/${GITLAB_CATALOG_PROJECT_ID}/terraform/state/cluster"

pushd catalog/manifests/openshift4-terraform/

terraform init \
  "-backend-config=address=${GITLAB_STATE_URL}" \
  "-backend-config=lock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=unlock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=username=${GITLAB_USER}" \
  "-backend-config=password=${GITLAB_TOKEN}" \
  "-backend-config=lock_method=POST" \
  "-backend-config=unlock_method=DELETE" \
  "-backend-config=retry_wait_min=5"

cat > override.tf <<EOF
module "cluster" {
  bootstrap_count          = 1
  infra_count              = 0
  storage_count            = 0
  worker_count             = 0
  additional_worker_groups = {}
}
EOF

echo "Running Terraform ..."

terraform apply -auto-approve

export KUBECONFIG="${INPUT_kubeconfig_path}"

echo "Waiting for masters to become ready ..."
kubectl wait --for create --timeout=600s node -l node-role.kubernetes.io/master
kubectl wait --for condition=ready --timeout=600s node -l node-role.kubernetes.io/master
popd


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_api_url=
# export INPUT_vault_address=
# export INPUT_kubeconfig_path=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export KUBECONFIG="${INPUT_kubeconfig_path}"

export VAULT_ADDR=${INPUT_vault_address}
vault login -method=oidc

echo '# Applying cert-manager ... #'
kubectl apply -f catalog/manifests/cert-manager/00_namespace.yaml
kubectl apply -Rf catalog/manifests/cert-manager/10_cert_manager
# shellcheck disable=2046
# we need word splitting here
kubectl -n syn-cert-manager patch --type=merge \
  $(kubectl -n syn-cert-manager get deploy -oname) \
  -p '{"spec":{"template":{"spec":{"tolerations":[{"operator":"Exists"}]}}}}'
echo '# Applied cert-manager. #'
echo


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_openshift_install_bin=

set -euo pipefail
openshift-install() {
  "${INPUT_openshift_install_bin}" "${@}"
}
installer_dir="$(pwd)/target"
openshift-install --dir "${installer_dir}" \
  wait-for bootstrap-complete --log-level debug


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=
# export INPUT_control_vshn_api_token=
# export INPUT_gitlab_user_name=
# export INPUT_gitlab_api_token=
# export INPUT_commodore_cluster_id=
# export INPUT_commodore_api_url=
# export INPUT_lb_fqdn_1=
# export INPUT_lb_fqdn_2=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export CLUSTER_ID="${INPUT_commodore_cluster_id}"
export GITLAB_TOKEN="${INPUT_gitlab_api_token}"
export GITLAB_USER="${INPUT_gitlab_user_name}"

cat <<EOF > ./terraform.env
EXOSCALE_API_KEY=${INPUT_exoscale_key}
EXOSCALE_API_SECRET=${INPUT_exoscale_secret}
TF_VAR_control_vshn_net_token=${INPUT_control_vshn_api_token}
GIT_AUTHOR_NAME=$(git config --global user.name)
GIT_AUTHOR_EMAIL=$(git config --global user.email)
HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token}
EOF

# Set terraform image and tag to be used
tf_image=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.image" \
  dependencies/openshift4-terraform/class/defaults.yml)
tf_tag=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.tag" \
  dependencies/openshift4-terraform/class/defaults.yml)

# Generate the terraform alias
base_dir=$(pwd)
terraform() {
  touch .terraformrc
  docker run --rm -e REAL_UID="$(id -u)" -e TF_CLI_CONFIG_FILE=/tf/.terraformrc --env-file "${base_dir}/terraform.env" -w /tf -v "$(pwd):/tf" --ulimit memlock=-1 "${tf_image}:${tf_tag}" /tf/terraform.sh "${@}"
}

GITLAB_REPOSITORY_URL=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r '.gitRepo.url' | sed 's|ssh://||; s|/|:|')
GITLAB_REPOSITORY_NAME=${GITLAB_REPOSITORY_URL##*/}
GITLAB_CATALOG_PROJECT_ID=$(curl -sH "Authorization: Bearer ${GITLAB_TOKEN}" "https://git.vshn.net/api/v4/projects?simple=true&search=${GITLAB_REPOSITORY_NAME/.git}" | jq -r ".[] | select(.ssh_url_to_repo == \"${GITLAB_REPOSITORY_URL}\") | .id")
GITLAB_STATE_URL="https://git.vshn.net/api/v4/projects/${GITLAB_CATALOG_PROJECT_ID}/terraform/state/cluster"

pushd catalog/manifests/openshift4-terraform/

terraform init \
  "-backend-config=address=${GITLAB_STATE_URL}" \
  "-backend-config=lock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=unlock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=username=${GITLAB_USER}" \
  "-backend-config=password=${GITLAB_TOKEN}" \
  "-backend-config=lock_method=POST" \
  "-backend-config=unlock_method=DELETE" \
  "-backend-config=retry_wait_min=5"

rm -f override.tf
terraform apply --auto-approve

echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
echo "@                                                                                       @"
echo "@  Please review and merge the LB hieradata MR listed in Terraform output hieradata_mr. @"
echo "@                                                                                       @"
echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
sleep 3
while (GITLAB_HOST=git.vshn.net GITLAB_TOKEN="${INPUT_gitlab_api_token}" glab mr list -R=appuio/appuio_hieradata | grep "${INPUT_commodore_cluster_id}")
do
  sleep 10
done
echo PR merged, waiting for CI to finish...
sleep 10
while (GITLAB_HOST=git.vshn.net GITLAB_TOKEN="${INPUT_gitlab_api_token}" glab mr list -R=appuio/appuio_hieradata | grep "running")
do
  sleep 10
done

ssh "${INPUT_lb_fqdn_1}" sudo puppetctl run
ssh "${INPUT_lb_fqdn_2}" sudo puppetctl run

popd


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_kubeconfig_path=

set -euo pipefail
export KUBECONFIG="${INPUT_kubeconfig_path}"

echo '##### CAUTION: This step does not currently wait for all CSRs to be created. #####'
echo '#####    Please re-run this step until no more CSRs are getting signed.      #####'
sleep 3

# TODO: approve CSRs until all nodes are labelled and ready.
echo "Approving node certs..."
csrs=$(oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')
if [ -n "$csrs" ]; then
  echo "$csrs" | xargs oc adm certificate approve
fi

echo "Label infra nodes..."
kubectl get node -ojson | \
  jq -r '.items[] | select(.metadata.name | test("infra-")).metadata.name' | \
  xargs -I {} kubectl label node {} node-role.kubernetes.io/infra=

echo "Label and taint storage nodes..."
kubectl get node -ojson | \
  jq -r '.items[] | select(.metadata.name | test("storage-")).metadata.name' | \
  xargs -I {} kubectl  label node {} node-role.kubernetes.io/storage=

kubectl taint node -lnode-role.kubernetes.io/storage storagenode=True:NoSchedule --overwrite

echo "Label worker nodes..."
kubectl get node -ojson | \
  jq -r '.items[] | select(.metadata.name | test("infra|master|storage-")|not).metadata.name' | \
  xargs -I {} kubectl label node {} node-role.kubernetes.io/app=

echo "✅ All new nodes approved and labeled successfully."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_cluster_id=
# export INPUT_commodore_api_url=
# export INPUT_kubeconfig_path=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export KUBECONFIG="${INPUT_kubeconfig_path}"

echo '# Enabling proxy protocol ... #'
kubectl -n openshift-ingress-operator patch ingresscontroller default --type=json \
  -p '[{
    "op":"replace",
    "path":"/spec/endpointPublishingStrategy",
    "value": {"type": "HostNetwork", "hostNetwork": {"protocol": "PROXY"}}
  }]'
echo '# Enabled proxy protocol. #'
echo

distribution="$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${INPUT_commodore_cluster_id} | jq -r .facts.distribution)"
if [[ "$distribution" != "oke" ]]
then
  echo '# Scheduling ingress controller on infra nodes ... #'
  kubectl -n openshift-ingress-operator patch ingresscontroller default --type=json \
    -p '[{
      "op":"replace",
      "path":"/spec/nodePlacement",
      "value":{"nodeSelector":{"matchLabels":{"node-role.kubernetes.io/infra":""}}}
    }]'
  echo '# Scheduled ingress controller on infra nodes. #'
  echo
fi

echo '# Removing temporary cert-manager tolerations ... #'
# shellcheck disable=2046
# we need word splitting here
kubectl -n syn-cert-manager patch --type=json \
  $(kubectl -n syn-cert-manager get deploy -oname) \
  -p '[{"op":"remove","path":"/spec/template/spec/tolerations"}]'
echo '# Removed temporary cert-manager tolerations. #'

echo "✅ Initial deployments configured successfully."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_openshift_install_bin=

set -euo pipefail
openshift-install() {
  "${INPUT_openshift_install_bin}" "${@}"
}
installer_dir="$(pwd)/target"
openshift-install --dir "${installer_dir}" \
  wait-for install-complete --log-level debug


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_kubeconfig_path=
# export INPUT_s3_key=
# export INPUT_s3_secret=

set -euo pipefail

export KUBECONFIG="${INPUT_kubeconfig_path}"
secret_name="image-registry-private-configuration-user"

if oc get secret "${secret_name}" -n openshift-image-registry &>/dev/null; then
  echo "Secret ${secret_name} already exists, skipping creation."
else
  oc create secret generic image-registry-private-configuration-user \
    --namespace openshift-image-registry \
    --from-literal=REGISTRY_STORAGE_S3_ACCESSKEY=${INPUT_s3_key} \
    --from-literal=REGISTRY_STORAGE_S3_SECRETKEY=${INPUT_s3_secret}
fi

echo "✅ Registry S3 secret created successfully."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_cluster_id=
# export INPUT_commodore_tenant_id=
# export INPUT_commodore_api_url=
# export INPUT_gitlab_user_name=
# export INPUT_gitlab_api_token=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export CLUSTER_ID="${INPUT_commodore_cluster_id}"
export TENANT_ID="${INPUT_commodore_tenant_id}"
export GITLAB_USER="${INPUT_gitlab_user_name}"
export GITLAB_TOKEN="${INPUT_gitlab_api_token}"

GITLAB_REPOSITORY_URL=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r '.gitRepo.url' | sed 's|ssh://||; s|/|:|')
GITLAB_REPOSITORY_NAME=${GITLAB_REPOSITORY_URL##*/}
GITLAB_CATALOG_PROJECT_ID=$(curl -sH "Authorization: Bearer ${GITLAB_TOKEN}" "https://git.vshn.net/api/v4/projects?simple=true&search=${GITLAB_REPOSITORY_NAME/.git}" | jq -r ".[] | select(.ssh_url_to_repo == \"${GITLAB_REPOSITORY_URL}\") | .id")
GITLAB_STATE_URL="https://git.vshn.net/api/v4/projects/${GITLAB_CATALOG_PROJECT_ID}/terraform/state/cluster"

pushd "inventory/classes/${TENANT_ID}/"

curl -fsu "${GITLAB_USER}:${GITLAB_TOKEN}" "$GITLAB_STATE_URL" |\
  jq '[.resources[] | select(.module == "module.cluster.module.worker" and .type == "exoscale_instance_pool")][0].instances[0].attributes.id' |\
  yq ea -i 'select(fileIndex == 0) * (select(fileIndex == 1) | {"parameters": {"exoscale_cloud_controller_manager":{"serviceLoadBalancerDefaultAnnotations":{"service.beta.kubernetes.io/exoscale-loadbalancer-service-instancepool-id": .}}}}) ' \
  "$CLUSTER_ID.yml" -
git commit -a -m "${CLUSTER_ID}: Enable default instance pool annotation injector for LoadBalancer services"
git push
popd


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_api_url=
# export INPUT_commodore_cluster_id=
# export INPUT_kubeconfig_path=

set -euo pipefail

export COMMODORE_API_URL="${INPUT_commodore_api_url}"
export KUBECONFIG="${INPUT_kubeconfig_path}"
LIEUTENANT_AUTH="Authorization:Bearer $(commodore fetch-token)"

if ! kubectl get deploy -n syn steward > /dev/null; then
  INSTALL_URL=$(curl -H "${LIEUTENANT_AUTH}" "${COMMODORE_API_URL}/clusters/${INPUT_commodore_cluster_id}" | jq -r ".installURL")

  if [[ $INSTALL_URL == "null" ]]
  # TODO(aa): consider doing this programmatically - especially if, at a later point, we add the lieutenant kubeconfig to the inputs anyway
  then
      echo '###################################################################################'
      echo '#                                                                                 #'
      echo '#  Could not fetch install URL! Please reset the bootstrap token and try again.   #'
      echo '#                                                                                 #'
      echo '###################################################################################'
      echo
      echo 'See https://kb.vshn.ch/corp-tech/projectsyn/explanation/bootstrap-token.html#_resetting_the_bootstrap_token'
      exit 1
  fi

  echo "# Deploying steward ..."
  kubectl create -f "$INSTALL_URL"
fi

echo "# Waiting for ArgoCD resource to exist ..."
kubectl wait --for=create crds/argocds.argoproj.io --timeout=5m
kubectl api-resources --api-group=argoproj.io # to refresh local K8s API cache
sleep 1 # to prevent race conditions

echo "# Waiting for ArgoCD instance to exist ..."
kubectl wait --for=create argocd/syn-argocd -nsyn --timeout=90s

echo "# Waiting for ArgoCD instance to be ready ..."
kubectl wait --for=jsonpath='{.status.phase}'=Available argocd/syn-argocd -nsyn --timeout=5m

echo "Done."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_kubeconfig_path=
# export INPUT_no_letsencrypt=
# export INPUT_cluster_domain=
# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=

set -euo pipefail

if [[ "${INPUT_no_letsencrypt}" == "y" ]]; then
  echo "no_letsencrypt is set to 'y', skipping ACME DNS record setup."
  exit 0
fi

export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"

export KUBECONFIG="${INPUT_kubeconfig_path}"

echo '# Waiting for cert-manager namespace ...'
kubectl wait --for=create ns/syn-cert-manager --timeout 5m
echo '# Waiting for cert-manager secret ...'
kubectl wait --for=create secret/acme-dns-client -nsyn-cert-manager --timeout 10m

fulldomain=""

while [[ -z "$fulldomain" ]]
do
  fulldomain=$(kubectl -n syn-cert-manager \
    get secret acme-dns-client \
    -o jsonpath='{.data.acmedns\.json}' | \
    base64 -d  | \
    jq -r '[.[]][0].fulldomain')
  echo "$fulldomain"
done

for cname in "api" "apps"; do
  exo dns add CNAME "${INPUT_cluster_domain}" -n "_acme-challenge.${cname}" -a "${fulldomain}." -t 600
done

echo
echo "✅ ACME DNS records created successfully."


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_kubeconfig_path=
# export INPUT_cluster_domain=
# export INPUT_commodore_cluster_id=
# export INPUT_passbolt_passphrase=

set -euo pipefail
export KUBECONFIG="${INPUT_kubeconfig_path}"

echo '# Waiting for emergency-credentials-controller namespace ...'
kubectl wait --for=create ns/appuio-emergency-credentials-controller
echo '# Waiting for emergency-credentials-controller ...'
kubectl wait --for=create secret/acme-dns-client -nsyn-cert-manager

echo '# Waiting for emergency credential tokens ...'
until kubectl -n appuio-emergency-credentials-controller get emergencyaccounts.cluster.appuio.io -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.lastTokenCreationTimestamp}{"\n"}{end}' | grep "$( date '+%Y' )" >/dev/null
do
  echo -n .
done

export EMR_KUBERNETES_ENDPOINT=https://api.${INPUT_cluster_domain}:6443
export EMR_PASSPHRASE="${INPUT_passbolt_passphrase}"
emergency-credentials-receive "${INPUT_commodore_cluster_id}"

yq -i e '.clusters[0].cluster.insecure-skip-tls-verify = true' "em-${INPUT_commodore_cluster_id}"
export KUBECONFIG="em-${INPUT_commodore_cluster_id}"
kubectl get nodes
oc whoami | grep system:serviceaccount:appuio-emergency-credentials-controller: || exit 1

env -i "kubeconfig_path=$(pwd)/em-${INPUT_commodore_cluster_id}" >> "$OUTPUT"

echo "#  Invalidating 10-year admin kubeconfig ..."
kubectl -n openshift-config patch cm admin-kubeconfig-client-ca --type=merge -p '{"data": {"ca-bundle.crt": ""}}'


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_kubeconfig_path=

set -euo pipefail
export KUBECONFIG="${INPUT_kubeconfig_path}"

kubectl wait --for create --timeout=10m -n openshift-monitoring cronjob silence
echo '# Installing default alert silence ...'
oc --as=system:admin -n openshift-monitoring get job silence-manual &>/dev/null || \
  oc --as=system:admin -n openshift-monitoring create job --from=cronjob/silence silence-manual
oc wait -n openshift-monitoring --for=condition=complete job/silence-manual
oc --as=system:admin -n openshift-monitoring delete job/silence-manual

echo '# Retrieving active alerts ...'
kubectl --as=system:admin -n openshift-monitoring exec sts/alertmanager-main -- \
  amtool --alertmanager.url=http://localhost:9093 alert --active

echo
echo '#######################################################'
echo '#                                                     #'
echo '#  Please review the list of open alerts above,       #'
echo '#  address any that require action before proceeding. #'
echo '#                                                     #'
echo '#######################################################'
sleep 2


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_cluster_id=
# export INPUT_commodore_tenant_id=

set -euo pipefail
pushd "inventory/classes/${INPUT_commodore_tenant_id}/"
yq eval -i 'del(.classes[] | select(. == "*.no-opsgenie"))' ${INPUT_commodore_cluster_id}.yml
git commit -a -m "Enable opsgenie alerting on cluster ${INPUT_commodore_cluster_id}"
git push
popd


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_kubeconfig_path=

set -euo pipefail
export KUBECONFIG="${INPUT_kubeconfig_path}"

numconfs="$( kubectl -n appuio-openshift-upgrade-controller get upgradeconfig -oyaml | yq '.items | length' )"

if (( numconfs < 1 ))
then
  kubectl -n appuio-openshift-upgrade-controller get upgradeconfig
  echo
  echo ERROR: did not find an upgradeconfig
  echo Please review the output above and ensure an upgradeconfig is present.
  echo
  echo "Double check the cluster's maintenance_window fact."
  exit 1
fi

echo '# Scheduling a first maintenance ...'

uc="$(yq .parameters.facts.maintenance_window inventory/classes/params/cluster.yml)"
kubectl -n appuio-openshift-upgrade-controller get upgradeconfig "$uc" -oyaml | \
  yq '
    .metadata.name = "first",
    .metadata.labels = {},
    .spec.jobTemplate.metadata.labels.upgradeconfig/name = "first",
    .spec.schedule.cron = ((now+"1m")| tz("Europe/Zurich") | format_datetime("4 15")) + " * * *",
    .spec.pinVersionWindow = "0m"
  ' | \
  kubectl create -f - --as=system:admin


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_lb_fqdn_1=
# export INPUT_lb_fqdn_2=
# export INPUT_gitlab_api_token=
# export INPUT_commodore_cluster_id=

set -euo pipefail

if [ -e nodes_hieradata ]
then
  rm -rf nodes_hieradata
fi
git clone git@git.vshn.net:vshn-puppet/nodes_hieradata.git
pushd nodes_hieradata

if ! grep "s_apt_dater::host::group" "${INPUT_lb_fqdn_1}"
then
# NOTE(aa): no indentation because here documents are ... something
cat >"${INPUT_lb_fqdn_1}.yaml" <<EOF
---
s_apt_dater::host::group: '2200_20_night_main'
EOF
fi

if ! grep "s_apt_dater::host::group" "${INPUT_lb_fqdn_2}"
then
# NOTE(aa): no indentation because here documents are ... something
cat >"${INPUT_lb_fqdn_2}.yaml" <<EOF
---
s_apt_dater::host::group: '2200_40_night_second'
EOF
fi

git add ./*.yaml

if git diff-index --quiet HEAD
then
  echo "No changes, skipping commit"
else
  git commit -m"Configure apt-dater groups for LBs for OCP4 cluster ${INPUT_commodore_cluster_id}"
  git push origin master

  echo Waiting for CI to finish...
  sleep 10
  while (GITLAB_HOST=git.vshn.net GITLAB_TOKEN="${INPUT_gitlab_api_token}" glab ci list -R=vshn-puppet/nodes_hieradata | grep "running")
  do
    sleep 10
  done

  echo Running puppet ...
  for fqdn in "${INPUT_lb_fqdn_1}" "${INPUT_lb_fqdn_2}"
  do
    ssh "${fqdn}" sudo puppetctl run
  done
fi || true
popd


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_cluster_id=
# export INPUT_exoscale_key=
# export INPUT_exoscale_secret=

set -euo pipefail

export EXOSCALE_API_KEY="${INPUT_exoscale_key}"
export EXOSCALE_API_SECRET="${INPUT_exoscale_secret}"

exo storage rb -r -f "sos://${INPUT_commodore_cluster_id}-bootstrap"

# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_commodore_cluster_id=
# export INPUT_kubeconfig_path=
# export INPUT_jumphost_fqdn=
# export INPUT_socks5_port=

set -euo pipefail
if [ -e openshift4-clusters ]
then
  rm -rf openshift4-clusters
fi
git clone git@git.vshn.net:vshn/openshift4-clusters.git
pushd openshift4-clusters

if [[ -d "${INPUT_commodore_cluster_id}" ]]
then
  echo "Cluster entry already exists - not touching that!"
  exit 0
else
  API_URL=$(yq e '.clusters[0].cluster.server' "${INPUT_kubeconfig_path}")

  mkdir -p "${INPUT_commodore_cluster_id}"
  pushd "${INPUT_commodore_cluster_id}"
  ln -s ../base_envrc .envrc
  cat >.connection_facts <<EOF
API=${API_URL}
EOF
  popd

  port="$( echo "${INPUT_socks5_port}" | tr '[:upper:]' '[:lower:]' )"
  jumphost="$( echo "${INPUT_jumphost_fqdn}" | tr '[:upper:]' '[:lower:]' )"

  if [[ "$port" != "none" ]] && [[ "$jumphost" != "none" ]]
  then
    cat >> "${INPUT_commodore_cluster_id}/.connection_facts" <<EOF
JUMPHOST=${INPUT_jumphost_fqdn}
SOCKS5_PORT=${INPUT_socks5_port}
EOF
    python foxyproxy_generate.py
  fi

  git add --force "${INPUT_commodore_cluster_id}"
  git add .

  if git diff-index --quiet HEAD
  then
    echo "No changes, skipping commit"
  else
    git commit -am "Add cluster ${INPUT_commodore_cluster_id}"
  fi || true
fi
popd

echo
echo '#########################################################'
echo '#                                                       #'
echo '#  Please test the cluster connection, and if it works  #'
echo '#  as expected, push the commit to the repository.      #'
echo '#                                                       #'
echo '#########################################################'
echo
echo "Run the following:"
echo "cd $(pwd)/openshift4-clusters/${INPUT_commodore_cluster_id}"
echo "direnv allow ."
echo "direnv exec . oc whoami"
echo "git push origin main   # only if everything is OK"
sleep 2


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"

OUTPUT=$(mktemp)

# export INPUT_kubeconfig_path=

set -euo pipefail
export KUBECONFIG="${INPUT_kubeconfig_path}"

echo "#  Waiting for initial maintenance to complete ..."
oc get clusterversion
until kubectl wait --for=condition=Succeeded upgradejob -l "upgradeconfig/name=first" -n appuio-openshift-upgrade-controller 2>/dev/null
do
  oc get clusterversion | grep -v NAME
done

echo "#  Deleting initial UpgradeConfig ..."
kubectl --as=system:admin -n appuio-openshift-upgrade-controller \
  delete upgradeconfig first


# echo "# Outputs"
# cat "$OUTPUT"
# rm -f "$OUTPUT"