#!/usr/bin/env bash
export DEBIAN_FRONTEND=noninteractive
export CURL_USER_AGENT=NimeOps-UA

LAYEROPS_USER=layerops
LAYEROPS_GROUP=$LAYEROPS_USER
LAYEROPS_HOME_DIR=/opt/layerops
LAYEROPS_BIN_DIR=${LAYEROPS_HOME_DIR}/bin
LAYEROPS_WORKER_PATH=${LAYEROPS_BIN_DIR}/worker
LAYEROPS_ETC_DIR=${LAYEROPS_HOME_DIR}/etc
LAYEROPS_ETC_DEFAULT_FILE=${LAYEROPS_ETC_DIR}/default
LAYEROPS_INSTANCES_SYSTEMD_FILE=/etc/systemd/system/layerops-instances.service
LAYEROPS_SERVICES_SYSTEMD_FILE=/etc/systemd/system/layerops-services.service
LAYEROPS_CERTBOT_SYSTEMD_FILE=/etc/systemd/system/layerops-certbot.service
LAYEROPS_MONITORING_SYSTEMD_FILE=/etc/systemd/system/layerops-monitoring.service

SUDO_FILE=/etc/sudoers.d/99-layerops-users
SYSTEMCTL_BIN_PATH=/usr/bin/systemctl

WIREGUARD_CONFIG_DIR=/etc/wireguard
WIREGUARD_INTERFACE=wg0
WIREGUARD_SUBNET=172.24.0.0/13
WIREGUARD_CONFIG_FILE=${WIREGUARD_CONFIG_DIR}/${WIREGUARD_INTERFACE}.conf
WIREGUARD_TEMPLATE_FILE=${WIREGUARD_CONFIG_FILE}.tmpl
WIREGUARD_QUICK_RELOAD_PATH=${LAYEROPS_BIN_DIR}/wg-quick-up-reload
WIREGUARD_RELOAD_PATH=${LAYEROPS_BIN_DIR}/wireguard_start_or_reload

DOCKER_DATA_DIR=${LAYEROPS_DATA_DIR}/docker
DOCKER_GROUP=docker
DOCKER_DAEMON_FILE=/etc/docker/daemon.json

CADVISOR_DOCKER_IMAGE=gcr.io/cadvisor/cadvisor:v0.52.1
CADVISOR_BIND_PORT=9889
CADVISOR_SYSTEMD_FILE=/etc/systemd/system/cadvisor.service

NODE_EXPORTER_VERSION=1.10.2
NODE_EXPORTER_DL_URL=https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz
NODE_EXPORTER_BIND_PORT=9888
NODE_EXPORTER_SYSTEMD_FILE=/etc/systemd/system/node_exporter.service
NODE_EXPORTER_USERGROUP=node_exporter

SPIRE_CONFIG_DIR=${LAYEROPS_ETC_DIR}/spire
SPIRE_AGENT_CONFIG_FILE=${SPIRE_CONFIG_DIR}/agent.conf
SPIRE_SERVER_CONFIG_FILE=${SPIRE_CONFIG_DIR}/server.conf
SPIRE_SERVER_DATA_DIR=${LAYEROPS_DATA_DIR}/spire/data/server
SPIRE_AGENT_DATA_DIR=${LAYEROPS_DATA_DIR}/spire/data/agent
SPIRE_URL=https://github.com/spiffe/spire/releases/download/v1.12.4/spire-1.12.4-linux-amd64-musl.tar.gz
SPIRE_SERVER_PATH=${LAYEROPS_BIN_DIR}/spire-server
SPIRE_AGENT_PATH=${LAYEROPS_BIN_DIR}/spire-agent
SPIRE_SERVER_LAUNCH_SCRIPT=${LAYEROPS_BIN_DIR}/start-spire-server
SPIRE_AGENT_LAUNCH_SCRIPT=${LAYEROPS_BIN_DIR}/start-spire-agent
SPIRE_SERVER_IP=172.24.0.1
SPIRE_BIND_PORT=8081
SPIRE_HTTP_CHALLENGE_PORT=1020
SPIRE_TRUST_DOMAIN=${ENVIRONMENT_GROUP_UUID:0:8}.layerops.io
SPIRE_SERVER_SYSTEMD_FILE=/etc/systemd/system/spire-server.service
SPIRE_AGENT_SYSTEMD_FILE=/etc/systemd/system/spire-agent.service

#VECTOR_BIND_PORT=6000
#VECTOR_SYSTEMD_OVERRIDE_DIR=/etc/systemd/system/vector.service.d
#VECTOR_SYSTEMD_OVERRIDE_FILE=${VECTOR_SYSTEMD_OVERRIDE_DIR}/override.conf
#VECTOR_ETC_DEFAULT_FILE=/etc/default/vector
#VECTOR_MAIN_CONFIG_FILE=/etc/vector/vector.yaml
#VECTOR_DYNAMIC_CONFIG_FILE=/etc/vector/dynamic.yaml
#VECTOR_LOGS_RESTORE_DIR=/data/vector/restore
#VECTOR_LOGS_RESTORE_SCRIPT=${LAYEROPS_BIN_DIR}/vector-restore-logs
#VECTOR_USERGROUP=vector
#VECTOR_RELOAD_SCRIPT=${LAYEROPS_BIN_DIR}/vector-reload

PROMETHEUS_VERSION=3.7.3
PROMETHEUS_DL_URL=https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz
PROMETHEUS_BIN_FILE=prometheus
PROMETHEUS_USERGROUP=prometheus
PROMETHEUS_CONFIG_DIR=/etc/prometheus
PROMETHEUS_CONFIG_FILE=${PROMETHEUS_CONFIG_DIR}/prom.yaml
PROMETHEUS_SYSTEMD_FILE=/etc/systemd/system/prometheus.service
PROMETHEUS_DATA_DIR=/data/prometheus
PROMETHEUS_BIND_PORT=9090
PROMETHEUS_RELOAD_SCRIPT=${LAYEROPS_BIN_DIR}/prometheus-reload

MIMIR_VERSION=2.17.2
MIMIR_DL_URL=https://github.com/grafana/mimir/releases/download/mimir-${MIMIR_VERSION}/mimir-linux-amd64
MIMIR_BIN_FILE=mimir
MIMIR_CONFIG_DIR=/etc/mimir
MIMIR_CONFIG_FILE=${MIMIR_CONFIG_DIR}/mimir.yaml
MIMIR_DATA_DIR=/data/mimir
MIMIR_TSDB_DIR=${MIMIR_DATA_DIR}/tsdb
MIMIR_TSDB_SYNC_DIR=${MIMIR_DATA_DIR}/tsdb-sync
MIMIR_RULES_DIR=${MIMIR_DATA_DIR}/rules
MIMIR_BIND_PORT=9009
MIMIR_SYSTEMD_FILE=/etc/systemd/system/mimir.service
MIMIR_RELOAD_SCRIPT=${LAYEROPS_BIN_DIR}/mimir-reload

#VICTORIALOGS_VERSION=1.36.1
#VICTORIALOGS_DL_URL=https://github.com/VictoriaMetrics/VictoriaLogs/releases/download/v${VICTORIALOGS_VERSION}/victoria-logs-linux-amd64-v${VICTORIALOGS_VERSION}.tar.gz
#VICTORIALOGS_BIN_FILE=victoria-logs-prod
#VICTORIALOGS_RELOAD_SCRIPT=${LAYEROPS_BIN_DIR}/victoria-logs-reload
#VICTORIALOGS_SNAPSHOT_SCRIPT=${LAYEROPS_BIN_DIR}/victoria-logs-snapshot
#VICTORIALOGS_DATA_DIR=/data/victorialogs
#VICTORIALOGS_BIND_PORT=9428
#VICTORIALOGS_HOMEDIR=/etc/victorialogs
#VICTORIALOGS_ENV_FILE=$VICTORIALOGS_HOMEDIR/config.env
#VICTORIALOGS_SYSTEMD_FILE=/etc/systemd/system/victorialogs.service
#VICTORIALOGS_BACKUP_SYSTEMD_FILE=/etc/systemd/system/victorialogs-backup.service
#VICTORIALOGS_BACKUP_SYSTEMD_TIMER_FILE=/etc/systemd/system/victorialogs-backup.timer
#VICTORIALOGS_USERGROUP=victorialogs
#RCLONE_CONFIG_DIR=/etc/rclone
#RCLONE_CONFIG_FILE=${RCLONE_CONFIG_DIR}/rclone.conf
#VICTORIALOGS_RCLONE_PATH_PREFIX=layerops/${ENVIRONMENT_GROUP_UUID}/victorialogs/snapshots
#LOGS_S3_BUCKET=s3-layerops

function usage {
  cat << EOF
Usage: $(basename "${BASH_SOURCE[0]}") [init|clean]
EOF
  exit 0
}

function check_envvars() {
  MISSING=""
  for envvar in ENVIRONMENT_GROUP_UUID SERVER_INSTANCE_TOKEN LAYEROPS_WORKER_URL LAYEROPS_ORCHESTRATOR_URL SPIRE_URL WIREGUARD_PRIVATE_IP
  do
    [ -z "${!envvar}" ] && MISSING="$envvar $MISSING"
  done

  if [ ! -z "$MISSING" ]
  then
    echo "Missing following environment variables:"
    for var in $MISSING
    do
      echo "  $var"
    done
    exit 1
  fi
}

function _init() {
  # Install required packages
  while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do sleep 1; done;
  while fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do sleep 1; done;
  apt-get update && apt-get install -y --no-install-recommends curl coreutils iptables jq rclone rsyslog uuid-runtime wireguard wireguard-tools zstd || exit 1

  # Disable systemd-resolved
  dpkg --purge systemd-resolved

  # Set instance signature
  SERVER_INSTANCE_SIGNATURE=$(uuidgen)
  INSTANCE_INIT_REQUEST=$(cat << EOF
{
  "instanceUuid": "${ENVIRONMENT_GROUP_UUID}",
  "instanceAccessToken": "${SERVER_INSTANCE_TOKEN}",
  "signature": "${SERVER_INSTANCE_SIGNATURE}",
  "environmentGroupUuid": "${ENVIRONMENT_GROUP_UUID}"
}
EOF
)
  curl --fail-with-body -A $CURL_USER_AGENT -X POST -H "Content-Type: application/json" -d "$INSTANCE_INIT_REQUEST" ${LAYEROPS_ORCHESTRATOR_URL}/environments/instance/init
  [ "$?" -ne "0" ] && exit 1

  # Install Docker
  mkdir -p /etc/apt/keyrings
  curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
  chmod a+r /etc/apt/keyrings/docker.asc
  echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/$(. /etc/os-release && echo $ID) \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \
    > /etc/apt/sources.list.d/docker.list
  apt-get update && apt-get install -y --no-install-recommends docker-ce docker-ce-cli

  cat <<EOF > $DOCKER_DAEMON_FILE
{
  "data-root": "$DOCKER_DATA_DIR",
}
EOF

  # Install Cadvisor
  cat <<EOF > $CADVISOR_SYSTEMD_FILE
[Unit]
Description=cadvisor
Wants=network-online.target docker.service
After=network-online.target docker.service

[Service]
ExecStart=docker run --rm --name cadvisor -p ${CADVISOR_BIND_PORT}:8080 -v "/:/rootfs:ro" -v "/var/run:/var/run:rw" -v "/sys:/sys:ro" -v "${DOCKER_DATA_DIR}/:/var/lib/docker:ro" ${CADVISOR_DOCKER_IMAGE}

[Install]
WantedBy=multi-user.target
EOF

  # Install Node_Exporter
  id $NODE_EXPORTER_USERGROUP > /dev/null 2>&1 || adduser --system --group --shell /bin/false --no-create-home $NODE_EXPORTER_USERGROUP
  curl -fsSL $NODE_EXPORTER_DL_URL \
    | tar -zxvf - -C ${LAYEROPS_BIN_DIR} --strip-components=1 node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter
  chown $NODE_EXPORTER_USERGROUP:$NODE_EXPORTER_USERGROUP ${LAYEROPS_BIN_DIR}/node_exporter
  cat <<EOF > $NODE_EXPORTER_SYSTEMD_FILE
[Unit]
Description=Node Exporter
Requires=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service
After=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service

[Service]
User=$NODE_EXPORTER_USERGROUP
Group=$NODE_EXPORTER_USERGROUP
ExecStart=${LAYEROPS_BIN_DIR}/node_exporter --web.listen-address=${WIREGUARD_PRIVATE_IP}:${NODE_EXPORTER_BIND_PORT}

[Install]
WantedBy=multi-user.target
EOF

  # Install Prometheus and Mimir
  id $PROMETHEUS_USERGROUP > /dev/null 2>&1 || adduser --system --group --shell /bin/false $PROMETHEUS_USERGROUP
  curl -fsSL $PROMETHEUS_DL_URL | tar -zxvf - -C ${LAYEROPS_BIN_DIR} --strip-components=1 prometheus-${PROMETHEUS_VERSION}.linux-amd64/${PROMETHEUS_BIN_FILE}
  curl -fsSL $MIMIR_DL_URL -o ${LAYEROPS_BIN_DIR}/${MIMIR_BIN_FILE}

  cat <<EOF > ${MIMIR_RELOAD_SCRIPT}
#!/bin/bash
$SYSTEMCTL_BIN_PATH try-reload-or-restart mimir
EOF

  cat <<EOF > ${PROMETHEUS_RELOAD_SCRIPT}
#!/bin/bash
$SYSTEMCTL_BIN_PATH try-reload-or-restart prometheus
EOF

  chmod 755 ${LAYEROPS_BIN_DIR}/${PROMETHEUS_BIN_FILE} ${LAYEROPS_BIN_DIR}/${MIMIR_BIN_FILE} ${PROMETHEUS_RELOAD_SCRIPT} ${MIMIR_RELOAD_SCRIPT}
  mkdir -p $MIMIR_CONFIG_DIR $MIMIR_DATA_DIR $MIMIR_TSDB_DIR $MIMIR_TSDB_SYNC_DIR $MIMIR_RULES_DIR $PROMETHEUS_CONFIG_DIR $PROMETHEUS_DATA_DIR

  cat <<EOF > $MIMIR_CONFIG_FILE
multitenancy_enabled: false
no_auth_tenant: ${ENVIRONMENT_GROUP_UUID}

server:
  http_listen_address: ${WIREGUARD_PRIVATE_IP}
  http_listen_port: ${MIMIR_BIND_PORT}
  log_level: error

limits:
  compactor_blocks_retention_period: 7d

blocks_storage:
  tsdb:
    dir: $MIMIR_TSDB_DIR
  bucket_store:
    sync_dir: $MIMIR_TSDB_SYNC_DIR

compactor:
  data_dir: /tmp/compactor
  sharding_ring:
    kvstore:
      store: memberlist

distributor:
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: memberlist

ingester:
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: memberlist
    replication_factor: 1

activity_tracker:
  filepath: $MIMIR_DATA_DIR/metrics-activity.log

ruler:
  rule_path: $MIMIR_RULES_DIR

store_gateway:
  sharding_ring:
    replication_factor: 1
EOF

  cat <<EOF > $PROMETHEUS_CONFIG_FILE
global:
  scrape_interval:     15s
  evaluation_interval: 15s

remote_write:
  - url: http://${WIREGUARD_PRIVATE_IP}:${MIMIR_BIND_PORT}/api/v1/push

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets: ['localhost:${PROMETHEUS_BIND_PORT}']
  - job_name: server
    static_configs:
      - targets: ["${WIREGUARD_PRIVATE_IP}:${NODE_EXPORTER_BIND_PORT}"]
EOF
  chown -R ${PROMETHEUS_USERGROUP}:${PROMETHEUS_USERGROUP} $MIMIR_CONFIG_DIR $MIMIR_DATA_DIR $PROMETHEUS_CONFIG_DIR $PROMETHEUS_DATA_DIR
  chmod 660 $MIMIR_CONFIG_FILE $PROMETHEUS_CONFIG_FILE

  cat <<EOF > $MIMIR_SYSTEMD_FILE
[Unit]
Description=Mimir
Requires=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service
After=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service

[Service]
User=${PROMETHEUS_USERGROUP}
Group=${PROMETHEUS_USERGROUP}
ExecStart=${LAYEROPS_BIN_DIR}/${MIMIR_BIN_FILE} -config.file=${MIMIR_CONFIG_FILE}

[Install]
WantedBy=multi-user.target
EOF

  cat <<EOF > $PROMETHEUS_SYSTEMD_FILE
[Unit]
Description=Prometheus
Requires=network-online.target mimir.service
After=network-online.target mimir.service

[Service]
User=${PROMETHEUS_USERGROUP}
Group=${PROMETHEUS_USERGROUP}
ExecStart=${LAYEROPS_BIN_DIR}/${PROMETHEUS_BIN_FILE} --config.file=${PROMETHEUS_CONFIG_FILE} --storage.tsdb.path=${PROMETHEUS_DATA_DIR}
ExecReload=/bin/kill -HUP \$MAINPID

[Install]
WantedBy=multi-user.target
EOF

#   # Install Victoria Logs
#   mkdir -p $VICTORIALOGS_DATA_DIR
#   id $VICTORIALOGS_USERGROUP > /dev/null 2>&1 || adduser --system --group --shell /bin/false --home=$VICTORIALOGS_HOMEDIR $VICTORIALOGS_USERGROUP
#   curl -fsSL $VICTORIALOGS_DL_URL | tar -zxvf - -C ${LAYEROPS_BIN_DIR} ${VICTORIALOGS_BIN_FILE}
#
#   cat <<EOF > $VICTORIALOGS_ENV_FILE
# VL_RETENTION_DISK=500MiB
# VL_RETENTION_PERIOD=7d
# EOF
#
#   mkdir -p $RCLONE_CONFIG_DIR
#   chmod 750 $RCLONE_CONFIG_DIR
#   touch $RCLONE_CONFIG_FILE
#   chmod 660 $VICTORIALOGS_ENV_FILE $RCLONE_CONFIG_FILE
#   chown -R $VICTORIALOGS_USERGROUP:$VICTORIALOGS_USERGROUP ${LAYEROPS_BIN_DIR}/${VICTORIALOGS_BIN_FILE} $VICTORIALOGS_DATA_DIR $RCLONE_CONFIG_DIR $VICTORIALOGS_ENV_FILE

#   cat <<EOF > $VICTORIALOGS_RESTORE_SCRIPT
# #!/bin/bash
# # If data is empty => try to restore snapshots, then start the systemd unit
# grep -q "\[${LOGS_S3_BUCKET}\]" $RCLONE_CONFIG_FILE
# if [ \$? -eq 0 ]
# then
#   for SNAPNAME in \$(rclone --config $RCLONE_CONFIG_FILE ls ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX} 2> /dev/null | grep .tar.zst | awk '{print \$2}')
#   do
#     PARTITION_DIR=$VICTORIALOGS_DATA_DIR/partitions/\${SNAPNAME%.tar.zst}
#     mkdir -p \$PARTITION_DIR
#     cd \$PARTITION_DIR
#     rclone --config $RCLONE_CONFIG_FILE cat ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/\${SNAPNAME} | tar -I zstd -xf -
#   done
# fi
#
# chown -R $VICTORIALOGS_USERGROUP:$VICTORIALOGS_USERGROUP $VICTORIALOGS_DATA_DIR
# EOF
#   cat <<EOF > $VICTORIALOGS_RELOAD_SCRIPT
# #!/bin/bash
#
# # If victorialogs is already running => reload the systemd unit
# $SYSTEMCTL_BIN_PATH status victorialogs > /dev/null 2>&1
# if [ \$? -eq 0 ]
# then
#   $SYSTEMCTL_BIN_PATH reload victorialogs
#   exit 0
# fi
#
# # If victorialogs is not running, and data is not empty => start the systemd unit
# ls $VICTORIALOGS_DATA_DIR/partitions > /dev/null 2>&1
# if [ \$? -eq 0 ]
# then
#   $SYSTEMCTL_BIN_PATH enable --now victorialogs victorialogs-backup.timer vector
#   exit 0
# fi
#
# # If data is empty => try to restore snapshots, then start the systemd unit
# grep -q "\[${LOGS_S3_BUCKET}\]" $RCLONE_CONFIG_FILE
# if [ \$? -eq 0 ]
# then
#   for SNAPNAME in \$(rclone --config $RCLONE_CONFIG_FILE ls ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX} 2> /dev/null | grep .tar.zst | awk '{print \$2}')
#   do
#     PARTITION_DIR=$VICTORIALOGS_DATA_DIR/partitions/\${SNAPNAME%.tar.zst}
#     mkdir -p \$PARTITION_DIR
#     cd \$PARTITION_DIR
#     rclone --config $RCLONE_CONFIG_FILE cat ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/\${SNAPNAME} | tar -I zstd -xf -
#   done
# fi
#
# chown -R $VICTORIALOGS_USERGROUP:$VICTORIALOGS_USERGROUP $VICTORIALOGS_DATA_DIR
#
# $SYSTEMCTL_BIN_PATH enable --now victorialogs victorialogs-backup.timer vector
#
# # Get the fresh logs from S3
# mkdir -p $VECTOR_LOGS_RESTORE_DIR
# latest=\$(rclone --config $RCLONE_CONFIG_FILE cat ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/.latest)
# now=\$(date "+%F %H:%M")
# while [ "\$(date -d "\$latest" +%Y%m%d%H%M)" -lt "\$(date -d "\$now" +%Y%m%d%H%M)" ]
# do
#   rclone --config $RCLONE_CONFIG_FILE copy ${LOGS_S3_BUCKET}:/layerops/${ENVIRONMENT_GROUP_UUID}/applogs/_all/\$(date -d "\$latest" +%Y/%m/%d)/ --include "layerops_\$(date -d "\$latest" +%F-%H)-* $VECTOR_LOGS_RESTORE_DIR/"
#   latest=\$(date -d "\$latest + 1 hour" +'%F %H:%M')
# done
#
# EOF

#   cat <<EOF > $VICTORIALOGS_SNAPSHOT_SCRIPT
# #!/bin/bash
#
# # If we are in the first hour of the day => backup a snasphot of last day
# if [ \$(date +%H) -eq 0 ]
# then
#   SNAPNAME=\$(date -d yesterday +%Y%m%d)
#   SNAPDIR=\$(curl http://${WIREGUARD_PRIVATE_IP}:${VICTORIALOGS_BIND_PORT}/internal/partition/snapshot/create?name=\$SNAPNAME | jq -r .)
#   if [ ! -z "\$SNAPDIR" ]
#   then
#     cd \$SNAPDIR
#     ls | tar -I zstd -cf - --files-from=/dev/stdin | rclone --config $RCLONE_CONFIG_FILE rcat --s3-chunk-size 100M ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/\${SNAPNAME}.tar.zst
#     cd -
#     rm -fR \$SNAPDIR
#   fi
# fi
#
# # backup a snapshot of current day
# SNAPNAME=\$(date +%Y%m%d)
# SNAPDIR=\$(curl http://${WIREGUARD_PRIVATE_IP}:$VICTORIALOGS_BIND_PORT/internal/partition/snapshot/create?name=\$SNAPNAME | jq -r .)
# [ -z "\$SNAPDIR" ] && exit 1
# cd \$SNAPDIR
# ls | tar -I zstd -cf - --files-from=/dev/stdin | rclone --config $RCLONE_CONFIG_FILE rcat --s3-chunk-size 100M ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/\${SNAPNAME}.tar.zst
# cd -
# rm -fR \$SNAPDIR
#
# date "+%F %H:%M" | rclone --config $RCLONE_CONFIG_FILE rcat ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/.latest
#
# EOF
#
#   chmod 755 $VICTORIALOGS_RELOAD_SCRIPT $VICTORIALOGS_SNAPSHOT_SCRIPT
#
#   cat <<EOF > $VICTORIALOGS_BACKUP_SYSTEMD_FILE
# [Unit]
# Description=Victoria Logs Backup
# Requires=victorialogs.service
# After=victorialogs.service
#
# [Service]
# Type=oneshot
# ExecStart=$VICTORIALOGS_SNAPSHOT_SCRIPT
# StandardOutput=journal
#
# [Install]
# WantedBy=multi-user.target
# EOF
#
#   cat <<EOF > $VICTORIALOGS_BACKUP_SYSTEMD_TIMER_FILE
# [Unit]
# Description=Victoria Logs Backup Timer
#
# [Timer]
# OnCalendar=hourly
# #OnCalendar=*-*-* *:00:00
#
# [Install]
# WantedBy=timers.target
# EOF
#
#   cat <<EOF > $VICTORIALOGS_SYSTEMD_FILE
# [Unit]
# Description=Victoria Logs
# Requires=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service
# After=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service
#
# [Service]
# User=victorialogs
# Group=victorialogs
# EnvironmentFile=-$VICTORIALOGS_ENV_FILE
# ExecStart=${LAYEROPS_BIN_DIR}/${VICTORIALOGS_BIN_FILE} -storageDataPath ${VICTORIALOGS_DATA_DIR}  -retention.maxDiskSpaceUsageBytes=\${VL_RETENTION_DISK} -retentionPeriod=\${VL_RETENTION_PERIOD} -httpListenAddr ${WIREGUARD_PRIVATE_IP}:${VICTORIALOGS_BIND_PORT}
# ExecReload=/bin/kill -HUP \$MAINPID
#
# [Install]
# WantedBy=multi-user.target
# EOF

#   # Install vector.dev
#   APT_GPG_KEYS=("DATADOG_APT_KEY_CURRENT.public" "DATADOG_APT_KEY_C0962C7D.public" "DATADOG_APT_KEY_F14F620E.public")
#   APT_USR_SHARE_KEYRING=/usr/share/keyrings/datadog-archive-keyring.gpg
#   touch $APT_USR_SHARE_KEYRING
#   for key in $APT_GPG_KEYS
#   do
#     curl -sSL --retry 5 https://keys.datadoghq.com/$key | gpg --import --batch --no-default-keyring --keyring "$APT_USR_SHARE_KEYRING"
#   done
#   echo "deb [signed-by=${APT_USR_SHARE_KEYRING}] https://apt.vector.dev/ stable vector-0" \
#     > /etc/apt/sources.list.d/vector.list
#   apt-get update && apt-get install -y --no-install-recommends vector
#   grep -qxF 'VECTOR_CONFIG_YAML' $VECTOR_ETC_DEFAULT_FILE || echo 'VECTOR_CONFIG_YAML=/etc/vector/*.yaml' >> $VECTOR_ETC_DEFAULT_FILE
#
#   mkdir -p $VECTOR_SYSTEMD_OVERRIDE_DIR
#   cat <<EOF > $VECTOR_SYSTEMD_OVERRIDE_FILE
# [Unit]
# Requires=network-online.target victorialogs.service-quick@${WIREGUARD_INTERFACE}.service
# After=network-online.target victorialogs.service wg-quick@${WIREGUARD_INTERFACE}.service
#
# [Service]
# # Disable ExecStartPre=/usr/bin/vector validate to keep vector running if sinks are down
# ExecStartPre=
# EOF
#
#   cat <<EOF > $VECTOR_RELOAD_SCRIPT
# #!/bin/bash
# $SYSTEMCTL_BIN_PATH try-reload-or-restart vector
# EOF
#
#   mkdir -p $VECTOR_LOGS_RESTORE_DIR
#   cat > $VECTOR_LOGS_RESTORE_SCRIPT <<EOF
# #!/bin/bash
# file=\$(ls -AU $VECTOR_LOGS_RESTORE_DIR 2> /dev/null | head -1) \
#   && [ -f "$VECTOR_LOGS_RESTORE_DIR/\$file" ] \
#   && cat $VECTOR_LOGS_RESTORE_DIR/\$file && rm -f $VECTOR_LOGS_RESTORE_DIR/\$file
# EOF
#   chmod 755 $VECTOR_LOGS_RESTORE_SCRIPT $VECTOR_RELOAD_SCRIPT
#
#   cat > $VECTOR_MAIN_CONFIG_FILE <<EOF
# sources:
#   vector:
#     type: vector
#     address: ${WIREGUARD_PRIVATE_IP}:${VECTOR_BIND_PORT}
#   backup_restore:
#     type: exec
#     command:
#       - $VECTOR_LOGS_RESTORE_SCRIPT
#     decoding:
#       codec: json
#     include_stderr: false
#     mode: scheduled
#
# sinks:
#   victorialogs:
#     inputs: ["vector", "backup_restore"]
#     type: http
#     uri: http://${WIREGUARD_PRIVATE_IP}:${VICTORIALOGS_BIND_PORT}/insert/jsonline?_stream_fields=host,container_name&_msg_field=message&_time_field=timestamp
#     compression: gzip
#     encoding:
#       codec: json
#       only_fields:
#         - timestamp
#         - stream
#         - environmentUuid
#         - host
#         - serviceId
#         - serviceUuid
#         - container_name
#         - message
#     framing:
#       method: newline_delimited
#     healthcheck:
#       enabled: false
# EOF

  # Create layerops user
  id $LAYEROPS_USER > /dev/null 2>&1 || adduser -q --gecos "" --disabled-password --home $LAYEROPS_HOME_DIR $LAYEROPS_USER
  mkdir -p $LAYEROPS_HOME_DIR/.ssh
  touch $LAYEROPS_HOME_DIR/.ssh/authorized_keys
  # adduser $LAYEROPS_USER $VICTORIALOGS_USERGROUP
  adduser $LAYEROPS_USER $PROMETHEUS_USERGROUP
  adduser $LAYEROPS_USER $DOCKER_GROUP

  # touch $VECTOR_DYNAMIC_CONFIG_FILE
  # chown $LAYEROPS_USER:$VECTOR_USERGROUP $VECTOR_DYNAMIC_CONFIG_FILE

  # Create mandatory folders
  mkdir -p \
    $WIREGUARD_CONFIG_DIR \
    $LAYEROPS_ETC_DIR \
    $LAYEROPS_BIN_DIR \
    $DOCKER_DATA_DIR \
    $SPIRE_CONFIG_DIR \
    $SPIRE_SERVER_DATA_DIR \
    $SPIRE_AGENT_DATA_DIR
  touch \
    $LAYEROPS_ETC_DEFAULT_FILE \
    $SPIRE_AGENT_CONFIG_FILE \
    $SPIRE_SERVER_CONFIG_FILE \
    $WIREGUARD_CONFIG_FILE \
    $WIREGUARD_TEMPLATE_FILE
  chmod 700 $WIREGUARD_CONFIG_DIR $LAYEROPS_ETC_DIR $SPIRE_CONFIG_DIR

  # Set sudo access to layerops user
  cat > $SUDO_FILE <<EOF
# Group rules for layerops
%layerops ALL=(ALL) NOPASSWD: /usr/bin/wg-quick, /usr/bin/wg, $WIREGUARD_QUICK_RELOAD_PATH
%layerops ALL=(ALL) NOPASSWD: $SPIRE_SERVER_LAUNCH_SCRIPT
%layerops ALL=(ALL) NOPASSWD: $SPIRE_AGENT_LAUNCH_SCRIPT
EOF
# %layerops ALL=(ALL) NOPASSWD: $VICTORIALOGS_RELOAD_SCRIPT
# %layerops ALL=(ALL) NOPASSWD: $VECTOR_RELOAD_SCRIPT
# %layerops ALL=(ALL) NOPASSWD: $MIMIR_RELOAD_SCRIPT
# %layerops ALL=(ALL) NOPASSWD: $PROMETHEUS_RELOAD_SCRIPT

  # Enable ip forwarding (for wireguard gateway)
  iptables -P FORWARD ACCEPT
  touch /etc/sysctl.conf
  grep -qxF 'net.ipv4.ip_forward = 1' /etc/sysctl.conf || echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf
  sysctl -p /etc/sysctl.conf

  # Init Wireguard:
  WIREGUARD_PRIVATE_KEY=$(wg genkey)
  WIREGUARD_PUBLIC_KEY=$(echo "$WIREGUARD_PRIVATE_KEY" | wg pubkey)
  WIREGUARD_PORT=51820
  cat > $WIREGUARD_CONFIG_FILE <<EOF
[Interface]
PrivateKey = $WIREGUARD_PRIVATE_KEY
Address = $WIREGUARD_PRIVATE_IP/32
ListenPort = $WIREGUARD_PORT
PostUp = ip -4 route add $WIREGUARD_SUBNET dev $WIREGUARD_INTERFACE
PostDown = ip -4 route del $WIREGUARD_SUBNET dev $WIREGUARD_INTERFACE
Table = off
EOF
  cat > $WIREGUARD_TEMPLATE_FILE <<EOF
[Interface]
PrivateKey = $WIREGUARD_PRIVATE_KEY
Address = $WIREGUARD_PRIVATE_IP/32
ListenPort = $WIREGUARD_PORT
PostUp = ip -4 route add $WIREGUARD_SUBNET dev $WIREGUARD_INTERFACE
PostDown = ip -4 route del $WIREGUARD_SUBNET dev $WIREGUARD_INTERFACE
Table = off

{{ range .WireguardPeers }}
# {{ .Role }} - {{ .InstanceUuid }}
[Peer]
PublicKey = {{ .WireguardPublicKey }}
AllowedIPs = {{ .PrivateIp }}/32
PersistentKeepalive = 15
{{ if and (eq .Role "lb") (ne .PublicIp "") }}Endpoint = {{ .PublicIp }}:$WIREGUARD_PORT{{ end }}

{{ end }}
EOF

  cat > $WIREGUARD_QUICK_RELOAD_PATH <<EOF
#!/bin/bash
#/usr/bin/wg-quick up \$1 > /dev/null 2>&1 || /usr/bin/wg syncconf \$1 <(exec /usr/bin/wg-quick strip \$1)
systemctl reload wg-quick@$1
EOF
  chmod 755 $WIREGUARD_QUICK_RELOAD_PATH

cat > $WIREGUARD_RELOAD_PATH <<EOF
#!/bin/bash
sudo $WIREGUARD_QUICK_RELOAD_PATH $WIREGUARD_INTERFACE
EOF
  chmod 755 $WIREGUARD_RELOAD_PATH

  # Install Spire server
  cd /tmp
  curl -s -N -L $SPIRE_URL | tar xz
  mv spire*/bin/spire-server $SPIRE_SERVER_PATH
  mv spire*/bin/spire-agent $SPIRE_AGENT_PATH
  chmod 755 $SPIRE_SERVER_PATH $SPIRE_AGENT_PATH

  # Configure Spire server
  cat > $SPIRE_SERVER_CONFIG_FILE <<EOF
server {
    bind_address = "$WIREGUARD_PRIVATE_IP"
    bind_port = "$SPIRE_BIND_PORT"
    trust_domain = "$SPIRE_TRUST_DOMAIN"
    data_dir = "$SPIRE_SERVER_DATA_DIR"
    log_level = "DEBUG"
    ca_ttl = "168h"
    default_x509_svid_ttl = "48h"
}

plugins {
    DataStore "sql" {
        plugin_data {
            database_type = "sqlite3"
            connection_string = "$SPIRE_SERVER_DATA_DIR/datastore.sqlite3"
        }
    }

    KeyManager "disk" {
        plugin_data {
            keys_path = "$SPIRE_SERVER_DATA_DIR/keys.json"
        }
    }

    NodeAttestor "http_challenge" {
        plugin_data {
            allowed_dns_patterns = ["172\\\.24\\\.[0-9]+\\\.[0-9]+"]
            required_port = $SPIRE_HTTP_CHALLENGE_PORT
            tofu = false
        }
    }
}
EOF

  # Configure Spire agent
  cat > $SPIRE_AGENT_CONFIG_FILE <<EOF
agent {
    data_dir = "$SPIRE_AGENT_DATA_DIR"
    log_level = "DEBUG"
    trust_domain = "$SPIRE_TRUST_DOMAIN"
    server_address = "$SPIRE_SERVER_IP"
    server_port = $SPIRE_BIND_PORT
}

plugins {
   KeyManager "disk" {
        plugin_data {
            directory = "$SPIRE_AGENT_DATA_DIR"
        }
    }

    NodeAttestor "http_challenge" {
        plugin_data {
            hostname = "$WIREGUARD_PRIVATE_IP"
            port = $SPIRE_HTTP_CHALLENGE_PORT
        }
    }

    WorkloadAttestor "unix" {
        plugin_data {}
    }
    WorkloadAttestor "docker" {
        plugin_data {}
    }
}
EOF

  cat > $SPIRE_SERVER_SYSTEMD_FILE <<EOF
[Unit]
Description=Spire Server
Requires=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service
After=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP

WorkingDirectory=$LAYEROPS_HOME_DIR
TimeoutStartSec=0
ExecStart=$SPIRE_SERVER_PATH run -config $SPIRE_SERVER_CONFIG_FILE
ExecStop=/bin/kill -9 \$MAINPID

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  cat > $SPIRE_AGENT_SYSTEMD_FILE <<EOF
[Unit]
Description=Spire Agent
Requires=network-online.target
Requires=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service
After=network-online.target wg-quick@${WIREGUARD_INTERFACE}.service

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP

TimeoutStartSec=0
ExecStartPre=+setcap 'cap_net_bind_service=+ep' $SPIRE_AGENT_PATH
ExecStart=$SPIRE_AGENT_PATH run -config $SPIRE_AGENT_CONFIG_FILE
ExecStop=/bin/kill -9 \$MAINPID

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  cat > $SPIRE_SERVER_LAUNCH_SCRIPT <<EOF
#!/bin/bash
systemctl enable --now spire-server
EOF
  chmod 755 $SPIRE_SERVER_LAUNCH_SCRIPT

  cat > $SPIRE_AGENT_LAUNCH_SCRIPT <<EOF
#!/bin/bash
systemctl enable --now spire-agent
EOF
  chmod 755 $SPIRE_AGENT_LAUNCH_SCRIPT

  # Install layerops worker
  curl -s -N -L -o $LAYEROPS_WORKER_PATH $LAYEROPS_WORKER_URL
  chmod 755 $LAYEROPS_WORKER_PATH

  LAYEROPS_WORKER_CMD="$LAYEROPS_WORKER_PATH --role=server --api-orchestrator=$LAYEROPS_ORCHESTRATOR_URL --environment-group-uuid=$ENVIRONMENT_GROUP_UUID --instance-uuid=$ENVIRONMENT_GROUP_UUID --signature=$SERVER_INSTANCE_SIGNATURE"

  # Setup layerops config
cat > $LAYEROPS_ETC_DEFAULT_FILE <<EOF
WIREGUARD_RELOAD_PATH=$WIREGUARD_RELOAD_PATH
WIREGUARD_CONFIG_FILE=$WIREGUARD_CONFIG_FILE
WIREGUARD_TEMPLATE_FILE=$WIREGUARD_TEMPLATE_FILE
WIREGUARD_PUBLIC_KEY=$WIREGUARD_PUBLIC_KEY
SPIRE_SERVER_LAUNCH_SCRIPT=$SPIRE_SERVER_LAUNCH_SCRIPT
SPIRE_AGENT_LAUNCH_SCRIPT=$SPIRE_AGENT_LAUNCH_SCRIPT
SPIRE_TRUST_DOMAIN=$SPIRE_TRUST_DOMAIN
EOF

  cat > $LAYEROPS_INSTANCES_SYSTEMD_FILE <<EOF
[Unit]
Description=Launch layerops instances worker as server mode

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP
EnvironmentFile=-$LAYEROPS_ETC_DEFAULT_FILE
ExecStart=$LAYEROPS_WORKER_CMD --watch=instances

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  cat > $LAYEROPS_SERVICES_SYSTEMD_FILE <<EOF
[Unit]
Description=Launch layerops services worker as server mode

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP
EnvironmentFile=-$LAYEROPS_ETC_DEFAULT_FILE
ExecStart=$LAYEROPS_WORKER_CMD --watch=services

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  cat > $LAYEROPS_CERTBOT_SYSTEMD_FILE <<EOF
[Unit]
Description=Launch layerops certbot worker as server mode

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP
EnvironmentFile=-$LAYEROPS_ETC_DEFAULT_FILE
ExecStart=$LAYEROPS_WORKER_CMD --watch=certbot

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  cat > $LAYEROPS_MONITORING_SYSTEMD_FILE <<EOF
[Unit]
Description=Launch layerops monitoring worker as server mode

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP
EnvironmentFile=-$LAYEROPS_ETC_DEFAULT_FILE
ExecStart=$LAYEROPS_WORKER_CMD --watch=monitoring

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  # Set ownership for /opt/layerops directory
  chown -R $LAYEROPS_USER:$LAYEROPS_GROUP $LAYEROPS_HOME_DIR $WIREGUARD_CONFIG_DIR $LAYEROPS_ETC_DIR $SPIRE_CONFIG_DIR
  chown root:root $DOCKER_DATA_DIR

  # Enable and start layerops-instances worker
  systemctl daemon-reload
  systemctl enable --now \
    wg-quick@${WIREGUARD_INTERFACE} \
    layerops-instances \
    layerops-services \
    layerops-certbot \
    layerops-monitoring \
    node_exporter \
    mimir \
    prometheus
  systemctl restart docker
}


function _clean() {
  # Remove layerops sudo config
  rm -f $SUDO_FILE

  # Stop services
  $SYSTEMCTL_BIN_PATH disable --now \
    layerops-instances \
    layerops-services \
    layerops-certbot \
    layerops-monitoring \
    spire-agent \
    spire-server \
    node_exporter \
    mimir \
    prometheus \
    # victorialogs \
    # vector \
    wg-quick@${WIREGUARD_INTERFACE}
  rm -f \
    $LAYEROPS_INSTANCES_SYSTEMD_FILE \
    $LAYEROPS_SERVICES_SYSTEMD_FILE \
    $LAYEROPS_CERTBOT_SYSTEMD_FILE \
    $LAYEROPS_MONITORING_SYSTEMD_FILE \
    $SPIRE_AGENT_SYSTEMD_FILE \
    $SPIRE_SERVER_SYSTEMD_FILE
    # $VICTORIALOGS_DATA_DIR
  systemctl daemon-reload

  # Delete layerops user
  id $LAYEROPS_USER > /dev/null 2>&1 && userdel $LAYEROPS_USER

  # Uninstall Wireguard
  apt-get uninstall -y wireguard wireguard-tools

  # Clean files
  rm -fR \
    $LAYEROPS_HOME_DIR \
    $LAYEROPS_WORKER_PATH \
    $WIREGUARD_CONFIG_FILE \
    $WIREGUARD_TEMPLATE_FILE \
    $LAYEROPS_ETC_DIR
}

if [[ "$#" -eq 0 ]]; then
  usage
fi

case "$1" in
  init)
    check_envvars
    _init
    ;;
  clean)
    _clean;;
  *)
    usage;;
esac