#!/usr/bin/env bash

# Source common variables and functions
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/layerops-install-common"

# === Server-specific variables ===
REQUIRED_ENVVARS="ENVIRONMENT_GROUP_UUID SERVER_INSTANCE_TOKEN LAYEROPS_WORKER_URL LAYEROPS_ORCHESTRATOR_URL SPIRE_URL WIREGUARD_PRIVATE_IP LAYEROPS_API_URL"
SYSTEMCTL_BIN_PATH=/usr/bin/systemctl

# SPIRE Server specific
SPIRE_SERVER_CONFIG_FILE=${SPIRE_CONFIG_DIR}/server.conf
SPIRE_SERVER_DATA_DIR=${LAYEROPS_DATA_DIR}/spire/server
SPIRE_SERVER_PATH=${LAYEROPS_BIN_DIR}/spire-server
SPIRE_SERVER_LAUNCH_SCRIPT=${LAYEROPS_BIN_DIR}/start-spire-server
SPIRE_BUNDLE_TRUST_PORT=8082
SPIRE_SERVER_SYSTEMD_FILE=/etc/systemd/system/spire-server.service
SPIRE_TRUST_BUNDLE_PATH=${LAYEROPS_DATA_DIR}/spire/trust-bundle.pem

# Logs backup and restore (victorialogs snapshots + vector server configuration)
RCLONE_CONFIG_DIR=$LAYEROPS_ETC_DIR/rclone
RCLONE_CONFIG_FILE=${RCLONE_CONFIG_DIR}/rclone.conf
VICTORIALOGS_RCLONE_PATH_PREFIX=layerops/${ENVIRONMENT_GROUP_UUID}/victorialogs/snapshots
LOGS_S3_BUCKET=s3-layerops
VICTORIALOGS_CONTAINER_ID=${ENVIRONMENT_GROUP_UUID:0:8}-monitoring-victorialogs-${ENVIRONMENT_GROUP_UUID:0:8}
VICTORIALOGS_CONTAINER_PORT=9428
VICTORIALOGS_DATA_DIR=${LAYEROPS_DATA_DIR}/volumes/${VICTORIALOGS_CONTAINER_ID}/data
VICTORIALOGS_SNAPSHOT_SCRIPT=${LAYEROPS_BIN_DIR}/victorialogs-snapshot
VICTORIALOGS_RESTORE_SCRIPT=${LAYEROPS_BIN_DIR}/victorialogs-restore
VICTORIALOGS_SNAPSHOT_SYSTEMD_FILE=/etc/systemd/system/victorialogs-snapshot.service
VICTORIALOGS_SNAPSHOT_SYSTEMD_TIMER_FILE=/etc/systemd/system/victorialogs-snapshot.timer
VICTORIALOGS_SNAPSHOT_FREQUENCY=1hour
VECTOR_SERVER_CONTAINER_ID=${ENVIRONMENT_GROUP_UUID:0:8}-monitoring-vector-server-${ENVIRONMENT_GROUP_UUID:0:8}
VECTOR_SERVER_RESTORE_DIR=${LAYEROPS_DATA_DIR}/volumes/${VECTOR_SERVER_CONTAINER_ID}/backups
VECTOR_SERVER_S3_PATH_PREFIX=layerops/${ENVIRONMENT_GROUP_UUID}/applogs/_all

function usage {
  cat << EOF
Usage: $(basename "${BASH_SOURCE[0]}") [init|clean]
EOF
  exit 0
}

# Setup Victoria Logs S3 Backups
function _setup_victorialogs_backup() {
  mkdir -p $RCLONE_CONFIG_DIR
  chmod 750 $RCLONE_CONFIG_DIR

  cat <<EOF > $VICTORIALOGS_SNAPSHOT_SCRIPT
#!/bin/bash

grep -q "\[${LOGS_S3_BUCKET}\]" $RCLONE_CONFIG_FILE
if [ \$? -ne 0 ]
then
  echo "[VICTORIALOGS SNAPSHOT] S3 backend is not configured yet => nothing to do"
  exit 0
fi

# Get Victorialogs container address
VICTORIALOGS_CONTAINER_IP=\$(docker inspect $VICTORIALOGS_CONTAINER_ID | jq -r '.[].NetworkSettings.Networks | to_entries[0].value.IPAddress' 2> /dev/null)
if [ -z "\$VICTORIALOGS_CONTAINER_IP" ]
then
  echo "[VICTORIALOGS SNAPSHOT] Victorialogs container is not running => nothing to do"
  exit 0
fi

VICTORIALOGS_SNAPHOST_URL="http://\$VICTORIALOGS_CONTAINER_IP:$VICTORIALOGS_CONTAINER_PORT/internal/partition/snapshot"

# If we are in the first hour of the day => backup a snasphot of last day
if [ \$(date +%H) -eq 0 ]
then
  SNAPNAME=\$(date -d yesterday +%Y%m%d)
  SNAPDIR=\$(curl \${VICTORIALOGS_SNAPHOST_URL}/create?name=\$SNAPNAME | jq -r .)
  if [ ! -z "\$SNAPDIR" ]
  then
    cd \$SNAPDIR
    ls | tar -I zstd -cf - --files-from=/dev/stdin | rclone --config $RCLONE_CONFIG_FILE rcat --s3-chunk-size 100M ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/\${SNAPNAME}.tar.zst
    cd -
    rm -fR \$SNAPDIR
  fi
fi

# backup a snapshot of current day
SNAPNAME=\$(date +%Y%m%d)
SNAPDIR=\$(curl \${VICTORIALOGS_SNAPHOST_URL}/create?name=\$SNAPNAME | jq -r .)
[ -z "\$SNAPDIR" ] && exit 1
cd \$SNAPDIR
ls | tar -I zstd -cf - --files-from=/dev/stdin | rclone --config $RCLONE_CONFIG_FILE rcat --s3-chunk-size 100M ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/\${SNAPNAME}.tar.zst
cd -
rm -fR \$SNAPDIR

# Force UTC time for .latest tag because vector.dev uploaded files are set in UTC (cannot be overriden in Vector.dev. cf. https://vector.dev/docs/architecture/data-model/log/#timestamps)
date -Iseconds | rclone --config $RCLONE_CONFIG_FILE rcat ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/.latest

EOF

  cat <<EOF > $VICTORIALOGS_SNAPSHOT_SYSTEMD_FILE
[Unit]
Description=Run Victorialogs snapshot

[Service]
Type=oneshot
ExecStart=$VICTORIALOGS_SNAPSHOT_SCRIPT
EnvironmentFile=-$LAYEROPS_ETC_DEFAULT_FILE
EOF

  cat <<EOF > $VICTORIALOGS_SNAPSHOT_SYSTEMD_TIMER_FILE
[Unit]
Description=Schedule Victorialogs snapshots

[Timer]
OnBootSec=$VICTORIALOGS_SNAPSHOT_FREQUENCY
OnUnitActiveSec=$VICTORIALOGS_SNAPSHOT_FREQUENCY

[Install]
WantedBy=timers.target
EOF

  cat <<EOF > $VICTORIALOGS_RESTORE_SCRIPT
#!/bin/bash

grep -q "\[${LOGS_S3_BUCKET}\]" $RCLONE_CONFIG_FILE
if [ \$? -ne 0 ]
then
  echo "[VICTORIALOGS RESTORE] S3 backend is not configured yet => nothing to restore"
  exit 0
fi

if [ "\$(docker inspect --format '{{.State.Status}}' $VICTORIALOGS_CONTAINER_ID 2> /dev/null)" == "running" ]
then
  echo "[VICTORIALOGS RESTORE] Victorialogs container is already running => nothing to restore"
  exit 0
fi

ls $VICTORIALOGS_DATA_DIR/partitions > /dev/null 2>&1
if [ \$? -eq 0 ]
then
  echo "[VICTORIALOGS RESTORE] Victorialogs data is not empty => nothing to restore"
  exit 0
fi

# If data is empty => try to restore snapshots
for SNAPNAME in \$(rclone --config $RCLONE_CONFIG_FILE ls ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX} 2> /dev/null | grep .tar.zst | awk '{print \$2}')
do
  PARTITION_DIR=$VICTORIALOGS_DATA_DIR/partitions/\${SNAPNAME%.tar.zst}
  mkdir -p \$PARTITION_DIR
  cd \$PARTITION_DIR
  rclone --config $RCLONE_CONFIG_FILE cat ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/\${SNAPNAME} | tar -I zstd -xf -
  cd -
done

# Get the fresh logs from S3 (@TODO: exécuter en parallèle, pour ne pas bloquer le process si gros volume de logs à restaurer)
latest=\$(rclone --config $RCLONE_CONFIG_FILE cat ${LOGS_S3_BUCKET}:/${VICTORIALOGS_RCLONE_PATH_PREFIX}/.latest)
[ -z "\$latest" ] && latest=\$(date -d "1 hour ago" -Iseconds)
now=\$(date -Iseconds)
while [ "\$(date -d "\$latest" +%Y%m%d%H%M)" -lt "\$(date -d "\$now" +%Y%m%d%H%M)" ]
do
  mkdir -p $VECTOR_SERVER_RESTORE_DIR
  chown $LAYEROPS_USER:$LAYEROPS_GROUP $VECTOR_SERVER_RESTORE_DIR
  rclone --config $RCLONE_CONFIG_FILE copy ${LOGS_S3_BUCKET}:/${VECTOR_SERVER_S3_PATH_PREFIX}/\$(date -d "\$latest" +%Y/%m/%d)/ --include "layerops_\$(date -u -d "\$latest" +%F-%H)-*" $VECTOR_SERVER_RESTORE_DIR/
  latest=\$(date -d "\$latest + 1 hour" -Iseconds)
done

EOF

  chmod 755 $VICTORIALOGS_SNAPSHOT_SCRIPT $VICTORIALOGS_RESTORE_SCRIPT
}

# Install SPIRE server and agent binaries
function _install_spire_binaries() {
  cd /tmp
  curl_with_retry -s -N -L $SPIRE_URL | tar xz || exit 1
  mv spire*/bin/spire-server $SPIRE_SERVER_PATH
  mv spire*/bin/spire-agent $SPIRE_AGENT_PATH
  chmod 755 $SPIRE_SERVER_PATH $SPIRE_AGENT_PATH
}

# Configure SPIRE server
function _configure_spire_server() {

  cat > /tmp/datastore.sql <<EOF
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "federated_registration_entries" ("bundle_id" integer,"registered_entry_id" integer, PRIMARY KEY ("bundle_id","registered_entry_id"));
CREATE TABLE IF NOT EXISTS "bundles" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"trust_domain" varchar(255) NOT NULL,"data" blob );
INSERT INTO bundles VALUES(1,'2026-05-22 00:24:16.353157464+02:00','2026-05-22 00:24:16.353157464+02:00','spiffe://1944506b.layerops.io',X'0a1d7370696666653a2f2f31393434353036622e6c617965726f70732e696f128e040a8b0430820207308201aea003020102021034e079a5db1fefd72c47025ceef4b769300a06082a8648ce3d040302304f310b3009060355040613025553310f300d060355040a1306535049464645312f302d060355040513263730323835333937353839313532313039393531363338333235353731353637373934303235301e170d3236303532313232323430365a170d3236303532323232323431365a304f310b3009060355040613025553310f300d060355040a1306535049464645312f302d0603550405132637303238353339373538393135323130393935313633383332353537313536373739343032353059301306072a8648ce3d020106082a8648ce3d03010703420004654059869551f91553e271ba9f70016ce1a04df75b3d024072a476883a806ba086158d04c99d92a7127c3d5628ac2183c4eb7ccc69c97320f44f51e7b33d47c2a36c306a300e0603551d0f0101ff040403020106300f0603551d130101ff040530030101ff301d0603551d0e04160414732af7f26d411b3144a8abfae03d65f43820170230280603551d110421301f861d7370696666653a2f2f31393434353036622e6c617965726f70732e696f300a06082a8648ce3d040302034700304402206600d4ff7eb2d3b728ad314673a5d0be3d86e79da79805a0a1974d09661c00a0022059636e3394a8f91a9b9e0475e6076bda5583b1f71d6984635912435dfafe081f');
CREATE TABLE IF NOT EXISTS "attested_node_entries" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"spiffe_id" varchar(255),"data_type" varchar(255),"serial_number" varchar(255),"expires_at" datetime,"new_serial_number" varchar(255),"new_expires_at" datetime,"can_reattest" bool );
INSERT INTO attested_node_entries VALUES(1,'2026-05-22 00:24:56.924534569+02:00','2026-05-22 00:24:56.924534569+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1','http_challenge','139305978110210085797385139789202822334','2026-05-22 01:24:56+02:00','',NULL,1);
INSERT INTO attested_node_entries VALUES(2,'2026-05-22 00:25:18.899071724+02:00','2026-05-22 00:25:18.899071724+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584','http_challenge','91610639479921082603756923122466006813','2026-05-22 01:25:18+02:00','',NULL,1);
INSERT INTO attested_node_entries VALUES(3,'2026-05-22 00:25:19.99573733+02:00','2026-05-22 00:25:19.99573733+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.1.1.16287f83-b00f-488a-8f5e-4de91264ce76','http_challenge','178132272467634680586315345112105029449','2026-05-22 01:25:19+02:00','',NULL,1);
CREATE TABLE IF NOT EXISTS "attested_node_entries_events" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"spiffe_id" varchar(255) );
INSERT INTO attested_node_entries_events VALUES(1,'2026-05-22 00:24:56.920027106+02:00','2026-05-22 00:24:56.920027106+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1');
INSERT INTO attested_node_entries_events VALUES(2,'2026-05-22 00:24:56.924667968+02:00','2026-05-22 00:24:56.924667968+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1');
INSERT INTO attested_node_entries_events VALUES(3,'2026-05-22 00:25:18.898734926+02:00','2026-05-22 00:25:18.898734926+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584');
INSERT INTO attested_node_entries_events VALUES(4,'2026-05-22 00:25:18.899189723+02:00','2026-05-22 00:25:18.899189723+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584');
INSERT INTO attested_node_entries_events VALUES(5,'2026-05-22 00:25:19.986214909+02:00','2026-05-22 00:25:19.986214909+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.1.1.16287f83-b00f-488a-8f5e-4de91264ce76');
INSERT INTO attested_node_entries_events VALUES(6,'2026-05-22 00:25:19.996230926+02:00','2026-05-22 00:25:19.996230926+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.1.1.16287f83-b00f-488a-8f5e-4de91264ce76');
CREATE TABLE IF NOT EXISTS "node_resolver_map_entries" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"spiffe_id" varchar(255),"type" varchar(255),"value" varchar(255) );
INSERT INTO node_resolver_map_entries VALUES(1,'2026-05-22 00:24:56.919893807+02:00','2026-05-22 00:24:56.919893807+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1','http_challenge','hostname:172.24.0.1');
INSERT INTO node_resolver_map_entries VALUES(2,'2026-05-22 00:25:18.897028641+02:00','2026-05-22 00:25:18.897028641+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584','http_challenge','hostname:172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584');
INSERT INTO node_resolver_map_entries VALUES(3,'2026-05-22 00:25:19.985145718+02:00','2026-05-22 00:25:19.985145718+02:00','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.1.1.16287f83-b00f-488a-8f5e-4de91264ce76','http_challenge','hostname:172.24.1.1.16287f83-b00f-488a-8f5e-4de91264ce76');
CREATE TABLE IF NOT EXISTS "registered_entries" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"entry_id" varchar(255),"spiffe_id" varchar(255),"parent_id" varchar(255),"ttl" integer,"admin" bool,"downstream" bool,"expiry" bigint,"revision_number" bigint,"store_svid" bool,"hint" varchar(255),"jwt_svid_ttl" integer );
INSERT INTO registered_entries VALUES(1,'2026-05-22 00:26:16.417537007+02:00','2026-05-22 00:26:16.417537007+02:00','62748503-b2f0-43dc-8066-92b58d79696b','spiffe://1944506b.layerops.io/1944506b-8512-4cf7-9674-06b709544b22/web','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.1.1.16287f83-b00f-488a-8f5e-4de91264ce76',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(2,'2026-05-22 00:26:16.449878845+02:00','2026-05-22 00:26:16.449878845+02:00','5d97742b-5edb-4d0e-8ce0-e24813d21214','spiffe://1944506b.layerops.io/load-balancer','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(3,'2026-05-22 00:26:16.471173871+02:00','2026-05-22 00:26:16.471173871+02:00','2de8058e-587a-4984-bbfe-1f81fb2852fb','spiffe://1944506b.layerops.io/custom-load-balancer','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(4,'2026-05-22 00:26:16.498861147+02:00','2026-05-22 00:26:16.498861147+02:00','9b9e5da0-d82e-4af2-8d38-e109249dfb36','spiffe://1944506b.layerops.io/monitoring-mimir','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(5,'2026-05-22 00:26:16.526819119+02:00','2026-05-22 00:26:16.526819119+02:00','0493a42e-39dd-4180-b31a-ee2669af24b0','spiffe://1944506b.layerops.io/monitoring-prometheus','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(6,'2026-05-22 00:26:16.553031706+02:00','2026-05-22 00:26:16.553031706+02:00','45da5ed8-a983-4102-9fdc-1d630e507d03','spiffe://1944506b.layerops.io/monitoring-alertmanager','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(7,'2026-05-22 00:26:16.563125324+02:00','2026-05-22 00:26:16.563125324+02:00','296d22a5-82a8-41ea-8bc8-f8f8a3dfdd6a','spiffe://1944506b.layerops.io/monitoring-victorialogs','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(8,'2026-05-22 00:26:16.571874353+02:00','2026-05-22 00:26:16.571874353+02:00','46b1219b-0db9-42ab-ba2f-f30121dc04f7','spiffe://1944506b.layerops.io/monitoring-vector-server','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.1',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(9,'2026-05-22 00:26:16.58089228+02:00','2026-05-22 00:26:16.58089228+02:00','0ca26328-0d4d-4074-94c7-1e3d0e9b8211','spiffe://1944506b.layerops.io/monitoring-vector-client','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.1.1.16287f83-b00f-488a-8f5e-4de91264ce76',0,0,0,0,0,0,'',0);
INSERT INTO registered_entries VALUES(10,'2026-05-22 00:26:16.590903799+02:00','2026-05-22 00:26:16.590903799+02:00','5cf6d345-8411-4037-aca7-38959f6c491c','spiffe://1944506b.layerops.io/monitoring-vector-client','spiffe://1944506b.layerops.io/spire/agent/http_challenge/172.24.0.10.a81bb52e-d58e-4023-ab55-41b22a26d584',0,0,0,0,0,0,'',0);
CREATE TABLE IF NOT EXISTS "registered_entries_events" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"entry_id" varchar(255) );
INSERT INTO registered_entries_events VALUES(1,'2026-05-22 00:26:16.419131294+02:00','2026-05-22 00:26:16.419131294+02:00','62748503-b2f0-43dc-8066-92b58d79696b');
INSERT INTO registered_entries_events VALUES(2,'2026-05-22 00:26:16.455157502+02:00','2026-05-22 00:26:16.455157502+02:00','5d97742b-5edb-4d0e-8ce0-e24813d21214');
INSERT INTO registered_entries_events VALUES(3,'2026-05-22 00:26:16.478951808+02:00','2026-05-22 00:26:16.478951808+02:00','2de8058e-587a-4984-bbfe-1f81fb2852fb');
INSERT INTO registered_entries_events VALUES(4,'2026-05-22 00:26:16.502356618+02:00','2026-05-22 00:26:16.502356618+02:00','9b9e5da0-d82e-4af2-8d38-e109249dfb36');
INSERT INTO registered_entries_events VALUES(5,'2026-05-22 00:26:16.528823603+02:00','2026-05-22 00:26:16.528823603+02:00','0493a42e-39dd-4180-b31a-ee2669af24b0');
INSERT INTO registered_entries_events VALUES(6,'2026-05-22 00:26:16.554683193+02:00','2026-05-22 00:26:16.554683193+02:00','45da5ed8-a983-4102-9fdc-1d630e507d03');
INSERT INTO registered_entries_events VALUES(7,'2026-05-22 00:26:16.564037217+02:00','2026-05-22 00:26:16.564037217+02:00','296d22a5-82a8-41ea-8bc8-f8f8a3dfdd6a');
INSERT INTO registered_entries_events VALUES(8,'2026-05-22 00:26:16.573424541+02:00','2026-05-22 00:26:16.573424541+02:00','46b1219b-0db9-42ab-ba2f-f30121dc04f7');
INSERT INTO registered_entries_events VALUES(9,'2026-05-22 00:26:16.582051871+02:00','2026-05-22 00:26:16.582051871+02:00','0ca26328-0d4d-4074-94c7-1e3d0e9b8211');
INSERT INTO registered_entries_events VALUES(10,'2026-05-22 00:26:16.592160188+02:00','2026-05-22 00:26:16.592160188+02:00','5cf6d345-8411-4037-aca7-38959f6c491c');
CREATE TABLE IF NOT EXISTS "join_tokens" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"token" varchar(255),"expiry" bigint );
CREATE TABLE IF NOT EXISTS "selectors" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"registered_entry_id" integer,"type" varchar(255),"value" varchar(255) );
INSERT INTO selectors VALUES(1,'2026-05-22 00:26:16.418182102+02:00','2026-05-22 00:26:16.418182102+02:00',1,'docker','label:io.layerops.service-uuid:d23fb9d6-9b0c-47cf-9680-d3003105b11c');
INSERT INTO selectors VALUES(2,'2026-05-22 00:26:16.418315501+02:00','2026-05-22 00:26:16.418315501+02:00',1,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(3,'2026-05-22 00:26:16.451236733+02:00','2026-05-22 00:26:16.451236733+02:00',2,'docker','label:io.layerops.service-uuid:1944506b-8512-4cf7-9674-06b709544b20');
INSERT INTO selectors VALUES(4,'2026-05-22 00:26:16.453863512+02:00','2026-05-22 00:26:16.453863512+02:00',2,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(5,'2026-05-22 00:26:16.473552352+02:00','2026-05-22 00:26:16.473552352+02:00',3,'docker','label:io.layerops.service-uuid:1944506b-8512-4cf7-9674-06b709544b21');
INSERT INTO selectors VALUES(6,'2026-05-22 00:26:16.474069448+02:00','2026-05-22 00:26:16.474069448+02:00',3,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(7,'2026-05-22 00:26:16.500407034+02:00','2026-05-22 00:26:16.500407034+02:00',4,'docker','label:io.layerops.service-uuid:75a88edb-5cc8-49c2-b4e1-bd0da31a0f2d');
INSERT INTO selectors VALUES(8,'2026-05-22 00:26:16.500598532+02:00','2026-05-22 00:26:16.500598532+02:00',4,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(9,'2026-05-22 00:26:16.527535214+02:00','2026-05-22 00:26:16.527535214+02:00',5,'docker','label:io.layerops.service-uuid:412ea221-903d-4476-8199-72120f6ba666');
INSERT INTO selectors VALUES(10,'2026-05-22 00:26:16.528153109+02:00','2026-05-22 00:26:16.528153109+02:00',5,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(11,'2026-05-22 00:26:16.553553302+02:00','2026-05-22 00:26:16.553553302+02:00',6,'docker','label:io.layerops.service-uuid:390c5808-1370-4ee8-948c-3383f2c886a2');
INSERT INTO selectors VALUES(12,'2026-05-22 00:26:16.553724701+02:00','2026-05-22 00:26:16.553724701+02:00',6,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(13,'2026-05-22 00:26:16.563375222+02:00','2026-05-22 00:26:16.563375222+02:00',7,'docker','label:io.layerops.service-uuid:ec2094ee-6866-42bc-9d14-37fc142f44b7');
INSERT INTO selectors VALUES(14,'2026-05-22 00:26:16.56360622+02:00','2026-05-22 00:26:16.56360622+02:00',7,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(15,'2026-05-22 00:26:16.57222305+02:00','2026-05-22 00:26:16.57222305+02:00',8,'docker','label:io.layerops.service-uuid:126bbd22-6655-4550-82b9-5052d8706854');
INSERT INTO selectors VALUES(16,'2026-05-22 00:26:16.572356349+02:00','2026-05-22 00:26:16.572356349+02:00',8,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(17,'2026-05-22 00:26:16.581220877+02:00','2026-05-22 00:26:16.581220877+02:00',9,'docker','label:io.layerops.service-uuid:94bf13c7-3170-49f8-891a-c6acd053211a');
INSERT INTO selectors VALUES(18,'2026-05-22 00:26:16.581366976+02:00','2026-05-22 00:26:16.581366976+02:00',9,'docker','label:io.layerops.side-task-name:envoy-proxy');
INSERT INTO selectors VALUES(19,'2026-05-22 00:26:16.591234596+02:00','2026-05-22 00:26:16.591234596+02:00',10,'docker','label:io.layerops.service-uuid:94bf13c7-3170-49f8-891a-c6acd053211a');
INSERT INTO selectors VALUES(20,'2026-05-22 00:26:16.591391095+02:00','2026-05-22 00:26:16.591391095+02:00',10,'docker','label:io.layerops.side-task-name:envoy-proxy');
CREATE TABLE IF NOT EXISTS "migrations" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"version" integer,"code_version" varchar(255) );
INSERT INTO migrations VALUES(1,'2026-05-22 00:24:16.299333389+02:00','2026-05-22 00:24:16.299333389+02:00',23,'1.14.0');
CREATE TABLE IF NOT EXISTS "dns_names" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"registered_entry_id" integer,"value" varchar(255) );
CREATE TABLE IF NOT EXISTS "federated_trust_domains" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"trust_domain" varchar(255) NOT NULL,"bundle_endpoint_url" varchar(255),"bundle_endpoint_profile" varchar(255),"endpoint_spiffe_id" varchar(255),"implicit" bool );
CREATE TABLE IF NOT EXISTS "ca_journals" ("id" integer primary key autoincrement,"created_at" datetime,"updated_at" datetime,"data" blob,"active_x509_authority_id" varchar(255),"active_jwt_authority_id" varchar(255) );
INSERT INTO ca_journals VALUES(1,'2026-05-22 00:24:16.35614464+02:00','2026-05-22 00:24:16.35614464+02:00',X'0ac9040a014110908cbed0061a8b0430820207308201aea003020102021034e079a5db1fefd72c47025ceef4b769300a06082a8648ce3d040302304f310b3009060355040613025553310f300d060355040a1306535049464645312f302d060355040513263730323835333937353839313532313039393531363338333235353731353637373934303235301e170d3236303532313232323430365a170d3236303532323232323431365a304f310b3009060355040613025553310f300d060355040a1306535049464645312f302d0603550405132637303238353339373538393135323130393935313633383332353537313536373739343032353059301306072a8648ce3d020106082a8648ce3d03010703420004654059869551f91553e271ba9f70016ce1a04df75b3d024072a476883a806ba086158d04c99d92a7127c3d5628ac2183c4eb7ccc69c97320f44f51e7b33d47c2a36c306a300e0603551d0f0101ff040403020106300f0603551d130101ff040530030101ff301d0603551d0e04160414732af7f26d411b3144a8abfae03d65f43820170230280603551d110421301f861d7370696666653a2f2f31393434353036622e6c617965726f70732e696f300a06082a8648ce3d040302034700304402206600d4ff7eb2d3b728ad314673a5d0be3d86e79da79805a0a1974d09661c00a0022059636e3394a8f91a9b9e0475e6076bda5583b1f71d6984635912435dfafe081f28033228373332616637663236643431316233313434613861626661653033643635663433383230313730323890afc3d006','732af7f26d411b3144a8abfae03d65f438201702','');
DELETE FROM sqlite_sequence;
INSERT INTO sqlite_sequence VALUES('migrations',1);
INSERT INTO sqlite_sequence VALUES('bundles',1);
INSERT INTO sqlite_sequence VALUES('ca_journals',1);
INSERT INTO sqlite_sequence VALUES('node_resolver_map_entries',3);
INSERT INTO sqlite_sequence VALUES('attested_node_entries_events',6);
INSERT INTO sqlite_sequence VALUES('attested_node_entries',3);
INSERT INTO sqlite_sequence VALUES('registered_entries',10);
INSERT INTO sqlite_sequence VALUES('selectors',20);
INSERT INTO sqlite_sequence VALUES('registered_entries_events',10);
CREATE UNIQUE INDEX uix_bundles_trust_domain ON "bundles"(trust_domain) ;
CREATE INDEX idx_attested_node_entries_expires_at ON "attested_node_entries"(expires_at) ;
CREATE UNIQUE INDEX uix_attested_node_entries_spiffe_id ON "attested_node_entries"(spiffe_id) ;
CREATE UNIQUE INDEX idx_node_resolver_map ON "node_resolver_map_entries"(spiffe_id, "type", "value") ;
CREATE INDEX idx_registered_entries_spiffe_id ON "registered_entries"(spiffe_id) ;
CREATE INDEX idx_registered_entries_parent_id ON "registered_entries"(parent_id) ;
CREATE INDEX idx_registered_entries_expiry ON "registered_entries"("expiry") ;
CREATE INDEX idx_registered_entries_hint ON "registered_entries"("hint") ;
CREATE UNIQUE INDEX uix_registered_entries_entry_id ON "registered_entries"(entry_id) ;
CREATE UNIQUE INDEX uix_join_tokens_token ON "join_tokens"("token") ;
CREATE INDEX idx_selectors_type_value ON "selectors"("type", "value") ;
CREATE UNIQUE INDEX idx_selector_entry ON "selectors"(registered_entry_id, "type", "value") ;
CREATE UNIQUE INDEX idx_dns_entry ON "dns_names"(registered_entry_id, "value") ;
CREATE UNIQUE INDEX uix_federated_trust_domains_trust_domain ON "federated_trust_domains"(trust_domain) ;
CREATE INDEX idx_ca_journals_active_x509_authority_id ON "ca_journals"(active_x509_authority_id) ;
CREATE INDEX idx_ca_journals_active_jwt_authority_id ON "ca_journals"(active_jwt_authority_id) ;
CREATE INDEX idx_federated_registration_entries_registered_entry_id ON "federated_registration_entries"(registered_entry_id) ;
COMMIT;
EOF
  sqlite3 $SPIRE_SERVER_DATA_DIR/datastore.sqlite3 < /tmp/datastore.sql

  cat > $SPIRE_SERVER_DATA_DIR/keys.json <<EOF
{
	"keys": {
		"x509-CA-A": "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgoyWGSNBjeYxc2k1qn5fDFbn9t7EO7zfGE7oteEaBdQehRANCAARlQFmGlVH5FVPicbqfcAFs4aBN91s9AkBypHaIOoBroIYVjQTJnZKnEnw9ViisIYPE63zMaclzIPRPUeezPUfC"
	}
}
EOF
  chown layerops:layerops $SPIRE_SERVER_DATA_DIR/*

  cat > $SPIRE_SERVER_CONFIG_FILE <<EOF
server {
    bind_address = "$WIREGUARD_PRIVATE_IP"
    bind_port = "$SPIRE_BIND_PORT"
    trust_domain = "$SPIRE_TRUST_DOMAIN"
    data_dir = "$SPIRE_SERVER_DATA_DIR"
    log_level = "INFO"
    ca_ttl = "24h"
    default_x509_svid_ttl = "1h"
    disable_jwt_svids = true
}

plugins {
    DataStore "sql" {
        plugin_data {
            database_type = "sqlite3"
            connection_string = "$SPIRE_SERVER_DATA_DIR/datastore.sqlite3"
        }
    }

    KeyManager "disk" {
        plugin_data {
            keys_path = "$SPIRE_SERVER_DATA_DIR/keys.json"
        }
    }

    NodeAttestor "http_challenge" {
        plugin_data {
            allowed_dns_patterns = ["172\\\.24\\\.[0-9]+\\\.[0-9]+(\\\..+)?"]
            required_port = $SPIRE_HTTP_CHALLENGE_PORT
            tofu = false
        }
    }
}
EOF
}

# Configure SPIRE agent for server (uses trust_bundle_path instead of URL)
function _configure_spire_agent_server() {
  cat > $SPIRE_AGENT_CONFIG_FILE <<EOF
agent {
    data_dir = "$SPIRE_AGENT_DATA_DIR"
    log_level = "DEBUG"
    trust_domain = "$SPIRE_TRUST_DOMAIN"
    server_address = "$SPIRE_SERVER_IP"
    server_port = $SPIRE_BIND_PORT
    trust_bundle_path = "$SPIRE_TRUST_BUNDLE_PATH"
    rebootstrap_mode = "auto"
}

plugins {
   KeyManager "disk" {
        plugin_data {
            directory = "$SPIRE_AGENT_DATA_DIR"
        }
    }

    NodeAttestor "http_challenge" {
        plugin_data {
            hostname = "$WIREGUARD_PRIVATE_IP"
            port = $SPIRE_HTTP_CHALLENGE_PORT
        }
    }

    WorkloadAttestor "unix" {
        plugin_data {}
    }
    WorkloadAttestor "docker" {
        plugin_data {}
    }
}
EOF
}

# Create SPIRE server systemd service
function _create_spire_server_systemd() {
  cat > $SPIRE_SERVER_SYSTEMD_FILE <<EOF
[Unit]
Description=Spire Server
Requires=network-online.target layerops-wireguard-init.service
After=network-online.target layerops-wireguard-init.service

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP

WorkingDirectory=$LAYEROPS_HOME_DIR
TimeoutStartSec=0
ExecStart=$SPIRE_SERVER_PATH run -config $SPIRE_SERVER_CONFIG_FILE
ExecStop=/bin/kill -9 \$MAINPID

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  cat > $SPIRE_SERVER_LAUNCH_SCRIPT <<EOF
#!/bin/bash
systemctl enable --now spire-server
EOF
  chmod 755 $SPIRE_SERVER_LAUNCH_SCRIPT
}

# Create SPIRE agent systemd service for server
function _create_spire_agent_systemd_server() {
  cat > $SPIRE_AGENT_SYSTEMD_FILE <<EOF
[Unit]
Description=Spire Agent
Requires=network-online.target layerops-wireguard-init.service
After=network-online.target layerops-wireguard-init.service

[Service]
User=$LAYEROPS_USER
Group=$LAYEROPS_GROUP

TimeoutStartSec=0
ExecStartPre=+setcap 'cap_net_bind_service=+ep' $SPIRE_AGENT_PATH
ExecStart=$SPIRE_AGENT_PATH run -config $SPIRE_AGENT_CONFIG_FILE
ExecStop=/bin/kill -9 \$MAINPID

Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

  cat > $SPIRE_AGENT_LAUNCH_SCRIPT <<EOF
#!/bin/bash
systemctl enable --now spire-agent
EOF
  chmod 755 $SPIRE_AGENT_LAUNCH_SCRIPT
}

# Create server-specific config file
function _create_config_file() {
  cat > $LAYEROPS_ETC_DEFAULT_FILE <<EOF
WIREGUARD_RELOAD_PATH=$WIREGUARD_RELOAD_PATH
WIREGUARD_CLEAN_PATH=$WIREGUARD_CLEAN_PATH
WIREGUARD_PRIVATE_IP=$WIREGUARD_PRIVATE_IP
WIREGUARD_PRIVATE_KEY=$WIREGUARD_PRIVATE_KEY
WIREGUARD_PUBLIC_KEY=$WIREGUARD_PUBLIC_KEY
WIREGUARD_MTU=$WIREGUARD_MTU
WIREGUARD_SUBNET=$WIREGUARD_SUBNET
HOSTFILE_RELOAD_PATH=$HOSTFILE_RELOAD_PATH
SPIRE_SERVER_LAUNCH_SCRIPT=$SPIRE_SERVER_LAUNCH_SCRIPT
SPIRE_AGENT_LAUNCH_SCRIPT=$SPIRE_AGENT_LAUNCH_SCRIPT
SPIRE_TRUST_DOMAIN=$SPIRE_TRUST_DOMAIN
LAYEROPS_DATA_DIR=$LAYEROPS_DATA_DIR
STATS_PORT=$STATS_PORT
STATS_CACHE_TTL=$STATS_CACHE_TTL
API_URL=$LAYEROPS_API_URL
LAYEROPS_WORKER_URL=$LAYEROPS_WORKER_URL
LAYEROPS_WORKER_VERSION_CHECK=$LAYEROPS_WORKER_VERSION_CHECK
LAYEROPS_WORKER_SIGNATURE_PUBLIC_KEY=$LAYEROPS_WORKER_SIGNATURE_PUBLIC_KEY
LAYEROPS_MARKETPLACE_REGISTRY=${MARKETPLACE_REGISTRY:-registry.nimeops.net/layerops-public/marketplace}
ROLE=server
API_ORCHESTRATOR=$LAYEROPS_ORCHESTRATOR_URL
ENVIRONMENT_GROUP_UUID=$ENVIRONMENT_GROUP_UUID
INSTANCE_UUID=$ENVIRONMENT_GROUP_UUID
SIGNATURE=$SERVER_INSTANCE_SIGNATURE
STATS=true
EOF
}

function _init() {
  # Install required packages
  install_base_packages curl coreutils gnupg iptables jq rclone rsyslog uuid-runtime wireguard wireguard-tools zstd

  # Disable systemd-resolved
  dpkg --purge systemd-resolved

  # Set instance signature
  SERVER_INSTANCE_SIGNATURE=$(uuidgen)
  INSTANCE_INIT_REQUEST=$(cat << EOF
{
  "instanceUuid": "${ENVIRONMENT_GROUP_UUID}",
  "instanceAccessToken": "${SERVER_INSTANCE_TOKEN}",
  "signature": "${SERVER_INSTANCE_SIGNATURE}",
  "environmentGroupUuid": "${ENVIRONMENT_GROUP_UUID}"
}
EOF
)
  curl_with_retry --fail-with-body -A $CURL_USER_AGENT -X POST -H "Content-Type: application/json" -d "$INSTANCE_INIT_REQUEST" ${LAYEROPS_ORCHESTRATOR_URL}/instances/init
  [ "$?" -ne "0" ] && exit 1

  # Create layerops user (without additional SSH keys for server)
  id $LAYEROPS_USER > /dev/null 2>&1 || adduser -q --gecos "" --disabled-password --home $LAYEROPS_HOME_DIR $LAYEROPS_USER
  mkdir -p $LAYEROPS_HOME_DIR/.ssh
  touch $LAYEROPS_HOME_DIR/.ssh/authorized_keys

  # Create mandatory folders (including server-specific ones)
  mkdir -p \
    $WIREGUARD_CONFIG_DIR \
    $LAYEROPS_ETC_DIR \
    $LAYEROPS_BIN_DIR \
    $DOCKER_DATA_DIR \
    $SPIRE_CONFIG_DIR \
    $SPIRE_SERVER_DATA_DIR \
    $SPIRE_AGENT_DATA_DIR
  touch \
    $LAYEROPS_ETC_DEFAULT_FILE \
    $SPIRE_AGENT_CONFIG_FILE \
    $SPIRE_SERVER_CONFIG_FILE
  chmod 700 $WIREGUARD_CONFIG_DIR $LAYEROPS_ETC_DIR $SPIRE_CONFIG_DIR

  # Install Docker
  install_docker

  # Install Node Exporter
  install_node_exporter

  # Setup Victoria Logs S3 Backups
  _setup_victorialogs_backup

  # Set sudo access to layerops user
  cat > $SUDO_FILE <<EOF
# Group rules for layerops
%layerops ALL=(ALL) NOPASSWD: /usr/bin/wg-quick, /usr/bin/wg, $WIREGUARD_QUICK_RELOAD_PATH, $WIREGUARD_QUICK_CLEAN_PATH
%layerops ALL=(ALL) NOPASSWD: $HOSTFILE_SUDO_RELOAD_PATH
%layerops ALL=(ALL) NOPASSWD: $SPIRE_SERVER_LAUNCH_SCRIPT
%layerops ALL=(ALL) NOPASSWD: $SPIRE_AGENT_LAUNCH_SCRIPT
%layerops ALL=(ALL) NOPASSWD: $VICTORIALOGS_RESTORE_SCRIPT
EOF
# %layerops ALL=(ALL) NOPASSWD: $VICTORIALOGS_RELOAD_SCRIPT
# %layerops ALL=(ALL) NOPASSWD: $VECTOR_RELOAD_SCRIPT
# %layerops ALL=(ALL) NOPASSWD: $MIMIR_RELOAD_SCRIPT
# %layerops ALL=(ALL) NOPASSWD: $PROMETHEUS_RELOAD_SCRIPT

  # Enable ip forwarding (for wireguard gateway)
  iptables -P FORWARD ACCEPT
  enable_ip_forwarding

  # Init Wireguard
  init_wireguard_base
  _launch_wireguard_ecmp_service

  # Install SPIRE server and agent
  _install_spire_binaries
  _configure_spire_server
  _configure_spire_agent_server
  _create_spire_server_systemd
  _create_spire_agent_systemd_server

  # Install layerops worker
  install_layerops_worker

  # Setup layerops config
  _create_config_file

  # Create worker systemd services
  create_worker_systemd_services

  # Set ownership for /opt/layerops directory
  chown -R $LAYEROPS_USER:$LAYEROPS_GROUP $LAYEROPS_HOME_DIR $LAYEROPS_DATA_DIR $WIREGUARD_CONFIG_DIR $LAYEROPS_ETC_DIR $SPIRE_CONFIG_DIR
  chown root:root $DOCKER_DATA_DIR

  # Enable and start layerops worker
  systemctl daemon-reload
  systemctl enable --now \
    layerops-wireguard-init \
    layerops-wireguard-ecmp \
    layerops-worker@${REMOTE_VERSION}.service \
    layerops-worker-update.timer \
    victorialogs-snapshot.timer \
    node_exporter
  systemctl restart docker
}


function _clean() {
  # Stop services
  $SYSTEMCTL_BIN_PATH disable --now \
    layerops-worker@${REMOTE_VERSION} \
    layerops-worker-update.timer \
    victorialogs-snapshot.timer \
    spire-agent \
    spire-server \
    node_exporter \
    layerops-wireguard-ecmp \
    layerops-wireguard-init
  rm -f \
    $LAYEROPS_WORKER_SYSTEMD_FILE \
    $LAYEROPS_WIREGUARD_INIT_SYSTEMD_FILE \
    $WIREGUARD_ECMP_SYSTEMD_UNIT \
    $SPIRE_AGENT_SYSTEMD_FILE \
    $SPIRE_SERVER_SYSTEMD_FILE
  systemctl daemon-reload

  # Clean base layerops installation
  clean_layerops_base
}

if [[ "$#" -eq 0 ]]; then
  usage
fi

case "$1" in
  init)
    check_envvars
    _init
    ;;
  clean)
    _clean;;
  *)
    usage;;
esac