feat: add backup cronjob

refactor: cleanup bootstrap scripts
2024-02-26 15:48:04 +03:00 · 2024-01-28 15:08:57 +00:00
parent 446fdf695a
commit d87720cb8a
11 changed files with 254 additions and 144 deletions
--- a/README.md
+++ b/README.md
@@ -28,6 +28,9 @@ updates and cleanup.
 | `UPDATE_CRON`         |          |                     | string (cron format) | Update game server files cron (eg. `*/30 * * * *` check for updates every 30 minutes)                                        |     |
 | `STEAM_API_PUBLIC_IP` |          |                     | string (ipv4)        | Server IP for the server empty check                                                                                         |     |
 | `STEAM_API_KEY`       |          |                     | string               | SteamApi key to authorize requests (needed for empty server check). Server restarts regardless of occupancy, if not provided |     |
+| `BACKUP_CRON`         |          |                     | string (cron format) | Backup game server files cron (eg. `*/15 * * *` backup saves every 15 minutes)                                               | ⚠️  |
+| `BACKUP_DIR`          |          | `./backup`          | string               | Folder for backups (relative and absolute paths are supported)                                                               | ⚠️  |
+| `BACKUP_COUNT`        |          | `0`                 | integer              | Number of backups to keep (0 means infinite)                                                                                 | ⚠️  |
 | `GAME_BRANCH`         |          | `public`            | string               | Steam branch (eg. testing) of the Enshrouded server                                                                          |     |
 | `STEAMCMD_ARGS`       |          | `validate`          | string               | Additional steamcmd args for the updater                                                                                     |     |

--- a/dockerfiles/Dockerfile
+++ b/dockerfiles/Dockerfile
@@ -10,7 +10,9 @@ RUN apt-get update \
        supervisor \
        cron \
        rsyslog \
-        jq
+        jq \
+        lsof \
+        zip

 # Install wine
 ARG WINE_BRANCH=stable
--- a/dockerfiles/proton.Dockerfile
+++ b/dockerfiles/proton.Dockerfile
@@ -12,6 +12,8 @@ RUN dpkg --add-architecture i386 \
        cron \
        rsyslog \
        jq \
+        lsof \
+        zip \
        tar \
        dbus \
        libfreetype6 \
--- a/scripts/default/bootstrap
+++ b/scripts/default/bootstrap
@@ -4,11 +4,39 @@

 main() {
  verifyVariables
+  createFolders
  applyPermissions
  setupSyslog
  exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
 }

+createFolders() {
+  info "Creating server folders (save, logs, backup)"
+  if [[ -n "$SERVER_SAVE_DIR" ]]; then
+    if [[ "$SERVER_SAVE_DIR" == /* ]]; then
+      mkdir -p "$SERVER_SAVE_DIR"
+    else
+      mkdir -p "$install_path/$SERVER_SAVE_DIR"
+    fi
+  fi
+
+  if [[ -n "$SERVER_LOG_DIR" ]]; then
+    if [[ "$SERVER_LOG_DIR" == /* ]]; then
+      mkdir -p "$SERVER_LOG_DIR"
+    else
+      mkdir -p "$install_path/$SERVER_LOG_DIR"
+    fi
+  fi
+
+  if [[ -n "$BACKUP_CRON" ]] && [[ -n "$BACKUP_DIR" ]]; then
+    if [[ "$BACKUP_DIR" == /* ]]; then
+      mkdir -p "$BACKUP_DIR"
+    else
+      mkdir -p "$install_path/$BACKUP_DIR"
+    fi
+  fi
+}
+
 # Apply user id and group id
 applyPermissions() {
  info "Setting uid:gid of enshrouded to $PUID:$PGID"
@@ -31,6 +59,11 @@ applyPermissions() {
    chown -R enshrouded:enshrouded "$SERVER_LOG_DIR"
  fi

+  if [[ "$BACKUP_DIR" == /* ]]; then
+    debug "Setting permissions for $BACKUP_DIR"
+    chown -R enshrouded:enshrouded "$BACKUP_DIR"
+  fi
+
  chgrp enshrouded /etc/supervisor/supervisord.conf
 }

--- a/scripts/default/common
+++ b/scripts/default/common
@@ -4,6 +4,7 @@ version_file_path=/opt/enshrouded/current_version
 steamcmd_path=/usr/bin/steamcmd
 steam_app_id=2278520
 enshrouded_binary_path=enshrouded_server.exe
+savefile_name=3ad85aea

 # trap SIGUSR1 as it is being used to check
 # for process aliveness when an existing
@@ -13,6 +14,7 @@ trap ':' USR1
 # Collection of PID files
 enshrouded_server_pidfile=/var/run/enshrouded/enshrouded-server.pid
 enshrouded_updater_pidfile=/var/run/enshrouded/enshrouded-updater.pid
+enshrouded_backup_pidfile=/var/run/enshrouded/enshrouded-backup.pid

 # Syslog supervisor config file
 supervisor_syslog_conf=/usr/local/etc/supervisor/conf.d/syslog.conf
--- a/scripts/default/defaults
+++ b/scripts/default/defaults
@@ -14,13 +14,20 @@ SERVER_LOG_DIR=${SERVER_LOG_DIR} # defaults to ./logs
 GAME_BRANCH=${GAME_BRANCH:-public}
 STEAMCMD_ARGS=${STEAMCMD_ARGS:-"$GAME_BRANCH" validate}

-# MISC
-PUID=${PUID:-4711}
-PGID=${PGID:-4711}
+# Update
 UPDATE_CRON=${UPDATE_CRON:-}
 STEAM_API_PUBLIC_IP=${STEAM_API_PUBLIC_IP:-}
 STEAM_API_KEY=${STEAM_API_KEY:-}

+# Backup
+BACKUP_CRON=${BACKUP_CRON:-}
+BACKUP_DIR=${BACKUP_DIR:-backups}
+BACKUP_MAX_COUNT=${BACKUP_MAX_COUNT:-0}
+
+# MISC
+PUID=${PUID:-4711}
+PGID=${PGID:-4711}
+
 # WINE
 WINEDEBUG=${WINEDEBUG:-fixme-all}

--- a/scripts/default/enshrouded-backup
+++ b/scripts/default/enshrouded-backup
@@ -0,0 +1,109 @@
+#!/bin/bash
+. "$(dirname "$0")/common"
+. "$(dirname "$0")/defaults"
+
+pidfile=$enshrouded_backup_pidfile
+save_dir=""
+backup_dir=""
+
+main() {
+  info "Running enshrouded-backup"
+  prepareBackup
+  info "enshrouded-backup complete"
+}
+
+prepareBackup() {
+  if [ -f "$pidfile" ]; then
+    info "Found existing PID file - checking process"
+    checkLock $pidfile
+  fi
+
+  getSaveDir
+  getBackupDir
+
+  if [ ! -f "$save_dir/$savefile_name" ]; then
+    warn "Save file not found - aborting backup"
+    return
+  fi
+
+  trap shutdown SIGINT SIGTERM
+
+  backupAndCleanup &
+  enshrouded_backup_pid=$!
+  echo $enshrouded_backup_pid >"$pidfile"
+  wait $enshrouded_backup_pid
+}
+
+backupAndCleanup() {
+  backup
+  cleanup
+  clearLock "$pidfile"
+}
+
+backup() {
+  local backup_file_name
+  debug "run backup"
+
+  # check if save file is older than 6 minutes
+  if [ $(($(date +%s) - $(stat -c %X "$save_dir/$savefile_name"))) -gt 360 ]; then
+    info "Save file is older than 6 minutes - server is probably just starting up - skipping backup"
+    return
+  fi
+
+  # check last access time > 4 minutes wait for 90 seconds
+  if [ $(($(date +%s) - $(stat -c %X "$save_dir/$savefile_name"))) -gt 240 ]; then
+    info "Save file is older than 4 minutes - waiting for 90 seconds to ensure save is complete"
+    sleep 90
+  fi
+
+  # check if file is still being written to with lsof
+  while [ -n "$(lsof -t "$save_dir/$savefile_name" "$save_dir/$savefile_name.backup")" ]; do
+    info "Save file is still being written to - waiting for 5 seconds to ensure save is complete"
+    sleep 5
+  done
+
+  backup_file_name="$(date +%Y-%m-%d_%H-%M-%S)-$savefile_name.zip"
+
+  info "Backing up save file to $backup_dir/$backup_file_name"
+  debug "zip -j "$backup_dir/$backup_file_name" "$save_dir/$savefile_name" "$save_dir/$savefile_name.backup""
+  zip -j "$backup_dir/$backup_file_name" "$save_dir/$savefile_name" "$save_dir/$savefile_name.backup"
+}
+
+cleanup() {
+  debug "run cleanup"
+
+  # skip if BACKUP_MAX_COUNT is not set or is 0
+  if [ -z "$BACKUP_MAX_COUNT" ] || [ "$BACKUP_MAX_COUNT" -eq 0 ]; then
+    warn "Skipping cleanup - BACKUP_MAX_COUNT is not set or is 0 (infinite)"
+    return
+  fi
+
+  info "Removing old backups with number of backups set to $BACKUP_MAX_COUNT"
+  find "$backup_dir" -type f -printf '%T@ %p\n' | sort -n | cut -d' ' -f 2- | head -n -$BACKUP_MAX_COUNT | xargs rm -fv
+}
+
+getSaveDir() {
+  if [ -z "$SERVER_SAVE_DIR" ]; then
+    SERVER_SAVE_DIR=$(jq -r '.saveDirectory' ${install_path}/enshrouded_server.json)
+  fi
+  if [[ "$SERVER_SAVE_DIR" == /* ]]; then
+    save_dir="$SERVER_SAVE_DIR"
+  else
+    save_dir="$install_path/$SERVER_SAVE_DIR"
+  fi
+}
+
+getBackupDir() {
+  if [[ "$BACKUP_DIR" == /* ]]; then
+    backup_dir="$BACKUP_DIR"
+  else
+    backup_dir="$install_path/$BACKUP_DIR"
+  fi
+}
+
+shutdown() {
+  debug "Received signal to shut down enshrouded-backup"
+  clearLock "$pidfile"
+}
+
+main
--- a/scripts/default/enshrouded-bootstrap-shared
+++ b/scripts/default/enshrouded-bootstrap-shared
@@ -0,0 +1,78 @@
+#!/bin/bash
+. "$(dirname "$0")/common"
+. "$(dirname "$0")/defaults"
+
+prepareEnshroudedAppFolders() {
+  mkdir -p "$install_path"
+}
+
+initCrontab() {
+  crontab=$(mktemp)
+
+  if [ -n "$UPDATE_CRON" ]; then
+    debug "creating cron for update checks (schedule: $UPDATE_CRON)"
+    echo "$UPDATE_CRON supervisorctl start enshrouded-updater >/dev/null 2>&1" >>"$crontab"
+  fi
+
+  if [ -n "$BACKUP_CRON" ]; then
+    debug "creating cron for backups (schedule: $BACKUP_CRON)"
+    echo "$BACKUP_CRON supervisorctl start enshrouded-backup >/dev/null 2>&1" >>"$crontab"
+  fi
+
+  crontab "$crontab"
+  rm -f "$crontab"
+}
+
+updateOrCreateEnshroudedServerConfig() {
+  if [[ ! -e ${install_path}/enshrouded_server.json ]]; then
+    mkdir -p ${install_path}
+    touch ${install_path}/enshrouded_server.json
+
+    # write json to file ${install_path}/enshrouded_server.json
+
+    cat >${install_path}/enshrouded_server.json << EOF
+{
+  "name": "Enshrouded Server",
+  "password": "",
+  "saveDirectory": "./savegame",
+  "logDirectory": "./logs",
+  "ip": "0.0.0.0",
+  "gamePort": 15636,
+  "queryPort": 15637,
+  "slotCount": 16
+}
+EOF
+  fi
+
+  if [[ -n "$SERVER_NAME" ]]; then
+    echo "$(jq --arg name "$SERVER_NAME" '.name = $name' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+
+  if [[ -n "$SERVER_PASSWORD" ]]; then
+    echo "$(jq --arg password "$SERVER_PASSWORD" '.password = $password' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+
+  if [[ -n "$SERVER_SAVE_DIR" ]]; then
+    echo "$(jq --arg saveDirectory "$SERVER_SAVE_DIR" '.saveDirectory = $saveDirectory' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+
+  if [[ -n "$SERVER_LOG_DIR" ]]; then
+    echo "$(jq --arg logDirectory "$SERVER_LOG_DIR" '.logDirectory = $logDirectory' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+
+  if [[ -n "$SERVER_IP" ]]; then
+    echo "$(jq --arg ip "$SERVER_IP" '.ip = $ip' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+
+  if [[ -n "$SERVER_PORT" ]]; then
+    echo "$(jq --argjson gamePort "$SERVER_PORT" '.gamePort = $gamePort' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+
+  if [[ -n "$SERVER_QUERYPORT" ]]; then
+    echo "$(jq --argjson queryPort "$SERVER_QUERYPORT" '.queryPort = $queryPort' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+
+  if [[ -n "$SERVER_SLOT_COUNT" ]]; then
+    echo "$(jq --argjson slotCount "$SERVER_SLOT_COUNT" '.slotCount = $slotCount' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
+  fi
+}
--- a/scripts/proton/enshrouded-bootstrap
+++ b/scripts/proton/enshrouded-bootstrap
@@ -1,6 +1,7 @@
 #!/bin/bash
 . "$(dirname "$0")/common"
 . "$(dirname "$0")/defaults"
+. "$(dirname "$0")/enshrouded-bootstrap-shared"

 main() {
  info "Running enshrouded-bootstrap"
@@ -13,10 +14,6 @@ main() {
  info "Bootstrap complete"
 }

-prepareEnshroudedAppFolders() {
-  mkdir -p "$install_path"
-}
-
 bootstrapProton() {
  info "bootstrap proton wine"
  export WINEPREFIX=$WINEPREFIX
@@ -41,70 +38,4 @@ fixSteamClient() {
  cp ~/Steam/steamapps/common/Steamworks\ SDK\ Redist/linux64/steamclient.so ~/.steam/sdk64/
 }

-initCrontab() {
-  crontab=$(mktemp)
-
-  if [ -n "$UPDATE_CRON" ]; then
-    debug "creating cron for update checks (schedule: $UPDATE_CRON)"
-    echo "$UPDATE_CRON supervisorctl start enshrouded-updater >/dev/null 2>&1" >>"$crontab"
-  fi
-
-  crontab "$crontab"
-  rm -f "$crontab"
-}
-
-updateOrCreateEnshroudedServerConfig() {
-  if [[ ! -e ${install_path}/enshrouded_server.json ]]; then
-    mkdir -p ${install_path}
-    touch ${install_path}/enshrouded_server.json
-
-    # write json to file ${install_path}/enshrouded_server.json
-
-    cat >${install_path}/enshrouded_server.json << EOF
-{
-  "name": "Enshrouded Server",
-  "password": "",
-  "saveDirectory": "./savegame",
-  "logDirectory": "./logs",
-  "ip": "0.0.0.0",
-  "gamePort": 15636,
-  "queryPort": 15637,
-  "slotCount": 16
-}
-EOF
-  fi
-
-  if [[ -n "$SERVER_NAME" ]]; then
-    echo "$(jq --arg name "$SERVER_NAME" '.name = $name' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_PASSWORD" ]]; then
-    echo "$(jq --arg password "$SERVER_PASSWORD" '.password = $password' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_SAVE_DIR" ]]; then
-    echo "$(jq --arg saveDirectory "$SERVER_SAVE_DIR" '.saveDirectory = $saveDirectory' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_LOG_DIR" ]]; then
-    echo "$(jq --arg logDirectory "$SERVER_LOG_DIR" '.logDirectory = $logDirectory' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_IP" ]]; then
-    echo "$(jq --arg ip "$SERVER_IP" '.ip = $ip' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_PORT" ]]; then
-    echo "$(jq --argjson gamePort "$SERVER_PORT" '.gamePort = $gamePort' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_QUERYPORT" ]]; then
-    echo "$(jq --argjson queryPort "$SERVER_QUERYPORT" '.queryPort = $queryPort' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_SLOT_COUNT" ]]; then
-    echo "$(jq --argjson slotCount "$SERVER_SLOT_COUNT" '.slotCount = $slotCount' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-}
-
 main
--- a/scripts/wine/enshrouded-bootstrap
+++ b/scripts/wine/enshrouded-bootstrap
@@ -1,6 +1,7 @@
 #!/bin/bash
 . "$(dirname "$0")/common"
 . "$(dirname "$0")/defaults"
+. "$(dirname "$0")/enshrouded-bootstrap-shared"

 main() {
  info "Running enshrouded-bootstrap"
@@ -12,10 +13,6 @@ main() {
  info "Bootstrap complete"
 }

-prepareEnshroudedAppFolders() {
-  mkdir -p "$install_path"
-}
-
 bootstrapWine() {
  info "bootstrap wine"
  export WINEPREFIX=$WINEPREFIX
@@ -33,70 +30,4 @@ bootstrapWine() {
  info "wine bootstrap finished"
 }

-initCrontab() {
-  crontab=$(mktemp)
-
-  if [ -n "$UPDATE_CRON" ]; then
-    debug "creating cron for update checks (schedule: $UPDATE_CRON)"
-    echo "$UPDATE_CRON supervisorctl start enshrouded-updater >/dev/null 2>&1" >>"$crontab"
-  fi
-
-  crontab "$crontab"
-  rm -f "$crontab"
-}
-
-updateOrCreateEnshroudedServerConfig() {
-  if [[ ! -e ${install_path}/enshrouded_server.json ]]; then
-    mkdir -p ${install_path}
-    touch ${install_path}/enshrouded_server.json
-
-    # write json to file ${install_path}/enshrouded_server.json
-
-    cat >${install_path}/enshrouded_server.json << EOF
-{
-  "name": "Enshrouded Server",
-  "password": "",
-  "saveDirectory": "./savegame",
-  "logDirectory": "./logs",
-  "ip": "0.0.0.0",
-  "gamePort": 15636,
-  "queryPort": 15637,
-  "slotCount": 16
-}
-EOF
-  fi
-
-  if [[ -n "$SERVER_NAME" ]]; then
-    echo "$(jq --arg name "$SERVER_NAME" '.name = $name' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_PASSWORD" ]]; then
-    echo "$(jq --arg password "$SERVER_PASSWORD" '.password = $password' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_SAVE_DIR" ]]; then
-    echo "$(jq --arg saveDirectory "$SERVER_SAVE_DIR" '.saveDirectory = $saveDirectory' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_LOG_DIR" ]]; then
-    echo "$(jq --arg logDirectory "$SERVER_LOG_DIR" '.logDirectory = $logDirectory' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_IP" ]]; then
-    echo "$(jq --arg ip "$SERVER_IP" '.ip = $ip' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_PORT" ]]; then
-    echo "$(jq --argjson gamePort "$SERVER_PORT" '.gamePort = $gamePort' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_QUERYPORT" ]]; then
-    echo "$(jq --argjson queryPort "$SERVER_QUERYPORT" '.queryPort = $queryPort' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-
-  if [[ -n "$SERVER_SLOT_COUNT" ]]; then
-    echo "$(jq --argjson slotCount "$SERVER_SLOT_COUNT" '.slotCount = $slotCount' ${install_path}/enshrouded_server.json)" > ${install_path}/enshrouded_server.json
-  fi
-}
-
 main
--- a/supervisord.conf
+++ b/supervisord.conf
@@ -74,6 +74,18 @@ autostart=false
 autorestart=false
 startsecs=0

+[program:enshrouded-backup]
+user=enshrouded
+environment=HOME="/home/enshrouded",USER="enshrouded",LANG="en_US.UTF-8",PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+command=/usr/local/etc/enshrouded/enshrouded-backup
+stdout_syslog=true
+stderr_syslog=true
+stdout_logfile_maxbytes=1MB
+stderr_logfile_maxbytes=1MB
+autostart=false
+autorestart=false
+startsecs=0
+
 [program:enshrouded-force-update]
 user=enshrouded
 environment=HOME="/home/enshrouded",USER="enshrouded",LANG="en_US.UTF-8",PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"