-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdocker_ZFS_AIO_backup.sh
489 lines (447 loc) · 31.8 KB
/
docker_ZFS_AIO_backup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
#!/bin/bash
################################################################################
# UNRAID DOCKER CONTAINER ZFS AIO BACKUP v1.000 #
# https://github.com/Blasman/Unraid_Docker_ZFS_AIO_Backup #
################################################################################
# Written/intended for Plex, but will work with any single docker container. Ideal for containers with large/complex appdata folders.
# The goal is to do a full/custom backup of docker containers appdata and to minimize docker container downtime.
# Simplified order of operations and functions of this script:
# 1. Stop docker.
# 2. Snapshot docker dataset.
# 3. Start docker.
# 4. Optional: Replicate docker dataset and snapshots to another pool/dataset using syncoid.
# The following options may use the snapshot from Step 2 as a source by creating a temporary clone of it:
# 5. Optional: Create timestamped rsynced folder any folders/files.
# 6. Optional: Create timestamped tarfile of any files/folders.
################################################################################
# USER CONFIG BELOW #
################################################################################
DOCKER_NAME="plex" # Name of docker container.
SOURCE_DATASET="pool_main/appdata/plex" # Docker container's appdata dataset.
REPLICATE_DATASET=false # Replicate most recent snapshot to another dataset using syncoid. If 'true', then EDIT THE 'REPLICATION SETTINGS' BELOW!
BACKUP_FILES=false # Create a timestamped rsync folder of the DB files and Preferences.xml (or any files/folders). If 'true', then EDIT THE 'BACKUP FILES SETTINGS' BELOW!
TAR_FILES=false # Create a timestamped tarfile of docker container's 'Media' and 'Metadata' folders (or any files/folders). If 'true', then EDIT THE 'TARFILE SETTINGS' BELOW!
MANAGE_SANOID_CONFIG=true # Have this script automatically create and update your sanoid .conf files.
SANOID_DEFAULT_CONFIG_DIR="/etc/sanoid" # You shouldn't need to change this.
SANOID_DOCKER_CONFIG_DIR="/etc/sanoid/$DOCKER_NAME" # You shouldn't need to change this. Path will be created if it does not exist.
# Set sanoid's snapshot retention policy below. "How many snapshots of X timeframe will be kept before deleting old snapshots of said timeframe?"
SNAPSHOT_HOURS="0"
SNAPSHOT_DAYS="7"
SNAPSHOT_WEEKS="4"
SNAPSHOT_MONTHS="3"
SNAPSHOT_YEARS="0"
ALLOW_SNAPSHOTS_OUTSIDE_OF_RETENTION_POLICY=false # sanoid will not take new snapshots if ran before its next retention policy interval. Set to 'true' to allow additional snapshots to be taken.
# DELETE_EXTRA_SNAPSHOTS_OLDER_THAN_X_DAYS="7" # Uncomment line to delete any '_extra' snapshots (taken when ALLOW_EXTRA_SNAPSHOTS_OUTSIDE_OF_RETENTION_POLICY=true) that are older than this many days.
# --------------------------- REPLICATION SETTINGS --------------------------- #
REPLICATED_DATASET="pool_ssds/backup_appdata/plex" # Define the name of the dataset that you want to replicate to. DATASET(S) WILL BE CREATED IF THEY DO NOT EXIST!
SYNCOID_ARGS="-r --delete-target-snapshots --force-delete --no-sync-snap --quiet" # OPTIONALLY (and carefully) customize the syncoid command line arguments. See: https://github.com/jimsalterjrs/sanoid/wiki/Syncoid#options
# --------------------- BACKUP FILES & TARFILE SETTINGS ---------------------- #
TEMP_DATASET_TO_CLONE_TO="pool_main/temp" # Dataset to temporarily clone the most recent snapshot of the docker container's appdata dataset. A dataset is created within this dataset during backup, and is then destoyed after backup.
# -------------------------- BACKUP FILES SETTINGS --------------------------- #
BACKUP_DIR="/mnt/pool_ssds/backup_plex_db" # Backup directory to rsync DB files and Preferences.xml to (or any other files/folders as specified in BACKUP_INCLUDES).
BACKUP_DATASET_PATH="pool_ssds/backup_plex_db" # If backing up files to a dataset, uncomment line and specify the dataset name to back up files to (needed for MOUNT_BACKUP_DATASET option). DATASET(S) WILL BE CREATED IF THEY DO NOT EXIST!
HOURS_TO_KEEP_BACKUPS_FOR="95" # Delete backups older than this many hours. [Hours=Days|72=3|96=4|120=5|144=6|168=7|336=14|720=30] Tip: subtract one hour if desired to ensure oldest backup is always deleted.
# Specify full paths to any files/folders (do *NOT* use '/mnt/user', use pool dir instead ie '/mnt/pool_name') and where to copy them to. Each line essentially becomes a 'rsync -a' command.
BACKUP_INCLUDES=( # <APPDATA_DIR> will be replaced with the folder path to your pool's docker container's appdata folder. <GENERATED_BACKUP_DIR> will be replaced with the scripts generated timestamped backup directory.
# "source" "destination"
"<APPDATA_DIR>/Library/Application Support/Plex Media Server/Preferences.xml" "<GENERATED_BACKUP_DIR>/Preferences.xml"
"<APPDATA_DIR>/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db" "<GENERATED_BACKUP_DIR>/com.plexapp.plugins.library.db"
"<APPDATA_DIR>/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.blobs.db" "<GENERATED_BACKUP_DIR>/com.plexapp.plugins.library.blobs.db"
)
# ---------------------- OPTIONAL BACKUP FILES SETTINGS ---------------------- #
MOUNT_BACKUP_DATASET=false # If using a database that needs to be mounted before backup and unmounted after backup, set to 'true' to use.
BACKUP_PERMISSIONS="755" # OPTIONALLY change to any 3 or 4 digit value to have chmod set those permissions on the timestamped backup sub-directory (but not the rsynced folders/files).
BACKUP_SUBDIR_TEXT="${DOCKER_NAME} Backup" # OPTIONALLY customize the text for the backup sub-directory name. As a precaution, the script only deletes old backups that match this pattern.
BACKUP_TIMESTAMP() { date +"%Y_%m_%d@%H.%M.%S"; } # OPTIONALLY customize TIMESTAMP for backup sub-directory name.
BACKUP_SUBDIR_COMPLETE_NAME() { echo "[$(BACKUP_TIMESTAMP)] $BACKUP_SUBDIR_TEXT"; } # OPTIONALLY customize the complete backup sub-directory name with the TIMESTAMP and BACKUP_SUBDIR_TEXT.
# ----------------------------- TARFILE SETTINGS ----------------------------- #
TARFILE_DIR="/mnt/pool_ssds/backup_plex_tarfiles" # Backup directory to store the created tarfiles.
TARFILE_DATASET_PATH="pool_ssds/backup_plex_tarfiles" # If saving tarfiles to a dataset, uncomment line and specify the dataset name (needed for MOUNT_TARFILE_DATASET option). DATASET(S) WILL BE CREATED IF THEY DO NOT EXIST!
HOURS_TO_KEEP_TARFILES_FOR="335" # Delete backups older than this many hours. [Hours=Days|72=3|96=4|120=5|144=6|168=7|336=14|720=30] Tip: subtract one hour if desired to ensure oldest backup is always deleted.
# Specify full paths to any files/folders (do *NOT* use '/mnt/user', use pool dir instead ie '/mnt/pool_name') to add to the tarfile (nothing is included by default).
TARFILE_INCLUDES=( # <APPDATA_DIR> will be replaced with the folder path to your pool's docker container's appdata folder.
"<APPDATA_DIR>/Library/Application Support/Plex Media Server/Media"
"<APPDATA_DIR>/Library/Application Support/Plex Media Server/Metadata"
)
# ------------------------ OPTIONAL TARFILE SETTINGS ------------------------- #
TARFILE_COMPRESSION_LEVEL="1" # Specify tarfile compression. "0" == none | "1" == GZIP | "2" or higher == ZSTD (number is the amount of CPU cores to use).
TARFILE_PERMISSIONS="640" # Set to any 3 or 4 digit value to have chmod set those permissions on the final tar file.
MOUNT_TARFILE_DATASET=false # If using a database that needs to be mounted before backup and unmounted after backup, set to 'true' to use.
TARFILE_TEXT="${DOCKER_NAME} Backup" # OPTIONALLY customize the text for the backup tar file. As a precaution, the script only deletes old backups that match this pattern.
TARFILE_TIMESTAMP() { date +"%Y_%m_%d@%H.%M.%S"; } # OPTIONALLY customize TIMESTAMP for the tar filename.
TARFILE_COMPLETE_FILENAME() { echo "[$(TARFILE_TIMESTAMP)] $TARFILE_TEXT.tar"; } # OPTIONALLY customize the complete tar file name with the TIMESTAMP and TARFILE_TEXT. (compression extensions are added automatically!)
# -------------------------- OPTIONAL MISC SETTINGS -------------------------- #
# Specify patterns to match to be added as '--exclude' arguments for the tar and rsync commands.
EXCLUDES=(
# "logs"
# "*.log"
)
STOP_DOCKER=true # Shutdown docker container before snapshot and always restart it after snapshot. Set to 'true' to use.
UNRAID_WEBGUI_START_MSG=false # Send backup start message to the Unraid Web GUI. Set to 'true' to use.
UNRAID_WEBGUI_SUCCESS_MSG=true # Send backup success message to the Unraid Web GUI. Set to 'true' to use.
UNRAID_WEBGUI_FAILURE_MSG=true # Send backup failure message to the Unraid Web GUI. Set to 'true' to use.
USE_LOCK_FILE=false # Set to 'true' to enable use of lock file to prevent overlapping backups. 'rm /tmp/zfs_backup_lock_file.tmp' to delete lock file if required.
# REPLICATION_SCHEDULE="1 2 3 4 5 6 7" # OPTIONALLY restrict the days of the week that replications can take place. 1 = Monday.
# BACKUP_SCHEDULE="1 2 3 4 5 6 7" # OPTIONALLY restrict the days of the week that files can be backed up. 1 = Monday.
# TARFILE_SCHEDULE="1 2 3 4 5 6 7" # OPTIONALLY restrict the days of the week that tarfiles can be created. 1 = Monday.
################################################################################
# END OF USER CONFIG #
################################################################################
echo_ts() { printf "[%(%Y_%m_%d)T %(%H:%M:%S)T.${EPOCHREALTIME: -6:3}] $@\\n"; }
unraid_notify() { /usr/local/emhttp/webGui/scripts/notify -s "$DOCKER_NAME ZFS Backup Script" -i $1 -d "$2"; }
run_timer() {
local run_time=$((${2/./} - ${1/./})) # Output Examples
if [[ $run_time -lt 1000000 ]]; then printf -v run_time "%06d" $run_time; echo ".${run_time: -6:4}s" # == .1234s
elif [[ $run_time -lt 10000000 ]]; then echo "${run_time:0:1}.${run_time: -6:3}s" # == 1.234s
elif [[ $run_time -lt 60000000 ]]; then echo "${run_time:0:2}.${run_time: -6:3}s" # == 12.345s
elif [[ $run_time -lt 3600000000 ]]; then echo "$((run_time % 3600000000 / 60000000))m $((run_time % 60000000 / 1000000))s" # == 1m 23s
elif [[ $run_time -lt 86400000000 ]]; then echo "$((run_time / 3600000000))h $((run_time % 3600000000 / 60000000))m" # == 1h 23m
else echo "$((run_time / 86400000000))d $((run_time / 3600000000 % 24))h $((run_time % 3600000000 / 60000000))m"; fi # == 1d 23h 45m
}
create_or_update_sanoid_config() {
if [[ ! -d "$SANOID_DOCKER_CONFIG_DIR" ]]; then mkdir -p "$SANOID_DOCKER_CONFIG_DIR"; fi
if [[ ! -f "$SANOID_DOCKER_CONFIG_DIR/sanoid.defaults.conf" ]]; then cp "$SANOID_DEFAULT_CONFIG_DIR/sanoid.defaults.conf" "$SANOID_DOCKER_CONFIG_DIR/sanoid.defaults.conf"; fi
sanoid_scripts_config_file="$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
if [[ -f "$sanoid_scripts_config_file" ]]; then
update_setting() {
local key=$1 new_value=$2 current_value
current_value=$(grep "^$key = " "$sanoid_scripts_config_file" | awk -F ' = ' '{print $2}')
if [[ "$current_value" != "$new_value" ]]; then
sed -i "s/^$key = .*/$key = $new_value/" "$sanoid_scripts_config_file"
echo_ts "[CONFIG CHANGE] Updated '$key' to '$new_value' in '$sanoid_scripts_config_file'."
fi
}
update_setting "hourly" "$SNAPSHOT_HOURS"
update_setting "daily" "$SNAPSHOT_DAYS"
update_setting "weekly" "$SNAPSHOT_WEEKS"
update_setting "monthly" "$SNAPSHOT_MONTHS"
update_setting "yearly" "$SNAPSHOT_YEARS"
else
echo "[$SOURCE_DATASET]" > "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "use_template = production" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "recursive = yes" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "[template_production]" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "hourly = $SNAPSHOT_HOURS" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "daily = $SNAPSHOT_DAYS" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "weekly = $SNAPSHOT_WEEKS" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "monthly = $SNAPSHOT_MONTHS" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "yearly = $SNAPSHOT_YEARS" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "autosnap = yes" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
echo "autoprune = yes" >> "$SANOID_DOCKER_CONFIG_DIR/sanoid.conf"
fi
}
exit_with_error() {
graceful_exit_with_error=true
echo_ts "[❌] $1"
if [[ $UNRAID_WEBGUI_FAILURE_MSG == true ]]; then unraid_notify alert "$1"; fi
exit 1
}
change_backup_types_to_false_if_ran_outside_of_schedule() {
local current_day_of_the_week=$(date +%u)
if [[ -n "$REPLICATION_SCHEDULE" && "$REPLICATION_SCHEDULE" != *"$current_day_of_the_week"* ]]; then REPLICATE_DATASET=false; fi
if [[ -n "$BACKUP_SCHEDULE" && "$BACKUP_SCHEDULE" != *"$current_day_of_the_week"* ]]; then BACKUP_FILES=false; fi
if [[ -n "$TARFILE_SCHEDULE" && "$TARFILE_SCHEDULE" != *"$current_day_of_the_week"* ]]; then TAR_FILES=false; fi
}
clean_up() {
if [ $? -ne 0 ] && [[ $UNRAID_WEBGUI_FAILURE_MSG == true ]] && [[ $graceful_exit_with_error != true ]]; then unraid_notify alert "Exited with an unexpected error."; fi
if [[ -n "$cloned_mount_point" ]] && zfs list -o name -H "$cloned_mount_point" &>/dev/null; then destroy_clone; fi
if [[ $MOUNT_BACKUP_DATASET == true ]] && [[ $(zfs get -H -o value mounted "$BACKUP_DATASET_PATH") == "yes" ]]; then unmount_dataset "$BACKUP_DATASET_PATH" yes; fi
if [[ $MOUNT_TARFILE_DATASET == true ]] && [[ $(zfs get -H -o value mounted "$TARFILE_DATASET_PATH") == "yes" ]]; then unmount_dataset "$TARFILE_DATASET_PATH" yes; fi
if [[ $USE_LOCK_FILE == true ]] && [[ -f "$lockfile" ]]; then rm -f "$lockfile"; fi
if [[ $STOP_DOCKER == true ]] && [[ $(get_docker_state) == "stopped" ]]; then docker start "$DOCKER_NAME" >/dev/null; fi
trap - EXIT
}
get_docker_state() {
local response=$(docker inspect -f '{{.State.Status}}' "$DOCKER_NAME" 2>/dev/null)
if [[ "$response" =~ ^(started|running|restarting)$ ]]; then echo "running";
elif [[ "$response" =~ ^(stopped|created|exited|paused)$ ]]; then echo "stopped";
else echo "unknown"; fi
}
pre_checks_and_start_script() {
script_start_time=$EPOCHREALTIME
if [[ $USE_LOCK_FILE == true ]]; then
lockfile="/tmp/zfs_backup_lock_file.tmp"
if [[ -f "$lockfile" ]]; then
exit_with_error "$DOCKER_NAME backup is currently active! Lock file can be removed by typing 'rm $lockfile'. Exiting."; fi
fi
if [[ ! -x "$(which zfs)" ]]; then
exit_with_error "ZFS not found on this system ('which zfs'). This script is meant for Unraid 6.12 or above (which includes ZFS support). Please ensure you are using the correct Unraid version."; fi
if [[ ! -x /usr/local/sbin/sanoid ]]; then
exit_with_error "sanoid not found or executable at '/usr/local/sbin/sanoid'. Please make sure that it is installed from the Unraid Community Apps."; fi
clean_directory_variables() { for var in "$@"; do if [ -n "${!var}" ]; then eval "$var=\"/$(echo ${!var} | sed 's|^/||; s|/$||')\""; fi; done }
clean_directory_variables SANOID_DEFAULT_CONFIG_DIR SANOID_DOCKER_CONFIG_DIR BACKUP_DIR TARFILE_DIR
if [[ ! -d "$SANOID_DEFAULT_CONFIG_DIR" ]]; then
exit_with_error "sanoid default config file directory 'SANOID_DEFAULT_CONFIG_DIR' not found at '$SANOID_DEFAULT_CONFIG_DIR'."; fi
if [[ ! -f "$SANOID_DEFAULT_CONFIG_DIR/sanoid.defaults.conf" ]] || [[ ! -f "$SANOID_DEFAULT_CONFIG_DIR/sanoid.conf" ]]; then
exit_with_error "sanoid config files not found at '$SANOID_DEFAULT_CONFIG_DIR'. You need 'sanoid.defaults.conf' and 'sanoid.conf' in this directory."; fi
if ! zfs list -o name -H "$SOURCE_DATASET" &>/dev/null; then
exit_with_error "The source dataset '$SOURCE_DATASET' does not exist."; fi
if [[ $(zfs get -H -o value used "$SOURCE_DATASET") == 0B ]]; then
exit_with_error "The source dataset '$SOURCE_DATASET' is empty. Nothing to do."; fi
if [[ $STOP_DOCKER == true ]]; then
docker_state="$(get_docker_state)"
if [[ "$docker_state" == "unknown" ]]; then
exit_with_error "Could not find '$DOCKER_NAME' docker. Exiting."; fi
fi
if [[ -n "$REPLICATION_SCHEDULE" || -n "$BACKUP_SCHEDULE" || -n "$TARFILE_SCHEDULE" ]]; then change_backup_types_to_false_if_ran_outside_of_schedule; fi
if [[ $REPLICATE_DATASET == true ]] && [[ ! -x /usr/local/sbin/syncoid ]]; then
echo_ts "[❌] Syncoid not found or executable at '/usr/local/sbin/syncoid'. Please install syncoid (part of sanoid) plugin. Skipping replication job."
REPLICATE_DATASET=false
fi
docker_folder_path=$(zfs get -H -o value mountpoint "$SOURCE_DATASET")
if [[ -z "$docker_folder_path" ]]; then exit_with_error "Could not get mountpoint for '$SOURCE_DATASET'."; fi
if [[ $MANAGE_SANOID_CONFIG == true ]]; then create_or_update_sanoid_config; fi
trap clean_up EXIT
if [[ $USE_LOCK_FILE == true ]]; then touch "$lockfile"; fi
echo_ts "[${DOCKER_NAME^^} BACKUP STARTED]"
if [[ $UNRAID_WEBGUI_START_MSG == true ]]; then unraid_notify normal "$DOCKER_NAME ZFS Backup Script Started."; fi
}
destroy_clone() {
zfs destroy "$cloned_mount_point"
if zfs list -o name -H "$cloned_mount_point" &>/dev/null; then echo_ts "[❌] Could not destroy '"$cloned_mount_point"'."; return 1; fi
echo_ts "Destroyed clone '"$cloned_mount_point"'."
}
stop_docker() {
if [[ "$docker_state" == "running" ]]; then
echo_ts "Stopping $DOCKER_NAME docker..."
stop_docker_start_time=$EPOCHREALTIME
docker stop "$DOCKER_NAME" >/dev/null
local stop_docker_finish_time=$EPOCHREALTIME
echo_ts "$DOCKER_NAME docker stopped in $(run_timer $stop_docker_start_time $stop_docker_finish_time)."
else
echo_ts "$DOCKER_NAME docker already stopped. Skipping docker stop."
fi
}
start_docker() {
docker_state="$(get_docker_state)"
if [[ "$docker_state" == "stopped" ]]; then
echo_ts "Starting $DOCKER_NAME docker..."
local start_docker_start_time=$EPOCHREALTIME
docker start "$DOCKER_NAME" >/dev/null
local start_docker_finish_time=$EPOCHREALTIME
echo_ts "$DOCKER_NAME docker started in $(run_timer $start_docker_start_time $start_docker_finish_time). ⏱️ $(run_timer $stop_docker_start_time $start_docker_finish_time) of total $DOCKER_NAME downtime since start of 'docker stop' command."
else
echo_ts "$DOCKER_NAME docker already started. Skipping docker start."
fi
}
get_age_in_seconds() { echo $(($(date +%s) - $(stat -c %Y "$1"))); }
delete_old_files() {
local dir=$1 hours_to_keep=$2 text_pattern=$3 type=$4
local cutoff_age=$(($hours_to_keep * 3600))
for match in "$dir"/*"$text_pattern"*; do
if [[ ( "$type" == "folder" && -d "$match" ) || ( "$type" == "file" && -f "$match" ) ]]; then
if [ "$(get_age_in_seconds "$match")" -gt "$cutoff_age" ]; then
if [ "$type" == "folder" ]; then rm -r "$match";
else rm -f "$match"; fi
echo_ts "Deleted old $type '$(basename "$match")'."
fi
fi
done
}
snapshot_dataset() {
echo_ts "Creating ZFS snapshot of '$SOURCE_DATASET' using sanoid..."
local snapshot_start_time=$EPOCHREALTIME
local sanoid_output=$(/usr/local/sbin/sanoid --configdir="$SANOID_DOCKER_CONFIG_DIR" --take-snapshots -v); local sanoid_exit=$?
local snapshot_finish_time=$EPOCHREALTIME
if [ $sanoid_exit -ne 0 ]; then exit_with_error "Automatic snapshot creation using sanoid failed for source '$SOURCE_DATASET'."; fi
most_recent_autosnap_name=$(zfs list -t snapshot -o name -S creation -r "$SOURCE_DATASET" | awk '/autosnap_/ {print; exit}')
most_recent_autosnap_age=$(( $(date +%s) - $(zfs get -Hp creation "$most_recent_autosnap_name" | awk '{print $3}') ))
if [[ $(echo "$sanoid_output" | tail -n 1) == *"INFO: taking snapshots..."* ]] || [[ "$most_recent_autosnap_age" -gt 15 ]]; then
if [[ $ALLOW_SNAPSHOTS_OUTSIDE_OF_RETENTION_POLICY == true ]]; then
echo_ts "[⚠️] Last 'autosnap' found is '$most_recent_autosnap_name' taken $most_recent_autosnap_age seconds ago. Taking snapshot with 'zfs snapshot' instead."
zfs snapshot "$SOURCE_DATASET@autosnap_$(date +"%Y-%m-%d_%H:%M:%S")_extra" &>/dev/null
if [ $? -ne 0 ]; then exit_with_error "Failed to create snapshot for source: '$SOURCE_DATASET'.";
else echo_ts "[✔️] '$(zfs list -t snapshot -o name -S creation -r "$SOURCE_DATASET" | awk '/autosnap_/ {print; exit}')' created in $(run_timer $snapshot_start_time $snapshot_finish_time)."; fi
else
exit_with_error "Last 'autosnap' found is '$most_recent_autosnap_name' taken $most_recent_autosnap_age seconds ago. Enable 'ALLOW_SNAPSHOTS_OUTSIDE_OF_RETENTION_POLICY' in script config to allow extra snapshots to be taken."
fi
else
echo_ts "[✔️] '$most_recent_autosnap_name' created in $(run_timer $snapshot_start_time $snapshot_finish_time)."
fi
}
create_all_required_datasets_from_path() {
local dataset="$1" type="$2"
if ! zfs list -o name -H "$dataset" &>/dev/null; then
IFS='/' read -r -a components <<< "$dataset"
local path="${components[0]}"
for ((i=1; i<${#components[@]}; i++)); do
path+="/${components[i]}"
if ! zfs list -o name -H "$path" &>/dev/null; then
echo_ts "Creating dataset '$path'..."
zfs create "$path"
if [ $? -ne 0 ]; then echo_ts "[❌] Failed to create dataset '$path'. Skipping $type."; return 1; fi
echo_ts "[✔️] Successfully created dataset '$path'."
fi
done
fi
}
pre_checks_for_various_backup_types() {
local operation_type=$1 dataset_path=$2 mount_dataset_flag=$3 dir=$4 output_path_var_name=$5 custom_name_var_name=$6
check_for_proper_dataset_paths() {
local -n array="$1"
for x in "${array[@]}"; do
if [[ "$x" == *"/mnt/user/"* ]]; then
echo_ts "[❌] Do not use '/mnt/user/' folder paths for '$1' variable. Use the pool devices folder paths instead."
return 1
fi
done
}
check_for_proper_dataset_paths "${operation_type^^}_INCLUDES"
if [ $? -ne 0 ]; then return 1; fi
if [[ -n "$dataset_path" ]]; then
create_all_required_datasets_from_path "$dataset_path" "$operation_type"
if [ $? -ne 0 ]; then return 1; fi
fi
if [[ $mount_dataset_flag == true ]]; then
mount_dataset "$dataset_path"
if [ $? -ne 0 ]; then
echo_ts "[❌] Failed to mount snapshot '$dataset_path'. Skipping $operation_type."
return 1
fi
fi
if [[ ! -d "$dir" ]]; then
echo_ts "[❌] '$operation_type directory' not found at '$dir'. Skipping $operation_type."
if [[ $mount_dataset_flag == true ]]; then unmount_dataset "$dataset_path"; fi
return 1
fi
eval "$output_path_var_name=\"$dir/$($custom_name_var_name)\""
}
get_file_or_folder_size() {
if [[ -f "$1" ]]; then local size=$(stat -c%s "$1")
elif [[ -d "$1" ]]; then local size=$(du -sb "$1" | cut -f1)
else return 1; fi
if [ "$size" -lt 1024 ]; then echo "${size}B"
elif [ "$size" -lt $((1024 * 1024)) ]; then echo "$((size / 1024))KB"
elif [ "$size" -lt $((1024 * 1024 * 1024)) ]; then echo "$((size / (1024 * 1024)))MB"
else echo "$((size / (1024 * 1024 * 1024)))GB"; fi
}
backup_files() {
pre_checks_for_various_backup_types "backup" "$BACKUP_DATASET_PATH" "$MOUNT_BACKUP_DATASET" "$BACKUP_DIR" "backup_path" "BACKUP_SUBDIR_COMPLETE_NAME"
if [ $? -ne 0 ]; then return 1; fi
local rsync_jobs backup_start_time backup_finish_time
for path in "${BACKUP_INCLUDES[@]}"; do rsync_jobs+=("$(echo "$path" | sed -E "s|$docker_folder_path|$cloned_folder_path|g; s|<APPDATA_DIR>|$cloned_folder_path|g; s|<GENERATED_BACKUP_DIR>|$backup_path|g")"); done
echo_ts "Copying files to '$backup_path'..."
backup_start_time=$EPOCHREALTIME
mkdir -p "$backup_path"
for ((i=0; i<${#rsync_jobs[@]}; i+=2)); do rsync -a ${exclude_arg[@]:+$exclude_arg[@]} "${rsync_jobs[$i]}" "${rsync_jobs[$i+1]}"; done
if [ $? -ne 0 ]; then echo_ts "[❌] Could not back up files to '$backup_path'."; return 1; fi
backup_finish_time=$EPOCHREALTIME
if [ -z "$(shopt -s nullglob dotglob; echo $backup_path/*)" ]; then echo_ts "[❌] Rsync didn't add any files to '$backup_path'."; return 1; fi
backup_path_filesize=$(get_file_or_folder_size "$backup_path")
echo_ts "[✔️] Copied $backup_path_filesize of data in $(run_timer $backup_start_time $backup_finish_time)."
chown nobody:users "$backup_path"
if [[ $BACKUP_PERMISSIONS =~ ^[0-9]{3,4}$ ]]; then chmod $BACKUP_PERMISSIONS "$backup_path"; fi
if [[ $HOURS_TO_KEEP_BACKUPS_FOR =~ ^[0-9]+$ ]]; then delete_old_files "$BACKUP_DIR" "$HOURS_TO_KEEP_BACKUPS_FOR" "$BACKUP_SUBDIR_TEXT" "folder"; fi
}
tar_files() {
pre_checks_for_various_backup_types "tarfile" "$TARFILE_DATASET_PATH" "$MOUNT_TARFILE_DATASET" "$TARFILE_DIR" "tarfile_complete_path" "TARFILE_COMPLETE_FILENAME"
if [ $? -ne 0 ]; then return 1; fi
local tar_paths=() compression ext tarfile_start_time tar_error_output tarfile_finish_time
for path in "${TARFILE_INCLUDES[@]}"; do tar_paths+=("$(echo "$path" | sed -E "s|$docker_folder_path|$cloned_folder_path|g; s|<APPDATA_DIR>|$cloned_folder_path|g")"); done
if [[ $TARFILE_COMPRESSION_LEVEL -eq 1 ]]; then compression="-z" ext=".gz"
elif [[ $TARFILE_COMPRESSION_LEVEL -ge 2 ]]; then compression="-I zstd -T$TARFILE_COMPRESSION_LEVEL" ext=".zst"; fi
tarfile_complete_path=$tarfile_complete_path$ext
echo_ts "Creating file '$tarfile_complete_path'..."
tarfile_start_time=$EPOCHREALTIME
tar_error_output=$(tar ${exclude_arg[@]:+$exclude_arg[@]} -cf "$tarfile_complete_path" ${compression:+"$compression"} --transform "s|^${cloned_folder_path#/}|$docker_folder_path|" "${tar_paths[@]}" 2>&1 >/dev/null)
if [ $? -ne 0 ]; then echo_ts "[❌] Could not create '$tarfile_complete_path'. Tar error: $tar_error_output"; return 1
elif [[ ! -f "$tarfile_complete_path" ]]; then echo_ts "[❌] File '$tarfile_complete_path' was not created by tar."; return 1; fi
tarfile_finish_time=$EPOCHREALTIME
tarfile_file_size=$(get_file_or_folder_size "$tarfile_complete_path")
echo_ts "[✔️] Created $tarfile_file_size size tarfile in $(run_timer $tarfile_start_time $tarfile_finish_time)."
chown nobody:users "$tarfile_complete_path"
if [[ $TARFILE_PERMISSIONS =~ ^[0-9]{3,4}$ ]]; then chmod $TARFILE_PERMISSIONS "$tarfile_complete_path"; fi
if [[ $HOURS_TO_KEEP_TARFILES_FOR =~ ^[0-9]+$ ]]; then delete_old_files "$TARFILE_DIR" "$HOURS_TO_KEEP_TARFILES_FOR" "$TARFILE_TEXT" "file"; fi
}
mount_dataset() {
local mount_status=$(zfs get -H -o value mounted "$1")
if [[ "$mount_status" == "no" ]]; then
zfs mount "$1" &>/dev/null
if [[ $(zfs get -H -o value mounted "$1") == "yes" ]]; then echo_ts "Mounted '$1'."
else return 1; fi
elif [[ "$mount_status" == "yes" ]]; then echo_ts "[⚠️] '$1' was already mounted."
else echo_ts "[❌] $mount_status."; return 1; fi
}
unmount_dataset() {
local mount_status
if [[ "$2" == "yes" ]]; then mount_status="yes"
else mount_status=$(zfs get -H -o value mounted "$1"); fi
if [[ "$mount_status" == "yes" ]]; then
zfs unmount "$1" &>/dev/null
if [[ $(zfs get -H -o value mounted "$1") == "no" ]]; then echo_ts "Unmounted '$1'."
else echo_ts "[⚠️] Could not unmount '$1'."; fi
elif [[ "$mount_status" == "no" ]]; then echo_ts "[⚠️] '$1' was not mounted."; fi
}
replicate_dataset() {
create_all_required_datasets_from_path "$REPLICATED_DATASET" "replication"
if [ $? -ne 0 ]; then return; fi
echo_ts "Starting ZFS replication using syncoid..."
replicate_dataset_start_time=$EPOCHREALTIME
/usr/local/sbin/syncoid $SYNCOID_ARGS "$SOURCE_DATASET" "$REPLICATED_DATASET" >/dev/null; local syncoid_exit=$?
replicate_dataset_finish_time=$EPOCHREALTIME
if [ $syncoid_exit -eq 0 ]; then echo_ts "[✔️] '$SOURCE_DATASET' >> '$REPLICATED_DATASET'. Successful Replication in $(run_timer $replicate_dataset_start_time $replicate_dataset_finish_time).";
else echo_ts "[❌] Replication failed from '$SOURCE_DATASET' to '$REPLICATED_DATASET'."; fi
}
delete_old_extra_snapshots() {
zfs list -t snapshot -o name,creation -S creation -r "$SOURCE_DATASET" | awk -v cutoff_date="$(date -d "$DELETE_EXTRA_SNAPSHOTS_OLDER_THAN_X_DAYS days ago" +%s)" '
/autosnap_.*_extra/ {
split($0, fields, " ")
creation_date = fields[length(fields)-4] " " fields[length(fields)-3] " " fields[length(fields)-2] " " fields[length(fields)-1] " " fields[length(fields)]
snapshot_name = substr($0, 1, length($0) - length(creation_date) - 1)
cmd = "date -d \"" creation_date "\" +%s"
cmd | getline snapshot_date
close(cmd)
if (snapshot_date < cutoff_date) { print snapshot_name }
}' | while read -r snapshot; do
zfs destroy "$snapshot" &>/dev/null
echo_ts "Deleted old snapshot '$snapshot'."
done
}
delete_old_snapshots() {
/usr/local/sbin/sanoid --configdir="$SANOID_DOCKER_CONFIG_DIR" --prune-snapshots
if [[ $DELETE_EXTRA_SNAPSHOTS_OLDER_THAN_X_DAYS =~ ^[0-9]+$ ]]; then delete_old_extra_snapshots; fi
}
complete_backup() {
clean_up
run_time=$(run_timer $script_start_time $EPOCHREALTIME)
echo_ts "[${DOCKER_NAME^^} BACKUP FINISHED] Run Time: $run_time."
if [[ $UNRAID_WEBGUI_SUCCESS_MSG == true ]]; then unraid_notify normal "Finished in $run_time."; fi
}
clone_recent_snapshot() {
if ! zfs list -o name -H "$TEMP_DATASET_TO_CLONE_TO" &>/dev/null; then echo_ts "[❌] The source dataset '$TEMP_DATASET_TO_CLONE_TO' does not exist.."; return 1; fi
cloned_mount_point=$TEMP_DATASET_TO_CLONE_TO/_temp_${SOURCE_DATASET##*/}
zfs clone "$most_recent_autosnap_name" "$cloned_mount_point"
if ! zfs list -o name -H "$cloned_mount_point" &>/dev/null; then echo_ts "[❌] Failed to clone '$most_recent_autosnap_name' to '$cloned_mount_point'."; return 1; fi
echo_ts "Created clone '$cloned_mount_point' from '$most_recent_autosnap_name'."
zfs set readonly=on "$cloned_mount_point"
cloned_folder_path=$(zfs get -H -o value mountpoint "$cloned_mount_point")
}
perform_additional_backups() {
clone_recent_snapshot
if [ $? -ne 0 ]; then return; fi
for pattern in "${EXCLUDES[@]}"; do exclude_arg+=("--exclude=${pattern}"); done
if [[ $BACKUP_FILES == true ]]; then backup_files; fi
if [[ $TAR_FILES == true ]]; then tar_files; fi
}
################################################################################
# BEGIN PROCESSING #
################################################################################
# Verify correctly set user variables and other error handling.
pre_checks_and_start_script
# Stop Docker.
if [[ $STOP_DOCKER == true ]]; then stop_docker; fi
# Create ZFS snapshot of docker container's appdata.
snapshot_dataset
# Start Docker.
if [[ $STOP_DOCKER == true ]]; then start_docker; fi
# Delete old snapshots.
delete_old_snapshots
# Replicate ZFS snapshot of docker container's appdata.
if [[ $REPLICATE_DATASET == true ]]; then replicate_dataset; fi
# Perform additional rsync and/or tarfile backups using a clone of most recent snapshot.
if [[ $BACKUP_FILES == true || $TAR_FILES == true ]]; then perform_additional_backups; fi
# Clean up checks and print backup complete message with run time for script.
complete_backup
# Exit with success.
exit 0