gabriel/ProxmoxBackupsServer-to-SMB
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

edit

Browse source
This commit is contained in:
Kaby_Kun 2025-05-23 22:37:55 +02:00
parent 65e21e5b30
commit 0692bf0a34
1 changed files with 276 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

328
backups2smb.sh
View file

@ -12,6 +12,52 @@ CHUNK_SIZE="10G"
MIN_SPACE_REQUIRED="5G" # Minimum space required in /tmp MIN_SPACE_REQUIRED="5G" # Minimum space required in /tmp
LOG="/var/log/pbs-smb-backup.log" LOG="/var/log/pbs-smb-backup.log"
SRC="/etc /root /mypool" SRC="/etc /root /mypool"
# Exclusion patterns
EXCLUDE_PATTERNS=(
"node_modules"
".cursor-server"
".git"
".vscode"
"*.log"
"*.tmp"
"*.temp"
"*.swp"
"*.swo"
"*.bak"
"*.cache"
"*.pid"
"*.lock"
"*.socket"
"*.sock"
"*.pid"
"*.log.*"
"*.gz"
"*.zip"
"*.tar"
"*.7z"
"*.rar"
"*.iso"
"*.img"
"*.vmdk"
"*.vhd"
"*.qcow2"
"*.raw"
"*.vdi"
"*.vbox"
"*.ova"
"*.ovf"
"*.vmx"
"*.vmsd"
"*.vmsn"
"*.vmss"
"*.vmtm"
"*.vmxf"
"*.nvram"
"*.vmem"
"*.vswp"
)
MAX_BACKUPS=3 # Keep exactly 3 backups MAX_BACKUPS=3 # Keep exactly 3 backups
# === Cleanup function === # === Cleanup function ===
@ -78,15 +124,96 @@ wait_for_space() {
done done
} }
cleanup_processes() {
local mount_point="$1"
local max_attempts=3
local attempt=1
echo "[$(date)] Cleaning up processes using $mount_point..." | tee -a "$LOG"
while [ $attempt -le $max_attempts ]; do
# Get list of processes using the mount point, excluding our own script
local pids=$(lsof -t "$mount_point" 2>/dev/null | grep -v "$$")
if [ -z "$pids" ]; then
echo "[$(date)] No processes found using $mount_point" | tee -a "$LOG"
return 0
fi
echo "[$(date)] Attempt $attempt: Found processes: $pids" | tee -a "$LOG"
# First try SIGTERM
kill -15 $pids 2>/dev/null
sleep 5
# Check if processes are still running
pids=$(lsof -t "$mount_point" 2>/dev/null | grep -v "$$")
if [ -n "$pids" ]; then
echo "[$(date)] Processes still running, sending SIGKILL..." | tee -a "$LOG"
kill -9 $pids 2>/dev/null
sleep 5
fi
# Final check
if [ -z "$(lsof -t "$mount_point" 2>/dev/null | grep -v "$$")" ]; then
echo "[$(date)] Successfully cleaned up all processes" | tee -a "$LOG"
return 0
fi
((attempt++))
done
echo "[$(date)] ⚠️ Warning: Could not clean up all processes after $max_attempts attempts" | tee -a "$LOG"
return 1
}
mount_share() { mount_share() {
echo "[$(date)] Mounting SMB share..." | tee -a "$LOG" echo "[$(date)] Mounting SMB share..." | tee -a "$LOG"
# Check if already mounted # Check if already mounted
if mountpoint -q "$MOUNT_POINT"; then if mountpoint -q "$MOUNT_POINT"; then
echo "[$(date)] Share already mounted, unmounting first..." | tee -a "$LOG" echo "[$(date)] Share already mounted, attempting to unmount..." | tee -a "$LOG"
umount "$MOUNT_POINT"
# Clean up any processes first
cleanup_processes "$MOUNT_POINT"
# Try to unmount gracefully first
umount "$MOUNT_POINT" 2>/dev/null
sleep 2
# If still mounted, try force unmount
if mountpoint -q "$MOUNT_POINT"; then
echo "[$(date)] Force unmounting..." | tee -a "$LOG"
umount -f "$MOUNT_POINT" 2>/dev/null
sleep 2
# If still mounted, try lazy unmount
if mountpoint -q "$MOUNT_POINT"; then
echo "[$(date)] Lazy unmounting..." | tee -a "$LOG"
umount -l "$MOUNT_POINT" 2>/dev/null
sleep 2
# If still mounted, try one more time with process cleanup
if mountpoint -q "$MOUNT_POINT"; then
cleanup_processes "$MOUNT_POINT"
sleep 2
umount -f "$MOUNT_POINT" 2>/dev/null
sleep 2
fi
fi
fi
# Final check - if still mounted, exit
if mountpoint -q "$MOUNT_POINT"; then
echo "[$(date)] ❌ Failed to unmount existing share. Please check for processes using $MOUNT_POINT" | tee -a "$LOG"
echo "[$(date)] Running processes:" | tee -a "$LOG"
lsof "$MOUNT_POINT" | grep -v "$$" | tee -a "$LOG"
exit 1
fi
fi fi
# Ensure mount point is clean
rm -rf "$MOUNT_POINT"/*
# Try mounting with different SMB versions if needed # Try mounting with different SMB versions if needed
for smb_ver in "3.0" "2.1" "2.0" "1.0"; do for smb_ver in "3.0" "2.1" "2.0" "1.0"; do
echo "[$(date)] Attempting mount with SMB version $smb_ver..." | tee -a "$LOG" echo "[$(date)] Attempting mount with SMB version $smb_ver..." | tee -a "$LOG"
@ -98,6 +225,7 @@ mount_share() {
# Show dmesg output for debugging # Show dmesg output for debugging
echo "[$(date)] Mount failed, checking dmesg for details..." | tee -a "$LOG" echo "[$(date)] Mount failed, checking dmesg for details..." | tee -a "$LOG"
dmesg | tail -n 20 | tee -a "$LOG" dmesg | tail -n 20 | tee -a "$LOG"
sleep 2
done done
echo "[$(date)] ❌ Failed to mount SMB share. Please check:" | tee -a "$LOG" echo "[$(date)] ❌ Failed to mount SMB share. Please check:" | tee -a "$LOG"
@ -111,10 +239,62 @@ mount_share() {
unmount_share() { unmount_share() {
echo "[$(date)] Unmounting SMB share..." | tee -a "$LOG" echo "[$(date)] Unmounting SMB share..." | tee -a "$LOG"
if mountpoint -q "$MOUNT_POINT"; then if mountpoint -q "$MOUNT_POINT"; then
umount "$MOUNT_POINT" # Clean up any processes first
cleanup_processes "$MOUNT_POINT"
# Try to unmount gracefully first
umount "$MOUNT_POINT" 2>/dev/null
sleep 2
# If still mounted, try force unmount
if mountpoint -q "$MOUNT_POINT"; then
echo "[$(date)] Force unmounting..." | tee -a "$LOG"
umount -f "$MOUNT_POINT" 2>/dev/null
sleep 2
# If still mounted, try lazy unmount
if mountpoint -q "$MOUNT_POINT"; then
echo "[$(date)] Lazy unmounting..." | tee -a "$LOG"
umount -l "$MOUNT_POINT" 2>/dev/null
sleep 2
# If still mounted, try one more time with process cleanup
if mountpoint -q "$MOUNT_POINT"; then
cleanup_processes "$MOUNT_POINT"
sleep 2
umount -f "$MOUNT_POINT" 2>/dev/null
sleep 2
fi
fi
fi
# Final check - if still mounted, warn but continue
if mountpoint -q "$MOUNT_POINT"; then
echo "[$(date)] ⚠️ Warning: Could not unmount $MOUNT_POINT completely" | tee -a "$LOG"
echo "[$(date)] Running processes:" | tee -a "$LOG"
lsof "$MOUNT_POINT" | grep -v "$$" | tee -a "$LOG"
fi
fi fi
} }
# Add trap to ensure cleanup on script exit
trap 'handle_exit' EXIT
handle_exit() {
local exit_code=$?
echo "[$(date)] Cleaning up on exit..." | tee -a "$LOG"
# Only attempt cleanup if we're not already in a cleanup process
if [ -z "$CLEANUP_IN_PROGRESS" ]; then
export CLEANUP_IN_PROGRESS=1
cleanup_processes "$MOUNT_POINT"
unmount_share
rm -rf "$TMP_BACKUP"
fi
exit $exit_code
}
# === Handle -u option for restore === # === Handle -u option for restore ===
if [ "$1" == "-u" ]; then if [ "$1" == "-u" ]; then
echo "[$(date)] 🔁 Restore mode selected. Starting cleanup..." | tee -a "$LOG" echo "[$(date)] 🔁 Restore mode selected. Starting cleanup..." | tee -a "$LOG"
@ -130,14 +310,14 @@ if [ "$1" == "-u" ]; then
exit 1 exit 1
fi fi
echo "[$(date)] ⬇️ Starting streaming restore process from $LATEST_BACKUP_DIR..." | tee -a "$LOG" echo "[$(date)] ⬇️ Starting restore process from $LATEST_BACKUP_DIR..." | tee -a "$LOG"
# Create a temporary directory for processing # Create a temporary directory for processing
TMP_RESTORE_DIR="/tmp/restore-$(basename "$LATEST_BACKUP_DIR")" TMP_RESTORE_DIR="/tmp/restore-$(basename "$LATEST_BACKUP_DIR")"
mkdir -p "$TMP_RESTORE_DIR" mkdir -p "$TMP_RESTORE_DIR"
# Process chunks one by one # Process chunks in order
for chunk in "$LATEST_BACKUP_DIR"/chunk_*; do for chunk in $(ls -v "$LATEST_BACKUP_DIR"/chunk_* 2>/dev/null); do
if [ ! -f "$chunk" ]; then if [ ! -f "$chunk" ]; then
continue continue
fi fi
@ -147,22 +327,14 @@ if [ "$1" == "-u" ]; then
# Copy chunk to temp # Copy chunk to temp
rsync -ah --progress "$chunk" "$TMP_RESTORE_DIR/" rsync -ah --progress "$chunk" "$TMP_RESTORE_DIR/"
# If this is the first chunk, start extraction # Extract the chunk
if [ "$(basename "$chunk")" = "chunk_aa" ]; then echo "[$(date)] Extracting chunk..." | tee -a "$LOG"
7z x -y "$TMP_RESTORE_DIR/$(basename "$chunk")" -o/ & 7z x -y "$TMP_RESTORE_DIR/$(basename "$chunk")" -o/ | tee -a "$LOG"
extract_pid=$!
else
# For subsequent chunks, append to the extraction
cat "$TMP_RESTORE_DIR/$(basename "$chunk")" >> "$TMP_RESTORE_DIR/combined.7z"
fi
# Clean up the processed chunk # Remove the processed chunk
rm -f "$TMP_RESTORE_DIR/$(basename "$chunk")" rm -f "$TMP_RESTORE_DIR/$(basename "$chunk")"
done done
# Wait for extraction to complete
wait $extract_pid
# Cleanup # Cleanup
rm -rf "$TMP_RESTORE_DIR" rm -rf "$TMP_RESTORE_DIR"
unmount_share unmount_share
@ -193,50 +365,102 @@ mkdir -p "$BACKUP_DIR"
# Step 5: Create backup in chunks with space management # Step 5: Create backup in chunks with space management
echo "[$(date)] 🗜️ Starting backup with space management..." | tee -a "$LOG" echo "[$(date)] 🗜️ Starting backup with space management..." | tee -a "$LOG"
# Create a temporary file list # Build find command exclusions
echo "[$(date)] 📋 Creating file list..." | tee -a "$LOG" build_find_exclusions() {
find $SRC -type f > "$TMP_BACKUP/filelist.txt" local exclusions=""
for pattern in "${EXCLUDE_PATTERNS[@]}"; do
exclusions+=" -not -path '*/$pattern/*' -not -name '$pattern'"
done
echo "$exclusions"
}
# Process files in chunks # Create a temporary file list with exclusions
echo "[$(date)] 📋 Creating file list (excluding unnecessary files)..." | tee -a "$LOG"
eval "find $SRC -type f $(build_find_exclusions) > \"$TMP_BACKUP/filelist.txt\""
# Initialize variables for chunk creation
chunk_num=1
current_chunk_size=0
current_chunk_files=()
# Process files and create chunks
while IFS= read -r file; do while IFS= read -r file; do
# Check space before processing each file # Skip if file doesn't exist
if ! check_space; then [ -f "$file" ] || continue
echo "[$(date)] ⏳ Waiting for more space..." | tee -a "$LOG"
wait_for_space
fi
# Get the relative path for the file # Get file size
rel_path="${file#/}" file_size=$(stat -c %s "$file")
chunk_dir="$TMP_BACKUP/$(dirname "$rel_path")"
mkdir -p "$chunk_dir"
# Compress the file # If adding this file would exceed chunk size, create the chunk
echo "[$(date)] 📦 Compressing: $file" | tee -a "$LOG" if [ $((current_chunk_size + file_size)) -gt $(numfmt --from=iec $CHUNK_SIZE) ]; then
7z a -t7z -m0=lzma2 -mx=5 "$TMP_BACKUP/$(echo "$rel_path" | tr / _).7z" "$file" if [ ${#current_chunk_files[@]} -gt 0 ]; then
echo "[$(date)] 📦 Creating chunk $chunk_num with ${#current_chunk_files[@]} files..." | tee -a "$LOG"
# If we have enough chunks or space is low, transfer them
if [ $(ls "$TMP_BACKUP"/*.7z 2>/dev/null | wc -l) -ge 5 ] || ! check_space; then # Create the chunk and wait for it to complete
echo "[$(date)] ⬆️ Transferring chunks to SMB..." | tee -a "$LOG" chunk_file="$TMP_BACKUP/chunk_$(printf "%03d" $chunk_num).7z"
for chunk in "$TMP_BACKUP"/*.7z; do 7z a -y -spf -t7z -m0=lzma2 -mx=5 "$chunk_file" "${current_chunk_files[@]}" | tee -a "$LOG"
if [ -f "$chunk" ]; then
rsync -ah --progress "$chunk" "$BACKUP_DIR/" # Wait for 7z to complete and verify the chunk exists
rm -f "$chunk" if [ -f "$chunk_file" ]; then
echo "[$(date)] ⬆️ Transferring chunk $chunk_num to SMB..." | tee -a "$LOG"
rsync -ah --progress "$chunk_file" "$BACKUP_DIR/"
# Verify the transfer was successful
if [ $? -eq 0 ]; then
echo "[$(date)] ✅ Chunk $chunk_num transferred successfully" | tee -a "$LOG"
rm -f "$chunk_file"
else
echo "[$(date)] ❌ Failed to transfer chunk $chunk_num" | tee -a "$LOG"
exit 1
fi
else
echo "[$(date)] ❌ Failed to create chunk $chunk_num" | tee -a "$LOG"
exit 1
fi fi
done
# Reset for next chunk
current_chunk_files=()
current_chunk_size=0
((chunk_num++))
# Check space before continuing
if ! check_space; then
wait_for_space
fi
fi
fi fi
# Add file to current chunk
current_chunk_files+=("$file")
current_chunk_size=$((current_chunk_size + file_size))
done < "$TMP_BACKUP/filelist.txt" done < "$TMP_BACKUP/filelist.txt"
# Transfer any remaining chunks # Create final chunk if there are remaining files
echo "[$(date)] ⬆️ Transferring remaining chunks..." | tee -a "$LOG" if [ ${#current_chunk_files[@]} -gt 0 ]; then
for chunk in "$TMP_BACKUP"/*.7z; do echo "[$(date)] 📦 Creating final chunk $chunk_num with ${#current_chunk_files[@]} files..." | tee -a "$LOG"
if [ -f "$chunk" ]; then chunk_file="$TMP_BACKUP/chunk_$(printf "%03d" $chunk_num).7z"
rsync -ah --progress "$chunk" "$BACKUP_DIR/" 7z a -y -spf -t7z -m0=lzma2 -mx=5 "$chunk_file" "${current_chunk_files[@]}" | tee -a "$LOG"
rm -f "$chunk"
# Wait for 7z to complete and verify the chunk exists
if [ -f "$chunk_file" ]; then
echo "[$(date)] ⬆️ Transferring final chunk to SMB..." | tee -a "$LOG"
rsync -ah --progress "$chunk_file" "$BACKUP_DIR/"
# Verify the transfer was successful
if [ $? -eq 0 ]; then
echo "[$(date)] ✅ Final chunk transferred successfully" | tee -a "$LOG"
rm -f "$chunk_file"
else
echo "[$(date)] ❌ Failed to transfer final chunk" | tee -a "$LOG"
exit 1
fi
else
echo "[$(date)] ❌ Failed to create final chunk" | tee -a "$LOG"
exit 1
fi fi
done fi
# Step 6: Final cleanup and finish # Step 6: Final cleanup and finish
rm -rf "$TMP_BACKUP" rm -rf "$TMP_BACKUP"
unmount_share unmount_share
echo "[$(date)] ✅ Backup completed successfully: $BACKUP_NAME" | tee -a "$LOG" echo "[$(date)] ✅ Backup completed successfully: $BACKUP_NAME" | tee -a "$LOG"