diff --git a/backups2smb.sh b/backups2smb.sh index 66c3a32..9db66d8 100755 --- a/backups2smb.sh +++ b/backups2smb.sh @@ -12,6 +12,52 @@ CHUNK_SIZE="10G" MIN_SPACE_REQUIRED="5G" # Minimum space required in /tmp LOG="/var/log/pbs-smb-backup.log" SRC="/etc /root /mypool" + +# Exclusion patterns +EXCLUDE_PATTERNS=( + "node_modules" + ".cursor-server" + ".git" + ".vscode" + "*.log" + "*.tmp" + "*.temp" + "*.swp" + "*.swo" + "*.bak" + "*.cache" + "*.pid" + "*.lock" + "*.socket" + "*.sock" + "*.pid" + "*.log.*" + "*.gz" + "*.zip" + "*.tar" + "*.7z" + "*.rar" + "*.iso" + "*.img" + "*.vmdk" + "*.vhd" + "*.qcow2" + "*.raw" + "*.vdi" + "*.vbox" + "*.ova" + "*.ovf" + "*.vmx" + "*.vmsd" + "*.vmsn" + "*.vmss" + "*.vmtm" + "*.vmxf" + "*.nvram" + "*.vmem" + "*.vswp" +) + MAX_BACKUPS=3 # Keep exactly 3 backups # === Cleanup function === @@ -78,15 +124,96 @@ wait_for_space() { done } +cleanup_processes() { + local mount_point="$1" + local max_attempts=3 + local attempt=1 + + echo "[$(date)] Cleaning up processes using $mount_point..." | tee -a "$LOG" + + while [ $attempt -le $max_attempts ]; do + # Get list of processes using the mount point, excluding our own script + local pids=$(lsof -t "$mount_point" 2>/dev/null | grep -v "$$") + if [ -z "$pids" ]; then + echo "[$(date)] No processes found using $mount_point" | tee -a "$LOG" + return 0 + fi + + echo "[$(date)] Attempt $attempt: Found processes: $pids" | tee -a "$LOG" + + # First try SIGTERM + kill -15 $pids 2>/dev/null + sleep 5 + + # Check if processes are still running + pids=$(lsof -t "$mount_point" 2>/dev/null | grep -v "$$") + if [ -n "$pids" ]; then + echo "[$(date)] Processes still running, sending SIGKILL..." | tee -a "$LOG" + kill -9 $pids 2>/dev/null + sleep 5 + fi + + # Final check + if [ -z "$(lsof -t "$mount_point" 2>/dev/null | grep -v "$$")" ]; then + echo "[$(date)] Successfully cleaned up all processes" | tee -a "$LOG" + return 0 + fi + + ((attempt++)) + done + + echo "[$(date)] ⚠️ Warning: Could not clean up all processes after $max_attempts attempts" | tee -a "$LOG" + return 1 +} + mount_share() { echo "[$(date)] Mounting SMB share..." | tee -a "$LOG" # Check if already mounted if mountpoint -q "$MOUNT_POINT"; then - echo "[$(date)] Share already mounted, unmounting first..." | tee -a "$LOG" - umount "$MOUNT_POINT" + echo "[$(date)] Share already mounted, attempting to unmount..." | tee -a "$LOG" + + # Clean up any processes first + cleanup_processes "$MOUNT_POINT" + + # Try to unmount gracefully first + umount "$MOUNT_POINT" 2>/dev/null + sleep 2 + + # If still mounted, try force unmount + if mountpoint -q "$MOUNT_POINT"; then + echo "[$(date)] Force unmounting..." | tee -a "$LOG" + umount -f "$MOUNT_POINT" 2>/dev/null + sleep 2 + + # If still mounted, try lazy unmount + if mountpoint -q "$MOUNT_POINT"; then + echo "[$(date)] Lazy unmounting..." | tee -a "$LOG" + umount -l "$MOUNT_POINT" 2>/dev/null + sleep 2 + + # If still mounted, try one more time with process cleanup + if mountpoint -q "$MOUNT_POINT"; then + cleanup_processes "$MOUNT_POINT" + sleep 2 + umount -f "$MOUNT_POINT" 2>/dev/null + sleep 2 + fi + fi + fi + + # Final check - if still mounted, exit + if mountpoint -q "$MOUNT_POINT"; then + echo "[$(date)] ❌ Failed to unmount existing share. Please check for processes using $MOUNT_POINT" | tee -a "$LOG" + echo "[$(date)] Running processes:" | tee -a "$LOG" + lsof "$MOUNT_POINT" | grep -v "$$" | tee -a "$LOG" + exit 1 + fi fi + # Ensure mount point is clean + rm -rf "$MOUNT_POINT"/* + # Try mounting with different SMB versions if needed for smb_ver in "3.0" "2.1" "2.0" "1.0"; do echo "[$(date)] Attempting mount with SMB version $smb_ver..." | tee -a "$LOG" @@ -98,6 +225,7 @@ mount_share() { # Show dmesg output for debugging echo "[$(date)] Mount failed, checking dmesg for details..." | tee -a "$LOG" dmesg | tail -n 20 | tee -a "$LOG" + sleep 2 done echo "[$(date)] ❌ Failed to mount SMB share. Please check:" | tee -a "$LOG" @@ -111,10 +239,62 @@ mount_share() { unmount_share() { echo "[$(date)] Unmounting SMB share..." | tee -a "$LOG" if mountpoint -q "$MOUNT_POINT"; then - umount "$MOUNT_POINT" + # Clean up any processes first + cleanup_processes "$MOUNT_POINT" + + # Try to unmount gracefully first + umount "$MOUNT_POINT" 2>/dev/null + sleep 2 + + # If still mounted, try force unmount + if mountpoint -q "$MOUNT_POINT"; then + echo "[$(date)] Force unmounting..." | tee -a "$LOG" + umount -f "$MOUNT_POINT" 2>/dev/null + sleep 2 + + # If still mounted, try lazy unmount + if mountpoint -q "$MOUNT_POINT"; then + echo "[$(date)] Lazy unmounting..." | tee -a "$LOG" + umount -l "$MOUNT_POINT" 2>/dev/null + sleep 2 + + # If still mounted, try one more time with process cleanup + if mountpoint -q "$MOUNT_POINT"; then + cleanup_processes "$MOUNT_POINT" + sleep 2 + umount -f "$MOUNT_POINT" 2>/dev/null + sleep 2 + fi + fi + fi + + # Final check - if still mounted, warn but continue + if mountpoint -q "$MOUNT_POINT"; then + echo "[$(date)] ⚠️ Warning: Could not unmount $MOUNT_POINT completely" | tee -a "$LOG" + echo "[$(date)] Running processes:" | tee -a "$LOG" + lsof "$MOUNT_POINT" | grep -v "$$" | tee -a "$LOG" + fi fi } +# Add trap to ensure cleanup on script exit +trap 'handle_exit' EXIT + +handle_exit() { + local exit_code=$? + echo "[$(date)] Cleaning up on exit..." | tee -a "$LOG" + + # Only attempt cleanup if we're not already in a cleanup process + if [ -z "$CLEANUP_IN_PROGRESS" ]; then + export CLEANUP_IN_PROGRESS=1 + cleanup_processes "$MOUNT_POINT" + unmount_share + rm -rf "$TMP_BACKUP" + fi + + exit $exit_code +} + # === Handle -u option for restore === if [ "$1" == "-u" ]; then echo "[$(date)] 🔁 Restore mode selected. Starting cleanup..." | tee -a "$LOG" @@ -130,14 +310,14 @@ if [ "$1" == "-u" ]; then exit 1 fi - echo "[$(date)] ⬇️ Starting streaming restore process from $LATEST_BACKUP_DIR..." | tee -a "$LOG" + echo "[$(date)] ⬇️ Starting restore process from $LATEST_BACKUP_DIR..." | tee -a "$LOG" # Create a temporary directory for processing TMP_RESTORE_DIR="/tmp/restore-$(basename "$LATEST_BACKUP_DIR")" mkdir -p "$TMP_RESTORE_DIR" - # Process chunks one by one - for chunk in "$LATEST_BACKUP_DIR"/chunk_*; do + # Process chunks in order + for chunk in $(ls -v "$LATEST_BACKUP_DIR"/chunk_* 2>/dev/null); do if [ ! -f "$chunk" ]; then continue fi @@ -147,22 +327,14 @@ if [ "$1" == "-u" ]; then # Copy chunk to temp rsync -ah --progress "$chunk" "$TMP_RESTORE_DIR/" - # If this is the first chunk, start extraction - if [ "$(basename "$chunk")" = "chunk_aa" ]; then - 7z x -y "$TMP_RESTORE_DIR/$(basename "$chunk")" -o/ & - extract_pid=$! - else - # For subsequent chunks, append to the extraction - cat "$TMP_RESTORE_DIR/$(basename "$chunk")" >> "$TMP_RESTORE_DIR/combined.7z" - fi + # Extract the chunk + echo "[$(date)] Extracting chunk..." | tee -a "$LOG" + 7z x -y "$TMP_RESTORE_DIR/$(basename "$chunk")" -o/ | tee -a "$LOG" - # Clean up the processed chunk + # Remove the processed chunk rm -f "$TMP_RESTORE_DIR/$(basename "$chunk")" done - # Wait for extraction to complete - wait $extract_pid - # Cleanup rm -rf "$TMP_RESTORE_DIR" unmount_share @@ -193,50 +365,102 @@ mkdir -p "$BACKUP_DIR" # Step 5: Create backup in chunks with space management echo "[$(date)] 🗜️ Starting backup with space management..." | tee -a "$LOG" -# Create a temporary file list -echo "[$(date)] 📋 Creating file list..." | tee -a "$LOG" -find $SRC -type f > "$TMP_BACKUP/filelist.txt" +# Build find command exclusions +build_find_exclusions() { + local exclusions="" + for pattern in "${EXCLUDE_PATTERNS[@]}"; do + exclusions+=" -not -path '*/$pattern/*' -not -name '$pattern'" + done + echo "$exclusions" +} -# Process files in chunks +# Create a temporary file list with exclusions +echo "[$(date)] 📋 Creating file list (excluding unnecessary files)..." | tee -a "$LOG" +eval "find $SRC -type f $(build_find_exclusions) > \"$TMP_BACKUP/filelist.txt\"" + +# Initialize variables for chunk creation +chunk_num=1 +current_chunk_size=0 +current_chunk_files=() + +# Process files and create chunks while IFS= read -r file; do - # Check space before processing each file - if ! check_space; then - echo "[$(date)] ⏳ Waiting for more space..." | tee -a "$LOG" - wait_for_space - fi + # Skip if file doesn't exist + [ -f "$file" ] || continue - # Get the relative path for the file - rel_path="${file#/}" - chunk_dir="$TMP_BACKUP/$(dirname "$rel_path")" - mkdir -p "$chunk_dir" + # Get file size + file_size=$(stat -c %s "$file") - # Compress the file - echo "[$(date)] 📦 Compressing: $file" | tee -a "$LOG" - 7z a -t7z -m0=lzma2 -mx=5 "$TMP_BACKUP/$(echo "$rel_path" | tr / _).7z" "$file" - - # If we have enough chunks or space is low, transfer them - if [ $(ls "$TMP_BACKUP"/*.7z 2>/dev/null | wc -l) -ge 5 ] || ! check_space; then - echo "[$(date)] ⬆️ Transferring chunks to SMB..." | tee -a "$LOG" - for chunk in "$TMP_BACKUP"/*.7z; do - if [ -f "$chunk" ]; then - rsync -ah --progress "$chunk" "$BACKUP_DIR/" - rm -f "$chunk" + # If adding this file would exceed chunk size, create the chunk + if [ $((current_chunk_size + file_size)) -gt $(numfmt --from=iec $CHUNK_SIZE) ]; then + if [ ${#current_chunk_files[@]} -gt 0 ]; then + echo "[$(date)] 📦 Creating chunk $chunk_num with ${#current_chunk_files[@]} files..." | tee -a "$LOG" + + # Create the chunk and wait for it to complete + chunk_file="$TMP_BACKUP/chunk_$(printf "%03d" $chunk_num).7z" + 7z a -y -spf -t7z -m0=lzma2 -mx=5 "$chunk_file" "${current_chunk_files[@]}" | tee -a "$LOG" + + # Wait for 7z to complete and verify the chunk exists + if [ -f "$chunk_file" ]; then + echo "[$(date)] ⬆️ Transferring chunk $chunk_num to SMB..." | tee -a "$LOG" + rsync -ah --progress "$chunk_file" "$BACKUP_DIR/" + + # Verify the transfer was successful + if [ $? -eq 0 ]; then + echo "[$(date)] ✅ Chunk $chunk_num transferred successfully" | tee -a "$LOG" + rm -f "$chunk_file" + else + echo "[$(date)] ❌ Failed to transfer chunk $chunk_num" | tee -a "$LOG" + exit 1 + fi + else + echo "[$(date)] ❌ Failed to create chunk $chunk_num" | tee -a "$LOG" + exit 1 fi - done + + # Reset for next chunk + current_chunk_files=() + current_chunk_size=0 + ((chunk_num++)) + + # Check space before continuing + if ! check_space; then + wait_for_space + fi + fi fi + + # Add file to current chunk + current_chunk_files+=("$file") + current_chunk_size=$((current_chunk_size + file_size)) done < "$TMP_BACKUP/filelist.txt" -# Transfer any remaining chunks -echo "[$(date)] ⬆️ Transferring remaining chunks..." | tee -a "$LOG" -for chunk in "$TMP_BACKUP"/*.7z; do - if [ -f "$chunk" ]; then - rsync -ah --progress "$chunk" "$BACKUP_DIR/" - rm -f "$chunk" +# Create final chunk if there are remaining files +if [ ${#current_chunk_files[@]} -gt 0 ]; then + echo "[$(date)] 📦 Creating final chunk $chunk_num with ${#current_chunk_files[@]} files..." | tee -a "$LOG" + chunk_file="$TMP_BACKUP/chunk_$(printf "%03d" $chunk_num).7z" + 7z a -y -spf -t7z -m0=lzma2 -mx=5 "$chunk_file" "${current_chunk_files[@]}" | tee -a "$LOG" + + # Wait for 7z to complete and verify the chunk exists + if [ -f "$chunk_file" ]; then + echo "[$(date)] ⬆️ Transferring final chunk to SMB..." | tee -a "$LOG" + rsync -ah --progress "$chunk_file" "$BACKUP_DIR/" + + # Verify the transfer was successful + if [ $? -eq 0 ]; then + echo "[$(date)] ✅ Final chunk transferred successfully" | tee -a "$LOG" + rm -f "$chunk_file" + else + echo "[$(date)] ❌ Failed to transfer final chunk" | tee -a "$LOG" + exit 1 + fi + else + echo "[$(date)] ❌ Failed to create final chunk" | tee -a "$LOG" + exit 1 fi -done +fi # Step 6: Final cleanup and finish rm -rf "$TMP_BACKUP" unmount_share -echo "[$(date)] ✅ Backup completed successfully: $BACKUP_NAME" | tee -a "$LOG" - +echo "[$(date)] ✅ Backup completed successfully: $BACKUP_NAME" | tee -a "$LOG" \ No newline at end of file