#!/usr/bin/env perl # # rbu - Rsync BackUp # Author: Vincent Stemen # # Rbu is the successor to bu and is being completely re-written in Perl. It # uses rsync to transfer the data and, unlike bu, can backup to remote # destinations (e.g. hostname:/backup/dir) as well as to local file systems. # It can use whatever remote shells that rsync supports as the transport # (generally ssh or rsh). # # Once it has all or most of bu's original features integrated in, the plan is # to rename it to bu with a new version and drop the original bu from the # package. In the mean time, they are capable of working along side one # another and sharing the same bu configuration file. # # Copyright (c) 2008, 2009 Vincent Stemen # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $Id$ # Written: Tue Oct 9 02:04:58 CDT 2007 # Updates: # Fri Apr 25 00:11:19 CDT 2008 # First public release # # Sat May 10 19:27:28 CDT 2008 # Now uses Perls native realpath() function from the Cwd module so that # it will work under Linux without a realpath utility. # # Wed May 28 18:41:51 CDT 2008 # Fixed bug. When more than one file was specified, it produced this # error: rsync: link_stat "..." failed: No such file or directory # It needed a newline appended to each file that gets sent to rsync's # stdin. ######################### # rbu configuration variables. These are set in the configuration file, # the environment, or on the command line. # # bu_rc_file # bu_rsh # bu_dest # bu_rsh # bu_rsync_options # bu_one_file_system # bu_exclude_file ######################## ### global variables ### ######################## $PREFIX = "/usr"; # Global variables are capitalized. # Variables with underscores (_) on each side are flags. local @File_list; local @Rsync; # rsync command ## Modes and Flags local $_Create_rcfile_ = 0; # Create a new rc file local $_Create_rcfile_only_ = 0; # Only create a new rc file local $_Dry_run_mode_ = 0; # Show what will be done but # Don't actually do it. local $_Help_mode_ = 0; # Indicates the help switch was used local $_One_file_system_mode_; # Do not cross device mount points local $_Sync_mode_; # Syncronize filesystems by deleting files # from the destination that do not exist in # the source file system. local $_Verbose_mode_ = 1; local $_Debug_mode_ = 0; # Flags indicating settings were set on the command line local $_CL_sync_mode_ = 0; local $_CL_exclude_file_ = 0; local $_CL_one_file_system_mode_ = 0; ######################### # These are the global configuration variables. They are set from the # command line, the environment, the RC file, or default settings - in that # order of precedence. local $Rc_file; # Configuration file local $Dest; # Backup destination directory local $Rsync_extra_opts; # Extra options to pass to rsync local $Rsh; # Remote shell (i.e. ssh or rsh) local $Log_dir; # Location of log files local $Etc_dir; # Configuration file directory local $Tmp_dir; # Location of bu temporary files local $Log; # Log file name local $Include_file; # File that lists file patterns to include in # the backup local $Exclude_file; # File that lists file patterns to exclude from # the backup local $Mail_addr; # The email address to mail the log to local $Log_access_perms; # File permissions for the log files # Default settings for dumps to removable media local $Device; # Removable media device local $Volume_size; # Volume size in MB local $_Differential_mode_; # Differential dump mode local $_Incremental_mode_; # Incremental mode local $_Backup_image_; # Flag - True if we are dumping an fs backup image local $Label; # Backup label string local $Speed; # CDRW write speed local $_Compress_; # Flag - Compress removable media bakups local $_Eject_; # Flag - Automatically eject the media local $_Blank_; # Flag - Blank each CDRW before writing local $_Prompt_; # Flag - Prompt for confirmation before # beginning a dump. ######################### sub usage() { my $sync_mode = $_Sync_mode_ ? "on" : "off"; my $verbose_mode = $_Verbose_mode_ ? "on" : "off"; my $one_file_system_mode = $_One_file_system_mode_ ? "on" : "off"; print < 0 to turn on debugging. Use higher levels of n for # more debug information. Currently there is just two levels, # 1 and 2. elsif (/^debug$/) { $_Debug_mode_ = $value; } else { die "\nUnrecognized setting: '$_'\n\n"; } } else { push(@File_list, $_); } } } # msg() # Usage: msg "line1", "line2", ... # Print a message to stdout. # sub msg { my $line1 = shift; print "rbu: $line1\n"; foreach (@_) { print " $_\n" } } # errmsg() # Usage: errmsg "line1", "line2", ... # Print a message to stderr. # sub errmsg { my $line1 = shift; print STDERR "rbu: $line1\n"; foreach (@_) { print " $_\n" } } # debug() # Usage: debug "line1", "line2", ... # Print a debug message # sub debug { my $line1 = shift; print "rbu-trace: $line1\n"; foreach (@_) { print " $_\n" } } # setflag() # Usage: $flag = setflag($value, ["variable_name"]) # Test $value for "yes", "no", "on", "off", "true", "false", 1 or 0. # Returns 1 if $value contains one of the true settings. Otherwise returns 0. # if "variable_name" is specified, it will print an error message if $value # contains an invalid value and exit(1). # sub setflag($;$) { local $value = shift; local $variable = shift; if (! $value) { return(0) } for ($value) { /^(yes | true | on | y | t | 1)$/xi && return(1); /^(no | false | off | n | f | 0)$/xi && return(0); if ($variable) { errmsg "\$$variable has an improper value of \"$value\".", 'It should be "yes", "no", "on", "off", "true", "false", 0 or 1.'; exit(1); } } return(0); } # get_environment() # Get environment settings. These override the configuration file. # sub get_environment() { $Env_rc_file = $ENV{bu_rc_file}; $Env_dest = $ENV{bu_dest}; $Env_backup_dir = $ENV{bu_backup_dir}; $Env_rsh = $ENV{bu_rsh}; $Env_rsync_options = $ENV{bu_rsync_options}; $Env_log = $ENV{bu_log}; $Env_incremental = $ENV{bu_incremental}; $Env_one_file_system = $ENV{bu_one_file_system}; $Env_include_file = $ENV{bu_include_file}; $Env_exclude_file = $ENV{bu_exclude_file}; $Env_mail_addr = $ENV{bu_mail_addr}; $Env_group_size = $ENV{bu_group_size}; $Env_delay = $ENV{bu_delay}; # Removable media dump settings $Env_device = $ENV{bu_device}; $Env_volume_size = $ENV{bu_volume_size}; $Env_differential = $ENV{bu_differential}; $Env_backup_image = $ENV{bu_backup_image}; $Env_label = $ENV{bu_label}; $Env_compress = $ENV{bu_compress}; $Env_speed = $ENV{bu_speed}; $Env_blank = $ENV{bu_blank}; $Env_eject = $ENV{bu_eject}; $Env_prompt = $ENV{bu_prompt}; } sub configure() { get_environment(); if (! $Rc_file) { $Rc_file = $Env_rc_file || $Default_rc_file } $Rc_file = glob($Rc_file); if (-f $Rc_file) { source($Rc_file) } elsif (! $_Help_mode_) { # If using the default rc file then create it if it does not exist if (($Rc_file eq $Default_rc_file) || $_Create_rcfile_) { $_Create_rcfile_ = 1; } else { errmsg "Configuration file, \"$Rc_file\" does not exist."; exit(1); } } # Settings that may be set only in the RC file $Log_dir = $bu_log_dir || $Default_log_dir; $Etc_dir = $bu_etc_dir || $Default_etc_dir; $Tmp_dir = $bu_tmp_dir || $Default_tmp_dir; $Log_access_perms = $bu_log_access_perms || $Default_log_access_perms; # Settings that can be set on the command line, in the environment, # or in the rc file. if (! $Dest) { $Dest = $Env_dest || $bu_dest || $Env_backup_dir || $bu_backup_dir || $Default_dest; } if (! $Backup_dir) { $Backup_dir = $Env_backup_dir || $bu_backup_dir || $Default_backup_dir } if (! $Rsh) { $Rsh = $Env_rsh || $bu_rsh || $Default_rsh } if (! $Rsync_extra_opts) { $Rsync_extra_opts = $Env_rsync_options || $bu_rsync_options } if (! $_CL_sync_mode_) { $_Sync_mode_ = $Env_bu_sync || $bu_sync || $Default_sync_mode; } if (! $_CL_one_file_system_mode_) { $_One_file_system_mode_ = $Env_one_file_system || $bu_one_file_system || $Default_one_file_system_mode; } $_Incremental_mode_ = $Env_incremental || $bu_incremental || $Default_incremental_mode; if (! $_CL_include_file) { $Include_file = $Env_include_file || $bu_include_file || $Default_include_file; } if ($Include_file) { $Include_file = glob($Include_file); } if (! $_CL_exclude_file_) { $Exclude_file = $Env_exclude_file || $bu_exclude_file || $Default_exclude_file; } if ($Exclude_file) { $Exclude_file = glob($Exclude_file); } # Set all the flags $_Sync_mode_ = setflag($_Sync_mode_, "bu_sync"); $_One_file_system_mode_ = setflag($_One_file_system_mode_, "bu_one_file_system"); $_Incremental_mode_ = setflag($_Incremental_mode_, "bu_incremental"); if ($_Help_mode_ || $_Create_rcfile_only_) { return } # Construct the rsync command @Rsync = qw(rsync -HOar --files-from=-); if (-s $Exclude_file) { push(@Rsync, "--exclude-from=$Exclude_file"); } if ($_Dry_run_mode_) { push(@Rsync, "--dry-run") } if ($_One_file_system_mode_) { push(@Rsync, "-x") } if ($_Verbose_mode_ || $_Dry_run_mode_) { push(@Rsync, "--log-format=/%n") } if ($_Sync_mode_) { push(@Rsync, qw(--delete-before --delete-excluded)) } # if $Dest is on a remote host if ($Dest =~ /^(.*):/) { if ($1 !~ /\//) { push(@Rsync, "--rsh=$Rsh"); } } if ($Rsync_extra_opts) { my @rsync_opts = split(' ', $Rsync_extra_opts); push(@Rsync, @rsync_opts); } push(@Rsync, "/", $Dest); } sub create_rcfile($) { my $file = shift; my $date = `date`; local *FILE; my $include_file; my $exclude_file; my $sync_mode = $_Sync_mode_ ? "on" : "off"; my $one_file_system_mode = $_One_file_system_mode_ ? "on" : "off"; my $incremental_mode = $_Incremental_mode_ ? "on" : "off"; my $differential_mode = $_Differential_mode_ ? "on" : "off"; if ($Include_file =~ /^$Etc_dir\//) { my $filename = $Include_file; $filename =~ s/^$Etc_dir\///; $include_file = "\$bu_etc_dir/$filename"; } else { $include_file = $Include_file; } if ($Exclude_file =~ /^$Etc_dir\//) { my $filename = $Exclude_file; $filename =~ s/^$Etc_dir\///; $exclude_file = "\$bu_etc_dir/$filename"; } else { $exclude_file = $Exclude_file; } chop($date); print <$file")) { errmsg "Error writing $file", $!; return(0); } print FILE <, ;, etc, you must put quotes around it. # # The term, "dump", throughout this file refers to a dump to removable # media, not to an NFS backup. ############################################################################# # rc_version # RC file format version rc_version=4 # bu_dest (env) [rbu] # Default destination where rbu backs up files. This can be a local directory # (which includes NFS mounts or symbolic links to the backup directory), or # a remote destination in the form of "hostname:/backup/dir". # If \$bu_dest is not set, rbu will use \$bu_backup_dir. This feature is for # compatibility with bu and will likely be removed eventually after rbu # completely replaces bu. bu_dest=$Dest # bu_backup_dir (env) [bu] # Default destination directory where bu backs up files. This should commonly # be either an NFS mount point or a symbolic link to an NFS or other file # system that houses your backups. This variable will probably become # deprecated once rbu completely replaces bu. bu_backup_dir=$Backup_dir # bu_sync (env) [rbu] # Syncronize the backup file system with the source file system. Deletes # backup files that no longer exist on the source file system. bu_sync=$sync_mode # bu_rsh (env) [rbu] # Remote shell to use as the transport for remote destinations. Usually either # ssh or rsh. ssh is recommended for security, especially if you are backing # up to a site outside your LAN, but you will want to use encryption keys so # that you do not have to type passwords all the time to do backups. See the # manual on ssh-keygen(1). bu_rsh=$Rsh # bu_rsync_options (env) [rbu] # Extra options to pass to rsync in addition to rbu's built in options. # This is an advanced feature. Leave it unset unless your know what you are # doing and are familiar with rsync. The primary built in options are "-HOar". # To see the entire rsync command line, run rbu with "--debug". bu_rsync_options="$Rsync_extra_opts" # bu_log_dir [bu] # Location of the log files. This directory will automatically be # created if it does not already exist. bu_log_dir=$Log_dir # bu_log (env) [bu] # Bu log file name. bu_log=\$bu_log_dir/log.\$(date +%y%m%d) # bu_tmp_dir # Bu creates sub-directories under here to store all of it's temp files # during operation. When doing a dump to CDRW, the ISO9660 image directory # is also created here, so there should be at least 650-675 MB of space # available. If running multiple instances of bu to dump to multiple CDRW's # at the same time, there should be enough space for multiple images. bu_tmp_dir=$Tmp_dir # bu_one_file_system - Flag (env) [bu] [rbu] # Stay in the local filesystem (Do not cross device mount points). This # applies to normal backups as well as dumps to removable media. This is # usually on for normal operation so that the root file system can be backed # up or dumped without unmounting filesystems that you do not want to be # included, such as NFS mount points. Must be off to gain control from the # command line with --one-file-system. bu_one_file_system=$one_file_system_mode # bu_etc_dir [bu] [rbu] # Location of the bu Include and Exclude configuration files bu_etc_dir=$Etc_dir # bu_include_file (env) [bu] # The name of the file containing the default list of files and/or # directories to back up. This filename can be overridden on the command # line with the -f switch. If files or directories to be backed up are # specified on the command line, then only the specified files are backed up # and this file is not read. bu_include_file=$include_file # bu_exclude_file (env) [bu] [rbu] # The name of the file containing the list of files to exclude from backups. bu_exclude_file=$exclude_file # bu_mail_addr (env) [bu] # The email address to mail the log to for NFS backups. If unset, no log # will be mailed. bu_mail_addr= # bu_log_access_perms [bu] # File permissions for the log files (man chmod). For security, logs should # not be world readable. Otherwise, users can look at the backup logs to see # what files are in directories that they do not have read access to. bu_log_access_perms=$Default_log_access_perms # bu_group_size and bu_delay (env) [bu] # If you specify a bu_group_size greater than 1, and a bu_delay greater than # 0, then it will back up the number of files specified by bu_group_size at # a time, sleeping bu_delay seconds between each group. This can be used to # tune the amount of network load when backing up over NFS. It will take # longer to do the backup but it could be handy if you need to do a back up # during the day in a high traffic environment and don't want to load the # network down so much. It only applies when backing up whole directories. # If individual files are specified, it is ignored. bu_group_size=$Default_group_size bu_delay=$Default_delay ############################################################################## # The next section is for multi-volume dumps to CDRW's ############################################################################## ## All settings below this point are still only used by bu (not rbu). # bu_device (env) # Default CD device bu_device=$Default_device # bu_volume_size (env) # Volume size in MB bu_volume_size=$Default_volume_size # bu_incremental - Flag (env) # When dumping to removable media, it will only dump files that have changed # since the last dump, no matter what dump mode was used. Differential mode, # when on, overrides incremental. bu_incremental=$incremental_mode # bu_differential - Flag (env) # Turns on or off differential dumps by default. This flag only applies to # dumps to removable media. When bu_differential is on, it will dump all # files that have changed since the date of the last full dump. If it is # off, and bu_incremental is on, it will dump all changed files since the # last incremental or differential dump. Whichever is more recent. If they # are both off, it will always do a full dump. It must be off to gain # command line control with --differential or to do an incremental dump. bu_differential=$differential_mode # bu_backup_image - Flag (env) # This flag should be turned off to back up anything other than a backup # image directory. When turned on, bu only accepts one directory argument # (or it will default to \$bu_backup_dir if not specified) and the directory # is assumed to be the root of a backup directory so that any preceding path # is ignored. # Example: # Assuming the specified directory is /usr/backup, and the directory # usr/bin resides under it. With bu_Backup_image on, it will be dumped as # usr/bin. With it off it will be dumped as usr/backup/usr/bin. bu_backup_image=$Default_backup_image # bu_label (env) # Backup label. If set, this string will go in the info file on each # CD volume as the backup label. It is useful for documenting and # identifying backups. bu_label="" # bu_compress - Flag (env) # When on, the dumped archive is compressed with gzip. Must be off to gain # command line control with --compress. bu_compress=$Default_compress # bu_speed (env) # CD write speed bu_speed=$Default_speed # bu_eject - Flag (env) # Automatically eject the media between each volume and at the end of the # backup. Must be off to gain command line control with --eject. bu_eject=$Default_eject # bu_blank - Flag (env) # Blank each CD before writing. This is for CDRW's that already have data # or old backups on them, so that they do not have to manually be blanked # ahead of time. This uses the fast blanking method. There will be no # prompting for confirmation, so make sure you do not put any CD's in with # data you do not want to loose if you turn this on. This must be off to # gain command line control with --blank. bu_blank=$Default_blank # bu_prompt - Flag (env) # Prompt for confirmation after displaying the settings before beginning a # dump. You might want to turn this off for scripts or if running under a # user interface. This must be on to gain command line control with # --prompt. bu_prompt=$Default_prompt EOF close(FILE); return(1); } # shell_expand() # Usage: shell_expand($variable) # Expand variable using the shell, all special shell characters in it's value # will be expanded and placed back into $variable. # sub shell_expand(\$) { $var = shift; $$var = `echo $$var`; chomp($$var); } # source() # Usage: source("file", ["variable_name"]) # # Reads a .conf or shell script style file of variables (ie. var=value) and # sets all the same variables in the perl script. Similar to the 'source' or # '.' command in shell script (i.e. ". filename") except it also sets them in # the environment. If variable_name is specified, it sets only that variable # and returns the value. Otherwise nothing is returned. White space around # the '=' in variable settings is allowed in the file but you should not have # any if you intend for shell scripts to also be able to source it. # # Shell expansion is also done, so the values may contain other variables as # well as wildcards, backticks, or any other special shell characters, and it # will be expanded if the value is not surounded by single quotes. # sub source($;$) { my ($filename, $specified_variable) = @_; my ($var, $val, $value); my $_single_quotes_; # Flag - For decision whether to expand variables. if (! -s $filename) { return(0) } if (! open(FILE, "<$filename")) { print STDERR "source(): Cannot read \"$filename\"\n"; return; } while () { $_single_quotes_ = 0; if (/^[ \t]*([A-Za-z]\w*)\s*=\s*(.*)$/) { $var = $1; $val = $2; if ($specified_variable) { if ($var ne $specified_variable) { next } } if ($val =~ /\"(.*)\"/) { $val = $1; } elsif ($val =~ /\'(.*)\'/) { $val = $1; $single_quotes = 1; } else { $val =~ s/#+.*$//; } # Strip trailing comments if (! $single_quotes) { shell_expand($val) } $$var = $val; $ENV{$var} = $$var; # Put it in the environment } } close(FILE); if ($specified_variable) { return($val); } } # run_command() # Usage: $pid = run_command(\*cmd_stdout, \*cmd_stdin, @cmd) # Runs @cmd in background and connects it stdin and stdout to the passed # file handles. # Returns the pid of the command or 0 if there is an error. # sub run_command($$@) { ($cmd_output, $cmd_input, @cmd) = @_; # open2() does not return on failure. Instead, it raises an exception, # which we catch with eval. eval { $pid = open2($cmd_output, $cmd_input, @cmd) }; if ($@) { print STDERR "rbu: Error running \"@cmd\"\n $@"; return(0); } return($pid); } sub verify_destination($) { my $dest = shift; my $rhost; # remote host my $rdir; # remote directory my @rcmd; # remote command if (($dest =~ /(.*?):(.*)/) && ($1 !~ /\//)) { $dest =~ /(.*):(.*)/; $rhost = $1; $rdir = $2; @rcmd = ($Rsh, $rhost); push(@rcmd, ("if [ ! -d \"$rdir\" ]; then exit 1; fi")); system(@rcmd) && return(0); } # if $dest is local elsif (! -d $dest) { return(0); } return(1); } # backup() # Usage: $result = backup(\@file_list, $dest) # Returns 1 on success, 0 on failure. # sub backup($$) { my $src = shift; my $dest = shift; my $destdir; # $dest with any leading 'user@' removed my $file; my @rsync; my $pid; my $_changed_files_ = 0; # flag - there were new or changed files to backup local (*RSYNC_INPUT, *RSYNC_OUTPUT); if ($_Debug_mode_ > 1 ) { my @arg1 = @$src; debug "backup(@$src, $dest)"; } $destdir = $dest; $destdir =~ s/^.+?\@//; if (! verify_destination($dest)) { errmsg "Backup directory, $destdir,", "does not exist."; return(0); } if ($_Verbose_mode_) { print "\nBackup directory: $destdir\n" ; if ($_Dry_run_mode_) { print "Dry run mode: No data will be transfered\n"; } print "\n"; } if ($_Debug_mode_) { debug "executing:\n @Rsync"; } $pid = run_command(\*RSYNC_OUTPUT, \*RSYNC_INPUT, @Rsync) || return(0); if ($_Debug_mode_ > 1 ) { debug "Sending filelist to rsync"; } foreach (@$src) { $file = realpath($_); if ($_Debug_mode_ > 1 ) { print " $file\n" } print RSYNC_INPUT "$file\n"; } if ($_Debug_mode_) { print "\n" } close(RSYNC_INPUT); if ($_Verbose_mode_ || $_Dry_run_mode_) { while () { if (/^\//) { print "$_"; $_changed_files_ = 1; } elsif (/^deleting (.*)/) { print "deleting $destdir/$1\n"; $_changed_files_ = 1; } } if (! $_changed_files_) { print "No changes\n"; } } waitpid($pid, 0); close(RSYNC_OUTPUT); if ($_Verbose_mode_) { print "\n"; } return(1); } use IPC::Open2; use Cwd 'realpath'; main: { set_defaults(); parse_cmdline(@ARGV); configure(); if ($_Help_mode_) { usage() } if ($_Create_rcfile_) { create_rcfile($Rc_file) } if ($_Create_rcfile_only_) { exit(0) } if (! @File_list) { errmsg "No source files specified"; exit(1); } if (! $Dest) { errmsg "No destination specified.", "Specify \$bu_dest or \$bu_backup_dir in $Rc_file", "or dest=[user\@host:]/path on the command line"; exit(1); } backup(\@File_list, $Dest) || exit(1); }