#!/usr/bin/perl
# Website Differential Backup
# Keeps several websites and their databases backed up
# by Ed Salisbury (ed@edsalisbury.net)
# http://www.edsalisbury.net
# (c)2009 Ed Salisbury, Some Rights Reserved
#
# Config File Format
# DIR /path/to/dir
# DB dbname
#
# Limitations:
# * Cannot do differential backups of databases
#
#
# License:
# Except where otherwise noted, this work is licensed under Creative Commons
#   Attribution ShareAlike 3.0.
#
# You are free:
#   * to Share - to copy, distribute and transmit the work
#   * to Remix - to adapt the work
#
# Under the following conditions:
#   * Attribution. You must attribute the work in the manner specified by the
#     author or licensor (but not in any way that suggests that they endorse
#     you or your use of the work).
#   * Share Alike. If you alter, transform, or build upon this work, you may
#     distribute the resulting work only under the same, similar or a
#     compatible license.
#   * For any reuse or distribution, you must make clear to others the license
#     terms of this work. The best way to do this is with a link to the
#     license's web page (http://creativecommons.org/licenses/by-sa/3.0/)
#   * Any of the above conditions can be waived if you get permission from the
#   * Nothing in this license impairs or restricts the author's moral rights.

use strict;
use warnings;

# Place to store backups
my $BACKUP_DIR = "/backup";
# Configuration file to read what to backup
my $BACKUP_CONF = "/usr/local/etc/backup.conf";
# Database user/pass
my $DB_USER = 'root';
my $DB_PASS = '**DBPASSWD**';

# Turn off output buffering
$|++;

# Tools
my $TAR = '/bin/tar';
my $GZIP = '/bin/gzip';
my $LS = '/bin/ls';
my $HEAD = '/usr/bin/head';
my $MKDIR = '/bin/mkdir';
my $RM = '/bin/rm';
my $MYSQLDUMP = '/usr/local/mysql/bin/mysqldump';

my @months = qw( Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec );
my @days = qw( Sun Mon Tue Wed Thu Fri Sat );
my @dirs;
my @dbs;

# Get current date
my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime time;
my $datestamp = sprintf("%04d%02d%02d", $year+1900, $mon+1, $mday);
my $day = $days[$wday];

# Read config file
unless (open(CFG, $BACKUP_CONF))
{
    print "Error: Cannot open $BACKUP_CONF\n";
    exit(1);
}
while (<CFG>)
{
    chomp;
    if (/^DIR\s+(.*)$/i)
    {
        push (@dirs, $1);
    }
    elsif (/^DB\s+(.*)$/i)
    {
        push (@dbs, $1);
    }
}
close (CFG);

# Do Directory Backups
foreach my $dir (@dirs)
{
    print "Backing up $dir... ";

    # Add backslashes for spaces
    $dir =~ s/ /\\ /g;

    my $backup_src = $dir;

    # Convert slashes to underscores
    $dir =~ s/\//_/g;
    # Remove leading slash
    $dir = substr($dir,1);

    unless (-d "$BACKUP_DIR/$dir")
    {
        system("$MKDIR -p $BACKUP_DIR/$dir");
    }

    # Get datestamp of last full backup
    my $last_full = `$LS -t $BACKUP_DIR/$dir/${dir}_full* 2>&1 | $HEAD -1`;
    chomp($last_full);
    if ($last_full =~ /No such file or directory/)
    {
        $last_full = '';
    }
    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,
        $blksize,$blocks) = stat($last_full);

    if ($mday == 1)
    {
        # Monthly Full Backup
        system("$TAR -cf $BACKUP_DIR/$dir/${dir}_full_monthly_$datestamp.tar $backup_src > /dev/null 2>&1");
        system("$GZIP -f $BACKUP_DIR/$dir/${dir}_full_monthly_$datestamp.tar");
        system("$RM $BACKUP_DIR/$dir/${dir}_full_weekly_*");
    }
    elsif ($wday == 0 || !$last_full)
    {
        # Weekly Full Backup
        system("$TAR -cf $BACKUP_DIR/$dir/${dir}_full_weekly_$datestamp.tar $backup_src  > /dev/null 2>&1");
        system("$GZIP -f $BACKUP_DIR/$dir/${dir}_full_weekly_$datestamp.tar");
    }

    # Daily Differential Backup
    if ($mtime)
    {
        my $newer = sprintf("%02d-%s", (localtime($mtime))[3], $months[(localtime($mtime))[4]]);
        system("$TAR --newer $newer -cf $BACKUP_DIR/$dir/${dir}_diff_daily_$day.tar $backup_src > /dev/null 2>&1");
    }
    else
    {
        system("$TAR -cf $BACKUP_DIR/$dir/${dir}_diff_daily_$day.tar $backup_src > /dev/null 2>&1");
    }
    system("$GZIP -f $BACKUP_DIR/$dir/${dir}_diff_daily_$day.tar");

    print "done.\n";
}

# Do Database Backups
foreach my $db (@dbs)
{
    my $db_name = $db;
    $db = "db_$db";

    unless (-d "$BACKUP_DIR/$db")
    {
        system("$MKDIR -p $BACKUP_DIR/$db");
    }

    print "Backing up $db_name database... ";

    if ($mday == 1)
    {
        # Monthly Backup
        system("$MYSQLDUMP --user=$DB_USER --password='$DB_PASS' $db_name > $BACKUP_DIR/$db/${db}_monthly_$datestamp.sql");
        system("$GZIP -f $BACKUP_DIR/$db/${db}_monthly_$datestamp.sql");
        system("$RM $BACKUP_DIR/$db/${db}_weekly_*");
    }
    elsif ($wday == 0)
    {
        # Weekly Backup
        system("$MYSQLDUMP --user=$DB_USER --password='$DB_PASS' $db_name > $BACKUP_DIR/$db/${db}_weekly_$datestamp.sql");
        system("$GZIP -f $BACKUP_DIR/$db/${db}_weekly_$datestamp.sql");
    }

    # Daily Backup
    system("$MYSQLDUMP --user=$DB_USER --password='$DB_PASS' $db_name > $BACKUP_DIR/$db/${db}_daily_$day.sql");
    system("$GZIP -f $BACKUP_DIR/$db/${db}_daily_$day.sql");

    print "done.\n";
}

Call the script something like /usr/local/bin/backup, and then create a config file /usr/local/etc/backup.conf, that has the following format:
DIR /www/domain1.com
DIR /www/domain2.com
DB domain1_wp_db
DB domain2_wp_db

(Substituting locations/db names appropriate for your environment of course). If you use a different config file than /usr/local/etc/backup.conf, be sure to change that line in the script. Also, you’ll need to change the mysql root password in the script to be whatever it is for your env (or make it non-root, whichever you want, it just needs to be able to do a mysqldump on the databases)

Next, set up your crontab to run nightly:

sudo crontab -e

Add the following line:

0 1 * * * /usr/local/bin/backup > /dev/null 2>&1

That should be it! Again, make sure to do some test restores to verify that it works ok! Here’s what a typical backup directory looks like after a couple of months running the script:

[ed@halo1:/backup/www_edsalisbury.net]$ ls -l
total 144240
-rw-r--r--  1 root root  5423759 Sep 11 01:00 www_edsalisbury.net_diff_daily_Fri.tar.gz
-rw-r--r--  1 root root  1839115 Sep  7 01:00 www_edsalisbury.net_diff_daily_Mon.tar.gz
-rw-r--r--  1 root root  5787136 Sep 12 01:00 www_edsalisbury.net_diff_daily_Sat.tar.gz
-rw-r--r--  1 root root  7193167 Sep 13 01:00 www_edsalisbury.net_diff_daily_Sun.tar.gz
-rw-r--r--  1 root root  3782727 Sep 10 01:00 www_edsalisbury.net_diff_daily_Thu.tar.gz
-rw-r--r--  1 root root  2455973 Sep  8 01:00 www_edsalisbury.net_diff_daily_Tue.tar.gz
-rw-r--r--  1 root root  3268897 Sep  9 01:00 www_edsalisbury.net_diff_daily_Wed.tar.gz
-rw-r--r--  1 root root 23705401 Aug  1 01:00 www_edsalisbury.net_full_monthly_20090801.tar.gz
-rw-r--r--  1 root root 33975458 Sep  1 01:00 www_edsalisbury.net_full_monthly_20090901.tar.gz
-rw-r--r--  1 root root 31453170 Sep  6 01:00 www_edsalisbury.net_full_weekly_20090906.tar.gz
-rw-r--r--  1 root root 28556424 Sep 13 01:00 www_edsalisbury.net_full_weekly_20090913.tar.gz

Not too bad, size-wise. For all of my sites, the backup directory is about 4 GB, and will stay around that size for quite a while. The differentials get deleted and so do the weekly full backups after the monthly full completes. This makes it so that the directory only really grows when there’s a new full monthly. If you find this script useful or have issues, be sure to post a comment!

First, let’s get the CPAN module up to date:

$ sudo perl -MCPAN -e shell
(take defaults to set everything up)
cpan> exit
$ sudo perl -MCPAN -e shell
cpan> install Bundle::CPAN
(answer yes to the question about saving data)
cpan> reload cpan

Then, to install a module, it’s simply:
$ sudo perl -MCPAN -e shell
cpan> install <name of module>
cpan> exit

You might ask, why not just set up a RAID5? Or, alternatively, why not buy a Drobo and be done with it? I hear the first one a lot, especially since I deal with RAID day in and day out for my day job. The main issue I don’t do either of these things is that I’m cheap! I want to be able to use all of the hardware I have without tossing out old stuff. On my server, I have the following drives:

• disk1: 120 GB ATA
• disk2: 200 GB ATA
• disk3: 300 GB USB
• disk4: 400 GB Firewire
• disk5: 600 GB SATA
• disk6: 1000 GB USB
• disk7: 1500 GB USB

Try to set up a RAID with all of that! The problem with this is the disparate drive sizes (RAID requires disks to be the same size, and will use the lowest common denominator if not.) I’ve been ‘collecting’ hard drives for quite a while, and I like the fact that I can just go to the local electronics store and pick up another drive when I run out of space. This configuration does have a downside though, and that is, how do you decide where to put everything? I decided to sit down and figure out how much space I actually need for each of my data types, and came up with this list (taking approximate growth rates into consideration):

• Photos: 100 GB
• Music: 300 GB
• Movies: 1 TB
• TV: 400 GB
• Software: 150 GB

I then tried to map things out onto the drives I have:

• disk1: Photos
• disk2: Software
• disk3: Music
• disk4: TV
• disk6: Movies

This leaves two disks, disk5 (600 GB) and disk7 (1500 GB) left to be able to mirror to. Since I can’t use standard mirroring software, I’m not able to have a real-time mirror. This is OK for my purposes, as I don’t intend on having them be hot-swappable, etc. — I just want another copy of my data out there, without having to worry about making backups all the time.

I set up my server to only export the data disks (I don’t want to be able to write to the backups from my desktop, as the changes will be overwritten by the backups) I also set up my filesystem in a convenient manner — everything is a directory off of /data:

• /data/Photos
• /data/Video
• /data/Software
• /data/Audio

This makes it very easy to get to any of my data (try that in Windows!)

One inherent advantage of having an asynchronous mirror is the fact that it can serve as a temporary backup if you accidentally delete a file/directory, whereas with a normal mirror, you’d still have to back up to tape or some other drive to be able to get this data back.

To do the actual mirroring, I opted to create a simple rsync script that will copy the data nightly from cron. Nothing overly complex, but something that makes it easy to add new filesystems via a config file:

#!/usr/bin/perl
# Mirror
# Asynchronously mirror filesystems on a local machine
# by Ed Salisbury (ed@edsalisbury.net)
# http://www.edsalisbury.net
# (c)2009 Ed Salisbury, Some Rights Reserved
#
# External Utilities Required:
# * rsync
#
# License:
# Except where otherwise noted, this work is licensed under Creative Commons
#   Attribution ShareAlike 3.0.
#
# You are free:
#   * to Share — to copy, distribute and transmit the work
#   * to Remix — to adapt the work
#
# Under the following conditions:
#   * Attribution. You must attribute the work in the manner specified by the
#     author or licensor (but not in any way that suggests that they endorse
#     you or your use of the work).
#   * Share Alike. If you alter, transform, or build upon this work, you may
#     distribute the resulting work only under the same, similar or a
#     compatible license.
#   * For any reuse or distribution, you must make clear to others the license
#     terms of this work. The best way to do this is with a link to the
#     license's web page (http://creativecommons.org/licenses/by-sa/3.0/)
#   * Any of the above conditions can be waived if you get permission from the
#     copyright holder.
#   * Nothing in this license impairs or restricts the author's moral rights.

use warnings;
use strict;

# Configuration file for mirrors
# Example Config:
# /data/Photos/       /backup/Backup/Photos/
# /data/Software/     /backup/Backup/Software/
# /data/Audio/        /backup/Backup/Audio/
# /data/Video/        /backup/Backup/Video/
# /data/Misc/         /backup/Backup/Misc/
my $CONFIG = "/usr/local/etc/mirror.cfg";

open(CFG, $CONFIG);
my @lines = <CFG>;
close(CFG);

foreach my $line (@lines)
{
    my ($src, $dest) = split(/\s+/, $line);
    system("/usr/bin/rsync -av --delete $src $dest");
}

I put this script into /usr/local/bin, and then created a config file /usr/local/etc/mirror.conf similar to this:

/data/Photos/       /backup/Backup/Photos/
/data/Software/     /backup/Backup/Software/
/data/Audio/        /backup/Backup/Audio/
/data/Video/        /backup/Backup/Video/
/data/Misc/         /backup/Backup/Misc/

I then added a cronjob to have it run nightly:

0 0 * * * /bin/backup > /dev/null 2>&1

A word of caution: Make *sure* you have the directories right! rsync is set up to *DELETE* files in the destination if the file isn’t in the source directory, so if you have it rsync to two different directories, it will delete files as needed in the destination to sync them up. You have been warned! I take no responsibility for you deleting your data!

Please let me know if you found this guide/script useful.