diff --git a/.bundle/config b/.bundle/config deleted file mode 100644 index 57d4641..0000000 --- a/.bundle/config +++ /dev/null @@ -1,3 +0,0 @@ ---- -BUNDLE_PATH: vendor/bundle -BUNDLE_DISABLE_SHARED_GEMS: '1' diff --git a/README.md b/README.md index a1fd3f2..bed29f7 100644 --- a/README.md +++ b/README.md @@ -133,10 +133,14 @@ rsnapshot::hosts: backup: '/var': './var' cron: - 'daily': - 'minute': '0-10' - 'hour': '1..5' - + mailto: 'bar1@example.com' + daily: + minute: '0-10' + hour: '1..5' + db1: + backup_scripts: + mysql: + misc: ``` @@ -243,10 +247,14 @@ The range notation is '$start..$end', so to pick a random hour from 8 pm to 2 am For the range feature to work, hours >0 and <10 must not have a preceding zero. Wrong: `00.09` Correct: `0..9` +Also, you can set a mailto for each host, or globally now. The settings will be merged bottom to top, so if you override a setting in a hosts cron, it will have precedence over the global setting, +which in turn has precedence over the default. + Example: ```puppet $cron = { + mailto => 'admin@example.com', hourly => { minute => '0..59', hour => [ '20..23','0..2' ], @@ -259,6 +267,7 @@ Or in hiera: ```yaml rsnapshot::cron: + mailto: 'admin@example.com' daily: minute: '20' weekly: @@ -270,16 +279,32 @@ rsnapshot::cron: ```yaml rsnapshot::hosts: webserver: - daily: - hour: [ '20..23','0..2' ] - weekly: - hour: [ '20..23','0..2' ] + cron: + mailto: 'support@example.com' + daily: + hour: [ '20..23','0..2' ] + weekly: + hour: [ '20..23','0..2' ] + + webhost: + + customervm.provider.com: + backup_user: 'customer' ``` +`webhost`: Mails will go to `admin@example.com` (from the global override). + +`webserver`: Mails will go to `support@example.com`. + +`customervm.provider.com`: The backup (and thus ssh) user will be `customer@customervm.provider.com` + + + Hash is of the form: ```puppet -$cron =>{ +$cron =>{ + mailto => param, daily => { minute => param, hour => param, @@ -297,6 +322,7 @@ Default is: ```puppet $cron = { + mailto => 'admin@example.com', hourly => { minute => '0..59', # random from 0 to 59 hour => '*', # you could also do: ['21..23','0..4','5'], @@ -366,20 +392,65 @@ Default is: #### `$backup_scripts` Additional scripts to create, possible values are: mysql, psql, misc +`mysql`: used for mysql backups + +`psql`: used for postgresql backups + +`misc`: custom commands to run on the node + +You can set + +`$dbbackup_user`: backup user + +`$dbbackup_password`: password for the backup user + +`$dumper`: path to the dump bin you wish to use + +`$dump_flags`: flags for your dump bin + +`$ignore_dbs`: databases to be ignored (the psql script ignores template and postgres databases by default) + +`$commands`: array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) + +See below for defaults + +NOTE: the psql and mysql scripts will SSH into your host and try and use $dumper. +Make sure you have those tools installed on your DB hosts. + +Also, this module will try and use pbzip to compress your databases. You can install pbzip2 (and additional packages you might need) by passing an array to [$rsnapshot::package_name](#package_name) + + Default is: ```puppet $backup_scripts = { - mysql => { + mysql => { dbbackup_user => 'root', - dbbackup_password => 'myPassWord', + dbbackup_password => '', + dumper => 'mysqldump', + dump_flags => '--single-transaction --quick --routines --ignore-table=mysql.event', + ignore_dbs => [ 'information_schema', 'performance_schema' ], }, psql => { dbbackup_user => 'postgres', dbbackup_password => '', + dumper => 'pg_dump', + dump_flags => '-Fc', + ignore_dbs => [], }, - misc => {}, + misc => { + commands => $::osfamily ? { + 'RedHat' => [ + 'rpm -qa --qf="%{name}," > packages.txt', + ], + 'Debian' => [ + 'dpkg --get-selections > packages.txt', + ], + default => [], + }, + } } + ``` Configuration example: @@ -398,21 +469,31 @@ rsnapshot::hosts: backup_scripts: mysql: psql: - dbbackup_user: 'backupuser' - dbbackup_password: 'password' + dumper: '/usr/local/bin/pg_dump' + dump_flags: '-Fc' + ignore_dbs: [ 'db1', 'tmp_db' ] + misc: bazqux:de: backup_scripts: mysql: dbbackup_user: 'myuser' dbbackup_password: 'mypassword' + misc: + commands: + - 'cat /etc/hostname > hostname.txt' + - 'date > date.txt' ``` This creates - a mysql and a psql backup script for `foobar.com` using the credentials `dbbackup:hunter2` for mysql and `dbbackup:yeshorsebatterystaple` for psql +- the psql script will use `/usr/local/bin/pg_dump` as the dump program with flags `-Fc` +- it will ignore the postgres databases `db1` and `tmp_db` for postgres - a mysql backup script for `bazqux.de` using the credentials `myuser:mypassword` +- a misc script for bazqux.de containing two commands to run on the node. the output will be redirected to hostname.txt and date.txt in the misc/ subfolder of the hosts backup directory (i.e. /snapshot_root/bazqux.de/daily.0/misc/hostname.txt) The scripts look like this: -mysql: + +##### `bazqux.de` ```bash #!/bin/bash @@ -420,34 +501,103 @@ host=bazqux.de user=myuser pass=mypassword -dbs=( $(mysql -h "$host" -u "$user" -p"$pass" -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d') ) +dbs=( + $(ssh -l root "$host" "mysql -u ${user} -p${pass} -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'") + ) for db in "${dbs[@]}"; do - mysqldump --host="$host" --user="$user" --password="$pass" --single-transaction --quick --routines --ignore-table=mysql.event "$db" > "$db".sql + ssh -l root "$host" "mysqldump --user=${user} --password=${pass} --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" wait - pbzip2 -p3 "$db".sql + pbzip2 "$db".sql done ``` +```bash +#!/bin/bash + +ssh bazqux.de 'cat /etc/hostname > hostname.txt' + +ssh bazqux.de 'date > date.txt' + +``` + + + +##### `foobar.com` + psql: ```bash #!/bin/bash host=foobar.com -user=backupuser -pass=password +user=dbbackup +pass=yeshorsebatterystaple PGPASSWORD="$pass" -dbs=( $(psql -h "$host" -U "$user" -Atc "SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'") ) +dbs=( + $(ssh -l root "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname ~ 'postgres|db1|tmp_db'\"" ) + ) for db in "${dbs[@]}"; do ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql wait - pbzip2 -p3 "$db".sql + pbzip2 "$db".sql done ``` +mysql: + + +```bash +#!/bin/bash +host=foobar.com +user=dbbackup +pass=hunter2 + +dbs=( + $(ssh -l root "$host" "mysql -u ${user} -p${pass} -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'") + ) + +for db in "${dbs[@]}"; do + ssh -l root "$host" "mysqldump --user=${user} --password=${pass} --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" + wait + pbzip2 "$db".sql +done + +``` + +misc (assuming foobar.com is a RedHat node): + +```bash +#!/bin/bash + +ssh foobar.com 'rpm -qa --qf "%{name}," > packages.txt' + +``` + +##### another example with root user and empty password + +mysql with root user: + +```bash +#!/bin/bash +host=bazqux.de +user=root +password= + +dbs=( + $(ssh -l root "$host" "mysql -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'") + ) + +for db in "${dbs[@]}"; do + ssh -l root "$host" "mysqldump --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" + wait + pbzip2 "$db".sql +done + +``` + ### rsnapshot configuration variables Please read up on the following in the [rsnapshot manpage](http://linux.die.net/man/1/rsnapshot) @@ -596,4 +746,4 @@ Norbert Varzariu (loomsen) ## Contributors Please see the [list of contributors.](https://github.com/loomsen/puppet-bloonix_agent/graphs/contributors) - +A big thank you to Hendrik Horeis for all his input and testing of this module. diff --git a/manifests/config.pp b/manifests/config.pp index 0a1d319..702765f 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -36,7 +36,7 @@ class rsnapshot::config ( $hosts_clean = assert_empty_hash($hosts) $hosts_clean.each |String $host, $hash | { - $backup_user = pick($hash['backup_user'], $rsnapshot::params::config_backup_user) + $backup_user = pick($hash['backup_user'], $rsnapshot::backup_user, $rsnapshot::params::config_backup_user, 'root') $default_backup_dirs = pick($rsnapshot::default_backup, $rsnapshot::params::config_default_backup) $backup_levels = pick($hash['backup_levels'], $rsnapshot::backup_levels, 'weekly') $backup = $hash['backup'] @@ -135,21 +135,22 @@ class rsnapshot::config ( content => template('rsnapshot/rsnapshot.erb'), } + + if has_key($hash, backup_scripts) { - - $hash[backup_scripts].each |$script, $credentials| { - - if is_hash($credentials) { - $dbbackup_user = $credentials['dbbackup_user'] - $dbbackup_password = $credentials['dbbackup_password'] - } else { - $dbbackup_user = $rsnapshot::default_backup_scripts[$script]['dbbackup_user'] - $dbbackup_password = $rsnapshot::default_backup_scripts[$script]['dbbackup_password'] - } + $hash[backup_scripts].each |$script, $scriptconf| { + $real_script = deep_merge($rsnapshot::params::backup_scripts[$script], $rsnapshot::backup_scripts[$script], $hash[backup_scripts][$script]) + $dbbackup_user = $real_script[dbbackup_user] + $dbbackup_password = $real_script[dbbackup_password] + $dumper = $real_script[dumper] + $dump_flags = $real_script[dump_flags] + $ignore_dbs = $real_script[ignore_dbs] + $compress = $real_script[compress] + $commands = $real_script[commands] concat::fragment { "${host}_${script}_backup": - target => $config, - content => "backup_script ${conf_d}/${host}.${script}.sh ./${script}\n", + target => $config, + content => "backup_script ${conf_d}/${host}.${script}.sh ./${script}\n", } file { "${conf_d}/${host}.${script}.sh": @@ -167,8 +168,14 @@ class rsnapshot::config ( # create cron files for each backup level # merge possible cron definitions to one $real_cron = deep_merge($rsnapshot::params::cron, $rsnapshot::cron, $hash[cron]) + concat::fragment { "mailto for $host": + content => "#This file is managed by puppet\nMAILTO=${real_cron[mailto]}\n\n", + target => $cronfile, + order => 1, + } $backup_levels.each |String $level| { + $mailto = $real_cron[mailto] $minute = rand_from_array($real_cron[$level][minute], "${host}.${level}.minute") $hour = rand_from_array($real_cron[$level][hour], "${host}.${level}.hour") $monthday = rand_from_array($real_cron[$level][monthday], "${host}.${level}.monthday") @@ -178,6 +185,7 @@ class rsnapshot::config ( concat::fragment { "${host}.${level}": target => $cronfile, content => template('rsnapshot/cron.erb'), + order => 2, } } } diff --git a/manifests/params.pp b/manifests/params.pp index 6a26d1a..254896e 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -7,7 +7,11 @@ class rsnapshot::params { $config_backup_user = 'root' $package_name = 'rsnapshot' $package_ensure = 'present' - $cron_service_name = 'crond' + $cron_service_name = $::osfamily ? { + 'RedHat' => 'crond', + 'Debian' => 'cron', + default => '', + } $cron_dir = '/etc/cron.d' $config_backup_levels = [ 'daily', 'weekly', 'monthly' ] $config_backup_defaults = true @@ -63,6 +67,7 @@ class rsnapshot::params { } $config_backup_scripts = {} $cron = { + mailto => 'admin@example.com', hourly => { minute => '0..59', hour => '*', # you could also do: ['21..23','0..4','5'], @@ -93,14 +98,32 @@ class rsnapshot::params { }, } $backup_scripts = { - mysql => { + mysql => { dbbackup_user => 'root', - dbbackup_password => 'myFancyPassWord', + dbbackup_password => '', + dumper => 'mysqldump', + dump_flags => '--single-transaction --quick --routines --ignore-table=mysql.event', + ignore_dbs => [ 'information_schema', 'performance_schema' ], + compress => 'pbzip2', }, psql => { dbbackup_user => 'postgres', dbbackup_password => '', + dumper => 'pg_dump', + dump_flags => '-Fc', + ignore_dbs => [], + compress => 'pbzip2', }, - misc => {}, + misc => { + commands => $::osfamily ? { + 'RedHat' => [ + 'rpm -qa --qf="%{name}," > packages.txt', + ], + 'Debian' => [ + 'dpkg --get-selections > packages.txt', + ], + default => [], + }, + } } } diff --git a/metadata.json b/metadata.json index 7f75e13..3daf1d9 100644 --- a/metadata.json +++ b/metadata.json @@ -1,6 +1,6 @@ { "name": "loomsen-rsnapshot", - "version": "0.5.2", + "version": "1.0.0", "author": "loomsen", "summary": "Configures rsnapshot.", "license": "Apache-2.0", diff --git a/templates/misc.sh.erb b/templates/misc.sh.erb index 8e19cf6..7185cc2 100644 --- a/templates/misc.sh.erb +++ b/templates/misc.sh.erb @@ -1,10 +1,7 @@ #!/bin/bash -# add custom stuff here -# FIXME: remember to concat for custom stuff after the last line of the puppet generated script -<% if @osfamily == 'RedHat' -%> -ssh <%=@host-%> rpm -qa --qf='%{name},' > packages.txt +#This file is managed by puppet +# +<%@commands.each do |command| -%> +ssh -l <%=@backup_user-%> <%=@host-%> '<%=command-%>' + <%end-%> -# -# -# -# ++++++++++++++++++++++++++++++++++++++++++++ diff --git a/templates/mysql.sh.erb b/templates/mysql.sh.erb index f07f69d..4a81e20 100644 --- a/templates/mysql.sh.erb +++ b/templates/mysql.sh.erb @@ -1,4 +1,5 @@ #!/bin/bash +# This file is managed by puppet # written by Norbert Varzariu # This Script is triggered by rsnapshot and dumps the specified dbs on the configured host. # @@ -6,11 +7,24 @@ host=<%=@host%> user=<%=@dbbackup_user%> pass=<%=@dbbackup_password%> -dbs=( $(mysql -h "$host" -u "$user" -p"$pass" -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d') ) - +<% if (@dbbackup_user == 'root' && @dbbackup_password == '') -%> +dbs=( + $(ssh -l <%=@backup_user-%> "$host" "mysql -e 'show databases' | sed '1d;<%@ignore_dbs.to_a.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) + ) +<%else-%> +dbs=( + $(ssh -l <%=@backup_user-%> "$host" "mysql -u <%=@dbbackup_user-%> -p'<%=@dbbackup_password-%>' -e 'show databases' | sed '1d;<%@ignore_dbs.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) + ) +<%end-%> for db in "${dbs[@]}"; do - mysqldump --host="$host" --user="$user" --password="$pass" --single-transaction --quick --routines --ignore-table=mysql.event "$db" > "$db".sql +<% if (@dbbackup_user == 'root' && @dbbackup_password = '' )-%> + ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> <%=@dump_flags-%> ${db}" > "$db".sql +<%else-%> + ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> --user=<%=@dbbackup_user-%> --password='<%=@dbbackup_password-%>' <%=@dump_flags-%> ${db}" > "$db".sql +<%end-%> wait - pbzip2 -p3 "$db".sql +<% if @compress != '' -%> + <%=@compress-%> "$db".sql +<%end-%> done diff --git a/templates/psql.sh.erb b/templates/psql.sh.erb index 596de34..0f3f9ab 100644 --- a/templates/psql.sh.erb +++ b/templates/psql.sh.erb @@ -1,4 +1,5 @@ #!/bin/bash +# This file is managed by puppet # written by Norbert Varzariu # This Script is triggered by rsnapshot and dumps the specified dbs on the configured host. # @@ -7,11 +8,14 @@ user=<%=@dbbackup_user%> pass=<%=@dbbackup_password%> PGPASSWORD="$pass" -dbs=( $(psql -h "$host" -U "$user" -Atc "SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'") ) - +dbs=( + $(ssh -l <%=@backup_user-%> "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND NOT datname ~ '<%@ignore_dbs.each do |db|-%><%if db == @ignore_dbs.last-%><%=db-%><%else-%><%=db-%>|<%end-%><%end-%>'\"") + ) for db in "${dbs[@]}"; do - ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql + ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> -U ${user} <%=@dump_flags-%> ${db}" > "$db".sql wait - pbzip2 -p3 "$db".sql +<% if @compress != '' -%> + <%=@compress-%> "$db".sql +<%end-%> done