bump version to 1.0.0 \o/

- parameterize backup scripts
- add custom commands to misc script
- add parameters to mysql and psql scripts
- update documentation

.bundle/config

@@ -1,3 +0,0 @@
----
-BUNDLE_PATH: vendor/bundle
-BUNDLE_DISABLE_SHARED_GEMS: '1'

README.md

@@ -133,10 +133,14 @@ rsnapshot::hosts:
     backup:
       '/var': './var'
     cron:
-      'daily':
-        'minute': '0-10'
-        'hour':   '1..5'
-
+      mailto: 'bar1@example.com'
+      daily:
+        minute: '0-10'
+        hour:   '1..5'
+  db1:
+    backup_scripts:
+      mysql:
+      misc:
 
 ```
 
@@ -240,10 +244,17 @@ Default is:
 Hash. Set time ranges for different backup levels. Each item (minute, hour...) allows for cron notation, an array to pick a random time from and a range to pick a random time from.
 The range notation is '$start..$end', so to pick a random hour from 8 pm to 2 am, you could set the hour of your desired backup level to 
 `[ '20..23','0..2' ]`
+For the range feature to work, hours >0 and <10 must not have a preceding zero. 
+Wrong: `00.09`
+Correct: `0..9`
+Also, you can set a mailto for each host, or globally now. The settings will be merged bottom to top, so if you override a setting in a hosts cron, it will have precedence over the global setting,
+which in turn has precedence over the default.
+
 Example:
 
 ```puppet
   $cron = {
+    mailto     => 'admin@example.com',
     hourly     => {
       minute   => '0..59',
       hour     => [ '20..23','0..2' ],
@@ -256,6 +267,7 @@ Or in hiera:
 
 ```yaml
 rsnapshot::cron:
+  mailto: 'admin@example.com'
   daily:
     minute: '20'
   weekly:
@@ -267,16 +279,32 @@ rsnapshot::cron:
 ```yaml
 rsnapshot::hosts:
   webserver:
+    cron:
+      mailto: 'support@example.com'
       daily:
         hour: [ '20..23','0..2' ]
       weekly:
         hour: [ '20..23','0..2' ]
+  
+  webhost:
+
+  customervm.provider.com:
+    backup_user: 'customer'
 ```
 
+`webhost`:  Mails will go to `admin@example.com` (from the global override).
+
+`webserver`: Mails will go to `support@example.com`.
+
+`customervm.provider.com`: The backup (and thus ssh) user will be `customer@customervm.provider.com`
+
+
+
 Hash is of the form:
 
 ```puppet
 $cron    =>{
+  mailto => param,
   daily => {
     minute => param,
     hour => param,
@@ -294,6 +322,7 @@ Default is:
 
 ```puppet
   $cron = {
+    mailto     => 'admin@example.com',
     hourly     => {
       minute   => '0..59',  # random from 0 to 59
       hour     => '*',      # you could also do:   ['21..23','0..4','5'],
@@ -363,20 +392,65 @@ Default is:
 #### `$backup_scripts`
 Additional scripts to create, possible values are: mysql, psql, misc
 
+`mysql`: used for mysql backups
+
+`psql`: used for postgresql backups
+
+`misc`: custom commands to run on the node
+
+You can set 
+
+`$dbbackup_user`:     backup user
+
+`$dbbackup_password`: password for the backup user
+
+`$dumper`:            path to the dump bin you wish to use
+
+`$dump_flags`:        flags for your dump bin
+
+`$ignore_dbs`:        databases to be ignored (the psql script ignores template and postgres databases by default)
+
+`$commands`:          array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section)
+
+See below for defaults
+
+NOTE: the psql and mysql scripts will SSH into your host and try and use $dumper.
+Make sure you have those tools installed on your DB hosts.
+
+Also, this module will try and use pbzip to compress your databases. You can install pbzip2 (and additional packages you might need) by passing an array to [$rsnapshot::package_name](#package_name)
+
+
 Default is:
 
 ```puppet
   $backup_scripts = {
     mysql               => {
       dbbackup_user     => 'root',
-      dbbackup_password => 'myFancyPassWord',
+      dbbackup_password => '',
+      dumper            => 'mysqldump',
+      dump_flags        => '--single-transaction --quick --routines --ignore-table=mysql.event',
+      ignore_dbs        => [ 'information_schema', 'performance_schema' ],
     },
     psql                => {
       dbbackup_user     => 'postgres',
       dbbackup_password => '',
+      dumper            => 'pg_dump',
+      dump_flags        => '-Fc',
+      ignore_dbs        => [],
+    },
+    misc         => {
+      commands   => $::osfamily ? {
+        'RedHat' =>  [
+          'rpm -qa --qf="%{name}," > packages.txt',
+        ],
+        'Debian' => [
+          'dpkg --get-selections > packages.txt',
+        ],
+        default => [],
       },
-    misc => {},
     }
+  }
+
 ```
 
 Configuration example:
@@ -395,16 +469,134 @@ rsnapshot::hosts:
     backup_scripts:
       mysql:
       psql:
+        dumper: '/usr/local/bin/pg_dump'
+        dump_flags: '-Fc'
+        ignore_dbs: [ 'db1', 'tmp_db' ]
+      misc:
   bazqux:de:
     backup_scripts:
       mysql:
         dbbackup_user: 'myuser'
         dbbackup_password: 'mypassword'
+      misc:
+        commands:
+          - 'cat /etc/hostname > hostname.txt'
+          - 'date > date.txt'
 ```
 
 This creates 
 - a mysql and a psql backup script for `foobar.com` using the credentials `dbbackup:hunter2` for mysql and `dbbackup:yeshorsebatterystaple` for psql
+- the psql script will use `/usr/local/bin/pg_dump` as the dump program with flags `-Fc`
+- it will ignore the postgres databases `db1` and `tmp_db` for postgres
 - a mysql backup script for `bazqux.de` using the credentials `myuser:mypassword`
+- a misc script for bazqux.de containing two commands to run on the node. the output will be redirected to hostname.txt and date.txt in the misc/ subfolder of the hosts backup directory (i.e. /snapshot_root/bazqux.de/daily.0/misc/hostname.txt)
+
+The scripts look like this:
+
+##### `bazqux.de`
+
+```bash
+#!/bin/bash
+host=bazqux.de
+user=myuser
+pass=mypassword
+
+dbs=( 
+      $(ssh -l root "$host" "mysql -u ${user} -p${pass} -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'")  
+    )
+
+for db in "${dbs[@]}"; do
+  ssh -l root "$host" "mysqldump --user=${user} --password=${pass} --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql"
+  wait
+  pbzip2 "$db".sql
+done      
+
+```
+
+```bash
+#!/bin/bash
+
+ssh bazqux.de 'cat /etc/hostname > hostname.txt'
+
+ssh bazqux.de 'date > date.txt'
+
+```
+
+
+
+##### `foobar.com`
+
+psql:
+
+```bash
+#!/bin/bash
+host=foobar.com
+user=dbbackup
+pass=yeshorsebatterystaple
+
+PGPASSWORD="$pass"
+dbs=( 
+      $(ssh -l root "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname ~ 'postgres|db1|tmp_db'\"" )
+    )
+
+for db in "${dbs[@]}"; do
+  ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql
+  wait
+  pbzip2 "$db".sql
+done
+```
+
+mysql:
+
+
+```bash
+#!/bin/bash
+host=foobar.com
+user=dbbackup
+pass=hunter2
+
+dbs=( 
+      $(ssh -l root "$host" "mysql -u ${user} -p${pass} -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'")  
+    )
+
+for db in "${dbs[@]}"; do
+  ssh -l root "$host" "mysqldump --user=${user} --password=${pass} --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql"
+  wait
+  pbzip2 "$db".sql
+done      
+
+```
+
+misc (assuming foobar.com is a RedHat node):
+
+```bash
+#!/bin/bash
+
+ssh foobar.com 'rpm -qa --qf "%{name}," > packages.txt'
+
+```
+
+##### another example with root user and empty password
+
+mysql with root user:
+
+```bash
+#!/bin/bash
+host=bazqux.de
+user=root
+password=
+
+dbs=( 
+      $(ssh -l root "$host" "mysql -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'")  
+    )
+
+for db in "${dbs[@]}"; do
+  ssh -l root "$host" "mysqldump --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql"
+  wait
+  pbzip2 "$db".sql
+done      
+
+```
 
 
 ### rsnapshot configuration variables
@@ -554,4 +746,4 @@ Norbert Varzariu (loomsen)
 
 ## Contributors
 Please see the [list of contributors.](https://github.com/loomsen/puppet-bloonix_agent/graphs/contributors)
-
+A big thank you to Hendrik Horeis <hendrik.horeis@gmail.com> for all his input and testing of this module.

manifests/config.pp

@@ -11,7 +11,7 @@ class rsnapshot::config (
   $lockpath               = pick($rsnapshot::lockpath, $rsnapshot::params::config_lockpath, '/var/run/rsnapshot')
   $conf_d                 = pick($rsnapshot::conf_d, $rsnapshot::params::conf_d, '/etc/rsnapshot')
   $snapshot_root          = pick($hosts['snapshot_root'], $rsnapshot::snapshot_root, '/backup')
-  $default_cron           = assert_empty_hash($::rsnapshot::cron)
+  $logpath                = pick($rsnapshot::logpath, $rsnapshot::params::config_logpath)
   # make sure lock path and conf path exist
   file { $conf_d:
     ensure => 'directory',
@@ -25,6 +25,10 @@ class rsnapshot::config (
   file { $snapshot_root:
     ensure => 'directory',
   }
+  file { $logpath:
+    ensure => directory,
+  }
+
   # custom function, if only a hostname is given as a param, this is an empty hash
   # the next loop would break as puppet does not allow to reassign variables
   # the function checks $hosts for elements like: 
@@ -32,7 +36,7 @@ class rsnapshot::config (
   $hosts_clean = assert_empty_hash($hosts)
 
   $hosts_clean.each |String $host, $hash | {
-    $backup_user            = pick($hash['backup_user'], $rsnapshot::params::config_backup_user)
+    $backup_user            = pick($hash['backup_user'], $rsnapshot::backup_user, $rsnapshot::params::config_backup_user, 'root')
     $default_backup_dirs    = pick($rsnapshot::default_backup, $rsnapshot::params::config_default_backup)
     $backup_levels          = pick($hash['backup_levels'], $rsnapshot::backup_levels, 'weekly')
     $backup                 = $hash['backup']
@@ -56,7 +60,6 @@ class rsnapshot::config (
     $linux_lvm_vgpath       = pick_undef($hash['linux_lvm_vgpath'], $rsnapshot::params::config_linux_lvm_vgpath)
     $linux_lvm_mountpath    = pick_undef($hash['linux_lvm_mountpath'], $rsnapshot::params::config_linux_lvm_mountpath)
     $no_create_root         = pick_undef($hash['no_create_root'], $rsnapshot::params::config_no_create_root)
-    $logpath                = pick($hash['logpath'], $rsnapshot::logpath, $rsnapshot::params::config_logpath)
     $verbose                = pick($hash['verbose'], $rsnapshot::params::config_verbose)
     $loglevel               = pick($hash['loglevel'], $rsnapshot::params::config_loglevel)
     $stop_on_stale_lockfile = pick_undef($hash['stop_on_stale_lockfile'], $rsnapshot::params::config_stop_on_stale_lockfile)
@@ -132,17 +135,18 @@ class rsnapshot::config (
       content => template('rsnapshot/rsnapshot.erb'),
     }
 
+    
+
     if has_key($hash, backup_scripts) {
-
-      $hash[backup_scripts].each |$script, $credentials| {
-
-        if is_hash($credentials) {
-          $dbbackup_user     = $credentials['dbbackup_user']
-          $dbbackup_password = $credentials['dbbackup_password']
-        } else {
-          $dbbackup_user     = $rsnapshot::default_backup_scripts[$script]['dbbackup_user']
-          $dbbackup_password = $rsnapshot::default_backup_scripts[$script]['dbbackup_password']
-        }
+      $hash[backup_scripts].each |$script, $scriptconf| {
+        $real_script       = deep_merge($rsnapshot::params::backup_scripts[$script], $rsnapshot::backup_scripts[$script], $hash[backup_scripts][$script])
+        $dbbackup_user     = $real_script[dbbackup_user]
+        $dbbackup_password = $real_script[dbbackup_password]
+        $dumper            = $real_script[dumper]
+        $dump_flags        = $real_script[dump_flags]
+        $ignore_dbs        = $real_script[ignore_dbs]
+        $compress          = $real_script[compress]
+        $commands          = $real_script[commands]
 
         concat::fragment { "${host}_${script}_backup":
         target  => $config,
@@ -162,60 +166,26 @@ class rsnapshot::config (
     concat { $cronfile:
     }
     # create cron files for each backup level
+    # merge possible cron definitions to one
+    $real_cron = deep_merge($rsnapshot::params::cron, $rsnapshot::cron, $hash[cron])
+    concat::fragment { "mailto for $host":
+      content => "#This file is managed by puppet\nMAILTO=${real_cron[mailto]}\n\n",
+      target  => $cronfile,
+      order   => 1,
+    }
+
     $backup_levels.each |String $level| {
+      $mailto   = $real_cron[mailto]
+      $minute   = rand_from_array($real_cron[$level][minute],   "${host}.${level}.minute")
+      $hour     = rand_from_array($real_cron[$level][hour],     "${host}.${level}.hour")
+      $monthday = rand_from_array($real_cron[$level][monthday], "${host}.${level}.monthday")
+      $month    = rand_from_array($real_cron[$level][month],    "${host}.${level}.month")
+      $weekday  = rand_from_array($real_cron[$level][weekday],  "${host}.${level}.weekday")
 
-      # allow to globally override ranges, create random numbers for backup_levels daily, weekly, monthly
-      if has_key($hash,cron){
-        if has_key($hash[cron], $level) {
-            $cron = $hash[cron][$level]
-          } else {
-            if has_key($default_cron, $level) {
-              $cron = $default_cron[$level]
-            } else {
-              $cron = $rsnapshot::params::cron[$level]
-            }
-        }
-      } else {
-        if has_key($default_cron, $level) {
-          $cron = $default_cron[$level]
-        } else {
-          $cron = $rsnapshot::params::cron[$level]
-        }
-      }
-
-      if has_key($cron, minute) {
-        $c_min                      = $cron[minute]
-      } else {
-        $c_min                      = $rsnapshot::params::cron[$level][minute]
-      }
-      if has_key($cron, hour) {
-        $c_hour                     = $cron[hour]
-      } else {
-        $c_hour                     = $rsnapshot::params::cron[$level][hour]
-      }
-      if has_key($cron, monthday) {
-        $c_monthday                 = $cron[monthday]
-      } else {
-        $c_monthday                 = $rsnapshot::params::cron[$level][monthday]
-      }
-      if has_key($cron, month) {
-        $c_month                    = $cron[month]
-      } else {
-        $c_month                    = $rsnapshot::params::cron[$level][month]
-      }
-      if has_key($cron, weekday)  {
-        $c_weekday                  = $cron[weekday]
-      } else {
-        $c_weekday                  = $rsnapshot::params::cron[$level][weekday]
-      }
-      $minute                           = rand_from_array($c_min, "${host}.${level}.minute")
-      $hour                             = rand_from_array($c_hour, "${host}.${level}.hour")
-      $monthday                         = rand_from_array($c_monthday, "${host}.${level}.monthday")
-      $month                            = rand_from_array($c_month, "${host}.${level}.month")
-      $weekday                          = rand_from_array($c_weekday, "${host}.${level}.weekday")
       concat::fragment { "${host}.${level}":
         target  => $cronfile,
         content => template('rsnapshot/cron.erb'),
+        order   => 2,
       }
     }
   }

manifests/init.pp

@@ -18,15 +18,18 @@ class rsnapshot (
   $exclude                       = $rsnapshot::params::config_exclude,
   $snapshot_root                 = $rsnapshot::params::config_snapshot_root,
   $backup_levels                 = $rsnapshot::params::config_backup_levels,
+  $cron_service_name             = $rsnapshot::params::cron_service_name,
 ) inherits rsnapshot::params {
 
   $default_backup_scripts = $rsnapshot::params::backup_scripts + $backup_scripts
   $default_exclude        = $rsnapshot::params::config_exclude + $exclude
   if $hosts {
     class { '::rsnapshot::install': }->
-    class { '::rsnapshot::config': }
+    class { '::rsnapshot::config': }~>
+    class { '::rsnapshot::service': }
     contain '::rsnapshot::install'
     contain '::rsnapshot::config'
+    contain '::rsnapshot::service'
   }
 }
 

manifests/params.pp

@@ -7,6 +7,11 @@ class rsnapshot::params {
   $config_backup_user            = 'root'
   $package_name                  = 'rsnapshot'
   $package_ensure                = 'present'
+  $cron_service_name             = $::osfamily ? {
+    'RedHat' => 'crond',
+    'Debian' => 'cron',
+    default  => '',
+    }
   $cron_dir                      = '/etc/cron.d'
   $config_backup_levels          = [ 'daily', 'weekly', 'monthly' ]
   $config_backup_defaults        = true
@@ -32,7 +37,7 @@ class rsnapshot::params {
   $config_logpath                = '/var/log/rsnapshot'
   $config_logfile                = '/var/log/rsnapshot.log'  # unused, we are logging to $logpath/$host.log
   $config_lockpath               = '/var/run/rsnapshot'
-  $config_snapshot_root          = '/backup/'
+  $config_snapshot_root          = '/backup'
   $config_no_create_root         = undef # bool, true or false
   $config_verbose                = '2'
   $config_loglevel               = '4'
@@ -62,6 +67,7 @@ class rsnapshot::params {
   }
   $config_backup_scripts         = {}
   $cron = {
+    mailto     => 'admin@example.com',
     hourly     => {
       minute   => '0..59',
       hour     => '*',      # you could also do:   ['21..23','0..4','5'],
@@ -94,12 +100,30 @@ class rsnapshot::params {
   $backup_scripts = {
     mysql               => {
       dbbackup_user     => 'root',
-      dbbackup_password => 'myFancyPassWord',
+      dbbackup_password => '',
+      dumper            => 'mysqldump',
+      dump_flags        => '--single-transaction --quick --routines --ignore-table=mysql.event',
+      ignore_dbs        => [ 'information_schema', 'performance_schema' ],
+      compress          => 'pbzip2',
     },
     psql                => {
       dbbackup_user     => 'postgres',
       dbbackup_password => '',
+      dumper            => 'pg_dump',
+      dump_flags        => '-Fc',
+      ignore_dbs        => [],
+      compress          => 'pbzip2',
+    },
+    misc         => {
+      commands   => $::osfamily ? {
+        'RedHat' =>  [
+          'rpm -qa --qf="%{name}," > packages.txt',
+        ],
+        'Debian' => [
+          'dpkg --get-selections > packages.txt',
+        ],
+        default => [],
       },
-    misc => {},
+    }
   }
 }

manifests/service.pp

@@ -0,0 +1,10 @@
+# == Class: rsnapshot::service
+#
+# Reloads cron 
+class rsnapshot::service {
+  service { $rsnapshot::cron_service_name:
+    ensure => running,
+  }
+
+}
+

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "loomsen-rsnapshot",
-  "version": "0.4.2",
+  "version": "1.0.0",
   "author": "loomsen",
   "summary": "Configures rsnapshot.",
   "license": "Apache-2.0",

templates/misc.sh.erb

@@ -1,10 +1,7 @@
 #!/bin/bash
-# add custom stuff here 
-# FIXME: remember to concat for custom stuff after the last line of the puppet generated script
-<% if @osfamily == 'RedHat' -%>
-ssh <%=@host-%> rpm -qa --qf='%{name},' > packages.txt
+#This file is managed by puppet
+#
+<%@commands.each do |command| -%>
+ssh -l <%=@backup_user-%> <%=@host-%> '<%=command-%>'
+
 <%end-%>
-#
-#
-#
-# ++++++++++++++++++++++++++++++++++++++++++++

templates/mysql.sh.erb

@@ -1,4 +1,5 @@
 #!/bin/bash
+# This file is managed by puppet
 # written by Norbert Varzariu <loomsen@gmail.com>
 # This Script is triggered by rsnapshot and dumps the specified dbs on the configured host.
 #
@@ -6,11 +7,24 @@ host=<%=@host%>
 user=<%=@dbbackup_user%>
 pass=<%=@dbbackup_password%>
 
-dbs=( $(mysql -h "$host" -u "$user" -p"$pass" -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d')  )
-
+<% if (@dbbackup_user == 'root' && @dbbackup_password == '') -%>
+dbs=( 
+      $(ssh -l <%=@backup_user-%> "$host" "mysql -e 'show databases' | sed '1d;<%@ignore_dbs.to_a.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) 
+    )
+<%else-%>
+dbs=( 
+      $(ssh -l <%=@backup_user-%> "$host" "mysql -u <%=@dbbackup_user-%> -p'<%=@dbbackup_password-%>' -e 'show databases' | sed '1d;<%@ignore_dbs.each do |db|-%>/<%=db-%>/d;<%end-%>'" )
+    )
+<%end-%>
 for db in "${dbs[@]}"; do
-  mysqldump --host="$host" --user="$user" --password="$pass" --single-transaction --quick --routines --ignore-table=mysql.event "$db" > "$db".sql
+<% if (@dbbackup_user == 'root' && @dbbackup_password = '' )-%>
+  ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> <%=@dump_flags-%> ${db}" > "$db".sql
+<%else-%>
+  ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%>  --user=<%=@dbbackup_user-%> --password='<%=@dbbackup_password-%>' <%=@dump_flags-%> ${db}" > "$db".sql
+<%end-%>
   wait
-  pbzip2 -p3 "$db".sql
+<% if @compress != '' -%>
+  <%=@compress-%> "$db".sql
+<%end-%>
 done
 

templates/psql.sh.erb

@@ -1,4 +1,5 @@
 #!/bin/bash
+# This file is managed by puppet
 # written by Norbert Varzariu <loomsen@gmail.com>
 # This Script is triggered by rsnapshot and dumps the specified dbs on the configured host.
 #
@@ -7,11 +8,14 @@ user=<%=@dbbackup_user%>
 pass=<%=@dbbackup_password%>
 
 PGPASSWORD="$pass"
-dbs=( $(psql -h "$host" -U "$user" -Atc "SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'")  )
-
+dbs=( 
+      $(ssh -l <%=@backup_user-%> "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND NOT datname ~ '<%@ignore_dbs.each do |db|-%><%if db == @ignore_dbs.last-%><%=db-%><%else-%><%=db-%>|<%end-%><%end-%>'\"")  
+    )
 for db in "${dbs[@]}"; do
-  ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql
+  ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> -U ${user} <%=@dump_flags-%> ${db}" > "$db".sql
   wait
-  pbzip2 -p3 "$db".sql
+<% if @compress != '' -%>
+  <%=@compress-%> "$db".sql
+<%end-%>
 done
 

templates/rsnapshot.erb

@@ -85,12 +85,12 @@ du_args	<%= @du_args %>
 <% if @one_fs  != '' -%>
 one_fs	<%= @one_fs_num %>
 <% end -%>
-<% if @include != '' -%>
+<% unless @real_include.empty? -%>
 <% if @include_file  != '' -%>
 include_file	<%= @include_file %>
 <% end -%>
 <% end -%>
-<% if @exclude != '' -%>
+<% unless @real_exclude.empty? -%>
 <% if @exclude_file != '' -%>
 exclude_file	<%= @exclude_file %>
 <% end -%>