From 942e9a76975f00f9c586c2d662031b4349f8a2a8 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 10:51:37 +0100 Subject: [PATCH 01/13] fix readme --- README.md | 2 ++ templates/mysql.sh.erb | 2 +- templates/psql.sh.erb | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a1fd3f2..c7267fe 100644 --- a/README.md +++ b/README.md @@ -365,6 +365,8 @@ Default is: #### `$backup_scripts` Additional scripts to create, possible values are: mysql, psql, misc +NOTE: this requires you to install the client packages you wish to use. + You can do this by passing an array to [$rsnapshot::package_name](#package_name) Default is: diff --git a/templates/mysql.sh.erb b/templates/mysql.sh.erb index f07f69d..d6f7b05 100644 --- a/templates/mysql.sh.erb +++ b/templates/mysql.sh.erb @@ -11,6 +11,6 @@ dbs=( $(mysql -h "$host" -u "$user" -p"$pass" -e 'show databases' | sed '1d;/inf for db in "${dbs[@]}"; do mysqldump --host="$host" --user="$user" --password="$pass" --single-transaction --quick --routines --ignore-table=mysql.event "$db" > "$db".sql wait - pbzip2 -p3 "$db".sql + pbzip2 "$db".sql done diff --git a/templates/psql.sh.erb b/templates/psql.sh.erb index 596de34..09296c8 100644 --- a/templates/psql.sh.erb +++ b/templates/psql.sh.erb @@ -12,6 +12,6 @@ dbs=( $(psql -h "$host" -U "$user" -Atc "SELECT datname FROM pg_database WHERE N for db in "${dbs[@]}"; do ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql wait - pbzip2 -p3 "$db".sql + pbzip2 "$db".sql done From 8ba0042a9481cf2ae18811a1ee15bd862ba974d3 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 13:17:03 +0100 Subject: [PATCH 02/13] parameterize backup scripts --- README.md | 44 +++++++++++++++++++++++++++++++++++++----- manifests/config.pp | 24 ++++++++++++----------- manifests/params.pp | 10 ++++++++-- templates/mysql.sh.erb | 17 +++++++++++++--- templates/psql.sh.erb | 7 ++++--- 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c7267fe..b4240e2 100644 --- a/README.md +++ b/README.md @@ -365,8 +365,18 @@ Default is: #### `$backup_scripts` Additional scripts to create, possible values are: mysql, psql, misc -NOTE: this requires you to install the client packages you wish to use. - You can do this by passing an array to [$rsnapshot::package_name](#package_name) + +NOTE: the psql and mysql scripts will SSH into your host and try and use pg_dump/mysqldump respectively. +Make sure you have those tools installed on your DB hosts. + +You can do this by passing an array to [$rsnapshot::package_name](#package_name) + +Example: +```yaml +rsnapshot::package_name: + - rsnapshot + - pbzip2 +``` Default is: @@ -414,6 +424,7 @@ This creates - a mysql backup script for `bazqux.de` using the credentials `myuser:mypassword` The scripts look like this: + mysql: ```bash @@ -422,10 +433,31 @@ host=bazqux.de user=myuser pass=mypassword -dbs=( $(mysql -h "$host" -u "$user" -p"$pass" -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d') ) +dbs=( + $(ssh -l root "$host" "mysql -u ${user} -p${pass} -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'") + ) for db in "${dbs[@]}"; do - mysqldump --host="$host" --user="$user" --password="$pass" --single-transaction --quick --routines --ignore-table=mysql.event "$db" > "$db".sql + ssh -l root "$host" "mysqldump --user=${user} --password=${pass} --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" + wait + pbzip2 -p3 "$db".sql +done + +``` + +mysql with root user: + +```bash +#!/bin/bash +host=bazqux.de +user=root + +dbs=( + $(ssh -l root "$host" "mysql -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'") + ) + +for db in "${dbs[@]}"; do + ssh -l root "$host" "mysqldump --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" wait pbzip2 -p3 "$db".sql done @@ -441,7 +473,9 @@ user=backupuser pass=password PGPASSWORD="$pass" -dbs=( $(psql -h "$host" -U "$user" -Atc "SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'") ) +dbs=( + $(ssh -l root "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'\"" ) + ) for db in "${dbs[@]}"; do ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql diff --git a/manifests/config.pp b/manifests/config.pp index 0a1d319..46d826b 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -135,21 +135,23 @@ class rsnapshot::config ( content => template('rsnapshot/rsnapshot.erb'), } + + if has_key($hash, backup_scripts) { + $hash[backup_scripts].each |$script, $scriptconf| { + $real_script = deep_merge($rsnapshot::params::backup_scripts[$script], $rsnapshot::backup_scripts[$script], $hash[backup_scripts][$script]) + notify { "$hash[backup_scripts][$script] for $host and $script": } + # notify { "$real_script for $host and $script": } + $dbbackup_user = $real_script[dbbackup_user] + $dbbackup_password = $real_script[dbbackup_password] + $dumper = $real_script[dumper] + $dump_flags = $real_script[dump_flags] + $ignore_dbs = $real_script[ignore_dbs] - $hash[backup_scripts].each |$script, $credentials| { - - if is_hash($credentials) { - $dbbackup_user = $credentials['dbbackup_user'] - $dbbackup_password = $credentials['dbbackup_password'] - } else { - $dbbackup_user = $rsnapshot::default_backup_scripts[$script]['dbbackup_user'] - $dbbackup_password = $rsnapshot::default_backup_scripts[$script]['dbbackup_password'] - } concat::fragment { "${host}_${script}_backup": - target => $config, - content => "backup_script ${conf_d}/${host}.${script}.sh ./${script}\n", + target => $config, + content => "backup_script ${conf_d}/${host}.${script}.sh ./${script}\n", } file { "${conf_d}/${host}.${script}.sh": diff --git a/manifests/params.pp b/manifests/params.pp index 6a26d1a..992bc59 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -93,13 +93,19 @@ class rsnapshot::params { }, } $backup_scripts = { - mysql => { + mysql => { dbbackup_user => 'root', - dbbackup_password => 'myFancyPassWord', + dbbackup_password => '', + dumper => 'mysqldump', + dump_flags => '--single-transaction --quick --routines --ignore-table=mysql.event', + ignore_dbs => [ 'information_schema', 'performance_schema' ], }, psql => { dbbackup_user => 'postgres', dbbackup_password => '', + dumper => 'pg_dump', + dump_flags => '-Fc', + ignore_dbs => [], }, misc => {}, } diff --git a/templates/mysql.sh.erb b/templates/mysql.sh.erb index d6f7b05..61fc790 100644 --- a/templates/mysql.sh.erb +++ b/templates/mysql.sh.erb @@ -6,10 +6,21 @@ host=<%=@host%> user=<%=@dbbackup_user%> pass=<%=@dbbackup_password%> -dbs=( $(mysql -h "$host" -u "$user" -p"$pass" -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d') ) - +<% if (@dbbackup_user == 'root' && @dbbackup_password == '') -%> +dbs=( + $(ssh -l root "$host" "mysql -e 'show databases' | sed '1d;<%@ignore_dbs.to_a.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) + ) +<%else-%> +dbs=( + $(ssh -l root "$host" "mysql -u <%=@dbbackup_user-%> -p'<%=@dbbackup_password-%>' -e 'show databases' | sed '1d;<%@ignore_dbs.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) + ) +<%end-%> for db in "${dbs[@]}"; do - mysqldump --host="$host" --user="$user" --password="$pass" --single-transaction --quick --routines --ignore-table=mysql.event "$db" > "$db".sql +<% if @dbbackup_user == 'root' -%> + ssh -l root "$host" "<%=@dumper-%> <%=@dump_flags-%> ${db}" > "$db".sql +<%else-%> + ssh -l root "$host" "<%=@dumper-%> --user=<%=@dbbackup_user-%> --password='<%=@dbbackup_password-%>' <%=@dump_flags-%> ${db}" > "$db".sql +<%end-%> wait pbzip2 "$db".sql done diff --git a/templates/psql.sh.erb b/templates/psql.sh.erb index 09296c8..01d47e8 100644 --- a/templates/psql.sh.erb +++ b/templates/psql.sh.erb @@ -7,10 +7,11 @@ user=<%=@dbbackup_user%> pass=<%=@dbbackup_password%> PGPASSWORD="$pass" -dbs=( $(psql -h "$host" -U "$user" -Atc "SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'") ) - +dbs=( + $(ssh "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND NOT datname ~ '<%@ignore_dbs.each do |db|-%><%if db == @ignore_dbs.last-%><%=db-%><%else-%><%=db-%>|<%end-%><%end-%>'\"") + ) for db in "${dbs[@]}"; do - ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql + ssh -l root "$host" "<%=@dumper-%> -U ${user} <%=@dump_flags-%> ${db}" > "$db".sql wait pbzip2 "$db".sql done From eb643443fd3c07dd8fb16def05212ea4c25a9cd1 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 14:07:06 +0100 Subject: [PATCH 03/13] update README --- README.md | 132 ++++++++++++++++++++++++++++++-------------- manifests/config.pp | 7 +++ manifests/params.pp | 1 + 3 files changed, 100 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index b4240e2..394f8d5 100644 --- a/README.md +++ b/README.md @@ -243,10 +243,14 @@ The range notation is '$start..$end', so to pick a random hour from 8 pm to 2 am For the range feature to work, hours >0 and <10 must not have a preceding zero. Wrong: `00.09` Correct: `0..9` +Also, you can set a mailto for each host, or globally now. The settings will be merged bottom to top, so if you override a setting in a hosts cron, it will have precedence over the global setting, +which in turn has precedence over the default. + Example: ```puppet $cron = { + mailto => 'admin@example.com', hourly => { minute => '0..59', hour => [ '20..23','0..2' ], @@ -259,6 +263,7 @@ Or in hiera: ```yaml rsnapshot::cron: + mailto: 'admin@example.com' daily: minute: '20' weekly: @@ -270,16 +275,24 @@ rsnapshot::cron: ```yaml rsnapshot::hosts: webserver: - daily: - hour: [ '20..23','0..2' ] - weekly: - hour: [ '20..23','0..2' ] + cron: + mailto: 'support@example.com' + daily: + hour: [ '20..23','0..2' ] + weekly: + hour: [ '20..23','0..2' ] + webhost: ``` +Mails for webhost will go to admin@example.com (from the global override), those for webserver will go to support@example.com. + + + Hash is of the form: ```puppet -$cron =>{ +$cron =>{ + mailto => param, daily => { minute => param, hour => param, @@ -297,6 +310,7 @@ Default is: ```puppet $cron = { + mailto => 'admin@example.com', hourly => { minute => '0..59', # random from 0 to 59 hour => '*', # you could also do: ['21..23','0..4','5'], @@ -365,33 +379,42 @@ Default is: #### `$backup_scripts` Additional scripts to create, possible values are: mysql, psql, misc +You can set +`$dbbackup_user` : backup user +`$dbbackup_password` : password for the backup user +`dumper` : path to the dump bin you wish to use +`dump_flags`: flags for your dump bin +`ignore_dbs` : databases to be ignored -NOTE: the psql and mysql scripts will SSH into your host and try and use pg_dump/mysqldump respectively. +See below for defaults + +NOTE: the psql and mysql scripts will SSH into your host and try and use $dumper. Make sure you have those tools installed on your DB hosts. -You can do this by passing an array to [$rsnapshot::package_name](#package_name) +Also, this module will try and use pbzip to compress your databases. You can install pbzip2 (and additional packages you might need) by passing an array to [$rsnapshot::package_name](#package_name) -Example: -```yaml -rsnapshot::package_name: - - rsnapshot - - pbzip2 -``` Default is: ```puppet $backup_scripts = { - mysql => { + mysql => { dbbackup_user => 'root', - dbbackup_password => 'myPassWord', + dbbackup_password => '', + dumper => 'mysqldump', + dump_flags => '--single-transaction --quick --routines --ignore-table=mysql.event', + ignore_dbs => [ 'information_schema', 'performance_schema' ], }, psql => { dbbackup_user => 'postgres', dbbackup_password => '', + dumper => 'pg_dump', + dump_flags => '-Fc', + ignore_dbs => [], }, misc => {}, } + ``` Configuration example: @@ -410,8 +433,9 @@ rsnapshot::hosts: backup_scripts: mysql: psql: - dbbackup_user: 'backupuser' - dbbackup_password: 'password' + dumper: '/usr/local/bin/pg_dump' + dump_flags: '-Fc' + ignore_dbs: [ 'db1', 'tmp_db' ] bazqux:de: backup_scripts: mysql: @@ -421,11 +445,13 @@ rsnapshot::hosts: This creates - a mysql and a psql backup script for `foobar.com` using the credentials `dbbackup:hunter2` for mysql and `dbbackup:yeshorsebatterystaple` for psql +- the psql script will use `/usr/local/bin/pg_dump` as the dump program with flags `-Fc` +- it will ignore the postgres databases `db1` and `tmp_db` for postgres - a mysql backup script for `bazqux.de` using the credentials `myuser:mypassword` The scripts look like this: -mysql: +##### `bazqux.de` ```bash #!/bin/bash @@ -440,17 +466,63 @@ dbs=( for db in "${dbs[@]}"; do ssh -l root "$host" "mysqldump --user=${user} --password=${pass} --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" wait - pbzip2 -p3 "$db".sql + pbzip2 "$db".sql done ``` +##### `foobar.com` + +psql: + +```bash +#!/bin/bash +host=foobar.com +user=dbbackup +pass=yeshorsebatterystaple + +PGPASSWORD="$pass" +dbs=( + $(ssh -l root "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname ~ 'postgres|db1|tmp_db'\"" ) + ) + +for db in "${dbs[@]}"; do + ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql + wait + pbzip2 "$db".sql +done +``` + +mysql: + + +```bash +#!/bin/bash +host=foobar.com +user=dbbackup +pass=hunter2 + +dbs=( + $(ssh -l root "$host" "mysql -u ${user} -p${pass} -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'") + ) + +for db in "${dbs[@]}"; do + ssh -l root "$host" "mysqldump --user=${user} --password=${pass} --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" + wait + pbzip2 "$db".sql +done + +``` + +##### another example with root user and empty password + mysql with root user: ```bash #!/bin/bash host=bazqux.de user=root +password= dbs=( $(ssh -l root "$host" "mysql -e 'show databases' | sed '1d;/information_schema/d;/performance_schema/d'") @@ -459,31 +531,11 @@ dbs=( for db in "${dbs[@]}"; do ssh -l root "$host" "mysqldump --single-transaction --quick --routines --ignore-table=mysql.event ${db}" > "${db}.sql" wait - pbzip2 -p3 "$db".sql + pbzip2 "$db".sql done ``` -psql: - -```bash -#!/bin/bash -host=foobar.com -user=backupuser -pass=password - -PGPASSWORD="$pass" -dbs=( - $(ssh -l root "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'\"" ) - ) - -for db in "${dbs[@]}"; do - ssh -l root "$host" "pg_dump -U ${user} -Fc ${db}" > "$db".sql - wait - pbzip2 -p3 "$db".sql -done -``` - ### rsnapshot configuration variables Please read up on the following in the [rsnapshot manpage](http://linux.die.net/man/1/rsnapshot) diff --git a/manifests/config.pp b/manifests/config.pp index 46d826b..81d3ca9 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -169,8 +169,14 @@ class rsnapshot::config ( # create cron files for each backup level # merge possible cron definitions to one $real_cron = deep_merge($rsnapshot::params::cron, $rsnapshot::cron, $hash[cron]) + concat::fragment { "mailto for $host": + content => "#This file is managed by puppet\nMAILTO=${real_cron[mailto]}\n\n", + target => $cronfile, + order => 1, + } $backup_levels.each |String $level| { + $mailto = $real_cron[mailto] $minute = rand_from_array($real_cron[$level][minute], "${host}.${level}.minute") $hour = rand_from_array($real_cron[$level][hour], "${host}.${level}.hour") $monthday = rand_from_array($real_cron[$level][monthday], "${host}.${level}.monthday") @@ -180,6 +186,7 @@ class rsnapshot::config ( concat::fragment { "${host}.${level}": target => $cronfile, content => template('rsnapshot/cron.erb'), + order => 2, } } } diff --git a/manifests/params.pp b/manifests/params.pp index 992bc59..a62a143 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -63,6 +63,7 @@ class rsnapshot::params { } $config_backup_scripts = {} $cron = { + mailto => 'admin@example.com', hourly => { minute => '0..59', hour => '*', # you could also do: ['21..23','0..4','5'], From 86073f42b19ee003d7c1e2a14a13bd23ed239699 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:01:32 +0100 Subject: [PATCH 04/13] add custom commands to misc script --- README.md | 39 +++++++++++++++++++++++++++++++++++++-- manifests/config.pp | 3 ++- manifests/params.pp | 20 ++++++++++++++++++-- templates/misc.sh.erb | 12 ++++-------- templates/mysql.sh.erb | 4 +++- templates/psql.sh.erb | 4 +++- 6 files changed, 67 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 394f8d5..915aa7a 100644 --- a/README.md +++ b/README.md @@ -385,7 +385,7 @@ You can set `dumper` : path to the dump bin you wish to use `dump_flags`: flags for your dump bin `ignore_dbs` : databases to be ignored - +`commands` : array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) See below for defaults NOTE: the psql and mysql scripts will SSH into your host and try and use $dumper. @@ -412,7 +412,16 @@ Default is: dump_flags => '-Fc', ignore_dbs => [], }, - misc => {}, + misc => { + commands => $::osfamily ? { + 'RedHat' => [ + 'rpm -qa --qf="%{name}," > packages.txt', + ], + 'Debian' => [ + 'dpkg --get-selections > packages.txt', + ], + default => [], + }, } ``` @@ -436,11 +445,16 @@ rsnapshot::hosts: dumper: '/usr/local/bin/pg_dump' dump_flags: '-Fc' ignore_dbs: [ 'db1', 'tmp_db' ] + misc: bazqux:de: backup_scripts: mysql: dbbackup_user: 'myuser' dbbackup_password: 'mypassword' + misc: + commands: + - 'cat /etc/hostname > hostname.txt' + - 'date > date.txt' ``` This creates @@ -448,6 +462,7 @@ This creates - the psql script will use `/usr/local/bin/pg_dump` as the dump program with flags `-Fc` - it will ignore the postgres databases `db1` and `tmp_db` for postgres - a mysql backup script for `bazqux.de` using the credentials `myuser:mypassword` +- a misc script for bazqux.de containing two commands to run on the node. the output will be redirected to hostname.txt and date.txt in the misc/ subfolder of the hosts backup directory (i.e. /snapshot_root/bazqux.de/daily.0/misc/hostname.txt) The scripts look like this: @@ -471,6 +486,17 @@ done ``` +```bash +#!/bin/bash + +ssh bazqux.de 'cat /etc/hostname > hostname.txt' + +ssh bazqux.de 'date > date.txt' + +``` + + + ##### `foobar.com` psql: @@ -514,6 +540,15 @@ done ``` +misc (assuming foobar.com is a RedHat node): + +```bash +#!/bin/bash + +ssh foobar.com 'rpm -qa --qf "%{name}," > packages.txt' + +``` + ##### another example with root user and empty password mysql with root user: diff --git a/manifests/config.pp b/manifests/config.pp index 81d3ca9..d04b941 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -147,7 +147,8 @@ class rsnapshot::config ( $dumper = $real_script[dumper] $dump_flags = $real_script[dump_flags] $ignore_dbs = $real_script[ignore_dbs] - + $compress = $real_script[compress] + $commands = $real_script[commands] concat::fragment { "${host}_${script}_backup": target => $config, diff --git a/manifests/params.pp b/manifests/params.pp index a62a143..254896e 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -7,7 +7,11 @@ class rsnapshot::params { $config_backup_user = 'root' $package_name = 'rsnapshot' $package_ensure = 'present' - $cron_service_name = 'crond' + $cron_service_name = $::osfamily ? { + 'RedHat' => 'crond', + 'Debian' => 'cron', + default => '', + } $cron_dir = '/etc/cron.d' $config_backup_levels = [ 'daily', 'weekly', 'monthly' ] $config_backup_defaults = true @@ -100,6 +104,7 @@ class rsnapshot::params { dumper => 'mysqldump', dump_flags => '--single-transaction --quick --routines --ignore-table=mysql.event', ignore_dbs => [ 'information_schema', 'performance_schema' ], + compress => 'pbzip2', }, psql => { dbbackup_user => 'postgres', @@ -107,7 +112,18 @@ class rsnapshot::params { dumper => 'pg_dump', dump_flags => '-Fc', ignore_dbs => [], + compress => 'pbzip2', }, - misc => {}, + misc => { + commands => $::osfamily ? { + 'RedHat' => [ + 'rpm -qa --qf="%{name}," > packages.txt', + ], + 'Debian' => [ + 'dpkg --get-selections > packages.txt', + ], + default => [], + }, + } } } diff --git a/templates/misc.sh.erb b/templates/misc.sh.erb index 8e19cf6..2940d97 100644 --- a/templates/misc.sh.erb +++ b/templates/misc.sh.erb @@ -1,10 +1,6 @@ #!/bin/bash -# add custom stuff here -# FIXME: remember to concat for custom stuff after the last line of the puppet generated script -<% if @osfamily == 'RedHat' -%> -ssh <%=@host-%> rpm -qa --qf='%{name},' > packages.txt + +<%@commands.each do |command| -%> +ssh <%=@host-%> '<%=command-%>' + <%end-%> -# -# -# -# ++++++++++++++++++++++++++++++++++++++++++++ diff --git a/templates/mysql.sh.erb b/templates/mysql.sh.erb index 61fc790..608bf15 100644 --- a/templates/mysql.sh.erb +++ b/templates/mysql.sh.erb @@ -22,6 +22,8 @@ for db in "${dbs[@]}"; do ssh -l root "$host" "<%=@dumper-%> --user=<%=@dbbackup_user-%> --password='<%=@dbbackup_password-%>' <%=@dump_flags-%> ${db}" > "$db".sql <%end-%> wait - pbzip2 "$db".sql +<% if @compress != '' -%> + <%=@compress-%> "$db".sql +<%end-%> done diff --git a/templates/psql.sh.erb b/templates/psql.sh.erb index 01d47e8..bc85740 100644 --- a/templates/psql.sh.erb +++ b/templates/psql.sh.erb @@ -13,6 +13,8 @@ dbs=( for db in "${dbs[@]}"; do ssh -l root "$host" "<%=@dumper-%> -U ${user} <%=@dump_flags-%> ${db}" > "$db".sql wait - pbzip2 "$db".sql +<% if @compress != '' -%> + <%=@compress-%> "$db".sql +<%end-%> done From 3bedb6efaa5596f32db16149f9123759b163349d Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:09:29 +0100 Subject: [PATCH 05/13] update README --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 915aa7a..1a61d0f 100644 --- a/README.md +++ b/README.md @@ -379,13 +379,27 @@ Default is: #### `$backup_scripts` Additional scripts to create, possible values are: mysql, psql, misc + +mysql: used for mysql backups + +psql: used for postgresql backups + +misc: custom commands to run on the node + You can set + `$dbbackup_user` : backup user + `$dbbackup_password` : password for the backup user + `dumper` : path to the dump bin you wish to use + `dump_flags`: flags for your dump bin + `ignore_dbs` : databases to be ignored + `commands` : array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) + See below for defaults NOTE: the psql and mysql scripts will SSH into your host and try and use $dumper. @@ -719,4 +733,4 @@ Norbert Varzariu (loomsen) ## Contributors Please see the [list of contributors.](https://github.com/loomsen/puppet-bloonix_agent/graphs/contributors) - +A big thank you to Hendrik Horeis for all his input and testing of this module. From eb7a73f4603a84a7adeeb59255cae6fe71aa3aa0 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:11:59 +0100 Subject: [PATCH 06/13] update README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1a61d0f..f18dbcd 100644 --- a/README.md +++ b/README.md @@ -388,17 +388,17 @@ misc: custom commands to run on the node You can set -`$dbbackup_user` : backup user +`$dbbackup_user`: backup user -`$dbbackup_password` : password for the backup user +`$dbbackup_password`: password for the backup user -`dumper` : path to the dump bin you wish to use +`$dumper`: path to the dump bin you wish to use -`dump_flags`: flags for your dump bin +`$dump_flags`: flags for your dump bin -`ignore_dbs` : databases to be ignored +`$ignore_dbs`: databases to be ignored (the psql script ignores template0 and template1 and postgres databases by default) -`commands` : array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) +`$commands`: array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) See below for defaults From 441c6a0584066586e7955950622f56bfd22485ee Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:16:55 +0100 Subject: [PATCH 07/13] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f18dbcd..2694397 100644 --- a/README.md +++ b/README.md @@ -396,7 +396,7 @@ You can set `$dump_flags`: flags for your dump bin -`$ignore_dbs`: databases to be ignored (the psql script ignores template0 and template1 and postgres databases by default) +`$ignore_dbs`: databases to be ignored (the psql script ignores template and postgres databases by default) `$commands`: array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) From 46a0a4faafa5d7273ee4a46e377add9da774c3f3 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:44:12 +0100 Subject: [PATCH 08/13] update README --- README.md | 37 +++++++++++++++++++++++++------------ manifests/config.pp | 2 +- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2694397..bed29f7 100644 --- a/README.md +++ b/README.md @@ -133,10 +133,14 @@ rsnapshot::hosts: backup: '/var': './var' cron: - 'daily': - 'minute': '0-10' - 'hour': '1..5' - + mailto: 'bar1@example.com' + daily: + minute: '0-10' + hour: '1..5' + db1: + backup_scripts: + mysql: + misc: ``` @@ -281,10 +285,18 @@ rsnapshot::hosts: hour: [ '20..23','0..2' ] weekly: hour: [ '20..23','0..2' ] + webhost: + + customervm.provider.com: + backup_user: 'customer' ``` -Mails for webhost will go to admin@example.com (from the global override), those for webserver will go to support@example.com. +`webhost`: Mails will go to `admin@example.com` (from the global override). + +`webserver`: Mails will go to `support@example.com`. + +`customervm.provider.com`: The backup (and thus ssh) user will be `customer@customervm.provider.com` @@ -380,11 +392,11 @@ Default is: #### `$backup_scripts` Additional scripts to create, possible values are: mysql, psql, misc -mysql: used for mysql backups +`mysql`: used for mysql backups -psql: used for postgresql backups +`psql`: used for postgresql backups -misc: custom commands to run on the node +`misc`: custom commands to run on the node You can set @@ -392,13 +404,13 @@ You can set `$dbbackup_password`: password for the backup user -`$dumper`: path to the dump bin you wish to use +`$dumper`: path to the dump bin you wish to use -`$dump_flags`: flags for your dump bin +`$dump_flags`: flags for your dump bin -`$ignore_dbs`: databases to be ignored (the psql script ignores template and postgres databases by default) +`$ignore_dbs`: databases to be ignored (the psql script ignores template and postgres databases by default) -`$commands`: array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) +`$commands`: array of commands to run on the host (this has no effect on psql and mysql scripts and is intended for your custom needs, see misc script section) See below for defaults @@ -436,6 +448,7 @@ Default is: ], default => [], }, + } } ``` diff --git a/manifests/config.pp b/manifests/config.pp index d04b941..9878cc4 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -36,7 +36,7 @@ class rsnapshot::config ( $hosts_clean = assert_empty_hash($hosts) $hosts_clean.each |String $host, $hash | { - $backup_user = pick($hash['backup_user'], $rsnapshot::params::config_backup_user) + $backup_user = pick($hash['backup_user'], $rsnapshot::backup_user, $rsnapshot::params::config_backup_user, 'root') $default_backup_dirs = pick($rsnapshot::default_backup, $rsnapshot::params::config_default_backup) $backup_levels = pick($hash['backup_levels'], $rsnapshot::backup_levels, 'weekly') $backup = $hash['backup'] From b9c0e0cebc4d2b6b98829a4e2d4a467c96f0f305 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:48:07 +0100 Subject: [PATCH 09/13] add headers to scripts --- templates/misc.sh.erb | 3 ++- templates/mysql.sh.erb | 1 + templates/psql.sh.erb | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/misc.sh.erb b/templates/misc.sh.erb index 2940d97..356882c 100644 --- a/templates/misc.sh.erb +++ b/templates/misc.sh.erb @@ -1,5 +1,6 @@ #!/bin/bash - +#This file is managed by puppet +# <%@commands.each do |command| -%> ssh <%=@host-%> '<%=command-%>' diff --git a/templates/mysql.sh.erb b/templates/mysql.sh.erb index 608bf15..59050ce 100644 --- a/templates/mysql.sh.erb +++ b/templates/mysql.sh.erb @@ -1,4 +1,5 @@ #!/bin/bash +# This file is managed by puppet # written by Norbert Varzariu # This Script is triggered by rsnapshot and dumps the specified dbs on the configured host. # diff --git a/templates/psql.sh.erb b/templates/psql.sh.erb index bc85740..cdbc79a 100644 --- a/templates/psql.sh.erb +++ b/templates/psql.sh.erb @@ -1,4 +1,5 @@ #!/bin/bash +# This file is managed by puppet # written by Norbert Varzariu # This Script is triggered by rsnapshot and dumps the specified dbs on the configured host. # From 03e385b2aa729250452617a49970b39f6be13fdc Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:50:22 +0100 Subject: [PATCH 10/13] clean up repo --- .bundle/config | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .bundle/config diff --git a/.bundle/config b/.bundle/config deleted file mode 100644 index 57d4641..0000000 --- a/.bundle/config +++ /dev/null @@ -1,3 +0,0 @@ ---- -BUNDLE_PATH: vendor/bundle -BUNDLE_DISABLE_SHARED_GEMS: '1' From 33e3fa685ae6e7ecafb5abdc6fc6b9910a3047d5 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:57:23 +0100 Subject: [PATCH 11/13] parameterize backup_user in scripts templates --- templates/misc.sh.erb | 2 +- templates/mysql.sh.erb | 10 +++++----- templates/psql.sh.erb | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/templates/misc.sh.erb b/templates/misc.sh.erb index 356882c..7185cc2 100644 --- a/templates/misc.sh.erb +++ b/templates/misc.sh.erb @@ -2,6 +2,6 @@ #This file is managed by puppet # <%@commands.each do |command| -%> -ssh <%=@host-%> '<%=command-%>' +ssh -l <%=@backup_user-%> <%=@host-%> '<%=command-%>' <%end-%> diff --git a/templates/mysql.sh.erb b/templates/mysql.sh.erb index 59050ce..4a81e20 100644 --- a/templates/mysql.sh.erb +++ b/templates/mysql.sh.erb @@ -9,18 +9,18 @@ pass=<%=@dbbackup_password%> <% if (@dbbackup_user == 'root' && @dbbackup_password == '') -%> dbs=( - $(ssh -l root "$host" "mysql -e 'show databases' | sed '1d;<%@ignore_dbs.to_a.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) + $(ssh -l <%=@backup_user-%> "$host" "mysql -e 'show databases' | sed '1d;<%@ignore_dbs.to_a.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) ) <%else-%> dbs=( - $(ssh -l root "$host" "mysql -u <%=@dbbackup_user-%> -p'<%=@dbbackup_password-%>' -e 'show databases' | sed '1d;<%@ignore_dbs.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) + $(ssh -l <%=@backup_user-%> "$host" "mysql -u <%=@dbbackup_user-%> -p'<%=@dbbackup_password-%>' -e 'show databases' | sed '1d;<%@ignore_dbs.each do |db|-%>/<%=db-%>/d;<%end-%>'" ) ) <%end-%> for db in "${dbs[@]}"; do -<% if @dbbackup_user == 'root' -%> - ssh -l root "$host" "<%=@dumper-%> <%=@dump_flags-%> ${db}" > "$db".sql +<% if (@dbbackup_user == 'root' && @dbbackup_password = '' )-%> + ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> <%=@dump_flags-%> ${db}" > "$db".sql <%else-%> - ssh -l root "$host" "<%=@dumper-%> --user=<%=@dbbackup_user-%> --password='<%=@dbbackup_password-%>' <%=@dump_flags-%> ${db}" > "$db".sql + ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> --user=<%=@dbbackup_user-%> --password='<%=@dbbackup_password-%>' <%=@dump_flags-%> ${db}" > "$db".sql <%end-%> wait <% if @compress != '' -%> diff --git a/templates/psql.sh.erb b/templates/psql.sh.erb index cdbc79a..0f3f9ab 100644 --- a/templates/psql.sh.erb +++ b/templates/psql.sh.erb @@ -9,10 +9,10 @@ pass=<%=@dbbackup_password%> PGPASSWORD="$pass" dbs=( - $(ssh "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND NOT datname ~ '<%@ignore_dbs.each do |db|-%><%if db == @ignore_dbs.last-%><%=db-%><%else-%><%=db-%>|<%end-%><%end-%>'\"") + $(ssh -l <%=@backup_user-%> "$host" "psql -U ${user} -Atc \"SELECT datname FROM pg_database WHERE NOT datistemplate AND NOT datname ~ '<%@ignore_dbs.each do |db|-%><%if db == @ignore_dbs.last-%><%=db-%><%else-%><%=db-%>|<%end-%><%end-%>'\"") ) for db in "${dbs[@]}"; do - ssh -l root "$host" "<%=@dumper-%> -U ${user} <%=@dump_flags-%> ${db}" > "$db".sql + ssh -l <%=@backup_user-%> "$host" "<%=@dumper-%> -U ${user} <%=@dump_flags-%> ${db}" > "$db".sql wait <% if @compress != '' -%> <%=@compress-%> "$db".sql From f67cd5a517c2bff22f35c9082d97aded6f65efe7 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 15:58:18 +0100 Subject: [PATCH 12/13] remove debug message --- manifests/config.pp | 2 -- 1 file changed, 2 deletions(-) diff --git a/manifests/config.pp b/manifests/config.pp index 9878cc4..702765f 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -140,8 +140,6 @@ class rsnapshot::config ( if has_key($hash, backup_scripts) { $hash[backup_scripts].each |$script, $scriptconf| { $real_script = deep_merge($rsnapshot::params::backup_scripts[$script], $rsnapshot::backup_scripts[$script], $hash[backup_scripts][$script]) - notify { "$hash[backup_scripts][$script] for $host and $script": } - # notify { "$real_script for $host and $script": } $dbbackup_user = $real_script[dbbackup_user] $dbbackup_password = $real_script[dbbackup_password] $dumper = $real_script[dumper] From 5cb6c2d2351ae4f1c4687e69cece7271b93ab315 Mon Sep 17 00:00:00 2001 From: Norbert Varzariu Date: Fri, 22 Jan 2016 16:03:46 +0100 Subject: [PATCH 13/13] bump version to 1.0.0 \o/ --- metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.json b/metadata.json index 7f75e13..3daf1d9 100644 --- a/metadata.json +++ b/metadata.json @@ -1,6 +1,6 @@ { "name": "loomsen-rsnapshot", - "version": "0.5.2", + "version": "1.0.0", "author": "loomsen", "summary": "Configures rsnapshot.", "license": "Apache-2.0",