diff --git a/acc-cleanup.conf b/acc-cleanup.conf new file mode 100644 index 0000000..6187e52 --- /dev/null +++ b/acc-cleanup.conf @@ -0,0 +1,93 @@ +# acc-cleanup.pl configuration file +# +############ +# +# This is a script style config file. The config consists of a series of +# statements, one per line, which are going to be executed in sequence. +# A statement can either just set a variable to some value, or perform an +# action. The behavior of action statement may be influenced by the value +# of certain variables. Variables retain their value until they're set to +# something else. +# +# Comments will only be recognized as such if the "#" character is the first +# character of the line. +# +# Variables can be set with a statement like this: +# variable = value +# For example, to set the variable called "foo" to the value "bar": +# foo = bar +# Leading and trailing whitespace are removed for both variable name and +# value. +# +# Variables have no default value. If an action statement requires a certain +# variable to be set and it isn't, then the script will exit with an error. +# Variables can be unset with the "unset" statement: +# unset foo +# +# If the script encounters an error at any point, it will stop executing the +# config and return an error. +# +############ +# +# Connect to a database: +# connect +# This statement will use the variables "host", "username" and "password" to +# connect to the database, all of which are optional as per the DBI +# specification. +# The database connection remains active up to the next "connect" statement. +# +# Create monthly backup tables: +# backup +# Moves old records out of the specified table and sorts them into monthly +# tables, which will be created as necessary using the same table name and a +# suffix of _YYYYMM. +# The following variables are used: +# "time-column": The name of the column in the table to use for determining +# which month a record belong to. Must be a "datetime" column. +# "batch": How many records to process within a single statement. If unset or +# <= 0, it does them all at once. +# "backup-months": How many months worth of records to keep in the table and +# not move into the monthly backup tables. Months are always processed as a +# whole and this specifies how many months to keep AT MOST. In other words, +# if the script is started on December 15th and this value is set to "2", then +# all of December and November is kept, and all of October will be backed up. +# "backup-retro": How many months to process for backups, going backwards in +# time. Using the example above, with this value set to "3", the months +# October, September and August would be backed up, while any older records +# would be left untouched. +# +# Archiving (dumping) old monthly backup tables: +# archive
+# Create an SQL dump out of too old tables created by the "backup" statement +# and drop them afterwards. Uses the following variables: +# "archive-months": Uses the same logic as the "backup-months" variable above. +# If set to "12" and the script was started on December 15th, it will start +# archiving with the December table of the previous year. Archiving continues +# month by month, going backwards in time, until the script encounters a +# missing table. +# "archive-target": Target directory for writing the SQL dump files. If +# explicitly specified as "/dev/null", then no actual archiving will be +# performed, but instead the tables will only be dropped. +# "compress": If set to "gzip", then gzip the dump files after creation. If +# unset, do not compress. +# "host", "username" and "password": As dumping is performed by an external +# command, those variables are reused from the "connect" statement. +# +# Cleaning up tables without backup: +# cleanup
+# Works just like the "backup" statement, but doesn't actually backup +# anything, but rather just deletes old records. Variables used: +# "time-column": See above. +# "batch": See above. +# "cleanup-days": Deleting everything older than this many days. +# +############ + +batch = 10000 +archive-target = /tmp +compress = gzip + +username = root +password = 1freibier! + +connect accounting diff --git a/acc-cleanup.pl b/acc-cleanup.pl new file mode 100755 index 0000000..bac25dc --- /dev/null +++ b/acc-cleanup.pl @@ -0,0 +1,187 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use DBI; + +#my $config_file = "/etc/ngcp-cleanup-tools/acc-cleanup.conf"; +my $config_file = "acc-cleanup.conf"; +open(CONFIG, "<", $config_file) or die("Program stopping, couldn't open the configuration file '$config_file'.\n"); + +######################################################################## + +my (%vars, $dbh); + +sub delete_loop { + my ($table, $mtable, $col, $mstart) = @_; + + my $limit = ''; + $vars{batch} && $vars{batch} > 0 and $limit = " limit $vars{batch}"; + + while (1) { + my $res = $dbh->selectcol_arrayref("select id from $table + where $col >= ? + and $col < date_add(?, interval 1 month) $limit", + undef, $mstart, $mstart); + + $res or last; + @$res or last; + + my $idlist = join(",", @$res); + $dbh->do("insert into $mtable select * from $table where id in ($idlist)") + or die("Failed to insert into monthly table $mtable"); + $dbh->do("delete from $table where id in ($idlist)") + or die("Failed to delete records out of $table"); + } +} + +sub archive_dump { + my ($table) = @_; + + my $month = $vars{"archive-months"}; + while (1) { + my $now = time(); + my $bt = $now - int(30.4375 * 86400 * $month); + my @bt = localtime($bt); + my $mtable = $table . "_" . sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1); + my $res = $dbh->selectcol_arrayref("show table status like ?", undef, $mtable); + ($res && @$res && $res->[0]) or last; + $month++; + if ($vars{"archive-target"} ne '/dev/null') { + my $target = $vars{"archive-target"} . "/$mtable." . sprintf('%04i%02i%02i%02i%02i%02i', $bt[5] + 1900, $bt[4] + 1, @bt[3,2,1,0]) . ".sql"; + + my @cmd = ('mysqldump'); + $vars{username} and push(@cmd, "-u" . $vars{username}); + $vars{password} and push(@cmd, "-p" . $vars{password}); + $vars{host} and push(@cmd, "-h" . $vars{host}); + push(@cmd, "--opt", $dbh->{private_db}, $mtable); + + for (@cmd) { s/'/'"'"'/g; $_ = "'$_'" } + my $cmd = join(' ', @cmd); + + if (system("$cmd > $target")) { + unlink($target); + die("MySQL DUMP of table $mtable into file $target failed\n"); + } + if ($vars{compress} && $vars{compress} eq 'gzip') { + if (system("nice gzip -9 $target")) { + unlink($target, "$target.gz"); + die("Gzipping of dump file $target failed\n"); + } + } + } + $dbh->do("drop table $mtable"); + } +} + +sub backup_table { + my ($table) = @_; + + for my $cmonth (0 .. ($vars{"backup-retro"} - 1)) { + my $tmonths = $cmonth + $vars{"backup-months"}; + my $bt = time() - int(30.4375 * 86400 * $tmonths); + my @bt = localtime($bt); + my $tstampl = sprintf('%04i-%02i', $bt[5] + 1900, $bt[4] + 1); + my $tstamp = sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1); + my $mstart = "$tstampl-01 00:00:00"; + + my $mtable = $table . "_$tstamp"; + $dbh->do("create table if not exists $mtable like $table"); + delete_loop($table, $mtable, $vars{"time-column"}, $mstart); + } + + return 1; +} + +sub cleanup { + my ($table) = @_; + + my $limit = ''; + $vars{batch} && $vars{batch} > 0 and $limit = " limit $vars{batch}"; + my $col = $vars{"time-column"}; + + while (1) { + my $aff = $dbh->do("delete from $table where $col < date_sub(now(), interval ? day) $limit", + undef, $vars{"cleanup-days"}); + $aff or die("Unable to delete records from $table"); + $aff == 0 and last; + } +} + +######################################################################## + +my %cmds; + +$cmds{connect} = sub { + my ($db) = @_; + + undef($dbh); + + $db or die("Missing DB name for connect command"); + + my $dbi = "dbi:mysql:$db"; + $vars{host} and $dbi .= ";host=$vars{host}"; + + $dbh = DBI->connect($dbi, $vars{username}, $vars{password}); + $dbh or die("Failed to connect to DB $db"); + + $dbh->{private_db} = $db; +}; + +$cmds{backup} = sub { + my ($table) = @_; + + $table or die("No table name given in backup command"); + $dbh or die("Not connected to a DB in backup command"); + $vars{"time-column"} or die("Variable time-column not set in backup command"); + $vars{"backup-months"} or die("Variable backup-months not set in backup command"); + $vars{"backup-retro"} or die("Variable backup-retro not set in backup command"); + + backup_table($table); +}; + +$cmds{archive} = sub { + my ($table) = @_; + + $table or die("No table name given in archive command"); + $dbh or die("Not connected to a DB in archive command"); + $vars{"archive-months"} or die("Variable archive-months not set in archive command"); + $vars{"archive-target"} or die("Variable archive-target not set in archive command"); + + archive_dump($table); +}; + +$cmds{cleanup} = sub { + my ($table) = @_; + + $table or die("No table name given in backup command"); + $dbh or die("Not connected to a DB in backup command"); + $vars{"time-column"} or die("Variable time-column not set in cleanup command"); + $vars{"cleanup-days"} or die("Variable cleanup-days not set in cleanup command"); + + cleanup($table); +}; + +while (my $line = ) { + $line =~ s/^\s*//s; + $line =~ s/\s*$//s; + + $line =~ /^#/ and next; + $line =~ /^$/ and next; + + if ($line =~ /^([\w-]+)\s*=\s*(\S*)$/) { + $vars{$1} = $2; + next; + } + + my ($cmd, $rest) = $line =~ /^([\w-]+)(?:\s+(.*?))?$/; + $cmd or die("Syntax error in config file: '$line'"); + + my $sub = $cmds{$cmd}; + $sub or die("Unrecognized statement '$cmd'"); + + my @rest; + $rest and @rest = split(/\s+/, $rest); + + $sub->($rest, \@rest); +} diff --git a/acc_backup.conf b/acc_backup.conf deleted file mode 100644 index c587c95..0000000 --- a/acc_backup.conf +++ /dev/null @@ -1,76 +0,0 @@ -# ngcp-cleanup-tools configuration file - -# ------ ACCOUNTING RECORDS BACKUP (acc_backup.pl) CONFIG ----- - -# Description of the logic and config options of the script: -# -# All tables specified by ACC_TABLES from the database ACC_DB will be -# processed, as well as all tables CDR_TABLES from the database CDR_DB. -# -# The script works in two steps: The first step moves old records out of the -# specified tables and into monthly backup tables, which will be created as -# necessary. The backup tables will have the same names as the source tables, -# with a suffix of "_YYYYMM". The script always performs this task on full -# months, i.e. first day of the month through last day of the month. The -# behavior of this step is controlled by the MONTHS and MONTHS_BACK config -# options. -# The second step is the archival of the older monthly backup tables. Archival -# simply consists of performing a mysqldump of a complete backup table, -# gzipping the dump file and then dropping the table. This step is controlled -# by the options ARCHIVE_MONTHS and ARCHIVE_DIR. - -# The MONTHS option specifies how many months worth of accounting records -# should be left untouched in the live DB tables and not be moved to the -# backup tables yet. As explained above, the script always works on complete -# months, so from today's date it will go back MONTHS months and start -# backing up the complete month from there. EXAMPLE: Assuming today is -# December 15th and MONTHS is set to 3: Going back 3 months from today gives -# the script September 15th, so the script will start to process the complete -# month of September and backup all accounting records from September 1st to -# September 30th. -# This value must be >= 1. -MONTHS = 3 -# Since there may be even older records in the DB tables than just from a -# single month (delayed or late entries, or the script hasn't been started in -# a while), the MONTHS_BACK options controls how many complete months should -# be processed. Using the example from above: If MONTHS_BACK is set to 6, then -# not only September will be processed, but also 5 more additional months, -# going backwards in time. In other words, all records between April 1st and -# September 30th (6 complete months) will be processed and moved to monthly backup -# tables. -# This value must be >= 1. -MONTHS_BACK = 6 - -# In the second step of the script, it will process all monthly backup tables -# older than ARCHIVE_MONTHS months and produce a gzipped SQL dump file in the -# target directory ARCHIVE_DIR. The DB tables will be dropped afterwards. This -# process continues for every month going backwards in time until there's no -# more backup tables left. -# EXAMPLE: If today is December 15th and ARCHIVE_MONTHS is set to 12, then the -# first table to be archived will be the December table from last year. After -# that, the November table will be archived, then October and so on, until the -# script tries to archive a non-existant table, at which point it stops. -# This value should be > (MONTHS + MONTHS_BACK) -ARCHIVE_MONTHS = 12 -ARCHIVE_DIR = /tmp - -# accounting database on proxies (openser) or db1 (accounting) -ACC_DB = accounting -# accounting tables on proxies (acc) or db1 (acc acc_backup acc_trash) -ACC_TABLES = acc,acc_backup,acc_trash -# cdr database (accounting) -CDR_DB = accounting -# cdr tables on proxies (empty) or db1 (cdr) -CDR_TABLES = cdr - -# How many entries to move and delete at the same time. Lower values make -# the script take longer to finish, higher values put more strain on the -# database (longer table locks or bigger transactions). -BATCH = 1000 - - -# ------ DATABASE ACCESS CONFIG ----- - -DBUSER = root -DBPASS = 1freibier! -DBHOST = localhost diff --git a/acc_backup.pl b/acc_backup.pl deleted file mode 100755 index e1117ac..0000000 --- a/acc_backup.pl +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; -use DBI; - -our $MONTHS; -our $MONTHS_BACK; -our $ARCHIVE_MONTHS; -our $ARCHIVE_DIR; -our $ACC_DB; -our $ACC_TABLES; -our $CDR_DB; -our $CDR_TABLES; -our $BATCH; -our $DBUSER; -our $DBPASS; -our $DBHOST; - - -my $config_file = "/etc/ngcp-cleanup-tools/acc_backup.conf"; -open CONFIG, "$config_file" or die "Program stopping, couldn't open the configuration file '$config_file'.\n"; - -while () { - chomp; # no newline - s/#.*//; # no comments - s/^\s+//; # no leading white - s/\s+$//; # no trailing white - next unless length; # anything left? - my ($var, $value) = split(/\s*=\s*/, $_, 2); - no strict 'refs'; - $$var = $value; -} -close CONFIG; - -my @ACC_TABLE_ARR = split(",",$ACC_TABLES); -my @CDR_TABLE_ARR = split(",",$CDR_TABLES); - -######################################################################## - -sub delete_loop { - my ($dbh, $table, $mtable, $col, $mstart) = @_; - - while (1) { - my $res = $dbh->selectcol_arrayref("select id from $table - where $col >= ? - and $col < date_add(?, interval 1 month) limit $BATCH", - undef, $mstart, $mstart); - - $res or last; - @$res or last; - - my $idlist = join(",", @$res); - $dbh->do("insert into $mtable select * from $table where id in ($idlist)") - or last; - $dbh->do("delete from $table where id in ($idlist)"); - } -} - -sub archive_dump { - my ($tables, $db) = @_; - - my $dbh = DBI->connect("dbi:mysql:$db;host=$DBHOST", $DBUSER, $DBPASS); - $dbh or return 0; - - for my $table (@$tables) { - my $month = $ARCHIVE_MONTHS; - while (1) { - my $now = time(); - my $bt = $now - int(30.4375 * 86400 * $month); - my @bt = localtime($bt); - my $mtable = $table . "_" . sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1); - my $res = $dbh->selectcol_arrayref("show table status like ?", undef, $mtable); - ($res && @$res && $res->[0]) or last; - $month++; - my $target = "$ARCHIVE_DIR/$mtable." . sprintf('%04i%02i%02i%02i%02i%02i', $bt[5] + 1900, $bt[4] + 1, @bt[3,2,1,0]) . ".sql"; - if (system("mysqldump -u$DBUSER -p$DBPASS -h$DBHOST --opt $db $mtable > $target")) { - print STDERR ("MySQL DUMP of table $mtable into file $target failed\n"); - next; - } - if (system("nice gzip -9 $target")) { - print STDERR ("Gzipping of dump file $target failed\n"); - unlink($target, "$target.gz"); - next; - } - $dbh->do("drop table $mtable"); - } - } - - return 1; -} - -sub backup_table { - my ($tables, $db, $col) = @_; - - my $dbh = DBI->connect("dbi:mysql:$db;host=$DBHOST", $DBUSER, $DBPASS); - $dbh or return 0; - - for my $cmonth (0 .. ($MONTHS_BACK - 1)) { - my $tmonths = $cmonth + $MONTHS; - my $bt = time() - int(30.4375 * 86400 * $tmonths); - my @bt = localtime($bt); - my $tstampl = sprintf('%04i-%02i', $bt[5] + 1900, $bt[4] + 1); - my $tstamp = sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1); - my $mstart = "$tstampl-01 00:00:00"; - - for my $table (@$tables) { - my $mtable = $table . "_$tstamp"; - - $dbh->do("create table if not exists $mtable like $table"); - - delete_loop($dbh, $table, $mtable, $col, $mstart); - } - } - - return 1; -} - -######################################################################## - - - -backup_table(\@ACC_TABLE_ARR, $ACC_DB, "time") or die(); -backup_table(\@CDR_TABLE_ARR, $CDR_DB, "start_time") or die(); - -archive_dump(\@ACC_TABLE_ARR, $ACC_DB); -archive_dump(\@CDR_TABLE_ARR, $CDR_DB); diff --git a/cleanup-tools.cron b/cleanup-tools.cron index 5ec9ec6..be15758 100644 --- a/cleanup-tools.cron +++ b/cleanup-tools.cron @@ -1,2 +1,2 @@ -00 03 * * * root /usr/sbin/acc_backup.pl > /dev/null 2>&1 -05 03 * * * root /usr/sbin/binlog-purge.pl > /dev/null 2>&1 +00 03 * * * root /usr/sbin/acc-cleanup.pl > /dev/null 2>&1 +#05 03 * * * root /usr/sbin/binlog-purge.pl > /dev/null 2>&1 diff --git a/debian/rules b/debian/rules index 3116973..19aea0d 100755 --- a/debian/rules +++ b/debian/rules @@ -32,10 +32,10 @@ install: build mkdir -p $(CURDIR)/debian/ngcp-cleanup-tools/etc/ngcp-cleanup-tools/ mkdir -p $(CURDIR)/debian/ngcp-cleanup-tools/etc/cron.d # Add here commands to install the package into debian/ngcp-cleanup-tools. - install -m 755 acc_backup.pl debian/ngcp-cleanup-tools/usr/sbin/acc_backup.pl - install -m 755 binlog-purge.pl debian/ngcp-cleanup-tools/usr/sbin/binlog-purge.pl - install -m 644 cleanup-tools.conf debian/ngcp-cleanup-tools/etc/ngcp-cleanup-tools/acc_backup.conf - install -m 644 cleanup-tools.conf debian/ngcp-cleanup-tools/etc/ngcp-cleanup-tools/binlog_purge.conf + install -m 755 acc-cleanup.pl debian/ngcp-cleanup-tools/usr/sbin/acc-cleanup.pl + #install -m 755 binlog-purge.pl debian/ngcp-cleanup-tools/usr/sbin/binlog-purge.pl + install -m 644 acc-cleanup.conf debian/ngcp-cleanup-tools/etc/ngcp-cleanup-tools/acc-cleanup.conf + #install -m 644 binlog-purge.conf debian/ngcp-cleanup-tools/etc/ngcp-cleanup-tools/binlog-purge.conf install -m 644 cleanup-tools.cron debian/ngcp-cleanup-tools/etc/cron.d/cleanup-tools # Build architecture-dependent files here.