Richard Fuchs
16 years ago
parent
ef61a22173
commit
4d0e21b34e
@ -0,0 +1,93 @@
|
||||
# acc-cleanup.pl configuration file
|
||||
#
|
||||
############
|
||||
#
|
||||
# This is a script style config file. The config consists of a series of
|
||||
# statements, one per line, which are going to be executed in sequence.
|
||||
# A statement can either just set a variable to some value, or perform an
|
||||
# action. The behavior of action statement may be influenced by the value
|
||||
# of certain variables. Variables retain their value until they're set to
|
||||
# something else.
|
||||
#
|
||||
# Comments will only be recognized as such if the "#" character is the first
|
||||
# character of the line.
|
||||
#
|
||||
# Variables can be set with a statement like this:
|
||||
# variable = value
|
||||
# For example, to set the variable called "foo" to the value "bar":
|
||||
# foo = bar
|
||||
# Leading and trailing whitespace are removed for both variable name and
|
||||
# value.
|
||||
#
|
||||
# Variables have no default value. If an action statement requires a certain
|
||||
# variable to be set and it isn't, then the script will exit with an error.
|
||||
# Variables can be unset with the "unset" statement:
|
||||
# unset foo
|
||||
#
|
||||
# If the script encounters an error at any point, it will stop executing the
|
||||
# config and return an error.
|
||||
#
|
||||
############
|
||||
#
|
||||
# Connect to a database:
|
||||
# connect <database name>
|
||||
# This statement will use the variables "host", "username" and "password" to
|
||||
# connect to the database, all of which are optional as per the DBI
|
||||
# specification.
|
||||
# The database connection remains active up to the next "connect" statement.
|
||||
#
|
||||
# Create monthly backup tables:
|
||||
# backup <table name>
|
||||
# Moves old records out of the specified table and sorts them into monthly
|
||||
# tables, which will be created as necessary using the same table name and a
|
||||
# suffix of _YYYYMM.
|
||||
# The following variables are used:
|
||||
# "time-column": The name of the column in the table to use for determining
|
||||
# which month a record belong to. Must be a "datetime" column.
|
||||
# "batch": How many records to process within a single statement. If unset or
|
||||
# <= 0, it does them all at once.
|
||||
# "backup-months": How many months worth of records to keep in the table and
|
||||
# not move into the monthly backup tables. Months are always processed as a
|
||||
# whole and this specifies how many months to keep AT MOST. In other words,
|
||||
# if the script is started on December 15th and this value is set to "2", then
|
||||
# all of December and November is kept, and all of October will be backed up.
|
||||
# "backup-retro": How many months to process for backups, going backwards in
|
||||
# time. Using the example above, with this value set to "3", the months
|
||||
# October, September and August would be backed up, while any older records
|
||||
# would be left untouched.
|
||||
#
|
||||
# Archiving (dumping) old monthly backup tables:
|
||||
# archive <table name>
|
||||
# Create an SQL dump out of too old tables created by the "backup" statement
|
||||
# and drop them afterwards. Uses the following variables:
|
||||
# "archive-months": Uses the same logic as the "backup-months" variable above.
|
||||
# If set to "12" and the script was started on December 15th, it will start
|
||||
# archiving with the December table of the previous year. Archiving continues
|
||||
# month by month, going backwards in time, until the script encounters a
|
||||
# missing table.
|
||||
# "archive-target": Target directory for writing the SQL dump files. If
|
||||
# explicitly specified as "/dev/null", then no actual archiving will be
|
||||
# performed, but instead the tables will only be dropped.
|
||||
# "compress": If set to "gzip", then gzip the dump files after creation. If
|
||||
# unset, do not compress.
|
||||
# "host", "username" and "password": As dumping is performed by an external
|
||||
# command, those variables are reused from the "connect" statement.
|
||||
#
|
||||
# Cleaning up tables without backup:
|
||||
# cleanup <table name>
|
||||
# Works just like the "backup" statement, but doesn't actually backup
|
||||
# anything, but rather just deletes old records. Variables used:
|
||||
# "time-column": See above.
|
||||
# "batch": See above.
|
||||
# "cleanup-days": Deleting everything older than this many days.
|
||||
#
|
||||
############
|
||||
|
||||
batch = 10000
|
||||
archive-target = /tmp
|
||||
compress = gzip
|
||||
|
||||
username = root
|
||||
password = 1freibier!
|
||||
|
||||
connect accounting
|
||||
@ -0,0 +1,187 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use DBI;
|
||||
|
||||
#my $config_file = "/etc/ngcp-cleanup-tools/acc-cleanup.conf";
|
||||
my $config_file = "acc-cleanup.conf";
|
||||
open(CONFIG, "<", $config_file) or die("Program stopping, couldn't open the configuration file '$config_file'.\n");
|
||||
|
||||
########################################################################
|
||||
|
||||
my (%vars, $dbh);
|
||||
|
||||
sub delete_loop {
|
||||
my ($table, $mtable, $col, $mstart) = @_;
|
||||
|
||||
my $limit = '';
|
||||
$vars{batch} && $vars{batch} > 0 and $limit = " limit $vars{batch}";
|
||||
|
||||
while (1) {
|
||||
my $res = $dbh->selectcol_arrayref("select id from $table
|
||||
where $col >= ?
|
||||
and $col < date_add(?, interval 1 month) $limit",
|
||||
undef, $mstart, $mstart);
|
||||
|
||||
$res or last;
|
||||
@$res or last;
|
||||
|
||||
my $idlist = join(",", @$res);
|
||||
$dbh->do("insert into $mtable select * from $table where id in ($idlist)")
|
||||
or die("Failed to insert into monthly table $mtable");
|
||||
$dbh->do("delete from $table where id in ($idlist)")
|
||||
or die("Failed to delete records out of $table");
|
||||
}
|
||||
}
|
||||
|
||||
sub archive_dump {
|
||||
my ($table) = @_;
|
||||
|
||||
my $month = $vars{"archive-months"};
|
||||
while (1) {
|
||||
my $now = time();
|
||||
my $bt = $now - int(30.4375 * 86400 * $month);
|
||||
my @bt = localtime($bt);
|
||||
my $mtable = $table . "_" . sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1);
|
||||
my $res = $dbh->selectcol_arrayref("show table status like ?", undef, $mtable);
|
||||
($res && @$res && $res->[0]) or last;
|
||||
$month++;
|
||||
if ($vars{"archive-target"} ne '/dev/null') {
|
||||
my $target = $vars{"archive-target"} . "/$mtable." . sprintf('%04i%02i%02i%02i%02i%02i', $bt[5] + 1900, $bt[4] + 1, @bt[3,2,1,0]) . ".sql";
|
||||
|
||||
my @cmd = ('mysqldump');
|
||||
$vars{username} and push(@cmd, "-u" . $vars{username});
|
||||
$vars{password} and push(@cmd, "-p" . $vars{password});
|
||||
$vars{host} and push(@cmd, "-h" . $vars{host});
|
||||
push(@cmd, "--opt", $dbh->{private_db}, $mtable);
|
||||
|
||||
for (@cmd) { s/'/'"'"'/g; $_ = "'$_'" }
|
||||
my $cmd = join(' ', @cmd);
|
||||
|
||||
if (system("$cmd > $target")) {
|
||||
unlink($target);
|
||||
die("MySQL DUMP of table $mtable into file $target failed\n");
|
||||
}
|
||||
if ($vars{compress} && $vars{compress} eq 'gzip') {
|
||||
if (system("nice gzip -9 $target")) {
|
||||
unlink($target, "$target.gz");
|
||||
die("Gzipping of dump file $target failed\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
$dbh->do("drop table $mtable");
|
||||
}
|
||||
}
|
||||
|
||||
sub backup_table {
|
||||
my ($table) = @_;
|
||||
|
||||
for my $cmonth (0 .. ($vars{"backup-retro"} - 1)) {
|
||||
my $tmonths = $cmonth + $vars{"backup-months"};
|
||||
my $bt = time() - int(30.4375 * 86400 * $tmonths);
|
||||
my @bt = localtime($bt);
|
||||
my $tstampl = sprintf('%04i-%02i', $bt[5] + 1900, $bt[4] + 1);
|
||||
my $tstamp = sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1);
|
||||
my $mstart = "$tstampl-01 00:00:00";
|
||||
|
||||
my $mtable = $table . "_$tstamp";
|
||||
$dbh->do("create table if not exists $mtable like $table");
|
||||
delete_loop($table, $mtable, $vars{"time-column"}, $mstart);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
sub cleanup {
|
||||
my ($table) = @_;
|
||||
|
||||
my $limit = '';
|
||||
$vars{batch} && $vars{batch} > 0 and $limit = " limit $vars{batch}";
|
||||
my $col = $vars{"time-column"};
|
||||
|
||||
while (1) {
|
||||
my $aff = $dbh->do("delete from $table where $col < date_sub(now(), interval ? day) $limit",
|
||||
undef, $vars{"cleanup-days"});
|
||||
$aff or die("Unable to delete records from $table");
|
||||
$aff == 0 and last;
|
||||
}
|
||||
}
|
||||
|
||||
########################################################################
|
||||
|
||||
my %cmds;
|
||||
|
||||
$cmds{connect} = sub {
|
||||
my ($db) = @_;
|
||||
|
||||
undef($dbh);
|
||||
|
||||
$db or die("Missing DB name for connect command");
|
||||
|
||||
my $dbi = "dbi:mysql:$db";
|
||||
$vars{host} and $dbi .= ";host=$vars{host}";
|
||||
|
||||
$dbh = DBI->connect($dbi, $vars{username}, $vars{password});
|
||||
$dbh or die("Failed to connect to DB $db");
|
||||
|
||||
$dbh->{private_db} = $db;
|
||||
};
|
||||
|
||||
$cmds{backup} = sub {
|
||||
my ($table) = @_;
|
||||
|
||||
$table or die("No table name given in backup command");
|
||||
$dbh or die("Not connected to a DB in backup command");
|
||||
$vars{"time-column"} or die("Variable time-column not set in backup command");
|
||||
$vars{"backup-months"} or die("Variable backup-months not set in backup command");
|
||||
$vars{"backup-retro"} or die("Variable backup-retro not set in backup command");
|
||||
|
||||
backup_table($table);
|
||||
};
|
||||
|
||||
$cmds{archive} = sub {
|
||||
my ($table) = @_;
|
||||
|
||||
$table or die("No table name given in archive command");
|
||||
$dbh or die("Not connected to a DB in archive command");
|
||||
$vars{"archive-months"} or die("Variable archive-months not set in archive command");
|
||||
$vars{"archive-target"} or die("Variable archive-target not set in archive command");
|
||||
|
||||
archive_dump($table);
|
||||
};
|
||||
|
||||
$cmds{cleanup} = sub {
|
||||
my ($table) = @_;
|
||||
|
||||
$table or die("No table name given in backup command");
|
||||
$dbh or die("Not connected to a DB in backup command");
|
||||
$vars{"time-column"} or die("Variable time-column not set in cleanup command");
|
||||
$vars{"cleanup-days"} or die("Variable cleanup-days not set in cleanup command");
|
||||
|
||||
cleanup($table);
|
||||
};
|
||||
|
||||
while (my $line = <CONFIG>) {
|
||||
$line =~ s/^\s*//s;
|
||||
$line =~ s/\s*$//s;
|
||||
|
||||
$line =~ /^#/ and next;
|
||||
$line =~ /^$/ and next;
|
||||
|
||||
if ($line =~ /^([\w-]+)\s*=\s*(\S*)$/) {
|
||||
$vars{$1} = $2;
|
||||
next;
|
||||
}
|
||||
|
||||
my ($cmd, $rest) = $line =~ /^([\w-]+)(?:\s+(.*?))?$/;
|
||||
$cmd or die("Syntax error in config file: '$line'");
|
||||
|
||||
my $sub = $cmds{$cmd};
|
||||
$sub or die("Unrecognized statement '$cmd'");
|
||||
|
||||
my @rest;
|
||||
$rest and @rest = split(/\s+/, $rest);
|
||||
|
||||
$sub->($rest, \@rest);
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
# ngcp-cleanup-tools configuration file
|
||||
|
||||
# ------ ACCOUNTING RECORDS BACKUP (acc_backup.pl) CONFIG -----
|
||||
|
||||
# Description of the logic and config options of the script:
|
||||
#
|
||||
# All tables specified by ACC_TABLES from the database ACC_DB will be
|
||||
# processed, as well as all tables CDR_TABLES from the database CDR_DB.
|
||||
#
|
||||
# The script works in two steps: The first step moves old records out of the
|
||||
# specified tables and into monthly backup tables, which will be created as
|
||||
# necessary. The backup tables will have the same names as the source tables,
|
||||
# with a suffix of "_YYYYMM". The script always performs this task on full
|
||||
# months, i.e. first day of the month through last day of the month. The
|
||||
# behavior of this step is controlled by the MONTHS and MONTHS_BACK config
|
||||
# options.
|
||||
# The second step is the archival of the older monthly backup tables. Archival
|
||||
# simply consists of performing a mysqldump of a complete backup table,
|
||||
# gzipping the dump file and then dropping the table. This step is controlled
|
||||
# by the options ARCHIVE_MONTHS and ARCHIVE_DIR.
|
||||
|
||||
# The MONTHS option specifies how many months worth of accounting records
|
||||
# should be left untouched in the live DB tables and not be moved to the
|
||||
# backup tables yet. As explained above, the script always works on complete
|
||||
# months, so from today's date it will go back MONTHS months and start
|
||||
# backing up the complete month from there. EXAMPLE: Assuming today is
|
||||
# December 15th and MONTHS is set to 3: Going back 3 months from today gives
|
||||
# the script September 15th, so the script will start to process the complete
|
||||
# month of September and backup all accounting records from September 1st to
|
||||
# September 30th.
|
||||
# This value must be >= 1.
|
||||
MONTHS = 3
|
||||
# Since there may be even older records in the DB tables than just from a
|
||||
# single month (delayed or late entries, or the script hasn't been started in
|
||||
# a while), the MONTHS_BACK options controls how many complete months should
|
||||
# be processed. Using the example from above: If MONTHS_BACK is set to 6, then
|
||||
# not only September will be processed, but also 5 more additional months,
|
||||
# going backwards in time. In other words, all records between April 1st and
|
||||
# September 30th (6 complete months) will be processed and moved to monthly backup
|
||||
# tables.
|
||||
# This value must be >= 1.
|
||||
MONTHS_BACK = 6
|
||||
|
||||
# In the second step of the script, it will process all monthly backup tables
|
||||
# older than ARCHIVE_MONTHS months and produce a gzipped SQL dump file in the
|
||||
# target directory ARCHIVE_DIR. The DB tables will be dropped afterwards. This
|
||||
# process continues for every month going backwards in time until there's no
|
||||
# more backup tables left.
|
||||
# EXAMPLE: If today is December 15th and ARCHIVE_MONTHS is set to 12, then the
|
||||
# first table to be archived will be the December table from last year. After
|
||||
# that, the November table will be archived, then October and so on, until the
|
||||
# script tries to archive a non-existant table, at which point it stops.
|
||||
# This value should be > (MONTHS + MONTHS_BACK)
|
||||
ARCHIVE_MONTHS = 12
|
||||
ARCHIVE_DIR = /tmp
|
||||
|
||||
# accounting database on proxies (openser) or db1 (accounting)
|
||||
ACC_DB = accounting
|
||||
# accounting tables on proxies (acc) or db1 (acc acc_backup acc_trash)
|
||||
ACC_TABLES = acc,acc_backup,acc_trash
|
||||
# cdr database (accounting)
|
||||
CDR_DB = accounting
|
||||
# cdr tables on proxies (empty) or db1 (cdr)
|
||||
CDR_TABLES = cdr
|
||||
|
||||
# How many entries to move and delete at the same time. Lower values make
|
||||
# the script take longer to finish, higher values put more strain on the
|
||||
# database (longer table locks or bigger transactions).
|
||||
BATCH = 1000
|
||||
|
||||
|
||||
# ------ DATABASE ACCESS CONFIG -----
|
||||
|
||||
DBUSER = root
|
||||
DBPASS = 1freibier!
|
||||
DBHOST = localhost
|
||||
@ -1,127 +0,0 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use DBI;
|
||||
|
||||
our $MONTHS;
|
||||
our $MONTHS_BACK;
|
||||
our $ARCHIVE_MONTHS;
|
||||
our $ARCHIVE_DIR;
|
||||
our $ACC_DB;
|
||||
our $ACC_TABLES;
|
||||
our $CDR_DB;
|
||||
our $CDR_TABLES;
|
||||
our $BATCH;
|
||||
our $DBUSER;
|
||||
our $DBPASS;
|
||||
our $DBHOST;
|
||||
|
||||
|
||||
my $config_file = "/etc/ngcp-cleanup-tools/acc_backup.conf";
|
||||
open CONFIG, "$config_file" or die "Program stopping, couldn't open the configuration file '$config_file'.\n";
|
||||
|
||||
while (<CONFIG>) {
|
||||
chomp; # no newline
|
||||
s/#.*//; # no comments
|
||||
s/^\s+//; # no leading white
|
||||
s/\s+$//; # no trailing white
|
||||
next unless length; # anything left?
|
||||
my ($var, $value) = split(/\s*=\s*/, $_, 2);
|
||||
no strict 'refs';
|
||||
$$var = $value;
|
||||
}
|
||||
close CONFIG;
|
||||
|
||||
my @ACC_TABLE_ARR = split(",",$ACC_TABLES);
|
||||
my @CDR_TABLE_ARR = split(",",$CDR_TABLES);
|
||||
|
||||
########################################################################
|
||||
|
||||
sub delete_loop {
|
||||
my ($dbh, $table, $mtable, $col, $mstart) = @_;
|
||||
|
||||
while (1) {
|
||||
my $res = $dbh->selectcol_arrayref("select id from $table
|
||||
where $col >= ?
|
||||
and $col < date_add(?, interval 1 month) limit $BATCH",
|
||||
undef, $mstart, $mstart);
|
||||
|
||||
$res or last;
|
||||
@$res or last;
|
||||
|
||||
my $idlist = join(",", @$res);
|
||||
$dbh->do("insert into $mtable select * from $table where id in ($idlist)")
|
||||
or last;
|
||||
$dbh->do("delete from $table where id in ($idlist)");
|
||||
}
|
||||
}
|
||||
|
||||
sub archive_dump {
|
||||
my ($tables, $db) = @_;
|
||||
|
||||
my $dbh = DBI->connect("dbi:mysql:$db;host=$DBHOST", $DBUSER, $DBPASS);
|
||||
$dbh or return 0;
|
||||
|
||||
for my $table (@$tables) {
|
||||
my $month = $ARCHIVE_MONTHS;
|
||||
while (1) {
|
||||
my $now = time();
|
||||
my $bt = $now - int(30.4375 * 86400 * $month);
|
||||
my @bt = localtime($bt);
|
||||
my $mtable = $table . "_" . sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1);
|
||||
my $res = $dbh->selectcol_arrayref("show table status like ?", undef, $mtable);
|
||||
($res && @$res && $res->[0]) or last;
|
||||
$month++;
|
||||
my $target = "$ARCHIVE_DIR/$mtable." . sprintf('%04i%02i%02i%02i%02i%02i', $bt[5] + 1900, $bt[4] + 1, @bt[3,2,1,0]) . ".sql";
|
||||
if (system("mysqldump -u$DBUSER -p$DBPASS -h$DBHOST --opt $db $mtable > $target")) {
|
||||
print STDERR ("MySQL DUMP of table $mtable into file $target failed\n");
|
||||
next;
|
||||
}
|
||||
if (system("nice gzip -9 $target")) {
|
||||
print STDERR ("Gzipping of dump file $target failed\n");
|
||||
unlink($target, "$target.gz");
|
||||
next;
|
||||
}
|
||||
$dbh->do("drop table $mtable");
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
sub backup_table {
|
||||
my ($tables, $db, $col) = @_;
|
||||
|
||||
my $dbh = DBI->connect("dbi:mysql:$db;host=$DBHOST", $DBUSER, $DBPASS);
|
||||
$dbh or return 0;
|
||||
|
||||
for my $cmonth (0 .. ($MONTHS_BACK - 1)) {
|
||||
my $tmonths = $cmonth + $MONTHS;
|
||||
my $bt = time() - int(30.4375 * 86400 * $tmonths);
|
||||
my @bt = localtime($bt);
|
||||
my $tstampl = sprintf('%04i-%02i', $bt[5] + 1900, $bt[4] + 1);
|
||||
my $tstamp = sprintf('%04i%02i', $bt[5] + 1900, $bt[4] + 1);
|
||||
my $mstart = "$tstampl-01 00:00:00";
|
||||
|
||||
for my $table (@$tables) {
|
||||
my $mtable = $table . "_$tstamp";
|
||||
|
||||
$dbh->do("create table if not exists $mtable like $table");
|
||||
|
||||
delete_loop($dbh, $table, $mtable, $col, $mstart);
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
########################################################################
|
||||
|
||||
|
||||
|
||||
backup_table(\@ACC_TABLE_ARR, $ACC_DB, "time") or die();
|
||||
backup_table(\@CDR_TABLE_ARR, $CDR_DB, "start_time") or die();
|
||||
|
||||
archive_dump(\@ACC_TABLE_ARR, $ACC_DB);
|
||||
archive_dump(\@CDR_TABLE_ARR, $CDR_DB);
|
||||
@ -1,2 +1,2 @@
|
||||
00 03 * * * root /usr/sbin/acc_backup.pl > /dev/null 2>&1
|
||||
05 03 * * * root /usr/sbin/binlog-purge.pl > /dev/null 2>&1
|
||||
00 03 * * * root /usr/sbin/acc-cleanup.pl > /dev/null 2>&1
|
||||
#05 03 * * * root /usr/sbin/binlog-purge.pl > /dev/null 2>&1
|
||||
|
||||
Loading…
Reference in new issue