mirror of https://github.com/sipwise/ngcpcfg.git
Switch away from the old daemon/wrapper architecture, which complicates things and requires more scaffolding code. We turn the daemon/wrapper and build_config into a single perl process that will load all YAML files, and then process each input/output pair on a parallel child, to try to speed up the processing as much as possible. For whole rebuilds, it might speed up the generation by at least a factor of x2, x4 or more, depending on the number of active processors. Change-Id: I51aa2f90336e34a20983d8733f45b64d9b6fea0bchanges/02/35902/18
Guillem Jover
5 years ago
parent
bfd9be075e
commit
a9166843c2
@ -1,156 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Filename: /usr/share/ngcp-ngcpcfg/helper/build_config
|
||||
# Purpose: builds output configuration file based on tt2 template file
|
||||
# using /usr/share/ngcp-ngcpcfg/helper/tt2-wrapper
|
||||
################################################################################
|
||||
|
||||
set -e
|
||||
set -u
|
||||
|
||||
if [ "${#:-}" -ne 3 ] ; then
|
||||
echo "Usage: /usr/share/ngcp-ngcpcfg/helper/build_config <input_file> <output_file> <tmp_file>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# support for testsuite, assume defaults if unset
|
||||
CONFIG_POOL="${CONFIG_POOL:-/etc}"
|
||||
FUNCTIONS="${FUNCTIONS:-/usr/share/ngcp-ngcpcfg/functions/}"
|
||||
HELPER="${HELPER:-/usr/share/ngcp-ngcpcfg/helper/}"
|
||||
|
||||
if ! [ -r "${FUNCTIONS}"/logs ] ; then
|
||||
printf "Error: %s/logs could not be read. Exiting.\n" "${FUNCTIONS}" >&2
|
||||
exit 1
|
||||
fi
|
||||
. "${FUNCTIONS}"/logs
|
||||
|
||||
# main script
|
||||
|
||||
input_file="$1" # like /etc/ngcp-config/templates/etc/mysql/my.cnf.tt2
|
||||
output_file="$2" # like /etc/mysql/my.cnf
|
||||
tmp_output_file="$3" # like /tmp/ngcpcfg.PID12345.KLJhiand/tmp_output_file
|
||||
|
||||
if [ -z "${input_file}" ] ; then
|
||||
log_error "Missing <input_file> parameter. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${output_file}" ] ; then
|
||||
log_error "Missing <output_file> parameter. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${tmp_output_file}" ] ; then
|
||||
log_error "Missing <tmp_output_file> parameter. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# export variable for usage within {pre,post}build scripts,
|
||||
# OUTPUT_DIRECTORY is for customization during testing
|
||||
if [ -n "${OUTPUT_DIRECTORY:-}" ] ; then
|
||||
log_debug "Using output directory ${OUTPUT_DIRECTORY}"
|
||||
export output_file="${OUTPUT_DIRECTORY}/${output_file}"
|
||||
else
|
||||
export output_file
|
||||
fi
|
||||
output_file_dirname="$(dirname "${output_file}")" # like /etc/mysql
|
||||
|
||||
# ensure we don't try to generate a file where a directory with same name exists already
|
||||
if [ -d "${output_file}" ] ; then
|
||||
log_error "Generating file ${output_file} not possible, it's an existing directory." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# pre-execution script in template store:
|
||||
if [ -r "${NGCPCTL_MAIN}/templates/${output_file}.prebuild" ] ; then
|
||||
log_info "Executing prebuild for ${output_file}"
|
||||
bash "${NGCPCTL_MAIN}/templates/${output_file}.prebuild"
|
||||
elif [ -r "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.prebuild" ] ; then
|
||||
log_info "Executing prebuild for ${output_file}"
|
||||
bash "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.prebuild"
|
||||
fi
|
||||
|
||||
# if output directory does not exist yet, create it
|
||||
if ! [ -d "${output_file_dirname}" ] ; then
|
||||
umask 0022 # directory permissions should be '755'
|
||||
mkdir -p "${output_file_dirname}"
|
||||
fi
|
||||
|
||||
# assume safe defaults
|
||||
umask 0077
|
||||
|
||||
# read host specific configuration file only if it exists
|
||||
[ -r "${HOST_CONFIG:-}" ] && host_conf="$HOST_CONFIG"
|
||||
|
||||
# read local config only if it exists
|
||||
[ -r "${LOCAL_CONFIG:-}" ] && local_conf="$LOCAL_CONFIG"
|
||||
|
||||
TT_WRAPPER="${HELPER}/tt2-wrapper"
|
||||
|
||||
# XXX: Docker breaks sane Unix expectations when moving a file into /etc/hosts,
|
||||
# as it creates a bind mount on that pathname. We need to use an implementation
|
||||
# that will fallback to use copy semantics in that case, but will default to
|
||||
# use rename semantics to avoid races on ETXTBSY on executable files.
|
||||
# <https://github.com/moby/moby/issues/22281>
|
||||
move()
|
||||
{
|
||||
local src="$1"
|
||||
local dst="$2"
|
||||
|
||||
if [ -e /.dockerinit ] || [ -e /.dockerenv ] ; then
|
||||
perl -MFile::Copy=mv \
|
||||
-E "mv('${src}', '${dst}') or die 'error: cannot move $src to $dst: \$!\n'"
|
||||
else
|
||||
mv "${src}" "${dst}"
|
||||
fi
|
||||
}
|
||||
|
||||
log_debug "Output file ${output_file} based on ${input_file}"
|
||||
log_debug "Executing: $TT_WRAPPER ${input_file} > ${tmp_output_file}"
|
||||
log_debug " and: move ${tmp_output_file} ${output_file}"
|
||||
# 1) We need to use «readlink -f» so that we do not destroy any symlink pointing
|
||||
# to the real file, which we were previously preserving while using «cat».
|
||||
# 2) Care about ">" below, do not trust tmp file you receive.
|
||||
if "$TT_WRAPPER" "${input_file}" > "${tmp_output_file}" 2>/dev/null &&
|
||||
! grep -q -E '^file error' "${tmp_output_file}" 2>/dev/null &&
|
||||
move "${tmp_output_file}" "$(readlink -f "${output_file}")" ; then
|
||||
log_info "Generating ${output_file}: OK"
|
||||
RC=0
|
||||
else
|
||||
log_error "Generating ${output_file} based on ${input_file}: FAILED"
|
||||
RC=1
|
||||
|
||||
if [[ -r "${tmp_output_file}" ]] && grep -q -E '^file error' "${tmp_output_file}" ; then
|
||||
log_error "from generated file:"
|
||||
log_error " $(grep -E '^file error' "${tmp_output_file}")"
|
||||
fi
|
||||
|
||||
log_info "NOTE: Check those files for valid syntax and encoding:"
|
||||
for f in "${input_file}" ${host_conf:-} ${local_conf:-} "$NGCPCTL_CONFIG" "${NETWORK_CONFIG:-}" "${EXTRA_CONFIG_FILES[@]}" "$CONSTANTS_CONFIG" ; do
|
||||
[ -r "$f" ] && log_info "$f"
|
||||
done
|
||||
log_info "Running /usr/share/ngcp-ngcpcfg/helper/tt2-wrapper <file>"
|
||||
log_info "or inspecting temporary ${tmp_output_file}"
|
||||
log_info "should provide more details."
|
||||
fi
|
||||
|
||||
if [ -L "$output_file" ] ; then
|
||||
log_warn "File $output_file is a symlink - NOT adjusting permissions"
|
||||
else
|
||||
# set permissions for generated config based on the ones of the template
|
||||
chmod --reference="${input_file}" "${output_file}"
|
||||
# finally drop all write permissions
|
||||
chmod a-w "${output_file}"
|
||||
fi
|
||||
|
||||
# post-execution script in template store:
|
||||
if [ -r "${NGCPCTL_MAIN}/templates/${output_file}.postbuild" ] ; then
|
||||
log_info "Executing postbuild for ${output_file}"
|
||||
bash "${NGCPCTL_MAIN}/templates/${output_file}.postbuild"
|
||||
elif [ -r "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.postbuild" ] ; then
|
||||
log_info "Executing postbuild for ${output_file}"
|
||||
bash "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.postbuild"
|
||||
fi
|
||||
|
||||
exit $RC
|
||||
|
||||
## END OF FILE #################################################################
|
||||
@ -1,91 +0,0 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use English;
|
||||
use Carp;
|
||||
use POSIX qw(setsid);
|
||||
use IO::Socket::UNIX;
|
||||
use Hash::Merge qw(merge);
|
||||
use YAML::XS qw(LoadFile);
|
||||
use Clone 'clone';
|
||||
use Getopt::Long;
|
||||
use NGCP::Template;
|
||||
|
||||
my $quiet = 0;
|
||||
|
||||
GetOptions("q|quiet" => \$quiet);
|
||||
|
||||
my $server_socket = get_server_socket();
|
||||
|
||||
daemonize();
|
||||
|
||||
handle_connections($server_socket, $quiet);
|
||||
|
||||
exit;
|
||||
|
||||
sub daemonize {
|
||||
my $NGCP_BASE_TT2 = $ENV{'NGCP_BASE_TT2'} //= '/';
|
||||
chdir $NGCP_BASE_TT2 or croak "Can't chdir to $NGCP_BASE_TT2: $ERRNO";
|
||||
open(STDIN, '<', '/dev/null') or croak "Can't read /dev/null: $ERRNO";
|
||||
open(STDOUT, '>', '/dev/null') or croak "Can't write to /dev/null: $ERRNO";
|
||||
defined(my $pid = fork) or croak "Can't fork: $ERRNO";
|
||||
exit if $pid;
|
||||
setsid() or croak "Can't start a new session: $ERRNO";
|
||||
open(STDERR, '>&', \*STDOUT) or croak "Can't dup stdout: $ERRNO";
|
||||
return;
|
||||
}
|
||||
|
||||
sub get_server_socket {
|
||||
my $NGCP_SOCKETFILE = $ENV{'NGCP_SOCKETFILE'} //= '/run/ngcpcfg.socket';
|
||||
|
||||
my $server = IO::Socket::UNIX->new(
|
||||
'Type' => SOCK_STREAM(),
|
||||
'Local' => $NGCP_SOCKETFILE,
|
||||
'Listen' => SOMAXCONN,
|
||||
);
|
||||
die "Error: can't setup tt2-daemon!\n$EVAL_ERROR\n$ERRNO\n" unless $server;
|
||||
|
||||
binmode $server => ":encoding(utf8)";
|
||||
|
||||
return $server;
|
||||
}
|
||||
|
||||
sub handle_connections {
|
||||
my $port = shift;
|
||||
my $quiet = shift;
|
||||
my $config = {};
|
||||
my %loaded_ymls = ();
|
||||
|
||||
while (my $client = $port->accept()) {
|
||||
my $input = <$client>;
|
||||
chomp $input;
|
||||
|
||||
my @argv = split(' ', $input) or
|
||||
print { $client } "Usage: echo '<template> [<config.yml> [<another.yml>]]' | netcat localhost 42042 \n";
|
||||
my $template = shift @argv;
|
||||
|
||||
foreach my $file (@argv) {
|
||||
next if exists $loaded_ymls{$file};
|
||||
$loaded_ymls{$file} = undef;
|
||||
|
||||
print { $client } "Loading $file in memory:" unless $quiet;
|
||||
my $hm = Hash::Merge->new('RIGHT_PRECEDENT');
|
||||
$config = $hm->merge($config, LoadFile($file));
|
||||
print { $client } " OK \n" unless $quiet;
|
||||
}
|
||||
|
||||
my $tt = NGCP::Template->new();
|
||||
|
||||
open my $fh, '<', $template or
|
||||
print { $client } "Unable to open file '$template' for reading: $ERRNO\n";
|
||||
$tt->process($fh, clone($config), $client) or
|
||||
print { $client } $tt->error;
|
||||
close $fh;
|
||||
|
||||
close $client;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
@ -0,0 +1,251 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use Getopt::Long qw(:config posix_default bundling_values no_ignorecase);
|
||||
use Cwd qw(realpath);
|
||||
use File::Basename;
|
||||
use File::Path qw(make_path);
|
||||
use File::Copy qw(mv);
|
||||
use Time::Piece;
|
||||
use POSIX qw(:sys_wait_h);
|
||||
use Hash::Merge qw(merge);
|
||||
use YAML::XS qw(LoadFile);
|
||||
use NGCP::Template;
|
||||
|
||||
my $HNAME = $ENV{HNAME} // '';
|
||||
my $TIME_FORMAT = $ENV{TIME_FORMAT} // '%F %T';
|
||||
$TIME_FORMAT =~ s/^\+//;
|
||||
|
||||
my %options = (
|
||||
help => sub { usage(); exit 0; },
|
||||
jobs => qx(nproc) // 1,
|
||||
);
|
||||
chomp $options{jobs};
|
||||
|
||||
GetOptions(\%options,
|
||||
'help|?',
|
||||
'quiet|q',
|
||||
'jobs|j:i',
|
||||
'config|c=s@',
|
||||
);
|
||||
|
||||
setup();
|
||||
exit process(%options);
|
||||
|
||||
sub usage {
|
||||
print <<HELP
|
||||
Usage: $0 [<option>...] (<input> <output>)...
|
||||
|
||||
Options:
|
||||
-c, --config <files> List of comma-separated config YAML files.
|
||||
Option can appear multiple times.
|
||||
-j, --jobs [<n>] Use up to <n> processing jobs (defaults to nproc).
|
||||
Missing argument means no limit of jobs.
|
||||
-q, --quiet Do not print progress information.
|
||||
-h, --help This help message.
|
||||
HELP
|
||||
}
|
||||
|
||||
sub output_prefix {
|
||||
my $t = Time::Piece->new;
|
||||
my $timestamp = $t->strftime($TIME_FORMAT);
|
||||
|
||||
return "$timestamp $HNAME";
|
||||
}
|
||||
|
||||
sub error {
|
||||
my $prefix = output_prefix();
|
||||
die "$prefix: Error: @_\n";
|
||||
}
|
||||
|
||||
sub warning {
|
||||
my $prefix = output_prefix();
|
||||
warn "$prefix: Warning: @_\n";
|
||||
}
|
||||
|
||||
sub info {
|
||||
return if $options{quiet};
|
||||
my $prefix = output_prefix();
|
||||
print "$prefix: @_\n";
|
||||
}
|
||||
|
||||
sub setup {
|
||||
my $NGCP_BASE_TT2 = $ENV{'NGCP_BASE_TT2'} //= '/';
|
||||
chdir $NGCP_BASE_TT2
|
||||
or error("Cannot chdir to $NGCP_BASE_TT2: $!");
|
||||
}
|
||||
|
||||
sub process_template {
|
||||
my ($tt, $config, $input, $output) = @_;
|
||||
|
||||
my $newfile = "$output.ngcpcfg-new";
|
||||
|
||||
open my $outfh, '>', $newfile
|
||||
or error("Cannot open template new file $newfile: $!");
|
||||
open my $infh, '<', $input
|
||||
or error("Cannot open file '$input' for reading: $!");
|
||||
$tt->process($infh, $config, $outfh)
|
||||
or error("Cannot process template '$input':\n " . $tt->error());
|
||||
close $infh;
|
||||
close $outfh;
|
||||
|
||||
# XXX: Docker breaks sane Unix expectations when moving a file into
|
||||
# /etc/hosts, as it creates a bind mount on that pathname. We need to
|
||||
# use an implementation that will fallback to use copy semantics in
|
||||
# that case, but will default to use rename semantics to avoid races
|
||||
# on ETXTBSY on executable files.
|
||||
# <https://github.com/moby/moby/issues/22281>
|
||||
#
|
||||
# In addition we need to dereference any target symlink, so that we do
|
||||
# not destroy any symlink pointing to the real file.
|
||||
my $target = realpath($output);
|
||||
mv($newfile, $target)
|
||||
or error("Cannot rename $newfile to $target: $!");
|
||||
}
|
||||
|
||||
sub process_input {
|
||||
my ($tt, $config, $input, $output) = @_;
|
||||
|
||||
# Export variable for usage within {pre,post}build scripts,
|
||||
# OUTPUT_DIRECTORY is for customization during testing.
|
||||
## no critic (Variables::RequireLocalizedPunctuationVars)
|
||||
if (length $ENV{OUTPUT_DIRECTORY}) {
|
||||
$output = "$ENV{OUTPUT_DIRECTORY}/$output";
|
||||
}
|
||||
$ENV{output_file} = $output;
|
||||
|
||||
# Ensure we do not try to generate a file where a directory with same
|
||||
# name exists already.
|
||||
if (-d $output) {
|
||||
error("Generating file $output not possible, it's an existing directory.");
|
||||
}
|
||||
|
||||
my $input_dirname = dirname($input);
|
||||
my $output_basename = basename($output);
|
||||
my $output_dirname = dirname($output);
|
||||
|
||||
# Execute prebuild script.
|
||||
for my $prebuild ((
|
||||
"$input_dirname/$output_basename.prebuild",
|
||||
"$input_dirname/ngcpcfg.prebuild")) {
|
||||
next unless -e $prebuild;
|
||||
|
||||
info("Executing prebuild for $output");
|
||||
system("bash $prebuild") == 0
|
||||
or error("Execution of prebuild script '$prebuild' failed: $?");
|
||||
last;
|
||||
}
|
||||
|
||||
# If output directory does not exist yet, create it
|
||||
if (not -d $output_dirname) {
|
||||
## no critic (ValuesAndExpressions::ProhibitLeadingZeros)
|
||||
make_path($output_dirname, { mode => 0755 });
|
||||
}
|
||||
|
||||
# Assume safe defaults.
|
||||
umask 0077;
|
||||
|
||||
eval {
|
||||
process_template($tt, $config, $input, $output);
|
||||
};
|
||||
if ($@) {
|
||||
warn $@;
|
||||
error("Generating $output based on $input: FAILED");
|
||||
} else {
|
||||
info("Generating $output: OK");
|
||||
}
|
||||
|
||||
if (-l $output) {
|
||||
warning("File $output is a symlink - NOT adjusting permissions");
|
||||
} else {
|
||||
# Set permissions for generated config based on the ones of the
|
||||
# template, plus dropping all write permissions.
|
||||
## no critic (ValuesAndExpressions::ProhibitLeadingZeros)
|
||||
my $mode = (stat $input)[2] & ~0222;
|
||||
|
||||
chmod $mode, $output;
|
||||
}
|
||||
|
||||
# Execute postbuild script.
|
||||
for my $postbuild ((
|
||||
"$input_dirname/$output_basename.postbuild",
|
||||
"$input_dirname/ngcpcfg.postbuild")) {
|
||||
next unless -e $postbuild;
|
||||
|
||||
info("Executing postbuid for $output");
|
||||
system("bash $postbuild") == 0
|
||||
or error("Execution of postbuild script '$postbuild' failed: $?");
|
||||
last;
|
||||
}
|
||||
}
|
||||
|
||||
sub process {
|
||||
my %options = @_;
|
||||
my $config = {};
|
||||
my %loaded_ymls = ();
|
||||
|
||||
my $visible_jobs = $options{jobs} || 'unlimited';
|
||||
info("Building configurations with $visible_jobs concurrent jobs");
|
||||
|
||||
foreach my $file (@{$options{config}}) {
|
||||
next if exists $loaded_ymls{$file};
|
||||
$loaded_ymls{$file} = undef;
|
||||
|
||||
my $prefix = output_prefix();
|
||||
print "$prefix: Loading $file in memory:" unless $options{quiet};
|
||||
my $hm = Hash::Merge->new('RIGHT_PRECEDENT');
|
||||
$config = $hm->merge($config, LoadFile($file));
|
||||
print " OK \n" unless $options{quiet};
|
||||
}
|
||||
|
||||
my $nprocs = 0;
|
||||
my $rc = 0;
|
||||
my $tt = NGCP::Template->new();
|
||||
|
||||
while (@ARGV) {
|
||||
my $input = shift @ARGV;
|
||||
my $output = shift @ARGV;
|
||||
|
||||
error('Missing input file') unless defined $input;
|
||||
error("Missing output file for $input") unless defined $output;
|
||||
|
||||
my $pid = fork;
|
||||
if (not defined $pid) {
|
||||
error("Cannot fork child process to process $input: $!");
|
||||
}
|
||||
if ($pid != 0) {
|
||||
# We are the parent.
|
||||
$nprocs++;
|
||||
|
||||
# If we have queued enough work, wait for some to finish.
|
||||
if ($options{jobs} > 0 && $nprocs >= $options{jobs}) {
|
||||
my $kid = waitpid(-1, 0);
|
||||
$nprocs-- if $kid > 0;
|
||||
$rc = 1 if $kid > 0 && $? != 0;
|
||||
}
|
||||
|
||||
# Queue more work if available.
|
||||
next;
|
||||
}
|
||||
|
||||
process_input($tt, $config, $input, $output);
|
||||
|
||||
exit 0;
|
||||
}
|
||||
|
||||
# Reap any remaining zombies.
|
||||
while (1) {
|
||||
my $pid = waitpid(-1, 0);
|
||||
last if $pid < 0;
|
||||
$nprocs--;
|
||||
$rc = 1 if $? != 0;
|
||||
}
|
||||
|
||||
if ($nprocs != 0) {
|
||||
warning("queued or reaped more jobs than expected, remaining $nprocs");
|
||||
}
|
||||
|
||||
return $rc;
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
NGCP_SOCKETFILE="${NGCP_SOCKETFILE:-/run/ngcpcfg.socket}"
|
||||
|
||||
if [ ! -e "$NGCP_SOCKETFILE" ] ; then
|
||||
echo "ERROR: Wrong ngcpcfg tt2-daemon socket file: $NGCP_SOCKETFILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$@" | netcat -U "$NGCP_SOCKETFILE"
|
||||
|
||||
exit $?
|
||||
@ -1,2 +0,0 @@
|
||||
file error - parse error - input file handle line 1: unexpected end of directive
|
||||
[% IF %]
|
||||
Loading…
Reference in new issue