TT#139455 support instances build

we need to produce different output for each instance defined
from the same source

* helper/instances-info, produces the needed output to feed
  helper/tt2-process with the proper arguments to build
  instances files. We inject ``instance_name`` with the
  proper value to templates and INSTANCE_NAME to hooks
  environment

* ${NGCPCTL_MAIN}/ngcpcfg.instances is where we define the
  supported instance templates and where the output path is

* helper/tt2-process, add -r --replace <path:path> option
  in order to allow generation of different output from
  the same source

Change-Id: I067266b5226485ea0d6c3bc3436275c5553a8177
mr10.2
Victor Seva 5 years ago committed by ngcp-config
parent f7d9d812a4
commit 83003649e3

@ -4,6 +4,7 @@ functions/logs usr/share/ngcp-ngcpcfg/functions/
functions/main usr/share/ngcp-ngcpcfg/functions/
helper/check-for-mysql usr/share/ngcp-ngcpcfg/helper/
helper/fileformat_version usr/share/ngcp-ngcpcfg/helper/
helper/instances-info usr/share/ngcp-ngcpcfg/helper/
helper/restore-permissions usr/share/ngcp-ngcpcfg/helper/
helper/sort-yml usr/share/ngcp-ngcpcfg/helper/
helper/sync-db usr/share/ngcp-ngcpcfg/helper/

@ -245,6 +245,17 @@ be executed under "bash $FILE", so you can use commands e.g. like
"ngcp-service foobar restart" and any further shell scripting syntax inside
the services files
NGCP instances
~~~~~~~~~~~~~~
ngcp-templates package provides the *TEMPLATE_INSTANCES* file defining what are
the supported templates for instances. Meaning that ngcpcfg will generate
one additional output per instance following the specifications provide by the
file.
ngcpcfg will generate that output with the template variable **instance_name** defined
and the environment variable **INSTANCE_NAME** set for ngcpcfg.service scripts
Global Options
--------------

@ -28,6 +28,9 @@ TEMPLATE_POOL_BASE="${NGCPCTL_MAIN}/templates"
# location of service definitions
SERVICES_POOL_BASE="${NGCPCTL_MAIN}/templates"
# location of instances info for templates
TEMPLATE_INSTANCES="${NGCPCTL_MAIN}/ngcpcfg.instances"
# Backward compatibility config for upgrade mr3.4*->mr3.5*
# it can be removed when the next LTS is released:
TEMPLATE_POOL="${TEMPLATE_POOL_BASE}/etc"

@ -0,0 +1,194 @@
#!/usr/bin/perl -CSD
# Purpose: format yaml instances file for bash script
################################################################################
use strict;
use warnings;
use YAML::XS;
use Getopt::Long qw(:config posix_default bundling_values no_ignorecase);
use Time::Piece;
use Data::Dumper;
use File::Spec;
use Storable qw(dclone);
my $DEBUG = $ENV{DEBUG} || 0;
my $HNAME = $ENV{HNAME} // '';
my $TIME_FORMAT = $ENV{TIME_FORMAT} // '%F %T';
$TIME_FORMAT =~ s/^\+//;
my $NETWORK_CONFIG = $ENV{NETWORK_CONFIG};
my $TEMPLATE_INSTANCES = $ENV{TEMPLATE_INSTANCES};
sub usage {
print <<HELP;
Usage: $0 [<option>...] <input>...
Options:
-h, --help This help message.
HELP
}
my %options = ( help => sub { usage(); exit 0; }, );
error("NETWORK_CONFIG is not defined") unless $NETWORK_CONFIG;
GetOptions( \%options, 'help|?', );
my $yaml = YAML::XS::LoadFile($NETWORK_CONFIG);
my $templ = YAML::XS::LoadFile($TEMPLATE_INSTANCES);
sub output_prefix {
my $t = Time::Piece->new;
my $timestamp = $t->strftime($TIME_FORMAT);
return "$timestamp $HNAME";
}
sub error {
my $prefix = output_prefix();
die "$prefix: Error: @_\n";
}
sub debug {
return unless $DEBUG;
my $prefix = output_prefix();
warn "$prefix: DEBUG @_\n";
}
my $instances = {};
my $dirs = {};
sub prefix_root {
my $path = shift;
return $path =~ s/^([^\/].*)/\/$1/r;
}
sub split_path {
my $input = shift;
my $val = ();
foreach(split /\//, $input){
error("relative paths are not supported, ${input}") if ($_ =~ '^\.\.?$');
if( defined $_ && !($_ =~ /^$/ )) {
push @{$val}, $_;
}
}
return $val;
}
sub show_match {
my $path;
my ($bpath, $spath, $suffix) = @_;
if(defined $spath) {
$path = $spath;
} else {
$path = $bpath;
}
foreach my $info (sort @{$dirs->{$path}}) {
my $dest;
if(defined $spath && defined $suffix) {
$dest = $info->{dest}."/${suffix}";
} else {
$dest = $info->{dest};
}
print "$bpath:$dest:".$info->{name}."\n";
}
}
sub prepare_info {
return 0 unless defined $yaml->{instances};
foreach my $instance ( sort @{ $yaml->{instances} } ) {
$instances->{ $instance->{label} } //= ();
push @{ $instances->{ $instance->{label} } }, $instance->{name};
}
foreach my $def ( sort @{ $templ->{sources} } ) {
my $inst = $instances->{ $def->{label} };
my $orig = prefix_root $def->{orig};
$dirs->{$orig} = ();
foreach ( sort @{$inst} ) {
my $dest = $def->{dest} =~ s/^(.*)\$\{INSTANCE_NAME\}(.*)$/$1$_$2/rg;
push @{ $dirs->{$orig} }, { name => $_, dest => prefix_root $dest};
}
}
debug("info:".Dumper({instances => $instances, dirs => $dirs}));
return 1;
}
sub match_source {
my ($build_path, $matched) = @_;
my $abs_path = File::Spec->rel2abs($build_path);
my $sbuild = split_path $abs_path;
my $size_build = $#{$sbuild};
my $search = keys %{$dirs};
debug("search ${build_path}");
foreach my $index (0 .. $size_build) {
my $b_part = @{$sbuild}[$index];
debug("index:${index} b_part:${b_part} b:".join '/', @{$sbuild}[0..$index]);
foreach my $source_path (sort keys %{$dirs}) {
next if defined $matched->{$source_path};
my $sorig = split_path $source_path;
my $size_orig = $#{$sorig};
my $i_part = defined(@{$sorig}[$index]) ? @{$sorig}[$index] : '';
my $equal = defined($i_part) ? $b_part eq $i_part : 0;
if (not $equal) {
$search--;
$matched->{$source_path} = 0;
debug("!! fail to match ${search} ${source_path}");
} elsif ($index eq $size_build) {
$search--;
$matched->{$source_path} = {build => $source_path, suffix=> undef};
debug(" match ${search} ${source_path}");
} elsif ($index eq $size_orig) {
$search--;
my $temp = join '/', @{$sbuild}[$index+1..$size_build];
$matched->{$source_path} = {
build => "${source_path}/$temp",
suffix => $temp,
};
debug(" match ${search} ${source_path}");
} else {
debug(" index:$index size_build:${size_build} size_orig:${size_orig} ${source_path}");
}
}
debug("***");
# nothing else to match
last unless $search;
}
# clean the non-matches for next round
foreach (sort keys %{$matched}) {
if ($matched->{$_}) {
show_match $matched->{$_}->{build}, $_, $matched->{$_}->{suffix};
} else {
delete($matched->{$_});
}
}
}
exit unless prepare_info;
if (@ARGV == 0) {
foreach (sort keys %{$dirs}) {
show_match $_;
}
} else {
my $matched;
foreach my $build (sort @ARGV) {
match_source $build, $matched;
}
}
exit;

@ -42,12 +42,19 @@ GetOptions(\%options,
'pairs|p',
'jobs|j:i',
'config|c=s@',
'replace|r=s',
);
if (exists $options{pairs} && @ARGV % 2 != 0) {
error("--pairs requires <input> <output> argument pairs");
}
if(exists $options{replace}) {
my ($orig, $dest) = split /:/, $options{replace};
$options{replace} = { orig => $orig, dest => $dest };
debug("replace orig:$orig dest:$dest");
}
setup();
exit process(%options);
@ -61,6 +68,7 @@ Options:
-j, --jobs [<n>] Use up to <n> processing jobs (defaults to nproc).
Missing argument means no limit of jobs.
-p, --pairs Expect the arguments to be <input> <output> pairs.
-r, --replace <path:path> Replace <input> path with <path> for output.
-q, --quiet Do not print progress information.
-h, --help This help message.
HELP
@ -207,6 +215,23 @@ sub process_input {
run_hook('postbuild', $file);
}
sub get_output_path {
my $file = shift;
my $output = ($file =~ s{\Q$NGCPCTL_MAIN\E/templates}{}r);
if(exists $options{replace}) {
my $orig = $options{replace}->{orig};
my $dest = $options{replace}->{dest};
$output = ($output =~ s{\Q$orig\E}{$dest}r);
}
# Add OUTPUT_DIRECTORY for customization during testing.
if (length $ENV{OUTPUT_DIRECTORY}) {
$output = "$ENV{OUTPUT_DIRECTORY}$output";
}
return $output;
}
sub generate_iofiles {
debug("Generating template file list from '$CONFIG_POOL'");
@ -297,11 +322,7 @@ sub generate_iofiles {
foreach my $ext (@match_ext) {
if (exists $filenames_scan{$file}{"$file$ext"}) {
my $input = "$file$ext";
my $output = ($file =~ s{\Q$NGCPCTL_MAIN\E/templates}{}r);
# Add OUTPUT_DIRECTORY for customization during testing.
if (length $ENV{OUTPUT_DIRECTORY}) {
$output = "$ENV{OUTPUT_DIRECTORY}$output";
}
my $output = get_output_path $file;
if ($file =~ m/ngcp-service/) {
push @filenames_prio, $input;

@ -48,12 +48,42 @@ for f in ${NGCPCTL_CONFIG:-} ${HA_CONFIG:-} ${PAIR_CONFIG:-} ${HOST_CONFIG:-} ${
fi
done
build_config_files_instances() {
local info
local source
local target
local instance
local inst_cfg
if [ -z "${TEMPLATE_INSTANCES:-}" ] || [ ! -f "${TEMPLATE_INSTANCES}" ] ; then
log_debug "no template info for instances, skip step"
return
fi
log_debug "${RUNNER} ${HELPER}/instances-info $*"
inst_cfg=$(mktemp)
while IFS=":" read -r -a info; do
source=${info[0]}
target=${info[1]}
instance=${info[2]}
log_debug "source:${source} target:${target} instance:${instance}"
cat <<EOF > "${inst_cfg}"
---
instance_name: ${instance}
EOF
log_debug "INSTANCE_NAME=${instance} ${RUNNER} ${HELPER}/tt2-process ${ARGS[*]} -c ${inst_cfg} -r ${source}:${target} ${source}"
INSTANCE_NAME="${instance}" ${RUNNER} "${HELPER}/tt2-process" "${ARGS[@]}" -c "${inst_cfg}" -r "${source}:${target}" "${source}"
done< <(${RUNNER} "${HELPER}/instances-info" "$@")
}
build_config_files() {
RUNNER="nice -n 19 ionice -c 3"
log_debug "${RUNNER} ${HELPER}/tt2-process ${ARGS[*]} $*"
${RUNNER} "${HELPER}/tt2-process" "${ARGS[@]}" "$@"
build_config_files_instances "$@"
record_commit_id
}

@ -10,8 +10,10 @@ set -u
# support testsuite
FUNCTIONS="${FUNCTIONS:-/usr/share/ngcp-ngcpcfg/functions/}"
OUTPUT_DIRECTORY="${OUTPUT_DIRECTORY:-}"
CLEANUP_FILES=()
INSTANCES=()
# load modules
@ -25,25 +27,55 @@ fi
# functions
instances_info() {
local target
local instance
local RUNNER="nice -n 19 ionice -c 3"
log_debug "${RUNNER} ${HELPER}/instances-info"
while IFS=":" read -r -a info; do
target=${info[0]}
instance=${info[2]}
log_debug "target:${target} instance:${instance}"
INSTANCES+=("${target}:${instance}")
done< <(${RUNNER} "${HELPER}/instances-info")
}
instance_env() {
local service_file=$1
local info
for instance in "${INSTANCES[@]}"; do
IFS=":" read -ra info <<< "${instance}"
if [[ "${service_file}" =~ ^${TEMPLATE_POOL_BASE}${info[0]} ]] ; then
log_debug "${service_file} matches ${info[1]}, scheduling instance restart"
exec_wrapper "${service_file}" "${info[1]}"
fi
done
}
exec_wrapper() {
local service_file=$1
local instance_name="${2:-}"
if ${DRYRUN} ; then
log_info "TEST MODE: Would execute action for ${line}"
log_info "TEST MODE: Would execute action for ${service_file}"
return 0
fi
log_info "Executing action for ${line}"
if [[ -x "${line}" ]]; then
log_debug "${line}"
if ! "${line}" ; then
log_warn "${line} returned with error code, continuing anyway."
log_info "Executing action for ${service_file}"
if [[ -x "${service_file}" ]]; then
log_debug "${service_file}"
if ! INSTANCE_NAME="${instance_name}" "${service_file}" ; then
log_warn "INSTANCE_NAME=${instance_name} ${service_file} returned with error code, continuing anyway."
fi
elif [[ -r "${line}" ]]; then
log_debug "bash ${line}"
if ! bash "${line}" ; then
log_warn "${line} returned with error code, continuing anyway."
elif [[ -r "${service_file}" ]]; then
log_debug "INSTANCE_NAME=${instance_name} bash ${service_file}"
if ! INSTANCE_NAME="${instance_name}" bash "${service_file}" ; then
log_warn "INSTANCE_NAME=${instance_name} ${service_file} returned with error code, continuing anyway."
fi
else
log_error "Error: ${line} could not be read."
log_error "Error: ${service_file} could not be read."
exit 1
fi
}
@ -94,6 +126,7 @@ services_start_queue()
execute() {
while read -r line ; do
exec_wrapper "${line}"
instance_env "${line}"
done < "${SORTED_LIST}"
}
@ -215,14 +248,20 @@ systemd_daemon_reload_preset
TMPFILE="$(mktemp)"
log_debug "TMPFILE = ${TMPFILE}"
log_debug "OUTPUT_DIRECTORY=${OUTPUT_DIRECTORY}"
CLEANUP_FILES+=("${TMPFILE}")
if [ -n "${TEMPLATE_INSTANCES:-}" ] && [ -f "${TEMPLATE_INSTANCES}" ] ; then
instances_info
fi
for dir in ${CONFIG_POOL} ; do
is_absolute_path "${dir}" || continue
is_non_git_folder "${dir}" || continue
is_absolute_path "${OUTPUT_DIRECTORY}${dir}" || continue
is_non_git_folder "${OUTPUT_DIRECTORY}${dir}" || continue
generate_list_to_process "${dir}"
generate_list_to_process "${OUTPUT_DIRECTORY}${dir}"
done
if [[ -s "${TMPFILE}" ]]; then

Loading…
Cancel
Save