diff --git a/debian/ngcp-ngcpcfg.install b/debian/ngcp-ngcpcfg.install index 36f9d8d7..fb125159 100644 --- a/debian/ngcp-ngcpcfg.install +++ b/debian/ngcp-ngcpcfg.install @@ -4,6 +4,7 @@ functions/logs usr/share/ngcp-ngcpcfg/functions/ functions/main usr/share/ngcp-ngcpcfg/functions/ helper/check-for-mysql usr/share/ngcp-ngcpcfg/helper/ helper/fileformat_version usr/share/ngcp-ngcpcfg/helper/ +helper/instances-info usr/share/ngcp-ngcpcfg/helper/ helper/restore-permissions usr/share/ngcp-ngcpcfg/helper/ helper/sort-yml usr/share/ngcp-ngcpcfg/helper/ helper/sync-db usr/share/ngcp-ngcpcfg/helper/ diff --git a/docs/ngcpcfg.txt b/docs/ngcpcfg.txt index 2dadd724..0a4262b8 100644 --- a/docs/ngcpcfg.txt +++ b/docs/ngcpcfg.txt @@ -245,6 +245,17 @@ be executed under "bash $FILE", so you can use commands e.g. like "ngcp-service foobar restart" and any further shell scripting syntax inside the services files +NGCP instances +~~~~~~~~~~~~~~ + +ngcp-templates package provides the *TEMPLATE_INSTANCES* file defining what are +the supported templates for instances. Meaning that ngcpcfg will generate +one additional output per instance following the specifications provide by the +file. + +ngcpcfg will generate that output with the template variable **instance_name** defined +and the environment variable **INSTANCE_NAME** set for ngcpcfg.service scripts + Global Options -------------- diff --git a/etc/ngcp-config/ngcpcfg.cfg b/etc/ngcp-config/ngcpcfg.cfg index bd9708c4..fd82b1d5 100644 --- a/etc/ngcp-config/ngcpcfg.cfg +++ b/etc/ngcp-config/ngcpcfg.cfg @@ -28,6 +28,9 @@ TEMPLATE_POOL_BASE="${NGCPCTL_MAIN}/templates" # location of service definitions SERVICES_POOL_BASE="${NGCPCTL_MAIN}/templates" +# location of instances info for templates +TEMPLATE_INSTANCES="${NGCPCTL_MAIN}/ngcpcfg.instances" + # Backward compatibility config for upgrade mr3.4*->mr3.5* # it can be removed when the next LTS is released: TEMPLATE_POOL="${TEMPLATE_POOL_BASE}/etc" diff --git a/helper/instances-info b/helper/instances-info new file mode 100755 index 00000000..4c7ef404 --- /dev/null +++ b/helper/instances-info @@ -0,0 +1,194 @@ +#!/usr/bin/perl -CSD +# Purpose: format yaml instances file for bash script +################################################################################ + +use strict; +use warnings; + +use YAML::XS; +use Getopt::Long qw(:config posix_default bundling_values no_ignorecase); +use Time::Piece; +use Data::Dumper; +use File::Spec; +use Storable qw(dclone); + +my $DEBUG = $ENV{DEBUG} || 0; +my $HNAME = $ENV{HNAME} // ''; +my $TIME_FORMAT = $ENV{TIME_FORMAT} // '%F %T'; +$TIME_FORMAT =~ s/^\+//; + +my $NETWORK_CONFIG = $ENV{NETWORK_CONFIG}; +my $TEMPLATE_INSTANCES = $ENV{TEMPLATE_INSTANCES}; + +sub usage { + print <...] ... + +Options: + -h, --help This help message. +HELP +} + +my %options = ( help => sub { usage(); exit 0; }, ); + +error("NETWORK_CONFIG is not defined") unless $NETWORK_CONFIG; + +GetOptions( \%options, 'help|?', ); + +my $yaml = YAML::XS::LoadFile($NETWORK_CONFIG); +my $templ = YAML::XS::LoadFile($TEMPLATE_INSTANCES); + + +sub output_prefix { + my $t = Time::Piece->new; + my $timestamp = $t->strftime($TIME_FORMAT); + + return "$timestamp $HNAME"; +} + + +sub error { + my $prefix = output_prefix(); + die "$prefix: Error: @_\n"; +} + + +sub debug { + return unless $DEBUG; + my $prefix = output_prefix(); + warn "$prefix: DEBUG @_\n"; +} + +my $instances = {}; +my $dirs = {}; + + +sub prefix_root { + my $path = shift; + return $path =~ s/^([^\/].*)/\/$1/r; +} + + +sub split_path { + my $input = shift; + my $val = (); + foreach(split /\//, $input){ + error("relative paths are not supported, ${input}") if ($_ =~ '^\.\.?$'); + if( defined $_ && !($_ =~ /^$/ )) { + push @{$val}, $_; + } + } + return $val; +} + + +sub show_match { + my $path; + my ($bpath, $spath, $suffix) = @_; + + if(defined $spath) { + $path = $spath; + } else { + $path = $bpath; + } + + foreach my $info (sort @{$dirs->{$path}}) { + my $dest; + if(defined $spath && defined $suffix) { + $dest = $info->{dest}."/${suffix}"; + } else { + $dest = $info->{dest}; + } + print "$bpath:$dest:".$info->{name}."\n"; + } +} + + +sub prepare_info { + return 0 unless defined $yaml->{instances}; + foreach my $instance ( sort @{ $yaml->{instances} } ) { + $instances->{ $instance->{label} } //= (); + push @{ $instances->{ $instance->{label} } }, $instance->{name}; + } + foreach my $def ( sort @{ $templ->{sources} } ) { + my $inst = $instances->{ $def->{label} }; + my $orig = prefix_root $def->{orig}; + $dirs->{$orig} = (); + foreach ( sort @{$inst} ) { + my $dest = $def->{dest} =~ s/^(.*)\$\{INSTANCE_NAME\}(.*)$/$1$_$2/rg; + push @{ $dirs->{$orig} }, { name => $_, dest => prefix_root $dest}; + } + } + debug("info:".Dumper({instances => $instances, dirs => $dirs})); + return 1; +} + + +sub match_source { + my ($build_path, $matched) = @_; + my $abs_path = File::Spec->rel2abs($build_path); + my $sbuild = split_path $abs_path; + my $size_build = $#{$sbuild}; + my $search = keys %{$dirs}; + + debug("search ${build_path}"); + foreach my $index (0 .. $size_build) { + my $b_part = @{$sbuild}[$index]; + debug("index:${index} b_part:${b_part} b:".join '/', @{$sbuild}[0..$index]); + foreach my $source_path (sort keys %{$dirs}) { + next if defined $matched->{$source_path}; + my $sorig = split_path $source_path; + my $size_orig = $#{$sorig}; + my $i_part = defined(@{$sorig}[$index]) ? @{$sorig}[$index] : ''; + my $equal = defined($i_part) ? $b_part eq $i_part : 0; + + if (not $equal) { + $search--; + $matched->{$source_path} = 0; + debug("!! fail to match ${search} ${source_path}"); + } elsif ($index eq $size_build) { + $search--; + $matched->{$source_path} = {build => $source_path, suffix=> undef}; + debug(" match ${search} ${source_path}"); + } elsif ($index eq $size_orig) { + $search--; + my $temp = join '/', @{$sbuild}[$index+1..$size_build]; + $matched->{$source_path} = { + build => "${source_path}/$temp", + suffix => $temp, + }; + debug(" match ${search} ${source_path}"); + } else { + debug(" index:$index size_build:${size_build} size_orig:${size_orig} ${source_path}"); + } + } + debug("***"); + + # nothing else to match + last unless $search; + } + + # clean the non-matches for next round + foreach (sort keys %{$matched}) { + if ($matched->{$_}) { + show_match $matched->{$_}->{build}, $_, $matched->{$_}->{suffix}; + } else { + delete($matched->{$_}); + } + } +} + +exit unless prepare_info; + +if (@ARGV == 0) { + foreach (sort keys %{$dirs}) { + show_match $_; + } +} else { + my $matched; + foreach my $build (sort @ARGV) { + match_source $build, $matched; + } +} +exit; + diff --git a/helper/tt2-process b/helper/tt2-process index c79bf5fa..a5943527 100755 --- a/helper/tt2-process +++ b/helper/tt2-process @@ -42,12 +42,19 @@ GetOptions(\%options, 'pairs|p', 'jobs|j:i', 'config|c=s@', + 'replace|r=s', ); if (exists $options{pairs} && @ARGV % 2 != 0) { error("--pairs requires argument pairs"); } +if(exists $options{replace}) { + my ($orig, $dest) = split /:/, $options{replace}; + $options{replace} = { orig => $orig, dest => $dest }; + debug("replace orig:$orig dest:$dest"); +} + setup(); exit process(%options); @@ -61,6 +68,7 @@ Options: -j, --jobs [] Use up to processing jobs (defaults to nproc). Missing argument means no limit of jobs. -p, --pairs Expect the arguments to be pairs. + -r, --replace Replace path with for output. -q, --quiet Do not print progress information. -h, --help This help message. HELP @@ -207,6 +215,23 @@ sub process_input { run_hook('postbuild', $file); } +sub get_output_path { + my $file = shift; + my $output = ($file =~ s{\Q$NGCPCTL_MAIN\E/templates}{}r); + + if(exists $options{replace}) { + my $orig = $options{replace}->{orig}; + my $dest = $options{replace}->{dest}; + $output = ($output =~ s{\Q$orig\E}{$dest}r); + } + + # Add OUTPUT_DIRECTORY for customization during testing. + if (length $ENV{OUTPUT_DIRECTORY}) { + $output = "$ENV{OUTPUT_DIRECTORY}$output"; + } + return $output; +} + sub generate_iofiles { debug("Generating template file list from '$CONFIG_POOL'"); @@ -297,11 +322,7 @@ sub generate_iofiles { foreach my $ext (@match_ext) { if (exists $filenames_scan{$file}{"$file$ext"}) { my $input = "$file$ext"; - my $output = ($file =~ s{\Q$NGCPCTL_MAIN\E/templates}{}r); - # Add OUTPUT_DIRECTORY for customization during testing. - if (length $ENV{OUTPUT_DIRECTORY}) { - $output = "$ENV{OUTPUT_DIRECTORY}$output"; - } + my $output = get_output_path $file; if ($file =~ m/ngcp-service/) { push @filenames_prio, $input; diff --git a/scripts/build b/scripts/build index 55e4dec6..82cefa69 100755 --- a/scripts/build +++ b/scripts/build @@ -48,12 +48,42 @@ for f in ${NGCPCTL_CONFIG:-} ${HA_CONFIG:-} ${PAIR_CONFIG:-} ${HOST_CONFIG:-} ${ fi done +build_config_files_instances() { + local info + local source + local target + local instance + local inst_cfg + + if [ -z "${TEMPLATE_INSTANCES:-}" ] || [ ! -f "${TEMPLATE_INSTANCES}" ] ; then + log_debug "no template info for instances, skip step" + return + fi + + log_debug "${RUNNER} ${HELPER}/instances-info $*" + inst_cfg=$(mktemp) + while IFS=":" read -r -a info; do + source=${info[0]} + target=${info[1]} + instance=${info[2]} + log_debug "source:${source} target:${target} instance:${instance}" + cat < "${inst_cfg}" +--- +instance_name: ${instance} +EOF + log_debug "INSTANCE_NAME=${instance} ${RUNNER} ${HELPER}/tt2-process ${ARGS[*]} -c ${inst_cfg} -r ${source}:${target} ${source}" + INSTANCE_NAME="${instance}" ${RUNNER} "${HELPER}/tt2-process" "${ARGS[@]}" -c "${inst_cfg}" -r "${source}:${target}" "${source}" + done< <(${RUNNER} "${HELPER}/instances-info" "$@") +} + build_config_files() { RUNNER="nice -n 19 ionice -c 3" log_debug "${RUNNER} ${HELPER}/tt2-process ${ARGS[*]} $*" ${RUNNER} "${HELPER}/tt2-process" "${ARGS[@]}" "$@" + build_config_files_instances "$@" + record_commit_id } diff --git a/scripts/services b/scripts/services index 87255c81..e32351c2 100755 --- a/scripts/services +++ b/scripts/services @@ -10,8 +10,10 @@ set -u # support testsuite FUNCTIONS="${FUNCTIONS:-/usr/share/ngcp-ngcpcfg/functions/}" +OUTPUT_DIRECTORY="${OUTPUT_DIRECTORY:-}" CLEANUP_FILES=() +INSTANCES=() # load modules @@ -25,25 +27,55 @@ fi # functions +instances_info() { + local target + local instance + local RUNNER="nice -n 19 ionice -c 3" + + log_debug "${RUNNER} ${HELPER}/instances-info" + while IFS=":" read -r -a info; do + target=${info[0]} + instance=${info[2]} + log_debug "target:${target} instance:${instance}" + INSTANCES+=("${target}:${instance}") + done< <(${RUNNER} "${HELPER}/instances-info") +} + +instance_env() { + local service_file=$1 + local info + + for instance in "${INSTANCES[@]}"; do + IFS=":" read -ra info <<< "${instance}" + if [[ "${service_file}" =~ ^${TEMPLATE_POOL_BASE}${info[0]} ]] ; then + log_debug "${service_file} matches ${info[1]}, scheduling instance restart" + exec_wrapper "${service_file}" "${info[1]}" + fi + done +} + exec_wrapper() { + local service_file=$1 + local instance_name="${2:-}" + if ${DRYRUN} ; then - log_info "TEST MODE: Would execute action for ${line}" + log_info "TEST MODE: Would execute action for ${service_file}" return 0 fi - log_info "Executing action for ${line}" - if [[ -x "${line}" ]]; then - log_debug "${line}" - if ! "${line}" ; then - log_warn "${line} returned with error code, continuing anyway." + log_info "Executing action for ${service_file}" + if [[ -x "${service_file}" ]]; then + log_debug "${service_file}" + if ! INSTANCE_NAME="${instance_name}" "${service_file}" ; then + log_warn "INSTANCE_NAME=${instance_name} ${service_file} returned with error code, continuing anyway." fi - elif [[ -r "${line}" ]]; then - log_debug "bash ${line}" - if ! bash "${line}" ; then - log_warn "${line} returned with error code, continuing anyway." + elif [[ -r "${service_file}" ]]; then + log_debug "INSTANCE_NAME=${instance_name} bash ${service_file}" + if ! INSTANCE_NAME="${instance_name}" bash "${service_file}" ; then + log_warn "INSTANCE_NAME=${instance_name} ${service_file} returned with error code, continuing anyway." fi else - log_error "Error: ${line} could not be read." + log_error "Error: ${service_file} could not be read." exit 1 fi } @@ -94,6 +126,7 @@ services_start_queue() execute() { while read -r line ; do exec_wrapper "${line}" + instance_env "${line}" done < "${SORTED_LIST}" } @@ -215,14 +248,20 @@ systemd_daemon_reload_preset TMPFILE="$(mktemp)" log_debug "TMPFILE = ${TMPFILE}" +log_debug "OUTPUT_DIRECTORY=${OUTPUT_DIRECTORY}" CLEANUP_FILES+=("${TMPFILE}") + +if [ -n "${TEMPLATE_INSTANCES:-}" ] && [ -f "${TEMPLATE_INSTANCES}" ] ; then + instances_info +fi + for dir in ${CONFIG_POOL} ; do - is_absolute_path "${dir}" || continue - is_non_git_folder "${dir}" || continue + is_absolute_path "${OUTPUT_DIRECTORY}${dir}" || continue + is_non_git_folder "${OUTPUT_DIRECTORY}${dir}" || continue - generate_list_to_process "${dir}" + generate_list_to_process "${OUTPUT_DIRECTORY}${dir}" done if [[ -s "${TMPFILE}" ]]; then