MT#4141 Speed up 'build' runs by loading YAML files only once

When runnning 'ngcpcfg build' (which is also executed when
running 'ngcpcfg apply') we waste quite some time with loading
the same YAML configuration files and merging them again and
again.

In a 'ngcpcfg build' run we don't have to re-load and merge those
configuration files again and again (since they won't modify
during a single run), instead just load+merge them *once* and
re-use the result on all the following execution steps in the
same run.

Original:

| # time ngcpcfg apply >/dev/null
| ngcpcfg apply > /dev/null  149.01s user 9.49s system 96% cpu 2:43.57 total

Reworked optimized runs:

| # time ngcpcfg apply >/dev/null
| ngcpcfg apply > /dev/null  81.25s user 8.62s system 95% cpu 1:34.17 total

For debugging purposes you can execute 'ngcpcfg build' under
NO_REUSE=1 to not use this feature, executing it under 'DEBUG=1'
won't remove the so called reuse file which stores the merged
configuration data.

Thanks: Alexander Lutay for the initial patch and idea

helper/build_config

@@ -7,9 +7,13 @@
 set -e
 set -u
 
-if [ "${#:-}" -ne 1 ] ; then
-  echo "Usage: /usr/share/ngcp-ngcpcfg/helper/build_config <input_file>" >&2
+usage() {
+  echo "Usage: /usr/share/ngcp-ngcpcfg/helper/build_config [--reuse <file>] <input_file>" >&2
   exit 1
+}
+
+if [ "${#:-}" -lt 1 ] ; then
+  usage
 fi
 
 # support for testsuite, assume defaults if unset
@@ -21,6 +25,20 @@ HELPER="${HELPER:-/usr/share/ngcp-ngcpcfg/helper/}"
 
 # main script
 
+if [ -n "${1:-}" ] ; then
+  case "$1" in
+    *--reuse*)
+      shift
+      if [ -z "${1:-}" ] ; then
+	usage
+      else
+        TT_WRAPER_ARGS="--reuse $1"
+	shift
+      fi
+      ;;
+  esac
+fi
+
 input_file="$1"       # like /etc/ngcp-config/templates/etc/mysql/my.cnf.tt2
 # calculate output file {{{
   x=${input_file##${NGCPCTL_MAIN}/templates/}     # drop leading /etc/ngcp-config
@@ -58,8 +76,8 @@ umask 0077
 TT_WRAPPER="${HELPER}/tt2-wrapper"
 
 log_debug "Output file ${output_file} based on ${input_file}"
-log_debug "Executing: $TT_WRAPPER ${input_file} ${host_conf:-} ${local_conf:-} $NGCPCTL_CONFIG ${NETWORK_CONFIG:-} ${EXTRA_CONFIG_FILES:-} $CONSTANTS_CONFIG > ${output_file}"
-if "$TT_WRAPPER" "${input_file}" ${host_conf:-} ${local_conf:-} "$NGCPCTL_CONFIG" "${NETWORK_CONFIG:-}" ${EXTRA_CONFIG_FILES:-} "$CONSTANTS_CONFIG" > "${output_file}" 2>/dev/null ; then
+log_debug "Executing: $TT_WRAPPER ${TT_WRAPER_ARGS:-} ${input_file} ${host_conf:-} ${local_conf:-} $NGCPCTL_CONFIG ${NETWORK_CONFIG:-} ${EXTRA_CONFIG_FILES:-} $CONSTANTS_CONFIG > ${output_file}"
+if "$TT_WRAPPER" ${TT_WRAPER_ARGS:-} "${input_file}" ${host_conf:-} ${local_conf:-} "$NGCPCTL_CONFIG" "${NETWORK_CONFIG:-}" ${EXTRA_CONFIG_FILES:-} "$CONSTANTS_CONFIG" > "${output_file}" 2>/dev/null ; then
   log_info "Generating ${output_file}: OK"
   RC=0
 else

helper/tt2-wrapper

@@ -5,16 +5,39 @@
 use strict;
 use warnings;
 
-use YAML qw/LoadFile/;
-use Template;
+use Getopt::Long;
 use Hash::Merge qw(merge);
+use Storable;
+use Template;
+use YAML qw/LoadFile/;
+
+my $help;
+my $reuse;
+
+GetOptions(
+    'help'    => \$help,
+    'reuse=s' => \$reuse,
+) or die "Usage: $0 [--reuse <file>] <template> <config.yml> [<another.yml>]";
 
 my $tt = Template->new({ ABSOLUTE => 1, RELATIVE => 1, EVAL_PERL => 1 });
-my $template = shift(@ARGV);
+my $template = shift(@ARGV) or die "Usage: $0 [--reuse <file>] <template> <config.yml> [<another.yml>]";
 
 my $config = {};
-foreach my $file (@ARGV) {
-  $config = merge($config, LoadFile($file) || die $!);
+
+if ($reuse) {
+  if (-e $reuse) {
+    $config = retrieve($reuse);
+  } else {
+    foreach my $file (@ARGV) {
+      $config = merge($config, LoadFile($file) || die $!);
+    }
+
+    store($config, $reuse);
+  }
+} else {
+  foreach my $file (@ARGV) {
+    $config = merge($config, LoadFile($file) || die $!);
+  }
 }
 
 # NOTE: we can't rely on "$tt->process($template, $config)" because the config

scripts/build

@@ -24,6 +24,13 @@ if [ -n "${1:-}" ] ; then
   esac
 fi
 
+if [ -n "${NO_REUSE:-}" ] ; then
+  log_info "Not using --reuse feature of tt2-wrapper as requested via NO_REUSE"
+else
+  tmpfile=$(mktemp --dry-run --tmpdir=/tmp/ ngcpcfgreuse.XXXXXX)
+  BUILD_CONFIG_ARGS="--reuse ${tmpfile}"
+fi
+
 # make sure encoding is OK
 for f in ${NGCPCTL_CONFIG:-} ${HOST_CONFIG:-} ${LOCAL_CONFIG:-} ${NETWORK_CONFIG:-} ${EXTRA_CONFIG_FILES:-} ${CONSTANTS_CONFIG:-} ; do
   if [ -r "$f" ] && ! file "$f" | grep -qe "UTF-8" -qe "ASCII" ; then
@@ -54,8 +61,8 @@ done
 
 build_config_files() {
   for file in $(generate_template_list $*) ; do
-    log_debug "${HELPER}/build_config $file"
-    "${HELPER}/build_config" "${file}"
+    log_debug "${HELPER}/build_config ${BUILD_CONFIG_ARGS:-} $file"
+    "${HELPER}/build_config" ${BUILD_CONFIG_ARGS:-} "${file}"
     RC=$(($RC + $?))
   done
 }
@@ -136,6 +143,14 @@ else
   fi
 fi
 
+if [ -z "${NO_REUSE:-}" ] ; then
+  if [ -n "${DEBUG:-}" ] ; then
+    log_debug "Not removing temporary rerun file ${tmpfile} since we are in debug mode"
+  else
+    rm -f "${tmpfile}"
+  fi
+fi
+
 exit "$RC"
 
 ## END OF FILE #################################################################