MT#5027 Speed up 'build' runs by using Perl daemon

Makefile

@@ -1,8 +1,8 @@
 # for syntax checks
-BASH_SCRIPTS =	scripts/* functions/* etc/ngcp-config/ngcpcfg.cfg helper/build_config sbin/ngcpcfg
+BASH_SCRIPTS =	scripts/* functions/* etc/ngcp-config/ngcpcfg.cfg helper/build_config sbin/ngcpcfg helper/tt2-wrapper
 PERL_SCRIPTS =	helper/sort-yml \
 		helper/sync-db \
-		helper/tt2-wrapper \
+		helper/tt2-daemon \
 		helper/validate-yml helper/fileformat_version \
 		sbin/ngcp-network \
 		sbin/ngcp-sync-constants

debian/ngcp-ngcpcfg.install

@@ -6,6 +6,7 @@ helper/fileformat_version   usr/share/ngcp-ngcpcfg/helper/
 helper/sort-yml             usr/share/ngcp-ngcpcfg/helper/
 helper/sync-db              usr/share/ngcp-ngcpcfg/helper/
 helper/tt2-wrapper          usr/share/ngcp-ngcpcfg/helper/
+helper/tt2-daemon           usr/share/ngcp-ngcpcfg/helper/
 helper/validate-yml         usr/share/ngcp-ngcpcfg/helper/
 lib/*                       usr/lib/ngcp-ngcpcfg/
 sbin/ngcp-network           usr/sbin/

helper/build_config

@@ -58,8 +58,8 @@ umask 0077
 TT_WRAPPER="${HELPER}/tt2-wrapper"
 
 log_debug "Output file ${output_file} based on ${input_file}"
-log_debug "Executing: $TT_WRAPPER ${input_file} ${host_conf:-} ${local_conf:-} $NGCPCTL_CONFIG ${NETWORK_CONFIG:-} ${EXTRA_CONFIG_FILES:-} $CONSTANTS_CONFIG > ${output_file}"
-if "$TT_WRAPPER" "${input_file}" ${host_conf:-} ${local_conf:-} "$NGCPCTL_CONFIG" "${NETWORK_CONFIG:-}" ${EXTRA_CONFIG_FILES:-} "$CONSTANTS_CONFIG" > "${output_file}" 2>/dev/null ; then
+log_debug "Executing: $TT_WRAPPER ${input_file} > ${output_file}"
+if "$TT_WRAPPER" "${input_file}" > "${output_file}" 2>/dev/null ; then
   log_info "Generating ${output_file}: OK"
   RC=0
 else

helper/tt2-daemon

@@ -0,0 +1,76 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use Carp;
+use POSIX qw( setsid );
+use IO::Socket;
+use Hash::Merge qw(merge);
+use Template;
+use YAML qw/LoadFile/;
+
+my $server_port = get_server_port();
+
+daemonize();
+
+handle_connections( $server_port );
+
+exit;
+
+sub daemonize {
+    chdir '/'                 or croak "Can't chdir to /: $!";
+    open STDIN, '/dev/null'   or croak "Can't read /dev/null: $!";
+    open STDOUT, '>/dev/null' or croak "Can't write to /dev/null: $!";
+    defined(my $pid = fork)   or croak "Can't fork: $!";
+    exit if $pid;
+    setsid                    or croak "Can't start a new session: $!";
+    open STDERR, '>&STDOUT'   or croak "Can't dup stdout: $!";
+}
+
+sub get_server_port {
+    my $server = IO::Socket::INET->new(
+        'Proto'     => 'tcp',
+        'LocalPort' => 42042,
+        'Listen'    => SOMAXCONN,
+        'Reuse'     => 1,
+    );
+    die "can't setup server" unless $server;
+
+    binmode $server => ":encoding(utf8)";
+
+    return $server;
+}
+
+sub handle_connections {
+    my $port = shift;
+    my $config = {};
+    my %loaded_ymls = ();
+    my $reuse;
+
+    while ( my $client = $port->accept() ) {
+    chomp ( my $input = <$client> );
+
+    my $tt = Template->new({ ABSOLUTE => 1, RELATIVE => 1, EVAL_PERL => 1 });
+    my @ARGV = split(' ', $input) or
+        print $client "Usage: echo '<template> [<config.yml> [<another.yml>]]' | netcat localhost 42042 \n";
+    my $template =  shift(@ARGV);
+
+    if (scalar @ARGV) {
+        foreach my $file (@ARGV)
+        {
+            next if(exists $loaded_ymls{$file});
+            $loaded_ymls{$file} = undef;
+            print $client "Loading $file in memory :";
+            $config = merge($config, LoadFile($file) || die $!);
+            print $client " OK \n";
+        };
+    };
+
+    open my $fh, '<', $template or print $client "Unable to open file '$template' for reading: $!\n";
+    $tt->process($fh, $config, $client) or print $client $tt->error;
+    close $fh;
+
+    close $client;
+    }
+
+    return;
+}

helper/tt2-wrapper

@@ -1,26 +1,5 @@
-#!/usr/bin/perl -CSD
-# Purpose: template toolkit helper script
-################################################################################
+#!/bin/bash
 
-use strict;
-use warnings;
+echo "$@" | netcat localhost 42042
 
-use YAML qw/LoadFile/;
-use Template;
-use Hash::Merge qw(merge);
-
-my $tt = Template->new({ ABSOLUTE => 1, RELATIVE => 1, EVAL_PERL => 1 });
-my $template = shift(@ARGV);
-
-my $config = {};
-foreach my $file (@ARGV) {
-  $config = merge($config, LoadFile($file) || die $!);
-}
-
-# NOTE: we can't rely on "$tt->process($template, $config)" because the config
-# file is UTF8 encoded but is then being read as a binary 8bit file and written
-# as UTF8 file again, causing wide characters getting encoded twice
-open my $fh, '<', $template or die "Unable to open file for reading: $!";
-$tt->process($fh, $config) or die $tt->error;
-
-## END OF FILE #################################################################
+exit $?

scripts/build

@@ -52,6 +52,23 @@ for f in ${NGCPCTL_CONFIG:-} ${HOST_CONFIG:-} ${LOCAL_CONFIG:-} ${NETWORK_CONFIG
   fi
 done
 
+# Kill all previous started tt2-daemon Perl processes if they were not stopped properly
+killall tt2-daemon 2>/dev/null || true
+
+# Start new tt2-daemon Perl process
+${HELPER}/tt2-daemon
+
+# Load all the configs in proper order and check their avialability and valid YAML syntax
+for f in ${NGCPCTL_CONFIG:-} ${HOST_CONFIG:-} ${LOCAL_CONFIG:-} ${NETWORK_CONFIG:-} ${EXTRA_CONFIG_FILES:-} ${CONSTANTS_CONFIG:-} ; do
+  if [ -r "$f" ] ; then
+    if ! "${HELPER}/tt2-wrapper" "/dev/null" "${f}" ; then
+      log_error "tt2-daemon cannot load ${f}:"
+      "${HELPER}/tt2-wrapper" "/dev/null" "${f}"
+      exit 1
+    fi
+  fi
+done
+
 build_config_files() {
   for file in $(generate_template_list $*) ; do
     log_debug "${HELPER}/build_config $file"
@@ -136,6 +153,9 @@ else
   fi
 fi
 
+# Kill all previous started tt2-daemon Perl processes
+killall tt2-daemon 2>/dev/null || true
+
 exit "$RC"
 
 ## END OF FILE #################################################################