From a9166843c20aa45b4a5393120c81254f5c54ee97 Mon Sep 17 00:00:00 2001 From: Guillem Jover Date: Wed, 4 Dec 2019 12:07:54 +0100 Subject: [PATCH] TT#71952 Parallelize templated output generation Switch away from the old daemon/wrapper architecture, which complicates things and requires more scaffolding code. We turn the daemon/wrapper and build_config into a single perl process that will load all YAML files, and then process each input/output pair on a parallel child, to try to speed up the processing as much as possible. For whole rebuilds, it might speed up the generation by at least a factor of x2, x4 or more, depending on the number of active processors. Change-Id: I51aa2f90336e34a20983d8733f45b64d9b6fea0b --- Makefile | 8 +- debian/control | 2 - debian/ngcp-ngcpcfg.install | 4 +- docs/hacking.txt | 19 +- docs/ngcpcfg.txt | 7 + helper/build_config | 156 ------------- helper/tt2-daemon | 91 -------- helper/tt2-process | 251 +++++++++++++++++++++ helper/tt2-wrapper | 12 - sbin/ngcpcfg | 2 + scripts/build | 40 +--- scripts/get | 30 +-- t/Dockerfile | 4 +- t/README.adoc | 13 +- t/fixtures/output/bad-syntax.txt | 2 - t/fixtures/programs.py | 1 - t/test_ngcpcfg.py | 2 - t/test_ngcpcfg_build_network_interfaces.py | 1 - t/test_ngcpcfg_build_syntax.py | 29 +-- t/test_ngcpcfg_lib_get_all_ips.py | 2 - t/test_ngcpcfg_lib_status.py | 1 - t/test_ngcpcfg_patch.py | 20 +- 22 files changed, 307 insertions(+), 390 deletions(-) delete mode 100755 helper/build_config delete mode 100755 helper/tt2-daemon create mode 100755 helper/tt2-process delete mode 100755 helper/tt2-wrapper delete mode 100644 t/fixtures/output/bad-syntax.txt diff --git a/Makefile b/Makefile index c6589183..46a7e836 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,14 @@ # for syntax checks -BASH_SCRIPTS = scripts/* functions/* etc/ngcp-config/ngcpcfg.cfg helper/build_config sbin/ngcpcfg helper/tt2-wrapper +BASH_SCRIPTS = \ + scripts/* \ + functions/* \ + etc/ngcp-config/ngcpcfg.cfg \ + sbin/ngcpcfg PERL_SCRIPTS = \ lib/NGCP/Template.pm \ helper/sort-yml \ helper/sync-db \ - helper/tt2-daemon \ + helper/tt2-process \ helper/validate-yml helper/fileformat_version \ sbin/ngcp-network \ sbin/ngcp-network-validator \ diff --git a/debian/control b/debian/control index 7f046143..a0e5f361 100644 --- a/debian/control +++ b/debian/control @@ -25,7 +25,6 @@ Build-Depends: libtemplate-perl, libterm-readpassword-perl, libyaml-libyaml-perl, - netcat-openbsd, pkwalify, python3-pytest, @@ -66,7 +65,6 @@ Depends: libsys-hostname-long-perl, libterm-readpassword-perl, libyaml-libyaml-perl, - netcat-openbsd, ngcp-system-tools, pkwalify, psmisc, diff --git a/debian/ngcp-ngcpcfg.install b/debian/ngcp-ngcpcfg.install index 7bde6fde..36b32a0a 100644 --- a/debian/ngcp-ngcpcfg.install +++ b/debian/ngcp-ngcpcfg.install @@ -2,14 +2,12 @@ etc/ngcp-config/ngcpcfg.cfg etc/ngcp-config/ functions/init usr/share/ngcp-ngcpcfg/functions/ functions/logs usr/share/ngcp-ngcpcfg/functions/ functions/main usr/share/ngcp-ngcpcfg/functions/ -helper/build_config usr/share/ngcp-ngcpcfg/helper/ helper/check-for-mysql usr/share/ngcp-ngcpcfg/helper/ helper/fileformat_version usr/share/ngcp-ngcpcfg/helper/ helper/restore-permissions usr/share/ngcp-ngcpcfg/helper/ helper/sort-yml usr/share/ngcp-ngcpcfg/helper/ helper/sync-db usr/share/ngcp-ngcpcfg/helper/ -helper/tt2-daemon usr/share/ngcp-ngcpcfg/helper/ -helper/tt2-wrapper usr/share/ngcp-ngcpcfg/helper/ +helper/tt2-process usr/share/ngcp-ngcpcfg/helper/ helper/validate-yml usr/share/ngcp-ngcpcfg/helper/ hooks/ usr/share/ngcp-ngcpcfg/ lib/get_* usr/lib/ngcp-ngcpcfg/ diff --git a/docs/hacking.txt b/docs/hacking.txt index ba0f440c..de9b9fd3 100644 --- a/docs/hacking.txt +++ b/docs/hacking.txt @@ -16,16 +16,15 @@ scripts and build system. Templating ---------- -Of very significant note here are the 'tt2-daemon' and 'tt2-wrapper' tandem, -which perform most of the work when processing the templates. The daemon is -a long-lived process that gets interacted with via a socket, and receives -"commands" from the 'tt2-wrapper'. The YAML configuration files are loaded -so that they only need to be validated and processed once. And then each -template file to be processed is fed to the daemon and the result returned -via the wrapper. All the configuration loaded from the YAML files is -flattened into a dot-separated namespaced tree, which can be accessed -directly by any template file even if no other code has references to those -configurations. +Of very significant note is the 'tt2-process' helper, which performs most +of the work when processing the templates. This process is started only +once during the whole ngcpcfg run, and loads and validates all the YAML +files, and then processes each input/output pairs on an independent process +to parallelize the execution. + +All the configuration loaded from the YAML files is flattened into a +dot-separated namespaced tree, which can be accessed directly by any +template file even if no other code has references to those configurations. ///////////////////////////// // vim: ft=asciidoc tw=80 ai diff --git a/docs/ngcpcfg.txt b/docs/ngcpcfg.txt index 25aa93bf..70b6d780 100644 --- a/docs/ngcpcfg.txt +++ b/docs/ngcpcfg.txt @@ -248,6 +248,13 @@ NGCP systems) as well as stderr. Display usage information and exit. + **--jobs[=]** []:: + +The _build_ and _apply_ actions process templates in parallel by default. +This option can be used to control the number of jobs used to process +templates, '0' for an unlimited amount, and '1' for serial execution to +for example get ordered output. + **--no-action-failure** []:: The _check_ and _apply_ actions check for any possibly outstanding pull diff --git a/helper/build_config b/helper/build_config deleted file mode 100755 index 700f5b21..00000000 --- a/helper/build_config +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash -# Filename: /usr/share/ngcp-ngcpcfg/helper/build_config -# Purpose: builds output configuration file based on tt2 template file -# using /usr/share/ngcp-ngcpcfg/helper/tt2-wrapper -################################################################################ - -set -e -set -u - -if [ "${#:-}" -ne 3 ] ; then - echo "Usage: /usr/share/ngcp-ngcpcfg/helper/build_config " >&2 - exit 1 -fi - -# support for testsuite, assume defaults if unset -CONFIG_POOL="${CONFIG_POOL:-/etc}" -FUNCTIONS="${FUNCTIONS:-/usr/share/ngcp-ngcpcfg/functions/}" -HELPER="${HELPER:-/usr/share/ngcp-ngcpcfg/helper/}" - -if ! [ -r "${FUNCTIONS}"/logs ] ; then - printf "Error: %s/logs could not be read. Exiting.\n" "${FUNCTIONS}" >&2 - exit 1 -fi -. "${FUNCTIONS}"/logs - -# main script - -input_file="$1" # like /etc/ngcp-config/templates/etc/mysql/my.cnf.tt2 -output_file="$2" # like /etc/mysql/my.cnf -tmp_output_file="$3" # like /tmp/ngcpcfg.PID12345.KLJhiand/tmp_output_file - -if [ -z "${input_file}" ] ; then - log_error "Missing parameter. Exiting." >&2 - exit 1 -fi - -if [ -z "${output_file}" ] ; then - log_error "Missing parameter. Exiting." >&2 - exit 1 -fi - -if [ -z "${tmp_output_file}" ] ; then - log_error "Missing parameter. Exiting." >&2 - exit 1 -fi - -# export variable for usage within {pre,post}build scripts, -# OUTPUT_DIRECTORY is for customization during testing -if [ -n "${OUTPUT_DIRECTORY:-}" ] ; then - log_debug "Using output directory ${OUTPUT_DIRECTORY}" - export output_file="${OUTPUT_DIRECTORY}/${output_file}" -else - export output_file -fi -output_file_dirname="$(dirname "${output_file}")" # like /etc/mysql - -# ensure we don't try to generate a file where a directory with same name exists already -if [ -d "${output_file}" ] ; then - log_error "Generating file ${output_file} not possible, it's an existing directory." >&2 - exit 1 -fi - -# pre-execution script in template store: -if [ -r "${NGCPCTL_MAIN}/templates/${output_file}.prebuild" ] ; then - log_info "Executing prebuild for ${output_file}" - bash "${NGCPCTL_MAIN}/templates/${output_file}.prebuild" -elif [ -r "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.prebuild" ] ; then - log_info "Executing prebuild for ${output_file}" - bash "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.prebuild" -fi - -# if output directory does not exist yet, create it -if ! [ -d "${output_file_dirname}" ] ; then - umask 0022 # directory permissions should be '755' - mkdir -p "${output_file_dirname}" -fi - -# assume safe defaults -umask 0077 - -# read host specific configuration file only if it exists -[ -r "${HOST_CONFIG:-}" ] && host_conf="$HOST_CONFIG" - -# read local config only if it exists -[ -r "${LOCAL_CONFIG:-}" ] && local_conf="$LOCAL_CONFIG" - -TT_WRAPPER="${HELPER}/tt2-wrapper" - -# XXX: Docker breaks sane Unix expectations when moving a file into /etc/hosts, -# as it creates a bind mount on that pathname. We need to use an implementation -# that will fallback to use copy semantics in that case, but will default to -# use rename semantics to avoid races on ETXTBSY on executable files. -# -move() -{ - local src="$1" - local dst="$2" - - if [ -e /.dockerinit ] || [ -e /.dockerenv ] ; then - perl -MFile::Copy=mv \ - -E "mv('${src}', '${dst}') or die 'error: cannot move $src to $dst: \$!\n'" - else - mv "${src}" "${dst}" - fi -} - -log_debug "Output file ${output_file} based on ${input_file}" -log_debug "Executing: $TT_WRAPPER ${input_file} > ${tmp_output_file}" -log_debug " and: move ${tmp_output_file} ${output_file}" -# 1) We need to use «readlink -f» so that we do not destroy any symlink pointing -# to the real file, which we were previously preserving while using «cat». -# 2) Care about ">" below, do not trust tmp file you receive. -if "$TT_WRAPPER" "${input_file}" > "${tmp_output_file}" 2>/dev/null && - ! grep -q -E '^file error' "${tmp_output_file}" 2>/dev/null && - move "${tmp_output_file}" "$(readlink -f "${output_file}")" ; then - log_info "Generating ${output_file}: OK" - RC=0 -else - log_error "Generating ${output_file} based on ${input_file}: FAILED" - RC=1 - - if [[ -r "${tmp_output_file}" ]] && grep -q -E '^file error' "${tmp_output_file}" ; then - log_error "from generated file:" - log_error " $(grep -E '^file error' "${tmp_output_file}")" - fi - - log_info "NOTE: Check those files for valid syntax and encoding:" - for f in "${input_file}" ${host_conf:-} ${local_conf:-} "$NGCPCTL_CONFIG" "${NETWORK_CONFIG:-}" "${EXTRA_CONFIG_FILES[@]}" "$CONSTANTS_CONFIG" ; do - [ -r "$f" ] && log_info "$f" - done - log_info "Running /usr/share/ngcp-ngcpcfg/helper/tt2-wrapper " - log_info "or inspecting temporary ${tmp_output_file}" - log_info "should provide more details." -fi - -if [ -L "$output_file" ] ; then - log_warn "File $output_file is a symlink - NOT adjusting permissions" -else - # set permissions for generated config based on the ones of the template - chmod --reference="${input_file}" "${output_file}" - # finally drop all write permissions - chmod a-w "${output_file}" -fi - -# post-execution script in template store: -if [ -r "${NGCPCTL_MAIN}/templates/${output_file}.postbuild" ] ; then - log_info "Executing postbuild for ${output_file}" - bash "${NGCPCTL_MAIN}/templates/${output_file}.postbuild" -elif [ -r "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.postbuild" ] ; then - log_info "Executing postbuild for ${output_file}" - bash "${NGCPCTL_MAIN}/templates/${output_file_dirname}/ngcpcfg.postbuild" -fi - -exit $RC - -## END OF FILE ################################################################# diff --git a/helper/tt2-daemon b/helper/tt2-daemon deleted file mode 100755 index 19c63e94..00000000 --- a/helper/tt2-daemon +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; - -use English; -use Carp; -use POSIX qw(setsid); -use IO::Socket::UNIX; -use Hash::Merge qw(merge); -use YAML::XS qw(LoadFile); -use Clone 'clone'; -use Getopt::Long; -use NGCP::Template; - -my $quiet = 0; - -GetOptions("q|quiet" => \$quiet); - -my $server_socket = get_server_socket(); - -daemonize(); - -handle_connections($server_socket, $quiet); - -exit; - -sub daemonize { - my $NGCP_BASE_TT2 = $ENV{'NGCP_BASE_TT2'} //= '/'; - chdir $NGCP_BASE_TT2 or croak "Can't chdir to $NGCP_BASE_TT2: $ERRNO"; - open(STDIN, '<', '/dev/null') or croak "Can't read /dev/null: $ERRNO"; - open(STDOUT, '>', '/dev/null') or croak "Can't write to /dev/null: $ERRNO"; - defined(my $pid = fork) or croak "Can't fork: $ERRNO"; - exit if $pid; - setsid() or croak "Can't start a new session: $ERRNO"; - open(STDERR, '>&', \*STDOUT) or croak "Can't dup stdout: $ERRNO"; - return; -} - -sub get_server_socket { - my $NGCP_SOCKETFILE = $ENV{'NGCP_SOCKETFILE'} //= '/run/ngcpcfg.socket'; - - my $server = IO::Socket::UNIX->new( - 'Type' => SOCK_STREAM(), - 'Local' => $NGCP_SOCKETFILE, - 'Listen' => SOMAXCONN, - ); - die "Error: can't setup tt2-daemon!\n$EVAL_ERROR\n$ERRNO\n" unless $server; - - binmode $server => ":encoding(utf8)"; - - return $server; -} - -sub handle_connections { - my $port = shift; - my $quiet = shift; - my $config = {}; - my %loaded_ymls = (); - - while (my $client = $port->accept()) { - my $input = <$client>; - chomp $input; - - my @argv = split(' ', $input) or - print { $client } "Usage: echo '