From a2a8b52f13b438a52cb8386812346498c2ab6da7 Mon Sep 17 00:00:00 2001 From: Alexander Lutay Date: Wed, 15 Nov 2017 16:32:13 +0100 Subject: [PATCH] TT#24920 Initial commit for 'ngcpcfg patch' The 'ngcpcfg' received support for 'patchtt' files, like > /etc/ngcp-config/templates/etc/foo/bar.patchtt.tt2 Those 'patchtt' are going to be applied on default 'tt2 template' file: > /etc/ngcp-config/templates/etc/foo/bar.tt2 and produce 'customtt' on 'ngcpcfg patch': > /etc/ngcp-config/templates/etc/foo/bar.customtt.tt2 Further 'customtt' will be used to overwrite 'tt2 templates' on 'ngcpcfg build' or 'ngcpcfg apply'. NOTE: 'ngcpcfg patch' is executed automatically on every 'ngcpcfg build'. It should allows to update ngcp-templates easily and support local modifications without the pain (until the patches can be applied). Change-Id: Ice4369386313c5d33e4d498346345eade6f3d0d7 --- debian/ngcp-ngcpcfg.install | 1 + docs/ngcpcfg.txt | 60 ++++++++++-- functions/main | 6 ++ sbin/ngcpcfg | 4 +- scripts/build | 2 + scripts/encrypt | 1 + scripts/patch | 179 ++++++++++++++++++++++++++++++++++++ 7 files changed, 242 insertions(+), 11 deletions(-) create mode 100755 scripts/patch diff --git a/debian/ngcp-ngcpcfg.install b/debian/ngcp-ngcpcfg.install index 18b26ac1..eebf5c45 100644 --- a/debian/ngcp-ngcpcfg.install +++ b/debian/ngcp-ngcpcfg.install @@ -26,6 +26,7 @@ scripts/diff usr/share/ngcp-ngcpcfg/scripts/ scripts/etckeeper usr/share/ngcp-ngcpcfg/scripts/ scripts/initialise usr/share/ngcp-ngcpcfg/scripts/ scripts/log usr/share/ngcp-ngcpcfg/scripts/ +scripts/patch usr/share/ngcp-ngcpcfg/scripts/ scripts/services usr/share/ngcp-ngcpcfg/scripts/ scripts/set usr/share/ngcp-ngcpcfg/scripts/ scripts/show usr/share/ngcp-ngcpcfg/scripts/ diff --git a/docs/ngcpcfg.txt b/docs/ngcpcfg.txt index 8db38245..1c83efaf 100644 --- a/docs/ngcpcfg.txt +++ b/docs/ngcpcfg.txt @@ -111,8 +111,8 @@ template file. $HA_NODE is determined using the content of /etc/ngcp_ha_node (usually being _sp1_ for the first node and _sp2_ for the second node on the Sipwise Next Generation Platform). Whereas _*customtt.tt2_ files are used on all nodes in a High Availability setup the _*.tt2.$HA_NODE*_ file is specific for -the single node only. A common usage case is master vs. slave configuration of a -service. Configuration file is usually provided by a Debian package. +the single node only. A common use case is master vs. slave configuration of a +service. The configuration file is usually provided by a Debian package. Note: Feature is available in the High Availability setup only. * _/etc/ngcp-config/templates/etc/foobar/baz.tt2.$PAIRNAME_: pair specific @@ -130,13 +130,13 @@ Note: Feature is available in the High Availability setup only. * _/etc/ngcp-config/templates/etc/foobar/baz.customtt.tt2_: system specific template file, but configuration usually isn't provided by a Debian package and -can be modified independent from any Debian package mechanism. +can be modified independently from any Debian package mechanism. * _/etc/ngcp-config/templates/etc/foobar/baz.customtt.tt2.$HA_NODE_: node specific template file. Regarding $HA_NODE the same as for _baz.tt2.$HA_NODE_ -applies (see previous bullet), but the configuration file usually isn't provided -by a Debian package but can be modified independent from any Debian package -mechanism. +applies (see the previous bullet), but the configuration file usually isn't +provided by a Debian package but can be modified independently from any +Debian package mechanism. Note: Feature is available in the High Availability setup only. * _/etc/ngcp-config/templates/etc/foobar/baz.customtt.tt2.$PAIRNAME_: @@ -150,13 +150,48 @@ configuration file similar to _/etc/ngcp-config/templates/etc/foobar/baz.tt2.$HOSTNAME_ but it's guaranteed that the file won't be part of any Debian package mechanism. Note: Feature is available in the High Availability setup only. -Note: Feature is available in the High Availability setup only. [IMPORTANT] -Configuration file priority: *.customtt.tt2.$HOSTNAME takes precedence over -*.customtt.tt2.$PAIRNAME, over .customtt.tt2.$HA_NODE, over *.customtt.tt2, over -*.tt2.$HOSTNAME, over *.tt2.$PAIRNAME, over *.tt2.$HA_NODE, over *.tt2. +Configuration file precedence (highest to lowest): +*.customtt.tt2.$HOSTNAME +*.customtt.tt2.$PAIRNAME +*.customtt.tt2.$HA_NODE +*.customtt.tt2, +*.tt2.$HOSTNAME +*.tt2.$PAIRNAME +*.tt2.$HA_NODE +*.tt2. + +Customisation for default template files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can overwrite the default template file +(e.g. _/etc/ngcp-config/templates/etc/foobar/baz.tt2_) +using a _customtt_ file, like _/etc/ngcp-config/templates/etc/foobar/baz.customtt.tt2_. +Or even host a specific _customtt_ file (see above in "Supported template files"). +This approach is NOT recommended as _customtt_ will become outdated very soon, +hence new template files can be released by upstream any time. + +The better way is to handle modifications using _patchtt_ files +(e.g. _/etc/ngcp-config/templates/etc/foobar/baz.patchtt.tt2_). +In this case, on every "ngcpcfg patch", _patchtt_ file will be applied on top +of the tt2 file and the result will be saved into the _customtt_ file, which +in the future will be used in a common way. "ngcpcfg patch" is the first step +on "ngcpcfg build" that guarantees the latest upstream templates with the +availability of the necessary local changes on every configuration apply. + +[IMPORTANT] +The patch to be applied to the corresponding tt2 template file is selected in +the following order (highest to lowest): +*.patchtt.tt2.$HOSTNAME +*.patchtt.tt2.$PAIRNAME +*.patchtt.tt2.$HA_NODE +*.patchtt.tt2 +[IMPORTANT] +If a suitable _patchtt_ file is found for a template, then the **ngcpcfg patch** +command will overwrite the corresponding _customtt_ file, if any exists. You can +find the old version of the customtt in ngcpcfg the git repository (if necessary). Support action related files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -362,6 +397,11 @@ Note: This feature is only available if the ngcp-ngcpcfg-carrier package is inst Prints the log of local changes. Expand all changes if '-p' option is specified. + **patch** [--help] []:: + +Generate customtt files using default templates and patchtt files. +Using patchtt files automates template updating and simplifies customtt management. + **pull**:: Retrieve modifications from shared repository. diff --git a/functions/main b/functions/main index 239c3399..39288a41 100644 --- a/functions/main +++ b/functions/main @@ -237,6 +237,12 @@ generate_template_list() { # foo.customtt.tt2.hostname > foo.customtt.tt2.pairname > foo.customtt.tt2.spX > foo.customtt.tt2 > foo.tt2.hostname > foo.tt2.pairname > foo.tt2.spX > foo.tt2 for line in $(cat ${filelist_prepared}); do + # ignoring foo.patchtt.tt2.* completely (it is not a tt2 template to be built) + if [[ "${line}" =~ .*\.patchtt\.tt2(.*)?$ ]]; then + log_debug "Ignored patchtt file '${line}'" + continue + fi + normalized_filename="$line" if [ -n "${HA_FILE:-}" ] ; then diff --git a/sbin/ngcpcfg b/sbin/ngcpcfg index 25338290..db84669f 100755 --- a/sbin/ngcpcfg +++ b/sbin/ngcpcfg @@ -61,6 +61,7 @@ Actions: clean [] clean /etc/ngcp-config folder configs/templates (see available options) set [] set YAML option in defined file del [] delete YAML option from defined file + patch [] create customtt files using patchtt and templates " "$PN" # display only if ngcp-ngcpcfg-ha is available @@ -118,7 +119,8 @@ case ${1:-} in status|\ values|\ set|\ - del) + del|\ + patch) action "$@" ;; --debug) export DEBUG=1 ; shift ; "$0" "$@" ;; --no-db-sync) export NO_DB_SYNC=1 ; shift ; "$0" "$@" ;; diff --git a/scripts/build b/scripts/build index a21fffe5..377c6b34 100755 --- a/scripts/build +++ b/scripts/build @@ -36,6 +36,8 @@ else "${SCRIPTS}"/check --ignore-branch-check fi +"${SCRIPTS}"/patch + # Kill all previous started tt2-daemon Perl processes if they were not stopped properly killall tt2-daemon 2>/dev/null || true diff --git a/scripts/encrypt b/scripts/encrypt index ba414225..b6f7b585 100755 --- a/scripts/encrypt +++ b/scripts/encrypt @@ -25,6 +25,7 @@ get_config_file_list() { y=${y%%.tt2.sp1} # drop trailing suffix '.tt2.sp1' y=${y%%.tt2.sp2} # drop trailing suffix '.tt2.sp2' y=${y%%.customtt} # drop trailing suffix '.customtt' + y=${y%%.patchtt} # drop trailing suffix '.patchtt' # if the file does not exist (e.g. because "ngcpcfg apply" # hasn't been executed yet for whatever reason, then don't # report missing files, otherwise tar will complain diff --git a/scripts/patch b/scripts/patch new file mode 100755 index 00000000..78807071 --- /dev/null +++ b/scripts/patch @@ -0,0 +1,179 @@ +#!/bin/bash +# Purpose: *.patchtt functionality for ngcpcfg config/templates +################################################################################ + +set -e +set -u + +# support testsuite +FUNCTIONS="${FUNCTIONS:-/usr/share/ngcp-ngcpcfg/functions/}" +HELPER="${HELPER:-/usr/share/ngcp-ngcpcfg/helper/}" +SCRIPTS="${SCRIPTS:-/usr/share/ngcp-ngcpcfg/scripts/}" + +if ! [ -r "${FUNCTIONS}"/main ] ; then + printf "Error: %s/main could not be read. Exiting.\n" "${FUNCTIONS}" >&2 + exit 1 +fi + +. "${FUNCTIONS}"/main + +cd "${NGCPCTL_MAIN}" + +## functions {{{ +patch_help() { + export TIME_FORMAT='' + log_info "'ngcpcfg patch' walks through all templates searching for '*.patchtt.tt2' files" + log_info "and generates '*.customtt.tt2' files based on the original template" + log_info "" + log_info "Sample:" + log_info " ngcpcfg patch [--help] []" + log_info "" + log_info "Run 'man ngcpcfg' for more information." +} + +patch_search() { + log_debug "Searching for patchtt files" + + local patchlist + patchlist=$(mktemp) + + local a="\.sp[12]?" + local b="\.(web|db|prx|lb|slb)[0-9]+[ab]?" + local awk_regexp=".*patchtt\.tt2(${a}|${b})?$" + + for dir in ${CONFIG_POOL} ; do + [ -n "${dir}" ] || log_error "${dir} doesn't exist" + # iterate over all files + while read -r patch ; do + # *NO* arguments provided via cmdline + if [ -z "${1:-}" ] ; then + log_debug "Found patch '${patch}'" + echo "${patch}" >> "${patchlist}" + else + # arguments (file list/pattern) provided via cmdline + for arg in "$@"; do + if echo "${patch}" | grep -q -- "${arg}" ; then + log_debug "Processing patch '${patch}' as requested" + echo "${patch}" >> "${patchlist}" + fi + done + fi + done < <(find "${TEMPLATE_POOL_BASE}/${dir}" -regextype awk -iregex "${awk_regexp}") + done + + # output patch list, make sure we provide the file names just once + sort -u "${patchlist}" + + if [ -n "${DEBUG:-}" ] ; then + # send to stderr since stdout is used from outside + log_debug "Not removing temporary patchlist files since we are in debug mode:" >&2 + log_debug " patchlist = ${patchlist}" >&2 + else + rm -f "${patchlist}" + fi +} + +patch_validate() { + local patch="$1" + log_debug "Validating patch: '${patch}'" + + if [ ! -f "${patch}" ] ; then + log_error "Missing patch file '${patch}'" + bad_patches+=("${patch}") + return 1 + fi + + local template="${patch%%.patchtt*}.tt2" + + if [ -f "${template}" ] ; then + log_debug "Found template for the patch: '${template}'" + else + log_error "Missing template for patch '${patch}'" + bad_patches+=("${patch}") + return 1 + fi + + local customtt="${patch//.patchtt/.customtt}" + if [ -f "${customtt}" ] ; then + log_debug "Overwriting customtt '${customtt}'" + else + log_debug "Not found customtt for the patch: '${customtt}'" + fi +} + +patch_apply() { + local patch="$1" + local apply="${2:-false}" + local template="${patch%%.patchtt*}.tt2" + local customtt="${patch//.patchtt/.customtt}" + local patch_output + patch_output=$(mktemp) + + local patch_opts=() + patch_opts+=(--input="${patch}") + patch_opts+=(--prefix=/dev/null) # do not produce .orig backup files + patch_opts+=(--reject-file=-) # do not produce .rej file + if "${apply}" ; then + patch_opts+=(--output="${customtt}") + else + patch_opts+=(--dry-run) + patch_opts+=(--output=/dev/null) + fi + + log_debug "Generating customtt '${customtt}' from '${patch}' (apply=${apply})" + + log_debug "Executing: patch ${patch_opts[*]} ${template}" + if patch "${patch_opts[@]}" "${template}" >"${patch_output}" 2>&1 ; then + if "${apply}" ; then + log_info "Successfully created '${customtt}'" + else + log_debug "Patch '${patch}' can be applied" + fi + else + log_error "The patch '${patch}' cannot be applied:" + cat "${patch_output}" >&2 + bad_patches+=("${patch}") + fi + + rm -f "${patch_output}" +} + +patch_main() { + for patch in $(patch_search "$@") ; do + log_info "Validating patch '${patch}'" + patch_validate "${patch}" && patch_apply "${patch}" "false" + done + + if [ "${#bad_patches[@]}" != "0" ] ; then + log_debug "Aborted here due to failed patch validation above" + return + fi + + for patch in $(patch_search "$@") ; do + log_info "Applying patch '${patch}'" + patch_apply "${patch}" "true" + done +} +## }}} + +if [ "${1:-}" = "--help" ]; then + patch_help + exit 0 +fi + +declare -a bad_patches=() + +patch_main "$@" + +if [ "${#bad_patches[@]}" = "0" ] ; then + log_info "Patch operation has finished successfully." +else + log_error "Some operations above finished with an error for the patch(es):" + bad_patches_unique=($(echo "${bad_patches[@]}" | tr ' ' '\n' | sort -u)) + printf '\t%s\n' "${bad_patches_unique[@]}" + RC=1 +fi + +exit "${RC:-0}" + +## END OF FILE #################################################################