TT#65907 Speedup 'ngcpcfg apply' on 1.5 seconds (re-use the same tmp file)

In the past helper/build_config generated new tmp file for every call
and we called it for hundred+ times. Let's generate it once and pass
to helper/build_config to be reused every time.

Command: time ngcpcfg apply "test"
Old code: real   0m17.194s
New code: real   0m15.477s

Change-Id: Ic1c35ccb5c5f92ddee6328502194a5301123bd9d

helper/build_config

@@ -7,8 +7,8 @@
 set -e
 set -u
 
-if [ "${#:-}" -ne 2 ] ; then
+if [ "${#:-}" -ne 3 ] ; then
-  echo "Usage: /usr/share/ngcp-ngcpcfg/helper/build_config <input_file> <output_file>" >&2
+  echo "Usage: /usr/share/ngcp-ngcpcfg/helper/build_config <input_file> <output_file> <tmp_file>" >&2
   exit 1
 fi
 
@@ -27,6 +27,7 @@ fi
 
 input_file="$1"       # like /etc/ngcp-config/templates/etc/mysql/my.cnf.tt2
 output_file="$2"      # like /etc/mysql/my.cnf
+tmp_output_file="$3"  # like /tmp/ngcpcfg.PID12345.KLJhiand/tmp_output_file
 
 if [ -z "${input_file}" ] ; then
   log_error "Missing <input_file> parameter. Exiting." >&2
@@ -38,6 +39,11 @@ if [ -z "${output_file}" ] ; then
   exit 1
 fi
 
+if [ -z "${tmp_output_file}" ] ; then
+  log_error "Missing <tmp_output_file> parameter. Exiting." >&2
+  exit 1
+fi
+
 # export variable for usage within {pre,post}build scripts,
 # OUTPUT_DIRECTORY is for customization during testing
 if [ -n "${OUTPUT_DIRECTORY:-}" ] ; then
@@ -80,10 +86,6 @@ umask 0077
 
 TT_WRAPPER="${HELPER}/tt2-wrapper"
 
-# generate in a temporary location, then move if successful
-tt_tmp_output_file_base="$(basename "${output_file}")"
-tt_tmp_output_file="$(mktemp --tmpdir ngcpcfg."${tt_tmp_output_file_base}".XXXXXXXXXX)"
-
 # XXX: Docker breaks sane Unix expectations when moving a file into /etc/hosts,
 # as it creates a bind mount on that pathname. We need to use an implementation
 # that will fallback to use copy semantics in that case, but will default to
@@ -103,22 +105,23 @@ move()
 }
 
 log_debug "Output file ${output_file} based on ${input_file}"
-log_debug "Executing: $TT_WRAPPER ${input_file} > ${tt_tmp_output_file}"
+log_debug "Executing: $TT_WRAPPER ${input_file} > ${tmp_output_file}"
-log_debug "      and: move ${tt_tmp_output_file} ${output_file}"
+log_debug "      and: move ${tmp_output_file} ${output_file}"
-# We need to use «readlink -f» so that we do not destroy any symlink pointing
+# 1) We need to use «readlink -f» so that we do not destroy any symlink pointing
 # to the real file, which we were previously preserving while using «cat».
-if "$TT_WRAPPER" "${input_file}" > "${tt_tmp_output_file}" 2>/dev/null &&
+# 2) Care about ">" below, do not trust tmp file you receive.
-   ! grep -q -E '^file error' "${tt_tmp_output_file}" 2>/dev/null &&
+if "$TT_WRAPPER" "${input_file}" > "${tmp_output_file}" 2>/dev/null &&
-   move "${tt_tmp_output_file}" "$(readlink -f "${output_file}")" ; then
+   ! grep -q -E '^file error' "${tmp_output_file}" 2>/dev/null &&
+   move "${tmp_output_file}" "$(readlink -f "${output_file}")" ; then
   log_info "Generating ${output_file}: OK"
   RC=0
 else
   log_error "Generating ${output_file} based on ${input_file}: FAILED"
   RC=1
 
-  if [[ -r "${tt_tmp_output_file}" ]] && grep -q -E '^file error' "${tt_tmp_output_file}" ; then
+  if [[ -r "${tmp_output_file}" ]] && grep -q -E '^file error' "${tmp_output_file}" ; then
     log_error "from generated file:"
-    log_error "  $(grep -E '^file error' "${tt_tmp_output_file}")"
+    log_error "  $(grep -E '^file error' "${tmp_output_file}")"
   fi
 
   log_info "NOTE: Check those files for valid syntax and encoding:"
@@ -126,7 +129,7 @@ else
     [ -r "$f" ] && log_info "$f"
   done
   log_info "Running /usr/share/ngcp-ngcpcfg/helper/tt2-wrapper <file>"
-  log_info "or inspecting temporary ${tt_tmp_output_file}"
+  log_info "or inspecting temporary ${tmp_output_file}"
   log_info "should provide more details."
 fi
 

scripts/build

@@ -61,6 +61,10 @@ for f in ${NGCPCTL_CONFIG:-} ${HOST_CONFIG:-} ${LOCAL_CONFIG:-} ${NETWORK_CONFIG
 done
 
 build_config_files() {
+  # prepare tmp file to be reused hundred times inside "${HELPER}/build_config"
+  tmp_output_dir="$(mktemp -d --tmpdir ngcpcfg.PID"${NGCPCFG_PID}".XXXXXXXXXX)"
+  tmp_output_file="${tmp_output_dir}/tmp_output_file"
+
   for input_file in $(generate_template_list "$@") ; do
 
     log_debug "calculate output file" # {{{
@@ -73,10 +77,13 @@ build_config_files() {
     # }}}
 
     log_debug "${HELPER}/build_config ${input_file} ${output_file}"
-    "${HELPER}/build_config" "${input_file}" "${output_file}"
+    "${HELPER}/build_config" "${input_file}" "${output_file}" "${tmp_output_file}"
   done
 
   record_commit_id
+
+  rm -f "${tmp_output_file}"
+  rmdir "${tmp_output_dir}"
 }
 
 # main script

t/test_ngcpcfg_build_syntax.py

@@ -28,7 +28,7 @@ def test_bad_syntax(ngcpcfgcli, tmpdir):
 
     regex1 = re.compile(r"NOTE: Check those files for valid syntax and "
                         "encoding.*etc/bad-syntax.txt.tt2.*or inspecting "
-                        "temporary /tmp/ngcpcfg.bad-syntax.txt.")
+                        "temporary /tmp/ngcpcfg.PID[0-9]+")
     assert re.search(regex1, out.stdout)
 
     regex2 = re.compile(r"Error: Generating /tmp/ngcp-.*-pytest-output.*"