From ef9ebde2f3155f0a18ec4cd9f155a125a1df127c Mon Sep 17 00:00:00 2001
From: Alexander Lutay <alutay@sipwise.com>
Date: Wed, 18 Sep 2019 10:39:17 +0200
Subject: [PATCH] TT#65907 Speedup generate_template_list() by removing
 unnecessary loops

The previous code had three including loops:
 - loop over all folders ${CONFIG_POOL}
 -  run find to get list of all files
 -  loop over all files in list
 -    check if no files requested (build all) then remember file
 -    loop over all requested files
 -     check if file was requested then remember file
 - process further with all files we remembered above

The new logic is much simpler and faster:
 - loop over all folders ${CONFIG_POOL}, remove missing
 - run find on all available folders to get list of all files 'filelist_allfiles'
 - check IF no files requested (build all)
 -   THEN just use all files from list 'filelist_allfiles'
 -   ELSE loop over all requested by user files (normally one/several)
 -     grep requested file from 'filelist_allfiles'
 - process further with all files we remembered above

The two main ideas here:
 - just use all files if we build all files, no need to loop one by one
 - grep requested files from "all files" instead of searching them in loop

At the end we are 30% faster in case if we build one file only:

> Command: time ngcpcfg build /etc/default/ngcp-roles
> Old code: real    0m1.587s
> New code: real    0m1.060s

Change-Id: I206557b004ff95c0607150a61a9e6e743ab8cd29
---
 functions/main | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/functions/main b/functions/main
index 8f8bd8fa..ba4e3762 100644
--- a/functions/main
+++ b/functions/main
@@ -217,28 +217,35 @@ fi
 generate_template_list() {
   [ -n "$TEMPLATE_POOL_BASE" ] || return 1
 
+  local filelist_allfiles=$(mktemp)
   local filelist_prepared=$(mktemp)
   local filelist_final=$(mktemp)
   local filelist_sorted=$(mktemp)
+  declare -a dirs_to_process=()
 
   for dir in ${CONFIG_POOL} ; do
     [ -n "${dir}" ] || ( echo "${dir} doesn't exist" >&2 ; continue )
-    # iterate over all files
-    for file in $(find "$TEMPLATE_POOL_BASE/${dir}" -name \*.tt2 -o -name \*.tt2"${HA_FILE:-}" -o -name \*.tt2"${HOST_FILE:-}" -o -name \*.tt2"${PAIR_FILE:-}") ; do
-      # *NO* arguments provided via cmdline
-      if [ -z "${1:-}" ] ; then
-        echo "$file" >> "${filelist_prepared}"
-      else
-        # arguments (file list/pattern) provided via cmdline
-        for arg in "$@"; do
-          if echo $file | grep -q -- "${arg}" ; then
-            echo "$file" >> "${filelist_prepared}"
-          fi
-        done
-      fi
-    done
+    dirs_to_process+=("$TEMPLATE_POOL_BASE/${dir}")
   done
 
+  # find all files we should process
+  find "${dirs_to_process[@]}" \
+    -name \*.tt2 \
+    -o -name \*.tt2"${HA_FILE:-}" \
+    -o -name \*.tt2"${HOST_FILE:-}" \
+    -o -name \*.tt2"${PAIR_FILE:-}" \
+      > "${filelist_allfiles}"
+
+  # argument(s) (file list/pattern) provided via cmdline
+  if [ -z "${1:-}" ] ; then
+    cat "${filelist_allfiles}" >> "${filelist_prepared}"
+  else
+    # limit processing to requested file(s) only
+    for arg in "$@"; do
+      grep -- "${arg}" "${filelist_allfiles}" >> "${filelist_prepared}"
+    done
+  fi
+
   # remove all filenames where a preferred filename exists
   # foo.customtt.tt2.hostname > foo.customtt.tt2.pairname > foo.customtt.tt2.spX > foo.customtt.tt2 > foo.tt2.hostname > foo.tt2.pairname > foo.tt2.spX > foo.tt2
   for line in $(cat ${filelist_prepared}); do
@@ -336,14 +343,15 @@ generate_template_list() {
   if [ -n "${DEBUG:-}" ] ; then
     # send to stderr since stdout is used from outside
     log_debug "Not removing temporary filelist files since we are in debug mode:" >&2
+    log_debug "  filelist_allfiles = ${filelist_allfiles}" >&2
     log_debug "  filelist_prepared = ${filelist_prepared}" >&2
     log_debug "  filelist_final    = ${filelist_final}" >&2
     log_debug "  filelist_sorted   = ${filelist_sorted}" >&2
   else
-    rm -f "${filelist_prepared}" "${filelist_final}" "${filelist_sorted}"
+    rm -f "${filelist_allfiles}" "${filelist_prepared}" "${filelist_final}" "${filelist_sorted}"
   fi
 
-  unset filelist_prepared filelist_final
+  unset filelist_allfiles filelist_prepared filelist_final dirs_to_process
 }
 
 record_commit_id() {