TT#65907 Speedup generate_template_list() by removing unnecessary loops

The previous code had three including loops:
 - loop over all folders ${CONFIG_POOL}
 -  run find to get list of all files
 -  loop over all files in list
 -    check if no files requested (build all) then remember file
 -    loop over all requested files
 -     check if file was requested then remember file
 - process further with all files we remembered above

The new logic is much simpler and faster:
 - loop over all folders ${CONFIG_POOL}, remove missing
 - run find on all available folders to get list of all files 'filelist_allfiles'
 - check IF no files requested (build all)
 -   THEN just use all files from list 'filelist_allfiles'
 -   ELSE loop over all requested by user files (normally one/several)
 -     grep requested file from 'filelist_allfiles'
 - process further with all files we remembered above

The two main ideas here:
 - just use all files if we build all files, no need to loop one by one
 - grep requested files from "all files" instead of searching them in loop

At the end we are 30% faster in case if we build one file only:

> Command: time ngcpcfg build /etc/default/ngcp-roles
> Old code: real    0m1.587s
> New code: real    0m1.060s

Change-Id: I206557b004ff95c0607150a61a9e6e743ab8cd29
changes/07/33607/6
Alexander Lutay 6 years ago
parent 2f69005c43
commit ef9ebde2f3

@ -217,27 +217,34 @@ fi
generate_template_list() {
[ -n "$TEMPLATE_POOL_BASE" ] || return 1
local filelist_allfiles=$(mktemp)
local filelist_prepared=$(mktemp)
local filelist_final=$(mktemp)
local filelist_sorted=$(mktemp)
declare -a dirs_to_process=()
for dir in ${CONFIG_POOL} ; do
[ -n "${dir}" ] || ( echo "${dir} doesn't exist" >&2 ; continue )
# iterate over all files
for file in $(find "$TEMPLATE_POOL_BASE/${dir}" -name \*.tt2 -o -name \*.tt2"${HA_FILE:-}" -o -name \*.tt2"${HOST_FILE:-}" -o -name \*.tt2"${PAIR_FILE:-}") ; do
# *NO* arguments provided via cmdline
dirs_to_process+=("$TEMPLATE_POOL_BASE/${dir}")
done
# find all files we should process
find "${dirs_to_process[@]}" \
-name \*.tt2 \
-o -name \*.tt2"${HA_FILE:-}" \
-o -name \*.tt2"${HOST_FILE:-}" \
-o -name \*.tt2"${PAIR_FILE:-}" \
> "${filelist_allfiles}"
# argument(s) (file list/pattern) provided via cmdline
if [ -z "${1:-}" ] ; then
echo "$file" >> "${filelist_prepared}"
cat "${filelist_allfiles}" >> "${filelist_prepared}"
else
# arguments (file list/pattern) provided via cmdline
# limit processing to requested file(s) only
for arg in "$@"; do
if echo $file | grep -q -- "${arg}" ; then
echo "$file" >> "${filelist_prepared}"
fi
grep -- "${arg}" "${filelist_allfiles}" >> "${filelist_prepared}"
done
fi
done
done
# remove all filenames where a preferred filename exists
# foo.customtt.tt2.hostname > foo.customtt.tt2.pairname > foo.customtt.tt2.spX > foo.customtt.tt2 > foo.tt2.hostname > foo.tt2.pairname > foo.tt2.spX > foo.tt2
@ -336,14 +343,15 @@ generate_template_list() {
if [ -n "${DEBUG:-}" ] ; then
# send to stderr since stdout is used from outside
log_debug "Not removing temporary filelist files since we are in debug mode:" >&2
log_debug " filelist_allfiles = ${filelist_allfiles}" >&2
log_debug " filelist_prepared = ${filelist_prepared}" >&2
log_debug " filelist_final = ${filelist_final}" >&2
log_debug " filelist_sorted = ${filelist_sorted}" >&2
else
rm -f "${filelist_prepared}" "${filelist_final}" "${filelist_sorted}"
rm -f "${filelist_allfiles}" "${filelist_prepared}" "${filelist_final}" "${filelist_sorted}"
fi
unset filelist_prepared filelist_final
unset filelist_allfiles filelist_prepared filelist_final dirs_to_process
}
record_commit_id() {

Loading…
Cancel
Save