merge_config.sh shell/sed/grep loop scales poorly and is slow. With Yocto genericarm64 kernel and around 190 config fragments the script takes more than 20 minutes to run on a fast build machine. Re-implementation with awk does the same job in 10 seconds. Using awk since it is likely available in the build environments and using perl, python etc would introduce more complex runtime dependencies. awk is good enough and lot better than shell/sed/grep. Output stays the same but changed execution time means that parallel job output may be ordered differently.
Unified · Split
| @@ -16,8 +16,8 @@ · 15 unmodified lines | |||
| 16 | set -e | 16 | set -e |
| 17 | 17 | ||
| 18 | clean_up() { | 18 | clean_up() { |
| 19 | rm -f $TMP_FILE | 19 | rm -f "$TMP_FILE" |
| 20 | rm -f $MERGE_FILE | 20 | rm -f "$TMP_FILE.new" |
| 21 | } | 21 | } |
| 22 | 22 | ||
| 23 | usage() { | 23 | usage() { |
| @@ -43,6 +43,10 @@ STRICT=false · 19 unmodified lines | |||
| 43 | CONFIG_PREFIX=${CONFIG_-CONFIG_} | 43 | CONFIG_PREFIX=${CONFIG_-CONFIG_} |
| 44 | WARNOVERRIDE=echo | 44 | WARNOVERRIDE=echo |
| 45 | 45 | ||
| 46 | if [ -z "$AWK" ]; then | ||
| 47 | AWK=awk | ||
| 48 | fi | ||
| 49 | |||
| 46 | while true; do | 50 | while true; do |
| 47 | case $1 in | 51 | case $1 in |
| 48 | "-n") | 52 | "-n") |
| @@ -117,11 +121,8 @@ if [ ! -r "$INITFILE" ]; then · 68 unmodified lines | |||
| 117 | fi | 121 | fi |
| 118 | 122 | ||
| 119 | MERGE_LIST=$* | 123 | MERGE_LIST=$* |
| 120 | SED_CONFIG_EXP1="s/^\(${CONFIG_PREFIX}[a-zA-Z0-9_]*\)=.*/\1/p" | ||
| 121 | SED_CONFIG_EXP2="s/^# \(${CONFIG_PREFIX}[a-zA-Z0-9_]*\) is not set$/\1/p" | ||
| 122 | 124 | ||
| 123 | TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX) | 125 | TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX) |
| 124 | MERGE_FILE=$(mktemp ./.merge_tmp.config.XXXXXXXXXX) | ||
| 125 | 126 | ||
| 126 | echo "Using $INITFILE as base" | 127 | echo "Using $INITFILE as base" |
| 127 | 128 | ||
| @@ -136,42 +137,129 @@ for ORIG_MERGE_FILE in $MERGE_LIST ; do · 8 unmodified lines | |||
| 136 | echo "The merge file '$ORIG_MERGE_FILE' does not exist. Exit." >&2 | 137 | echo "The merge file '$ORIG_MERGE_FILE' does not exist. Exit." >&2 |
| 137 | exit 1 | 138 | exit 1 |
| 138 | fi | 139 | fi |
| 139 | cat $ORIG_MERGE_FILE > $MERGE_FILE | 140 | # Use awk for single-pass processing instead of per-symbol grep/sed |
| 140 | CFG_LIST=$(sed -n -e "$SED_CONFIG_EXP1" -e "$SED_CONFIG_EXP2" $MERGE_FILE) | 141 | if ! "$AWK" -v prefix="$CONFIG_PREFIX" \ |
| 141 | 142 | -v warnoverride="$WARNOVERRIDE" \ | |
| 142 | for CFG in $CFG_LIST ; do | 143 | -v strict="$STRICT" \ |
| 143 | grep -q -w $CFG $TMP_FILE || continue | 144 | -v builtin="$BUILTIN" \ |
| 144 | PREV_VAL=$(grep -w $CFG $TMP_FILE) | 145 | -v warnredun="$WARNREDUN" ' |
| 145 | NEW_VAL=$(grep -w $CFG $MERGE_FILE) | 146 | BEGIN { |
| 146 | BUILTIN_FLAG=false | 147 | strict_violated = 0 |
| 147 | if [ "$BUILTIN" = "true" ] && [ "${NEW_VAL#CONFIG_*=}" = "m" ] && [ "${PREV_VAL#CONFIG_*=}" = "y" ]; then | 148 | cfg_regex = "^" prefix "[a-zA-Z0-9_]+" |
| 148 | ${WARNOVERRIDE} Previous value: $PREV_VAL | 149 | notset_regex = "^# " prefix "[a-zA-Z0-9_]+ is not set$" |
| 149 | ${WARNOVERRIDE} New value: $NEW_VAL | 150 | } |
| 150 | ${WARNOVERRIDE} -y passed, will not demote y to m | 151 | |
| 151 | ${WARNOVERRIDE} | 152 | # Extract config name from a line, returns "" if not a config line |
| 152 | BUILTIN_FLAG=true | 153 | function get_cfg(line) { |
| 153 | elif [ "x$PREV_VAL" != "x$NEW_VAL" ] ; then | 154 | if (match(line, cfg_regex)) { |
| 154 | ${WARNOVERRIDE} Value of $CFG is redefined by fragment $ORIG_MERGE_FILE: | 155 | return substr(line, RSTART, RLENGTH) |
| 155 | ${WARNOVERRIDE} Previous value: $PREV_VAL | 156 | } else if (match(line, notset_regex)) { |
| 156 | ${WARNOVERRIDE} New value: $NEW_VAL | 157 | # Extract CONFIG_FOO from "# CONFIG_FOO is not set" |
| 157 | ${WARNOVERRIDE} | 158 | sub(/^# /, "", line) |
| 158 | if [ "$STRICT" = "true" ]; then | 159 | sub(/ is not set$/, "", line) |
| 159 | STRICT_MODE_VIOLATED=true | 160 | return line |
| 160 | fi | 161 | } |
| 161 | elif [ "$WARNREDUN" = "true" ]; then | 162 | return "" |
| 162 | ${WARNOVERRIDE} Value of $CFG is redundant by fragment $ORIG_MERGE_FILE: | 163 | } |
| 163 | fi | 164 | |
| 164 | if [ "$BUILTIN_FLAG" = "false" ]; then | 165 | function warn_builtin(cfg, prev, new) { |
| 165 | sed -i "/$CFG[ =]/d" $TMP_FILE | 166 | if (warnoverride == "true") return |
| 166 | else | 167 | print cfg ": -y passed, will not demote y to m" |
| 167 | sed -i "/$CFG[ =]/d" $MERGE_FILE | 168 | print "Previous value: " prev |
| 168 | fi | 169 | print "New value: " new |
| 169 | done | 170 | print "" |
| 170 | # In case the previous file lacks a new line at the end | 171 | } |
| 171 | echo >> $TMP_FILE | 172 | |
| 172 | cat $MERGE_FILE >> $TMP_FILE | 173 | function warn_redefined(cfg, prev, new) { |
| 173 | done | 174 | if (warnoverride == "true") return |
| 175 | print "Value of " cfg " is redefined by fragment " mergefile ":" | ||
| 176 | print "Previous value: " prev | ||
| 177 | print "New value: " new | ||
| 178 | print "" | ||
| 179 | } | ||
| 180 | |||
| 181 | function warn_redundant(cfg) { | ||
| 182 | if (warnredun != "true" || warnoverride == "true") return | ||
| 183 | print "Value of " cfg " is redundant by fragment " mergefile ":" | ||
| 184 | } | ||
| 185 | |||
| 186 | # First pass: read merge file, store all lines and index | ||
| 187 | FILENAME == ARGV[1] { | ||
| 188 | mergefile = FILENAME | ||
| 189 | merge_lines[FNR] = $0 | ||
| 190 | merge_total = FNR | ||
| 191 | cfg = get_cfg($0) | ||
| 192 | if (cfg != "") { | ||
| 193 | merge_cfg[cfg] = $0 | ||
| 194 | merge_cfg_line[cfg] = FNR | ||
| 195 | } | ||
| 196 | next | ||
| 197 | } | ||
| 174 | 198 | ||
| 199 | # Second pass: process base file (TMP_FILE) | ||
| 200 | FILENAME == ARGV[2] { | ||
| 201 | cfg = get_cfg($0) | ||
| 202 | |||
| 203 | # Not a config or not in merge file - keep it | ||
| 204 | if (cfg == "" || !(cfg in merge_cfg)) { | ||
| 205 | print $0 >> ARGV[3] | ||
| 206 | next | ||
| 207 | } | ||
| 208 | |||
| 209 | prev_val = $0 | ||
| 210 | new_val = merge_cfg[cfg] | ||
| 211 | |||
| 212 | # BUILTIN: do not demote y to m | ||
| 213 | if (builtin == "true" && new_val ~ /=m$/ && prev_val ~ /=y$/) { | ||
| 214 | warn_builtin(cfg, prev_val, new_val) | ||
| 215 | print $0 >> ARGV[3] | ||
| 216 | skip_merge[merge_cfg_line[cfg]] = 1 | ||
| 217 | next | ||
| 218 | } | ||
| 219 | |||
| 220 | # Values equal - redundant | ||
| 221 | if (prev_val == new_val) { | ||
| 222 | warn_redundant(cfg) | ||
| 223 | next | ||
| 224 | } | ||
| 225 | |||
| 226 | # "=n" is the same as "is not set" | ||
| 227 | if (prev_val ~ /=n$/ && new_val ~ / is not set$/) { | ||
| 228 | print $0 >> ARGV[3] | ||
| 229 | next | ||
| 230 | } | ||
| 231 | |||
| 232 | # Values differ - redefined | ||
| 233 | warn_redefined(cfg, prev_val, new_val) | ||
| 234 | if (strict == "true") { | ||
| 235 | strict_violated = 1 | ||
| 236 | } | ||
| 237 | } | ||
| 238 | |||
| 239 | # output file, skip all lines | ||
| 240 | FILENAME == ARGV[3] { | ||
| 241 | nextfile | ||
| 242 | } | ||
| 243 | |||
| 244 | END { | ||
| 245 | # Newline in case base file lacks trailing newline | ||
| 246 | print "" >> ARGV[3] | ||
| 247 | # Append merge file, skipping lines marked for builtin preservation | ||
| 248 | for (i = 1; i <= merge_total; i++) { | ||
| 249 | if (!(i in skip_merge)) { | ||
| 250 | print merge_lines[i] >> ARGV[3] | ||
| 251 | } | ||
| 252 | } | ||
| 253 | if (strict_violated) { | ||
| 254 | exit 1 | ||
| 255 | } | ||
| 256 | }' \ | ||
| 257 | "$ORIG_MERGE_FILE" "$TMP_FILE" "$TMP_FILE.new"; then | ||
| 258 | # awk exited non-zero, strict mode was violated | ||
| 259 | STRICT_MODE_VIOLATED=true | ||
| 260 | fi | ||
| 261 | mv "$TMP_FILE.new" "$TMP_FILE" | ||
| 262 | done | ||
| 175 | if [ "$STRICT_MODE_VIOLATED" = "true" ]; then | 263 | if [ "$STRICT_MODE_VIOLATED" = "true" ]; then |
| 176 | echo "The fragment redefined a value and strict mode had been passed." | 264 | echo "The fragment redefined a value and strict mode had been passed." |
| 177 | exit 1 | 265 | exit 1 |