* commit '35d1f726eb9fdd376ab900587fb02122b72f2b9a': fate: Add --ignore-tests configure option for omitting specific FATE tests
Unified · Split
| @@ -440,6 +440,8 @@ Developer options (useful when working on FFmpeg itself): · 439 unmodified lines | |||
| 440 | (only applies to --disable-optimizations builds) | 440 | (only applies to --disable-optimizations builds) |
| 441 | --enable-osfuzz Enable building fuzzer tool | 441 | --enable-osfuzz Enable building fuzzer tool |
| 442 | --libfuzzer=PATH path to libfuzzer | 442 | --libfuzzer=PATH path to libfuzzer |
| 443 | --ignore-tests=TESTS comma-separated list (without "fate-" prefix | ||
| 444 | in the name) of tests whose result is ignored | ||
| 443 | 445 | ||
| 444 | NOTE: Object files are built at the place where configure is launched. | 446 | NOTE: Object files are built at the place where configure is launched. |
| 445 | EOF | 447 | EOF |
| @@ -2209,6 +2211,7 @@ CMDLINE_SET=" · 1763 unmodified lines | |||
| 2209 | host_ld | 2211 | host_ld |
| 2210 | host_ldflags | 2212 | host_ldflags |
| 2211 | host_os | 2213 | host_os |
| 2214 | ignore_tests | ||
| 2212 | install | 2215 | install |
| 2213 | ld | 2216 | ld |
| 2214 | ln_s | 2217 | ln_s |
| @@ -6617,6 +6620,13 @@ for type in decoder encoder hwaccel parser demuxer muxer protocol filter bsf ind · 4402 unmodified lines | |||
| 6617 | echo | 6620 | echo |
| 6618 | done | 6621 | done |
| 6619 | 6622 | ||
| 6623 | if test -n "$ignore_tests"; then | ||
| 6624 | ignore_tests=$(echo $ignore_tests | tr ',' ' ') | ||
| 6625 | echo "Ignored FATE tests:" | ||
| 6626 | echo $ignore_tests | print_in_columns | ||
| 6627 | echo | ||
| 6628 | fi | ||
| 6629 | |||
| 6620 | echo "License: $license" | 6630 | echo "License: $license" |
| 6621 | 6631 | ||
| 6622 | echo "Creating configuration files ..." | 6632 | echo "Creating configuration files ..." |
| @@ -6754,6 +6764,7 @@ VERSION_SCRIPT_POSTPROCESS_CMD=${VERSION_SCRIPT_POSTPROCESS_CMD} · 131 unmodified lines | |||
| 6754 | SAMPLES:=${samples:-\$(FATE_SAMPLES)} | 6764 | SAMPLES:=${samples:-\$(FATE_SAMPLES)} |
| 6755 | NOREDZONE_FLAGS=$noredzone_flags | 6765 | NOREDZONE_FLAGS=$noredzone_flags |
| 6756 | LIBFUZZER_PATH=$libfuzzer_path | 6766 | LIBFUZZER_PATH=$libfuzzer_path |
| 6767 | IGNORE_TESTS=$ignore_tests | ||
| 6757 | EOF | 6768 | EOF |
| 6758 | 6769 | ||
| 6759 | map 'eval echo "${v}_FFLIBS=\$${v}_deps" >> ffbuild/config.mak' $LIBRARY_LIST | 6770 | map 'eval echo "${v}_FFLIBS=\$${v}_deps" >> ffbuild/config.mak' $LIBRARY_LIST |
| @@ -6,6 +6,7 @@ workdir= # directory in which to do all the work · 5 unmodified lines | |||
| 6 | #fate_recv="ssh -T fate@fate.ffmpeg.org" # command to submit report | 6 | #fate_recv="ssh -T fate@fate.ffmpeg.org" # command to submit report |
| 7 | comment= # optional description | 7 | comment= # optional description |
| 8 | build_only= # set to "yes" for a compile-only instance that skips tests | 8 | build_only= # set to "yes" for a compile-only instance that skips tests |
| 9 | ignore_tests= | ||
| 9 | 10 | ||
| 10 | # the following are optional and map to configure options | 11 | # the following are optional and map to configure options |
| 11 | arch= | 12 | arch= |
| @@ -212,12 +212,14 @@ FATE_UTILS = base64 tiny_psnr tiny_ssim audiomatch · 211 unmodified lines | |||
| 212 | 212 | ||
| 213 | TOOL = ffmpeg | 213 | TOOL = ffmpeg |
| 214 | 214 | ||
| 215 | $(addprefix fate-, $(IGNORE_TESTS)): REPORT=ignore | ||
| 216 | |||
| 215 | fate:: $(FATE) | 217 | fate:: $(FATE) |
| 216 | 218 | ||
| 217 | $(FATE) $(FATE_TESTS-no): export PROGSUF = $(PROGSSUF) | 219 | $(FATE) $(FATE_TESTS-no): export PROGSUF = $(PROGSSUF) |
| 218 | $(FATE) $(FATE_TESTS-no): $(FATE_UTILS:%=tests/%$(HOSTEXESUF)) | 220 | $(FATE) $(FATE_TESTS-no): $(FATE_UTILS:%=tests/%$(HOSTEXESUF)) |
| 219 | @echo "TEST $(@:fate-%=%)" | 221 | @echo "TEST $(@:fate-%=%)" |
| 220 | $(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' | 222 | $(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' |
| 221 | 223 | ||
| 222 | fate-list: | 224 | fate-list: |
| 223 | @printf '%s\n' $(sort $(FATE)) | 225 | @printf '%s\n' $(sort $(FATE)) |
| @@ -24,6 +24,7 @@ size_tolerance=${14:-0} · 23 unmodified lines | |||
| 24 | cmp_unit=${15:-2} | 24 | cmp_unit=${15:-2} |
| 25 | gen=${16:-no} | 25 | gen=${16:-no} |
| 26 | hwaccel=${17:-none} | 26 | hwaccel=${17:-none} |
| 27 | report_type=${18:-standard} | ||
| 27 | 28 | ||
| 28 | outdir="tests/data/fate" | 29 | outdir="tests/data/fate" |
| 29 | outfile="${outdir}/${test}" | 30 | outfile="${outdir}/${test}" |
| @@ -362,13 +363,17 @@ if test -e "$ref" || test $cmp = "oneline" || test $cmp = "grep" ; then · 332 unmodified lines | |||
| 362 | esac | 363 | esac |
| 363 | cmperr=$? | 364 | cmperr=$? |
| 364 | test $err = 0 && err=$cmperr | 365 | test $err = 0 && err=$cmperr |
| 365 | test $err = 0 || cat $cmpfile | 366 | if [ "$report_type" = "ignore" ]; then |
| 367 | test $err = 0 || echo "IGNORE fate-${test}" && err=0 | ||
| 368 | else | ||
| 369 | test $err = 0 || cat $cmpfile | ||
| 370 | fi | ||
| 366 | else | 371 | else |
| 367 | echo "reference file '$ref' not found" | 372 | echo "reference file '$ref' not found" |
| 368 | err=1 | 373 | err=1 |
| 369 | fi | 374 | fi |
| 370 | 375 | ||
| 371 | if [ $err -eq 0 ]; then | 376 | if [ $err -eq 0 ] && test $report_type = "standard" ; then |
| 372 | unset cmpo erro | 377 | unset cmpo erro |
| 373 | else | 378 | else |
| 374 | cmpo="$($base64 <$cmpfile)" | 379 | cmpo="$($base64 <$cmpfile)" |
| @@ -49,6 +49,7 @@ configure()( · 48 unmodified lines | |||
| 49 | --enable-gpl \ | 49 | --enable-gpl \ |
| 50 | --enable-memory-poisoning \ | 50 | --enable-memory-poisoning \ |
| 51 | --enable-avresample \ | 51 | --enable-avresample \ |
| 52 | ${ignore_tests:+--ignore-tests="$ignore_tests"} \ | ||
| 52 | ${arch:+--arch=$arch} \ | 53 | ${arch:+--arch=$arch} \ |
| 53 | ${cpu:+--cpu="$cpu"} \ | 54 | ${cpu:+--cpu="$cpu"} \ |
| 54 | ${toolchain:+--toolchain="$toolchain"} \ | 55 | ${toolchain:+--toolchain="$toolchain"} \ |