run_tests.sh

   1 #!/usr/bin/env bash
   2
   3 line='........................................'
   4 ucode='ucode -S -T, -L./tests/lib -L./root/usr/share/ucode'
   5
   6 extract_sections() {
   7         local file=$1
   8         local dir=$2
   9         local count=0
  10         local tag line outfile
  11
  12         while IFS= read -r line; do
  13                 case "$line" in
  14                         "-- Testcase --")
  15                                 tag="test"
  16                                 count=$((count + 1))
  17                                 outfile=$(printf "%s/%03d.in" "$dir" $count)
  18                                 printf "" > "$outfile"
  19                         ;;
  20                         "-- Environment --")
  21                                 tag="env"
  22                                 count=$((count + 1))
  23                                 outfile=$(printf "%s/%03d.env" "$dir" $count)
  24                                 printf "" > "$outfile"
  25                         ;;
  26                         "-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --")
  27                                 tag="${line#-- Expect }"
  28                                 tag="${tag% --}"
  29                                 count=$((count + 1))
  30                                 outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag")
  31                                 printf "" > "$outfile"
  32                         ;;
  33                         "-- File "*" --")
  34                                 tag="file"
  35                                 outfile="${line#-- File }"
  36                                 outfile="$(echo "${outfile% --}" | xargs)"
  37                                 outfile="$dir/files$(readlink -m "/${outfile:-file}")"
  38                                 mkdir -p "$(dirname "$outfile")"
  39                                 printf "" > "$outfile"
  40                         ;;
  41                         "-- End --")
  42                                 tag=""
  43                                 outfile=""
  44                         ;;
  45                         *)
  46                                 if [ -n "$tag" ]; then
  47                                         printf "%s\\n" "$line" >> "$outfile"
  48                                 fi
  49                         ;;
  50                 esac
  51         done < "$file"
  52
  53         return $(ls -l "$dir/"*.in 2>/dev/null | wc -l)
  54 }
  55
  56 run_testcase() {
  57         local num=$1
  58         local dir=$2
  59         local in=$3
  60         local env=$4
  61         local out=$5
  62         local err=$6
  63         local code=$7
  64         local fail=0
  65
  66         $ucode \
  67                 -D MOCK_SEARCH_PATH='["'"$dir"'/files", "./tests/mocks"]' \
  68                 ${env:+-F "$env"} \
  69                 -l mocklib -l fw4 \
  70                 - <"$in" >"$dir/res.out" 2>"$dir/res.err"
  71
  72         printf "%d\n" $? > "$dir/res.code"
  73
  74         touch "$dir/empty"
  75
  76         if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then
  77                 [ $fail = 0 ] && printf "!\n"
  78                 printf "Testcase #%d: Expected stderr did not match:\n" $num
  79                 diff -u --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err"
  80                 printf -- "---\n"
  81                 fail=1
  82         fi
  83
  84         if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then
  85                 [ $fail = 0 ] && printf "!\n"
  86                 printf "Testcase #%d: Expected stdout did not match:\n" $num
  87                 diff -u --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out"
  88                 printf -- "---\n"
  89                 fail=1
  90         fi
  91
  92         if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then
  93                 [ $fail = 0 ] && printf "!\n"
  94                 printf "Testcase #%d: Expected exit code did not match:\n" $num
  95                 diff -u --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code"
  96                 printf -- "---\n"
  97                 fail=1
  98         fi
  99
 100         return $fail
 101 }
 102
 103 run_test() {
 104         local file=$1
 105         local name=${file##*/}
 106         local res ecode eout eerr ein eenv tests
 107         local testcase_first=0 failed=0 count=0
 108
 109         printf "%s %s " "$name" "${line:${#name}}"
 110
 111         mkdir "/tmp/test.$$"
 112
 113         extract_sections "$file" "/tmp/test.$$"
 114         tests=$?
 115
 116         [ -f "/tmp/test.$$/001.in" ] && testcase_first=1
 117
 118         for res in "/tmp/test.$$/"[0-9]*; do
 119                 case "$res" in
 120                         *.in)
 121                                 count=$((count + 1))
 122
 123                                 if [ $testcase_first = 1 ]; then
 124                                         # Flush previous test
 125                                         if [ -n "$ein" ]; then
 126                                                 run_testcase $count "/tmp/test.$$" "$ein" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 127
 128                                                 eout=""
 129                                                 eerr=""
 130                                                 ecode=""
 131                                                 eenv=""
 132                                         fi
 133
 134                                         ein=$res
 135                                 else
 136                                         run_testcase $count "/tmp/test.$$" "$res" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 137
 138                                         eout=""
 139                                         eerr=""
 140                                         ecode=""
 141                                         eenv=""
 142                                 fi
 143
 144                         ;;
 145                         *.env) eenv=$res ;;
 146                         *.stdout) eout=$res ;;
 147                         *.stderr) eerr=$res ;;
 148                         *.exitcode) ecode=$res ;;
 149                 esac
 150         done
 151
 152         # Flush last test
 153         if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then
 154                 run_testcase $count "/tmp/test.$$" "$ein" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 155         fi
 156
 157         rm -r "/tmp/test.$$"
 158
 159         if [ $failed = 0 ]; then
 160                 printf "OK\n"
 161         else
 162                 printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests
 163         fi
 164
 165         return $failed
 166 }
 167
 168
 169 n_tests=0
 170 n_fails=0
 171
 172 select_tests="$@"
 173
 174 use_test() {
 175         local input="$(readlink -f "$1")"
 176         local test
 177
 178         [ -f "$input" ] || return 1
 179         [ -n "$select_tests" ] || return 0
 180
 181         for test in "$select_tests"; do
 182                 test="$(readlink -f "$test")"
 183
 184                 [ "$test" != "$input" ] || return 0
 185         done
 186
 187         return 1
 188 }
 189
 190 for catdir in tests/[0-9][0-9]_*; do
 191         [ -d "$catdir" ] || continue
 192
 193         printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}"
 194
 195         for testfile in "$catdir/"[0-9][0-9]_*; do
 196                 use_test "$testfile" || continue
 197
 198                 n_tests=$((n_tests + 1))
 199                 run_test "$testfile" || n_fails=$((n_fails + 1))
 200         done
 201 done
 202
 203 printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails
 204 exit $n_fails