run_tests.sh

   1 #!/usr/bin/env bash
   2
   3 line='........................................'
   4 uenv='{
   5         "REQUIRE_SEARCH_PATH": [
   6                 "./tests/lib/*.uc",
   7                 "./root/usr/share/ucode/*.uc",
   8                 "/usr/local/lib/ucode/*.so",
   9                 "/usr/lib/ucode/*.so"
  10         ]
  11 }'
  12
  13 extract_sections() {
  14         local file=$1
  15         local dir=$2
  16         local count=0
  17         local tag line outfile
  18
  19         while IFS= read -r line; do
  20                 case "$line" in
  21                         "-- Testcase --")
  22                                 tag="test"
  23                                 count=$((count + 1))
  24                                 outfile=$(printf "%s/%03d.in" "$dir" $count)
  25                                 printf "" > "$outfile"
  26                         ;;
  27                         "-- Environment --")
  28                                 tag="env"
  29                                 count=$((count + 1))
  30                                 outfile=$(printf "%s/%03d.env" "$dir" $count)
  31                                 printf "" > "$outfile"
  32                         ;;
  33                         "-- Expect stdout --"|"-- Expect stderr --"|"-- Expect exitcode --")
  34                                 tag="${line#-- Expect }"
  35                                 tag="${tag% --}"
  36                                 count=$((count + 1))
  37                                 outfile=$(printf "%s/%03d.%s" "$dir" $count "$tag")
  38                                 printf "" > "$outfile"
  39                         ;;
  40                         "-- File "*" --")
  41                                 tag="file"
  42                                 outfile="${line#-- File }"
  43                                 outfile="$(echo "${outfile% --}" | xargs)"
  44                                 outfile="$dir/files$(readlink -m "/${outfile:-file}")"
  45                                 mkdir -p "$(dirname "$outfile")"
  46                                 printf "" > "$outfile"
  47                         ;;
  48                         "-- End --")
  49                                 tag=""
  50                                 outfile=""
  51                         ;;
  52                         *)
  53                                 if [ -n "$tag" ]; then
  54                                         printf "%s\\n" "$line" >> "$outfile"
  55                                 fi
  56                         ;;
  57                 esac
  58         done < "$file"
  59
  60         return $(ls -l "$dir/"*.in 2>/dev/null | wc -l)
  61 }
  62
  63 run_testcase() {
  64         local num=$1
  65         local dir=$2
  66         local in=$3
  67         local env=$4
  68         local out=$5
  69         local err=$6
  70         local code=$7
  71         local fail=0
  72
  73         ucode -S ${uenv:+-e "$uenv"} -e '{
  74                 "MOCK_SEARCH_PATH": [
  75                         "'"$dir"'/files",
  76                         "./tests/mocks"
  77                 ]
  78         }' ${env:+-e "$(cat "$env")"} -m mocklib -m fw4 -i - <"$in" >"$dir/res.out" 2>"$dir/res.err"
  79
  80         printf "%d\n" $? > "$dir/res.code"
  81
  82         touch "$dir/empty"
  83
  84         if ! cmp -s "$dir/res.err" "${err:-$dir/empty}"; then
  85                 [ $fail = 0 ] && printf "!\n"
  86                 printf "Testcase #%d: Expected stderr did not match:\n" $num
  87                 diff -u --color=always --label="Expected stderr" --label="Resulting stderr" "${err:-$dir/empty}" "$dir/res.err"
  88                 printf -- "---\n"
  89                 fail=1
  90         fi
  91
  92         if ! cmp -s "$dir/res.out" "${out:-$dir/empty}"; then
  93                 [ $fail = 0 ] && printf "!\n"
  94                 printf "Testcase #%d: Expected stdout did not match:\n" $num
  95                 diff -u --color=always --label="Expected stdout" --label="Resulting stdout" "${out:-$dir/empty}" "$dir/res.out"
  96                 printf -- "---\n"
  97                 fail=1
  98         fi
  99
 100         if [ -n "$code" ] && ! cmp -s "$dir/res.code" "$code"; then
 101                 [ $fail = 0 ] && printf "!\n"
 102                 printf "Testcase #%d: Expected exit code did not match:\n" $num
 103                 diff -u --color=always --label="Expected code" --label="Resulting code" "$code" "$dir/res.code"
 104                 printf -- "---\n"
 105                 fail=1
 106         fi
 107
 108         return $fail
 109 }
 110
 111 run_test() {
 112         local file=$1
 113         local name=${file##*/}
 114         local res ecode eout eerr ein eenv tests
 115         local testcase_first=0 failed=0 count=0
 116
 117         printf "%s %s " "$name" "${line:${#name}}"
 118
 119         mkdir "/tmp/test.$$"
 120
 121         extract_sections "$file" "/tmp/test.$$"
 122         tests=$?
 123
 124         [ -f "/tmp/test.$$/001.in" ] && testcase_first=1
 125
 126         for res in "/tmp/test.$$/"[0-9]*; do
 127                 case "$res" in
 128                         *.in)
 129                                 count=$((count + 1))
 130
 131                                 if [ $testcase_first = 1 ]; then
 132                                         # Flush previous test
 133                                         if [ -n "$ein" ]; then
 134                                                 run_testcase $count "/tmp/test.$$" "$ein" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 135
 136                                                 eout=""
 137                                                 eerr=""
 138                                                 ecode=""
 139                                                 eenv=""
 140                                         fi
 141
 142                                         ein=$res
 143                                 else
 144                                         run_testcase $count "/tmp/test.$$" "$res" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 145
 146                                         eout=""
 147                                         eerr=""
 148                                         ecode=""
 149                                         eenv=""
 150                                 fi
 151
 152                         ;;
 153                         *.env) eenv=$res ;;
 154                         *.stdout) eout=$res ;;
 155                         *.stderr) eerr=$res ;;
 156                         *.exitcode) ecode=$res ;;
 157                 esac
 158         done
 159
 160         # Flush last test
 161         if [ $testcase_first = 1 ] && [ -n "$eout$eerr$ecode" ]; then
 162                 run_testcase $count "/tmp/test.$$" "$ein" "$eenv" "$eout" "$eerr" "$ecode" || failed=$((failed + 1))
 163         fi
 164
 165         rm -r "/tmp/test.$$"
 166
 167         if [ $failed = 0 ]; then
 168                 printf "OK\n"
 169         else
 170                 printf "%s %s FAILED (%d/%d)\n" "$name" "${line:${#name}}" $failed $tests
 171         fi
 172
 173         return $failed
 174 }
 175
 176
 177 n_tests=0
 178 n_fails=0
 179
 180 select_tests="$@"
 181
 182 use_test() {
 183         local input="$(readlink -f "$1")"
 184         local test
 185
 186         [ -f "$input" ] || return 1
 187         [ -n "$select_tests" ] || return 0
 188
 189         for test in "$select_tests"; do
 190                 test="$(readlink -f "$test")"
 191
 192                 [ "$test" != "$input" ] || return 0
 193         done
 194
 195         return 1
 196 }
 197
 198 for catdir in tests/[0-9][0-9]_*; do
 199         [ -d "$catdir" ] || continue
 200
 201         printf "\n##\n## Running %s tests\n##\n\n" "${catdir##*/[0-9][0-9]_}"
 202
 203         for testfile in "$catdir/"[0-9][0-9]_*; do
 204                 use_test "$testfile" || continue
 205
 206                 n_tests=$((n_tests + 1))
 207                 run_test "$testfile" || n_fails=$((n_fails + 1))
 208         done
 209 done
 210
 211 printf "\nRan %d tests, %d okay, %d failures\n" $n_tests $((n_tests - n_fails)) $n_fails
 212 exit $n_fails