CI: add runtime tests for packages

Additional to manual runtime tests this CI addition runs a custom test
script per package. Ideally this lowers the errors of package bumps,
something which is time consuming when done manually for multiple
architectures.

This CI uses the official OpenWrt containers and tries to install and
run compiled packages. The run depends on the content of `test.sh`,
which is an `ash` script. It's called with the *packge name* and
*package version* as arguments. This allows different behaviour if
a single package generates multiple IPK files. The version is usable for
the most trivial runtime check, e.g. `tmux -V | grep "$2"`.

The current approach uses the qus project[1] which contains multiple
QEMU binaries to run various architectures.

[1]: https://github.com/dbhi/qus

Signed-off-by: Paul Spooren <mail@aparcar.org>

diff --git a/.github/workflows/Dockerfile b/.github/workflows/Dockerfile
new file mode 100644 (file)
index 0000000..aa269b7
--- /dev/null
+++ b/.github/workflows/Dockerfile
@@ -0,0 +1,6 @@
+ARG ARCH=x86-64
+FROM openwrtorg/rootfs:$ARCH
+
+ADD entrypoint.sh /entrypoint.sh
+
+CMD ["/entrypoint.sh"]

diff --git a/.github/workflows/entrypoint.sh b/.github/workflows/entrypoint.sh
new file mode 100755 (executable)
index 0000000..21e9035
--- /dev/null
+++ b/.github/workflows/entrypoint.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+mkdir -p /var/lock/
+
+opkg update
+
+for PKG in /ci/*.ipk; do
+       tar -xzOf "$PKG" ./control.tar.gz | tar xzf - ./control 
+       PKG_NAME=$(sed -ne 's#^Package: \(.*\)$#\1#p' ./control)
+       PKG_VERSION=$(sed -ne 's#^Version: \(.*\)$#\1#p' ./control)
+
+       echo "Testing package $PKG_NAME ($PKG_VERSION)"
+
+       opkg install "$PKG"
+
+       TEST_SCRIPT=$(find /ci/ -name "$PKG_NAME" -type d)/test.sh
+       if [ -f "$TEST_SCRIPT" ]; then
+               echo "Use package specific test.sh"
+               if sh "$TEST_SCRIPT" "$PKG_NAME" "$PKG_VERSION"; then
+                       echo "Test successfull"
+               else
+                       echo "Test failed"
+                       exit 1
+               fi
+       else
+               echo "No test.sh script available"
+       fi
+
+       opkg remove "$PKG_NAME"
+done

diff --git a/.github/workflows/multi-arch-test-build.yml b/.github/workflows/multi-arch-test-build.yml
index b026b10b29975b4ef26577cd10592d884ef12f78..97fa8176b6dac2c90099085677f33d7a8cd2ec68 100644 (file)
--- a/.github/workflows/multi-arch-test-build.yml
+++ b/.github/workflows/multi-arch-test-build.yml
@@ -7,23 +7,29 @@ on:
 
 jobs:
   build:
-    name: ${{ matrix.arch }} build
+    name: Test ${{ matrix.arch }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         arch:
-          - aarch64_generic
           - arc_arc700
           - arc_archs
-          - arm_cortex-a15_neon-vfpv4
           - arm_cortex-a9_neon
           - arm_cortex-a9_vfpv3-d16
-          - i386_pentium4
           - mips_24kc
           - powerpc_464fp
           - powerpc_8540
-          - x86_64
+        runtime_test: [false]
+        include:
+          - arch: aarch64_generic
+            runtime_test: true
+          - arch: arm_cortex-a15_neon-vfpv4
+            runtime_test: true
+          - arch: i386_pentium4
+            runtime_test: true
+          - arch: x86_64
+            runtime_test: true
 
     steps:
       - uses: actions/checkout@v2
@@ -50,11 +56,14 @@ jobs:
           ARCH: ${{ matrix.arch }}
           FEEDNAME: packages_ci
 
+      - name: Move created packages to project dir
+        run: cp bin/packages/${{ matrix.arch }}/packages_ci/*.ipk . || true
+
       - name: Store packages
         uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.arch}}-packages
-          path: bin/packages/${{ matrix.arch }}/packages_ci/*.ipk
+          path: "*.ipk"
 
       - name: Store logs
         uses: actions/upload-artifact@v2
@@ -62,3 +71,22 @@ jobs:
           name: ${{ matrix.arch}}-logs
           path: logs/
 
+      - name: Remove logs
+        run: sudo rm -rf logs/ || true
+
+      - name: Register QEMU
+        if: ${{ matrix.runtime_test }}
+        run: |
+          sudo docker run --rm --privileged aptman/qus -s -- -p
+
+      - name: Build Docker container
+        if: ${{ matrix.runtime_test }}
+        run: |
+          docker build -t test-container --build-arg ARCH .github/workflows/
+        env:
+          ARCH: ${{ matrix.arch }}
+
+      - name: Test via Docker container
+        if: ${{ matrix.runtime_test }}
+        run: |
+          docker run --rm -v $GITHUB_WORKSPACE:/ci test-container