.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       container_name:
   9         type: string
  10         default: tools
  11       target:
  12         required: true
  13         type: string
  14       subtarget:
  15         required: true
  16         type: string
  17       testing:
  18         type: boolean
  19       build_toolchain:
  20         type: boolean
  21       include_feeds:
  22         type: boolean
  23       build_full:
  24         type: boolean
  25       build_kernel:
  26         type: boolean
  27       build_all_modules:
  28         type: boolean
  29       build_all_kmods:
  30         type: boolean
  31       build_all_boards:
  32         type: boolean
  33       use_openwrt_container:
  34         type: boolean
  35         default: true
  36       coverity_project_name:
  37         type: string
  38         default: OpenWrt
  39       coverity_check_packages:
  40         type: string
  41       coverity_compiler_template_list:
  42         type: string
  43         default: >-
  44           arm-openwrt-linux-gcc
  45       coverity_force_compile_packages:
  46         type: string
  47         default: >-
  48           curl
  49           libnl
  50           mbedtls
  51           wolfssl
  52           openssl
  53       build_external_toolchain:
  54         type: boolean
  55       upload_external_toolchain:
  56         type: boolean
  57
  58 permissions:
  59   contents: read
  60
  61 jobs:
  62   setup_build:
  63     name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
  64     runs-on: ubuntu-latest
  65     outputs:
  66       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  67       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  68       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  69       container_name: ${{ steps.determine_tools_container.outputs.container_name }}
  70
  71     steps:
  72       - name: Checkout
  73         uses: actions/checkout@v3
  74
  75       - name: Set lower case owner name
  76         id: lower_owner
  77         run: |
  78           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  79             | tr '[:upper:]' '[:lower:]')
  80
  81           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  82             OWNER_LC=openwrt
  83           fi
  84
  85           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  86
  87       - name: Generate ccache hash
  88         id: ccache_hash
  89         run: |
  90           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  91            | md5sum | awk '{ print $1 }')
  92           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  93
  94       # Per branch tools container tag
  95       # By default stick to latest
  96       # For official test targetting openwrt stable branch
  97       # Get the branch or parse the tag and push dedicated tools containers
  98       # For local test to use the correct container for stable release testing
  99       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
 100       - name: Determine tools container tag
 101         id: determine_tools_container
 102         run: |
 103           CONTAINER_NAME=${{ inputs.container_name }}
 104           CONTAINER_TAG=latest
 105           if [ -n "${{ github.base_ref }}" ]; then
 106             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 107               CONTAINER_TAG="${{ github.base_ref }}"
 108             fi
 109           elif [ ${{ github.ref_type }} == "branch" ]; then
 110             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 111               CONTAINER_TAG=${{ github.ref_name }}
 112             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 113               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 114             fi
 115           elif [ ${{ github.ref_type }} == "tag" ]; then
 116             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 117               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 118             fi
 119           fi
 120
 121           if [ "$CONTAINER_NAME" = "toolchain" ]; then
 122             GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
 123             GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
 124             GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 125             # Check if container exist
 126             if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
 127               CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 128             else
 129               CONTAINER_NAME=tools
 130             fi
 131           fi
 132
 133           echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
 134           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 135           echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT
 136
 137   build:
 138     name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
 139     needs: setup_build
 140     runs-on: ubuntu-latest
 141
 142     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
 143
 144     permissions:
 145       contents: read
 146       packages: read
 147
 148     steps:
 149       - name: Checkout master directory
 150         uses: actions/checkout@v3
 151         with:
 152           path: openwrt
 153
 154       - name: Checkout packages feed
 155         if: inputs.include_feeds == true
 156         uses: actions/checkout@v3
 157         with:
 158           repository: openwrt/packages
 159           path: openwrt/feeds/packages
 160
 161       - name: Checkout luci feed
 162         if: inputs.include_feeds == true
 163         uses: actions/checkout@v3
 164         with:
 165           repository: openwrt/luci
 166           path: openwrt/feeds/luci
 167
 168       - name: Checkout routing feed
 169         if: inputs.include_feeds == true
 170         uses: actions/checkout@v3
 171         with:
 172           repository: openwrt/routing
 173           path: openwrt/feeds/routing
 174
 175       - name: Checkout telephony feed
 176         if: inputs.include_feeds == true
 177         uses: actions/checkout@v3
 178         with:
 179           repository: openwrt/telephony
 180           path: openwrt/feeds/telephony
 181
 182       - name: Fix permission
 183         run: |
 184           chown -R buildbot:buildbot openwrt
 185
 186       - name: Prepare prebuilt tools
 187         shell: su buildbot -c "sh -e {0}"
 188         working-directory: openwrt
 189         run: |
 190           mkdir -p staging_dir build_dir
 191           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 192           ln -s /prebuilt_tools/build_dir/host build_dir/host
 193
 194           ./scripts/ext-tools.sh --refresh
 195
 196       - name: Update & Install feeds
 197         if: inputs.include_feeds == true
 198         shell: su buildbot -c "sh -e {0}"
 199         working-directory: openwrt
 200         run: |
 201           ./scripts/feeds update -a
 202           ./scripts/feeds install -a
 203
 204       - name: Parse toolchain file
 205         if: inputs.build_toolchain == false
 206         id: parse-toolchain
 207         working-directory: openwrt
 208         run: |
 209           TOOLCHAIN_PATH=snapshots
 210
 211           if [ -n "${{ github.base_ref }}" ]; then
 212             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 213               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 214             fi
 215           elif [ "${{ github.ref_type }}" = "branch" ]; then
 216             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 217               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 218             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 219               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 220             fi
 221           elif [ "${{ github.ref_type }}" = "tag" ]; then
 222             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 223               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 224             fi
 225           fi
 226
 227           if [ -n "$major_ver" ]; then
 228             git fetch --tags -f
 229             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 230             if [ -n "$latest_tag" ]; then
 231               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 232             fi
 233           fi
 234
 235           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
 236           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 237             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 238             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 239
 240             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 241           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 242             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 243             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 244
 245             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 246           else
 247             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 248           fi
 249
 250           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 251           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 252
 253       - name: Cache ccache
 254         uses: actions/cache@v3
 255         with:
 256           path: openwrt/.ccache
 257           key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ needs.setup_build.outputs.ccache_hash }}
 258           restore-keys: |
 259             ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
 260
 261       - name: Download external toolchain/sdk
 262         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 263         shell: su buildbot -c "sh -e {0}"
 264         working-directory: openwrt
 265         run: |
 266           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 267             | tar --xz -xf -
 268
 269       - name: Configure testing kernel
 270         if: inputs.testing == true
 271         shell: su buildbot -c "sh -e {0}"
 272         working-directory: openwrt
 273         run: |
 274           echo CONFIG_TESTING_KERNEL=y >> .config
 275
 276       - name: Configure all kernel modules
 277         if: inputs.build_all_kmods == true
 278         shell: su buildbot -c "sh -e {0}"
 279         working-directory: openwrt
 280         run: |
 281           echo CONFIG_ALL_KMODS=y >> .config
 282
 283       - name: Configure all modules
 284         if: inputs.build_all_modules == true
 285         shell: su buildbot -c "sh -e {0}"
 286         working-directory: openwrt
 287         run: |
 288           echo CONFIG_ALL=y >> .config
 289
 290       - name: Configure all boards
 291         if: inputs.build_all_boards == true
 292         shell: su buildbot -c "sh -e {0}"
 293         working-directory: openwrt
 294         run: |
 295           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 296           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 297           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 298
 299       - name: Configure external toolchain
 300         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 301         shell: su buildbot -c "sh -e {0}"
 302         working-directory: openwrt
 303         run: |
 304           echo CONFIG_DEVEL=y >> .config
 305           echo CONFIG_AUTOREMOVE=y >> .config
 306           echo CONFIG_CCACHE=y >> .config
 307
 308           ./scripts/ext-toolchain.sh \
 309             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 310             --overwrite-config \
 311             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 312
 313       - name: Adapt external sdk to external toolchain format
 314         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 315         shell: su buildbot -c "sh -e {0}"
 316         working-directory: openwrt
 317         run: |
 318           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 319           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 320           OPENWRT_DIR=$(pwd)
 321
 322           # Find target name from toolchain info.mk
 323           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 324
 325           cd $TOOLCHAIN_BIN
 326
 327           # Revert sdk wrapper scripts applied to all the bins
 328           for app in $(find . -name "*.bin"); do
 329             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 330             rm $TARGET_APP
 331             mv .$TARGET_APP.bin $TARGET_APP
 332           done
 333
 334           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 335           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 336           for app in cc gcc g++ c++ cpp ld as ; do
 337             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 338             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 339           done
 340
 341       - name: Configure external toolchain with sdk
 342         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 343         shell: su buildbot -c "sh -e {0}"
 344         working-directory: openwrt
 345         run: |
 346           echo CONFIG_DEVEL=y >> .config
 347           echo CONFIG_AUTOREMOVE=y >> .config
 348           echo CONFIG_CCACHE=y >> .config
 349
 350           ./scripts/ext-toolchain.sh \
 351             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 352             --overwrite-config \
 353             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 354
 355       - name: Configure internal toolchain
 356         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 357         shell: su buildbot -c "sh -e {0}"
 358         working-directory: openwrt
 359         run: |
 360           echo CONFIG_DEVEL=y >> .config
 361           echo CONFIG_AUTOREMOVE=y >> .config
 362           echo CONFIG_CCACHE=y >> .config
 363
 364           echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
 365           echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
 366
 367           make defconfig
 368
 369       - name: Show configuration
 370         shell: su buildbot -c "sh -e {0}"
 371         working-directory: openwrt
 372         run: ./scripts/diffconfig.sh
 373
 374       - name: Build tools
 375         shell: su buildbot -c "sh -e {0}"
 376         working-directory: openwrt
 377         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 378
 379       - name: Build toolchain
 380         shell: su buildbot -c "sh -e {0}"
 381         working-directory: openwrt
 382         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 383
 384       - name: Build Kernel
 385         if: inputs.build_kernel == true
 386         shell: su buildbot -c "sh -e {0}"
 387         working-directory: openwrt
 388         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 389
 390       - name: Build Kernel Kmods
 391         if: inputs.build_kernel == true
 392         shell: su buildbot -c "sh -e {0}"
 393         working-directory: openwrt
 394         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 395
 396       - name: Build everything
 397         if: inputs.build_full == true
 398         shell: su buildbot -c "sh -e {0}"
 399         working-directory: openwrt
 400         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 401
 402       - name: Build external toolchain
 403         if: inputs.build_external_toolchain == true
 404         shell: su buildbot -c "sh -e {0}"
 405         working-directory: openwrt
 406         run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 407
 408       - name: Coverity prepare toolchain
 409         if: inputs.coverity_check_packages != ''
 410         shell: su buildbot -c "sh -e {0}"
 411         working-directory: openwrt
 412         run: |
 413           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 414           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 415           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 416           md5sum -c coverity.tar.gz.md5
 417
 418           mkdir cov-analysis-linux64
 419           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 420           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 421
 422           for template in ${{ inputs.coverity_compiler_template_list }}; do
 423             cov-configure --template --comptype gcc --compiler "$template"
 424           done
 425
 426       - name: Clean and recompile packages with Coverity toolchain
 427         if: inputs.coverity_check_packages != ''
 428         shell: su buildbot -c "bash {0}"
 429         working-directory: openwrt
 430         run: |
 431           set -o pipefail -o errexit
 432
 433           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 434           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 435           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 436
 437           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 438           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 439           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 440
 441           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 442           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 443           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 444
 445       - name: Upload build to Coverity for analysis
 446         if: inputs.coverity_check_packages != ''
 447         shell: su buildbot -c "sh -e {0}"
 448         working-directory: openwrt
 449         run: |
 450           tar czf cov-int.tar.gz ./cov-int
 451           curl \
 452             --form token="${{ secrets.coverity_api_token }}" \
 453             --form email="contact@openwrt.org" \
 454             --form file=@cov-int.tar.gz \
 455             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 456             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 457             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 458
 459       - name: Upload logs
 460         if: failure()
 461         uses: actions/upload-artifact@v3
 462         with:
 463           name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
 464           path: "openwrt/logs"
 465
 466       - name: Find external toolchain name
 467         id: get-toolchain-name
 468         if: inputs.upload_external_toolchain == true
 469         working-directory: openwrt
 470         run: |
 471           TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
 472           echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
 473
 474       - name: Upload prebuilt toolchain
 475         if: inputs.upload_external_toolchain == true
 476         uses: actions/upload-artifact@v3
 477         with:
 478           name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
 479           path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
 480           retention-days: 1