.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       target:
   9         required: true
  10         type: string
  11       subtarget:
  12         required: true
  13         type: string
  14       testing:
  15         type: boolean
  16       build_toolchain:
  17         type: boolean
  18       include_feeds:
  19         type: boolean
  20       build_full:
  21         type: boolean
  22       build_kernel:
  23         type: boolean
  24       build_all_modules:
  25         type: boolean
  26       build_all_kmods:
  27         type: boolean
  28       build_all_boards:
  29         type: boolean
  30       use_openwrt_container:
  31         type: boolean
  32         default: true
  33       coverity_project_name:
  34         type: string
  35         default: OpenWrt
  36       coverity_check_packages:
  37         type: string
  38       coverity_compiler_template_list:
  39         type: string
  40         default: >-
  41           arm-openwrt-linux-gcc
  42       coverity_force_compile_packages:
  43         type: string
  44         default: >-
  45           curl
  46           libnl
  47           mbedtls
  48           wolfssl
  49           openssl
  50       build_external_toolchain:
  51         type: boolean
  52       upload_external_toolchain:
  53         type: boolean
  54
  55 permissions:
  56   contents: read
  57
  58 jobs:
  59   setup_build:
  60     name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
  61     runs-on: ubuntu-latest
  62     outputs:
  63       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  64       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  65       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  66
  67     steps:
  68       - name: Checkout
  69         uses: actions/checkout@v3
  70
  71       - name: Set lower case owner name
  72         id: lower_owner
  73         run: |
  74           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  75             | tr '[:upper:]' '[:lower:]')
  76
  77           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  78             OWNER_LC=openwrt
  79           fi
  80
  81           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  82
  83       - name: Generate ccache hash
  84         id: ccache_hash
  85         run: |
  86           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  87            | md5sum | awk '{ print $1 }')
  88           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  89
  90       # Per branch tools container tag
  91       # By default stick to latest
  92       # For official test targetting openwrt stable branch
  93       # Get the branch or parse the tag and push dedicated tools containers
  94       # For local test to use the correct container for stable release testing
  95       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  96       - name: Determine tools container tag
  97         id: determine_tools_container
  98         run: |
  99           CONTAINER_TAG=latest
 100           if [ -n "${{ github.base_ref }}" ]; then
 101             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 102               CONTAINER_TAG="${{ github.base_ref }}"
 103             fi
 104           elif [ ${{ github.ref_type }} == "branch" ]; then
 105             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 106               CONTAINER_TAG=${{ github.ref_name }}
 107             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 108               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 109             fi
 110           elif [ ${{ github.ref_type }} == "tag" ]; then
 111             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 112               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 113             fi
 114           fi
 115           echo "Tools container to use tools:$CONTAINER_TAG"
 116           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 117
 118   build:
 119     name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
 120     needs: setup_build
 121     runs-on: ubuntu-latest
 122
 123     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:${{ needs.setup_build.outputs.container_tag }}
 124
 125     permissions:
 126       contents: read
 127       packages: read
 128
 129     steps:
 130       - name: Checkout master directory
 131         uses: actions/checkout@v3
 132         with:
 133           path: openwrt
 134
 135       - name: Checkout packages feed
 136         if: inputs.include_feeds == true
 137         uses: actions/checkout@v3
 138         with:
 139           repository: openwrt/packages
 140           path: openwrt/feeds/packages
 141
 142       - name: Checkout luci feed
 143         if: inputs.include_feeds == true
 144         uses: actions/checkout@v3
 145         with:
 146           repository: openwrt/luci
 147           path: openwrt/feeds/luci
 148
 149       - name: Checkout routing feed
 150         if: inputs.include_feeds == true
 151         uses: actions/checkout@v3
 152         with:
 153           repository: openwrt/routing
 154           path: openwrt/feeds/routing
 155
 156       - name: Checkout telephony feed
 157         if: inputs.include_feeds == true
 158         uses: actions/checkout@v3
 159         with:
 160           repository: openwrt/telephony
 161           path: openwrt/feeds/telephony
 162
 163       - name: Fix permission
 164         run: |
 165           chown -R buildbot:buildbot openwrt
 166
 167       - name: Prepare prebuilt tools
 168         shell: su buildbot -c "sh -e {0}"
 169         working-directory: openwrt
 170         run: |
 171           mkdir -p staging_dir build_dir
 172           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 173           ln -s /prebuilt_tools/build_dir/host build_dir/host
 174
 175           ./scripts/ext-tools.sh --refresh
 176
 177       - name: Update & Install feeds
 178         if: inputs.include_feeds == true
 179         shell: su buildbot -c "sh -e {0}"
 180         working-directory: openwrt
 181         run: |
 182           ./scripts/feeds update -a
 183           ./scripts/feeds install -a
 184
 185       - name: Parse toolchain file
 186         if: inputs.build_toolchain == false
 187         id: parse-toolchain
 188         working-directory: openwrt
 189         run: |
 190           TOOLCHAIN_PATH=snapshots
 191
 192           if [ -n "${{ github.base_ref }}" ]; then
 193             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 194               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 195             fi
 196           elif [ "${{ github.ref_type }}" = "branch" ]; then
 197             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 198               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 199             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 200               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 201             fi
 202           elif [ "${{ github.ref_type }}" = "tag" ]; then
 203             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 204               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 205             fi
 206           fi
 207
 208           if [ -n "$major_ver" ]; then
 209             git fetch --tags -f
 210             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 211             if [ -n "$latest_tag" ]; then
 212               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 213             fi
 214           fi
 215
 216           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
 217           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 218             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 219             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 220
 221             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 222           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 223             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 224             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 225
 226             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 227           else
 228             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 229           fi
 230
 231           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 232           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 233
 234       - name: Cache ccache
 235         uses: actions/cache@v3
 236         with:
 237           path: openwrt/.ccache
 238           key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ needs.setup_build.outputs.ccache_hash }}
 239           restore-keys: |
 240             ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
 241
 242       - name: Download external toolchain/sdk
 243         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 244         shell: su buildbot -c "sh -e {0}"
 245         working-directory: openwrt
 246         run: |
 247           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 248             | tar --xz -xf -
 249
 250       - name: Configure testing kernel
 251         if: inputs.testing == true
 252         shell: su buildbot -c "sh -e {0}"
 253         working-directory: openwrt
 254         run: |
 255           echo CONFIG_TESTING_KERNEL=y >> .config
 256
 257       - name: Configure all kernel modules
 258         if: inputs.build_all_kmods == true
 259         shell: su buildbot -c "sh -e {0}"
 260         working-directory: openwrt
 261         run: |
 262           echo CONFIG_ALL_KMODS=y >> .config
 263
 264       - name: Configure all modules
 265         if: inputs.build_all_modules == true
 266         shell: su buildbot -c "sh -e {0}"
 267         working-directory: openwrt
 268         run: |
 269           echo CONFIG_ALL=y >> .config
 270
 271       - name: Configure all boards
 272         if: inputs.build_all_boards == true
 273         shell: su buildbot -c "sh -e {0}"
 274         working-directory: openwrt
 275         run: |
 276           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 277           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 278           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 279
 280       - name: Configure external toolchain
 281         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 282         shell: su buildbot -c "sh -e {0}"
 283         working-directory: openwrt
 284         run: |
 285           echo CONFIG_DEVEL=y >> .config
 286           echo CONFIG_AUTOREMOVE=y >> .config
 287           echo CONFIG_CCACHE=y >> .config
 288
 289           ./scripts/ext-toolchain.sh \
 290             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 291             --overwrite-config \
 292             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 293
 294       - name: Adapt external sdk to external toolchain format
 295         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 296         shell: su buildbot -c "sh -e {0}"
 297         working-directory: openwrt
 298         run: |
 299           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 300           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 301           OPENWRT_DIR=$(pwd)
 302
 303           # Find target name from toolchain info.mk
 304           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 305
 306           cd $TOOLCHAIN_BIN
 307
 308           # Revert sdk wrapper scripts applied to all the bins
 309           for app in $(find . -name "*.bin"); do
 310             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 311             rm $TARGET_APP
 312             mv .$TARGET_APP.bin $TARGET_APP
 313           done
 314
 315           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 316           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 317           for app in cc gcc g++ c++ cpp ld as ; do
 318             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 319             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 320           done
 321
 322       - name: Configure external toolchain with sdk
 323         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 324         shell: su buildbot -c "sh -e {0}"
 325         working-directory: openwrt
 326         run: |
 327           echo CONFIG_DEVEL=y >> .config
 328           echo CONFIG_AUTOREMOVE=y >> .config
 329           echo CONFIG_CCACHE=y >> .config
 330
 331           ./scripts/ext-toolchain.sh \
 332             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 333             --overwrite-config \
 334             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 335
 336       - name: Configure internal toolchain
 337         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 338         shell: su buildbot -c "sh -e {0}"
 339         working-directory: openwrt
 340         run: |
 341           echo CONFIG_DEVEL=y >> .config
 342           echo CONFIG_AUTOREMOVE=y >> .config
 343           echo CONFIG_CCACHE=y >> .config
 344
 345           echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
 346           echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
 347
 348           make defconfig
 349
 350       - name: Show configuration
 351         shell: su buildbot -c "sh -e {0}"
 352         working-directory: openwrt
 353         run: ./scripts/diffconfig.sh
 354
 355       - name: Build tools
 356         shell: su buildbot -c "sh -e {0}"
 357         working-directory: openwrt
 358         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 359
 360       - name: Build toolchain
 361         shell: su buildbot -c "sh -e {0}"
 362         working-directory: openwrt
 363         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 364
 365       - name: Build Kernel
 366         if: inputs.build_kernel == true
 367         shell: su buildbot -c "sh -e {0}"
 368         working-directory: openwrt
 369         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 370
 371       - name: Build Kernel Kmods
 372         if: inputs.build_kernel == true
 373         shell: su buildbot -c "sh -e {0}"
 374         working-directory: openwrt
 375         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 376
 377       - name: Build everything
 378         if: inputs.build_full == true
 379         shell: su buildbot -c "sh -e {0}"
 380         working-directory: openwrt
 381         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 382
 383       - name: Build external toolchain
 384         if: inputs.build_external_toolchain == true
 385         shell: su buildbot -c "sh -e {0}"
 386         working-directory: openwrt
 387         run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 388
 389       - name: Coverity prepare toolchain
 390         if: inputs.coverity_check_packages != ''
 391         shell: su buildbot -c "sh -e {0}"
 392         working-directory: openwrt
 393         run: |
 394           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 395           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 396           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 397           md5sum -c coverity.tar.gz.md5
 398
 399           mkdir cov-analysis-linux64
 400           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 401           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 402
 403           for template in ${{ inputs.coverity_compiler_template_list }}; do
 404             cov-configure --template --comptype gcc --compiler "$template"
 405           done
 406
 407       - name: Clean and recompile packages with Coverity toolchain
 408         if: inputs.coverity_check_packages != ''
 409         shell: su buildbot -c "bash {0}"
 410         working-directory: openwrt
 411         run: |
 412           set -o pipefail -o errexit
 413
 414           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 415           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 416           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 417
 418           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 419           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 420           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 421
 422           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 423           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 424           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 425
 426       - name: Upload build to Coverity for analysis
 427         if: inputs.coverity_check_packages != ''
 428         shell: su buildbot -c "sh -e {0}"
 429         working-directory: openwrt
 430         run: |
 431           tar czf cov-int.tar.gz ./cov-int
 432           curl \
 433             --form token="${{ secrets.coverity_api_token }}" \
 434             --form email="contact@openwrt.org" \
 435             --form file=@cov-int.tar.gz \
 436             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 437             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 438             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 439
 440       - name: Upload logs
 441         if: failure()
 442         uses: actions/upload-artifact@v3
 443         with:
 444           name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
 445           path: "openwrt/logs"
 446
 447       - name: Find external toolchain name
 448         id: get-toolchain-name
 449         if: inputs.upload_external_toolchain == true
 450         working-directory: openwrt
 451         run: |
 452           TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
 453           echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
 454
 455       - name: Upload prebuilt toolchain
 456         if: inputs.upload_external_toolchain == true
 457         uses: actions/upload-artifact@v3
 458         with:
 459           name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
 460           path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
 461           retention-days: 1