.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       container_name:
   9         type: string
  10         default: tools
  11       target:
  12         required: true
  13         type: string
  14       subtarget:
  15         required: true
  16         type: string
  17       testing:
  18         type: boolean
  19       build_toolchain:
  20         type: boolean
  21       include_feeds:
  22         type: boolean
  23       build_full:
  24         type: boolean
  25       build_kernel:
  26         type: boolean
  27       build_all_modules:
  28         type: boolean
  29       build_all_kmods:
  30         type: boolean
  31       build_all_boards:
  32         type: boolean
  33       use_openwrt_container:
  34         type: boolean
  35         default: true
  36       coverity_project_name:
  37         type: string
  38         default: OpenWrt
  39       coverity_check_packages:
  40         type: string
  41       coverity_compiler_template_list:
  42         type: string
  43         default: >-
  44           arm-openwrt-linux-gcc
  45       coverity_force_compile_packages:
  46         type: string
  47         default: >-
  48           curl
  49           libnl
  50           mbedtls
  51           wolfssl
  52           openssl
  53       build_external_toolchain:
  54         type: boolean
  55       upload_external_toolchain:
  56         type: boolean
  57
  58 permissions:
  59   contents: read
  60
  61 jobs:
  62   setup_build:
  63     name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
  64     runs-on: ubuntu-latest
  65     outputs:
  66       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  67       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  68       container_name: ${{ steps.determine_tools_container.outputs.container_name }}
  69
  70     steps:
  71       - name: Checkout
  72         uses: actions/checkout@v3
  73
  74       - name: Set lower case owner name
  75         id: lower_owner
  76         run: |
  77           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  78             | tr '[:upper:]' '[:lower:]')
  79
  80           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  81             OWNER_LC=openwrt
  82           fi
  83
  84           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  85
  86       # Per branch tools container tag
  87       # By default stick to latest
  88       # For official test targetting openwrt stable branch
  89       # Get the branch or parse the tag and push dedicated tools containers
  90       # For local test to use the correct container for stable release testing
  91       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  92       - name: Determine tools container tag
  93         id: determine_tools_container
  94         run: |
  95           CONTAINER_NAME=${{ inputs.container_name }}
  96           CONTAINER_TAG=latest
  97           if [ -n "${{ github.base_ref }}" ]; then
  98             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  99               CONTAINER_TAG="${{ github.base_ref }}"
 100             fi
 101           elif [ ${{ github.ref_type }} == "branch" ]; then
 102             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 103               CONTAINER_TAG=${{ github.ref_name }}
 104             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 105               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 106             fi
 107           elif [ ${{ github.ref_type }} == "tag" ]; then
 108             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 109               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 110             fi
 111           fi
 112
 113           if [ "$CONTAINER_NAME" = "toolchain" ]; then
 114             GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
 115             GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
 116             GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 117             # Check if container exist
 118             if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
 119               CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
 120             else
 121               CONTAINER_NAME=tools
 122             fi
 123           fi
 124
 125           echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
 126           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 127           echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT
 128
 129   build:
 130     name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
 131     needs: setup_build
 132     runs-on: ubuntu-latest
 133
 134     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
 135
 136     permissions:
 137       contents: read
 138       packages: read
 139
 140     steps:
 141       - name: Checkout master directory
 142         uses: actions/checkout@v3
 143         with:
 144           path: openwrt
 145
 146       - name: Checkout packages feed
 147         if: inputs.include_feeds == true
 148         uses: actions/checkout@v3
 149         with:
 150           repository: openwrt/packages
 151           path: openwrt/feeds/packages
 152
 153       - name: Checkout luci feed
 154         if: inputs.include_feeds == true
 155         uses: actions/checkout@v3
 156         with:
 157           repository: openwrt/luci
 158           path: openwrt/feeds/luci
 159
 160       - name: Checkout routing feed
 161         if: inputs.include_feeds == true
 162         uses: actions/checkout@v3
 163         with:
 164           repository: openwrt/routing
 165           path: openwrt/feeds/routing
 166
 167       - name: Checkout telephony feed
 168         if: inputs.include_feeds == true
 169         uses: actions/checkout@v3
 170         with:
 171           repository: openwrt/telephony
 172           path: openwrt/feeds/telephony
 173
 174       - name: Parse toolchain file
 175         if: inputs.build_toolchain == false
 176         id: parse-toolchain
 177         working-directory: openwrt
 178         run: |
 179           if [ -d /external-toolchain/ ]; then
 180             echo "toolchain-type=external_container" >> $GITHUB_OUTPUT
 181             exit 0
 182           fi
 183
 184           TOOLCHAIN_PATH=snapshots
 185
 186           if [ -n "${{ github.base_ref }}" ]; then
 187             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 188               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 189             fi
 190           elif [ "${{ github.ref_type }}" = "branch" ]; then
 191             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 192               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 193             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 194               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 195             fi
 196           elif [ "${{ github.ref_type }}" = "tag" ]; then
 197             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 198               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 199             fi
 200           fi
 201
 202           if [ -n "$major_ver" ]; then
 203             git fetch --tags -f
 204             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 205             if [ -n "$latest_tag" ]; then
 206               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 207             fi
 208           fi
 209
 210           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
 211           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 212             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 213             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 214
 215             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 216           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 217             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 218             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 219
 220             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 221           else
 222             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 223           fi
 224
 225           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 226           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 227
 228       - name: Fix permission
 229         run: |
 230           chown -R buildbot:buildbot openwrt
 231
 232       - name: Prepare prebuilt tools
 233         shell: su buildbot -c "sh -e {0}"
 234         working-directory: openwrt
 235         run: |
 236           mkdir -p staging_dir build_dir
 237           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 238           ln -s /prebuilt_tools/build_dir/host build_dir/host
 239
 240           ./scripts/ext-tools.sh --refresh
 241
 242       - name: Update & Install feeds
 243         if: inputs.include_feeds == true
 244         shell: su buildbot -c "sh -e {0}"
 245         working-directory: openwrt
 246         run: |
 247           ./scripts/feeds update -a
 248           ./scripts/feeds install -a
 249
 250       - name: Cache ccache
 251         uses: actions/cache@v3
 252         with:
 253           path: openwrt/.ccache
 254           key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ hashFiles('openwrt/include/kernel-**') }}
 255           restore-keys: |
 256             ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
 257
 258       - name: Download external toolchain/sdk
 259         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container'
 260         shell: su buildbot -c "sh -e {0}"
 261         working-directory: openwrt
 262         run: |
 263           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 264             | tar --xz -xf -
 265
 266       - name: Configure testing kernel
 267         if: inputs.testing == true
 268         shell: su buildbot -c "sh -e {0}"
 269         working-directory: openwrt
 270         run: |
 271           echo CONFIG_TESTING_KERNEL=y >> .config
 272
 273       - name: Configure all kernel modules
 274         if: inputs.build_all_kmods == true
 275         shell: su buildbot -c "sh -e {0}"
 276         working-directory: openwrt
 277         run: |
 278           echo CONFIG_ALL_KMODS=y >> .config
 279
 280       - name: Configure all modules
 281         if: inputs.build_all_modules == true
 282         shell: su buildbot -c "sh -e {0}"
 283         working-directory: openwrt
 284         run: |
 285           echo CONFIG_ALL=y >> .config
 286
 287       - name: Configure all boards
 288         if: inputs.build_all_boards == true
 289         shell: su buildbot -c "sh -e {0}"
 290         working-directory: openwrt
 291         run: |
 292           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 293           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 294           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 295
 296       - name: Configure external toolchain in container
 297         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container'
 298         shell: su buildbot -c "sh -e {0}"
 299         working-directory: openwrt
 300         run: |
 301           echo CONFIG_DEVEL=y >> .config
 302           echo CONFIG_AUTOREMOVE=y >> .config
 303           echo CONFIG_CCACHE=y >> .config
 304
 305           ./scripts/ext-toolchain.sh \
 306             --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \
 307             --overwrite-config \
 308             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 309
 310       - name: Configure external toolchain
 311         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 312         shell: su buildbot -c "sh -e {0}"
 313         working-directory: openwrt
 314         run: |
 315           echo CONFIG_DEVEL=y >> .config
 316           echo CONFIG_AUTOREMOVE=y >> .config
 317           echo CONFIG_CCACHE=y >> .config
 318
 319           ./scripts/ext-toolchain.sh \
 320             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 321             --overwrite-config \
 322             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 323
 324       - name: Adapt external sdk to external toolchain format
 325         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 326         shell: su buildbot -c "sh -e {0}"
 327         working-directory: openwrt
 328         run: |
 329           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 330           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 331           OPENWRT_DIR=$(pwd)
 332
 333           # Find target name from toolchain info.mk
 334           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 335
 336           cd $TOOLCHAIN_BIN
 337
 338           # Revert sdk wrapper scripts applied to all the bins
 339           for app in $(find . -name "*.bin"); do
 340             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 341             rm $TARGET_APP
 342             mv .$TARGET_APP.bin $TARGET_APP
 343           done
 344
 345           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 346           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 347           for app in cc gcc g++ c++ cpp ld as ; do
 348             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 349             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 350           done
 351
 352       - name: Configure external toolchain with sdk
 353         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 354         shell: su buildbot -c "sh -e {0}"
 355         working-directory: openwrt
 356         run: |
 357           echo CONFIG_DEVEL=y >> .config
 358           echo CONFIG_AUTOREMOVE=y >> .config
 359           echo CONFIG_CCACHE=y >> .config
 360
 361           ./scripts/ext-toolchain.sh \
 362             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 363             --overwrite-config \
 364             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 365
 366       - name: Configure internal toolchain
 367         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 368         shell: su buildbot -c "sh -e {0}"
 369         working-directory: openwrt
 370         run: |
 371           echo CONFIG_DEVEL=y >> .config
 372           echo CONFIG_AUTOREMOVE=y >> .config
 373           echo CONFIG_CCACHE=y >> .config
 374
 375           echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
 376           echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
 377
 378           make defconfig
 379
 380       - name: Show configuration
 381         shell: su buildbot -c "sh -e {0}"
 382         working-directory: openwrt
 383         run: ./scripts/diffconfig.sh
 384
 385       - name: Build tools
 386         shell: su buildbot -c "sh -e {0}"
 387         working-directory: openwrt
 388         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 389
 390       - name: Build toolchain
 391         shell: su buildbot -c "sh -e {0}"
 392         working-directory: openwrt
 393         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 394
 395       - name: Build Kernel
 396         if: inputs.build_kernel == true
 397         shell: su buildbot -c "sh -e {0}"
 398         working-directory: openwrt
 399         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 400
 401       - name: Build Kernel Kmods
 402         if: inputs.build_kernel == true
 403         shell: su buildbot -c "sh -e {0}"
 404         working-directory: openwrt
 405         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 406
 407       - name: Build everything
 408         if: inputs.build_full == true
 409         shell: su buildbot -c "sh -e {0}"
 410         working-directory: openwrt
 411         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 412
 413       - name: Build external toolchain
 414         if: inputs.build_external_toolchain == true
 415         shell: su buildbot -c "sh -e {0}"
 416         working-directory: openwrt
 417         run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 418
 419       - name: Coverity prepare toolchain
 420         if: inputs.coverity_check_packages != ''
 421         shell: su buildbot -c "sh -e {0}"
 422         working-directory: openwrt
 423         run: |
 424           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 425           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 426           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 427           md5sum -c coverity.tar.gz.md5
 428
 429           mkdir cov-analysis-linux64
 430           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 431           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 432
 433           for template in ${{ inputs.coverity_compiler_template_list }}; do
 434             cov-configure --template --comptype gcc --compiler "$template"
 435           done
 436
 437       - name: Clean and recompile packages with Coverity toolchain
 438         if: inputs.coverity_check_packages != ''
 439         shell: su buildbot -c "bash {0}"
 440         working-directory: openwrt
 441         run: |
 442           set -o pipefail -o errexit
 443
 444           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 445           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 446           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 447
 448           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 449           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 450           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 451
 452           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 453           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 454           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 455
 456       - name: Upload build to Coverity for analysis
 457         if: inputs.coverity_check_packages != ''
 458         shell: su buildbot -c "sh -e {0}"
 459         working-directory: openwrt
 460         run: |
 461           tar czf cov-int.tar.gz ./cov-int
 462           curl \
 463             --form token="${{ secrets.coverity_api_token }}" \
 464             --form email="contact@openwrt.org" \
 465             --form file=@cov-int.tar.gz \
 466             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 467             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 468             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 469
 470       - name: Upload logs
 471         if: failure()
 472         uses: actions/upload-artifact@v3
 473         with:
 474           name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
 475           path: "openwrt/logs"
 476
 477       - name: Find external toolchain name
 478         id: get-toolchain-name
 479         if: inputs.upload_external_toolchain == true
 480         working-directory: openwrt
 481         run: |
 482           TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
 483           echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
 484
 485       - name: Upload prebuilt toolchain
 486         if: inputs.upload_external_toolchain == true
 487         uses: actions/upload-artifact@v3
 488         with:
 489           name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
 490           path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
 491           retention-days: 1