.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     secrets:
   6       coverity_api_token:
   7     inputs:
   8       container_name:
   9         type: string
  10         default: tools
  11       target:
  12         required: true
  13         type: string
  14       subtarget:
  15         required: true
  16         type: string
  17       testing:
  18         type: boolean
  19       build_toolchain:
  20         type: boolean
  21       include_feeds:
  22         type: boolean
  23       build_full:
  24         type: boolean
  25       build_kernel:
  26         type: boolean
  27       build_all_modules:
  28         type: boolean
  29       build_all_kmods:
  30         type: boolean
  31       build_all_boards:
  32         type: boolean
  33       use_openwrt_container:
  34         type: boolean
  35         default: true
  36       coverity_project_name:
  37         type: string
  38         default: OpenWrt
  39       coverity_check_packages:
  40         type: string
  41       coverity_compiler_template_list:
  42         type: string
  43         default: >-
  44           arm-openwrt-linux-gcc
  45       coverity_force_compile_packages:
  46         type: string
  47         default: >-
  48           curl
  49           libnl
  50           mbedtls
  51           wolfssl
  52           openssl
  53       build_external_toolchain:
  54         type: boolean
  55       upload_external_toolchain:
  56         type: boolean
  57
  58 permissions:
  59   contents: read
  60
  61 jobs:
  62   setup_build:
  63     name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
  64     runs-on: ubuntu-latest
  65     outputs:
  66       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  67       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  68       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  69
  70     steps:
  71       - name: Checkout
  72         uses: actions/checkout@v3
  73
  74       - name: Set lower case owner name
  75         id: lower_owner
  76         run: |
  77           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  78             | tr '[:upper:]' '[:lower:]')
  79
  80           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  81             OWNER_LC=openwrt
  82           fi
  83
  84           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  85
  86       - name: Generate ccache hash
  87         id: ccache_hash
  88         run: |
  89           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  90            | md5sum | awk '{ print $1 }')
  91           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  92
  93       # Per branch tools container tag
  94       # By default stick to latest
  95       # For official test targetting openwrt stable branch
  96       # Get the branch or parse the tag and push dedicated tools containers
  97       # For local test to use the correct container for stable release testing
  98       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  99       - name: Determine tools container tag
 100         id: determine_tools_container
 101         run: |
 102           CONTAINER_TAG=latest
 103           if [ -n "${{ github.base_ref }}" ]; then
 104             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 105               CONTAINER_TAG="${{ github.base_ref }}"
 106             fi
 107           elif [ ${{ github.ref_type }} == "branch" ]; then
 108             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 109               CONTAINER_TAG=${{ github.ref_name }}
 110             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 111               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
 112             fi
 113           elif [ ${{ github.ref_type }} == "tag" ]; then
 114             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 115               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 116             fi
 117           fi
 118           echo "Tools container to use tools:$CONTAINER_TAG"
 119           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
 120
 121   build:
 122     name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
 123     needs: setup_build
 124     runs-on: ubuntu-latest
 125
 126     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ inputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
 127
 128     permissions:
 129       contents: read
 130       packages: read
 131
 132     steps:
 133       - name: Checkout master directory
 134         uses: actions/checkout@v3
 135         with:
 136           path: openwrt
 137
 138       - name: Checkout packages feed
 139         if: inputs.include_feeds == true
 140         uses: actions/checkout@v3
 141         with:
 142           repository: openwrt/packages
 143           path: openwrt/feeds/packages
 144
 145       - name: Checkout luci feed
 146         if: inputs.include_feeds == true
 147         uses: actions/checkout@v3
 148         with:
 149           repository: openwrt/luci
 150           path: openwrt/feeds/luci
 151
 152       - name: Checkout routing feed
 153         if: inputs.include_feeds == true
 154         uses: actions/checkout@v3
 155         with:
 156           repository: openwrt/routing
 157           path: openwrt/feeds/routing
 158
 159       - name: Checkout telephony feed
 160         if: inputs.include_feeds == true
 161         uses: actions/checkout@v3
 162         with:
 163           repository: openwrt/telephony
 164           path: openwrt/feeds/telephony
 165
 166       - name: Fix permission
 167         run: |
 168           chown -R buildbot:buildbot openwrt
 169
 170       - name: Prepare prebuilt tools
 171         shell: su buildbot -c "sh -e {0}"
 172         working-directory: openwrt
 173         run: |
 174           mkdir -p staging_dir build_dir
 175           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 176           ln -s /prebuilt_tools/build_dir/host build_dir/host
 177
 178           ./scripts/ext-tools.sh --refresh
 179
 180       - name: Update & Install feeds
 181         if: inputs.include_feeds == true
 182         shell: su buildbot -c "sh -e {0}"
 183         working-directory: openwrt
 184         run: |
 185           ./scripts/feeds update -a
 186           ./scripts/feeds install -a
 187
 188       - name: Parse toolchain file
 189         if: inputs.build_toolchain == false
 190         id: parse-toolchain
 191         working-directory: openwrt
 192         run: |
 193           TOOLCHAIN_PATH=snapshots
 194
 195           if [ -n "${{ github.base_ref }}" ]; then
 196             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 197               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 198             fi
 199           elif [ "${{ github.ref_type }}" = "branch" ]; then
 200             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 201               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 202             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 203               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 204             fi
 205           elif [ "${{ github.ref_type }}" = "tag" ]; then
 206             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 207               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 208             fi
 209           fi
 210
 211           if [ -n "$major_ver" ]; then
 212             git fetch --tags -f
 213             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 214             if [ -n "$latest_tag" ]; then
 215               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 216             fi
 217           fi
 218
 219           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
 220           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 221             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 222             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 223
 224             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 225           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 226             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 227             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 228
 229             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 230           else
 231             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 232           fi
 233
 234           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 235           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 236
 237       - name: Cache ccache
 238         uses: actions/cache@v3
 239         with:
 240           path: openwrt/.ccache
 241           key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ needs.setup_build.outputs.ccache_hash }}
 242           restore-keys: |
 243             ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
 244
 245       - name: Download external toolchain/sdk
 246         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 247         shell: su buildbot -c "sh -e {0}"
 248         working-directory: openwrt
 249         run: |
 250           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 251             | tar --xz -xf -
 252
 253       - name: Configure testing kernel
 254         if: inputs.testing == true
 255         shell: su buildbot -c "sh -e {0}"
 256         working-directory: openwrt
 257         run: |
 258           echo CONFIG_TESTING_KERNEL=y >> .config
 259
 260       - name: Configure all kernel modules
 261         if: inputs.build_all_kmods == true
 262         shell: su buildbot -c "sh -e {0}"
 263         working-directory: openwrt
 264         run: |
 265           echo CONFIG_ALL_KMODS=y >> .config
 266
 267       - name: Configure all modules
 268         if: inputs.build_all_modules == true
 269         shell: su buildbot -c "sh -e {0}"
 270         working-directory: openwrt
 271         run: |
 272           echo CONFIG_ALL=y >> .config
 273
 274       - name: Configure all boards
 275         if: inputs.build_all_boards == true
 276         shell: su buildbot -c "sh -e {0}"
 277         working-directory: openwrt
 278         run: |
 279           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 280           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 281           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 282
 283       - name: Configure external toolchain
 284         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 285         shell: su buildbot -c "sh -e {0}"
 286         working-directory: openwrt
 287         run: |
 288           echo CONFIG_DEVEL=y >> .config
 289           echo CONFIG_AUTOREMOVE=y >> .config
 290           echo CONFIG_CCACHE=y >> .config
 291
 292           ./scripts/ext-toolchain.sh \
 293             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 294             --overwrite-config \
 295             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 296
 297       - name: Adapt external sdk to external toolchain format
 298         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 299         shell: su buildbot -c "sh -e {0}"
 300         working-directory: openwrt
 301         run: |
 302           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 303           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 304           OPENWRT_DIR=$(pwd)
 305
 306           # Find target name from toolchain info.mk
 307           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 308
 309           cd $TOOLCHAIN_BIN
 310
 311           # Revert sdk wrapper scripts applied to all the bins
 312           for app in $(find . -name "*.bin"); do
 313             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 314             rm $TARGET_APP
 315             mv .$TARGET_APP.bin $TARGET_APP
 316           done
 317
 318           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 319           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 320           for app in cc gcc g++ c++ cpp ld as ; do
 321             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 322             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 323           done
 324
 325       - name: Configure external toolchain with sdk
 326         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 327         shell: su buildbot -c "sh -e {0}"
 328         working-directory: openwrt
 329         run: |
 330           echo CONFIG_DEVEL=y >> .config
 331           echo CONFIG_AUTOREMOVE=y >> .config
 332           echo CONFIG_CCACHE=y >> .config
 333
 334           ./scripts/ext-toolchain.sh \
 335             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 336             --overwrite-config \
 337             --config ${{ inputs.target }}/${{ inputs.subtarget }}
 338
 339       - name: Configure internal toolchain
 340         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 341         shell: su buildbot -c "sh -e {0}"
 342         working-directory: openwrt
 343         run: |
 344           echo CONFIG_DEVEL=y >> .config
 345           echo CONFIG_AUTOREMOVE=y >> .config
 346           echo CONFIG_CCACHE=y >> .config
 347
 348           echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
 349           echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
 350
 351           make defconfig
 352
 353       - name: Show configuration
 354         shell: su buildbot -c "sh -e {0}"
 355         working-directory: openwrt
 356         run: ./scripts/diffconfig.sh
 357
 358       - name: Build tools
 359         shell: su buildbot -c "sh -e {0}"
 360         working-directory: openwrt
 361         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 362
 363       - name: Build toolchain
 364         shell: su buildbot -c "sh -e {0}"
 365         working-directory: openwrt
 366         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 367
 368       - name: Build Kernel
 369         if: inputs.build_kernel == true
 370         shell: su buildbot -c "sh -e {0}"
 371         working-directory: openwrt
 372         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 373
 374       - name: Build Kernel Kmods
 375         if: inputs.build_kernel == true
 376         shell: su buildbot -c "sh -e {0}"
 377         working-directory: openwrt
 378         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 379
 380       - name: Build everything
 381         if: inputs.build_full == true
 382         shell: su buildbot -c "sh -e {0}"
 383         working-directory: openwrt
 384         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 385
 386       - name: Build external toolchain
 387         if: inputs.build_external_toolchain == true
 388         shell: su buildbot -c "sh -e {0}"
 389         working-directory: openwrt
 390         run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 391
 392       - name: Coverity prepare toolchain
 393         if: inputs.coverity_check_packages != ''
 394         shell: su buildbot -c "sh -e {0}"
 395         working-directory: openwrt
 396         run: |
 397           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
 398           wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
 399           echo ' coverity.tar.gz' >> coverity.tar.gz.md5
 400           md5sum -c coverity.tar.gz.md5
 401
 402           mkdir cov-analysis-linux64
 403           tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
 404           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 405
 406           for template in ${{ inputs.coverity_compiler_template_list }}; do
 407             cov-configure --template --comptype gcc --compiler "$template"
 408           done
 409
 410       - name: Clean and recompile packages with Coverity toolchain
 411         if: inputs.coverity_check_packages != ''
 412         shell: su buildbot -c "bash {0}"
 413         working-directory: openwrt
 414         run: |
 415           set -o pipefail -o errexit
 416
 417           coverity_check_packages=(${{ inputs.coverity_check_packages }})
 418           printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
 419           make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 420
 421           coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
 422           printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
 423           make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 424
 425           printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
 426           export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
 427           cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
 428
 429       - name: Upload build to Coverity for analysis
 430         if: inputs.coverity_check_packages != ''
 431         shell: su buildbot -c "sh -e {0}"
 432         working-directory: openwrt
 433         run: |
 434           tar czf cov-int.tar.gz ./cov-int
 435           curl \
 436             --form token="${{ secrets.coverity_api_token }}" \
 437             --form email="contact@openwrt.org" \
 438             --form file=@cov-int.tar.gz \
 439             --form version="${{ github.ref_name }}-${{ github.sha }}" \
 440             --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
 441             "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
 442
 443       - name: Upload logs
 444         if: failure()
 445         uses: actions/upload-artifact@v3
 446         with:
 447           name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
 448           path: "openwrt/logs"
 449
 450       - name: Find external toolchain name
 451         id: get-toolchain-name
 452         if: inputs.upload_external_toolchain == true
 453         working-directory: openwrt
 454         run: |
 455           TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
 456           echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
 457
 458       - name: Upload prebuilt toolchain
 459         if: inputs.upload_external_toolchain == true
 460         uses: actions/upload-artifact@v3
 461         with:
 462           name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
 463           path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
 464           retention-days: 1