.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     inputs:
   6       target:
   7         required: true
   8         type: string
   9       testing:
  10         type: boolean
  11       build_toolchain:
  12         type: boolean
  13       include_feeds:
  14         type: boolean
  15       build_full:
  16         type: boolean
  17       build_kernel:
  18         type: boolean
  19       build_all_modules:
  20         type: boolean
  21       build_all_kmods:
  22         type: boolean
  23       build_all_boards:
  24         type: boolean
  25       use_openwrt_container:
  26         type: boolean
  27         default: true
  28
  29 permissions:
  30   contents: read
  31
  32 jobs:
  33   setup_build:
  34     name: Setup build ${{ inputs.target }}
  35     runs-on: ubuntu-latest
  36     outputs:
  37       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  38       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  39       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  40
  41     steps:
  42       - name: Checkout
  43         uses: actions/checkout@v3
  44
  45       - name: Set lower case owner name
  46         id: lower_owner
  47         run: |
  48           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  49             | tr '[:upper:]' '[:lower:]')
  50
  51           if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
  52             OWNER_LC=openwrt
  53           fi
  54
  55           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  56
  57       - name: Generate ccache hash
  58         id: ccache_hash
  59         run: |
  60           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  61            | md5sum | awk '{ print $1 }')
  62           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  63
  64       # Per branch tools container tag
  65       # By default stick to latest
  66       # For official test targetting openwrt stable branch
  67       # Get the branch or parse the tag and push dedicated tools containers
  68       # For local test to use the correct container for stable release testing
  69       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  70       - name: Determine tools container tag
  71         id: determine_tools_container
  72         run: |
  73           CONTAINER_TAG=latest
  74           if [ -n "${{ github.base_ref }}" ]; then
  75             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  76               CONTAINER_TAG="${{ github.base_ref }}"
  77             fi
  78           elif [ ${{ github.ref_type }} == "branch" ]; then
  79             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  80               CONTAINER_TAG=${{ github.ref_name }}
  81             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
  82               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
  83             fi
  84           elif [ ${{ github.ref_type }} == "tag" ]; then
  85             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
  86               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
  87             fi
  88           fi
  89           echo "Tools container to use tools:$CONTAINER_TAG"
  90           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
  91
  92   build:
  93     name: Build ${{ inputs.target }}
  94     needs: setup_build
  95     runs-on: ubuntu-latest
  96
  97     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:${{ needs.setup_build.outputs.container_tag }}
  98
  99     permissions:
 100       contents: read
 101       packages: read
 102
 103     steps:
 104       - name: Checkout master directory
 105         uses: actions/checkout@v3
 106         with:
 107           path: openwrt
 108
 109       - name: Checkout packages feed
 110         if: inputs.include_feeds == true
 111         uses: actions/checkout@v3
 112         with:
 113           repository: openwrt/packages
 114           path: openwrt/feeds/packages
 115
 116       - name: Checkout luci feed
 117         if: inputs.include_feeds == true
 118         uses: actions/checkout@v3
 119         with:
 120           repository: openwrt/luci
 121           path: openwrt/feeds/luci
 122
 123       - name: Checkout routing feed
 124         if: inputs.include_feeds == true
 125         uses: actions/checkout@v3
 126         with:
 127           repository: openwrt/routing
 128           path: openwrt/feeds/routing
 129
 130       - name: Checkout telephony feed
 131         if: inputs.include_feeds == true
 132         uses: actions/checkout@v3
 133         with:
 134           repository: openwrt/telephony
 135           path: openwrt/feeds/telephony
 136
 137       - name: Fix permission
 138         run: |
 139           chown -R buildbot:buildbot openwrt
 140
 141       - name: Initialization environment
 142         run: |
 143           TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1)
 144           SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2)
 145           echo "TARGET=$TARGET" >> "$GITHUB_ENV"
 146           echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
 147
 148       - name: Prepare prebuilt tools
 149         shell: su buildbot -c "sh -e {0}"
 150         working-directory: openwrt
 151         run: |
 152           mkdir -p staging_dir build_dir
 153           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 154           ln -s /prebuilt_tools/build_dir/host build_dir/host
 155
 156           ./scripts/ext-tools.sh --refresh
 157
 158       - name: Update & Install feeds
 159         if: inputs.include_feeds == true
 160         shell: su buildbot -c "sh -e {0}"
 161         working-directory: openwrt
 162         run: |
 163           ./scripts/feeds update -a
 164           ./scripts/feeds install -a
 165
 166       - name: Parse toolchain file
 167         if: inputs.build_toolchain == false
 168         id: parse-toolchain
 169         working-directory: openwrt
 170         run: |
 171           TOOLCHAIN_PATH=snapshots
 172
 173           if [ -n "${{ github.base_ref }}" ]; then
 174             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 175               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 176             fi
 177           elif [ "${{ github.ref_type }}" = "branch" ]; then
 178             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 179               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 180             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 181               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 182             fi
 183           elif [ "${{ github.ref_type }}" = "tag" ]; then
 184             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 185               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 186             fi
 187           fi
 188
 189           if [ -n "$major_ver" ]; then
 190             git fetch --tags -f
 191             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 192             if [ -n "$latest_tag" ]; then
 193               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 194             fi
 195           fi
 196
 197           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums"
 198           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 199             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 200             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 201             TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
 202
 203             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 204           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 205             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 206             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 207             TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
 208
 209             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 210           else
 211             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 212           fi
 213
 214           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 215           echo "TOOLCHAIN_SHA256=$TOOLCHAIN_SHA256" >> "$GITHUB_ENV"
 216           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 217
 218       - name: Cache external toolchain/sdk
 219         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 220         id: cache-external-toolchain
 221         uses: actions/cache@v3
 222         with:
 223           path: openwrt/${{ env.TOOLCHAIN_FILE }}
 224           key: ${{ env.TOOLCHAIN_FILE }}-${{ steps.parse-toolchain.outputs.toolchain-type }}-${{ env.TOOLCHAIN_SHA256 }}
 225
 226       - name: Cache ccache
 227         uses: actions/cache@v3
 228         with:
 229           path: openwrt/.ccache
 230           key: ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-${{ needs.setup_build.outputs.ccache_hash }}
 231           restore-keys: |
 232             ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-
 233
 234       - name: Download external toolchain/sdk
 235         if: inputs.build_toolchain == false && steps.cache-external-toolchain.outputs.cache-hit != 'true' && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 236         shell: su buildbot -c "sh -e {0}"
 237         working-directory: openwrt
 238         run: |
 239           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 240             | tar --xz -xf -
 241
 242       - name: Configure testing kernel
 243         if: inputs.testing == true
 244         shell: su buildbot -c "sh -e {0}"
 245         working-directory: openwrt
 246         run: |
 247           echo CONFIG_TESTING_KERNEL=y >> .config
 248
 249       - name: Configure all kernel modules
 250         if: inputs.build_all_kmods == true
 251         shell: su buildbot -c "sh -e {0}"
 252         working-directory: openwrt
 253         run: |
 254           echo CONFIG_ALL_KMODS=y >> .config
 255
 256       - name: Configure all modules
 257         if: inputs.build_all_modules == true
 258         shell: su buildbot -c "sh -e {0}"
 259         working-directory: openwrt
 260         run: |
 261           echo CONFIG_ALL=y >> .config
 262
 263       - name: Configure all boards
 264         if: inputs.build_all_boards == true
 265         shell: su buildbot -c "sh -e {0}"
 266         working-directory: openwrt
 267         run: |
 268           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 269           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 270           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 271
 272       - name: Configure external toolchain
 273         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 274         shell: su buildbot -c "sh -e {0}"
 275         working-directory: openwrt
 276         run: |
 277           echo CONFIG_DEVEL=y >> .config
 278           echo CONFIG_AUTOREMOVE=y >> .config
 279           echo CONFIG_CCACHE=y >> .config
 280
 281           ./scripts/ext-toolchain.sh \
 282             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 283             --overwrite-config \
 284             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 285
 286       - name: Adapt external sdk to external toolchain format
 287         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk' && steps.cache-external-toolchain.outputs.cache-hit != 'true'
 288         shell: su buildbot -c "sh -e {0}"
 289         working-directory: openwrt
 290         run: |
 291           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 292           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 293           OPENWRT_DIR=$(pwd)
 294
 295           # Find target name from toolchain info.mk
 296           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 297
 298           cd $TOOLCHAIN_BIN
 299
 300           # Revert sdk wrapper scripts applied to all the bins
 301           for app in $(find . -name "*.bin"); do
 302             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 303             rm $TARGET_APP
 304             mv .$TARGET_APP.bin $TARGET_APP
 305           done
 306
 307           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 308           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 309           for app in cc gcc g++ c++ cpp ld as ; do
 310             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 311             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 312           done
 313
 314       - name: Configure external toolchain with sdk
 315         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 316         shell: su buildbot -c "sh -e {0}"
 317         working-directory: openwrt
 318         run: |
 319           echo CONFIG_DEVEL=y >> .config
 320           echo CONFIG_AUTOREMOVE=y >> .config
 321           echo CONFIG_CCACHE=y >> .config
 322
 323           ./scripts/ext-toolchain.sh \
 324             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 325             --overwrite-config \
 326             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 327
 328       - name: Configure internal toolchain
 329         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 330         shell: su buildbot -c "sh -e {0}"
 331         working-directory: openwrt
 332         run: |
 333           echo CONFIG_DEVEL=y >> .config
 334           echo CONFIG_AUTOREMOVE=y >> .config
 335           echo CONFIG_CCACHE=y >> .config
 336
 337           echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config
 338           echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config
 339
 340           make defconfig
 341
 342       - name: Show configuration
 343         shell: su buildbot -c "sh -e {0}"
 344         working-directory: openwrt
 345         run: ./scripts/diffconfig.sh
 346
 347       - name: Build tools
 348         shell: su buildbot -c "sh -e {0}"
 349         working-directory: openwrt
 350         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 351
 352       - name: Build toolchain
 353         shell: su buildbot -c "sh -e {0}"
 354         working-directory: openwrt
 355         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 356
 357       - name: Build Kernel
 358         if: inputs.build_kernel == true
 359         shell: su buildbot -c "sh -e {0}"
 360         working-directory: openwrt
 361         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 362
 363       - name: Build Kernel Kmods
 364         if: inputs.build_kernel == true
 365         shell: su buildbot -c "sh -e {0}"
 366         working-directory: openwrt
 367         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 368
 369       - name: Build everything
 370         if: inputs.build_full == true
 371         shell: su buildbot -c "sh -e {0}"
 372         working-directory: openwrt
 373         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 374
 375       - name: Upload logs
 376         if: failure()
 377         uses: actions/upload-artifact@v3
 378         with:
 379           name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs
 380           path: "openwrt/logs"