.github/workflows/build.yml

   1 name: Build sub target
   2
   3 on:
   4   workflow_call:
   5     inputs:
   6       target:
   7         required: true
   8         type: string
   9       testing:
  10         type: boolean
  11       build_toolchain:
  12         type: boolean
  13       include_feeds:
  14         type: boolean
  15       build_full:
  16         type: boolean
  17       build_kernel:
  18         type: boolean
  19       build_all_modules:
  20         type: boolean
  21       build_all_kmods:
  22         type: boolean
  23       build_all_boards:
  24         type: boolean
  25
  26 permissions:
  27   contents: read
  28
  29 jobs:
  30   setup_build:
  31     name: Setup build ${{ inputs.target }}
  32     runs-on: ubuntu-latest
  33     outputs:
  34       owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
  35       ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
  36       container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
  37
  38     steps:
  39       - name: Checkout
  40         uses: actions/checkout@v3
  41
  42       - name: Set lower case owner name
  43         id: lower_owner
  44         run: |
  45           OWNER_LC=$(echo "${{ github.repository_owner }}" \
  46             | tr '[:upper:]' '[:lower:]')
  47           echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
  48
  49       - name: Generate ccache hash
  50         id: ccache_hash
  51         run: |
  52           CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
  53            | md5sum | awk '{ print $1 }')
  54           echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
  55
  56       # Per branch tools container tag
  57       # By default stick to latest
  58       # For official test targetting openwrt stable branch
  59       # Get the branch or parse the tag and push dedicated tools containers
  60       # For local test to use the correct container for stable release testing
  61       # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
  62       - name: Determine tools container tag
  63         id: determine_tools_container
  64         run: |
  65           CONTAINER_TAG=latest
  66           if [ -n "${{ github.base_ref }}" ]; then
  67             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  68               CONTAINER_TAG="${{ github.base_ref }}"
  69             fi
  70           elif [ ${{ github.ref_type }} == "branch" ]; then
  71             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
  72               CONTAINER_TAG=${{ github.ref_name }}
  73             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
  74               CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
  75             fi
  76           elif [ ${{ github.ref_type }} == "tag" ]; then
  77             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
  78               CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
  79             fi
  80           fi
  81           echo "Tools container to use tools:$CONTAINER_TAG"
  82           echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
  83
  84   build:
  85     name: Build ${{ inputs.target }}
  86     needs: setup_build
  87     runs-on: ubuntu-latest
  88
  89     container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:${{ needs.setup_build.outputs.container_tag }}
  90
  91     permissions:
  92       contents: read
  93       packages: read
  94
  95     steps:
  96       - name: Checkout master directory
  97         uses: actions/checkout@v3
  98         with:
  99           path: openwrt
 100
 101       - name: Checkout packages feed
 102         if: inputs.include_feeds == true
 103         uses: actions/checkout@v3
 104         with:
 105           repository: openwrt/packages
 106           path: openwrt/feeds/packages
 107
 108       - name: Checkout luci feed
 109         if: inputs.include_feeds == true
 110         uses: actions/checkout@v3
 111         with:
 112           repository: openwrt/luci
 113           path: openwrt/feeds/luci
 114
 115       - name: Checkout routing feed
 116         if: inputs.include_feeds == true
 117         uses: actions/checkout@v3
 118         with:
 119           repository: openwrt/routing
 120           path: openwrt/feeds/routing
 121
 122       - name: Checkout telephony feed
 123         if: inputs.include_feeds == true
 124         uses: actions/checkout@v3
 125         with:
 126           repository: openwrt/telephony
 127           path: openwrt/feeds/telephony
 128
 129       - name: Fix permission
 130         run: |
 131           chown -R buildbot:buildbot openwrt
 132
 133       - name: Initialization environment
 134         run: |
 135           TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1)
 136           SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2)
 137           echo "TARGET=$TARGET" >> "$GITHUB_ENV"
 138           echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
 139
 140       - name: Prepare prebuilt tools
 141         shell: su buildbot -c "sh -e {0}"
 142         working-directory: openwrt
 143         run: |
 144           mkdir -p staging_dir build_dir
 145           ln -s /prebuilt_tools/staging_dir/host staging_dir/host
 146           ln -s /prebuilt_tools/build_dir/host build_dir/host
 147
 148           ./scripts/ext-tools.sh --refresh
 149
 150       - name: Update & Install feeds
 151         if: inputs.include_feeds == true
 152         shell: su buildbot -c "sh -e {0}"
 153         working-directory: openwrt
 154         run: |
 155           ./scripts/feeds update -a
 156           ./scripts/feeds install -a
 157
 158       - name: Parse toolchain file
 159         if: inputs.build_toolchain == false
 160         id: parse-toolchain
 161         working-directory: openwrt
 162         run: |
 163           TOOLCHAIN_PATH=snapshots
 164
 165           if [ -n "${{ github.base_ref }}" ]; then
 166             if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 167               major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
 168             fi
 169           elif [ "${{ github.ref_type }}" = "branch" ]; then
 170             if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
 171               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
 172             elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
 173               major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
 174             fi
 175           elif [ "${{ github.ref_type }}" = "tag" ]; then
 176             if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
 177               major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
 178             fi
 179           fi
 180
 181           if [ -n "$major_ver" ]; then
 182             git fetch --tags -f
 183             latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
 184             if [ -n "$latest_tag" ]; then
 185               TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
 186             fi
 187           fi
 188
 189           SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums"
 190           if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
 191             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
 192             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
 193             TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
 194
 195             echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
 196           elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
 197             TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
 198             TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
 199             TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
 200
 201             echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
 202           else
 203             echo "toolchain-type=internal" >> $GITHUB_OUTPUT
 204           fi
 205
 206           echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
 207           echo "TOOLCHAIN_SHA256=$TOOLCHAIN_SHA256" >> "$GITHUB_ENV"
 208           echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
 209
 210       - name: Cache external toolchain/sdk
 211         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 212         id: cache-external-toolchain
 213         uses: actions/cache@v3
 214         with:
 215           path: openwrt/${{ env.TOOLCHAIN_FILE }}
 216           key: ${{ env.TOOLCHAIN_FILE }}-${{ steps.parse-toolchain.outputs.toolchain-type }}-${{ env.TOOLCHAIN_SHA256 }}
 217
 218       - name: Cache ccache
 219         uses: actions/cache@v3
 220         with:
 221           path: openwrt/.ccache
 222           key: ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-${{ needs.setup_build.outputs.ccache_hash }}
 223           restore-keys: |
 224             ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-
 225
 226       - name: Download external toolchain/sdk
 227         if: inputs.build_toolchain == false && steps.cache-external-toolchain.outputs.cache-hit != 'true' && steps.parse-toolchain.outputs.toolchain-type != 'internal'
 228         shell: su buildbot -c "sh -e {0}"
 229         working-directory: openwrt
 230         run: |
 231           wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
 232             | tar --xz -xf -
 233
 234       - name: Configure testing kernel
 235         if: inputs.testing == true
 236         shell: su buildbot -c "sh -e {0}"
 237         working-directory: openwrt
 238         run: |
 239           echo CONFIG_TESTING_KERNEL=y >> .config
 240
 241       - name: Configure all kernel modules
 242         if: inputs.build_all_kmods == true
 243         shell: su buildbot -c "sh -e {0}"
 244         working-directory: openwrt
 245         run: |
 246           echo CONFIG_ALL_KMODS=y >> .config
 247
 248       - name: Configure all modules
 249         if: inputs.build_all_modules == true
 250         shell: su buildbot -c "sh -e {0}"
 251         working-directory: openwrt
 252         run: |
 253           echo CONFIG_ALL=y >> .config
 254
 255       - name: Configure all boards
 256         if: inputs.build_all_boards == true
 257         shell: su buildbot -c "sh -e {0}"
 258         working-directory: openwrt
 259         run: |
 260           echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
 261           echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
 262           echo CONFIG_TARGET_ALL_PROFILES=y >> .config
 263
 264       - name: Configure external toolchain
 265         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
 266         shell: su buildbot -c "sh -e {0}"
 267         working-directory: openwrt
 268         run: |
 269           echo CONFIG_DEVEL=y >> .config
 270           echo CONFIG_AUTOREMOVE=y >> .config
 271           echo CONFIG_CCACHE=y >> .config
 272
 273           ./scripts/ext-toolchain.sh \
 274             --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
 275             --overwrite-config \
 276             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 277
 278       - name: Adapt external sdk to external toolchain format
 279         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk' && steps.cache-external-toolchain.outputs.cache-hit != 'true'
 280         shell: su buildbot -c "sh -e {0}"
 281         working-directory: openwrt
 282         run: |
 283           TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
 284           TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
 285           OPENWRT_DIR=$(pwd)
 286
 287           # Find target name from toolchain info.mk
 288           GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
 289
 290           cd $TOOLCHAIN_BIN
 291
 292           # Revert sdk wrapper scripts applied to all the bins
 293           for app in $(find . -name "*.bin"); do
 294             TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
 295             rm $TARGET_APP
 296             mv .$TARGET_APP.bin $TARGET_APP
 297           done
 298
 299           # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
 300           cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
 301           for app in cc gcc g++ c++ cpp ld as ; do
 302             [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
 303             ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
 304           done
 305
 306       - name: Configure external toolchain with sdk
 307         if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
 308         shell: su buildbot -c "sh -e {0}"
 309         working-directory: openwrt
 310         run: |
 311           echo CONFIG_DEVEL=y >> .config
 312           echo CONFIG_AUTOREMOVE=y >> .config
 313           echo CONFIG_CCACHE=y >> .config
 314
 315           ./scripts/ext-toolchain.sh \
 316             --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
 317             --overwrite-config \
 318             --config ${{ env.TARGET }}/${{ env.SUBTARGET }}
 319
 320       - name: Configure internal toolchain
 321         if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
 322         shell: su buildbot -c "sh -e {0}"
 323         working-directory: openwrt
 324         run: |
 325           echo CONFIG_DEVEL=y >> .config
 326           echo CONFIG_AUTOREMOVE=y >> .config
 327           echo CONFIG_CCACHE=y >> .config
 328
 329           echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config
 330           echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config
 331
 332           make defconfig
 333
 334       - name: Show configuration
 335         shell: su buildbot -c "sh -e {0}"
 336         working-directory: openwrt
 337         run: ./scripts/diffconfig.sh
 338
 339       - name: Build tools
 340         shell: su buildbot -c "sh -e {0}"
 341         working-directory: openwrt
 342         run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 343
 344       - name: Build toolchain
 345         shell: su buildbot -c "sh -e {0}"
 346         working-directory: openwrt
 347         run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 348
 349       - name: Build Kernel
 350         if: inputs.build_kernel == true
 351         shell: su buildbot -c "sh -e {0}"
 352         working-directory: openwrt
 353         run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 354
 355       - name: Build Kernel Kmods
 356         if: inputs.build_kernel == true
 357         shell: su buildbot -c "sh -e {0}"
 358         working-directory: openwrt
 359         run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 360
 361       - name: Build everything
 362         if: inputs.build_full == true
 363         shell: su buildbot -c "sh -e {0}"
 364         working-directory: openwrt
 365         run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
 366
 367       - name: Upload logs
 368         if: failure()
 369         uses: actions/upload-artifact@v3
 370         with:
 371           name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs
 372           path: "openwrt/logs"