27ed5cc218e370e54a14488161a173e1f971d2f9
[openwrt/staging/stintel.git] / .github / workflows / build.yml
1 name: Build sub target
2
3 on:
4 workflow_call:
5 secrets:
6 coverity_api_token:
7 inputs:
8 container_name:
9 type: string
10 default: tools
11 target:
12 required: true
13 type: string
14 subtarget:
15 required: true
16 type: string
17 testing:
18 type: boolean
19 build_toolchain:
20 type: boolean
21 include_feeds:
22 type: boolean
23 build_full:
24 type: boolean
25 build_kernel:
26 type: boolean
27 build_all_modules:
28 type: boolean
29 build_all_kmods:
30 type: boolean
31 build_all_boards:
32 type: boolean
33 use_openwrt_container:
34 type: boolean
35 default: true
36 coverity_project_name:
37 type: string
38 default: OpenWrt
39 coverity_check_packages:
40 type: string
41 coverity_compiler_template_list:
42 type: string
43 default: >-
44 arm-openwrt-linux-gcc
45 coverity_force_compile_packages:
46 type: string
47 default: >-
48 curl
49 libnl
50 mbedtls
51 wolfssl
52 openssl
53 build_external_toolchain:
54 type: boolean
55 upload_external_toolchain:
56 type: boolean
57 use_ccache_cache:
58 type: boolean
59 default: true
60
61 permissions:
62 contents: read
63
64 jobs:
65 setup_build:
66 name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
67 runs-on: ubuntu-latest
68 outputs:
69 owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
70 container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
71 container_name: ${{ steps.determine_tools_container.outputs.container_name }}
72
73 steps:
74 - name: Checkout
75 uses: actions/checkout@v3
76
77 - name: Set lower case owner name
78 id: lower_owner
79 run: |
80 OWNER_LC=$(echo "${{ github.repository_owner }}" \
81 | tr '[:upper:]' '[:lower:]')
82
83 if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
84 OWNER_LC=openwrt
85 fi
86
87 echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
88
89 # Per branch tools container tag
90 # By default stick to latest
91 # For official test targetting openwrt stable branch
92 # Get the branch or parse the tag and push dedicated tools containers
93 # For local test to use the correct container for stable release testing
94 # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
95 - name: Determine tools container tag
96 id: determine_tools_container
97 run: |
98 CONTAINER_NAME=${{ inputs.container_name }}
99 CONTAINER_TAG=latest
100 if [ -n "${{ github.base_ref }}" ]; then
101 if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
102 CONTAINER_TAG="${{ github.base_ref }}"
103 fi
104 elif [ ${{ github.ref_type }} == "branch" ]; then
105 if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
106 CONTAINER_TAG=${{ github.ref_name }}
107 elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
108 CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
109 fi
110 elif [ ${{ github.ref_type }} == "tag" ]; then
111 if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
112 CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
113 fi
114 fi
115
116 if [ "$CONTAINER_NAME" = "toolchain" ]; then
117 GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
118 GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
119 GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
120 # Check if container exist
121 if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
122 CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
123 else
124 CONTAINER_NAME=tools
125 fi
126 fi
127
128 echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
129 echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
130 echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT
131
132 build:
133 name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
134 needs: setup_build
135 runs-on: ubuntu-latest
136
137 container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
138
139 permissions:
140 contents: read
141 packages: read
142 actions: write
143
144 steps:
145 - name: Checkout master directory
146 uses: actions/checkout@v3
147 with:
148 path: openwrt
149
150 - name: Checkout packages feed
151 if: inputs.include_feeds == true
152 uses: actions/checkout@v3
153 with:
154 repository: openwrt/packages
155 path: openwrt/feeds/packages
156
157 - name: Checkout luci feed
158 if: inputs.include_feeds == true
159 uses: actions/checkout@v3
160 with:
161 repository: openwrt/luci
162 path: openwrt/feeds/luci
163
164 - name: Checkout routing feed
165 if: inputs.include_feeds == true
166 uses: actions/checkout@v3
167 with:
168 repository: openwrt/routing
169 path: openwrt/feeds/routing
170
171 - name: Checkout telephony feed
172 if: inputs.include_feeds == true
173 uses: actions/checkout@v3
174 with:
175 repository: openwrt/telephony
176 path: openwrt/feeds/telephony
177
178 - name: Parse toolchain file
179 if: inputs.build_toolchain == false
180 id: parse-toolchain
181 working-directory: openwrt
182 run: |
183 if [ -d /external-toolchain/ ]; then
184 echo "toolchain-type=external_container" >> $GITHUB_OUTPUT
185 exit 0
186 fi
187
188 TOOLCHAIN_PATH=snapshots
189
190 if [ -n "${{ github.base_ref }}" ]; then
191 if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
192 major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
193 fi
194 elif [ "${{ github.ref_type }}" = "branch" ]; then
195 if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
196 major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
197 elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
198 major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
199 fi
200 elif [ "${{ github.ref_type }}" = "tag" ]; then
201 if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
202 major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
203 fi
204 fi
205
206 if [ -n "$major_ver" ]; then
207 git fetch --tags -f
208 latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
209 if [ -n "$latest_tag" ]; then
210 TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
211 fi
212 fi
213
214 SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
215 if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
216 TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
217 TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
218
219 echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
220 elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
221 TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
222 TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
223
224 echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
225 else
226 echo "toolchain-type=internal" >> $GITHUB_OUTPUT
227 fi
228
229 echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
230 echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
231
232 - name: Fix permission
233 run: |
234 chown -R buildbot:buildbot openwrt
235
236 - name: Prepare prebuilt tools
237 shell: su buildbot -c "sh -e {0}"
238 working-directory: openwrt
239 run: |
240 mkdir -p staging_dir build_dir
241 ln -s /prebuilt_tools/staging_dir/host staging_dir/host
242 ln -s /prebuilt_tools/build_dir/host build_dir/host
243
244 ./scripts/ext-tools.sh --refresh
245
246 - name: Update & Install feeds
247 if: inputs.include_feeds == true
248 shell: su buildbot -c "sh -e {0}"
249 working-directory: openwrt
250 run: |
251 ./scripts/feeds update -a
252 ./scripts/feeds install -a
253
254 - name: Restore ccache cache
255 id: restore-ccache-cache
256 if: inputs.use_ccache_cache == true
257 uses: actions/cache/restore@v3
258 with:
259 path: openwrt/.ccache
260 key: ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-${{ hashFiles('openwrt/include/kernel-**') }}
261 restore-keys: |
262 ccache-kernel-${{ inputs.target }}/${{ inputs.subtarget }}-
263
264 - name: Download external toolchain/sdk
265 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container'
266 shell: su buildbot -c "sh -e {0}"
267 working-directory: openwrt
268 run: |
269 wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
270 | tar --xz -xf -
271
272 - name: Configure testing kernel
273 if: inputs.testing == true
274 shell: su buildbot -c "sh -e {0}"
275 working-directory: openwrt
276 run: |
277 echo CONFIG_TESTING_KERNEL=y >> .config
278
279 - name: Configure all kernel modules
280 if: inputs.build_all_kmods == true
281 shell: su buildbot -c "sh -e {0}"
282 working-directory: openwrt
283 run: |
284 echo CONFIG_ALL_KMODS=y >> .config
285
286 - name: Configure all modules
287 if: inputs.build_all_modules == true
288 shell: su buildbot -c "sh -e {0}"
289 working-directory: openwrt
290 run: |
291 echo CONFIG_ALL=y >> .config
292
293 - name: Configure all boards
294 if: inputs.build_all_boards == true
295 shell: su buildbot -c "sh -e {0}"
296 working-directory: openwrt
297 run: |
298 echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
299 echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
300 echo CONFIG_TARGET_ALL_PROFILES=y >> .config
301
302 - name: Configure ccache
303 if: inputs.use_ccache_cache == true
304 shell: su buildbot -c "sh -e {0}"
305 working-directory: openwrt
306 run: |
307 echo CONFIG_CCACHE=y >> .config
308
309 - name: Configure external toolchain in container
310 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container'
311 shell: su buildbot -c "sh -e {0}"
312 working-directory: openwrt
313 run: |
314 echo CONFIG_DEVEL=y >> .config
315 echo CONFIG_AUTOREMOVE=y >> .config
316
317 ./scripts/ext-toolchain.sh \
318 --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \
319 --overwrite-config \
320 --config ${{ inputs.target }}/${{ inputs.subtarget }}
321
322 - name: Configure external toolchain
323 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
324 shell: su buildbot -c "sh -e {0}"
325 working-directory: openwrt
326 run: |
327 echo CONFIG_DEVEL=y >> .config
328 echo CONFIG_AUTOREMOVE=y >> .config
329
330 ./scripts/ext-toolchain.sh \
331 --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
332 --overwrite-config \
333 --config ${{ inputs.target }}/${{ inputs.subtarget }}
334
335 - name: Adapt external sdk to external toolchain format
336 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
337 shell: su buildbot -c "sh -e {0}"
338 working-directory: openwrt
339 run: |
340 TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
341 TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
342 OPENWRT_DIR=$(pwd)
343
344 # Find target name from toolchain info.mk
345 GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
346
347 cd $TOOLCHAIN_BIN
348
349 # Revert sdk wrapper scripts applied to all the bins
350 for app in $(find . -name "*.bin"); do
351 TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
352 rm $TARGET_APP
353 mv .$TARGET_APP.bin $TARGET_APP
354 done
355
356 # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
357 cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
358 for app in cc gcc g++ c++ cpp ld as ; do
359 [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
360 ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
361 done
362
363 - name: Configure external toolchain with sdk
364 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
365 shell: su buildbot -c "sh -e {0}"
366 working-directory: openwrt
367 run: |
368 echo CONFIG_DEVEL=y >> .config
369 echo CONFIG_AUTOREMOVE=y >> .config
370
371 ./scripts/ext-toolchain.sh \
372 --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
373 --overwrite-config \
374 --config ${{ inputs.target }}/${{ inputs.subtarget }}
375
376 - name: Configure internal toolchain
377 if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
378 shell: su buildbot -c "sh -e {0}"
379 working-directory: openwrt
380 run: |
381 echo CONFIG_DEVEL=y >> .config
382 echo CONFIG_AUTOREMOVE=y >> .config
383
384 echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
385 echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
386
387 make defconfig
388
389 - name: Show configuration
390 shell: su buildbot -c "sh -e {0}"
391 working-directory: openwrt
392 run: ./scripts/diffconfig.sh
393
394 - name: Build tools
395 shell: su buildbot -c "sh -e {0}"
396 working-directory: openwrt
397 run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
398
399 - name: Build toolchain
400 shell: su buildbot -c "sh -e {0}"
401 working-directory: openwrt
402 run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
403
404 - name: Build Kernel
405 if: inputs.build_kernel == true
406 shell: su buildbot -c "sh -e {0}"
407 working-directory: openwrt
408 run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
409
410 - name: Build Kernel Kmods
411 if: inputs.build_kernel == true
412 shell: su buildbot -c "sh -e {0}"
413 working-directory: openwrt
414 run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
415
416 - name: Build everything
417 if: inputs.build_full == true
418 shell: su buildbot -c "sh -e {0}"
419 working-directory: openwrt
420 run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
421
422 - name: Build external toolchain
423 if: inputs.build_external_toolchain == true
424 shell: su buildbot -c "sh -e {0}"
425 working-directory: openwrt
426 run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
427
428 - name: Coverity prepare toolchain
429 if: inputs.coverity_check_packages != ''
430 shell: su buildbot -c "sh -e {0}"
431 working-directory: openwrt
432 run: |
433 wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
434 wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
435 echo ' coverity.tar.gz' >> coverity.tar.gz.md5
436 md5sum -c coverity.tar.gz.md5
437
438 mkdir cov-analysis-linux64
439 tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
440 export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
441
442 for template in ${{ inputs.coverity_compiler_template_list }}; do
443 cov-configure --template --comptype gcc --compiler "$template"
444 done
445
446 - name: Clean and recompile packages with Coverity toolchain
447 if: inputs.coverity_check_packages != ''
448 shell: su buildbot -c "bash {0}"
449 working-directory: openwrt
450 run: |
451 set -o pipefail -o errexit
452
453 coverity_check_packages=(${{ inputs.coverity_check_packages }})
454 printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
455 make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
456
457 coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
458 printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
459 make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
460
461 printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
462 export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
463 cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
464
465 - name: Upload build to Coverity for analysis
466 if: inputs.coverity_check_packages != ''
467 shell: su buildbot -c "sh -e {0}"
468 working-directory: openwrt
469 run: |
470 tar czf cov-int.tar.gz ./cov-int
471 curl \
472 --form token="${{ secrets.coverity_api_token }}" \
473 --form email="contact@openwrt.org" \
474 --form file=@cov-int.tar.gz \
475 --form version="${{ github.ref_name }}-${{ github.sha }}" \
476 --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
477 "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
478
479 - name: Upload logs
480 if: failure()
481 uses: actions/upload-artifact@v3
482 with:
483 name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
484 path: "openwrt/logs"
485
486 - name: Delete already present ccache cache
487 if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true
488 uses: octokit/request-action@v2.x
489 with:
490 route: DELETE /repos/{repository}/actions/caches?key={key}
491 env:
492 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
493 INPUT_REPOSITORY: ${{ github.repository }}
494 INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
495
496 - name: Save ccache cache
497 if: inputs.use_ccache_cache == true
498 uses: actions/cache/save@v3
499 with:
500 path: openwrt/.ccache
501 key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
502
503 - name: Find external toolchain name
504 id: get-toolchain-name
505 if: inputs.upload_external_toolchain == true
506 working-directory: openwrt
507 run: |
508 TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
509 echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
510
511 - name: Upload prebuilt toolchain
512 if: inputs.upload_external_toolchain == true
513 uses: actions/upload-artifact@v3
514 with:
515 name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
516 path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
517 retention-days: 1