0c0c05f402b7048a290e320bab8d21c86dae15c0
[openwrt/staging/hauke.git] / .github / workflows / build.yml
1 name: Build sub target
2
3 on:
4 workflow_call:
5 secrets:
6 coverity_api_token:
7 inputs:
8 container_name:
9 type: string
10 default: tools
11 target:
12 required: true
13 type: string
14 subtarget:
15 required: true
16 type: string
17 testing:
18 type: boolean
19 build_toolchain:
20 type: boolean
21 include_feeds:
22 type: boolean
23 build_full:
24 type: boolean
25 build_kernel:
26 type: boolean
27 build_all_modules:
28 type: boolean
29 build_all_kmods:
30 type: boolean
31 build_all_boards:
32 type: boolean
33 use_openwrt_container:
34 type: boolean
35 default: true
36 coverity_project_name:
37 type: string
38 default: OpenWrt
39 coverity_check_packages:
40 type: string
41 coverity_compiler_template_list:
42 type: string
43 default: >-
44 arm-openwrt-linux-gcc
45 coverity_force_compile_packages:
46 type: string
47 default: >-
48 curl
49 libnl
50 mbedtls
51 wolfssl
52 openssl
53 build_external_toolchain:
54 type: boolean
55 upload_external_toolchain:
56 type: boolean
57 use_ccache_cache:
58 type: boolean
59 default: true
60 ccache_type:
61 type: string
62 default: kernel
63
64 permissions:
65 contents: read
66
67 jobs:
68 setup_build:
69 name: Setup build ${{ inputs.target }}/${{ inputs.subtarget }}
70 runs-on: ubuntu-latest
71 outputs:
72 owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
73 container_tag: ${{ steps.determine_tools_container.outputs.container_tag }}
74 container_name: ${{ steps.determine_tools_container.outputs.container_name }}
75
76 steps:
77 - name: Checkout
78 uses: actions/checkout@v3
79
80 - name: Set lower case owner name
81 id: lower_owner
82 run: |
83 OWNER_LC=$(echo "${{ github.repository_owner }}" \
84 | tr '[:upper:]' '[:lower:]')
85
86 if [ ${{ inputs.use_openwrt_container }} == "true" ]; then
87 OWNER_LC=openwrt
88 fi
89
90 echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
91
92 # Per branch tools container tag
93 # By default stick to latest
94 # For official test targetting openwrt stable branch
95 # Get the branch or parse the tag and push dedicated tools containers
96 # For local test to use the correct container for stable release testing
97 # you need to use for the branch name a prefix of openwrt-[0-9][0-9].[0-9][0-9]-
98 - name: Determine tools container tag
99 id: determine_tools_container
100 run: |
101 CONTAINER_NAME=${{ inputs.container_name }}
102 CONTAINER_TAG=latest
103 if [ -n "${{ github.base_ref }}" ]; then
104 if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
105 CONTAINER_TAG="${{ github.base_ref }}"
106 fi
107 elif [ ${{ github.ref_type }} == "branch" ]; then
108 if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
109 CONTAINER_TAG=${{ github.ref_name }}
110 elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
111 CONTAINER_TAG="$(echo ${{ github.ref_name }} | sed 's/^\(openwrt-[0-9][0-9]\.[0-9][0-9]\)-.*/\1/')"
112 fi
113 elif [ ${{ github.ref_type }} == "tag" ]; then
114 if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
115 CONTAINER_TAG=openwrt-"$(echo ${{ github.ref_name }} | sed 's/^v\([0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
116 fi
117 fi
118
119 if [ "$CONTAINER_NAME" = "toolchain" ]; then
120 GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
121 GHCR_HEADER="Authorization: Bearer ${GHCR_TOKEN}"
122 GHCR_MANIFEST_LINK=https://ghcr.io/v2/${{ steps.lower_owner.outputs.owner_lc }}/${{ inputs.container_name }}/manifests/${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
123 # Check if container exist
124 if [ $(curl -s -o /dev/null -w "%{http_code}" -H "$GHCR_HEADER" -I "$GHCR_MANIFEST_LINK") = 200 ]; then
125 CONTAINER_TAG=${{ inputs.target }}-${{ inputs.subtarget }}-"$CONTAINER_TAG"
126 else
127 CONTAINER_NAME=tools
128 fi
129 fi
130
131 echo "Tools container to use $CONTAINER_NAME:$CONTAINER_TAG"
132 echo "container_tag=$CONTAINER_TAG" >> $GITHUB_OUTPUT
133 echo "container_name=$CONTAINER_NAME" >> $GITHUB_OUTPUT
134
135 build:
136 name: Build ${{ inputs.target }}/${{ inputs.subtarget }}
137 needs: setup_build
138 runs-on: ubuntu-latest
139
140 container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/${{ needs.setup_build.outputs.container_name }}:${{ needs.setup_build.outputs.container_tag }}
141
142 permissions:
143 contents: read
144 packages: read
145 actions: write
146
147 steps:
148 - name: Checkout master directory
149 uses: actions/checkout@v3
150 with:
151 path: openwrt
152
153 - name: Checkout packages feed
154 if: inputs.include_feeds == true
155 uses: actions/checkout@v3
156 with:
157 repository: openwrt/packages
158 path: openwrt/feeds/packages
159
160 - name: Checkout luci feed
161 if: inputs.include_feeds == true
162 uses: actions/checkout@v3
163 with:
164 repository: openwrt/luci
165 path: openwrt/feeds/luci
166
167 - name: Checkout routing feed
168 if: inputs.include_feeds == true
169 uses: actions/checkout@v3
170 with:
171 repository: openwrt/routing
172 path: openwrt/feeds/routing
173
174 - name: Checkout telephony feed
175 if: inputs.include_feeds == true
176 uses: actions/checkout@v3
177 with:
178 repository: openwrt/telephony
179 path: openwrt/feeds/telephony
180
181 - name: Parse toolchain file
182 if: inputs.build_toolchain == false
183 id: parse-toolchain
184 working-directory: openwrt
185 run: |
186 if [ -d /external-toolchain/ ]; then
187 echo "toolchain-type=external_container" >> $GITHUB_OUTPUT
188 exit 0
189 fi
190
191 TOOLCHAIN_PATH=snapshots
192
193 if [ -n "${{ github.base_ref }}" ]; then
194 if echo "${{ github.base_ref }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
195 major_ver="$(echo ${{ github.base_ref }} | sed 's/^openwrt-/v/')"
196 fi
197 elif [ "${{ github.ref_type }}" = "branch" ]; then
198 if echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]$'; then
199 major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-/v/')"
200 elif echo "${{ github.ref_name }}" | grep -q -E '^openwrt-[0-9][0-9]\.[0-9][0-9]-'; then
201 major_ver="$(echo ${{ github.ref_name }} | sed 's/^openwrt-\([0-9][0-9]\.[0-9][0-9]\)-.*/v\1/')"
202 fi
203 elif [ "${{ github.ref_type }}" = "tag" ]; then
204 if echo "${{ github.ref_name }}" | grep -q -E '^v[0-9][0-9]\.[0-9][0-9]\..+'; then
205 major_ver="$(echo ${{ github.ref_name }} | sed 's/^\(v[0-9][0-9]\.[0-9][0-9]\)\..\+/\1/')"
206 fi
207 fi
208
209 if [ -n "$major_ver" ]; then
210 git fetch --tags -f
211 latest_tag="$(git tag --sort=-creatordate -l $major_ver* | head -n1)"
212 if [ -n "$latest_tag" ]; then
213 TOOLCHAIN_PATH=releases/$(echo $latest_tag | sed 's/^v//')
214 fi
215 fi
216
217 SUMS_FILE="https://downloads.cdn.openwrt.org/$TOOLCHAIN_PATH/targets/${{ inputs.target }}/${{ inputs.subtarget }}/sha256sums"
218 if curl $SUMS_FILE | grep -q ".*openwrt-toolchain.*tar.xz"; then
219 TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-toolchain.*tar.xz")"
220 TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
221
222 echo "toolchain-type=external_toolchain" >> $GITHUB_OUTPUT
223 elif curl $SUMS_FILE | grep -q ".*openwrt-sdk.*tar.xz"; then
224 TOOLCHAIN_STRING="$( curl $SUMS_FILE | grep ".*openwrt-sdk.*tar.xz")"
225 TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-sdk.*\).tar.xz/\1/p')
226
227 echo "toolchain-type=external_sdk" >> $GITHUB_OUTPUT
228 else
229 echo "toolchain-type=internal" >> $GITHUB_OUTPUT
230 fi
231
232 echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
233 echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
234
235 - name: Fix permission
236 run: |
237 chown -R buildbot:buildbot openwrt
238
239 - name: Prepare prebuilt tools
240 shell: su buildbot -c "sh -e {0}"
241 working-directory: openwrt
242 run: |
243 mkdir -p staging_dir build_dir
244 ln -s /prebuilt_tools/staging_dir/host staging_dir/host
245 ln -s /prebuilt_tools/build_dir/host build_dir/host
246
247 ./scripts/ext-tools.sh --refresh
248
249 - name: Update & Install feeds
250 if: inputs.include_feeds == true
251 shell: su buildbot -c "sh -e {0}"
252 working-directory: openwrt
253 run: |
254 ./scripts/feeds update -a
255 ./scripts/feeds install -a
256
257 - name: Restore ccache cache
258 id: restore-ccache-cache
259 if: inputs.use_ccache_cache == true
260 uses: actions/cache/restore@v3
261 with:
262 path: openwrt/.ccache
263 key: ccache-${{ inputs.ccache_type }}-${{ inputs.target }}/${{ inputs.subtarget }}-${{ hashFiles('openwrt/include/kernel-**') }}
264 restore-keys: |
265 ccache-${{ inputs.ccache_type }}-${{ inputs.target }}/${{ inputs.subtarget }}-
266
267 - name: Download external toolchain/sdk
268 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type != 'internal' && steps.parse-toolchain.outputs.toolchain-type != 'external_container'
269 shell: su buildbot -c "sh -e {0}"
270 working-directory: openwrt
271 run: |
272 wget -O - https://downloads.cdn.openwrt.org/${{ env.TOOLCHAIN_PATH }}/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ env.TOOLCHAIN_FILE }}.tar.xz \
273 | tar --xz -xf -
274
275 - name: Configure testing kernel
276 if: inputs.testing == true
277 shell: su buildbot -c "sh -e {0}"
278 working-directory: openwrt
279 run: |
280 echo CONFIG_TESTING_KERNEL=y >> .config
281
282 - name: Configure all kernel modules
283 if: inputs.build_all_kmods == true
284 shell: su buildbot -c "sh -e {0}"
285 working-directory: openwrt
286 run: |
287 echo CONFIG_ALL_KMODS=y >> .config
288
289 - name: Configure all modules
290 if: inputs.build_all_modules == true
291 shell: su buildbot -c "sh -e {0}"
292 working-directory: openwrt
293 run: |
294 echo CONFIG_ALL=y >> .config
295
296 - name: Configure all boards
297 if: inputs.build_all_boards == true
298 shell: su buildbot -c "sh -e {0}"
299 working-directory: openwrt
300 run: |
301 echo CONFIG_TARGET_MULTI_PROFILE=y >> .config
302 echo CONFIG_TARGET_PER_DEVICE_ROOTFS=y >> .config
303 echo CONFIG_TARGET_ALL_PROFILES=y >> .config
304
305 # ccache for some reason have problem detecting compiler type
306 # with external toolchain. This cause the complete malfunction
307 # of ccache with the result of tons of unsupported compiler
308 # option error.
309 # To fix this force compiler type to gcc.
310 - name: Configure ccache and apply fixes
311 if: inputs.use_ccache_cache == true
312 shell: su buildbot -c "sh -e {0}"
313 working-directory: openwrt
314 env:
315 SYSTEM_CCACHE_CONF: staging_dir/host/etc/ccache.conf
316 run: |
317 touch $SYSTEM_CCACHE_CONF
318
319 echo compiler_type=gcc >> $SYSTEM_CCACHE_CONF
320
321 echo CONFIG_CCACHE=y >> .config
322
323 - name: Configure external toolchain in container
324 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_container'
325 shell: su buildbot -c "sh -e {0}"
326 working-directory: openwrt
327 run: |
328 echo CONFIG_DEVEL=y >> .config
329 echo CONFIG_AUTOREMOVE=y >> .config
330
331 ./scripts/ext-toolchain.sh \
332 --toolchain /external-toolchain/$(ls /external-toolchain/ | grep openwrt-toolchain)/toolchain-* \
333 --overwrite-config \
334 --config ${{ inputs.target }}/${{ inputs.subtarget }}
335
336 - name: Configure external toolchain
337 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_toolchain'
338 shell: su buildbot -c "sh -e {0}"
339 working-directory: openwrt
340 run: |
341 echo CONFIG_DEVEL=y >> .config
342 echo CONFIG_AUTOREMOVE=y >> .config
343
344 ./scripts/ext-toolchain.sh \
345 --toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
346 --overwrite-config \
347 --config ${{ inputs.target }}/${{ inputs.subtarget }}
348
349 - name: Adapt external sdk to external toolchain format
350 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
351 shell: su buildbot -c "sh -e {0}"
352 working-directory: openwrt
353 run: |
354 TOOLCHAIN_DIR=${{ env.TOOLCHAIN_FILE }}/staging_dir/$(ls ${{ env.TOOLCHAIN_FILE }}/staging_dir | grep toolchain)
355 TOOLCHAIN_BIN=$TOOLCHAIN_DIR/bin
356 OPENWRT_DIR=$(pwd)
357
358 # Find target name from toolchain info.mk
359 GNU_TARGET_NAME=$(cat $TOOLCHAIN_DIR/info.mk | grep TARGET_CROSS | sed 's/^TARGET_CROSS=\(.*\)-$/\1/')
360
361 cd $TOOLCHAIN_BIN
362
363 # Revert sdk wrapper scripts applied to all the bins
364 for app in $(find . -name "*.bin"); do
365 TARGET_APP=$(echo $app | sed 's/\.\/\.\(.*\)\.bin/\1/')
366 rm $TARGET_APP
367 mv .$TARGET_APP.bin $TARGET_APP
368 done
369
370 # Setup the wrapper script in the sdk toolchain dir simulating an external toolchain build
371 cp $OPENWRT_DIR/target/toolchain/files/wrapper.sh $GNU_TARGET_NAME-wrapper.sh
372 for app in cc gcc g++ c++ cpp ld as ; do
373 [ -f $GNU_TARGET_NAME-$app ] && mv $GNU_TARGET_NAME-$app $GNU_TARGET_NAME-$app.bin
374 ln -sf $GNU_TARGET_NAME-wrapper.sh $GNU_TARGET_NAME-$app
375 done
376
377 - name: Configure external toolchain with sdk
378 if: inputs.build_toolchain == false && steps.parse-toolchain.outputs.toolchain-type == 'external_sdk'
379 shell: su buildbot -c "sh -e {0}"
380 working-directory: openwrt
381 run: |
382 echo CONFIG_DEVEL=y >> .config
383 echo CONFIG_AUTOREMOVE=y >> .config
384
385 ./scripts/ext-toolchain.sh \
386 --toolchain ${{ env.TOOLCHAIN_FILE }}/staging_dir/toolchain-* \
387 --overwrite-config \
388 --config ${{ inputs.target }}/${{ inputs.subtarget }}
389
390 - name: Configure internal toolchain
391 if: inputs.build_toolchain == true || steps.parse-toolchain.outputs.toolchain-type == 'internal'
392 shell: su buildbot -c "sh -e {0}"
393 working-directory: openwrt
394 run: |
395 echo CONFIG_DEVEL=y >> .config
396 echo CONFIG_AUTOREMOVE=y >> .config
397
398 echo "CONFIG_TARGET_${{ inputs.target }}=y" >> .config
399 echo "CONFIG_TARGET_${{ inputs.target }}_${{ inputs.subtarget }}=y" >> .config
400
401 make defconfig
402
403 - name: Show configuration
404 shell: su buildbot -c "sh -e {0}"
405 working-directory: openwrt
406 run: ./scripts/diffconfig.sh
407
408 - name: Build tools
409 shell: su buildbot -c "sh -e {0}"
410 working-directory: openwrt
411 run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
412
413 - name: Build toolchain
414 shell: su buildbot -c "sh -e {0}"
415 working-directory: openwrt
416 run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
417
418 - name: Build Kernel
419 if: inputs.build_kernel == true
420 shell: su buildbot -c "sh -e {0}"
421 working-directory: openwrt
422 run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
423
424 - name: Build Kernel Kmods
425 if: inputs.build_kernel == true
426 shell: su buildbot -c "sh -e {0}"
427 working-directory: openwrt
428 run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
429
430 - name: Build everything
431 if: inputs.build_full == true
432 shell: su buildbot -c "sh -e {0}"
433 working-directory: openwrt
434 run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
435
436 - name: Build external toolchain
437 if: inputs.build_external_toolchain == true
438 shell: su buildbot -c "sh -e {0}"
439 working-directory: openwrt
440 run: make target/toolchain/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
441
442 - name: Coverity prepare toolchain
443 if: inputs.coverity_check_packages != ''
444 shell: su buildbot -c "sh -e {0}"
445 working-directory: openwrt
446 run: |
447 wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}" -O coverity.tar.gz
448 wget -q https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.coverity_api_token }}&project=${{ inputs.coverity_project_name }}&md5=1" -O coverity.tar.gz.md5
449 echo ' coverity.tar.gz' >> coverity.tar.gz.md5
450 md5sum -c coverity.tar.gz.md5
451
452 mkdir cov-analysis-linux64
453 tar xzf coverity.tar.gz --strip 1 -C cov-analysis-linux64
454 export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
455
456 for template in ${{ inputs.coverity_compiler_template_list }}; do
457 cov-configure --template --comptype gcc --compiler "$template"
458 done
459
460 - name: Clean and recompile packages with Coverity toolchain
461 if: inputs.coverity_check_packages != ''
462 shell: su buildbot -c "bash {0}"
463 working-directory: openwrt
464 run: |
465 set -o pipefail -o errexit
466
467 coverity_check_packages=(${{ inputs.coverity_check_packages }})
468 printf -v clean_packages "package/%s/clean " "${coverity_check_packages[@]}"
469 make -j$(nproc) BUILD_LOG=1 $clean_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
470
471 coverity_force_compile_packages=(${{ inputs.coverity_force_compile_packages }})
472 printf -v force_compile_packages "package/%s/compile " "${coverity_force_compile_packages[@]}"
473 make -j$(nproc) BUILD_LOG=1 $force_compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
474
475 printf -v compile_packages "package/%s/compile " "${coverity_check_packages[@]}"
476 export PATH=$(pwd)/cov-analysis-linux64/bin:$PATH
477 cov-build --dir cov-int make -j $(nproc) BUILD_LOG=1 $compile_packages || ret=$? .github/workflows/scripts/show_build_failures.sh
478
479 - name: Upload build to Coverity for analysis
480 if: inputs.coverity_check_packages != ''
481 shell: su buildbot -c "sh -e {0}"
482 working-directory: openwrt
483 run: |
484 tar czf cov-int.tar.gz ./cov-int
485 curl \
486 --form token="${{ secrets.coverity_api_token }}" \
487 --form email="contact@openwrt.org" \
488 --form file=@cov-int.tar.gz \
489 --form version="${{ github.ref_name }}-${{ github.sha }}" \
490 --form description="OpenWrt ${{ github.ref_name }}-${{ github.sha }}" \
491 "https://scan.coverity.com/builds?project=${{ inputs.coverity_project_name }}"
492
493 - name: Upload logs
494 if: failure()
495 uses: actions/upload-artifact@v3
496 with:
497 name: ${{ inputs.target }}-${{ inputs.subtarget }}-logs
498 path: "openwrt/logs"
499
500 - name: Delete already present ccache cache
501 if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true
502 uses: octokit/request-action@v2.x
503 with:
504 route: DELETE /repos/{repository}/actions/caches?key={key}
505 env:
506 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
507 INPUT_REPOSITORY: ${{ github.repository }}
508 INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
509
510 - name: Save ccache cache
511 if: inputs.use_ccache_cache == true
512 uses: actions/cache/save@v3
513 with:
514 path: openwrt/.ccache
515 key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
516
517 - name: Find external toolchain name
518 id: get-toolchain-name
519 if: inputs.upload_external_toolchain == true
520 working-directory: openwrt
521 run: |
522 TOOLCHAIN_NAME=$(ls bin/targets/${{inputs.target }}/${{ inputs.subtarget }} | grep toolchain)
523 echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
524
525 - name: Upload prebuilt toolchain
526 if: inputs.upload_external_toolchain == true
527 uses: actions/upload-artifact@v3
528 with:
529 name: ${{ inputs.target }}-${{ inputs.subtarget }}-external-toolchain
530 path: openwrt/bin/targets/${{ inputs.target }}/${{ inputs.subtarget }}/${{ steps.get-toolchain-name.outputs.toolchain-name }}
531 retention-days: 1