build: use zstd for IB, toolchain, SDK and LLVM compression
[openwrt/staging/stintel.git] / target / sdk / Makefile
index bf8c3ca2c66cba4c680a3d42603423b4de472d7a..f4b7640d2cd3b32d1cf2d4d60deef13437e0deb6 100644 (file)
@@ -9,6 +9,7 @@
 include $(TOPDIR)/rules.mk
 include $(INCLUDE_DIR)/kernel.mk
 include $(INCLUDE_DIR)/version.mk
+include $(INCLUDE_DIR)/download.mk
 
 override MAKEFLAGS=
 
@@ -19,17 +20,18 @@ STAGING_SUBDIR_HOST := staging_dir/host
 STAGING_SUBDIR_TARGET := staging_dir/$(TARGET_DIR_NAME)
 STAGING_SUBDIR_TOOLCHAIN := staging_dir/toolchain-$(ARCH)$(ARCH_SUFFIX)_gcc-$(GCCV)_$(LIBC)$(if $(CONFIG_arm),_eabi)
 
-EXCLUDE_DIRS:=*/ccache/* \
+BUNDLER_PATH := $(subst $(space),:,$(filter-out $(TOPDIR)/%,$(subst :,$(space),$(PATH))))
+BUNDLER_COMMAND := PATH=$(BUNDLER_PATH) $(XARGS) $(SCRIPT_DIR)/bundle-libraries.sh $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_HOST)
+
+EXCLUDE_DIRS:= \
        */stamp \
        */stampfiles \
        */man \
        */info \
        */root-* \
-       initial \
        *.install.clean \
        *.install.flags \
        *.install \
-       */doc \
        */share/locale
 
 SDK_DIRS = \
@@ -42,12 +44,18 @@ GIT_COMMIT:=$(shell git rev-parse HEAD 2>/dev/null)
 GIT_BRANCH:=$(filter-out master HEAD,$(shell git rev-parse --abbrev-ref HEAD 2>/dev/null))
 GIT_TAGNAME:=$(shell git show-ref --tags --dereference 2>/dev/null | sed -ne '/^$(GIT_COMMIT) / { s|^.*/||; s|\^.*||; p }')
 
-BASE_FEED:=$(if $(GIT_URL),src-git base $(GIT_URL)$(if $(GIT_BRANCH),;$(GIT_BRANCH),$(if $(GIT_TAGNAME),;$(GIT_TAGNAME))))
+BASE_FEED:=$(if $(GIT_URL),src-git-full base $(GIT_URL)$(if $(GIT_BRANCH),;$(GIT_BRANCH),$(if $(GIT_TAGNAME),;$(GIT_TAGNAME))))
 BASE_FEED:=$(if $(BASE_FEED),$(BASE_FEED),$(shell cd $(TOPDIR); LC_ALL=C git svn info 2>/dev/null | sed -ne 's/^URL: /src-gitsvn base /p'))
 BASE_FEED:=$(if $(BASE_FEED),$(BASE_FEED),$(shell cd $(TOPDIR); LC_ALL=C svn info 2>/dev/null | sed -ne 's/^URL: /src-svn base /p'))
-BASE_FEED:=$(if $(BASE_FEED),$(BASE_FEED),src-git base https://git.lede-project.org/source.git$(if $(GIT_BRANCH),;$(GIT_BRANCH),$(if $(GIT_TAGNAME),;$(GIT_TAGNAME))))
+BASE_FEED:=$(if $(BASE_FEED),$(BASE_FEED),src-git-full base $(PROJECT_GIT)/openwrt/openwrt.git$(if $(GIT_BRANCH),;$(GIT_BRANCH),$(if $(GIT_TAGNAME),;$(GIT_TAGNAME))))
 
 KDIR_BASE = $(patsubst $(TOPDIR)/%,%,$(LINUX_DIR))
+KDIR_ARCHES = $(LINUX_KARCH)
+
+# arch/arm64/ includes reference files in arch/arm/, so we'll need both
+ifeq ($(LINUX_KARCH),arm64)
+  KDIR_ARCHES += arm
+endif
 
 KERNEL_FILES_ARCH = \
        Makefile* \
@@ -57,35 +65,65 @@ KERNEL_FILES_ARCH = \
        include \
        */include \
        scripts \
-       kernel/asm-offsets.s
+       kernel/asm-offsets.s \
+       kernel/module.lds
+
+ifeq ($(LINUX_KARCH),powerpc)
+  KERNEL_FILES_ARCH += lib/crtsavres.o
+endif
 
 KERNEL_FILES_BASE := \
        .config \
        Makefile \
        scripts \
+       tools/objtool \
        include \
        Module.symvers \
        modules.builtin \
-       $(addprefix arch/$(LINUX_KARCH)/,$(KERNEL_FILES_ARCH))
+       $(foreach arch,$(KDIR_ARCHES),$(addprefix arch/$(arch)/,$(KERNEL_FILES_ARCH)))
 
 KERNEL_FILES := $(patsubst $(TOPDIR)/%,%,$(wildcard $(addprefix $(LINUX_DIR)/,$(KERNEL_FILES_BASE))))
 
+# The kernel source hosts various userspace utilities sources.
+# These are packaged separately from the kernel and kernel modules.
+# The source has to be included here to be buildable by the SDK.
+#
+USERSPACE_UTILS_FILES := \
+       tools/build \
+       tools/leds \
+       tools/power/cpupower \
+       tools/scripts \
+       tools/spi \
+       tools/thermal/tmon \
+       tools/usb/usbip
+
+USERSPACE_FILES := $(patsubst $(TOPDIR)/%,%,$(wildcard $(addprefix $(LINUX_DIR)/,$(USERSPACE_UTILS_FILES))))
+
 all: compile
 
-$(BIN_DIR)/$(SDK_NAME).tar.xz: clean
-       mkdir -p $(SDK_BUILD_DIR)/dl $(SDK_BUILD_DIR)/package
+$(BIN_DIR)/$(SDK_NAME).tar.zst: clean
+       mkdir -p \
+               $(SDK_BUILD_DIR)/dl \
+               $(SDK_BUILD_DIR)/package \
+               $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_TARGET)/include \
+               $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_TARGET)/usr/include
+
        $(CP) -L $(INCLUDE_DIR) $(SCRIPT_DIR) $(SDK_BUILD_DIR)/
        $(TAR) -cf - -C $(TOPDIR) \
-               `cd $(TOPDIR); find $(KDIR_BASE) -name \*.ko` \
+               `cd $(TOPDIR); find $(KDIR_BASE)/ -name \*.ko` \
                `cd $(TOPDIR); find $(KDIR_BASE)/firmware/ -newer $(KDIR_BASE)/firmware/Makefile \
                        -type f -name '*.bin' -or -name '*.cis' -or -name '*.csp' -or -name '*.dsp' -or -name '*.fw'` \
                $(foreach exclude,$(EXCLUDE_DIRS),--exclude="$(exclude)") \
                $(SDK_DIRS) $(KERNEL_FILES) | \
                $(TAR) -xf - -C $(SDK_BUILD_DIR)
 
+       $(TAR) -cf - -C $(TOPDIR) $(USERSPACE_FILES) | \
+               $(TAR) -xf - -C $(SDK_BUILD_DIR)
+
        (cd $(SDK_BUILD_DIR); find $(STAGING_SUBDIR_HOST)/bin $(STAGING_SUBDIR_HOST)/usr/bin \
                $(STAGING_SUBDIR_TOOLCHAIN)/bin $(STAGING_SUBDIR_TOOLCHAIN)/*/bin $(STAGING_SUBDIR_TOOLCHAIN)/libexec \
-               -type f | $(XARGS) $(SCRIPT_DIR)/bundle-libraries.sh $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_HOST))
+               $(KDIR_BASE) \
+               -type f | $(BUNDLER_COMMAND))
 
        @-( \
                find \
@@ -108,23 +146,26 @@ $(BIN_DIR)/$(SDK_NAME).tar.xz: clean
                $(SDK_BUILD_DIR)/target/linux/*/patches*
        ./convert-config.pl $(TOPDIR)/.config > $(SDK_BUILD_DIR)/Config-build.in
        $(CP) -L \
-               $(TOPDIR)/LICENSE \
+               $(TOPDIR)/LICENSES \
+               $(TOPDIR)/COPYING \
                $(TOPDIR)/rules.mk \
                ./files/Config.in \
                ./files/Makefile \
                ./files/include/prepare.mk \
-               ./files/README.SDK \
+               ./files/README.md \
                $(SDK_BUILD_DIR)/
+       mkdir -p $(SDK_BUILD_DIR)/package/kernel
        $(CP) \
                $(TOPDIR)/package/Makefile \
                $(TOPDIR)/package/libs/toolchain \
-               $(TOPDIR)/package/kernel/linux \
                $(SDK_BUILD_DIR)/package/
+       $(CP) \
+               $(TOPDIR)/package/kernel/linux \
+               $(SDK_BUILD_DIR)/package/kernel/
 
-       -rm -rf \
-               $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_HOST)/ccache \
-               $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_TARGET)/ccache \
-               $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_TOOLCHAIN)/ccache
+       -rm -rf $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_HOST)/.prereq-build
+       -rm -rf $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_HOST)/doc
+       -rm -rf $(SDK_BUILD_DIR)/$(STAGING_SUBDIR_HOST)/share/doc
 
        -rm -f $(SDK_BUILD_DIR)/feeds.conf.default
        $(if $(BASE_FEED),echo "$(BASE_FEED)" > $(SDK_BUILD_DIR)/feeds.conf.default)
@@ -140,14 +181,16 @@ $(BIN_DIR)/$(SDK_NAME).tar.xz: clean
        find $(SDK_BUILD_DIR) -name .svn | $(XARGS) rm -rf
        find $(SDK_BUILD_DIR) -name CVS | $(XARGS) rm -rf
        -make -C $(SDK_BUILD_DIR)/scripts/config clean
+
        (cd $(BUILD_DIR); \
-               tar -I 'xz -7e' -cf $@ $(SDK_NAME); \
+               tar -I '$(STAGING_DIR_HOST)/bin/zstd -T0 --ultra -20' -cf $@ $(SDK_NAME) \
+               --mtime="$(shell date --date=@$(SOURCE_DATE_EPOCH))"; \
        )
 
 download:
 prepare:
-compile: $(BIN_DIR)/$(SDK_NAME).tar.xz
+compile: $(BIN_DIR)/$(SDK_NAME).tar.zst
 install: compile
 
 clean:
-       rm -rf $(SDK_BUILD_DIR) $(BIN_DIR)/$(SDK_NAME).tar.xz
+       rm -rf $(SDK_BUILD_DIR) $(BIN_DIR)/$(SDK_NAME).tar.zst