From 07449f692ce4c4525e946401f4c3ed0cbbc8c4df Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Thu, 12 Mar 2020 12:55:41 -1000 Subject: [PATCH 1/4] build: refactor JSON info files to `profiles.json` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JSON info files contain machine readable information of built profiles and resulting images. These files were added in commit 881ed09ee6e2 ("build: create JSON files containing image info"). They are useful for firmware wizards and script checking for reproducibility. Currently all JSON files are stored next to the built images, resulting in up to 168 individual files for the ath79/generic target. This patch refactors the JSON creation to store individual per image (not per profile) files in $(BUILD_DIR)/json_info_files and create an single overview file called `profiles.json` in the target directory. Storing per image files and not per profile solves the problem of parallel file writes. If a profiles sysupgrade and factory image are finished at the same time both processes would write to the same JSON file, resulting in randomly broken outputs. Some target like x86/64 do not use the image code yet, resulting in missing JSON files. If no JSON info files were created, no `profiles.json` files is created as it would be empty anyway. As before, this creation is enabled by default only if `BUILDBOT` is set. Tested via buildroot & ImageBuilder on ath79/generic, imx6 and x86/64. Signed-off-by: Paul Spooren [json_info_files dir handling in Make, if case refactoring] Signed-off-by: Petr Štetiar --- Makefile | 9 ++++ config/Config-build.in | 9 ++-- include/image.mk | 61 +++++++++++++------------ scripts/json_add_image_info.py | 70 ++++++++++++++++------------- scripts/json_overview_image_info.py | 39 ++++++++++++++++ target/imagebuilder/files/Makefile | 10 +++++ 6 files changed, 134 insertions(+), 64 deletions(-) create mode 100755 scripts/json_overview_image_info.py diff --git a/Makefile b/Makefile index fd705a37ad..32c050bb48 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,14 @@ prereq: $(target/stamp-prereq) tmp/.prereq_packages exit 1; \ fi +$(BIN_DIR)/profiles.json: FORCE + $(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \ + WORK_DIR=$(BUILD_DIR)/json_info_files \ + $(SCRIPT_DIR)/json_overview_image_info.py $@ \ + ) + +json_overview_image_info: $(BIN_DIR)/profiles.json + checksum: FORCE $(call sha256sums,$(BIN_DIR),$(CONFIG_BUILDBOT)) @@ -109,6 +117,7 @@ prepare: .config $(tools/stamp-compile) $(toolchain/stamp-compile) world: prepare $(target/stamp-compile) $(package/stamp-compile) $(package/stamp-install) $(target/stamp-install) FORCE $(_SINGLE)$(SUBMAKE) -r package/index + $(_SINGLE)$(SUBMAKE) -r json_overview_image_info $(_SINGLE)$(SUBMAKE) -r checksum .PHONY: clean dirclean prereq prepare world package/symlinks package/symlinks-install package/symlinks-clean diff --git a/config/Config-build.in b/config/Config-build.in index 6a6fb2882c..61a9265ad7 100644 --- a/config/Config-build.in +++ b/config/Config-build.in @@ -7,12 +7,13 @@ menu "Global build settings" - config JSON_ADD_IMAGE_INFO - bool "Create JSON info files per build image" + config JSON_OVERVIEW_IMAGE_INFO + bool "Create JSON info file overview per target" default BUILDBOT help - The JSON info files contain information about the device and - build images, stored next to the firmware images. + Create a JSON info file called profiles.json in the target + directory containing machine readable list of built profiles + and resulting images. config ALL_NONSHARED bool "Select all target specific packages by default" diff --git a/include/image.mk b/include/image.mk index 6204e8ab61..441f55b741 100644 --- a/include/image.mk +++ b/include/image.mk @@ -545,8 +545,11 @@ endef define Device/Build/image GZ_SUFFIX := $(if $(filter %dtb %gz,$(2)),,$(if $(and $(findstring ext4,$(1)),$(CONFIG_TARGET_IMAGES_GZIP)),.gz)) - $$(_TARGET): $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX) + $$(_TARGET): $(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \ + $(BUILD_DIR)/json_info_files/$(call IMAGE_NAME,$(1),$(2)).json, \ + $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX)) $(eval $(call Device/Export,$(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2)),$(1))) + ROOTFS/$(1)/$(3) := \ $(KDIR)/root.$(1)$$(strip \ $$(if $$(FS_OPTIONS/$(1)),+fs=$$(call param_mangle,$$(FS_OPTIONS/$(1)))) \ @@ -568,32 +571,33 @@ define Device/Build/image $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2)): $(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2)) cp $$^ $$@ - $(if $(CONFIG_JSON_ADD_IMAGE_INFO), \ - DEVICE_ID="$(DEVICE_NAME)" \ - BIN_DIR="$(BIN_DIR)" \ - IMAGE_NAME="$(IMAGE_NAME)" \ - IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \ - IMAGE_PREFIX="$(IMAGE_PREFIX)" \ - DEVICE_VENDOR="$(DEVICE_VENDOR)" \ - DEVICE_MODEL="$(DEVICE_MODEL)" \ - DEVICE_VARIANT="$(DEVICE_VARIANT)" \ - DEVICE_ALT0_VENDOR="$(DEVICE_ALT0_VENDOR)" \ - DEVICE_ALT0_MODEL="$(DEVICE_ALT0_MODEL)" \ - DEVICE_ALT0_VARIANT="$(DEVICE_ALT0_VARIANT)" \ - DEVICE_ALT1_VENDOR="$(DEVICE_ALT1_VENDOR)" \ - DEVICE_ALT1_MODEL="$(DEVICE_ALT1_MODEL)" \ - DEVICE_ALT1_VARIANT="$(DEVICE_ALT1_VARIANT)" \ - DEVICE_ALT2_VENDOR="$(DEVICE_ALT2_VENDOR)" \ - DEVICE_ALT2_MODEL="$(DEVICE_ALT2_MODEL)" \ - DEVICE_ALT2_VARIANT="$(DEVICE_ALT2_VARIANT)" \ - DEVICE_TITLE="$(DEVICE_TITLE)" \ - TARGET="$(BOARD)" \ - SUBTARGET="$(if $(SUBTARGET),$(SUBTARGET),generic)" \ - VERSION_NUMBER="$(VERSION_NUMBER)" \ - VERSION_CODE="$(VERSION_CODE)" \ - SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \ - $(TOPDIR)/scripts/json_add_image_info.py \ - ) + + $(BUILD_DIR)/json_info_files/$(call IMAGE_NAME,$(1),$(2)).json: $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX) + @mkdir -p $$(shell dirname $$@) + DEVICE_ID="$(DEVICE_NAME)" \ + BIN_DIR="$(BIN_DIR)" \ + IMAGE_NAME="$(IMAGE_NAME)" \ + IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \ + IMAGE_PREFIX="$(IMAGE_PREFIX)" \ + DEVICE_VENDOR="$(DEVICE_VENDOR)" \ + DEVICE_MODEL="$(DEVICE_MODEL)" \ + DEVICE_VARIANT="$(DEVICE_VARIANT)" \ + DEVICE_ALT0_VENDOR="$(DEVICE_ALT0_VENDOR)" \ + DEVICE_ALT0_MODEL="$(DEVICE_ALT0_MODEL)" \ + DEVICE_ALT0_VARIANT="$(DEVICE_ALT0_VARIANT)" \ + DEVICE_ALT1_VENDOR="$(DEVICE_ALT1_VENDOR)" \ + DEVICE_ALT1_MODEL="$(DEVICE_ALT1_MODEL)" \ + DEVICE_ALT1_VARIANT="$(DEVICE_ALT1_VARIANT)" \ + DEVICE_ALT2_VENDOR="$(DEVICE_ALT2_VENDOR)" \ + DEVICE_ALT2_MODEL="$(DEVICE_ALT2_MODEL)" \ + DEVICE_ALT2_VARIANT="$(DEVICE_ALT2_VARIANT)" \ + DEVICE_TITLE="$(DEVICE_TITLE)" \ + TARGET="$(BOARD)" \ + SUBTARGET="$(if $(SUBTARGET),$(SUBTARGET),generic)" \ + VERSION_NUMBER="$(VERSION_NUMBER)" \ + VERSION_CODE="$(VERSION_CODE)" \ + SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \ + $(TOPDIR)/scripts/json_add_image_info.py $$@ endef @@ -612,8 +616,6 @@ define Device/Build/artifact endef define Device/Build - $(shell rm -f $(BIN_DIR)/$(IMG_PREFIX)-$(1).json) - $(if $(CONFIG_TARGET_ROOTFS_INITRAMFS),$(call Device/Build/initramfs,$(1))) $(call Device/Build/kernel,$(1)) @@ -699,6 +701,7 @@ define BuildImage image_prepare: compile mkdir -p $(BIN_DIR) $(KDIR)/tmp + rm -rf $(BUILD_DIR)/json_info_files $(call Image/Prepare) legacy-images-prepare-make: image_prepare diff --git a/scripts/json_add_image_info.py b/scripts/json_add_image_info.py index 44b4031f85..b6628113ac 100755 --- a/scripts/json_add_image_info.py +++ b/scripts/json_add_image_info.py @@ -1,18 +1,22 @@ #!/usr/bin/env python3 -import json -import os +from os import getenv +from pathlib import Path +from sys import argv import hashlib +import json +if len(argv) != 2: + print("ERROR: JSON info script requires output arg") + exit(1) -def e(variable, default=None): - return os.environ.get(variable, default) +json_path = Path(argv[1]) +bin_dir = Path(getenv("BIN_DIR")) +image_file = bin_dir / getenv("IMAGE_NAME") - -json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX")) - -with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file: - image_hash = hashlib.sha256(image_file.read()).hexdigest() +if not image_file.is_file(): + print("Skip JSON creation for non existing image ", image_file) + exit(0) def get_titles(): @@ -20,36 +24,40 @@ def get_titles(): for prefix in ["", "ALT0_", "ALT1_", "ALT2_"]: title = {} for var in ["vendor", "model", "variant"]: - if e("DEVICE_{}{}".format(prefix, var.upper())): - title[var] = e("DEVICE_{}{}".format(prefix, var.upper())) + if getenv("DEVICE_{}{}".format(prefix, var.upper())): + title[var] = getenv("DEVICE_{}{}".format(prefix, var.upper())) if title: titles.append(title) if not titles: - titles.append({"title": e("DEVICE_TITLE")}) + titles.append({"title": getenv("DEVICE_TITLE")}) return titles -if not os.path.exists(json_path): - device_info = { - "id": e("DEVICE_ID"), - "image_prefix": e("IMAGE_PREFIX"), - "images": [], - "metadata_version": 1, - "supported_devices": e("SUPPORTED_DEVICES").split(), - "target": "{}/{}".format(e("TARGET"), e("SUBTARGET", "generic")), - "titles": get_titles(), - "version_commit": e("VERSION_CODE"), - "version_number": e("VERSION_NUMBER"), - } -else: - with open(json_path, "r") as json_file: - device_info = json.load(json_file) +device_id = getenv("DEVICE_ID") +image_hash = hashlib.sha256(image_file.read_bytes()).hexdigest() -image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash} -device_info["images"].append(image_info) +image_info = { + "metadata_version": 1, + "target": "{}/{}".format(getenv("TARGET"), getenv("SUBTARGET")), + "version_code": getenv("VERSION_CODE"), + "version_number": getenv("VERSION_NUMBER"), + "profiles": { + device_id: { + "image_prefix": getenv("IMAGE_PREFIX"), + "images": [ + { + "type": getenv("IMAGE_TYPE"), + "name": getenv("IMAGE_NAME"), + "sha256": image_hash, + } + ], + "supported_devices": getenv("SUPPORTED_DEVICES").split(), + "titles": get_titles(), + } + }, +} -with open(json_path, "w") as json_file: - json.dump(device_info, json_file, sort_keys=True, indent=" ") +json_path.write_text(json.dumps(image_info, separators=(",", ":"))) diff --git a/scripts/json_overview_image_info.py b/scripts/json_overview_image_info.py new file mode 100755 index 0000000000..5ed829249b --- /dev/null +++ b/scripts/json_overview_image_info.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import json +from pathlib import Path +from os import getenv +from sys import argv + +if len(argv) != 2: + print("JSON info files script requires ouput file as argument") + exit(1) + +output_path = Path(argv[1]) + +assert getenv("WORK_DIR"), "$WORK_DIR required" + +work_dir = Path(getenv("WORK_DIR")) + +assert work_dir.is_dir(), "$WORK_DIR not a directory" + +output = {} + +for json_file in work_dir.glob("*.json"): + image_info = json.loads(json_file.read_text()) + if not output: + output.update(image_info) + else: + # get first (and only) profile in json file + device_id = next(iter(image_info["profiles"].keys())) + if device_id not in output["profiles"]: + output["profiles"].update(image_info["profiles"]) + else: + output["profiles"][device_id]["images"].append( + image_info["profiles"][device_id]["images"][0] + ) + +if output: + output_path.write_text(json.dumps(output, sort_keys=True, separators=(",", ":"))) +else: + print("JSON info file script could not find any JSON files for target") diff --git a/target/imagebuilder/files/Makefile b/target/imagebuilder/files/Makefile index 15b3d5c35c..835dd98ca5 100644 --- a/target/imagebuilder/files/Makefile +++ b/target/imagebuilder/files/Makefile @@ -118,6 +118,7 @@ _call_image: staging_dir/host/.prereq-build $(MAKE) package_install $(MAKE) -s prepare_rootfs $(MAKE) -s build_image + $(MAKE) -s json_overview_image_info $(MAKE) -s checksum _call_manifest: FORCE @@ -163,12 +164,21 @@ prepare_rootfs: FORCE $(CP) $(TARGET_DIR) $(TARGET_DIR_ORIG) $(call prepare_rootfs,$(TARGET_DIR),$(USER_FILES),$(DISABLED_SERVICES)) + build_image: FORCE @echo @echo Building images... $(NO_TRACE_MAKE) -C target/linux/$(BOARD)/image install TARGET_BUILD=1 IB=1 EXTRA_IMAGE_NAME="$(EXTRA_IMAGE_NAME)" \ $(if $(USER_PROFILE),PROFILE="$(USER_PROFILE)") +$(BIN_DIR)/profiles.json: FORCE + $(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \ + WORK_DIR=$(BUILD_DIR)/json_info_files \ + $(SCRIPT_DIR)/json_overview_image_info.py $@ \ + ) + +json_overview_image_info: $(BIN_DIR)/profiles.json + checksum: FORCE @echo @echo Calculating checksums... From 4298339b238ae9be0692e800042cc25a61f87fad Mon Sep 17 00:00:00 2001 From: Tim Harvey Date: Wed, 1 Apr 2020 10:24:12 -0700 Subject: [PATCH 2/4] octeontx: switch to kernel 5.4 5.4 is stable on Gateworks Newport GW610x/GW620x/GW630x/GW640x Signed-off-by: Tim Harvey --- target/linux/octeontx/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target/linux/octeontx/Makefile b/target/linux/octeontx/Makefile index 19a264d986..67addbd424 100644 --- a/target/linux/octeontx/Makefile +++ b/target/linux/octeontx/Makefile @@ -11,8 +11,7 @@ BOARD:=octeontx BOARDNAME:=Octeon-TX FEATURES:=targz pcie gpio rtc usb fpu -KERNEL_PATCHVER:=4.14 -KERNEL_TESTING_PATCHVER:=5.4 +KERNEL_PATCHVER:=5.4 define Target/Description Build images for Octeon-TX CN80XX/CN81XX based boards From ff6b092121ba96745553584f5a5b0897805edfc6 Mon Sep 17 00:00:00 2001 From: Tim Harvey Date: Wed, 1 Apr 2020 10:23:12 -0700 Subject: [PATCH 3/4] imx6: bootscript: use partition UUID for rootfs if possible Specifying root filesystem by device is non-deterministic for several reasons: - USB device unmeration order is not garunteeed for USB storage devs - MMC devs ordering is determined by the instance of the MMC host controller including non-storage SDIO devices which can throw off numbering depending on kernel versions. It is recommended to use partition UUID Signed-off-by: Tim Harvey --- target/linux/imx6/image/bootscript-ventana | 34 +++++++++++++--------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/target/linux/imx6/image/bootscript-ventana b/target/linux/imx6/image/bootscript-ventana index 941afb519a..734f74a30c 100644 --- a/target/linux/imx6/image/bootscript-ventana +++ b/target/linux/imx6/image/bootscript-ventana @@ -1,4 +1,4 @@ -echo "Gateworks Ventana OpenWrt Boot script v1.01" +echo "Gateworks Ventana OpenWrt Boot script v1.02" # set some defaults # set some defaults @@ -9,7 +9,7 @@ setenv bootargs console=${console},${baudrate} setenv loadaddr 10800000 setenv fdt_addr 18000000 -# detect dtype and bootdev by looking for kernel on media the bootloader +# detect dtype by looking for kernel on media the bootloader # has mounted (in order of preference: usb/mmc/sata) # # This assumes the bootloader has already started the respective subsystem @@ -31,18 +31,9 @@ else fi echo "detected dtype:$dtype" fi -if test -n "$bootdev" ; then - echo "Using bootdev from env: $bootdev" -else - if itest.s "x${dtype}" == "xmmc" ; then - bootdev=mmcblk0p1 - else - bootdev=sda1 - fi -fi +echo "Booting from ${dtype}..." if itest.s "x${dtype}" == "xnand" ; then - echo "Booting from NAND..." # fix partition name # OpenWrt kernel bug prevents partition name of 'rootfs' from booting # instead name the partition ubi which is what is looked for by @@ -52,9 +43,24 @@ if itest.s "x${dtype}" == "xnand" ; then setenv fsload ubifsload setenv root "ubi0:ubi ubi.mtd=2 rootfstype=squashfs,ubifs" else - echo "Booting from block device ${bootdev}..." setenv fsload "${fs}load ${dtype} ${disk}:1" - setenv root "root=/dev/${bootdev} rootfstype=${fs} rootwait rw" + part uuid ${dtype} ${disk}:1 uuid + if test -z "${uuid}"; then + # fallback to bootdev + if test -n "$bootdev" ; then + echo "Using bootdev from env: $bootdev" + else + if itest.s "x${dtype}" == "xmmc" ; then + bootdev=mmcblk0p1 + else + bootdev=sda1 + fi + fi + setenv root "root=/dev/${bootdev}" + else + setenv root "root=PARTUUID=${uuid}" + fi + setenv root "$root rootfstype=${fs} rootwait rw" fi setenv bootargs "${bootargs}" "${root}" "${video}" "${extra}" From 3ec70052c5d82ffbc8d13e597f2833a1c30bec54 Mon Sep 17 00:00:00 2001 From: Tan Zien Date: Mon, 6 Jan 2020 05:16:43 +0800 Subject: [PATCH 4/4] kernel: add module for Mellanox mlx Network Driver add module to support Mellanox Connect-X card mlx4 supports ConnectX-3 series and previous cards mlx5 supports Connect-IB/ConnectX-4 series and later cards Signed-off-by: Tan Zien --- package/kernel/linux/modules/netdevices.mk | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/package/kernel/linux/modules/netdevices.mk b/package/kernel/linux/modules/netdevices.mk index 944fdaec5b..34503f1e83 100644 --- a/package/kernel/linux/modules/netdevices.mk +++ b/package/kernel/linux/modules/netdevices.mk @@ -1090,3 +1090,38 @@ define KernelPackage/be2net/description endef $(eval $(call KernelPackage,be2net)) + +define KernelPackage/mlx4-core + SUBMENU:=$(NETWORK_DEVICES_MENU) + TITLE:=Mellanox ConnectX(R) mlx4 core Network Driver + DEPENDS:=@PCI_SUPPORT +kmod-ptp + FILES:=$(LINUX_DIR)/drivers/net/ethernet/mellanox/mlx4/mlx4_core.ko + KCONFIG:= CONFIG_MLX4_EN \ + CONFIG_MLX4_CORE=y \ + CONFIG_MLX4_CORE_GEN2=y \ + CONFIG_MLX4_DEBUG=n + AUTOLOAD:=$(call AutoProbe,mlx4_core) +endef + +define KernelPackage/mlx4-core/description + Supports Mellanox ConnectX-3 series and previous cards +endef + +$(eval $(call KernelPackage,mlx4-core)) + +define KernelPackage/mlx5-core + SUBMENU:=$(NETWORK_DEVICES_MENU) + TITLE:=Mellanox ConnectX(R) mlx5 core Network Driver + DEPENDS:=@PCI_SUPPORT +kmod-ptp + FILES:=$(LINUX_DIR)/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.ko + KCONFIG:= CONFIG_MLX5_CORE \ + CONFIG_MLX5_CORE_EN=y \ + CONFIG_MLX5_EN_RXNFC=y + AUTOLOAD:=$(call AutoProbe,mlx5_core) +endef + +define KernelPackage/mlx5-core/description + Supports Mellanox Connect-IB/ConnectX-4 series and later cards +endef + +$(eval $(call KernelPackage,mlx5-core))