diff --git a/include/host-build.mk b/include/host-build.mk index 22fcc31f15..2cc1ec5842 100644 --- a/include/host-build.mk +++ b/include/host-build.mk @@ -206,5 +206,9 @@ endif define HostBuild $(HostBuild/Core) - $(if $(if $(PKG_HOST_ONLY),,$(if $(and $(filter host-%,$(MAKECMDGOALS)),$(PKG_SKIP_DOWNLOAD)),,$(STAMP_PREPARED))),,$(if $(strip $(PKG_SOURCE_URL)),$(call Download,default))) + $(if $(if $(PKG_HOST_ONLY),,$(if $(and $(filter host-%,$(MAKECMDGOALS)),$(PKG_SKIP_DOWNLOAD)),,$(STAMP_PREPARED))),, + $(if $(and $(CONFIG_AUTOREMOVE), $(wildcard $(HOST_STAMP_INSTALLED), $(wildcard $(HOST_STAMP_BUILT)))),, + $(if $(strip $(PKG_SOURCE_URL)),$(call Download,default)) + ) + ) endef diff --git a/include/image.mk b/include/image.mk index 87ba60d954..b801ef993c 100644 --- a/include/image.mk +++ b/include/image.mk @@ -333,6 +333,8 @@ define Device/InitProfile DEVICE_ALT0_TITLE = $$(DEVICE_ALT0_VENDOR) $$(DEVICE_ALT0_MODEL)$$(if $$(DEVICE_ALT0_VARIANT), $$(DEVICE_ALT0_VARIANT)) DEVICE_ALT1_TITLE = $$(DEVICE_ALT1_VENDOR) $$(DEVICE_ALT1_MODEL)$$(if $$(DEVICE_ALT1_VARIANT), $$(DEVICE_ALT1_VARIANT)) DEVICE_ALT2_TITLE = $$(DEVICE_ALT2_VENDOR) $$(DEVICE_ALT2_MODEL)$$(if $$(DEVICE_ALT2_VARIANT), $$(DEVICE_ALT2_VARIANT)) + DEVICE_ALT3_TITLE = $$(DEVICE_ALT3_VENDOR) $$(DEVICE_ALT3_MODEL)$$(if $$(DEVICE_ALT3_VARIANT), $$(DEVICE_ALT3_VARIANT)) + DEVICE_ALT4_TITLE = $$(DEVICE_ALT4_VENDOR) $$(DEVICE_ALT4_MODEL)$$(if $$(DEVICE_ALT4_VARIANT), $$(DEVICE_ALT4_VARIANT)) DEVICE_VENDOR := DEVICE_MODEL := DEVICE_VARIANT := @@ -345,6 +347,12 @@ define Device/InitProfile DEVICE_ALT2_VENDOR := DEVICE_ALT2_MODEL := DEVICE_ALT2_VARIANT := + DEVICE_ALT3_VENDOR := + DEVICE_ALT3_MODEL := + DEVICE_ALT3_VARIANT := + DEVICE_ALT4_VENDOR := + DEVICE_ALT4_MODEL := + DEVICE_ALT4_VARIANT := DEVICE_PACKAGES := DEVICE_DESCRIPTION = Build firmware images for $$(DEVICE_TITLE) endef @@ -424,7 +432,9 @@ DEFAULT_DEVICE_VARS := \ DEVICE_VENDOR DEVICE_MODEL DEVICE_VARIANT \ DEVICE_ALT0_VENDOR DEVICE_ALT0_MODEL DEVICE_ALT0_VARIANT \ DEVICE_ALT1_VENDOR DEVICE_ALT1_MODEL DEVICE_ALT1_VARIANT \ - DEVICE_ALT2_VENDOR DEVICE_ALT2_MODEL DEVICE_ALT2_VARIANT + DEVICE_ALT2_VENDOR DEVICE_ALT2_MODEL DEVICE_ALT2_VARIANT \ + DEVICE_ALT3_VENDOR DEVICE_ALT3_MODEL DEVICE_ALT3_VARIANT \ + DEVICE_ALT4_VENDOR DEVICE_ALT4_MODEL DEVICE_ALT4_VARIANT define Device/ExportVar $(1) : $(2):=$$($(2)) @@ -507,6 +517,12 @@ define Device/Build/initramfs DEVICE_ALT2_VENDOR="$$(DEVICE_ALT2_VENDOR)" \ DEVICE_ALT2_MODEL="$$(DEVICE_ALT2_MODEL)" \ DEVICE_ALT2_VARIANT="$$(DEVICE_ALT2_VARIANT)" \ + DEVICE_ALT3_VENDOR="$$(DEVICE_ALT3_VENDOR)" \ + DEVICE_ALT3_MODEL="$$(DEVICE_ALT3_MODEL)" \ + DEVICE_ALT3_VARIANT="$$(DEVICE_ALT3_VARIANT)" \ + DEVICE_ALT4_VENDOR="$$(DEVICE_ALT4_VENDOR)" \ + DEVICE_ALT4_MODEL="$$(DEVICE_ALT4_MODEL)" \ + DEVICE_ALT4_VARIANT="$$(DEVICE_ALT4_VARIANT)" \ DEVICE_TITLE="$$(DEVICE_TITLE)" \ DEVICE_PACKAGES="$$(DEVICE_PACKAGES)" \ TARGET="$(BOARD)" \ @@ -615,6 +631,12 @@ define Device/Build/image DEVICE_ALT2_VENDOR="$(DEVICE_ALT2_VENDOR)" \ DEVICE_ALT2_MODEL="$(DEVICE_ALT2_MODEL)" \ DEVICE_ALT2_VARIANT="$(DEVICE_ALT2_VARIANT)" \ + DEVICE_ALT3_VENDOR="$(DEVICE_ALT3_VENDOR)" \ + DEVICE_ALT3_MODEL="$(DEVICE_ALT3_MODEL)" \ + DEVICE_ALT3_VARIANT="$(DEVICE_ALT3_VARIANT)" \ + DEVICE_ALT4_VENDOR="$(DEVICE_ALT4_VENDOR)" \ + DEVICE_ALT4_MODEL="$(DEVICE_ALT4_MODEL)" \ + DEVICE_ALT4_VARIANT="$(DEVICE_ALT4_VARIANT)" \ DEVICE_TITLE="$(DEVICE_TITLE)" \ DEVICE_PACKAGES="$(DEVICE_PACKAGES)" \ TARGET="$(BOARD)" \ @@ -660,6 +682,12 @@ define Device/Build/artifact DEVICE_ALT2_VENDOR="$(DEVICE_ALT2_VENDOR)" \ DEVICE_ALT2_MODEL="$(DEVICE_ALT2_MODEL)" \ DEVICE_ALT2_VARIANT="$(DEVICE_ALT2_VARIANT)" \ + DEVICE_ALT3_VENDOR="$(DEVICE_ALT3_VENDOR)" \ + DEVICE_ALT3_MODEL="$(DEVICE_ALT3_MODEL)" \ + DEVICE_ALT3_VARIANT="$(DEVICE_ALT3_VARIANT)" \ + DEVICE_ALT4_VENDOR="$(DEVICE_ALT4_VENDOR)" \ + DEVICE_ALT4_MODEL="$(DEVICE_ALT4_MODEL)" \ + DEVICE_ALT4_VARIANT="$(DEVICE_ALT4_VARIANT)" \ DEVICE_TITLE="$(DEVICE_TITLE)" \ DEVICE_PACKAGES="$(DEVICE_PACKAGES)" \ TARGET="$(BOARD)" \ @@ -701,6 +729,8 @@ $(if $(strip $(DEVICE_ALT0_TITLE)),Alternative device titles: - $(DEVICE_ALT0_TITLE)) $(if $(strip $(DEVICE_ALT1_TITLE)),- $(DEVICE_ALT1_TITLE)) $(if $(strip $(DEVICE_ALT2_TITLE)),- $(DEVICE_ALT2_TITLE)) +$(if $(strip $(DEVICE_ALT3_TITLE)),- $(DEVICE_ALT3_TITLE)) +$(if $(strip $(DEVICE_ALT4_TITLE)),- $(DEVICE_ALT4_TITLE)) @@ endef @@ -718,6 +748,14 @@ ifneq ($$(strip $$(DEVICE_ALT2_TITLE)),) DEVICE_DISPLAY = $$(DEVICE_ALT2_TITLE) ($$(DEVICE_TITLE)) $$(info $$(call Device/DumpInfo,$(1))) endif +ifneq ($$(strip $$(DEVICE_ALT3_TITLE)),) +DEVICE_DISPLAY = $$(DEVICE_ALT3_TITLE) ($$(DEVICE_TITLE)) +$$(info $$(call Device/DumpInfo,$(1))) +endif +ifneq ($$(strip $$(DEVICE_ALT4_TITLE)),) +DEVICE_DISPLAY = $$(DEVICE_ALT4_TITLE) ($$(DEVICE_TITLE)) +$$(info $$(call Device/DumpInfo,$(1))) +endif DEVICE_DISPLAY = $$(DEVICE_TITLE) $$(eval $$(if $$(DEVICE_TITLE),$$(info $$(call Device/DumpInfo,$(1))))) endef diff --git a/include/kernel-5.10 b/include/kernel-5.10 index df67985cca..7a91cd12fa 100644 --- a/include/kernel-5.10 +++ b/include/kernel-5.10 @@ -1,2 +1,2 @@ -LINUX_VERSION-5.10 = .163 -LINUX_KERNEL_HASH-5.10.163 = 96e226e2d388abc0600434e0f4f365a8829ef901f4d8e761e7ffe2799dc09b20 +LINUX_VERSION-5.10 = .165 +LINUX_KERNEL_HASH-5.10.165 = 971defc48f19ed0a2a7ffd4b48234619cac28895c985c6d747f5b707ba47af0d diff --git a/package/boot/uboot-envtools/files/ath79 b/package/boot/uboot-envtools/files/ath79 index cd94be66de..71dd104f55 100644 --- a/package/boot/uboot-envtools/files/ath79 +++ b/package/boot/uboot-envtools/files/ath79 @@ -42,7 +42,6 @@ etactica,eg200|\ glinet,gl-ar750s-nor|\ glinet,gl-ar750s-nor-nand|\ librerouter,librerouter-v1|\ -netgear,ex6400|\ netgear,ex7300|\ netgear,ex7300-v2|\ netgear,wndr4300-v2|\ diff --git a/scripts/ext-tools.sh b/scripts/ext-tools.sh index bf56f4d9ed..b58296be10 100755 --- a/scripts/ext-tools.sh +++ b/scripts/ext-tools.sh @@ -5,16 +5,14 @@ HOST_BUILD_DIR=$(pwd)/"build_dir/host" HOST_STAGING_DIR_STAMP=$(pwd)/"staging_dir/host/stamp" refresh_timestamps() { - find "$1" -not -type l -print0 | xargs -0 touch + find -H "$1" -not -type l -print0 | xargs -0 touch } extract_prebuilt_tar() { tar -xf "$1" } -install_prebuilt_tools() { - extract_prebuilt_tar "$TOOLS_TAR" - +refresh_prebuilt_tools() { if [ ! -d "$HOST_BUILD_DIR" ]; then echo "Can't find Host Build Dir "$HOST_BUILD_DIR"" >&2 exit 1 @@ -33,6 +31,14 @@ install_prebuilt_tools() { return 0 } +install_prebuilt_tools() { + extract_prebuilt_tar "$TOOLS_TAR" + + refresh_prebuilt_tools + + return 0 +} + while [ -n "$1" ]; do arg="$1"; shift case "$arg" in @@ -63,6 +69,12 @@ while [ -n "$1" ]; do exit $? ;; + --refresh) + refresh_prebuilt_tools + + exit $? + ;; + -h|--help) me="$(basename "$0")" echo -e "\nUsage:\n" >&2 @@ -81,8 +93,12 @@ while [ -n "$1" ]; do echo -e " $me --tools {tar}" >&2 echo -e " Install the prebuilt tools present in the passed" >&2 echo -e " tar and prepare them." >&2 - echo -e " To correctly use them it's needed to update the." >&2 + echo -e " To correctly use them it's needed to update the" >&2 echo -e " timestamp of each tools to skip recompilation.\n" >&2 + echo -e " $me --refresh" >&2 + echo -e " Refresh timestamps of already extracted prebuilt" >&2 + echo -e " tools to correctly use them and skip" >&2 + echo -e " recompilation.\n" >&2 echo -e " $me --help" >&2 echo -e " Display this help text and exit.\n\n" >&2 exit 1 diff --git a/scripts/json_add_image_info.py b/scripts/json_add_image_info.py index 9aa2a19e45..0c441b9334 100755 --- a/scripts/json_add_image_info.py +++ b/scripts/json_add_image_info.py @@ -21,7 +21,7 @@ if not file_path.is_file(): def get_titles(): titles = [] - for prefix in ["", "ALT0_", "ALT1_", "ALT2_"]: + for prefix in ["", "ALT0_", "ALT1_", "ALT2_", "ALT3_", "ALT4_"]: title = {} for var in ["vendor", "model", "variant"]: if getenv("DEVICE_{}{}".format(prefix, var.upper())): diff --git a/target/linux/apm821xx/patches-5.10/802-usb-xhci-force-msi-renesas-xhci.patch b/target/linux/apm821xx/patches-5.10/802-usb-xhci-force-msi-renesas-xhci.patch index 00ca3fbade..8aee7c0bc4 100644 --- a/target/linux/apm821xx/patches-5.10/802-usb-xhci-force-msi-renesas-xhci.patch +++ b/target/linux/apm821xx/patches-5.10/802-usb-xhci-force-msi-renesas-xhci.patch @@ -13,7 +13,7 @@ produce a noisy warning. --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c -@@ -276,6 +276,7 @@ static void xhci_pci_quirks(struct devic +@@ -279,6 +279,7 @@ static void xhci_pci_quirks(struct devic pdev->device == 0x0015) { xhci->quirks |= XHCI_RESET_ON_RESUME; xhci->quirks |= XHCI_ZERO_64B_REGS; @@ -43,7 +43,7 @@ produce a noisy warning. hcd->msi_enabled = 1; --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h -@@ -1897,6 +1897,7 @@ struct xhci_hcd { +@@ -1902,6 +1902,7 @@ struct xhci_hcd { struct xhci_hub usb2_rhub; struct xhci_hub usb3_rhub; /* support xHCI 1.0 spec USB2 hardware LPM */ diff --git a/target/linux/ath79/dts/qca9558_netgear_ex6400.dts b/target/linux/ath79/dts/qca9558_netgear_ex6400.dts deleted file mode 100644 index 273c872b6d..0000000000 --- a/target/linux/ath79/dts/qca9558_netgear_ex6400.dts +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later OR MIT - -#include "qca9558_netgear_ex7300.dtsi" - -/ { - model = "Netgear EX6400"; - compatible = "netgear,ex6400", "qca,qca9558"; -}; diff --git a/target/linux/ath79/dts/qca9558_netgear_ex7300.dts b/target/linux/ath79/dts/qca9558_netgear_ex7300.dts index 9802210b32..b94ccd30b7 100644 --- a/target/linux/ath79/dts/qca9558_netgear_ex7300.dts +++ b/target/linux/ath79/dts/qca9558_netgear_ex7300.dts @@ -1,8 +1,246 @@ // SPDX-License-Identifier: GPL-2.0-or-later OR MIT -#include "qca9558_netgear_ex7300.dtsi" +#include "qca955x.dtsi" + +#include +#include / { model = "Netgear EX7300"; compatible = "netgear,ex7300", "qca,qca9558"; + + aliases { + led-boot = &led_power_green; + led-failsafe = &led_power_amber; + led-running = &led_power_green; + led-upgrade = &led_power_amber; + label-mac-device = ð0; + }; + + led_spi { + compatible = "spi-gpio"; + #address-cells = <1>; + #size-cells = <0>; + + sck-gpios = <&gpio 18 GPIO_ACTIVE_HIGH>; + mosi-gpios = <&gpio 15 GPIO_ACTIVE_HIGH>; + num-chipselects = <0>; + + led_gpio: led_gpio@0 { + compatible = "nxp,74lvc594"; + reg = <0>; + gpio-controller; + #gpio-cells = <2>; + registers-number = <1>; + spi-max-frequency = <500000>; + + gpio_latch_bit { + gpio-hog; + gpios = <4 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "gpio-latch-bit"; + }; + }; + }; + + leds { + compatible = "gpio-leds"; + + led_power_green: power_green { + label = "green:power"; + gpios = <&gpio 19 GPIO_ACTIVE_LOW>; + }; + + led_power_amber: power_amber { + label = "amber:power"; + gpios = <&gpio 20 GPIO_ACTIVE_LOW>; + }; + + left_blue { + label = "blue:left"; + gpios = <&led_gpio 7 GPIO_ACTIVE_LOW>; + }; + + right_blue { + label = "blue:right"; + gpios = <&led_gpio 6 GPIO_ACTIVE_LOW>; + }; + + wps_green { + label = "green:wps"; + gpios = <&led_gpio 5 GPIO_ACTIVE_LOW>; + }; + + client_red { + label = "red:client"; + gpios = <&led_gpio 3 GPIO_ACTIVE_LOW>; + }; + + client_green { + label = "green:client"; + gpios = <&led_gpio 2 GPIO_ACTIVE_LOW>; + }; + + router_red { + label = "red:router"; + gpios = <&led_gpio 1 GPIO_ACTIVE_LOW>; + }; + + router_green { + label = "green:router"; + gpios = <&led_gpio 0 GPIO_ACTIVE_LOW>; + }; + }; + + keys { + compatible = "gpio-keys"; + + reset { + label = "Reset button"; + linux,code = ; + gpios = <&gpio 11 GPIO_ACTIVE_LOW>; + debounce-interval = <60>; + }; + + wps { + label = "WPS button"; + linux,code = ; + gpios = <&gpio 22 GPIO_ACTIVE_LOW>; + debounce-interval = <60>; + }; + + extender_apmode { + label = "EXTENDER/APMODE switch"; + gpios = <&gpio 23 GPIO_ACTIVE_LOW>; + linux,code = ; + linux,input-type = ; + debounce-interval = <60>; + }; + }; +}; + +&pcie0 { + status = "okay"; + + wifi@0,0 { + compatible = "qcom,ath10k"; + reg = <0 0 0 0 0>; + + nvmem-cells = <&macaddr_caldata_c>, <&precal_caldata_5000>; + nvmem-cell-names = "mac-address", "pre-calibration"; + }; +}; + +&spi { + status = "okay"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <25000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + uboot: partition@0 { + label = "u-boot"; + reg = <0x000000 0x040000>; + read-only; + }; + + partition@40000 { + label = "u-boot-env"; + reg = <0x040000 0x010000>; + }; + + caldata: partition@50000 { + label = "caldata"; + reg = <0x050000 0x010000>; + read-only; + }; + + partition@60000 { + label = "caldata-backup"; + reg = <0x060000 0x010000>; + read-only; + }; + + partition@70000 { + label = "config"; + reg = <0x070000 0x010000>; + }; + + partition@80000 { + label = "pot"; + reg = <0x080000 0x010000>; + }; + + partition@90000 { + label = "firmware"; + reg = <0x090000 0xf30000>; + compatible = "denx,uimage"; + }; + + partition@fc0000 { + label = "language"; + reg = <0xfc0000 0x040000>; + }; + }; + }; +}; + +&wmac { + status = "okay"; + + nvmem-cells = <&macaddr_caldata_6>, <&cal_caldata_1000>; + nvmem-cell-names = "mac-address", "calibration"; +}; + +&mdio0 { + status = "okay"; + + phy4: ethernet-phy@4 { + reg = <4>; + phy-mode = "rgmii"; + }; +}; + +ð0 { + status = "okay"; + + nvmem-cells = <&macaddr_caldata_0>; + nvmem-cell-names = "mac-address"; + + phy-handle = <&phy4>; + phy-mode = "rgmii-rxid"; + + pll-data = <0x86000000 0x80000101 0x80001313>; +}; + +&caldata { + compatible = "nvmem-cells"; + #address-cells = <1>; + #size-cells = <1>; + + macaddr_caldata_0: macaddr@0 { + reg = <0x0 0x6>; + }; + + macaddr_caldata_6: macaddr@6 { + reg = <0x6 0x6>; + }; + + macaddr_caldata_c: macaddr@c { + reg = <0xc 0x6>; + }; + + cal_caldata_1000: cal@1000 { + reg = <0x1000 0x440>; + }; + + precal_caldata_5000: precal@5000 { + reg = <0x5000 0x2f20>; + }; }; diff --git a/target/linux/ath79/dts/qca9558_netgear_ex7300.dtsi b/target/linux/ath79/dts/qca9558_netgear_ex7300.dtsi deleted file mode 100644 index c266c52dad..0000000000 --- a/target/linux/ath79/dts/qca9558_netgear_ex7300.dtsi +++ /dev/null @@ -1,224 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later OR MIT - -#include "qca955x.dtsi" - -#include -#include - -/ { - aliases { - led-boot = &led_power_green; - led-failsafe = &led_power_amber; - led-running = &led_power_green; - led-upgrade = &led_power_amber; - label-mac-device = ð0; - }; - - led_spi { - compatible = "spi-gpio"; - #address-cells = <1>; - #size-cells = <0>; - - sck-gpios = <&gpio 18 GPIO_ACTIVE_HIGH>; - mosi-gpios = <&gpio 15 GPIO_ACTIVE_HIGH>; - num-chipselects = <0>; - - led_gpio: led_gpio@0 { - compatible = "nxp,74lvc594"; - reg = <0>; - gpio-controller; - #gpio-cells = <2>; - registers-number = <1>; - spi-max-frequency = <500000>; - - gpio_latch_bit { - gpio-hog; - gpios = <4 GPIO_ACTIVE_HIGH>; - output-high; - line-name = "gpio-latch-bit"; - }; - }; - }; - - leds { - compatible = "gpio-leds"; - - led_power_green: power_green { - label = "green:power"; - gpios = <&gpio 19 GPIO_ACTIVE_LOW>; - }; - - led_power_amber: power_amber { - label = "amber:power"; - gpios = <&gpio 20 GPIO_ACTIVE_LOW>; - }; - - left_blue { - label = "blue:left"; - gpios = <&led_gpio 7 GPIO_ACTIVE_LOW>; - }; - - right_blue { - label = "blue:right"; - gpios = <&led_gpio 6 GPIO_ACTIVE_LOW>; - }; - - wps_green { - label = "green:wps"; - gpios = <&led_gpio 5 GPIO_ACTIVE_LOW>; - }; - - client_red { - label = "red:client"; - gpios = <&led_gpio 3 GPIO_ACTIVE_LOW>; - }; - - client_green { - label = "green:client"; - gpios = <&led_gpio 2 GPIO_ACTIVE_LOW>; - }; - - router_red { - label = "red:router"; - gpios = <&led_gpio 1 GPIO_ACTIVE_LOW>; - }; - - router_green { - label = "green:router"; - gpios = <&led_gpio 0 GPIO_ACTIVE_LOW>; - }; - }; - - keys { - compatible = "gpio-keys"; - - reset { - label = "Reset button"; - linux,code = ; - gpios = <&gpio 11 GPIO_ACTIVE_LOW>; - debounce-interval = <60>; - }; - - wps { - label = "WPS button"; - linux,code = ; - gpios = <&gpio 22 GPIO_ACTIVE_LOW>; - debounce-interval = <60>; - }; - - extender_apmode { - label = "EXTENDER/APMODE switch"; - gpios = <&gpio 23 GPIO_ACTIVE_LOW>; - linux,code = ; - linux,input-type = ; - debounce-interval = <60>; - }; - }; -}; - -&pcie0 { - status = "okay"; -}; - -&spi { - status = "okay"; - - flash@0 { - compatible = "jedec,spi-nor"; - reg = <0>; - spi-max-frequency = <25000000>; - - partitions { - compatible = "fixed-partitions"; - #address-cells = <1>; - #size-cells = <1>; - - uboot: partition@0 { - label = "u-boot"; - reg = <0x000000 0x040000>; - read-only; - }; - - partition@40000 { - label = "u-boot-env"; - reg = <0x040000 0x010000>; - }; - - caldata: partition@50000 { - label = "caldata"; - reg = <0x050000 0x010000>; - read-only; - }; - - partition@60000 { - label = "caldata-backup"; - reg = <0x060000 0x010000>; - read-only; - }; - - partition@70000 { - label = "config"; - reg = <0x070000 0x010000>; - }; - - partition@80000 { - label = "pot"; - reg = <0x080000 0x010000>; - }; - - partition@90000 { - label = "firmware"; - reg = <0x090000 0xf30000>; - compatible = "denx,uimage"; - }; - - partition@fc0000 { - label = "language"; - reg = <0xfc0000 0x040000>; - }; - }; - }; -}; - -&wmac { - status = "okay"; - - mtd-cal-data = <&caldata 0x1000>; - nvmem-cells = <&macaddr_caldata_6>; - nvmem-cell-names = "mac-address"; -}; - -&mdio0 { - status = "okay"; - - phy4: ethernet-phy@4 { - reg = <4>; - phy-mode = "rgmii"; - }; -}; - -ð0 { - status = "okay"; - - nvmem-cells = <&macaddr_caldata_0>; - nvmem-cell-names = "mac-address"; - - phy-handle = <&phy4>; - phy-mode = "rgmii-rxid"; - - pll-data = <0x86000000 0x80000101 0x80001313>; -}; - -&caldata { - compatible = "nvmem-cells"; - #address-cells = <1>; - #size-cells = <1>; - - macaddr_caldata_0: macaddr@0 { - reg = <0x0 0x6>; - }; - - macaddr_caldata_6: macaddr@6 { - reg = <0x6 0x6>; - }; -}; diff --git a/target/linux/ath79/generic/base-files/etc/board.d/02_network b/target/linux/ath79/generic/base-files/etc/board.d/02_network index 8b0fba7c69..20352095a4 100644 --- a/target/linux/ath79/generic/base-files/etc/board.d/02_network +++ b/target/linux/ath79/generic/base-files/etc/board.d/02_network @@ -50,7 +50,6 @@ ath79_setup_interfaces() glinet,gl-usb150|\ hak5,wifi-pineapple-nano|\ meraki,mr16|\ - netgear,ex6400|\ netgear,ex7300|\ netgear,ex7300-v2|\ netgear,wndap360|\ diff --git a/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/11-ath10k-caldata b/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/11-ath10k-caldata index 74e6738162..b0b91f1c8a 100644 --- a/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/11-ath10k-caldata +++ b/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/11-ath10k-caldata @@ -216,11 +216,6 @@ case "$FIRMWARE" in ln -sf /lib/firmware/ath10k/pre-cal-pci-0000\:00\:00.0.bin \ /lib/firmware/ath10k/QCA9888/hw2.0/board.bin ;; - netgear,ex6400|\ - netgear,ex7300) - caldata_extract "caldata" 0x5000 0x2f20 - ath10k_patch_mac $(mtd_get_mac_binary caldata 0xc) - ;; phicomm,k2t) caldata_extract "art" 0x5000 0x2f20 ath10k_patch_mac $(k2t_get_mac "5g_mac") diff --git a/target/linux/ath79/generic/base-files/lib/preinit/02_sysinfo_fixup b/target/linux/ath79/generic/base-files/lib/preinit/02_sysinfo_fixup new file mode 100644 index 0000000000..e01469f0d9 --- /dev/null +++ b/target/linux/ath79/generic/base-files/lib/preinit/02_sysinfo_fixup @@ -0,0 +1,42 @@ +. /lib/functions.sh + +do_sysinfo_ath79_fixup() { + local model="" + + case $(board_name) in + netgear,ex7300) + local part=$(find_mtd_part caldata) + local board_hw_id=$(dd if=$part bs=1 skip=67 count=10 2>/dev/null) + case "$board_hw_id" in + 5508013406) + model="Netgear EX6400" + ;; + 5508013271) + model="Netgear EX7300" + ;; + esac + ;; + netgear,ex7300-v2) + local part=$(find_mtd_part artmtd) + local antenna_cfg=$(dd if=$part bs=1 skip=59 count=7 2>/dev/null) + local board_hw_id=$(dd if=$part bs=1 skip=67 count=6 2>/dev/null) + case "$antenna_cfg" in + 3X3+3X3) + model="Netgear EX6250" + ;; + 3X3+4X4) + # EX6400 v2, EX6410, EX6420 + model="Netgear ${board_hw_id:-EX6400 v2}" + ;; + 4X4+4X4) + # EX7300 v2, EX7320 + model="Netgear ${board_hw_id:-EX7300 v2}" + ;; + esac + ;; + esac + + [ -n "$model" ] && echo "$model" > /tmp/sysinfo/model +} + +boot_hook_add preinit_main do_sysinfo_ath79_fixup diff --git a/target/linux/ath79/image/generic.mk b/target/linux/ath79/image/generic.mk index 2b758df6d7..59cb910d02 100644 --- a/target/linux/ath79/image/generic.mk +++ b/target/linux/ath79/image/generic.mk @@ -1734,41 +1734,45 @@ define Device/nec_wg800hp endef TARGET_DEVICES += nec_wg800hp -define Device/netgear_ex6400_ex7300 - $(Device/netgear_generic) +define Device/netgear_ex7300 SOC := qca9558 - UIMAGE_MAGIC := 0x27051956 + DEVICE_VENDOR := NETGEAR + DEVICE_MODEL := EX7300 + DEVICE_ALT0_VENDOR := NETGEAR + DEVICE_ALT0_MODEL := EX6400 NETGEAR_BOARD_ID := EX7300series NETGEAR_HW_ID := 29765104+16+0+128 IMAGE_SIZE := 15552k + IMAGES += factory.img IMAGE/default := append-kernel | pad-offset $$$$(BLOCKSIZE) 64 | \ netgear-rootfs | pad-rootfs IMAGE/sysupgrade.bin := $$(IMAGE/default) | check-size | append-metadata IMAGE/factory.img := $$(IMAGE/default) | netgear-dni | check-size DEVICE_PACKAGES := kmod-ath10k-ct ath10k-firmware-qca99x0-ct -endef - -define Device/netgear_ex6400 - $(Device/netgear_ex6400_ex7300) - DEVICE_MODEL := EX6400 -endef -TARGET_DEVICES += netgear_ex6400 - -define Device/netgear_ex7300 - $(Device/netgear_ex6400_ex7300) - DEVICE_MODEL := EX7300 + SUPPORTED_DEVICES += netgear,ex6400 endef TARGET_DEVICES += netgear_ex7300 define Device/netgear_ex7300-v2 - $(Device/netgear_generic) SOC := qcn5502 + DEVICE_VENDOR := NETGEAR DEVICE_MODEL := EX7300 DEVICE_VARIANT := v2 - UIMAGE_MAGIC := 0x27051956 + DEVICE_ALT0_VENDOR := NETGEAR + DEVICE_ALT0_MODEL := EX6250 + DEVICE_ALT1_VENDOR := NETGEAR + DEVICE_ALT1_MODEL := EX6400 + DEVICE_ALT1_VARIANT := v2 + DEVICE_ALT2_VENDOR := NETGEAR + DEVICE_ALT2_MODEL := EX6410 + DEVICE_ALT3_VENDOR := NETGEAR + DEVICE_ALT3_MODEL := EX6420 + DEVICE_ALT4_VENDOR := NETGEAR + DEVICE_ALT4_MODEL := EX7320 NETGEAR_BOARD_ID := EX7300v2series NETGEAR_HW_ID := 29765907+16+0+128 IMAGE_SIZE := 14528k + IMAGES += factory.img IMAGE/default := append-kernel | pad-offset $$$$(BLOCKSIZE) 64 | \ netgear-rootfs | pad-rootfs IMAGE/sysupgrade.bin := $$(IMAGE/default) | check-size | append-metadata diff --git a/target/linux/bcm53xx/patches-5.10/180-usb-xhci-add-support-for-performing-fake-doorbell.patch b/target/linux/bcm53xx/patches-5.10/180-usb-xhci-add-support-for-performing-fake-doorbell.patch index 022b2945ad..6b7897c41b 100644 --- a/target/linux/bcm53xx/patches-5.10/180-usb-xhci-add-support-for-performing-fake-doorbell.patch +++ b/target/linux/bcm53xx/patches-5.10/180-usb-xhci-add-support-for-performing-fake-doorbell.patch @@ -127,7 +127,7 @@ it on BCM4708 family. /* --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h -@@ -1890,6 +1890,7 @@ struct xhci_hcd { +@@ -1895,6 +1895,7 @@ struct xhci_hcd { #define XHCI_EP_CTX_BROKEN_DCS BIT_ULL(42) #define XHCI_SUSPEND_RESUME_CLKS BIT_ULL(43) #define XHCI_RESET_TO_DEFAULT BIT_ULL(44) diff --git a/target/linux/generic/backport-5.10/828-v6.3-of-property-fix-nvmem-cell-cells-parsing.patch b/target/linux/generic/backport-5.10/828-v6.3-of-property-fix-nvmem-cell-cells-parsing.patch new file mode 100644 index 0000000000..848ec3731b --- /dev/null +++ b/target/linux/generic/backport-5.10/828-v6.3-of-property-fix-nvmem-cell-cells-parsing.patch @@ -0,0 +1,44 @@ +From ef26c0349eb5a615dab2272d08d1d5de4ac9cd4c Mon Sep 17 00:00:00 2001 +From: Michael Walle +Date: Wed, 11 Jan 2023 00:30:56 +0100 +Subject: [PATCH] of: property: fix #nvmem-cell-cells parsing + +Commit 67b8497f005f ("of: property: make #.*-cells optional for simple +props") claims to make the cells-name property optional for simple +properties, but changed the code for the wrong property, i.e. for +DEFINE_SUFFIX_PROP(). Fix that. + +Fixes: 67b8497f005f ("of: property: make #.*-cells optional for simple props") +Reported-by: Peng Fan +Signed-off-by: Michael Walle +Acked-by: Rob Herring +Tested-by: Robert Marko +Signed-off-by: Srinivas Kandagatla +--- + drivers/of/property.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +--- a/drivers/of/property.c ++++ b/drivers/of/property.c +@@ -1213,8 +1213,8 @@ static struct device_node *parse_prop_ce + if (strcmp(prop_name, list_name)) + return NULL; + +- if (of_parse_phandle_with_args(np, list_name, cells_name, index, +- &sup_args)) ++ if (__of_parse_phandle_with_args(np, list_name, cells_name, 0, index, ++ &sup_args)) + return NULL; + + return sup_args.np; +@@ -1267,8 +1267,8 @@ static struct device_node *parse_suffix_ + if (strcmp_suffix(prop_name, suffix)) + return NULL; + +- if (__of_parse_phandle_with_args(np, prop_name, cells_name, 0, index, +- &sup_args)) ++ if (of_parse_phandle_with_args(np, prop_name, cells_name, index, ++ &sup_args)) + return NULL; + + return sup_args.np; diff --git a/target/linux/generic/backport-5.15/828-v6.3-of-property-fix-nvmem-cell-cells-parsing.patch b/target/linux/generic/backport-5.15/828-v6.3-of-property-fix-nvmem-cell-cells-parsing.patch new file mode 100644 index 0000000000..f17cc1f4fa --- /dev/null +++ b/target/linux/generic/backport-5.15/828-v6.3-of-property-fix-nvmem-cell-cells-parsing.patch @@ -0,0 +1,44 @@ +From ef26c0349eb5a615dab2272d08d1d5de4ac9cd4c Mon Sep 17 00:00:00 2001 +From: Michael Walle +Date: Wed, 11 Jan 2023 00:30:56 +0100 +Subject: [PATCH] of: property: fix #nvmem-cell-cells parsing + +Commit 67b8497f005f ("of: property: make #.*-cells optional for simple +props") claims to make the cells-name property optional for simple +properties, but changed the code for the wrong property, i.e. for +DEFINE_SUFFIX_PROP(). Fix that. + +Fixes: 67b8497f005f ("of: property: make #.*-cells optional for simple props") +Reported-by: Peng Fan +Signed-off-by: Michael Walle +Acked-by: Rob Herring +Tested-by: Robert Marko +Signed-off-by: Srinivas Kandagatla +--- + drivers/of/property.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +--- a/drivers/of/property.c ++++ b/drivers/of/property.c +@@ -1173,8 +1173,8 @@ static struct device_node *parse_prop_ce + if (strcmp(prop_name, list_name)) + return NULL; + +- if (of_parse_phandle_with_args(np, list_name, cells_name, index, +- &sup_args)) ++ if (__of_parse_phandle_with_args(np, list_name, cells_name, 0, index, ++ &sup_args)) + return NULL; + + return sup_args.np; +@@ -1227,8 +1227,8 @@ static struct device_node *parse_suffix_ + if (strcmp_suffix(prop_name, suffix)) + return NULL; + +- if (__of_parse_phandle_with_args(np, prop_name, cells_name, 0, index, +- &sup_args)) ++ if (of_parse_phandle_with_args(np, prop_name, cells_name, index, ++ &sup_args)) + return NULL; + + return sup_args.np; diff --git a/target/linux/generic/hack-5.10/780-usb-net-MeigLink_modem_support.patch b/target/linux/generic/hack-5.10/780-usb-net-MeigLink_modem_support.patch index 6d499b88fe..e006221a97 100644 --- a/target/linux/generic/hack-5.10/780-usb-net-MeigLink_modem_support.patch +++ b/target/linux/generic/hack-5.10/780-usb-net-MeigLink_modem_support.patch @@ -33,7 +33,7 @@ Submitted-by: Daniel Golle #define QUECTEL_VENDOR_ID 0x2c7c /* These Quectel products use Quectel's vendor ID */ -@@ -1156,6 +1158,11 @@ static const struct usb_device_id option +@@ -1162,6 +1164,11 @@ static const struct usb_device_id option .driver_info = ZLP }, { USB_DEVICE(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_BG96), .driver_info = RSVD(4) }, diff --git a/target/linux/pistachio/Makefile b/target/linux/pistachio/Makefile index cec8614a13..6ccf63142e 100644 --- a/target/linux/pistachio/Makefile +++ b/target/linux/pistachio/Makefile @@ -12,7 +12,7 @@ CPU_TYPE:=24kc CPU_SUBTYPE:=24kf SUBTARGETS:=generic -KERNEL_PATCHVER:=5.10 +KERNEL_PATCHVER:=5.15 include $(INCLUDE_DIR)/target.mk diff --git a/target/linux/pistachio/config-5.15 b/target/linux/pistachio/config-5.15 new file mode 100644 index 0000000000..c303e99987 --- /dev/null +++ b/target/linux/pistachio/config-5.15 @@ -0,0 +1,344 @@ +CONFIG_ARCH_32BIT_OFF_T=y +CONFIG_ARCH_HIBERNATION_POSSIBLE=y +CONFIG_ARCH_KEEP_MEMBLOCK=y +CONFIG_ARCH_MMAP_RND_BITS_MAX=15 +CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX=15 +CONFIG_ARCH_SUSPEND_POSSIBLE=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_SD=y +# CONFIG_BOARD_INGENIC is not set +CONFIG_BOARD_SCACHE=y +CONFIG_BUILTIN_DTB=y +CONFIG_CEVT_R4K=y +CONFIG_CLKSRC_MIPS_GIC=y +CONFIG_CLKSRC_PISTACHIO=y +CONFIG_CLOCKSOURCE_WATCHDOG=y +CONFIG_CLONE_BACKWARDS=y +CONFIG_COMMON_CLK=y +CONFIG_COMMON_CLK_PISTACHIO=y +CONFIG_COMPAT_32BIT_TIME=y +CONFIG_CONNECTOR=y +CONFIG_CPU_GENERIC_DUMP_TLB=y +CONFIG_CPU_HAS_DIEI=y +CONFIG_CPU_HAS_PREFETCH=y +CONFIG_CPU_HAS_RIXI=y +# CONFIG_CPU_HAS_SMARTMIPS is not set +CONFIG_CPU_HAS_SYNC=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_CPU_IDLE_GOV_MENU=y +CONFIG_CPU_LITTLE_ENDIAN=y +# CONFIG_CPU_MICROMIPS is not set +CONFIG_CPU_MIPS32=y +# CONFIG_CPU_MIPS32_R1 is not set +CONFIG_CPU_MIPS32_R2=y +# CONFIG_CPU_MIPS32_R6 is not set +# CONFIG_CPU_MIPS64_R1 is not set +# CONFIG_CPU_MIPS64_R2 is not set +# CONFIG_CPU_MIPS64_R6 is not set +CONFIG_CPU_MIPSR2=y +CONFIG_CPU_MIPSR2_IRQ_EI=y +CONFIG_CPU_MIPSR2_IRQ_VI=y +CONFIG_CPU_NEEDS_NO_SMARTMIPS_OR_MICROMIPS=y +CONFIG_CPU_PM=y +CONFIG_CPU_R4K_CACHE_TLB=y +CONFIG_CPU_RMAP=y +CONFIG_CPU_SUPPORTS_32BIT_KERNEL=y +CONFIG_CPU_SUPPORTS_HIGHMEM=y +CONFIG_CPU_SUPPORTS_MSA=y +CONFIG_CRC16=y +CONFIG_CRC_CCITT=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_HASH_INFO=y +CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC=y +CONFIG_CRYPTO_LIB_POLY1305_RSIZE=2 +CONFIG_CRYPTO_LIB_SHA256=y +CONFIG_CRYPTO_LZO=y +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_RNG=y +CONFIG_CRYPTO_RNG2=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_ZSTD=y +CONFIG_CSRC_R4K=y +CONFIG_DMADEVICES=y +CONFIG_DMA_ENGINE=y +CONFIG_DMA_NONCOHERENT=y +CONFIG_DMA_OF=y +CONFIG_DMA_VIRTUAL_CHANNELS=y +CONFIG_DTC=y +CONFIG_DWMAC_GENERIC=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +# CONFIG_FIT_IMAGE_FDT_BOSTON is not set +# CONFIG_FIT_IMAGE_FDT_JAGUAR2 is not set +# CONFIG_FIT_IMAGE_FDT_LUTON is not set +CONFIG_FIT_IMAGE_FDT_MARDUK=y +# CONFIG_FIT_IMAGE_FDT_NI169445 is not set +# CONFIG_FIT_IMAGE_FDT_OCELOT is not set +# CONFIG_FIT_IMAGE_FDT_SERVAL is not set +# CONFIG_FIT_IMAGE_FDT_XILFPGA is not set +CONFIG_FIXED_PHY=y +CONFIG_FS_IOMAP=y +CONFIG_FS_MBCACHE=y +CONFIG_FS_POSIX_ACL=y +CONFIG_FWNODE_MDIO=y +CONFIG_FW_LOADER_PAGED_BUF=y +CONFIG_GENERIC_ALLOCATOR=y +CONFIG_GENERIC_ATOMIC64=y +CONFIG_GENERIC_CLOCKEVENTS=y +CONFIG_GENERIC_CMOS_UPDATE=y +CONFIG_GENERIC_CPU_AUTOPROBE=y +CONFIG_GENERIC_FIND_FIRST_BIT=y +CONFIG_GENERIC_GETTIMEOFDAY=y +CONFIG_GENERIC_IOMAP=y +CONFIG_GENERIC_IRQ_CHIP=y +CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK=y +CONFIG_GENERIC_IRQ_MIGRATION=y +CONFIG_GENERIC_IRQ_SHOW=y +CONFIG_GENERIC_LIB_ASHLDI3=y +CONFIG_GENERIC_LIB_ASHRDI3=y +CONFIG_GENERIC_LIB_CMPDI2=y +CONFIG_GENERIC_LIB_LSHRDI3=y +CONFIG_GENERIC_LIB_UCMPDI2=y +CONFIG_GENERIC_PCI_IOMAP=y +CONFIG_GENERIC_PHY=y +CONFIG_GENERIC_PINCONF=y +CONFIG_GENERIC_PINCTRL_GROUPS=y +CONFIG_GENERIC_PINMUX_FUNCTIONS=y +CONFIG_GENERIC_SCHED_CLOCK=y +CONFIG_GENERIC_SMP_IDLE_THREAD=y +CONFIG_GENERIC_TIME_VSYSCALL=y +CONFIG_GPIOLIB_IRQCHIP=y +CONFIG_GPIO_CDEV=y +CONFIG_HANDLE_DOMAIN_IRQ=y +CONFIG_HARDWARE_WATCHPOINTS=y +CONFIG_HAS_DMA=y +CONFIG_HAS_IOMEM=y +CONFIG_HAS_IOPORT_MAP=y +CONFIG_HOTPLUG_CPU=y +CONFIG_I2C=y +CONFIG_I2C_BOARDINFO=y +CONFIG_I2C_IMG=y +CONFIG_IMGPDC_WDT=y +CONFIG_IMG_MDC_DMA=y +CONFIG_INITRAMFS_SOURCE="" +CONFIG_IRQCHIP=y +CONFIG_IRQ_DOMAIN=y +CONFIG_IRQ_DOMAIN_HIERARCHY=y +CONFIG_IRQ_FORCED_THREADING=y +CONFIG_IRQ_MIPS_CPU=y +CONFIG_IRQ_WORK=y +CONFIG_JBD2=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_PWM=y +# CONFIG_LEGACY_BOARD_OCELOT is not set +# CONFIG_LEGACY_BOARD_SEAD3 is not set +CONFIG_LIBFDT=y +CONFIG_LKDTM=y +CONFIG_LOCK_DEBUGGING_SUPPORT=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_LZO_COMPRESS=y +CONFIG_LZO_DECOMPRESS=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0 +CONFIG_MDIO_BUS=y +CONFIG_MDIO_DEVICE=y +CONFIG_MDIO_DEVRES=y +CONFIG_MEMFD_CREATE=y +CONFIG_MFD_SYSCON=y +CONFIG_MICREL_PHY=y +CONFIG_MIGRATION=y +CONFIG_MIPS=y +CONFIG_MIPS_ASID_BITS=8 +CONFIG_MIPS_ASID_SHIFT=0 +CONFIG_MIPS_AUTO_PFN_OFFSET=y +CONFIG_MIPS_CLOCK_VSYSCALL=y +CONFIG_MIPS_CM=y +CONFIG_MIPS_CMDLINE_DTB_EXTEND=y +# CONFIG_MIPS_CMDLINE_FROM_BOOTLOADER is not set +CONFIG_MIPS_CPC=y +CONFIG_MIPS_CPS=y +# CONFIG_MIPS_CPS_CPUIDLE is not set +# CONFIG_MIPS_CPS_NS16550_BOOL is not set +CONFIG_MIPS_CPS_PM=y +CONFIG_MIPS_CPU_SCACHE=y +CONFIG_MIPS_EBPF_JIT=y +CONFIG_MIPS_GENERIC=y +CONFIG_MIPS_GENERIC_KERNEL=y +CONFIG_MIPS_GIC=y +CONFIG_MIPS_L1_CACHE_SHIFT=7 +CONFIG_MIPS_L1_CACHE_SHIFT_7=y +CONFIG_MIPS_LD_CAN_LINK_VDSO=y +CONFIG_MIPS_MT=y +CONFIG_MIPS_MT_FPAFF=y +CONFIG_MIPS_MT_SMP=y +CONFIG_MIPS_NO_APPENDED_DTB=y +CONFIG_MIPS_NR_CPU_NR_MAP=4 +CONFIG_MIPS_PERF_SHARED_TC_COUNTERS=y +CONFIG_MIPS_SPRAM=y +CONFIG_MMC=y +CONFIG_MMC_BLOCK=y +CONFIG_MMC_DW=y +# CONFIG_MMC_DW_BLUEFIELD is not set +# CONFIG_MMC_DW_EXYNOS is not set +# CONFIG_MMC_DW_HI3798CV200 is not set +# CONFIG_MMC_DW_K3 is not set +CONFIG_MMC_DW_PLTFM=y +CONFIG_MODULES_USE_ELF_REL=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_NAND_CORE=y +CONFIG_MTD_NAND_ECC=y +CONFIG_MTD_SPI_NAND=y +CONFIG_MTD_SPI_NOR=y +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y +CONFIG_MTD_UBI=y +CONFIG_MTD_UBI_BEB_LIMIT=20 +CONFIG_MTD_UBI_BLOCK=y +CONFIG_MTD_UBI_FASTMAP=y +CONFIG_MTD_UBI_WL_THRESHOLD=4096 +CONFIG_NAMESPACES=y +CONFIG_NEED_DMA_MAP_STATE=y +CONFIG_NET_FLOW_LIMIT=y +CONFIG_NET_NS=y +CONFIG_NET_PTP_CLASSIFY=y +CONFIG_NET_SELFTESTS=y +CONFIG_NLS=y +CONFIG_NO_EXCEPT_FILL=y +CONFIG_NO_HZ=y +CONFIG_NO_HZ_COMMON=y +CONFIG_NO_HZ_IDLE=y +CONFIG_NR_CPUS=4 +CONFIG_OF=y +CONFIG_OF_ADDRESS=y +CONFIG_OF_EARLY_FLATTREE=y +CONFIG_OF_FLATTREE=y +CONFIG_OF_GPIO=y +CONFIG_OF_IRQ=y +CONFIG_OF_KOBJ=y +CONFIG_OF_MDIO=y +CONFIG_PADATA=y +CONFIG_PAGE_POOL=y +CONFIG_PCI_DRIVERS_GENERIC=y +CONFIG_PCS_XPCS=y +CONFIG_PERF_USE_VMALLOC=y +CONFIG_PGTABLE_LEVELS=2 +CONFIG_PHYLIB=y +CONFIG_PHYLINK=y +CONFIG_PHY_PISTACHIO_USB=y +CONFIG_PINCTRL=y +CONFIG_PINCTRL_PISTACHIO=y +CONFIG_POWER_SUPPLY=y +CONFIG_PPS=y +# CONFIG_PREEMPT_NONE is not set +CONFIG_PREEMPT_VOLUNTARY=y +CONFIG_PRINTK_TIME=y +CONFIG_PROC_EVENTS=y +CONFIG_PROFILING=y +CONFIG_PTP_1588_CLOCK=y +CONFIG_PTP_1588_CLOCK_OPTIONAL=y +CONFIG_PWM=y +CONFIG_PWM_IMG=y +CONFIG_PWM_SYSFS=y +CONFIG_QUEUED_RWLOCKS=y +CONFIG_QUEUED_SPINLOCKS=y +CONFIG_RATIONAL=y +CONFIG_REGMAP=y +CONFIG_REGMAP_MMIO=y +CONFIG_REGMAP_SPI=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_GPIO=y +CONFIG_RESET_CONTROLLER=y +CONFIG_RESET_PISTACHIO=y +CONFIG_RFS_ACCEL=y +CONFIG_RPS=y +CONFIG_SCHEDSTATS=y +CONFIG_SCHED_INFO=y +CONFIG_SCSI=y +CONFIG_SCSI_COMMON=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SERIAL_8250_DW=y +CONFIG_SERIAL_8250_DWLIB=y +CONFIG_SERIAL_MCTRL_GPIO=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_SC16IS7XX=y +CONFIG_SERIAL_SC16IS7XX_CORE=y +# CONFIG_SERIAL_SC16IS7XX_I2C is not set +CONFIG_SERIAL_SC16IS7XX_SPI=y +CONFIG_SGL_ALLOC=y +CONFIG_SG_POOL=y +CONFIG_SMP=y +CONFIG_SMP_UP=y +CONFIG_SOCK_RX_QUEUE_MAPPING=y +CONFIG_SPI=y +CONFIG_SPI_IMG_SPFI=y +CONFIG_SPI_MASTER=y +CONFIG_SPI_MEM=y +CONFIG_SRAM=y +CONFIG_SRCU=y +CONFIG_STMMAC_ETH=y +CONFIG_STMMAC_PLATFORM=y +CONFIG_SWAP_IO_SPACE=y +CONFIG_SWPHY=y +CONFIG_SYNC_R4K=y +CONFIG_SYSCTL_EXCEPTION_TRACE=y +CONFIG_SYS_HAS_CPU_MIPS32_R1=y +CONFIG_SYS_HAS_CPU_MIPS32_R2=y +CONFIG_SYS_HAS_CPU_MIPS32_R6=y +CONFIG_SYS_HAS_CPU_MIPS64_R1=y +CONFIG_SYS_HAS_CPU_MIPS64_R2=y +CONFIG_SYS_HAS_CPU_MIPS64_R6=y +CONFIG_SYS_SUPPORTS_32BIT_KERNEL=y +CONFIG_SYS_SUPPORTS_64BIT_KERNEL=y +CONFIG_SYS_SUPPORTS_ARBIT_HZ=y +CONFIG_SYS_SUPPORTS_BIG_ENDIAN=y +CONFIG_SYS_SUPPORTS_HIGHMEM=y +CONFIG_SYS_SUPPORTS_HOTPLUG_CPU=y +CONFIG_SYS_SUPPORTS_LITTLE_ENDIAN=y +CONFIG_SYS_SUPPORTS_MICROMIPS=y +CONFIG_SYS_SUPPORTS_MIPS16=y +CONFIG_SYS_SUPPORTS_MIPS_CPS=y +CONFIG_SYS_SUPPORTS_MULTITHREADING=y +CONFIG_SYS_SUPPORTS_RELOCATABLE=y +CONFIG_SYS_SUPPORTS_SCHED_SMT=y +CONFIG_SYS_SUPPORTS_SMARTMIPS=y +CONFIG_SYS_SUPPORTS_SMP=y +CONFIG_SYS_SUPPORTS_ZBOOT=y +CONFIG_TARGET_ISA_REV=2 +CONFIG_TICK_CPU_ACCOUNTING=y +CONFIG_TIMER_OF=y +CONFIG_TIMER_PROBE=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_TREE_RCU=y +CONFIG_TREE_SRCU=y +CONFIG_UBIFS_FS=y +CONFIG_UHI_BOOT=y +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_COMMON=y +CONFIG_USB_DWC2=y +CONFIG_USB_DWC2_DUAL_ROLE=y +CONFIG_USB_EHCI_HCD=y +# CONFIG_USB_EHCI_HCD_PLATFORM is not set +CONFIG_USB_GADGET=y +CONFIG_USB_ROLE_SWITCH=y +CONFIG_USB_STORAGE=y +CONFIG_USB_SUPPORT=y +CONFIG_USER_NS=y +CONFIG_USE_OF=y +# CONFIG_VIRT_BOARD_RANCHU is not set +CONFIG_WATCHDOG_CORE=y +CONFIG_WEAK_ORDERING=y +CONFIG_XPS=y +CONFIG_XXHASH=y +CONFIG_ZLIB_DEFLATE=y +CONFIG_ZLIB_INFLATE=y +CONFIG_ZSMALLOC=y +# CONFIG_ZSMALLOC_STAT is not set +CONFIG_ZSTD_COMPRESS=y +CONFIG_ZSTD_DECOMPRESS=y diff --git a/target/linux/pistachio/image/Makefile b/target/linux/pistachio/image/Makefile index 2e15005ac2..9c0e9da91f 100644 --- a/target/linux/pistachio/image/Makefile +++ b/target/linux/pistachio/image/Makefile @@ -5,7 +5,11 @@ include $(TOPDIR)/rules.mk include $(INCLUDE_DIR)/image.mk -KERNEL_LOADADDR := 0x80400000 +ifdef CONFIG_LINUX_5_10 + KERNEL_LOADADDR := 0x80400000 +else + KERNEL_LOADADDR := 0x80100000 +endif define Device/Default PROFILES := Default diff --git a/target/linux/pistachio/patches-5.15/101-dmaengine-img-mdc-Handle-early-status-read.patch b/target/linux/pistachio/patches-5.15/101-dmaengine-img-mdc-Handle-early-status-read.patch new file mode 100644 index 0000000000..031a4e3e5e --- /dev/null +++ b/target/linux/pistachio/patches-5.15/101-dmaengine-img-mdc-Handle-early-status-read.patch @@ -0,0 +1,68 @@ +From a2dd154377c9aa6ddda00d39b8c7c334e4fa16ff Mon Sep 17 00:00:00 2001 +From: Damien Horsley +Date: Tue, 22 Mar 2016 12:46:09 +0000 +Subject: dmaengine: img-mdc: Handle early status read + +It is possible that mdc_tx_status may be called before the first +node has been read from memory. + +In this case, the residue value stored in the register is undefined. +Return the transfer size instead. + +Signed-off-by: Damien Horsley +--- + drivers/dma/img-mdc-dma.c | 40 ++++++++++++++++++++++++---------------- + 1 file changed, 24 insertions(+), 16 deletions(-) + +--- a/drivers/dma/img-mdc-dma.c ++++ b/drivers/dma/img-mdc-dma.c +@@ -618,25 +618,33 @@ static enum dma_status mdc_tx_status(str + (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1); + + /* +- * If the command loaded event hasn't been processed yet, then +- * the difference above includes an extra command. ++ * If the first node has not yet been read from memory, ++ * the residue register value is undefined + */ +- if (!mdesc->cmd_loaded) +- cmds--; +- else +- cmds += mdesc->list_cmds_done; +- +- bytes = mdesc->list_xfer_size; +- ldesc = mdesc->list; +- for (i = 0; i < cmds; i++) { +- bytes -= ldesc->xfer_size + 1; +- ldesc = ldesc->next_desc; +- } +- if (ldesc) { +- if (residue != MDC_TRANSFER_SIZE_MASK) +- bytes -= ldesc->xfer_size - residue; ++ if (!mdesc->cmd_loaded && !cmds) { ++ bytes = mdesc->list_xfer_size; ++ } else { ++ /* ++ * If the command loaded event hasn't been processed yet, then ++ * the difference above includes an extra command. ++ */ ++ if (!mdesc->cmd_loaded) ++ cmds--; + else ++ cmds += mdesc->list_cmds_done; ++ ++ bytes = mdesc->list_xfer_size; ++ ldesc = mdesc->list; ++ for (i = 0; i < cmds; i++) { + bytes -= ldesc->xfer_size + 1; ++ ldesc = ldesc->next_desc; ++ } ++ if (ldesc) { ++ if (residue != MDC_TRANSFER_SIZE_MASK) ++ bytes -= ldesc->xfer_size - residue; ++ else ++ bytes -= ldesc->xfer_size + 1; ++ } + } + } + spin_unlock_irqrestore(&mchan->vc.lock, flags); diff --git a/target/linux/pistachio/patches-5.15/102-spi-img-spfi-Implement-dual-and-quad-mode.patch b/target/linux/pistachio/patches-5.15/102-spi-img-spfi-Implement-dual-and-quad-mode.patch new file mode 100644 index 0000000000..83f21a5c0a --- /dev/null +++ b/target/linux/pistachio/patches-5.15/102-spi-img-spfi-Implement-dual-and-quad-mode.patch @@ -0,0 +1,198 @@ +From cd2a6af51553d38072cd31699b58d16ca6176ef5 Mon Sep 17 00:00:00 2001 +From: Ionela Voinescu +Date: Thu, 2 Feb 2017 16:46:14 +0000 +Subject: spi: img-spfi: Implement dual and quad mode + +For dual and quad modes to work the SPFI controller needs +to have information about command/address/dummy bytes in the +transaction register. This information is not relevant for +single mode, and therefore it can have any value in the +allowed range. Therefore, for any read or write transfers of less +than 8 bytes (cmd = 1 byte, addr up to 7 bytes), SPFI will be +configured, but not enabled (unless it is the last transfer in +the queue). The transfer will be enabled by the subsequent tranfer. +A pending transfer is determined by the content of the transaction +register: if command part is set and tsize is not. + +This way we ensure that for dual and quad transactions +the command request size will apear in the command/address part +of the transaction register, while the data size will be in +tsize, all data being sent/received in the same transaction (as +set up in the transaction register). + +Signed-off-by: Ionela Voinescu +Signed-off-by: Ezequiel Garcia +--- + drivers/spi/spi-img-spfi.c | 96 ++++++++++++++++++++++++++++++++++++++++------ + 1 file changed, 85 insertions(+), 11 deletions(-) + +--- a/drivers/spi/spi-img-spfi.c ++++ b/drivers/spi/spi-img-spfi.c +@@ -36,7 +36,8 @@ + #define SPFI_CONTROL_SOFT_RESET BIT(11) + #define SPFI_CONTROL_SEND_DMA BIT(10) + #define SPFI_CONTROL_GET_DMA BIT(9) +-#define SPFI_CONTROL_SE BIT(8) ++#define SPFI_CONTROL_SE BIT(8) ++#define SPFI_CONTROL_TX_RX BIT(1) + #define SPFI_CONTROL_TMODE_SHIFT 5 + #define SPFI_CONTROL_TMODE_MASK 0x7 + #define SPFI_CONTROL_TMODE_SINGLE 0 +@@ -47,6 +48,10 @@ + #define SPFI_TRANSACTION 0x18 + #define SPFI_TRANSACTION_TSIZE_SHIFT 16 + #define SPFI_TRANSACTION_TSIZE_MASK 0xffff ++#define SPFI_TRANSACTION_CMD_SHIFT 13 ++#define SPFI_TRANSACTION_CMD_MASK 0x7 ++#define SPFI_TRANSACTION_ADDR_SHIFT 10 ++#define SPFI_TRANSACTION_ADDR_MASK 0x7 + + #define SPFI_PORT_STATE 0x1c + #define SPFI_PORT_STATE_DEV_SEL_SHIFT 20 +@@ -83,6 +88,7 @@ + */ + #define SPFI_32BIT_FIFO_SIZE 64 + #define SPFI_8BIT_FIFO_SIZE 16 ++#define SPFI_DATA_REQUEST_MAX_SIZE 8 + + struct img_spfi { + struct device *dev; +@@ -99,6 +105,8 @@ struct img_spfi { + struct dma_chan *tx_ch; + bool tx_dma_busy; + bool rx_dma_busy; ++ ++ bool complete; + }; + + static inline u32 spfi_readl(struct img_spfi *spfi, u32 reg) +@@ -115,9 +123,11 @@ static inline void spfi_start(struct img + { + u32 val; + +- val = spfi_readl(spfi, SPFI_CONTROL); +- val |= SPFI_CONTROL_SPFI_EN; +- spfi_writel(spfi, val, SPFI_CONTROL); ++ if (spfi->complete) { ++ val = spfi_readl(spfi, SPFI_CONTROL); ++ val |= SPFI_CONTROL_SPFI_EN; ++ spfi_writel(spfi, val, SPFI_CONTROL); ++ } + } + + static inline void spfi_reset(struct img_spfi *spfi) +@@ -130,12 +140,21 @@ static int spfi_wait_all_done(struct img + { + unsigned long timeout = jiffies + msecs_to_jiffies(50); + ++ if (!(spfi->complete)) ++ return 0; ++ + while (time_before(jiffies, timeout)) { + u32 status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS); + + if (status & SPFI_INTERRUPT_ALLDONETRIG) { + spfi_writel(spfi, SPFI_INTERRUPT_ALLDONETRIG, + SPFI_INTERRUPT_CLEAR); ++ /* ++ * Disable SPFI for it not to interfere with ++ * pending transactions ++ */ ++ spfi_writel(spfi, spfi_readl(spfi, SPFI_CONTROL) ++ & ~SPFI_CONTROL_SPFI_EN, SPFI_CONTROL); + return 0; + } + cpu_relax(); +@@ -441,9 +460,32 @@ static void img_spfi_config(struct spi_m + struct spi_transfer *xfer) + { + struct img_spfi *spfi = spi_master_get_devdata(spi->master); +- u32 val, div; ++ u32 val, div, transact; ++ bool is_pending; + + /* ++ * For read or write transfers of less than 8 bytes (cmd = 1 byte, ++ * addr up to 7 bytes), SPFI will be configured, but not enabled ++ * (unless it is the last transfer in the queue).The transfer will ++ * be enabled by the subsequent transfer. ++ * A pending transfer is determined by the content of the ++ * transaction register: if command part is set and tsize ++ * is not ++ */ ++ transact = spfi_readl(spfi, SPFI_TRANSACTION); ++ is_pending = ((transact >> SPFI_TRANSACTION_CMD_SHIFT) & ++ SPFI_TRANSACTION_CMD_MASK) && ++ (!((transact >> SPFI_TRANSACTION_TSIZE_SHIFT) & ++ SPFI_TRANSACTION_TSIZE_MASK)); ++ ++ /* If there are no pending transactions it's OK to soft reset */ ++ if (!is_pending) { ++ /* Start the transaction from a known (reset) state */ ++ spfi_reset(spfi); ++ } ++ ++ /* ++ * Before anything else, set up parameters. + * output = spfi_clk * (BITCLK / 512), where BITCLK must be a + * power of 2 up to 128 + */ +@@ -456,20 +498,52 @@ static void img_spfi_config(struct spi_m + val |= div << SPFI_DEVICE_PARAMETER_BITCLK_SHIFT; + spfi_writel(spfi, val, SPFI_DEVICE_PARAMETER(spi->chip_select)); + +- spfi_writel(spfi, xfer->len << SPFI_TRANSACTION_TSIZE_SHIFT, +- SPFI_TRANSACTION); ++ if (!list_is_last(&xfer->transfer_list, &master->cur_msg->transfers) && ++ /* ++ * For duplex mode (both the tx and rx buffers are !NULL) the ++ * CMD, ADDR, and DUMMY byte parts of the transaction register ++ * should always be 0 and therefore the pending transfer ++ * technique cannot be used. ++ */ ++ (xfer->tx_buf) && (!xfer->rx_buf) && ++ (xfer->len <= SPFI_DATA_REQUEST_MAX_SIZE) && !is_pending) { ++ transact = (1 & SPFI_TRANSACTION_CMD_MASK) << ++ SPFI_TRANSACTION_CMD_SHIFT; ++ transact |= ((xfer->len - 1) & SPFI_TRANSACTION_ADDR_MASK) << ++ SPFI_TRANSACTION_ADDR_SHIFT; ++ spfi->complete = false; ++ } else { ++ spfi->complete = true; ++ if (is_pending) { ++ /* Keep setup from pending transfer */ ++ transact |= ((xfer->len & SPFI_TRANSACTION_TSIZE_MASK) << ++ SPFI_TRANSACTION_TSIZE_SHIFT); ++ } else { ++ transact = ((xfer->len & SPFI_TRANSACTION_TSIZE_MASK) << ++ SPFI_TRANSACTION_TSIZE_SHIFT); ++ } ++ } ++ spfi_writel(spfi, transact, SPFI_TRANSACTION); + + val = spfi_readl(spfi, SPFI_CONTROL); + val &= ~(SPFI_CONTROL_SEND_DMA | SPFI_CONTROL_GET_DMA); +- if (xfer->tx_buf) ++ /* ++ * We set up send DMA for pending transfers also, as ++ * those are always send transfers ++ */ ++ if ((xfer->tx_buf) || is_pending) + val |= SPFI_CONTROL_SEND_DMA; +- if (xfer->rx_buf) ++ if (xfer->tx_buf) ++ val |= SPFI_CONTROL_TX_RX; ++ if (xfer->rx_buf) { + val |= SPFI_CONTROL_GET_DMA; ++ val &= ~SPFI_CONTROL_TX_RX; ++ } + val &= ~(SPFI_CONTROL_TMODE_MASK << SPFI_CONTROL_TMODE_SHIFT); +- if (xfer->tx_nbits == SPI_NBITS_DUAL && ++ if (xfer->tx_nbits == SPI_NBITS_DUAL || + xfer->rx_nbits == SPI_NBITS_DUAL) + val |= SPFI_CONTROL_TMODE_DUAL << SPFI_CONTROL_TMODE_SHIFT; +- else if (xfer->tx_nbits == SPI_NBITS_QUAD && ++ else if (xfer->tx_nbits == SPI_NBITS_QUAD || + xfer->rx_nbits == SPI_NBITS_QUAD) + val |= SPFI_CONTROL_TMODE_QUAD << SPFI_CONTROL_TMODE_SHIFT; + val |= SPFI_CONTROL_SE; diff --git a/target/linux/pistachio/patches-5.15/104-spi-img-spfi-use-device-0-configuration-for-all-devi.patch b/target/linux/pistachio/patches-5.15/104-spi-img-spfi-use-device-0-configuration-for-all-devi.patch new file mode 100644 index 0000000000..2995b7dd88 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/104-spi-img-spfi-use-device-0-configuration-for-all-devi.patch @@ -0,0 +1,64 @@ +From 905ee06a9966113fe51d6bad1819759cb30fd0bd Mon Sep 17 00:00:00 2001 +From: Ionela Voinescu +Date: Tue, 9 Feb 2016 10:18:31 +0000 +Subject: spi: img-spfi: use device 0 configuration for all devices + +Given that we control the chip select line externally +we can use only one parameter register (device 0 parameter +register) and one set of configuration bits (port configuration +bits for device 0) for all devices (all chip select lines). + +Signed-off-by: Ionela Voinescu +--- + drivers/spi/spi-img-spfi.c | 23 ++++++++++++++++------- + 1 file changed, 16 insertions(+), 7 deletions(-) + +--- a/drivers/spi/spi-img-spfi.c ++++ b/drivers/spi/spi-img-spfi.c +@@ -429,18 +429,23 @@ static int img_spfi_prepare(struct spi_m + struct img_spfi *spfi = spi_master_get_devdata(master); + u32 val; + ++ /* ++ * The chip select line is controlled externally so ++ * we can use the CS0 configuration for all devices ++ */ + val = spfi_readl(spfi, SPFI_PORT_STATE); ++ ++ /* 0 for device selection */ + val &= ~(SPFI_PORT_STATE_DEV_SEL_MASK << + SPFI_PORT_STATE_DEV_SEL_SHIFT); +- val |= msg->spi->chip_select << SPFI_PORT_STATE_DEV_SEL_SHIFT; + if (msg->spi->mode & SPI_CPHA) +- val |= SPFI_PORT_STATE_CK_PHASE(msg->spi->chip_select); ++ val |= SPFI_PORT_STATE_CK_PHASE(0); + else +- val &= ~SPFI_PORT_STATE_CK_PHASE(msg->spi->chip_select); ++ val &= ~SPFI_PORT_STATE_CK_PHASE(0); + if (msg->spi->mode & SPI_CPOL) +- val |= SPFI_PORT_STATE_CK_POL(msg->spi->chip_select); ++ val |= SPFI_PORT_STATE_CK_POL(0); + else +- val &= ~SPFI_PORT_STATE_CK_POL(msg->spi->chip_select); ++ val &= ~SPFI_PORT_STATE_CK_POL(0); + spfi_writel(spfi, val, SPFI_PORT_STATE); + + return 0; +@@ -492,11 +497,15 @@ static void img_spfi_config(struct spi_m + div = DIV_ROUND_UP(clk_get_rate(spfi->spfi_clk), xfer->speed_hz); + div = clamp(512 / (1 << get_count_order(div)), 1, 128); + +- val = spfi_readl(spfi, SPFI_DEVICE_PARAMETER(spi->chip_select)); ++ /* ++ * The chip select line is controlled externally so ++ * we can use the CS0 parameters for all devices ++ */ ++ val = spfi_readl(spfi, SPFI_DEVICE_PARAMETER(0)); + val &= ~(SPFI_DEVICE_PARAMETER_BITCLK_MASK << + SPFI_DEVICE_PARAMETER_BITCLK_SHIFT); + val |= div << SPFI_DEVICE_PARAMETER_BITCLK_SHIFT; +- spfi_writel(spfi, val, SPFI_DEVICE_PARAMETER(spi->chip_select)); ++ spfi_writel(spfi, val, SPFI_DEVICE_PARAMETER(0)); + + if (!list_is_last(&xfer->transfer_list, &master->cur_msg->transfers) && + /* diff --git a/target/linux/pistachio/patches-5.15/105-spi-img-spfi-RX-maximum-burst-size-for-DMA-is-8.patch b/target/linux/pistachio/patches-5.15/105-spi-img-spfi-RX-maximum-burst-size-for-DMA-is-8.patch new file mode 100644 index 0000000000..5418503816 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/105-spi-img-spfi-RX-maximum-burst-size-for-DMA-is-8.patch @@ -0,0 +1,59 @@ +From 56466f505f58f44b69feb7eaed3b506842800456 Mon Sep 17 00:00:00 2001 +From: Ionela Voinescu +Date: Tue, 1 Mar 2016 17:49:45 +0000 +Subject: spi: img-spfi: RX maximum burst size for DMA is 8 + +The depth of the FIFOs is 16 bytes. The DMA request line is tied +to the half full/empty (depending on the use of the TX or RX FIFO) +threshold. For the TX FIFO, if you set a burst size of 8 (equal to +half the depth) the first burst goes into FIFO without any issues, +but due the latency involved (the time the data leaves the DMA +engine to the time it arrives at the FIFO), the DMA might trigger +another burst of 8. But given that there is no space for 2 additonal +bursts of 8, this would result in a failure. Therefore, we have to +keep the burst size for TX to 4 to accomodate for an extra burst. + +For the read (RX) scenario, the DMA request line goes high when +there is at least 8 entries in the FIFO (half full), and we can +program the burst size to be 8 because the risk of accidental burst +does not exist. The DMA engine will not trigger another read until +the read data for all the burst it has sent out has been received. + +While here, move the burst size setting outside of the if/else branches +as they have the same value for both 8 and 32 bit data widths. + +Signed-off-by: Ionela Voinescu +--- + drivers/spi/spi-img-spfi.c | 6 ++---- + 1 file changed, 2 insertions(+), 4 deletions(-) + +--- a/drivers/spi/spi-img-spfi.c ++++ b/drivers/spi/spi-img-spfi.c +@@ -338,12 +338,11 @@ static int img_spfi_start_dma(struct spi + if (xfer->len % 4 == 0) { + rxconf.src_addr = spfi->phys + SPFI_RX_32BIT_VALID_DATA; + rxconf.src_addr_width = 4; +- rxconf.src_maxburst = 4; + } else { + rxconf.src_addr = spfi->phys + SPFI_RX_8BIT_VALID_DATA; + rxconf.src_addr_width = 1; +- rxconf.src_maxburst = 4; + } ++ rxconf.src_maxburst = 8; + dmaengine_slave_config(spfi->rx_ch, &rxconf); + + rxdesc = dmaengine_prep_slave_sg(spfi->rx_ch, xfer->rx_sg.sgl, +@@ -362,12 +361,11 @@ static int img_spfi_start_dma(struct spi + if (xfer->len % 4 == 0) { + txconf.dst_addr = spfi->phys + SPFI_TX_32BIT_VALID_DATA; + txconf.dst_addr_width = 4; +- txconf.dst_maxburst = 4; + } else { + txconf.dst_addr = spfi->phys + SPFI_TX_8BIT_VALID_DATA; + txconf.dst_addr_width = 1; +- txconf.dst_maxburst = 4; + } ++ txconf.dst_maxburst = 4; + dmaengine_slave_config(spfi->tx_ch, &txconf); + + txdesc = dmaengine_prep_slave_sg(spfi->tx_ch, xfer->tx_sg.sgl, diff --git a/target/linux/pistachio/patches-5.15/106-spi-img-spfi-finish-every-transfer-cleanly.patch b/target/linux/pistachio/patches-5.15/106-spi-img-spfi-finish-every-transfer-cleanly.patch new file mode 100644 index 0000000000..ea1f9f28cc --- /dev/null +++ b/target/linux/pistachio/patches-5.15/106-spi-img-spfi-finish-every-transfer-cleanly.patch @@ -0,0 +1,120 @@ +From 5fcca3fd4b621d7b5bdeca18d36dfc6ca6cfe383 Mon Sep 17 00:00:00 2001 +From: Ionela Voinescu +Date: Wed, 10 Aug 2016 11:42:26 +0100 +Subject: spi: img-spfi: finish every transfer cleanly + +Before this change, the interrupt status bit that signaled +the end of a tranfers was cleared in the wait_all_done +function. That functionality triggered issues for DMA +duplex transactions where the wait function was called +twice, in both the TX and RX callbacks. + +In order to fix the issue, clear all interrupt data bits +at the end of a PIO transfer or at the end of both TX and RX +duplex transfers, if the transfer is not a pending tranfer +(command waiting for data). After that, the status register +is checked for new incoming data or new data requests to be +signaled. If SPFI finished cleanly, no new interrupt data +bits should be set. + +Signed-off-by: Ionela Voinescu +--- + drivers/spi/spi-img-spfi.c | 49 +++++++++++++++++++++++++++++++++------------- + 1 file changed, 35 insertions(+), 14 deletions(-) + +--- a/drivers/spi/spi-img-spfi.c ++++ b/drivers/spi/spi-img-spfi.c +@@ -79,6 +79,14 @@ + #define SPFI_INTERRUPT_SDE BIT(1) + #define SPFI_INTERRUPT_SDTRIG BIT(0) + ++#define SPFI_INTERRUPT_DATA_BITS (SPFI_INTERRUPT_SDHF |\ ++ SPFI_INTERRUPT_SDFUL |\ ++ SPFI_INTERRUPT_GDEX32BIT |\ ++ SPFI_INTERRUPT_GDHF |\ ++ SPFI_INTERRUPT_GDFUL |\ ++ SPFI_INTERRUPT_ALLDONETRIG |\ ++ SPFI_INTERRUPT_GDEX8BIT) ++ + /* + * There are four parallel FIFOs of 16 bytes each. The word buffer + * (*_32BIT_VALID_DATA) accesses all four FIFOs at once, resulting in an +@@ -136,6 +144,23 @@ static inline void spfi_reset(struct img + spfi_writel(spfi, 0, SPFI_CONTROL); + } + ++static inline void spfi_finish(struct img_spfi *spfi) ++{ ++ if (!(spfi->complete)) ++ return; ++ ++ /* Clear data bits as all transfers(TX and RX) have finished */ ++ spfi_writel(spfi, SPFI_INTERRUPT_DATA_BITS, SPFI_INTERRUPT_CLEAR); ++ if (spfi_readl(spfi, SPFI_INTERRUPT_STATUS) & SPFI_INTERRUPT_DATA_BITS) { ++ dev_err(spfi->dev, "SPFI did not finish transfer cleanly.\n"); ++ spfi_reset(spfi); ++ } ++ /* Disable SPFI for it not to interfere with pending transactions */ ++ spfi_writel(spfi, ++ spfi_readl(spfi, SPFI_CONTROL) & ~SPFI_CONTROL_SPFI_EN, ++ SPFI_CONTROL); ++} ++ + static int spfi_wait_all_done(struct img_spfi *spfi) + { + unsigned long timeout = jiffies + msecs_to_jiffies(50); +@@ -144,19 +169,9 @@ static int spfi_wait_all_done(struct img + return 0; + + while (time_before(jiffies, timeout)) { +- u32 status = spfi_readl(spfi, SPFI_INTERRUPT_STATUS); +- +- if (status & SPFI_INTERRUPT_ALLDONETRIG) { +- spfi_writel(spfi, SPFI_INTERRUPT_ALLDONETRIG, +- SPFI_INTERRUPT_CLEAR); +- /* +- * Disable SPFI for it not to interfere with +- * pending transactions +- */ +- spfi_writel(spfi, spfi_readl(spfi, SPFI_CONTROL) +- & ~SPFI_CONTROL_SPFI_EN, SPFI_CONTROL); ++ if (spfi_readl(spfi, SPFI_INTERRUPT_STATUS) & ++ SPFI_INTERRUPT_ALLDONETRIG) + return 0; +- } + cpu_relax(); + } + +@@ -288,6 +303,8 @@ static int img_spfi_start_pio(struct spi + } + + ret = spfi_wait_all_done(spfi); ++ spfi_finish(spfi); ++ + if (ret < 0) + return ret; + +@@ -303,8 +320,10 @@ static void img_spfi_dma_rx_cb(void *dat + + spin_lock_irqsave(&spfi->lock, flags); + spfi->rx_dma_busy = false; +- if (!spfi->tx_dma_busy) ++ if (!spfi->tx_dma_busy) { ++ spfi_finish(spfi); + spi_finalize_current_transfer(spfi->master); ++ } + spin_unlock_irqrestore(&spfi->lock, flags); + } + +@@ -317,8 +336,10 @@ static void img_spfi_dma_tx_cb(void *dat + + spin_lock_irqsave(&spfi->lock, flags); + spfi->tx_dma_busy = false; +- if (!spfi->rx_dma_busy) ++ if (!spfi->rx_dma_busy) { ++ spfi_finish(spfi); + spi_finalize_current_transfer(spfi->master); ++ } + spin_unlock_irqrestore(&spfi->lock, flags); + } + diff --git a/target/linux/pistachio/patches-5.15/108-clk-pistachio-Fix-wrong-SDHost-card-speed.patch b/target/linux/pistachio/patches-5.15/108-clk-pistachio-Fix-wrong-SDHost-card-speed.patch new file mode 100644 index 0000000000..6fddbe269a --- /dev/null +++ b/target/linux/pistachio/patches-5.15/108-clk-pistachio-Fix-wrong-SDHost-card-speed.patch @@ -0,0 +1,49 @@ +From 3642843a06025ec333d7e92580cf52cb8db2a652 Mon Sep 17 00:00:00 2001 +From: Govindraj Raja +Date: Fri, 8 Jan 2016 16:36:07 +0000 +Subject: clk: pistachio: Fix wrong SDHost card speed + +The SDHost currently clocks the card 4x slower than it +should do, because there is fixed divide by 4 in the +sdhost wrapper that is not present in the clock tree. +To model this add a fixed divide by 4 clock node in +the SDHost clock path. + +This will ensure the right clock frequency is selected when +the mmc driver tries to configure frequency on card insert. + +Signed-off-by: Govindraj Raja +--- + drivers/clk/pistachio/clk-pistachio.c | 3 ++- + include/dt-bindings/clock/pistachio-clk.h | 1 + + 2 files changed, 3 insertions(+), 1 deletion(-) + +--- a/drivers/clk/pistachio/clk-pistachio.c ++++ b/drivers/clk/pistachio/clk-pistachio.c +@@ -41,7 +41,7 @@ static struct pistachio_gate pistachio_g + GATE(CLK_AUX_ADC_INTERNAL, "aux_adc_internal", "sys_internal_div", + 0x104, 22), + GATE(CLK_AUX_ADC, "aux_adc", "aux_adc_div", 0x104, 23), +- GATE(CLK_SD_HOST, "sd_host", "sd_host_div", 0x104, 24), ++ GATE(CLK_SD_HOST, "sd_host", "sd_host_div4", 0x104, 24), + GATE(CLK_BT, "bt", "bt_div", 0x104, 25), + GATE(CLK_BT_DIV4, "bt_div4", "bt_div4_div", 0x104, 26), + GATE(CLK_BT_DIV8, "bt_div8", "bt_div8_div", 0x104, 27), +@@ -51,6 +51,7 @@ static struct pistachio_gate pistachio_g + static struct pistachio_fixed_factor pistachio_ffs[] __initdata = { + FIXED_FACTOR(CLK_WIFI_DIV4, "wifi_div4", "wifi_pll", 4), + FIXED_FACTOR(CLK_WIFI_DIV8, "wifi_div8", "wifi_pll", 8), ++ FIXED_FACTOR(CLK_SDHOST_DIV4, "sd_host_div4", "sd_host_div", 4), + }; + + static struct pistachio_div pistachio_divs[] __initdata = { +--- a/include/dt-bindings/clock/pistachio-clk.h ++++ b/include/dt-bindings/clock/pistachio-clk.h +@@ -18,6 +18,7 @@ + /* Fixed-factor clocks */ + #define CLK_WIFI_DIV4 16 + #define CLK_WIFI_DIV8 17 ++#define CLK_SDHOST_DIV4 18 + + /* Gate clocks */ + #define CLK_MIPS 32 diff --git a/target/linux/pistachio/patches-5.15/109-MIPS-DTS-img-marduk-switch-mmc-to-1-bit-mode.patch b/target/linux/pistachio/patches-5.15/109-MIPS-DTS-img-marduk-switch-mmc-to-1-bit-mode.patch new file mode 100644 index 0000000000..faba23c5f1 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/109-MIPS-DTS-img-marduk-switch-mmc-to-1-bit-mode.patch @@ -0,0 +1,47 @@ +From 981c1d416af45eff207227aec106381ac23aac99 Mon Sep 17 00:00:00 2001 +From: Ian Pozella +Date: Mon, 20 Feb 2017 10:00:52 +0000 +Subject: MIPS: DTS: img: marduk: switch mmc to 1 bit mode + +The mmc block in Pistachio allows 1 to 8 data bits to be used. +Marduk uses 4 bits allowing the upper 4 bits to be allocated +to the Mikrobus ports. However these bits are still connected +internally meaning the mmc block recieves signals on all data lines +and seems the internal HW CRC checks get corrupted by this erroneous +data. + +We cannot control what data is sent on these lines because they go +to external ports. 1 bit mode does not exhibit the issue hence the +safe default is to use this. If a user knows that in their use case +they will not use the upper bits then they can set to 4 bit mode in +order to improve performance. + +Also make sure that the upper 4 bits don't get allocated to the mmc +driver (the default is to assign all 8 pins) so they can be allocated +to other drivers. Allocating all 4 despite setting 1 bit mode as this +matches what is there in hardware. + +Signed-off-by: Ian Pozella +--- + arch/mips/boot/dts/img/pistachio_marduk.dts | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +--- a/arch/mips/boot/dts/img/pistachio_marduk.dts ++++ b/arch/mips/boot/dts/img/pistachio_marduk.dts +@@ -118,7 +118,7 @@ + + &sdhost { + status = "okay"; +- bus-width = <4>; ++ bus-width = <1>; + disable-wp; + }; + +@@ -128,6 +128,7 @@ + + &pin_sdhost_data { + drive-strength = <2>; ++ pins = "mfio17", "mfio18", "mfio19", "mfio20"; + }; + + &pwm { diff --git a/target/linux/pistachio/patches-5.15/401-mtd-nor-support-mtd-name-from-device-tree.patch b/target/linux/pistachio/patches-5.15/401-mtd-nor-support-mtd-name-from-device-tree.patch new file mode 100644 index 0000000000..da3ca270fa --- /dev/null +++ b/target/linux/pistachio/patches-5.15/401-mtd-nor-support-mtd-name-from-device-tree.patch @@ -0,0 +1,54 @@ +From f32bc2aa01edcba2f2ed5db151cf183eac9ef919 Mon Sep 17 00:00:00 2001 +From: Abhimanyu Vishwakarma +Date: Sat, 25 Feb 2017 16:42:50 +0000 +Subject: mtd: nor: support mtd name from device tree + +Signed-off-by: Abhimanyu Vishwakarma +--- + drivers/mtd/spi-nor/spi-nor.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +--- a/drivers/mtd/spi-nor/core.c ++++ b/drivers/mtd/spi-nor/core.c +@@ -3098,6 +3098,7 @@ int spi_nor_scan(struct spi_nor *nor, co + struct device *dev = nor->dev; + struct mtd_info *mtd = &nor->mtd; + struct device_node *np = spi_nor_get_flash_node(nor); ++ const char __maybe_unused *of_mtd_name = NULL; + int ret; + int i; + +@@ -3152,7 +3153,12 @@ int spi_nor_scan(struct spi_nor *nor, co + if (ret) + return ret; + +- if (!mtd->name) ++#ifdef CONFIG_MTD_OF_PARTS ++ of_property_read_string(np, "linux,mtd-name", &of_mtd_name); ++#endif ++ if (of_mtd_name) ++ mtd->name = of_mtd_name; ++ else if (!mtd->name) + mtd->name = dev_name(dev); + mtd->priv = nor; + mtd->type = MTD_NORFLASH; +--- a/drivers/mtd/mtdcore.c ++++ b/drivers/mtd/mtdcore.c +@@ -847,6 +847,17 @@ out_error: + */ + static void mtd_set_dev_defaults(struct mtd_info *mtd) + { ++#ifdef CONFIG_MTD_OF_PARTS ++ const char __maybe_unused *of_mtd_name = NULL; ++ struct device_node *np; ++ ++ np = mtd_get_of_node(mtd); ++ if (np && !mtd->name) { ++ of_property_read_string(np, "linux,mtd-name", &of_mtd_name); ++ if (of_mtd_name) ++ mtd->name = of_mtd_name; ++ } else ++#endif + if (mtd->dev.parent) { + if (!mtd->owner && mtd->dev.parent->driver) + mtd->owner = mtd->dev.parent->driver->owner; diff --git a/target/linux/pistachio/patches-5.15/901-MIPS-DTS-img-marduk-Add-SPI-NAND-flash.patch b/target/linux/pistachio/patches-5.15/901-MIPS-DTS-img-marduk-Add-SPI-NAND-flash.patch new file mode 100644 index 0000000000..4b28f46833 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/901-MIPS-DTS-img-marduk-Add-SPI-NAND-flash.patch @@ -0,0 +1,30 @@ +From 0023c706f7e0f0f02bd48a63a2f3c04c839532ae Mon Sep 17 00:00:00 2001 +From: Hauke Mehrtens +Date: Sat, 15 Aug 2020 16:04:53 +0200 +Subject: [PATCH 901/904] MIPS: DTS: img: marduk: Add SPI NAND flash + +Add Gigadevice GD5F4GQ4UCYIGT SPI NAND flash to the device tree. + +The NAND flash chip is connected with quad SPI, but reading currently +fails in quad SPI mode. + +Signed-off-by: Hauke Mehrtens +--- + arch/mips/boot/dts/img/pistachio_marduk.dts | 6 ++++++ + 1 file changed, 6 insertions(+) + +--- a/arch/mips/boot/dts/img/pistachio_marduk.dts ++++ b/arch/mips/boot/dts/img/pistachio_marduk.dts +@@ -89,6 +89,12 @@ + reg = <0>; + spi-max-frequency = <50000000>; + }; ++ ++ flash@1 { ++ compatible = "spi-nand"; ++ reg = <1>; ++ spi-max-frequency = <50000000>; ++ }; + }; + + &uart0 { diff --git a/target/linux/pistachio/patches-5.15/902-MIPS-DTS-img-marduk-Add-Cascoda-CA8210-6LoWPAN.patch b/target/linux/pistachio/patches-5.15/902-MIPS-DTS-img-marduk-Add-Cascoda-CA8210-6LoWPAN.patch new file mode 100644 index 0000000000..d4c4ccac53 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/902-MIPS-DTS-img-marduk-Add-Cascoda-CA8210-6LoWPAN.patch @@ -0,0 +1,43 @@ +From b7700154d75e8d7c9a2022f09c2d5430137606fa Mon Sep 17 00:00:00 2001 +From: Hauke Mehrtens +Date: Sat, 15 Aug 2020 16:05:25 +0200 +Subject: [PATCH 902/904] MIPS: DTS: img: marduk: Add Cascoda CA8210 6LoWPAN + +Add Cascoda CA8210 6LoWPAN controller to device tree. + +Signed-off-by: Hauke Mehrtens +--- + arch/mips/boot/dts/img/pistachio_marduk.dts | 22 +++++++++++++++++++++ + 1 file changed, 22 insertions(+) + +--- a/arch/mips/boot/dts/img/pistachio_marduk.dts ++++ b/arch/mips/boot/dts/img/pistachio_marduk.dts +@@ -76,6 +76,28 @@ + VDD-supply = <&internal_dac_supply>; + }; + ++&spfi0 { ++ status = "okay"; ++ pinctrl-0 = <&spim0_pins>, <&spim0_cs0_alt_pin>, <&spim0_cs2_alt_pin>, <&spim0_cs3_alt_pin>, <&spim0_cs4_alt_pin>; ++ pinctrl-names = "default"; ++ ++ cs-gpios = <&gpio1 14 GPIO_ACTIVE_HIGH>, <&gpio0 2 GPIO_ACTIVE_HIGH>, ++ <&gpio1 12 GPIO_ACTIVE_HIGH>, <&gpio1 13 GPIO_ACTIVE_HIGH>; ++ ++ ca8210: ca8210@0 { ++ status = "okay"; ++ compatible = "cascoda,ca8210"; ++ reg = <0>; ++ spi-max-frequency = <4000000>; ++ spi-cpol; ++ reset-gpio = <&gpio0 12 GPIO_ACTIVE_HIGH>; ++ irq-gpio = <&gpio2 12 GPIO_ACTIVE_HIGH>; ++ extclock-enable; ++ extclock-freq = <16000000>; ++ extclock-gpio = <2>; ++ }; ++}; ++ + &spfi1 { + status = "okay"; + diff --git a/target/linux/pistachio/patches-5.15/903-MIPS-DTS-img-marduk-Add-NXP-SC16IS752IPW.patch b/target/linux/pistachio/patches-5.15/903-MIPS-DTS-img-marduk-Add-NXP-SC16IS752IPW.patch new file mode 100644 index 0000000000..b1070c3d30 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/903-MIPS-DTS-img-marduk-Add-NXP-SC16IS752IPW.patch @@ -0,0 +1,81 @@ +From ad4eba0c36ce8af6ab9ea1bc163e4c1ac7c271c3 Mon Sep 17 00:00:00 2001 +From: Hauke Mehrtens +Date: Sat, 15 Aug 2020 16:09:02 +0200 +Subject: [PATCH 903/904] MIPS: DTS: img: marduk: Add NXP SC16IS752IPW + +Add NXP SC16IS752IPW SPI-UART controller to device tree. + +This controller drives 2 UARTs and 7 LEDs on the board. + +Signed-off-by: Hauke Mehrtens +--- + arch/mips/boot/dts/img/pistachio_marduk.dts | 51 +++++++++++++++++++++ + 1 file changed, 51 insertions(+) + +--- a/arch/mips/boot/dts/img/pistachio_marduk.dts ++++ b/arch/mips/boot/dts/img/pistachio_marduk.dts +@@ -46,6 +46,46 @@ + regulator-max-microvolt = <1800000>; + }; + ++ /* EXT clock from ca8210 is fed to sc16is752 */ ++ ca8210_ext_clk: ca8210-ext-clk { ++ compatible = "fixed-clock"; ++ #clock-cells = <0>; ++ clock-frequency = <16000000>; ++ clock-output-names = "ca8210_ext_clock"; ++ }; ++ ++ gpioleds { ++ compatible = "gpio-leds"; ++ user1 { ++ label = "marduk:red:user1"; ++ gpios = <&sc16is752 0 GPIO_ACTIVE_LOW>; ++ }; ++ user2 { ++ label = "marduk:red:user2"; ++ gpios = <&sc16is752 1 GPIO_ACTIVE_LOW>; ++ }; ++ user3 { ++ label = "marduk:red:user3"; ++ gpios = <&sc16is752 2 GPIO_ACTIVE_LOW>; ++ }; ++ user4 { ++ label = "marduk:red:user4"; ++ gpios = <&sc16is752 3 GPIO_ACTIVE_LOW>; ++ }; ++ user5 { ++ label = "marduk:red:user5"; ++ gpios = <&sc16is752 4 GPIO_ACTIVE_LOW>; ++ }; ++ user6 { ++ label = "marduk:red:user6"; ++ gpios = <&sc16is752 5 GPIO_ACTIVE_LOW>; ++ }; ++ user7 { ++ label = "marduk:red:user7"; ++ gpios = <&sc16is752 6 GPIO_ACTIVE_LOW>; ++ }; ++ }; ++ + led-controller { + compatible = "pwm-leds"; + +@@ -96,6 +136,17 @@ + extclock-freq = <16000000>; + extclock-gpio = <2>; + }; ++ ++ sc16is752: sc16is752@1 { ++ compatible = "nxp,sc16is752"; ++ reg = <1>; ++ clocks = <&ca8210_ext_clk>; ++ spi-max-frequency = <4000000>; ++ interrupt-parent = <&gpio0>; ++ interrupts = <11 IRQ_TYPE_EDGE_FALLING>; ++ gpio-controller; ++ #gpio-cells = <2>; ++ }; + }; + + &spfi1 { diff --git a/target/linux/pistachio/patches-5.15/904-MIPS-DTS-img-marduk-Add-partition-name.patch b/target/linux/pistachio/patches-5.15/904-MIPS-DTS-img-marduk-Add-partition-name.patch new file mode 100644 index 0000000000..490027a702 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/904-MIPS-DTS-img-marduk-Add-partition-name.patch @@ -0,0 +1,27 @@ +From ff0e950b605047bf50d470023e0fb2fc2003a0f0 Mon Sep 17 00:00:00 2001 +From: Ian Pozella +Date: Mon, 20 Feb 2017 10:38:07 +0000 +Subject: [PATCH 904/904] MIPS: DTS: img: marduk: Add partition name + +Signed-off-by: Ian Pozella +--- + arch/mips/boot/dts/img/pistachio_marduk.dts | 2 ++ + 1 file changed, 2 insertions(+) + +--- a/arch/mips/boot/dts/img/pistachio_marduk.dts ++++ b/arch/mips/boot/dts/img/pistachio_marduk.dts +@@ -161,12 +161,14 @@ + compatible = "spansion,s25fl016k", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; ++ linux,mtd-name = "spi-nor"; + }; + + flash@1 { + compatible = "spi-nand"; + reg = <1>; + spi-max-frequency = <50000000>; ++ linux,mtd-name = "spi-nand"; + }; + }; + diff --git a/target/linux/pistachio/patches-5.15/905-MIPS-DTS-img-marduk-Add-led-aliases.patch b/target/linux/pistachio/patches-5.15/905-MIPS-DTS-img-marduk-Add-led-aliases.patch new file mode 100644 index 0000000000..8c03ddeea2 --- /dev/null +++ b/target/linux/pistachio/patches-5.15/905-MIPS-DTS-img-marduk-Add-led-aliases.patch @@ -0,0 +1,27 @@ +--- a/arch/mips/boot/dts/img/pistachio_marduk.dts ++++ b/arch/mips/boot/dts/img/pistachio_marduk.dts +@@ -19,6 +19,11 @@ + ethernet0 = &enet; + spi0 = &spfi0; + spi1 = &spfi1; ++ ++ led-boot = &led_heartbeat; ++ led-failsafe = &led_heartbeat; ++ led-running = &led_heartbeat; ++ led-upgrade = &led_heartbeat; + }; + + chosen { +@@ -89,11 +94,10 @@ + led-controller { + compatible = "pwm-leds"; + +- led-1 { ++ led_heartbeat: heartbeat { + label = "marduk:red:heartbeat"; + pwms = <&pwm 3 300000>; + max-brightness = <255>; +- linux,default-trigger = "heartbeat"; + }; + }; + diff --git a/target/linux/realtek/dts-5.15/rtl9302_zyxel_xgs1250-12.dts b/target/linux/realtek/dts-5.15/rtl9302_zyxel_xgs1250-12.dts index a57fc00c6e..a9821b2cc6 100644 --- a/target/linux/realtek/dts-5.15/rtl9302_zyxel_xgs1250-12.dts +++ b/target/linux/realtek/dts-5.15/rtl9302_zyxel_xgs1250-12.dts @@ -64,6 +64,8 @@ led_set: led_set@0 { compatible = "realtek,rtl9300-leds"; + active-low; + led_set0 = <0x0000 0xffff 0x0a20 0x0b80>; // LED set 0: 1000Mbps, 10/100Mbps led_set1 = <0x0a0b 0x0a28 0x0a82 0x0a0b>; // LED set 1: (10G, 5G, 2.5G) (2.5G, 1G) // (5G, 10/100) (10G, 5G, 2.5G) diff --git a/target/linux/realtek/files-5.15/drivers/net/dsa/rtl83xx/rtl930x.c b/target/linux/realtek/files-5.15/drivers/net/dsa/rtl83xx/rtl930x.c index 7e4f13fbad..5a899f32ba 100644 --- a/target/linux/realtek/files-5.15/drivers/net/dsa/rtl83xx/rtl930x.c +++ b/target/linux/realtek/files-5.15/drivers/net/dsa/rtl83xx/rtl930x.c @@ -21,6 +21,8 @@ #define RTL930X_VLAN_PORT_TAG_STS_CTRL_IGR_P_OTAG_KEEP_MASK GENMASK(1,1) #define RTL930X_VLAN_PORT_TAG_STS_CTRL_IGR_P_ITAG_KEEP_MASK GENMASK(0,0) +#define RTL930X_LED_GLB_ACTIVE_LOW BIT(22) + extern struct mutex smi_lock; extern struct rtl83xx_soc_info soc_info; @@ -2431,6 +2433,12 @@ static void rtl930x_led_init(struct rtl838x_switch_priv *priv) /* Set LED mode to serial (0x1) */ sw_w32_mask(0x3, 0x1, RTL930X_LED_GLB_CTRL); + /* Set LED active state */ + if (of_property_read_bool(node, "active-low")) + sw_w32_mask(RTL930X_LED_GLB_ACTIVE_LOW, 0, RTL930X_LED_GLB_CTRL); + else + sw_w32_mask(0, RTL930X_LED_GLB_ACTIVE_LOW, RTL930X_LED_GLB_CTRL); + /* Set port type masks */ sw_w32(pm, RTL930X_LED_PORT_COPR_MASK_CTRL); sw_w32(pm, RTL930X_LED_PORT_FIB_MASK_CTRL); diff --git a/target/linux/realtek/image/Makefile b/target/linux/realtek/image/Makefile index 3dfbcf67c1..a2afb39d40 100644 --- a/target/linux/realtek/image/Makefile +++ b/target/linux/realtek/image/Makefile @@ -4,7 +4,6 @@ include $(TOPDIR)/rules.mk include $(INCLUDE_DIR)/image.mk KERNEL_LOADADDR = 0x80100000 -KERNEL_ENTRY = 0x80100400 DEVICE_VARS += \ CAMEO_BOARD_VERSION \ diff --git a/target/linux/realtek/patches-5.10/300-mips-add-rtl838x-platform.patch b/target/linux/realtek/patches-5.10/300-mips-add-rtl838x-platform.patch index 3834ba7c61..f54bd28242 100644 --- a/target/linux/realtek/patches-5.10/300-mips-add-rtl838x-platform.patch +++ b/target/linux/realtek/patches-5.10/300-mips-add-rtl838x-platform.patch @@ -25,7 +25,7 @@ Submitted-by: Birger Koblitz platform-$(CONFIG_SGI_IP28) += sgi-ip22/ --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1037,8 +1037,58 @@ config NLM_XLP_BOARD +@@ -1037,8 +1037,59 @@ config NLM_XLP_BOARD This board is based on Netlogic XLP Processor. Say Y here if you have a XLP based board. @@ -33,6 +33,7 @@ Submitted-by: Birger Koblitz + bool "Realtek based platforms" + select DMA_NONCOHERENT + select IRQ_MIPS_CPU ++ select NO_EXCEPT_FILL + select SYS_HAS_CPU_MIPS32_R1 + select SYS_HAS_CPU_MIPS32_R2 + select SYS_SUPPORTS_BIG_ENDIAN diff --git a/target/linux/realtek/patches-5.10/312-rt9313-support.patch b/target/linux/realtek/patches-5.10/312-rt9313-support.patch index 516bca2b6e..7626cc9c25 100644 --- a/target/linux/realtek/patches-5.10/312-rt9313-support.patch +++ b/target/linux/realtek/patches-5.10/312-rt9313-support.patch @@ -43,23 +43,6 @@ Submitted-by: Birger Koblitz ifdef CONFIG_32BIT bootvars-y += ADDR_BITS=32 endif ---- a/arch/mips/kernel/head.S -+++ b/arch/mips/kernel/head.S -@@ -60,12 +60,14 @@ - .endm - - #ifndef CONFIG_NO_EXCEPT_FILL -+#ifndef CONFIG_RTL931X - /* - * Reserved space for exception handlers. - * Necessary for machines which link their kernels at KSEG0. - */ - .fill 0x400 - #endif -+#endif - - EXPORT(_stext) - --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -55,7 +55,11 @@ SECTIONS diff --git a/target/linux/realtek/patches-5.15/300-mips-add-rtl838x-platform.patch b/target/linux/realtek/patches-5.15/300-mips-add-rtl838x-platform.patch index 3929096888..8428c32e4e 100644 --- a/target/linux/realtek/patches-5.15/300-mips-add-rtl838x-platform.patch +++ b/target/linux/realtek/patches-5.15/300-mips-add-rtl838x-platform.patch @@ -25,7 +25,7 @@ Submitted-by: Birger Koblitz platform-$(CONFIG_SGI_IP28) += sgi-ip22/ --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1053,8 +1053,58 @@ config NLM_XLP_BOARD +@@ -1053,8 +1053,59 @@ config NLM_XLP_BOARD This board is based on Netlogic XLP Processor. Say Y here if you have a XLP based board. @@ -33,6 +33,7 @@ Submitted-by: Birger Koblitz + bool "Realtek based platforms" + select DMA_NONCOHERENT + select IRQ_MIPS_CPU ++ select NO_EXCEPT_FILL + select SYS_HAS_CPU_MIPS32_R1 + select SYS_HAS_CPU_MIPS32_R2 + select SYS_SUPPORTS_BIG_ENDIAN diff --git a/target/linux/realtek/patches-5.15/312-rt9313-support.patch b/target/linux/realtek/patches-5.15/312-rt9313-support.patch index 67f5580cd6..e8edc996f3 100644 --- a/target/linux/realtek/patches-5.15/312-rt9313-support.patch +++ b/target/linux/realtek/patches-5.15/312-rt9313-support.patch @@ -43,23 +43,6 @@ Submitted-by: Birger Koblitz ifdef CONFIG_32BIT bootvars-y += ADDR_BITS=32 endif ---- a/arch/mips/kernel/head.S -+++ b/arch/mips/kernel/head.S -@@ -60,12 +60,14 @@ - .endm - - #ifndef CONFIG_NO_EXCEPT_FILL -+#ifndef CONFIG_RTL931X - /* - * Reserved space for exception handlers. - * Necessary for machines which link their kernels at KSEG0. - */ - .fill 0x400 - #endif -+#endif - - EXPORT(_stext) - --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -55,7 +55,11 @@ SECTIONS diff --git a/target/linux/realtek/rtl838x/config-5.10 b/target/linux/realtek/rtl838x/config-5.10 index 982dd6cb8f..9749eec27b 100644 --- a/target/linux/realtek/rtl838x/config-5.10 +++ b/target/linux/realtek/rtl838x/config-5.10 @@ -164,6 +164,7 @@ CONFIG_NET_DSA_RTL83XX=y CONFIG_NET_DSA_TAG_TRAILER=y CONFIG_NET_RTL838X=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NVMEM=y CONFIG_OF=y diff --git a/target/linux/realtek/rtl838x/config-5.15 b/target/linux/realtek/rtl838x/config-5.15 index 326fee1651..4e4ed9f44b 100644 --- a/target/linux/realtek/rtl838x/config-5.15 +++ b/target/linux/realtek/rtl838x/config-5.15 @@ -161,6 +161,7 @@ CONFIG_NET_DSA_TAG_TRAILER=y CONFIG_NET_RTL838X=y CONFIG_NET_SELFTESTS=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NVMEM=y CONFIG_OF=y diff --git a/target/linux/realtek/rtl839x/config-5.10 b/target/linux/realtek/rtl839x/config-5.10 index ad2ed1e590..9377d482d7 100644 --- a/target/linux/realtek/rtl839x/config-5.10 +++ b/target/linux/realtek/rtl839x/config-5.10 @@ -160,6 +160,7 @@ CONFIG_NET_DSA_RTL83XX=y CONFIG_NET_DSA_TAG_TRAILER=y CONFIG_NET_RTL838X=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NVMEM=y CONFIG_OF=y diff --git a/target/linux/realtek/rtl839x/config-5.15 b/target/linux/realtek/rtl839x/config-5.15 index b16476c3c6..17edb2b3bb 100644 --- a/target/linux/realtek/rtl839x/config-5.15 +++ b/target/linux/realtek/rtl839x/config-5.15 @@ -164,6 +164,7 @@ CONFIG_NET_FLOW_LIMIT=y CONFIG_NET_RTL838X=y CONFIG_NET_SELFTESTS=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NR_CPUS=2 CONFIG_NVMEM=y diff --git a/target/linux/realtek/rtl930x/config-5.10 b/target/linux/realtek/rtl930x/config-5.10 index 4c1a4ffaf2..bd275a28ba 100644 --- a/target/linux/realtek/rtl930x/config-5.10 +++ b/target/linux/realtek/rtl930x/config-5.10 @@ -150,6 +150,7 @@ CONFIG_NET_DSA_RTL83XX=y CONFIG_NET_DSA_TAG_TRAILER=y CONFIG_NET_RTL838X=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NVMEM=y CONFIG_OF=y diff --git a/target/linux/realtek/rtl930x/config-5.15 b/target/linux/realtek/rtl930x/config-5.15 index 7804bcc83d..5e03616678 100644 --- a/target/linux/realtek/rtl930x/config-5.15 +++ b/target/linux/realtek/rtl930x/config-5.15 @@ -144,6 +144,7 @@ CONFIG_NET_DSA_TAG_TRAILER=y CONFIG_NET_RTL838X=y CONFIG_NET_SELFTESTS=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NVMEM=y CONFIG_OF=y diff --git a/target/linux/realtek/rtl931x/config-5.10 b/target/linux/realtek/rtl931x/config-5.10 index fb7443a865..1a6546c031 100644 --- a/target/linux/realtek/rtl931x/config-5.10 +++ b/target/linux/realtek/rtl931x/config-5.10 @@ -149,6 +149,7 @@ CONFIG_NET_DSA_TAG_RTL83XX=y CONFIG_NET_DSA_TAG_TRAILER=y CONFIG_NET_RTL838X=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NVMEM=y CONFIG_OF=y diff --git a/target/linux/realtek/rtl931x/config-5.15 b/target/linux/realtek/rtl931x/config-5.15 index 0c149bdf6d..3abc49a4e9 100644 --- a/target/linux/realtek/rtl931x/config-5.15 +++ b/target/linux/realtek/rtl931x/config-5.15 @@ -159,6 +159,7 @@ CONFIG_NET_FLOW_LIMIT=y CONFIG_NET_RTL838X=y CONFIG_NET_SELFTESTS=y CONFIG_NET_SWITCHDEV=y +CONFIG_NO_EXCEPT_FILL=y CONFIG_NO_GENERIC_PCI_IOPORT_MAP=y CONFIG_NR_CPUS=2 CONFIG_NVMEM=y diff --git a/tools/cmake/Makefile b/tools/cmake/Makefile index b7dadee733..ed2580fe4c 100644 --- a/tools/cmake/Makefile +++ b/tools/cmake/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=cmake -PKG_VERSION:=3.25.1 +PKG_VERSION:=3.25.2 PKG_VERSION_MAJOR:=$(word 1,$(subst ., ,$(PKG_VERSION))).$(word 2,$(subst ., ,$(PKG_VERSION))) PKG_RELEASE:=1 PKG_CPE_ID:=cpe:/a:kitware:cmake @@ -15,7 +15,7 @@ PKG_CPE_ID:=cpe:/a:kitware:cmake PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://github.com/Kitware/CMake/releases/download/v$(PKG_VERSION)/ \ https://cmake.org/files/v$(PKG_VERSION_MAJOR)/ -PKG_HASH:=1c511d09516af493694ed9baf13c55947a36389674d657a2d5e0ccedc6b291d8 +PKG_HASH:=c026f22cb931dd532f648f087d587f07a1843c6e66a3dfca4fb0ea21944ed33c HOST_BUILD_PARALLEL:=1 HOST_CONFIGURE_PARALLEL:=1 diff --git a/tools/ninja/Makefile b/tools/ninja/Makefile index c5c83d9b14..4763e759d8 100644 --- a/tools/ninja/Makefile +++ b/tools/ninja/Makefile @@ -1,12 +1,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ninja -PKG_VERSION:=1.11.0 +PKG_VERSION:=1.11.1 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/ninja-build/ninja/tar.gz/v$(PKG_VERSION)? -PKG_HASH:=3c6ba2e66400fe3f1ae83deb4b235faf3137ec20bd5b08c29bfc368db143e4c6 +PKG_HASH:=31747ae633213f1eda3842686f83c2aa1412e0f5691d1c14dbbcc67fe7400cea include $(INCLUDE_DIR)/host-build.mk diff --git a/tools/ninja/patches/100-make_jobserver_support.patch b/tools/ninja/patches/100-make_jobserver_support.patch index 7dac8ef814..34d2b6c431 100644 --- a/tools/ninja/patches/100-make_jobserver_support.patch +++ b/tools/ninja/patches/100-make_jobserver_support.patch @@ -1,4 +1,4 @@ -From 17d13fd7881fd3ce9f9b9d44ce435d6caf4b8f28 Mon Sep 17 00:00:00 2001 +From afec30f5caf4b051827ffdd822ebd27c58219fee Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Tue, 22 Mar 2016 13:48:07 +0200 Subject: [PATCH 01/11] Add GNU make jobserver client support @@ -31,28 +31,41 @@ Fixes https://github.com/ninja-build/ninja/issues/1139 create mode 100644 src/tokenpool-none.cc create mode 100644 src/tokenpool.h -diff --git a/configure.py b/configure.py -index 43904349a8..db3492c93c 100755 --- a/configure.py +++ b/configure.py -@@ -522,6 +522,7 @@ def has_re2c(): +@@ -517,11 +517,13 @@ for name in ['build', + 'state', + 'status', + 'string_piece_util', ++ 'tokenpool-gnu-make', + 'util', + 'version']: objs += cxx(name, variables=cxxvariables) if platform.is_windows(): for name in ['subprocess-win32', -+ 'tokenpool-none', ++ 'tokenpool-gnu-make-win32', 'includes_normalize-win32', 'msvc_helper-win32', 'msvc_helper_main-win32']: -@@ -531,6 +532,7 @@ def has_re2c(): +@@ -530,7 +532,9 @@ if platform.is_windows(): + objs += cxx('minidump-win32', variables=cxxvariables) objs += cc('getopt') else: - objs += cxx('subprocess-posix') -+ objs += cxx('tokenpool-gnu-make') +- objs += cxx('subprocess-posix') ++ for name in ['subprocess-posix', ++ 'tokenpool-gnu-make-posix']: ++ objs += cxx(name) if platform.is_aix(): objs += cc('getopt') if platform.is_msvc(): -diff --git a/src/build.cc b/src/build.cc -index 6f11ed7a3c..fa096eac33 100644 +@@ -588,6 +592,7 @@ for name in ['build_log_test', + 'string_piece_util_test', + 'subprocess_test', + 'test', ++ 'tokenpool_test', + 'util_test']: + objs += cxx(name, variables=cxxvariables) + if platform.is_windows(): --- a/src/build.cc +++ b/src/build.cc @@ -35,6 +35,7 @@ @@ -63,7 +76,36 @@ index 6f11ed7a3c..fa096eac33 100644 #include "util.h" using namespace std; -@@ -149,7 +150,7 @@ void Plan::EdgeWanted(const Edge* edge) { +@@ -47,8 +48,9 @@ struct DryRunCommandRunner : public Comm + + // Overridden from CommandRunner: + virtual bool CanRunMore() const; ++ virtual bool AcquireToken(); + virtual bool StartCommand(Edge* edge); +- virtual bool WaitForCommand(Result* result); ++ virtual bool WaitForCommand(Result* result, bool more_ready); + + private: + queue finished_; +@@ -58,12 +60,16 @@ bool DryRunCommandRunner::CanRunMore() c + return true; + } + ++bool DryRunCommandRunner::AcquireToken() { ++ return true; ++} ++ + bool DryRunCommandRunner::StartCommand(Edge* edge) { + finished_.push(edge); + return true; + } + +-bool DryRunCommandRunner::WaitForCommand(Result* result) { ++bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) { + if (finished_.empty()) + return false; + +@@ -149,7 +155,7 @@ void Plan::EdgeWanted(const Edge* edge) } Edge* Plan::FindWork() { @@ -72,7 +114,7 @@ index 6f11ed7a3c..fa096eac33 100644 return NULL; EdgeSet::iterator e = ready_.begin(); Edge* edge = *e; -@@ -448,8 +449,8 @@ void Plan::Dump() const { +@@ -448,19 +454,39 @@ void Plan::Dump() const { } struct RealCommandRunner : public CommandRunner { @@ -81,18 +123,31 @@ index 6f11ed7a3c..fa096eac33 100644 + explicit RealCommandRunner(const BuildConfig& config); + virtual ~RealCommandRunner(); virtual bool CanRunMore() const; ++ virtual bool AcquireToken(); virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); -@@ -458,9 +459,18 @@ struct RealCommandRunner : public CommandRunner { +- virtual bool WaitForCommand(Result* result); ++ virtual bool WaitForCommand(Result* result, bool more_ready); + virtual vector GetActiveEdges(); + virtual void Abort(); const BuildConfig& config_; ++ // copy of config_.max_load_average; can be modified by TokenPool setup ++ double max_load_average_; SubprocessSet subprocs_; -+ TokenPool *tokens_; ++ TokenPool* tokens_; map subproc_to_edge_; }; +RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { -+ tokens_ = TokenPool::Get(); ++ max_load_average_ = config.max_load_average; ++ if ((tokens_ = TokenPool::Get()) != NULL) { ++ if (!tokens_->Setup(config_.parallelism_from_cmdline, ++ config_.verbosity == BuildConfig::VERBOSE, ++ max_load_average_)) { ++ delete tokens_; ++ tokens_ = NULL; ++ } ++ } +} + +RealCommandRunner::~RealCommandRunner() { @@ -102,7 +157,7 @@ index 6f11ed7a3c..fa096eac33 100644 vector RealCommandRunner::GetActiveEdges() { vector edges; for (map::iterator e = subproc_to_edge_.begin(); -@@ -471,14 +481,18 @@ vector RealCommandRunner::GetActiveEdges() { +@@ -471,14 +497,23 @@ vector RealCommandRunner::GetActi void RealCommandRunner::Abort() { subprocs_.Clear(); @@ -111,19 +166,27 @@ index 6f11ed7a3c..fa096eac33 100644 } bool RealCommandRunner::CanRunMore() const { - size_t subproc_number = - subprocs_.running_.size() + subprocs_.finished_.size(); - return (int)subproc_number < config_.parallelism +- size_t subproc_number = +- subprocs_.running_.size() + subprocs_.finished_.size(); +- return (int)subproc_number < config_.parallelism - && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) - || GetLoadAverage() < config_.max_load_average); ++ bool parallelism_limit_not_reached = ++ tokens_ || // ignore config_.parallelism ++ ((int) (subprocs_.running_.size() + ++ subprocs_.finished_.size()) < config_.parallelism); ++ return parallelism_limit_not_reached + && (subprocs_.running_.empty() || -+ ((config_.max_load_average <= 0.0f || -+ GetLoadAverage() < config_.max_load_average) -+ && (!tokens_ || tokens_->Acquire()))); ++ (max_load_average_ <= 0.0f || ++ GetLoadAverage() < max_load_average_)); ++} ++ ++bool RealCommandRunner::AcquireToken() { ++ return (!tokens_ || tokens_->Acquire()); } bool RealCommandRunner::StartCommand(Edge* edge) { -@@ -486,6 +500,8 @@ bool RealCommandRunner::StartCommand(Edge* edge) { +@@ -486,19 +521,33 @@ bool RealCommandRunner::StartCommand(Edg Subprocess* subproc = subprocs_.Add(command, edge->use_console()); if (!subproc) return false; @@ -132,25 +195,50 @@ index 6f11ed7a3c..fa096eac33 100644 subproc_to_edge_.insert(make_pair(subproc, edge)); return true; -@@ -499,6 +515,9 @@ bool RealCommandRunner::WaitForCommand(Result* result) { + } + +-bool RealCommandRunner::WaitForCommand(Result* result) { ++bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) { + Subprocess* subproc; +- while ((subproc = subprocs_.NextFinished()) == NULL) { +- bool interrupted = subprocs_.DoWork(); ++ subprocs_.ResetTokenAvailable(); ++ while (((subproc = subprocs_.NextFinished()) == NULL) && ++ !subprocs_.IsTokenAvailable()) { ++ bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL); + if (interrupted) return false; } ++ // token became available ++ if (subproc == NULL) { ++ result->status = ExitTokenAvailable; ++ return true; ++ } ++ ++ // command completed + if (tokens_) + tokens_->Release(); + result->status = subproc->Finish(); result->output = subproc->GetOutput(); -@@ -621,31 +640,31 @@ bool Builder::Build(string* err) { +@@ -620,38 +669,43 @@ bool Builder::Build(string* err) { + // command runner. // Second, we attempt to wait for / reap the next finished command. while (plan_.more_to_do()) { - // See if we can start any more commands. +- // See if we can start any more commands. - if (failures_allowed && command_runner_->CanRunMore()) { - if (Edge* edge = plan_.FindWork()) { - if (edge->GetBindingBool("generator")) { -+ if (failures_allowed && plan_.more_ready() && -+ command_runner_->CanRunMore()) { ++ // See if we can start any more commands... ++ bool can_run_more = ++ failures_allowed && ++ plan_.more_ready() && ++ command_runner_->CanRunMore(); ++ ++ // ... but we also need a token to do that. ++ if (can_run_more && command_runner_->AcquireToken()) { + Edge* edge = plan_.FindWork(); + if (edge->GetBindingBool("generator")) { scan_.build_log()->Close(); @@ -191,8 +279,24 @@ index 6f11ed7a3c..fa096eac33 100644 } // See if we can reap any finished commands. -diff --git a/src/build.h b/src/build.h -index d697dfb89e..7dcd111e61 100644 + if (pending_commands) { + CommandRunner::Result result; +- if (!command_runner_->WaitForCommand(&result) || ++ if (!command_runner_->WaitForCommand(&result, can_run_more) || + result.status == ExitInterrupted) { + Cleanup(); + status_->BuildFinished(); +@@ -659,6 +713,10 @@ bool Builder::Build(string* err) { + return false; + } + ++ // We might be able to start another command; start the main loop over. ++ if (result.status == ExitTokenAvailable) ++ continue; ++ + --pending_commands; + if (!FinishCommand(&result, err)) { + Cleanup(); --- a/src/build.h +++ b/src/build.h @@ -52,6 +52,9 @@ struct Plan { @@ -205,13 +309,47 @@ index d697dfb89e..7dcd111e61 100644 /// Dumps the current state of the plan. void Dump() const; -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -new file mode 100644 -index 0000000000..a8f9b7139d +@@ -136,6 +139,7 @@ private: + struct CommandRunner { + virtual ~CommandRunner() {} + virtual bool CanRunMore() const = 0; ++ virtual bool AcquireToken() = 0; + virtual bool StartCommand(Edge* edge) = 0; + + /// The result of waiting for a command. +@@ -147,7 +151,9 @@ struct CommandRunner { + bool success() const { return status == ExitSuccess; } + }; + /// Wait for a command to complete, or return false if interrupted. +- virtual bool WaitForCommand(Result* result) = 0; ++ /// If more_ready is true then the optional TokenPool is monitored too ++ /// and we return when a token becomes available. ++ virtual bool WaitForCommand(Result* result, bool more_ready) = 0; + + virtual std::vector GetActiveEdges() { return std::vector(); } + virtual void Abort() {} +@@ -155,7 +161,8 @@ struct CommandRunner { + + /// Options (e.g. verbosity, parallelism) passed to a build. + struct BuildConfig { +- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), ++ BuildConfig() : verbosity(NORMAL), dry_run(false), ++ parallelism(1), parallelism_from_cmdline(false), + failures_allowed(1), max_load_average(-0.0f) {} + + enum Verbosity { +@@ -167,6 +174,7 @@ struct BuildConfig { + Verbosity verbosity; + bool dry_run; + int parallelism; ++ bool parallelism_from_cmdline; + int failures_allowed; + /// The maximum load average we must not exceed. A negative value + /// means that we do not have any limit. --- /dev/null +++ b/src/tokenpool-gnu-make.cc -@@ -0,0 +1,211 @@ -+// Copyright 2016 Google Inc. All Rights Reserved. +@@ -0,0 +1,108 @@ ++// Copyright 2016-2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. @@ -225,111 +363,55 @@ index 0000000000..a8f9b7139d +// See the License for the specific language governing permissions and +// limitations under the License. + -+#include "tokenpool.h" ++#include "tokenpool-gnu-make.h" + -+#include -+#include -+#include -+#include -+#include ++#include +#include +#include -+#include + -+// TokenPool implementation for GNU make jobserver -+// (http://make.mad-scientist.net/papers/jobserver-implementation/) -+struct GNUmakeTokenPool : public TokenPool { -+ GNUmakeTokenPool(); -+ virtual ~GNUmakeTokenPool(); -+ -+ virtual bool Acquire(); -+ virtual void Reserve(); -+ virtual void Release(); -+ virtual void Clear(); -+ -+ bool Setup(); -+ -+ private: -+ int available_; -+ int used_; -+ -+#ifdef _WIN32 -+ // @TODO -+#else -+ int rfd_; -+ int wfd_; -+ -+ struct sigaction old_act_; -+ bool restore_; -+ -+ static int dup_rfd_; -+ static void CloseDupRfd(int signum); -+ -+ bool CheckFd(int fd); -+ bool SetAlarmHandler(); -+#endif -+ -+ void Return(); -+}; ++#include "line_printer.h" + ++// TokenPool implementation for GNU make jobserver - common bits +// every instance owns an implicit token -> available_ == 1 -+GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0), -+ rfd_(-1), wfd_(-1), restore_(false) { ++GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) { +} + +GNUmakeTokenPool::~GNUmakeTokenPool() { -+ Clear(); -+ if (restore_) -+ sigaction(SIGALRM, &old_act_, NULL); +} + -+bool GNUmakeTokenPool::CheckFd(int fd) { -+ if (fd < 0) ++bool GNUmakeTokenPool::Setup(bool ignore, ++ bool verbose, ++ double& max_load_average) { ++ const char* value = GetEnv("MAKEFLAGS"); ++ if (!value) + return false; -+ int ret = fcntl(fd, F_GETFD); -+ if (ret < 0) -+ return false; -+ return true; -+} + -+int GNUmakeTokenPool::dup_rfd_ = -1; -+ -+void GNUmakeTokenPool::CloseDupRfd(int signum) { -+ close(dup_rfd_); -+ dup_rfd_ = -1; -+} -+ -+bool GNUmakeTokenPool::SetAlarmHandler() { -+ struct sigaction act; -+ memset(&act, 0, sizeof(act)); -+ act.sa_handler = CloseDupRfd; -+ if (sigaction(SIGALRM, &act, &old_act_) < 0) { -+ perror("sigaction:"); -+ return(false); -+ } else { -+ restore_ = true; -+ return(true); -+ } -+} -+ -+bool GNUmakeTokenPool::Setup() { -+ const char *value = getenv("MAKEFLAGS"); -+ if (value) { -+ // GNU make <= 4.1 -+ const char *jobserver = strstr(value, "--jobserver-fds="); ++ // GNU make <= 4.1 ++ const char* jobserver = strstr(value, "--jobserver-fds="); ++ if (!jobserver) + // GNU make => 4.2 -+ if (!jobserver) -+ jobserver = strstr(value, "--jobserver-auth="); -+ if (jobserver) { -+ int rfd = -1; -+ int wfd = -1; -+ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && -+ CheckFd(rfd) && -+ CheckFd(wfd) && -+ SetAlarmHandler()) { -+ printf("ninja: using GNU make jobserver.\n"); -+ rfd_ = rfd; -+ wfd_ = wfd; ++ jobserver = strstr(value, "--jobserver-auth="); ++ if (jobserver) { ++ LinePrinter printer; ++ ++ if (ignore) { ++ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ } else { ++ if (ParseAuth(jobserver)) { ++ const char* l_arg = strstr(value, " -l"); ++ int load_limit = -1; ++ ++ if (verbose) { ++ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); ++ } ++ ++ // translate GNU make -lN to ninja -lN ++ if (l_arg && ++ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && ++ (load_limit > 0)) { ++ max_load_average = load_limit; ++ } ++ + return true; + } + } @@ -342,44 +424,13 @@ index 0000000000..a8f9b7139d + if (available_ > 0) + return true; + -+#ifdef USE_PPOLL -+ pollfd pollfds[] = {{rfd_, POLLIN, 0}}; -+ int ret = poll(pollfds, 1, 0); -+#else -+ fd_set set; -+ struct timeval timeout = { 0, 0 }; -+ FD_ZERO(&set); -+ FD_SET(rfd_, &set); -+ int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); -+#endif -+ if (ret > 0) { -+ dup_rfd_ = dup(rfd_); -+ -+ if (dup_rfd_ != -1) { -+ struct sigaction act, old_act; -+ int ret = 0; -+ -+ memset(&act, 0, sizeof(act)); -+ act.sa_handler = CloseDupRfd; -+ if (sigaction(SIGCHLD, &act, &old_act) == 0) { -+ char buf; -+ -+ // block until token read, child exits or timeout -+ alarm(1); -+ ret = read(dup_rfd_, &buf, 1); -+ alarm(0); -+ -+ sigaction(SIGCHLD, &old_act, NULL); -+ } -+ -+ CloseDupRfd(0); -+ -+ if (ret > 0) { -+ available_++; -+ return true; -+ } -+ } ++ if (AcquireToken()) { ++ // token acquired ++ available_++; ++ return true; + } ++ ++ // no token available + return false; +} + @@ -389,15 +440,8 @@ index 0000000000..a8f9b7139d +} + +void GNUmakeTokenPool::Return() { -+ const char buf = '+'; -+ while (1) { -+ int ret = write(wfd_, &buf, 1); -+ if (ret > 0) -+ available_--; -+ if ((ret != -1) || (errno != EINTR)) -+ return; -+ // write got interrupted - retry -+ } ++ if (ReturnToken()) ++ available_--; +} + +void GNUmakeTokenPool::Release() { @@ -413,55 +457,10 @@ index 0000000000..a8f9b7139d + while (available_ > 1) + Return(); +} -+ -+struct TokenPool *TokenPool::Get(void) { -+ GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; -+ if (tokenpool->Setup()) -+ return tokenpool; -+ else -+ delete tokenpool; -+ return NULL; -+} -diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc -new file mode 100644 -index 0000000000..602b3316f5 ---- /dev/null -+++ b/src/tokenpool-none.cc -@@ -0,0 +1,27 @@ -+// Copyright 2016 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ -+#include "tokenpool.h" -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+// No-op TokenPool implementation -+struct TokenPool *TokenPool::Get(void) { -+ return NULL; -+} -diff --git a/src/tokenpool.h b/src/tokenpool.h -new file mode 100644 -index 0000000000..f560b1083b --- /dev/null +++ b/src/tokenpool.h -@@ -0,0 +1,26 @@ -+// Copyright 2016 Google Inc. All Rights Reserved. +@@ -0,0 +1,42 @@ ++// Copyright 2016-2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. @@ -475,6 +474,10 @@ index 0000000000..f560b1083b +// See the License for the specific language governing permissions and +// limitations under the License. + ++#ifdef _WIN32 ++#include ++#endif ++ +// interface to token pool +struct TokenPool { + virtual ~TokenPool() {} @@ -484,1532 +487,21 @@ index 0000000000..f560b1083b + virtual void Release() = 0; + virtual void Clear() = 0; + -+ // returns NULL if token pool is not available -+ static struct TokenPool *Get(void); -+}; - -From ccaccc610cd456f6068758f82e72006364c7380b Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Fri, 27 May 2016 16:47:10 +0300 -Subject: [PATCH 02/11] Add TokenPool monitoring to SubprocessSet::DoWork() - -Improve on the original jobserver client implementation. This makes -ninja a more aggressive GNU make jobserver client. - -- add monitor interface to TokenPool -- TokenPool is passed down when main loop indicates that more work is - ready and would be allowed to start if a token becomes available -- posix: update DoWork() to monitor TokenPool read file descriptor -- WaitForCommand() exits when DoWork() sets token flag -- Main loop starts over when WaitForCommand() sets token exit status ---- - src/build.cc | 53 +++++++++++++++++++++++++++++---------- - src/build.h | 3 ++- - src/build_test.cc | 9 +++++-- - src/exit_status.h | 3 ++- - src/subprocess-posix.cc | 33 ++++++++++++++++++++++-- - src/subprocess-win32.cc | 2 +- - src/subprocess.h | 8 +++++- - src/subprocess_test.cc | 47 +++++++++++++++++++++++----------- - src/tokenpool-gnu-make.cc | 5 ++++ - src/tokenpool.h | 6 +++++ - 10 files changed, 134 insertions(+), 35 deletions(-) - -diff --git a/src/build.cc b/src/build.cc -index fa096eac33..a25c349050 100644 ---- a/src/build.cc -+++ b/src/build.cc -@@ -48,8 +48,9 @@ struct DryRunCommandRunner : public CommandRunner { - - // Overridden from CommandRunner: - virtual bool CanRunMore() const; -+ virtual bool AcquireToken(); - virtual bool StartCommand(Edge* edge); -- virtual bool WaitForCommand(Result* result); -+ virtual bool WaitForCommand(Result* result, bool more_ready); - - private: - queue finished_; -@@ -59,12 +60,16 @@ bool DryRunCommandRunner::CanRunMore() const { - return true; - } - -+bool DryRunCommandRunner::AcquireToken() { -+ return true; -+} ++ // returns false if token pool setup failed ++ virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0; + - bool DryRunCommandRunner::StartCommand(Edge* edge) { - finished_.push(edge); - return true; - } - --bool DryRunCommandRunner::WaitForCommand(Result* result) { -+bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) { - if (finished_.empty()) - return false; - -@@ -452,8 +457,9 @@ struct RealCommandRunner : public CommandRunner { - explicit RealCommandRunner(const BuildConfig& config); - virtual ~RealCommandRunner(); - virtual bool CanRunMore() const; -+ virtual bool AcquireToken(); - virtual bool StartCommand(Edge* edge); -- virtual bool WaitForCommand(Result* result); -+ virtual bool WaitForCommand(Result* result, bool more_ready); - virtual vector GetActiveEdges(); - virtual void Abort(); - -@@ -490,9 +496,12 @@ bool RealCommandRunner::CanRunMore() const { - subprocs_.running_.size() + subprocs_.finished_.size(); - return (int)subproc_number < config_.parallelism - && (subprocs_.running_.empty() || -- ((config_.max_load_average <= 0.0f || -- GetLoadAverage() < config_.max_load_average) -- && (!tokens_ || tokens_->Acquire()))); -+ (config_.max_load_average <= 0.0f || -+ GetLoadAverage() < config_.max_load_average)); -+} -+ -+bool RealCommandRunner::AcquireToken() { -+ return (!tokens_ || tokens_->Acquire()); - } - - bool RealCommandRunner::StartCommand(Edge* edge) { -@@ -507,14 +516,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) { - return true; - } - --bool RealCommandRunner::WaitForCommand(Result* result) { -+bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) { - Subprocess* subproc; -- while ((subproc = subprocs_.NextFinished()) == NULL) { -- bool interrupted = subprocs_.DoWork(); -+ subprocs_.ResetTokenAvailable(); -+ while (((subproc = subprocs_.NextFinished()) == NULL) && -+ !subprocs_.IsTokenAvailable()) { -+ bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL); - if (interrupted) - return false; - } - -+ // token became available -+ if (subproc == NULL) { -+ result->status = ExitTokenAvailable; -+ return true; -+ } -+ -+ // command completed - if (tokens_) - tokens_->Release(); - -@@ -639,9 +657,14 @@ bool Builder::Build(string* err) { - // command runner. - // Second, we attempt to wait for / reap the next finished command. - while (plan_.more_to_do()) { -- // See if we can start any more commands. -- if (failures_allowed && plan_.more_ready() && -- command_runner_->CanRunMore()) { -+ // See if we can start any more commands... -+ bool can_run_more = -+ failures_allowed && -+ plan_.more_ready() && -+ command_runner_->CanRunMore(); -+ -+ // ... but we also need a token to do that. -+ if (can_run_more && command_runner_->AcquireToken()) { - Edge* edge = plan_.FindWork(); - if (edge->GetBindingBool("generator")) { - scan_.build_log()->Close(); -@@ -670,7 +693,7 @@ bool Builder::Build(string* err) { - // See if we can reap any finished commands. - if (pending_commands) { - CommandRunner::Result result; -- if (!command_runner_->WaitForCommand(&result) || -+ if (!command_runner_->WaitForCommand(&result, can_run_more) || - result.status == ExitInterrupted) { - Cleanup(); - status_->BuildFinished(); -@@ -678,6 +701,10 @@ bool Builder::Build(string* err) { - return false; - } - -+ // We might be able to start another command; start the main loop over. -+ if (result.status == ExitTokenAvailable) -+ continue; -+ - --pending_commands; - if (!FinishCommand(&result, err)) { - Cleanup(); -diff --git a/src/build.h b/src/build.h -index 7dcd111e61..35c7b97d12 100644 ---- a/src/build.h -+++ b/src/build.h -@@ -139,6 +139,7 @@ struct Plan { - struct CommandRunner { - virtual ~CommandRunner() {} - virtual bool CanRunMore() const = 0; -+ virtual bool AcquireToken() = 0; - virtual bool StartCommand(Edge* edge) = 0; - - /// The result of waiting for a command. -@@ -150,7 +151,7 @@ struct CommandRunner { - bool success() const { return status == ExitSuccess; } - }; - /// Wait for a command to complete, or return false if interrupted. -- virtual bool WaitForCommand(Result* result) = 0; -+ virtual bool WaitForCommand(Result* result, bool more_ready) = 0; - - virtual std::vector GetActiveEdges() { return std::vector(); } - virtual void Abort() {} -diff --git a/src/build_test.cc b/src/build_test.cc -index 4ef62b2113..7a5ff4015a 100644 ---- a/src/build_test.cc -+++ b/src/build_test.cc -@@ -474,8 +474,9 @@ struct FakeCommandRunner : public CommandRunner { - - // CommandRunner impl - virtual bool CanRunMore() const; -+ virtual bool AcquireToken(); - virtual bool StartCommand(Edge* edge); -- virtual bool WaitForCommand(Result* result); -+ virtual bool WaitForCommand(Result* result, bool more_ready); - virtual vector GetActiveEdges(); - virtual void Abort(); - -@@ -578,6 +579,10 @@ bool FakeCommandRunner::CanRunMore() const { - return active_edges_.size() < max_active_edges_; - } - -+bool FakeCommandRunner::AcquireToken() { -+ return true; -+} -+ - bool FakeCommandRunner::StartCommand(Edge* edge) { - assert(active_edges_.size() < max_active_edges_); - assert(find(active_edges_.begin(), active_edges_.end(), edge) -@@ -649,7 +654,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { - return true; - } - --bool FakeCommandRunner::WaitForCommand(Result* result) { -+bool FakeCommandRunner::WaitForCommand(Result* result, bool more_ready) { - if (active_edges_.empty()) - return false; - -diff --git a/src/exit_status.h b/src/exit_status.h -index a714ece791..75ebf6a7a0 100644 ---- a/src/exit_status.h -+++ b/src/exit_status.h -@@ -18,7 +18,8 @@ - enum ExitStatus { - ExitSuccess, - ExitFailure, -- ExitInterrupted -+ ExitTokenAvailable, -+ ExitInterrupted, - }; - - #endif // NINJA_EXIT_STATUS_H_ -diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc -index 8e785406c9..74451b0be2 100644 ---- a/src/subprocess-posix.cc -+++ b/src/subprocess-posix.cc -@@ -13,6 +13,7 @@ - // limitations under the License. - - #include "subprocess.h" -+#include "tokenpool.h" - - #include - #include -@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { - } - - #ifdef USE_PPOLL --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(struct TokenPool* tokens) { - vector fds; - nfds_t nfds = 0; - -@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() { - ++nfds; - } - -+ if (tokens) { -+ pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 }; -+ fds.push_back(pfd); -+ ++nfds; -+ } -+ - interrupted_ = 0; - int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); - if (ret == -1) { -@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() { - ++i; - } - -+ if (tokens) { -+ pollfd *pfd = &fds[nfds - 1]; -+ if (pfd->fd >= 0) { -+ assert(pfd->fd == tokens->GetMonitorFd()); -+ if (pfd->revents != 0) -+ token_available_ = true; -+ } -+ } -+ - return IsInterrupted(); - } - - #else // !defined(USE_PPOLL) --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(struct TokenPool* tokens) { - fd_set set; - int nfds = 0; - FD_ZERO(&set); -@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() { - } - } - -+ if (tokens) { -+ int fd = tokens->GetMonitorFd(); -+ FD_SET(fd, &set); -+ if (nfds < fd+1) -+ nfds = fd+1; -+ } -+ - interrupted_ = 0; - int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); - if (ret == -1) { -@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() { - ++i; - } - -+ if (tokens) { -+ int fd = tokens->GetMonitorFd(); -+ if ((fd >= 0) && FD_ISSET(fd, &set)) -+ token_available_ = true; -+ } -+ - return IsInterrupted(); - } - #endif // !defined(USE_PPOLL) -diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc -index ff3baaca7f..66d2c2c430 100644 ---- a/src/subprocess-win32.cc -+++ b/src/subprocess-win32.cc -@@ -251,7 +251,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { - return subprocess; - } - --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(struct TokenPool* tokens) { - DWORD bytes_read; - Subprocess* subproc; - OVERLAPPED* overlapped; -diff --git a/src/subprocess.h b/src/subprocess.h -index 9e3d2ee98f..9ea67ea477 100644 ---- a/src/subprocess.h -+++ b/src/subprocess.h -@@ -76,6 +76,8 @@ struct Subprocess { - friend struct SubprocessSet; - }; - -+struct TokenPool; -+ - /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses. - /// DoWork() waits for any state change in subprocesses; finished_ - /// is a queue of subprocesses as they finish. -@@ -84,13 +86,17 @@ struct SubprocessSet { - ~SubprocessSet(); - - Subprocess* Add(const std::string& command, bool use_console = false); -- bool DoWork(); -+ bool DoWork(struct TokenPool* tokens); - Subprocess* NextFinished(); - void Clear(); - - std::vector running_; - std::queue finished_; - -+ bool token_available_; -+ bool IsTokenAvailable() { return token_available_; } -+ void ResetTokenAvailable() { token_available_ = false; } -+ - #ifdef _WIN32 - static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType); - static HANDLE ioport_; -diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc -index 073fe86931..4bc8083e26 100644 ---- a/src/subprocess_test.cc -+++ b/src/subprocess_test.cc -@@ -45,10 +45,12 @@ TEST_F(SubprocessTest, BadCommandStderr) { - Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { - // Pretend we discovered that stderr was ready for writing. -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -59,10 +61,12 @@ TEST_F(SubprocessTest, NoSuchCommand) { - Subprocess* subproc = subprocs_.Add("ninja_no_such_command"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { - // Pretend we discovered that stderr was ready for writing. -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -78,9 +82,11 @@ TEST_F(SubprocessTest, InterruptChild) { - Subprocess* subproc = subprocs_.Add("kill -INT $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -90,7 +96,7 @@ TEST_F(SubprocessTest, InterruptParent) { - ASSERT_NE((Subprocess *) 0, subproc); - - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -102,9 +108,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) { - Subprocess* subproc = subprocs_.Add("kill -TERM $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -114,7 +122,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigTerm) { - ASSERT_NE((Subprocess *) 0, subproc); - - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -126,9 +134,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) { - Subprocess* subproc = subprocs_.Add("kill -HUP $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -138,7 +148,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigHup) { - ASSERT_NE((Subprocess *) 0, subproc); - - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -153,9 +163,11 @@ TEST_F(SubprocessTest, Console) { - subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true); - ASSERT_NE((Subprocess*)0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitSuccess, subproc->Finish()); - } -@@ -167,9 +179,11 @@ TEST_F(SubprocessTest, SetWithSingle) { - Subprocess* subproc = subprocs_.Add(kSimpleCommand); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_NE("", subproc->GetOutput()); - -@@ -200,12 +214,13 @@ TEST_F(SubprocessTest, SetWithMulti) { - ASSERT_EQ("", processes[i]->GetOutput()); - } - -+ subprocs_.ResetTokenAvailable(); - while (!processes[0]->Done() || !processes[1]->Done() || - !processes[2]->Done()) { - ASSERT_GT(subprocs_.running_.size(), 0u); -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -- -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - ASSERT_EQ(0u, subprocs_.running_.size()); - ASSERT_EQ(3u, subprocs_.finished_.size()); - -@@ -237,8 +252,10 @@ TEST_F(SubprocessTest, SetWithLots) { - ASSERT_NE((Subprocess *) 0, subproc); - procs.push_back(subproc); - } -+ subprocs_.ResetTokenAvailable(); - while (!subprocs_.running_.empty()) -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - for (size_t i = 0; i < procs.size(); ++i) { - ASSERT_EQ(ExitSuccess, procs[i]->Finish()); - ASSERT_NE("", procs[i]->GetOutput()); -@@ -254,9 +271,11 @@ TEST_F(SubprocessTest, SetWithLots) { - // that stdin is closed. - TEST_F(SubprocessTest, ReadStdin) { - Subprocess* subproc = subprocs_.Add("cat -"); -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_EQ(1u, subprocs_.finished_.size()); - } -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -index a8f9b7139d..396bb7d874 100644 ---- a/src/tokenpool-gnu-make.cc -+++ b/src/tokenpool-gnu-make.cc -@@ -33,6 +33,7 @@ struct GNUmakeTokenPool : public TokenPool { - virtual void Reserve(); - virtual void Release(); - virtual void Clear(); -+ virtual int GetMonitorFd(); - - bool Setup(); - -@@ -201,6 +202,10 @@ void GNUmakeTokenPool::Clear() { - Return(); - } - -+int GNUmakeTokenPool::GetMonitorFd() { -+ return(rfd_); -+} -+ - struct TokenPool *TokenPool::Get(void) { - GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; - if (tokenpool->Setup()) -diff --git a/src/tokenpool.h b/src/tokenpool.h -index f560b1083b..301e1998ee 100644 ---- a/src/tokenpool.h -+++ b/src/tokenpool.h -@@ -21,6 +21,12 @@ struct TokenPool { - virtual void Release() = 0; - virtual void Clear() = 0; - +#ifdef _WIN32 -+ // @TODO ++ virtual void WaitForTokenAvailability(HANDLE ioport) = 0; ++ // returns true if a token has become available ++ // key is result from GetQueuedCompletionStatus() ++ virtual bool TokenIsAvailable(ULONG_PTR key) = 0; +#else + virtual int GetMonitorFd() = 0; +#endif + - // returns NULL if token pool is not available - static struct TokenPool *Get(void); - }; - -From d09f3d77821b3b1fdf09fc0ef8e814907675eafb Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Sun, 12 Nov 2017 16:58:55 +0200 -Subject: [PATCH 03/11] Ignore jobserver when -jN is forced on command line - -This emulates the behaviour of GNU make. - -- add parallelism_from_cmdline flag to build configuration -- set the flag when -jN is given on command line -- pass the flag to TokenPool::Get() -- GNUmakeTokenPool::Setup() - * prints a warning when the flag is true and jobserver was detected - * returns false, i.e. jobserver will be ignored -- ignore config.parallelism in CanRunMore() when we have a valid - TokenPool, because it gets always initialized to a default when not - given on the command line ---- - src/build.cc | 10 ++++++---- - src/build.h | 4 +++- - src/ninja.cc | 1 + - src/tokenpool-gnu-make.cc | 34 +++++++++++++++++++--------------- - src/tokenpool-none.cc | 4 ++-- - src/tokenpool.h | 4 ++-- - 6 files changed, 33 insertions(+), 24 deletions(-) - -diff --git a/src/build.cc b/src/build.cc -index a25c349050..406a84ec39 100644 ---- a/src/build.cc -+++ b/src/build.cc -@@ -470,7 +470,7 @@ struct RealCommandRunner : public CommandRunner { - }; - - RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { -- tokens_ = TokenPool::Get(); -+ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); - } - - RealCommandRunner::~RealCommandRunner() { -@@ -492,9 +492,11 @@ void RealCommandRunner::Abort() { - } - - bool RealCommandRunner::CanRunMore() const { -- size_t subproc_number = -- subprocs_.running_.size() + subprocs_.finished_.size(); -- return (int)subproc_number < config_.parallelism -+ bool parallelism_limit_not_reached = -+ tokens_ || // ignore config_.parallelism -+ ((int) (subprocs_.running_.size() + -+ subprocs_.finished_.size()) < config_.parallelism); -+ return parallelism_limit_not_reached - && (subprocs_.running_.empty() || - (config_.max_load_average <= 0.0f || - GetLoadAverage() < config_.max_load_average)); -diff --git a/src/build.h b/src/build.h -index 35c7b97d12..dfde576573 100644 ---- a/src/build.h -+++ b/src/build.h -@@ -159,7 +159,8 @@ struct CommandRunner { - - /// Options (e.g. verbosity, parallelism) passed to a build. - struct BuildConfig { -- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), -+ BuildConfig() : verbosity(NORMAL), dry_run(false), -+ parallelism(1), parallelism_from_cmdline(false), - failures_allowed(1), max_load_average(-0.0f) {} - - enum Verbosity { -@@ -171,6 +172,7 @@ struct BuildConfig { - Verbosity verbosity; - bool dry_run; - int parallelism; -+ bool parallelism_from_cmdline; - int failures_allowed; - /// The maximum load average we must not exceed. A negative value - /// means that we do not have any limit. -diff --git a/src/ninja.cc b/src/ninja.cc -index df39ba92d1..d904c56c4e 100644 ---- a/src/ninja.cc -+++ b/src/ninja.cc -@@ -1447,6 +1447,7 @@ int ReadFlags(int* argc, char*** argv, - // We want to run N jobs in parallel. For N = 0, INT_MAX - // is close enough to infinite for most sane builds. - config->parallelism = value > 0 ? value : INT_MAX; -+ config->parallelism_from_cmdline = true; - deferGuessParallelism.needGuess = false; - break; - } -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -index 396bb7d874..af4be05a31 100644 ---- a/src/tokenpool-gnu-make.cc -+++ b/src/tokenpool-gnu-make.cc -@@ -1,4 +1,4 @@ --// Copyright 2016 Google Inc. All Rights Reserved. -+// Copyright 2016-2017 Google Inc. All Rights Reserved. - // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. -@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { - virtual void Clear(); - virtual int GetMonitorFd(); - -- bool Setup(); -+ bool Setup(bool ignore); - - private: - int available_; -@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { - } - } - --bool GNUmakeTokenPool::Setup() { -+bool GNUmakeTokenPool::Setup(bool ignore) { - const char *value = getenv("MAKEFLAGS"); - if (value) { - // GNU make <= 4.1 -@@ -109,16 +109,20 @@ bool GNUmakeTokenPool::Setup() { - if (!jobserver) - jobserver = strstr(value, "--jobserver-auth="); - if (jobserver) { -- int rfd = -1; -- int wfd = -1; -- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && -- CheckFd(rfd) && -- CheckFd(wfd) && -- SetAlarmHandler()) { -- printf("ninja: using GNU make jobserver.\n"); -- rfd_ = rfd; -- wfd_ = wfd; -- return true; -+ if (ignore) { -+ printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); -+ } else { -+ int rfd = -1; -+ int wfd = -1; -+ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && -+ CheckFd(rfd) && -+ CheckFd(wfd) && -+ SetAlarmHandler()) { -+ printf("ninja: using GNU make jobserver.\n"); -+ rfd_ = rfd; -+ wfd_ = wfd; -+ return true; -+ } - } - } - } -@@ -206,9 +210,9 @@ int GNUmakeTokenPool::GetMonitorFd() { - return(rfd_); - } - --struct TokenPool *TokenPool::Get(void) { -+struct TokenPool *TokenPool::Get(bool ignore) { - GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; -- if (tokenpool->Setup()) -+ if (tokenpool->Setup(ignore)) - return tokenpool; - else - delete tokenpool; -diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc -index 602b3316f5..199b22264b 100644 ---- a/src/tokenpool-none.cc -+++ b/src/tokenpool-none.cc -@@ -1,4 +1,4 @@ --// Copyright 2016 Google Inc. All Rights Reserved. -+// Copyright 2016-2017 Google Inc. All Rights Reserved. - // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. -@@ -22,6 +22,6 @@ - #include - - // No-op TokenPool implementation --struct TokenPool *TokenPool::Get(void) { -+struct TokenPool *TokenPool::Get(bool ignore) { - return NULL; - } -diff --git a/src/tokenpool.h b/src/tokenpool.h -index 301e1998ee..878a0933c2 100644 ---- a/src/tokenpool.h -+++ b/src/tokenpool.h -@@ -1,4 +1,4 @@ --// Copyright 2016 Google Inc. All Rights Reserved. -+// Copyright 2016-2017 Google Inc. All Rights Reserved. - // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. -@@ -28,5 +28,5 @@ struct TokenPool { - #endif - - // returns NULL if token pool is not available -- static struct TokenPool *Get(void); -+ static struct TokenPool *Get(bool ignore); - }; - -From dfe4ca753caee65bf9041e2b4e883dfa172a5c6a Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Sun, 12 Nov 2017 18:04:12 +0200 -Subject: [PATCH 04/11] Honor -lN from MAKEFLAGS - -This emulates the behaviour of GNU make. - -- build: make a copy of max_load_average and pass it to TokenPool. -- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in - MAKEFLAGS then set max_load_average to N. ---- - src/build.cc | 10 +++++++--- - src/tokenpool-gnu-make.cc | 19 +++++++++++++++---- - src/tokenpool-none.cc | 2 +- - src/tokenpool.h | 2 +- - 4 files changed, 24 insertions(+), 9 deletions(-) - -diff --git a/src/build.cc b/src/build.cc -index 406a84ec39..9e6272d035 100644 ---- a/src/build.cc -+++ b/src/build.cc -@@ -464,13 +464,17 @@ struct RealCommandRunner : public CommandRunner { - virtual void Abort(); - - const BuildConfig& config_; -+ // copy of config_.max_load_average; can be modified by TokenPool setup -+ double max_load_average_; - SubprocessSet subprocs_; - TokenPool *tokens_; - map subproc_to_edge_; - }; - - RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { -- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); -+ max_load_average_ = config.max_load_average; -+ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, -+ max_load_average_); - } - - RealCommandRunner::~RealCommandRunner() { -@@ -498,8 +502,8 @@ bool RealCommandRunner::CanRunMore() const { - subprocs_.finished_.size()) < config_.parallelism); - return parallelism_limit_not_reached - && (subprocs_.running_.empty() || -- (config_.max_load_average <= 0.0f || -- GetLoadAverage() < config_.max_load_average)); -+ (max_load_average_ <= 0.0f || -+ GetLoadAverage() < max_load_average_)); - } - - bool RealCommandRunner::AcquireToken() { -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -index af4be05a31..fb654c4d88 100644 ---- a/src/tokenpool-gnu-make.cc -+++ b/src/tokenpool-gnu-make.cc -@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { - virtual void Clear(); - virtual int GetMonitorFd(); - -- bool Setup(bool ignore); -+ bool Setup(bool ignore, double& max_load_average); - - private: - int available_; -@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { - } - } - --bool GNUmakeTokenPool::Setup(bool ignore) { -+bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { - const char *value = getenv("MAKEFLAGS"); - if (value) { - // GNU make <= 4.1 -@@ -118,9 +118,20 @@ bool GNUmakeTokenPool::Setup(bool ignore) { - CheckFd(rfd) && - CheckFd(wfd) && - SetAlarmHandler()) { -+ const char *l_arg = strstr(value, " -l"); -+ int load_limit = -1; -+ - printf("ninja: using GNU make jobserver.\n"); - rfd_ = rfd; - wfd_ = wfd; -+ -+ // translate GNU make -lN to ninja -lN -+ if (l_arg && -+ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && -+ (load_limit > 0)) { -+ max_load_average = load_limit; -+ } -+ - return true; - } - } -@@ -210,9 +221,9 @@ int GNUmakeTokenPool::GetMonitorFd() { - return(rfd_); - } - --struct TokenPool *TokenPool::Get(bool ignore) { -+struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { - GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; -- if (tokenpool->Setup(ignore)) -+ if (tokenpool->Setup(ignore, max_load_average)) - return tokenpool; - else - delete tokenpool; -diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc -index 199b22264b..e8e25426c3 100644 ---- a/src/tokenpool-none.cc -+++ b/src/tokenpool-none.cc -@@ -22,6 +22,6 @@ - #include - - // No-op TokenPool implementation --struct TokenPool *TokenPool::Get(bool ignore) { -+struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { - return NULL; - } -diff --git a/src/tokenpool.h b/src/tokenpool.h -index 878a0933c2..f9e8cc2ee0 100644 ---- a/src/tokenpool.h -+++ b/src/tokenpool.h -@@ -28,5 +28,5 @@ struct TokenPool { - #endif - - // returns NULL if token pool is not available -- static struct TokenPool *Get(bool ignore); -+ static struct TokenPool *Get(bool ignore, double& max_load_average); - }; - -From 1c10047fc6a3269ba42839da19361e09cbc06ff0 Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Wed, 6 Dec 2017 22:14:21 +0200 -Subject: [PATCH 05/11] Use LinePrinter for TokenPool messages - -- replace printf() with calls to LinePrinter -- print GNU make jobserver message only when verbose build is requested ---- - src/build.cc | 1 + - src/tokenpool-gnu-make.cc | 22 ++++++++++++++++------ - src/tokenpool-none.cc | 4 +++- - src/tokenpool.h | 4 +++- - 4 files changed, 23 insertions(+), 8 deletions(-) - -diff --git a/src/build.cc b/src/build.cc -index 9e6272d035..662e4bd7be 100644 ---- a/src/build.cc -+++ b/src/build.cc -@@ -474,6 +474,7 @@ struct RealCommandRunner : public CommandRunner { - RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { - max_load_average_ = config.max_load_average; - tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, -+ config_.verbosity == BuildConfig::VERBOSE, - max_load_average_); - } - -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -index fb654c4d88..b0d3e6ebc4 100644 ---- a/src/tokenpool-gnu-make.cc -+++ b/src/tokenpool-gnu-make.cc -@@ -23,6 +23,8 @@ - #include - #include - -+#include "line_printer.h" -+ - // TokenPool implementation for GNU make jobserver - // (http://make.mad-scientist.net/papers/jobserver-implementation/) - struct GNUmakeTokenPool : public TokenPool { -@@ -35,7 +37,7 @@ struct GNUmakeTokenPool : public TokenPool { - virtual void Clear(); - virtual int GetMonitorFd(); - -- bool Setup(bool ignore, double& max_load_average); -+ bool Setup(bool ignore, bool verbose, double& max_load_average); - - private: - int available_; -@@ -100,7 +102,9 @@ bool GNUmakeTokenPool::SetAlarmHandler() { - } - } - --bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { -+bool GNUmakeTokenPool::Setup(bool ignore, -+ bool verbose, -+ double& max_load_average) { - const char *value = getenv("MAKEFLAGS"); - if (value) { - // GNU make <= 4.1 -@@ -109,8 +113,10 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { - if (!jobserver) - jobserver = strstr(value, "--jobserver-auth="); - if (jobserver) { -+ LinePrinter printer; -+ - if (ignore) { -- printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); -+ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); - } else { - int rfd = -1; - int wfd = -1; -@@ -121,7 +127,9 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { - const char *l_arg = strstr(value, " -l"); - int load_limit = -1; - -- printf("ninja: using GNU make jobserver.\n"); -+ if (verbose) { -+ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); -+ } - rfd_ = rfd; - wfd_ = wfd; - -@@ -221,9 +229,11 @@ int GNUmakeTokenPool::GetMonitorFd() { - return(rfd_); - } - --struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { -+struct TokenPool *TokenPool::Get(bool ignore, -+ bool verbose, -+ double& max_load_average) { - GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; -- if (tokenpool->Setup(ignore, max_load_average)) -+ if (tokenpool->Setup(ignore, verbose, max_load_average)) - return tokenpool; - else - delete tokenpool; -diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc -index e8e25426c3..1c1c499c8d 100644 ---- a/src/tokenpool-none.cc -+++ b/src/tokenpool-none.cc -@@ -22,6 +22,8 @@ - #include - - // No-op TokenPool implementation --struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { -+struct TokenPool *TokenPool::Get(bool ignore, -+ bool verbose, -+ double& max_load_average) { - return NULL; - } -diff --git a/src/tokenpool.h b/src/tokenpool.h -index f9e8cc2ee0..4bf477f20c 100644 ---- a/src/tokenpool.h -+++ b/src/tokenpool.h -@@ -28,5 +28,7 @@ struct TokenPool { - #endif - - // returns NULL if token pool is not available -- static struct TokenPool *Get(bool ignore, double& max_load_average); -+ static struct TokenPool *Get(bool ignore, -+ bool verbose, -+ double& max_load_average); - }; - -From fdbf68416e3574add3bffd0b637d0694fbaba320 Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Sat, 7 Apr 2018 17:11:21 +0300 -Subject: [PATCH 06/11] Prepare PR for merging - -- fix Windows build error in no-op TokenPool implementation -- improve Acquire() to block for a maximum of 100ms -- address review comments ---- - src/build.h | 2 ++ - src/tokenpool-gnu-make.cc | 53 +++++++++++++++++++++++++++++++++------ - src/tokenpool-none.cc | 7 +----- - 3 files changed, 49 insertions(+), 13 deletions(-) - -diff --git a/src/build.h b/src/build.h -index dfde576573..66ddefb888 100644 ---- a/src/build.h -+++ b/src/build.h -@@ -151,6 +151,8 @@ struct CommandRunner { - bool success() const { return status == ExitSuccess; } - }; - /// Wait for a command to complete, or return false if interrupted. -+ /// If more_ready is true then the optional TokenPool is monitored too -+ /// and we return when a token becomes available. - virtual bool WaitForCommand(Result* result, bool more_ready) = 0; - - virtual std::vector GetActiveEdges() { return std::vector(); } -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -index b0d3e6ebc4..4132bb06d9 100644 ---- a/src/tokenpool-gnu-make.cc -+++ b/src/tokenpool-gnu-make.cc -@@ -1,4 +1,4 @@ --// Copyright 2016-2017 Google Inc. All Rights Reserved. -+// Copyright 2016-2018 Google Inc. All Rights Reserved. - // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -153,6 +154,15 @@ bool GNUmakeTokenPool::Acquire() { - if (available_ > 0) - return true; - -+ // Please read -+ // -+ // http://make.mad-scientist.net/papers/jobserver-implementation/ -+ // -+ // for the reasoning behind the following code. -+ // -+ // Try to read one character from the pipe. Returns true on success. -+ // -+ // First check if read() would succeed without blocking. - #ifdef USE_PPOLL - pollfd pollfds[] = {{rfd_, POLLIN, 0}}; - int ret = poll(pollfds, 1, 0); -@@ -164,33 +174,62 @@ bool GNUmakeTokenPool::Acquire() { - int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); - #endif - if (ret > 0) { -+ // Handle potential race condition: -+ // - the above check succeeded, i.e. read() should not block -+ // - the character disappears before we call read() -+ // -+ // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_ -+ // can safely be closed by signal handlers without affecting rfd_. - dup_rfd_ = dup(rfd_); - - if (dup_rfd_ != -1) { - struct sigaction act, old_act; - int ret = 0; - -+ // Temporarily replace SIGCHLD handler with our own - memset(&act, 0, sizeof(act)); - act.sa_handler = CloseDupRfd; - if (sigaction(SIGCHLD, &act, &old_act) == 0) { -- char buf; -- -- // block until token read, child exits or timeout -- alarm(1); -- ret = read(dup_rfd_, &buf, 1); -- alarm(0); -+ struct itimerval timeout; -+ -+ // install a 100ms timeout that generates SIGALARM on expiration -+ memset(&timeout, 0, sizeof(timeout)); -+ timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] -+ if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { -+ char buf; -+ -+ // Now try to read() from dup_rfd_. Return values from read(): -+ // -+ // 1. token read -> 1 -+ // 2. pipe closed -> 0 -+ // 3. alarm expires -> -1 (EINTR) -+ // 4. child exits -> -1 (EINTR) -+ // 5. alarm expired before entering read() -> -1 (EBADF) -+ // 6. child exited before entering read() -> -1 (EBADF) -+ // 7. child exited before handler is installed -> go to 1 - 3 -+ ret = read(dup_rfd_, &buf, 1); -+ -+ // disarm timer -+ memset(&timeout, 0, sizeof(timeout)); -+ setitimer(ITIMER_REAL, &timeout, NULL); -+ } - - sigaction(SIGCHLD, &old_act, NULL); - } - - CloseDupRfd(0); - -+ // Case 1 from above list - if (ret > 0) { - available_++; - return true; - } - } - } -+ -+ // read() would block, i.e. no token available, -+ // cases 2-6 from above list or -+ // select() / poll() / dup() / sigaction() / setitimer() failed - return false; - } - -diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc -index 1c1c499c8d..4c592875b4 100644 ---- a/src/tokenpool-none.cc -+++ b/src/tokenpool-none.cc -@@ -1,4 +1,4 @@ --// Copyright 2016-2017 Google Inc. All Rights Reserved. -+// Copyright 2016-2018 Google Inc. All Rights Reserved. - // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. -@@ -14,11 +14,6 @@ - - #include "tokenpool.h" - --#include --#include --#include --#include --#include - #include - - // No-op TokenPool implementation - -From ec6220a0baf7d3a6eaf1a2b75bf8960ddfe24c2f Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Fri, 25 May 2018 00:17:07 +0300 -Subject: [PATCH 07/11] Add tests for TokenPool - -- TokenPool setup -- GetMonitorFd() API -- implicit token and tokens in jobserver pipe -- Acquire() / Reserve() / Release() protocol -- Clear() API ---- - configure.py | 1 + - src/tokenpool_test.cc | 198 ++++++++++++++++++++++++++++++++++++++++++ - 2 files changed, 199 insertions(+) - create mode 100644 src/tokenpool_test.cc - -diff --git a/configure.py b/configure.py -index db3492c93c..dc8a0066b7 100755 ---- a/configure.py -+++ b/configure.py -@@ -590,6 +590,7 @@ def has_re2c(): - 'string_piece_util_test', - 'subprocess_test', - 'test', -+ 'tokenpool_test', - 'util_test']: - objs += cxx(name, variables=cxxvariables) - if platform.is_windows(): -diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc -new file mode 100644 -index 0000000000..6c89064ca4 ---- /dev/null -+++ b/src/tokenpool_test.cc -@@ -0,0 +1,198 @@ -+// Copyright 2018 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ -+#include "tokenpool.h" -+ -+#include "test.h" -+ -+#ifndef _WIN32 -+#include -+#include -+#include -+ -+#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") -+#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true); -+#endif -+ -+namespace { -+ -+const double kLoadAverageDefault = -1.23456789; -+ -+struct TokenPoolTest : public testing::Test { -+ double load_avg_; -+ TokenPool *tokens_; -+#ifndef _WIN32 -+ char buf_[1024]; -+ int fds_[2]; -+#endif -+ -+ virtual void SetUp() { -+ load_avg_ = kLoadAverageDefault; -+ tokens_ = NULL; -+#ifndef _WIN32 -+ ENVIRONMENT_CLEAR(); -+ if (pipe(fds_) < 0) -+ ASSERT_TRUE(false); -+#endif -+ } -+ -+ void CreatePool(const char *format, bool ignore_jobserver) { -+#ifndef _WIN32 -+ if (format) { -+ sprintf(buf_, format, fds_[0], fds_[1]); -+ ENVIRONMENT_INIT(buf_); -+ } -+#endif -+ tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_); -+ } -+ -+ void CreateDefaultPool() { -+ CreatePool("foo --jobserver-auth=%d,%d bar", false); -+ } -+ -+ virtual void TearDown() { -+ if (tokens_) -+ delete tokens_; -+#ifndef _WIN32 -+ close(fds_[0]); -+ close(fds_[1]); -+ ENVIRONMENT_CLEAR(); -+#endif -+ } ++ // returns NULL if token pool is not available ++ static TokenPool* Get(); +}; -+ -+} // anonymous namespace -+ -+// verifies none implementation -+TEST_F(TokenPoolTest, NoTokenPool) { -+ CreatePool(NULL, false); -+ -+ EXPECT_EQ(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+#ifndef _WIN32 -+TEST_F(TokenPoolTest, SuccessfulOldSetup) { -+ // GNUmake <= 4.1 -+ CreatePool("foo --jobserver-fds=%d,%d bar", false); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, SuccessfulNewSetup) { -+ // GNUmake => 4.2 -+ CreateDefaultPool(); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, IgnoreWithJN) { -+ CreatePool("foo --jobserver-auth=%d,%d bar", true); -+ -+ EXPECT_EQ(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, HonorLN) { -+ CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(9.0, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, MonitorFD) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); -+} -+ -+TEST_F(TokenPoolTest, ImplicitToken) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+} -+ -+TEST_F(TokenPoolTest, TwoTokens) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // jobserver offers 2nd token -+ ASSERT_EQ(1u, write(fds_[1], "T", 1)); -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // release 2nd token -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ -+ // release implict token - must return 2nd token back to jobserver -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ -+ // there must be one token in the pipe -+ EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+} -+ -+TEST_F(TokenPoolTest, Clear) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // jobserver offers 2nd & 3rd token -+ ASSERT_EQ(2u, write(fds_[1], "TT", 2)); -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ tokens_->Clear(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ -+ // there must be two tokens in the pipe -+ EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+} -+#endif - -From e59d8858327126d1593fd0b8e607975a79072e92 Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Thu, 24 May 2018 18:52:45 +0300 -Subject: [PATCH 08/11] Add tests for subprocess module - -- add TokenPoolTest stub to provide TokenPool::GetMonitorFd() -- add two tests - * both tests set up a dummy GNUmake jobserver pipe - * both tests call DoWork() with TokenPoolTest - * test 1: verify that DoWork() detects when a token is available - * test 2: verify that DoWork() works as before without a token -- the tests are not compiled in under Windows ---- - src/subprocess_test.cc | 76 ++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 76 insertions(+) - -diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc -index 4bc8083e26..6264c8bf11 100644 ---- a/src/subprocess_test.cc -+++ b/src/subprocess_test.cc -@@ -13,6 +13,7 @@ - // limitations under the License. - - #include "subprocess.h" -+#include "tokenpool.h" - - #include "test.h" - -@@ -34,8 +35,23 @@ const char* kSimpleCommand = "cmd /c dir \\"; - const char* kSimpleCommand = "ls /"; - #endif - -+struct TokenPoolTest : public TokenPool { -+ bool Acquire() { return false; } -+ void Reserve() {} -+ void Release() {} -+ void Clear() {} -+ -+#ifdef _WIN32 -+ // @TODO -+#else -+ int _fd; -+ int GetMonitorFd() { return _fd; } -+#endif -+}; -+ - struct SubprocessTest : public testing::Test { - SubprocessSet subprocs_; -+ TokenPoolTest tokens_; - }; - - } // anonymous namespace -@@ -280,3 +296,63 @@ TEST_F(SubprocessTest, ReadStdin) { - ASSERT_EQ(1u, subprocs_.finished_.size()); - } - #endif // _WIN32 -+ -+// @TODO: remove once TokenPool implementation for Windows is available -+#ifndef _WIN32 -+TEST_F(SubprocessTest, TokenAvailable) { -+ Subprocess* subproc = subprocs_.Add(kSimpleCommand); -+ ASSERT_NE((Subprocess *) 0, subproc); -+ -+ // simulate GNUmake jobserver pipe with 1 token -+ int fds[2]; -+ ASSERT_EQ(0u, pipe(fds)); -+ tokens_._fd = fds[0]; -+ ASSERT_EQ(1u, write(fds[1], "T", 1)); -+ -+ subprocs_.ResetTokenAvailable(); -+ subprocs_.DoWork(&tokens_); -+ -+ EXPECT_TRUE(subprocs_.IsTokenAvailable()); -+ EXPECT_EQ(0u, subprocs_.finished_.size()); -+ -+ // remove token to let DoWork() wait for command again -+ char token; -+ ASSERT_EQ(1u, read(fds[0], &token, 1)); -+ -+ while (!subproc->Done()) { -+ subprocs_.DoWork(&tokens_); -+ } -+ -+ close(fds[1]); -+ close(fds[0]); -+ -+ EXPECT_EQ(ExitSuccess, subproc->Finish()); -+ EXPECT_NE("", subproc->GetOutput()); -+ -+ EXPECT_EQ(1u, subprocs_.finished_.size()); -+} -+ -+TEST_F(SubprocessTest, TokenNotAvailable) { -+ Subprocess* subproc = subprocs_.Add(kSimpleCommand); -+ ASSERT_NE((Subprocess *) 0, subproc); -+ -+ // simulate GNUmake jobserver pipe with 0 tokens -+ int fds[2]; -+ ASSERT_EQ(0u, pipe(fds)); -+ tokens_._fd = fds[0]; -+ -+ subprocs_.ResetTokenAvailable(); -+ while (!subproc->Done()) { -+ subprocs_.DoWork(&tokens_); -+ } -+ -+ close(fds[1]); -+ close(fds[0]); -+ -+ EXPECT_FALSE(subprocs_.IsTokenAvailable()); -+ EXPECT_EQ(ExitSuccess, subproc->Finish()); -+ EXPECT_NE("", subproc->GetOutput()); -+ -+ EXPECT_EQ(1u, subprocs_.finished_.size()); -+} -+#endif // _WIN32 - -From 0145e2d4db64ea6c21aeb371928e4071f65164eb Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Sat, 26 May 2018 23:17:51 +0300 -Subject: [PATCH 09/11] Add tests for build module - -Add tests that verify the token functionality of the builder main loop. -We replace the default fake command runner with a special version where -the tests can control each call to AcquireToken(), CanRunMore() and -WaitForCommand(). ---- - src/build_test.cc | 364 ++++++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 364 insertions(+) - -diff --git a/src/build_test.cc b/src/build_test.cc -index 7a5ff4015a..dd41dfbe1d 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -15,6 +15,7 @@ @@ -2020,7 +512,38 @@ index 7a5ff4015a..dd41dfbe1d 100644 #include "build_log.h" #include "deps_log.h" -@@ -3990,3 +3991,366 @@ TEST_F(BuildTest, ValidationWithCircularDependency) { +@@ -474,8 +475,9 @@ struct FakeCommandRunner : public Comman + + // CommandRunner impl + virtual bool CanRunMore() const; ++ virtual bool AcquireToken(); + virtual bool StartCommand(Edge* edge); +- virtual bool WaitForCommand(Result* result); ++ virtual bool WaitForCommand(Result* result, bool more_ready); + virtual vector GetActiveEdges(); + virtual void Abort(); + +@@ -578,6 +580,10 @@ bool FakeCommandRunner::CanRunMore() con + return active_edges_.size() < max_active_edges_; + } + ++bool FakeCommandRunner::AcquireToken() { ++ return true; ++} ++ + bool FakeCommandRunner::StartCommand(Edge* edge) { + assert(active_edges_.size() < max_active_edges_); + assert(find(active_edges_.begin(), active_edges_.end(), edge) +@@ -649,7 +655,7 @@ bool FakeCommandRunner::StartCommand(Edg + return true; + } + +-bool FakeCommandRunner::WaitForCommand(Result* result) { ++bool FakeCommandRunner::WaitForCommand(Result* result, bool more_ready) { + if (active_edges_.empty()) + return false; + +@@ -3985,3 +3991,356 @@ TEST_F(BuildTest, ValidationWithCircular EXPECT_FALSE(builder_.AddTarget("out", &err)); EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err); } @@ -2131,7 +654,7 @@ index 7a5ff4015a..dd41dfbe1d 100644 + void ExpectWaitForCommand(int count, ...); + +private: -+ void EnqueueBooleans(vector& booleans, int count, va_list ao); ++ void EnqueueBooleans(vector& booleans, int count, va_list ap); +}; + +void BuildTokenTest::SetUp() { @@ -2177,16 +700,6 @@ index 7a5ff4015a..dd41dfbe1d 100644 + } +} + -+TEST_F(BuildTokenTest, CompleteNoWork) { -+ // plan should not execute anything -+ string err; -+ -+ EXPECT_TRUE(builder_.Build(&err)); -+ EXPECT_EQ("", err); -+ -+ EXPECT_EQ(0u, token_command_runner_.commands_ran_.size()); -+} -+ +TEST_F(BuildTokenTest, DoNotAquireToken) { + // plan should execute one command + string err; @@ -2387,106 +900,99 @@ index 7a5ff4015a..dd41dfbe1d 100644 + token_command_runner_.commands_ran_[1] == "cat in1 > out1")); + EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat out1 out2 > out12"); +} - -From f016e5430c9123d34a73ea7ad28693b20ee59d6d Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Mon, 8 Oct 2018 17:47:50 +0300 -Subject: [PATCH 10/11] Add Win32 implementation for GNUmakeTokenPool - -GNU make uses a semaphore as jobserver protocol on Win32. See also - - https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html - -Usage is pretty simple and straightforward, i.e. WaitForSingleObject() -to obtain a token and ReleaseSemaphore() to return it. - -Unfortunately subprocess-win32.cc uses an I/O completion port (IOCP). -IOCPs aren't waitable objects, i.e. we can't use WaitForMultipleObjects() -to wait on the IOCP and the token semaphore at the same time. - -Therefore GNUmakeTokenPoolWin32 creates a child thread that waits on the -token semaphore and posts a dummy I/O completion status on the IOCP when -it was able to obtain a token. That unblocks SubprocessSet::DoWork() and -it can then check if a token became available or not. - -- split existing GNUmakeTokenPool into common and platform bits -- add GNUmakeTokenPool interface -- move the Posix bits to GNUmakeTokenPoolPosix -- add the Win32 bits as GNUmakeTokenPoolWin32 -- move Setup() method up to TokenPool interface -- update Subprocess & TokenPool tests accordingly ---- - configure.py | 8 +- - src/build.cc | 11 +- - src/subprocess-win32.cc | 9 ++ - src/subprocess_test.cc | 34 ++++- - src/tokenpool-gnu-make-posix.cc | 203 +++++++++++++++++++++++++++ - src/tokenpool-gnu-make-win32.cc | 237 ++++++++++++++++++++++++++++++++ - src/tokenpool-gnu-make.cc | 203 ++------------------------- - src/tokenpool-gnu-make.h | 40 ++++++ - src/tokenpool-none.cc | 4 +- - src/tokenpool.h | 18 ++- - src/tokenpool_test.cc | 113 ++++++++++++--- - 11 files changed, 653 insertions(+), 227 deletions(-) - create mode 100644 src/tokenpool-gnu-make-posix.cc - create mode 100644 src/tokenpool-gnu-make-win32.cc - create mode 100644 src/tokenpool-gnu-make.h - -diff --git a/configure.py b/configure.py -index dc8a0066b7..a239b90eef 100755 ---- a/configure.py -+++ b/configure.py -@@ -517,12 +517,13 @@ def has_re2c(): - 'state', - 'status', - 'string_piece_util', -+ 'tokenpool-gnu-make', - 'util', - 'version']: - objs += cxx(name, variables=cxxvariables) - if platform.is_windows(): - for name in ['subprocess-win32', -- 'tokenpool-none', -+ 'tokenpool-gnu-make-win32', - 'includes_normalize-win32', - 'msvc_helper-win32', - 'msvc_helper_main-win32']: -@@ -531,8 +532,9 @@ def has_re2c(): - objs += cxx('minidump-win32', variables=cxxvariables) - objs += cc('getopt') - else: -- objs += cxx('subprocess-posix') -- objs += cxx('tokenpool-gnu-make') -+ for name in ['subprocess-posix', -+ 'tokenpool-gnu-make-posix']: -+ objs += cxx(name) - if platform.is_aix(): - objs += cc('getopt') - if platform.is_msvc(): -diff --git a/src/build.cc b/src/build.cc -index 662e4bd7be..20c3bdc2a0 100644 ---- a/src/build.cc -+++ b/src/build.cc -@@ -473,9 +473,14 @@ struct RealCommandRunner : public CommandRunner { +--- a/src/exit_status.h ++++ b/src/exit_status.h +@@ -18,7 +18,8 @@ + enum ExitStatus { + ExitSuccess, + ExitFailure, +- ExitInterrupted ++ ExitTokenAvailable, ++ ExitInterrupted, + }; - RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { - max_load_average_ = config.max_load_average; -- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, -- config_.verbosity == BuildConfig::VERBOSE, -- max_load_average_); -+ if ((tokens_ = TokenPool::Get()) != NULL) { -+ if (!tokens_->Setup(config_.parallelism_from_cmdline, -+ config_.verbosity == BuildConfig::VERBOSE, -+ max_load_average_)) { -+ delete tokens_; -+ tokens_ = NULL; -+ } -+ } + #endif // NINJA_EXIT_STATUS_H_ +--- a/src/subprocess-posix.cc ++++ b/src/subprocess-posix.cc +@@ -13,6 +13,7 @@ + // limitations under the License. + + #include "subprocess.h" ++#include "tokenpool.h" + + #include + #include +@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const str } - RealCommandRunner::~RealCommandRunner() { -diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc -index 66d2c2c430..ce3e2c20a4 100644 + #ifdef USE_PPOLL +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + vector fds; + nfds_t nfds = 0; + +@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() { + ++nfds; + } + ++ if (tokens) { ++ pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 }; ++ fds.push_back(pfd); ++ ++nfds; ++ } ++ + interrupted_ = 0; + int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); + if (ret == -1) { +@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() { + ++i; + } + ++ if (tokens) { ++ pollfd *pfd = &fds[nfds - 1]; ++ if (pfd->fd >= 0) { ++ assert(pfd->fd == tokens->GetMonitorFd()); ++ if (pfd->revents != 0) ++ token_available_ = true; ++ } ++ } ++ + return IsInterrupted(); + } + + #else // !defined(USE_PPOLL) +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + fd_set set; + int nfds = 0; + FD_ZERO(&set); +@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() { + } + } + ++ if (tokens) { ++ int fd = tokens->GetMonitorFd(); ++ FD_SET(fd, &set); ++ if (nfds < fd+1) ++ nfds = fd+1; ++ } ++ + interrupted_ = 0; + int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); + if (ret == -1) { +@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() { + ++i; + } + ++ if (tokens) { ++ int fd = tokens->GetMonitorFd(); ++ if ((fd >= 0) && FD_ISSET(fd, &set)) ++ token_available_ = true; ++ } ++ + return IsInterrupted(); + } + #endif // !defined(USE_PPOLL) --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -13,6 +13,7 @@ @@ -2497,7 +1003,13 @@ index 66d2c2c430..ce3e2c20a4 100644 #include #include -@@ -256,6 +257,9 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { +@@ -251,11 +252,14 @@ Subprocess *SubprocessSet::Add(const str + return subprocess; + } + +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + DWORD bytes_read; Subprocess* subproc; OVERLAPPED* overlapped; @@ -2507,7 +1019,7 @@ index 66d2c2c430..ce3e2c20a4 100644 if (!GetQueuedCompletionStatus(ioport_, &bytes_read, (PULONG_PTR)&subproc, &overlapped, INFINITE)) { if (GetLastError() != ERROR_BROKEN_PIPE) -@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { +@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork() { // delivered by NotifyInterrupted above. return true; @@ -2519,18 +1031,58 @@ index 66d2c2c430..ce3e2c20a4 100644 subproc->OnPipeReady(); if (subproc->Done()) { -diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc -index 6264c8bf11..f625963462 100644 +--- a/src/subprocess.h ++++ b/src/subprocess.h +@@ -76,6 +76,8 @@ struct Subprocess { + friend struct SubprocessSet; + }; + ++struct TokenPool; ++ + /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses. + /// DoWork() waits for any state change in subprocesses; finished_ + /// is a queue of subprocesses as they finish. +@@ -84,13 +86,17 @@ struct SubprocessSet { + ~SubprocessSet(); + + Subprocess* Add(const std::string& command, bool use_console = false); +- bool DoWork(); ++ bool DoWork(TokenPool* tokens); + Subprocess* NextFinished(); + void Clear(); + + std::vector running_; + std::queue finished_; + ++ bool token_available_; ++ bool IsTokenAvailable() { return token_available_; } ++ void ResetTokenAvailable() { token_available_ = false; } ++ + #ifdef _WIN32 + static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType); + static HANDLE ioport_; --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc -@@ -40,9 +40,16 @@ struct TokenPoolTest : public TokenPool { - void Reserve() {} - void Release() {} - void Clear() {} -+ bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; } +@@ -13,6 +13,7 @@ + // limitations under the License. - #ifdef _WIN32 -- // @TODO + #include "subprocess.h" ++#include "tokenpool.h" + + #include "test.h" + +@@ -34,8 +35,30 @@ const char* kSimpleCommand = "cmd /c dir + const char* kSimpleCommand = "ls /"; + #endif + ++struct TestTokenPool : public TokenPool { ++ bool Acquire() { return false; } ++ void Reserve() {} ++ void Release() {} ++ void Clear() {} ++ bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; } ++ ++#ifdef _WIN32 + bool _token_available; + void WaitForTokenAvailability(HANDLE ioport) { + if (_token_available) @@ -2538,95 +1090,554 @@ index 6264c8bf11..f625963462 100644 + PostQueuedCompletionStatus(ioport, 0, (ULONG_PTR) this, NULL); + } + bool TokenIsAvailable(ULONG_PTR key) { return key == (ULONG_PTR) this; } - #else - int _fd; - int GetMonitorFd() { return _fd; } -@@ -297,34 +304,48 @@ TEST_F(SubprocessTest, ReadStdin) { - } - #endif // _WIN32 ++#else ++ int _fd; ++ int GetMonitorFd() { return _fd; } ++#endif ++}; ++ + struct SubprocessTest : public testing::Test { + SubprocessSet subprocs_; ++ TestTokenPool tokens_; + }; --// @TODO: remove once TokenPool implementation for Windows is available --#ifndef _WIN32 - TEST_F(SubprocessTest, TokenAvailable) { + } // anonymous namespace +@@ -45,10 +68,12 @@ TEST_F(SubprocessTest, BadCommandStderr) + Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + // Pretend we discovered that stderr was ready for writing. +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -59,10 +84,12 @@ TEST_F(SubprocessTest, NoSuchCommand) { + Subprocess* subproc = subprocs_.Add("ninja_no_such_command"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + // Pretend we discovered that stderr was ready for writing. +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -78,9 +105,11 @@ TEST_F(SubprocessTest, InterruptChild) { + Subprocess* subproc = subprocs_.Add("kill -INT $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -90,7 +119,7 @@ TEST_F(SubprocessTest, InterruptParent) + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -102,9 +131,11 @@ TEST_F(SubprocessTest, InterruptChildWit + Subprocess* subproc = subprocs_.Add("kill -TERM $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -114,7 +145,7 @@ TEST_F(SubprocessTest, InterruptParentWi + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -126,9 +157,11 @@ TEST_F(SubprocessTest, InterruptChildWit + Subprocess* subproc = subprocs_.Add("kill -HUP $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -138,7 +171,7 @@ TEST_F(SubprocessTest, InterruptParentWi + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -153,9 +186,11 @@ TEST_F(SubprocessTest, Console) { + subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true); + ASSERT_NE((Subprocess*)0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } +@@ -167,9 +202,11 @@ TEST_F(SubprocessTest, SetWithSingle) { Subprocess* subproc = subprocs_.Add(kSimpleCommand); ASSERT_NE((Subprocess *) 0, subproc); - // simulate GNUmake jobserver pipe with 1 token ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_NE("", subproc->GetOutput()); + +@@ -200,12 +237,13 @@ TEST_F(SubprocessTest, SetWithMulti) { + ASSERT_EQ("", processes[i]->GetOutput()); + } + ++ subprocs_.ResetTokenAvailable(); + while (!processes[0]->Done() || !processes[1]->Done() || + !processes[2]->Done()) { + ASSERT_GT(subprocs_.running_.size(), 0u); +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } +- ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(0u, subprocs_.running_.size()); + ASSERT_EQ(3u, subprocs_.finished_.size()); + +@@ -237,8 +275,10 @@ TEST_F(SubprocessTest, SetWithLots) { + ASSERT_NE((Subprocess *) 0, subproc); + procs.push_back(subproc); + } ++ subprocs_.ResetTokenAvailable(); + while (!subprocs_.running_.empty()) +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + for (size_t i = 0; i < procs.size(); ++i) { + ASSERT_EQ(ExitSuccess, procs[i]->Finish()); + ASSERT_NE("", procs[i]->GetOutput()); +@@ -254,10 +294,91 @@ TEST_F(SubprocessTest, SetWithLots) { + // that stdin is closed. + TEST_F(SubprocessTest, ReadStdin) { + Subprocess* subproc = subprocs_.Add("cat -"); ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_EQ(1u, subprocs_.finished_.size()); + } + #endif // _WIN32 ++ ++TEST_F(SubprocessTest, TokenAvailable) { ++ Subprocess* subproc = subprocs_.Add(kSimpleCommand); ++ ASSERT_NE((Subprocess *) 0, subproc); ++ ++ // simulate GNUmake jobserver pipe with 1 token +#ifdef _WIN32 + tokens_._token_available = true; +#else - int fds[2]; - ASSERT_EQ(0u, pipe(fds)); - tokens_._fd = fds[0]; - ASSERT_EQ(1u, write(fds[1], "T", 1)); ++ int fds[2]; ++ ASSERT_EQ(0u, pipe(fds)); ++ tokens_._fd = fds[0]; ++ ASSERT_EQ(1u, write(fds[1], "T", 1)); +#endif - - subprocs_.ResetTokenAvailable(); - subprocs_.DoWork(&tokens_); ++ ++ subprocs_.ResetTokenAvailable(); ++ subprocs_.DoWork(&tokens_); +#ifdef _WIN32 + tokens_._token_available = false; -+ // we need to loop here as we have no conrol where the token ++ // we need to loop here as we have no control where the token + // I/O completion post ends up in the queue + while (!subproc->Done() && !subprocs_.IsTokenAvailable()) { + subprocs_.DoWork(&tokens_); + } +#endif - - EXPECT_TRUE(subprocs_.IsTokenAvailable()); - EXPECT_EQ(0u, subprocs_.finished_.size()); - - // remove token to let DoWork() wait for command again ++ ++ EXPECT_TRUE(subprocs_.IsTokenAvailable()); ++ EXPECT_EQ(0u, subprocs_.finished_.size()); ++ ++ // remove token to let DoWork() wait for command again +#ifndef _WIN32 - char token; - ASSERT_EQ(1u, read(fds[0], &token, 1)); ++ char token; ++ ASSERT_EQ(1u, read(fds[0], &token, 1)); +#endif - - while (!subproc->Done()) { - subprocs_.DoWork(&tokens_); - } - ++ ++ while (!subproc->Done()) { ++ subprocs_.DoWork(&tokens_); ++ } ++ +#ifndef _WIN32 - close(fds[1]); - close(fds[0]); ++ close(fds[1]); ++ close(fds[0]); +#endif - - EXPECT_EQ(ExitSuccess, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -337,17 +358,23 @@ TEST_F(SubprocessTest, TokenNotAvailable) { - ASSERT_NE((Subprocess *) 0, subproc); - - // simulate GNUmake jobserver pipe with 0 tokens ++ ++ EXPECT_EQ(ExitSuccess, subproc->Finish()); ++ EXPECT_NE("", subproc->GetOutput()); ++ ++ EXPECT_EQ(1u, subprocs_.finished_.size()); ++} ++ ++TEST_F(SubprocessTest, TokenNotAvailable) { ++ Subprocess* subproc = subprocs_.Add(kSimpleCommand); ++ ASSERT_NE((Subprocess *) 0, subproc); ++ ++ // simulate GNUmake jobserver pipe with 0 tokens +#ifdef _WIN32 + tokens_._token_available = false; +#else - int fds[2]; - ASSERT_EQ(0u, pipe(fds)); - tokens_._fd = fds[0]; ++ int fds[2]; ++ ASSERT_EQ(0u, pipe(fds)); ++ tokens_._fd = fds[0]; +#endif - - subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { - subprocs_.DoWork(&tokens_); - } - ++ ++ subprocs_.ResetTokenAvailable(); ++ while (!subproc->Done()) { ++ subprocs_.DoWork(&tokens_); ++ } ++ +#ifndef _WIN32 - close(fds[1]); - close(fds[0]); ++ close(fds[1]); ++ close(fds[0]); +#endif - - EXPECT_FALSE(subprocs_.IsTokenAvailable()); - EXPECT_EQ(ExitSuccess, subproc->Finish()); -@@ -355,4 +382,3 @@ TEST_F(SubprocessTest, TokenNotAvailable) { - - EXPECT_EQ(1u, subprocs_.finished_.size()); - } --#endif // _WIN32 -diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc -new file mode 100644 -index 0000000000..70d84bfff7 ++ ++ EXPECT_FALSE(subprocs_.IsTokenAvailable()); ++ EXPECT_EQ(ExitSuccess, subproc->Finish()); ++ EXPECT_NE("", subproc->GetOutput()); ++ ++ EXPECT_EQ(1u, subprocs_.finished_.size()); ++} +--- a/src/ninja.cc ++++ b/src/ninja.cc +@@ -1447,6 +1447,7 @@ int ReadFlags(int* argc, char*** argv, + // We want to run N jobs in parallel. For N = 0, INT_MAX + // is close enough to infinite for most sane builds. + config->parallelism = value > 0 ? value : INT_MAX; ++ config->parallelism_from_cmdline = true; + deferGuessParallelism.needGuess = false; + break; + } +--- /dev/null ++++ b/src/tokenpool_test.cc +@@ -0,0 +1,279 @@ ++// Copyright 2018 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "tokenpool.h" ++ ++#include "test.h" ++ ++#ifdef _WIN32 ++#include ++#else ++#include ++#endif ++ ++#include ++#include ++ ++#ifdef _WIN32 ++// should contain all valid characters ++#define SEMAPHORE_NAME "abcdefghijklmnopqrstwxyz01234567890_" ++#define AUTH_FORMAT(tmpl) "foo " tmpl "=%s bar" ++#define ENVIRONMENT_CLEAR() SetEnvironmentVariable("MAKEFLAGS", NULL) ++#define ENVIRONMENT_INIT(v) SetEnvironmentVariable("MAKEFLAGS", v) ++#else ++#define AUTH_FORMAT(tmpl) "foo " tmpl "=%d,%d bar" ++#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") ++#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true) ++#endif ++ ++namespace { ++ ++const double kLoadAverageDefault = -1.23456789; ++ ++struct TokenPoolTest : public testing::Test { ++ double load_avg_; ++ TokenPool* tokens_; ++ char buf_[1024]; ++#ifdef _WIN32 ++ const char* semaphore_name_; ++ HANDLE semaphore_; ++#else ++ int fds_[2]; ++ ++ char random() { ++ return int((rand() / double(RAND_MAX)) * 256); ++ } ++#endif ++ ++ virtual void SetUp() { ++ load_avg_ = kLoadAverageDefault; ++ tokens_ = NULL; ++ ENVIRONMENT_CLEAR(); ++#ifdef _WIN32 ++ semaphore_name_ = SEMAPHORE_NAME; ++ if ((semaphore_ = CreateSemaphore(0, 0, 2, SEMAPHORE_NAME)) == NULL) ++#else ++ if (pipe(fds_) < 0) ++#endif ++ ASSERT_TRUE(false); ++ } ++ ++ void CreatePool(const char* format, bool ignore_jobserver = false) { ++ if (format) { ++ sprintf(buf_, format, ++#ifdef _WIN32 ++ semaphore_name_ ++#else ++ fds_[0], fds_[1] ++#endif ++ ); ++ ENVIRONMENT_INIT(buf_); ++ } ++ if ((tokens_ = TokenPool::Get()) != NULL) { ++ if (!tokens_->Setup(ignore_jobserver, false, load_avg_)) { ++ delete tokens_; ++ tokens_ = NULL; ++ } ++ } ++ } ++ ++ void CreateDefaultPool() { ++ CreatePool(AUTH_FORMAT("--jobserver-auth")); ++ } ++ ++ virtual void TearDown() { ++ if (tokens_) ++ delete tokens_; ++#ifdef _WIN32 ++ CloseHandle(semaphore_); ++#else ++ close(fds_[0]); ++ close(fds_[1]); ++#endif ++ ENVIRONMENT_CLEAR(); ++ } ++}; ++ ++} // anonymous namespace ++ ++// verifies none implementation ++TEST_F(TokenPoolTest, NoTokenPool) { ++ CreatePool(NULL, false); ++ ++ EXPECT_EQ(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, SuccessfulOldSetup) { ++ // GNUmake <= 4.1 ++ CreatePool(AUTH_FORMAT("--jobserver-fds")); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, SuccessfulNewSetup) { ++ // GNUmake => 4.2 ++ CreateDefaultPool(); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, IgnoreWithJN) { ++ CreatePool(AUTH_FORMAT("--jobserver-auth"), true); ++ ++ EXPECT_EQ(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, HonorLN) { ++ CreatePool(AUTH_FORMAT("-l9 --jobserver-auth")); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(9.0, load_avg_); ++} ++ ++#ifdef _WIN32 ++TEST_F(TokenPoolTest, SemaphoreNotFound) { ++ semaphore_name_ = SEMAPHORE_NAME "_foobar"; ++ CreateDefaultPool(); ++ ++ EXPECT_EQ(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, TokenIsAvailable) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ EXPECT_TRUE(tokens_->TokenIsAvailable((ULONG_PTR)tokens_)); ++} ++#else ++TEST_F(TokenPoolTest, MonitorFD) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); ++} ++#endif ++ ++TEST_F(TokenPoolTest, ImplicitToken) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++ ++TEST_F(TokenPoolTest, TwoTokens) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // jobserver offers 2nd token ++#ifdef _WIN32 ++ LONG previous; ++ ASSERT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); ++ ASSERT_EQ(0, previous); ++#else ++ char test_tokens[1] = { random() }; ++ ASSERT_EQ(1u, write(fds_[1], test_tokens, sizeof(test_tokens))); ++#endif ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // release 2nd token ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // release implicit token - must return 2nd token back to jobserver ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // there must be one token available ++#ifdef _WIN32 ++ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); ++ EXPECT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); ++ EXPECT_EQ(0, previous); ++#else ++ EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); ++ EXPECT_EQ(test_tokens[0], buf_[0]); ++#endif ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++ ++TEST_F(TokenPoolTest, Clear) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // jobserver offers 2nd & 3rd token ++#ifdef _WIN32 ++ LONG previous; ++ ASSERT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); ++ ASSERT_EQ(0, previous); ++#else ++ char test_tokens[2] = { random(), random() }; ++ ASSERT_EQ(2u, write(fds_[1], test_tokens, sizeof(test_tokens))); ++#endif ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ tokens_->Clear(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // there must be two tokens available ++#ifdef _WIN32 ++ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); ++ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); ++ EXPECT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); ++ EXPECT_EQ(0, previous); ++#else ++ EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); ++ // tokens are pushed onto a stack, hence returned in reverse order ++ EXPECT_EQ(test_tokens[0], buf_[1]); ++ EXPECT_EQ(test_tokens[1], buf_[0]); ++#endif ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++} --- /dev/null +++ b/src/tokenpool-gnu-make-posix.cc -@@ -0,0 +1,203 @@ +@@ -0,0 +1,214 @@ +// Copyright 2016-2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); @@ -2652,6 +1663,7 @@ index 0000000000..70d84bfff7 +#include +#include +#include ++#include + +// TokenPool implementation for GNU make jobserver - POSIX implementation +// (http://make.mad-scientist.net/papers/jobserver-implementation/) @@ -2661,8 +1673,8 @@ index 0000000000..70d84bfff7 + + virtual int GetMonitorFd(); + -+ virtual const char *GetEnv(const char *name) { return getenv(name); }; -+ virtual bool ParseAuth(const char *jobserver); ++ virtual const char* GetEnv(const char* name) { return getenv(name); }; ++ virtual bool ParseAuth(const char* jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + @@ -2673,6 +1685,16 @@ index 0000000000..70d84bfff7 + struct sigaction old_act_; + bool restore_; + ++ // See https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html ++ // ++ // It’s important that when you release the job slot, you write back ++ // the same character you read. Don’t assume that all tokens are the ++ // same character different characters may have different meanings to ++ // GNU make. The order is not important, since make has no idea in ++ // what order jobs will complete anyway. ++ // ++ std::stack tokens_; ++ + static int dup_rfd_; + static void CloseDupRfd(int signum); + @@ -2693,9 +1715,7 @@ index 0000000000..70d84bfff7 + if (fd < 0) + return false; + int ret = fcntl(fd, F_GETFD); -+ if (ret < 0) -+ return false; -+ return true; ++ return ret >= 0; +} + +int GNUmakeTokenPoolPosix::dup_rfd_ = -1; @@ -2711,14 +1731,13 @@ index 0000000000..70d84bfff7 + act.sa_handler = CloseDupRfd; + if (sigaction(SIGALRM, &act, &old_act_) < 0) { + perror("sigaction:"); -+ return(false); -+ } else { -+ restore_ = true; -+ return(true); ++ return false; + } ++ restore_ = true; ++ return true; +} + -+bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) { ++bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) { + int rfd = -1; + int wfd = -1; + if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && @@ -2765,6 +1784,7 @@ index 0000000000..70d84bfff7 + if (dup_rfd_ != -1) { + struct sigaction act, old_act; + int ret = 0; ++ char buf; + + // Temporarily replace SIGCHLD handler with our own + memset(&act, 0, sizeof(act)); @@ -2776,8 +1796,6 @@ index 0000000000..70d84bfff7 + memset(&timeout, 0, sizeof(timeout)); + timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] + if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { -+ char buf; -+ + // Now try to read() from dup_rfd_. Return values from read(): + // + // 1. token read -> 1 @@ -2800,8 +1818,10 @@ index 0000000000..70d84bfff7 + CloseDupRfd(0); + + // Case 1 from above list -+ if (ret > 0) ++ if (ret > 0) { ++ tokens_.push(buf); + return true; ++ } + } + } + @@ -2812,11 +1832,13 @@ index 0000000000..70d84bfff7 +} + +bool GNUmakeTokenPoolPosix::ReturnToken() { -+ const char buf = '+'; ++ const char buf = tokens_.top(); + while (1) { + int ret = write(wfd_, &buf, 1); -+ if (ret > 0) ++ if (ret > 0) { ++ tokens_.pop(); + return true; ++ } + if ((ret != -1) || (errno != EINTR)) + return false; + // write got interrupted - retry @@ -2824,18 +1846,15 @@ index 0000000000..70d84bfff7 +} + +int GNUmakeTokenPoolPosix::GetMonitorFd() { -+ return(rfd_); ++ return rfd_; +} + -+struct TokenPool *TokenPool::Get() { ++TokenPool* TokenPool::Get() { + return new GNUmakeTokenPoolPosix; +} -diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc -new file mode 100644 -index 0000000000..2719f2c1fc --- /dev/null +++ b/src/tokenpool-gnu-make-win32.cc -@@ -0,0 +1,237 @@ +@@ -0,0 +1,239 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); @@ -2852,7 +1871,8 @@ index 0000000000..2719f2c1fc + +#include "tokenpool-gnu-make.h" + -+// always include first to make sure other headers do the correct thing... ++// Always include this first. ++// Otherwise the other system headers don't work correctly under Win32 +#include + +#include @@ -2870,8 +1890,8 @@ index 0000000000..2719f2c1fc + virtual void WaitForTokenAvailability(HANDLE ioport); + virtual bool TokenIsAvailable(ULONG_PTR key); + -+ virtual const char *GetEnv(const char *name); -+ virtual bool ParseAuth(const char *jobserver); ++ virtual const char* GetEnv(const char* name); ++ virtual bool ParseAuth(const char* jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + @@ -2936,19 +1956,19 @@ index 0000000000..2719f2c1fc + } +} + -+const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) { ++const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) { + // getenv() does not work correctly together with tokenpool_tests.cc + static char buffer[MAX_PATH + 1]; -+ if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0) ++ if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0) + return NULL; -+ return(buffer); ++ return buffer; +} + -+bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { ++bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) { + // match "--jobserver-auth=gmake_semaphore_..." -+ const char *start = strchr(jobserver, '='); ++ const char* start = strchr(jobserver, '='); + if (start) { -+ const char *end = start; ++ const char* end = start; + unsigned int len; + char c, *auth; + @@ -2957,14 +1977,15 @@ index 0000000000..2719f2c1fc + break; + len = end - start; // includes string terminator in count + -+ if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) { ++ if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) { + strncpy(auth, start + 1, len - 1); + auth[len - 1] = '\0'; + -+ if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ -+ FALSE, /* Child processes DON'T inherit */ -+ auth /* Semaphore name */ -+ )) != NULL) { ++ if ((semaphore_jobserver_ = ++ OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ ++ FALSE, /* Child processes DON'T inherit */ ++ auth /* Semaphore name */ ++ )) != NULL) { + free(auth); + return true; + } @@ -3009,7 +2030,7 @@ index 0000000000..2719f2c1fc +} + +DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) { -+ GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param; ++ GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param; + return This->SemaphoreThread(); +} + @@ -3054,7 +2075,7 @@ index 0000000000..2719f2c1fc + +bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) { + // alert child thread to break wait on token semaphore -+ QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL); ++ QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL); + + // return true when GetQueuedCompletionStatus() returned our key + return key == (ULONG_PTR) this; @@ -3070,273 +2091,9 @@ index 0000000000..2719f2c1fc + Win32Fatal("WaitForSingleObject"); +} + -+struct TokenPool *TokenPool::Get() { ++TokenPool* TokenPool::Get() { + return new GNUmakeTokenPoolWin32; +} -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -index 4132bb06d9..92ff611721 100644 ---- a/src/tokenpool-gnu-make.cc -+++ b/src/tokenpool-gnu-make.cc -@@ -12,101 +12,26 @@ - // See the License for the specific language governing permissions and - // limitations under the License. - --#include "tokenpool.h" -+#include "tokenpool-gnu-make.h" - --#include --#include --#include --#include --#include --#include -+#include - #include - #include --#include - - #include "line_printer.h" - --// TokenPool implementation for GNU make jobserver --// (http://make.mad-scientist.net/papers/jobserver-implementation/) --struct GNUmakeTokenPool : public TokenPool { -- GNUmakeTokenPool(); -- virtual ~GNUmakeTokenPool(); -- -- virtual bool Acquire(); -- virtual void Reserve(); -- virtual void Release(); -- virtual void Clear(); -- virtual int GetMonitorFd(); -- -- bool Setup(bool ignore, bool verbose, double& max_load_average); -- -- private: -- int available_; -- int used_; -- --#ifdef _WIN32 -- // @TODO --#else -- int rfd_; -- int wfd_; -- -- struct sigaction old_act_; -- bool restore_; -- -- static int dup_rfd_; -- static void CloseDupRfd(int signum); -- -- bool CheckFd(int fd); -- bool SetAlarmHandler(); --#endif -- -- void Return(); --}; -- -+// TokenPool implementation for GNU make jobserver - common bits - // every instance owns an implicit token -> available_ == 1 --GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0), -- rfd_(-1), wfd_(-1), restore_(false) { -+GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) { - } - - GNUmakeTokenPool::~GNUmakeTokenPool() { -- Clear(); -- if (restore_) -- sigaction(SIGALRM, &old_act_, NULL); --} -- --bool GNUmakeTokenPool::CheckFd(int fd) { -- if (fd < 0) -- return false; -- int ret = fcntl(fd, F_GETFD); -- if (ret < 0) -- return false; -- return true; --} -- --int GNUmakeTokenPool::dup_rfd_ = -1; -- --void GNUmakeTokenPool::CloseDupRfd(int signum) { -- close(dup_rfd_); -- dup_rfd_ = -1; --} -- --bool GNUmakeTokenPool::SetAlarmHandler() { -- struct sigaction act; -- memset(&act, 0, sizeof(act)); -- act.sa_handler = CloseDupRfd; -- if (sigaction(SIGALRM, &act, &old_act_) < 0) { -- perror("sigaction:"); -- return(false); -- } else { -- restore_ = true; -- return(true); -- } - } - - bool GNUmakeTokenPool::Setup(bool ignore, - bool verbose, - double& max_load_average) { -- const char *value = getenv("MAKEFLAGS"); -+ const char *value = GetEnv("MAKEFLAGS"); - if (value) { - // GNU make <= 4.1 - const char *jobserver = strstr(value, "--jobserver-fds="); -@@ -119,20 +44,13 @@ bool GNUmakeTokenPool::Setup(bool ignore, - if (ignore) { - printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); - } else { -- int rfd = -1; -- int wfd = -1; -- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && -- CheckFd(rfd) && -- CheckFd(wfd) && -- SetAlarmHandler()) { -+ if (ParseAuth(jobserver)) { - const char *l_arg = strstr(value, " -l"); - int load_limit = -1; - - if (verbose) { - printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); - } -- rfd_ = rfd; -- wfd_ = wfd; - - // translate GNU make -lN to ninja -lN - if (l_arg && -@@ -154,83 +72,14 @@ bool GNUmakeTokenPool::Acquire() { - if (available_ > 0) - return true; - -- // Please read -- // -- // http://make.mad-scientist.net/papers/jobserver-implementation/ -- // -- // for the reasoning behind the following code. -- // -- // Try to read one character from the pipe. Returns true on success. -- // -- // First check if read() would succeed without blocking. --#ifdef USE_PPOLL -- pollfd pollfds[] = {{rfd_, POLLIN, 0}}; -- int ret = poll(pollfds, 1, 0); --#else -- fd_set set; -- struct timeval timeout = { 0, 0 }; -- FD_ZERO(&set); -- FD_SET(rfd_, &set); -- int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); --#endif -- if (ret > 0) { -- // Handle potential race condition: -- // - the above check succeeded, i.e. read() should not block -- // - the character disappears before we call read() -- // -- // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_ -- // can safely be closed by signal handlers without affecting rfd_. -- dup_rfd_ = dup(rfd_); -- -- if (dup_rfd_ != -1) { -- struct sigaction act, old_act; -- int ret = 0; -- -- // Temporarily replace SIGCHLD handler with our own -- memset(&act, 0, sizeof(act)); -- act.sa_handler = CloseDupRfd; -- if (sigaction(SIGCHLD, &act, &old_act) == 0) { -- struct itimerval timeout; -- -- // install a 100ms timeout that generates SIGALARM on expiration -- memset(&timeout, 0, sizeof(timeout)); -- timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] -- if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { -- char buf; -- -- // Now try to read() from dup_rfd_. Return values from read(): -- // -- // 1. token read -> 1 -- // 2. pipe closed -> 0 -- // 3. alarm expires -> -1 (EINTR) -- // 4. child exits -> -1 (EINTR) -- // 5. alarm expired before entering read() -> -1 (EBADF) -- // 6. child exited before entering read() -> -1 (EBADF) -- // 7. child exited before handler is installed -> go to 1 - 3 -- ret = read(dup_rfd_, &buf, 1); -- -- // disarm timer -- memset(&timeout, 0, sizeof(timeout)); -- setitimer(ITIMER_REAL, &timeout, NULL); -- } -- -- sigaction(SIGCHLD, &old_act, NULL); -- } -- -- CloseDupRfd(0); -- -- // Case 1 from above list -- if (ret > 0) { -- available_++; -- return true; -- } -- } -+ if (AcquireToken()) { -+ // token acquired -+ available_++; -+ return true; -+ } else { -+ // no token available -+ return false; - } -- -- // read() would block, i.e. no token available, -- // cases 2-6 from above list or -- // select() / poll() / dup() / sigaction() / setitimer() failed -- return false; - } - - void GNUmakeTokenPool::Reserve() { -@@ -239,15 +88,8 @@ void GNUmakeTokenPool::Reserve() { - } - - void GNUmakeTokenPool::Return() { -- const char buf = '+'; -- while (1) { -- int ret = write(wfd_, &buf, 1); -- if (ret > 0) -- available_--; -- if ((ret != -1) || (errno != EINTR)) -- return; -- // write got interrupted - retry -- } -+ if (ReturnToken()) -+ available_--; - } - - void GNUmakeTokenPool::Release() { -@@ -263,18 +105,3 @@ void GNUmakeTokenPool::Clear() { - while (available_ > 1) - Return(); - } -- --int GNUmakeTokenPool::GetMonitorFd() { -- return(rfd_); --} -- --struct TokenPool *TokenPool::Get(bool ignore, -- bool verbose, -- double& max_load_average) { -- GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; -- if (tokenpool->Setup(ignore, verbose, max_load_average)) -- return tokenpool; -- else -- delete tokenpool; -- return NULL; --} -diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h -new file mode 100644 -index 0000000000..d3852088e2 --- /dev/null +++ b/src/tokenpool-gnu-make.h @@ -0,0 +1,40 @@ @@ -3359,7 +2116,7 @@ index 0000000000..d3852088e2 +// interface to GNU make token pool +struct GNUmakeTokenPool : public TokenPool { + GNUmakeTokenPool(); -+ virtual ~GNUmakeTokenPool(); ++ ~GNUmakeTokenPool(); + + // token pool implementation + virtual bool Acquire(); @@ -3369,8 +2126,8 @@ index 0000000000..d3852088e2 + virtual bool Setup(bool ignore, bool verbose, double& max_load_average); + + // platform specific implementation -+ virtual const char *GetEnv(const char *name) = 0; -+ virtual bool ParseAuth(const char *jobserver) = 0; ++ virtual const char* GetEnv(const char* name) = 0; ++ virtual bool ParseAuth(const char* jobserver) = 0; + virtual bool AcquireToken() = 0; + virtual bool ReturnToken() = 0; + @@ -3380,332 +2137,6 @@ index 0000000000..d3852088e2 + + void Return(); +}; -diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc -index 4c592875b4..613d16882d 100644 ---- a/src/tokenpool-none.cc -+++ b/src/tokenpool-none.cc -@@ -17,8 +17,6 @@ - #include - - // No-op TokenPool implementation --struct TokenPool *TokenPool::Get(bool ignore, -- bool verbose, -- double& max_load_average) { -+struct TokenPool *TokenPool::Get() { - return NULL; - } -diff --git a/src/tokenpool.h b/src/tokenpool.h -index 4bf477f20c..1be8e1d5ce 100644 ---- a/src/tokenpool.h -+++ b/src/tokenpool.h -@@ -1,4 +1,4 @@ --// Copyright 2016-2017 Google Inc. All Rights Reserved. -+// Copyright 2016-2018 Google Inc. All Rights Reserved. - // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. -@@ -12,6 +12,10 @@ - // See the License for the specific language governing permissions and - // limitations under the License. - -+#ifdef _WIN32 -+#include -+#endif -+ - // interface to token pool - struct TokenPool { - virtual ~TokenPool() {} -@@ -21,14 +25,18 @@ struct TokenPool { - virtual void Release() = 0; - virtual void Clear() = 0; - -+ // returns false if token pool setup failed -+ virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0; -+ - #ifdef _WIN32 -- // @TODO -+ virtual void WaitForTokenAvailability(HANDLE ioport) = 0; -+ // returns true if a token has become available -+ // key is result from GetQueuedCompletionStatus() -+ virtual bool TokenIsAvailable(ULONG_PTR key) = 0; - #else - virtual int GetMonitorFd() = 0; - #endif - - // returns NULL if token pool is not available -- static struct TokenPool *Get(bool ignore, -- bool verbose, -- double& max_load_average); -+ static struct TokenPool *Get(); - }; -diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc -index 6c89064ca4..8d4fd7d33a 100644 ---- a/src/tokenpool_test.cc -+++ b/src/tokenpool_test.cc -@@ -16,13 +16,25 @@ - - #include "test.h" - --#ifndef _WIN32 -+#ifdef _WIN32 -+#include -+#else -+#include -+#endif -+ - #include - #include --#include - -+#ifdef _WIN32 -+// should contain all valid characters -+#define SEMAPHORE_NAME "abcdefghijklmnopqrstwxyz01234567890_" -+#define AUTH_FORMAT(tmpl) "foo " tmpl "=%s bar" -+#define ENVIRONMENT_CLEAR() SetEnvironmentVariable("MAKEFLAGS", NULL) -+#define ENVIRONMENT_INIT(v) SetEnvironmentVariable("MAKEFLAGS", v) -+#else -+#define AUTH_FORMAT(tmpl) "foo " tmpl "=%d,%d bar" - #define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") --#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true); -+#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true) - #endif - - namespace { -@@ -32,43 +44,60 @@ const double kLoadAverageDefault = -1.23456789; - struct TokenPoolTest : public testing::Test { - double load_avg_; - TokenPool *tokens_; --#ifndef _WIN32 - char buf_[1024]; -+#ifdef _WIN32 -+ const char *semaphore_name_; -+ HANDLE semaphore_; -+#else - int fds_[2]; - #endif - - virtual void SetUp() { - load_avg_ = kLoadAverageDefault; - tokens_ = NULL; --#ifndef _WIN32 - ENVIRONMENT_CLEAR(); -+#ifdef _WIN32 -+ semaphore_name_ = SEMAPHORE_NAME; -+ if ((semaphore_ = CreateSemaphore(0, 0, 2, SEMAPHORE_NAME)) == NULL) -+#else - if (pipe(fds_) < 0) -- ASSERT_TRUE(false); - #endif -+ ASSERT_TRUE(false); - } - -- void CreatePool(const char *format, bool ignore_jobserver) { --#ifndef _WIN32 -+ void CreatePool(const char *format, bool ignore_jobserver = false) { - if (format) { -- sprintf(buf_, format, fds_[0], fds_[1]); -+ sprintf(buf_, format, -+#ifdef _WIN32 -+ semaphore_name_ -+#else -+ fds_[0], fds_[1] -+#endif -+ ); - ENVIRONMENT_INIT(buf_); - } --#endif -- tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_); -+ if ((tokens_ = TokenPool::Get()) != NULL) { -+ if (!tokens_->Setup(ignore_jobserver, false, load_avg_)) { -+ delete tokens_; -+ tokens_ = NULL; -+ } -+ } - } - - void CreateDefaultPool() { -- CreatePool("foo --jobserver-auth=%d,%d bar", false); -+ CreatePool(AUTH_FORMAT("--jobserver-auth")); - } - - virtual void TearDown() { - if (tokens_) - delete tokens_; --#ifndef _WIN32 -+#ifdef _WIN32 -+ CloseHandle(semaphore_); -+#else - close(fds_[0]); - close(fds_[1]); -- ENVIRONMENT_CLEAR(); - #endif -+ ENVIRONMENT_CLEAR(); - } - }; - -@@ -82,10 +111,9 @@ TEST_F(TokenPoolTest, NoTokenPool) { - EXPECT_EQ(kLoadAverageDefault, load_avg_); - } - --#ifndef _WIN32 - TEST_F(TokenPoolTest, SuccessfulOldSetup) { - // GNUmake <= 4.1 -- CreatePool("foo --jobserver-fds=%d,%d bar", false); -+ CreatePool(AUTH_FORMAT("--jobserver-fds")); - - EXPECT_NE(NULL, tokens_); - EXPECT_EQ(kLoadAverageDefault, load_avg_); -@@ -100,19 +128,37 @@ TEST_F(TokenPoolTest, SuccessfulNewSetup) { - } - - TEST_F(TokenPoolTest, IgnoreWithJN) { -- CreatePool("foo --jobserver-auth=%d,%d bar", true); -+ CreatePool(AUTH_FORMAT("--jobserver-auth"), true); - - EXPECT_EQ(NULL, tokens_); - EXPECT_EQ(kLoadAverageDefault, load_avg_); - } - - TEST_F(TokenPoolTest, HonorLN) { -- CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false); -+ CreatePool(AUTH_FORMAT("-l9 --jobserver-auth")); - - EXPECT_NE(NULL, tokens_); - EXPECT_EQ(9.0, load_avg_); - } - -+#ifdef _WIN32 -+TEST_F(TokenPoolTest, SemaphoreNotFound) { -+ semaphore_name_ = SEMAPHORE_NAME "_foobar"; -+ CreateDefaultPool(); -+ -+ EXPECT_EQ(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, TokenIsAvailable) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ EXPECT_TRUE(tokens_->TokenIsAvailable((ULONG_PTR)tokens_)); -+} -+#else - TEST_F(TokenPoolTest, MonitorFD) { - CreateDefaultPool(); - -@@ -121,6 +167,7 @@ TEST_F(TokenPoolTest, MonitorFD) { - - EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); - } -+#endif - - TEST_F(TokenPoolTest, ImplicitToken) { - CreateDefaultPool(); -@@ -147,7 +194,13 @@ TEST_F(TokenPoolTest, TwoTokens) { - EXPECT_FALSE(tokens_->Acquire()); - - // jobserver offers 2nd token -+#ifdef _WIN32 -+ LONG previous; -+ ASSERT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); -+ ASSERT_EQ(0, previous); -+#else - ASSERT_EQ(1u, write(fds_[1], "T", 1)); -+#endif - EXPECT_TRUE(tokens_->Acquire()); - tokens_->Reserve(); - EXPECT_FALSE(tokens_->Acquire()); -@@ -160,8 +213,14 @@ TEST_F(TokenPoolTest, TwoTokens) { - tokens_->Release(); - EXPECT_TRUE(tokens_->Acquire()); - -- // there must be one token in the pipe -+ // there must be one token available -+#ifdef _WIN32 -+ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); -+ EXPECT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); -+ EXPECT_EQ(0, previous); -+#else - EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); -+#endif - - // implicit token - EXPECT_TRUE(tokens_->Acquire()); -@@ -179,7 +238,13 @@ TEST_F(TokenPoolTest, Clear) { - EXPECT_FALSE(tokens_->Acquire()); - - // jobserver offers 2nd & 3rd token -+#ifdef _WIN32 -+ LONG previous; -+ ASSERT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); -+ ASSERT_EQ(0, previous); -+#else - ASSERT_EQ(2u, write(fds_[1], "TT", 2)); -+#endif - EXPECT_TRUE(tokens_->Acquire()); - tokens_->Reserve(); - EXPECT_TRUE(tokens_->Acquire()); -@@ -189,10 +254,16 @@ TEST_F(TokenPoolTest, Clear) { - tokens_->Clear(); - EXPECT_TRUE(tokens_->Acquire()); - -- // there must be two tokens in the pipe -+ // there must be two tokens available -+#ifdef _WIN32 -+ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); -+ EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); -+ EXPECT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); -+ EXPECT_EQ(0, previous); -+#else - EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); -+#endif - - // implicit token - EXPECT_TRUE(tokens_->Acquire()); - } --#endif - -From 2b9c81c0ec1226d8795e7725529f13be41eaa385 Mon Sep 17 00:00:00 2001 -From: Stefan Becker -Date: Fri, 14 Dec 2018 13:27:11 +0200 -Subject: [PATCH 11/11] Prepare PR for merging - part II - -- remove unnecessary "struct" from TokenPool -- add PAPCFUNC cast to QueryUserAPC() -- remove hard-coded MAKEFLAGS string from win32 -- remove useless build test CompleteNoWork -- rename TokenPoolTest to TestTokenPool -- add tokenpool modules to CMake build -- remove unused no-op TokenPool implementation -- fix errors flagged by codespell & clang-tidy -- address review comments from - -https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-195195803 -https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-185089255 -https://github.com/ninja-build/ninja/pull/1140#issuecomment-473898963 -https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 ---- - CMakeLists.txt | 8 ++++- - src/build.cc | 2 +- - src/build_test.cc | 12 +------ - src/subprocess-posix.cc | 4 +-- - src/subprocess-win32.cc | 2 +- - src/subprocess.h | 2 +- - src/subprocess_test.cc | 26 +++++++------- - src/tokenpool-gnu-make-posix.cc | 21 +++++------ - src/tokenpool-gnu-make-win32.cc | 36 ++++++++++--------- - src/tokenpool-gnu-make.cc | 63 +++++++++++++++++---------------- - src/tokenpool-gnu-make.h | 6 ++-- - src/tokenpool-none.cc | 22 ------------ - src/tokenpool.h | 2 +- - src/tokenpool_test.cc | 8 ++--- - 14 files changed, 94 insertions(+), 120 deletions(-) - delete mode 100644 src/tokenpool-none.cc - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 57ae548f5b..e2876fe413 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,7 @@ add_library(libninja OBJECT @@ -3716,14 +2147,14 @@ index 57ae548f5b..e2876fe413 100644 src/util.cc src/version.cc ) -@@ -123,9 +124,13 @@ if(WIN32) +@@ -123,9 +124,14 @@ if(WIN32) src/msvc_helper_main-win32.cc src/getopt.c src/minidump-win32.cc + src/tokenpool-gnu-make-win32.cc ) else() -- target_sources(libninja PRIVATE src/subprocess-posix.cc) + target_sources(libninja PRIVATE src/subprocess-posix.cc) + target_sources(libninja PRIVATE + src/subprocess-posix.cc + src/tokenpool-gnu-make-posix.cc @@ -3731,7 +2162,7 @@ index 57ae548f5b..e2876fe413 100644 if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") target_sources(libninja PRIVATE src/getopt.c) endif() -@@ -204,6 +209,7 @@ if(BUILD_TESTING) +@@ -204,6 +210,7 @@ if(BUILD_TESTING) src/string_piece_util_test.cc src/subprocess_test.cc src/test.cc @@ -3739,552 +2170,3 @@ index 57ae548f5b..e2876fe413 100644 src/util_test.cc ) if(WIN32) -diff --git a/src/build.cc b/src/build.cc -index 20c3bdc2a0..854df08c2a 100644 ---- a/src/build.cc -+++ b/src/build.cc -@@ -467,7 +467,7 @@ struct RealCommandRunner : public CommandRunner { - // copy of config_.max_load_average; can be modified by TokenPool setup - double max_load_average_; - SubprocessSet subprocs_; -- TokenPool *tokens_; -+ TokenPool* tokens_; - map subproc_to_edge_; - }; - -diff --git a/src/build_test.cc b/src/build_test.cc -index dd41dfbe1d..8901c9518f 100644 ---- a/src/build_test.cc -+++ b/src/build_test.cc -@@ -4098,7 +4098,7 @@ struct BuildTokenTest : public BuildTest { - void ExpectWaitForCommand(int count, ...); - - private: -- void EnqueueBooleans(vector& booleans, int count, va_list ao); -+ void EnqueueBooleans(vector& booleans, int count, va_list ap); - }; - - void BuildTokenTest::SetUp() { -@@ -4144,16 +4144,6 @@ void BuildTokenTest::EnqueueBooleans(vector& booleans, int count, va_list - } - } - --TEST_F(BuildTokenTest, CompleteNoWork) { -- // plan should not execute anything -- string err; -- -- EXPECT_TRUE(builder_.Build(&err)); -- EXPECT_EQ("", err); -- -- EXPECT_EQ(0u, token_command_runner_.commands_ran_.size()); --} -- - TEST_F(BuildTokenTest, DoNotAquireToken) { - // plan should execute one command - string err; -diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc -index 74451b0be2..31839276c4 100644 ---- a/src/subprocess-posix.cc -+++ b/src/subprocess-posix.cc -@@ -250,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { - } - - #ifdef USE_PPOLL --bool SubprocessSet::DoWork(struct TokenPool* tokens) { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - vector fds; - nfds_t nfds = 0; - -@@ -315,7 +315,7 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { - } - - #else // !defined(USE_PPOLL) --bool SubprocessSet::DoWork(struct TokenPool* tokens) { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - fd_set set; - int nfds = 0; - FD_ZERO(&set); -diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc -index ce3e2c20a4..2926e9a221 100644 ---- a/src/subprocess-win32.cc -+++ b/src/subprocess-win32.cc -@@ -252,7 +252,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { - return subprocess; - } - --bool SubprocessSet::DoWork(struct TokenPool* tokens) { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - DWORD bytes_read; - Subprocess* subproc; - OVERLAPPED* overlapped; -diff --git a/src/subprocess.h b/src/subprocess.h -index 9ea67ea477..1ec78171e8 100644 ---- a/src/subprocess.h -+++ b/src/subprocess.h -@@ -86,7 +86,7 @@ struct SubprocessSet { - ~SubprocessSet(); - - Subprocess* Add(const std::string& command, bool use_console = false); -- bool DoWork(struct TokenPool* tokens); -+ bool DoWork(TokenPool* tokens); - Subprocess* NextFinished(); - void Clear(); - -diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc -index f625963462..7b146f31be 100644 ---- a/src/subprocess_test.cc -+++ b/src/subprocess_test.cc -@@ -35,7 +35,7 @@ const char* kSimpleCommand = "cmd /c dir \\"; - const char* kSimpleCommand = "ls /"; - #endif - --struct TokenPoolTest : public TokenPool { -+struct TestTokenPool : public TokenPool { - bool Acquire() { return false; } - void Reserve() {} - void Release() {} -@@ -58,7 +58,7 @@ struct TokenPoolTest : public TokenPool { - - struct SubprocessTest : public testing::Test { - SubprocessSet subprocs_; -- TokenPoolTest tokens_; -+ TestTokenPool tokens_; - }; - - } // anonymous namespace -@@ -73,7 +73,7 @@ TEST_F(SubprocessTest, BadCommandStderr) { - // Pretend we discovered that stderr was ready for writing. - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -89,7 +89,7 @@ TEST_F(SubprocessTest, NoSuchCommand) { - // Pretend we discovered that stderr was ready for writing. - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -109,7 +109,7 @@ TEST_F(SubprocessTest, InterruptChild) { - while (!subproc->Done()) { - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -135,7 +135,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) { - while (!subproc->Done()) { - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -161,7 +161,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) { - while (!subproc->Done()) { - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -190,7 +190,7 @@ TEST_F(SubprocessTest, Console) { - while (!subproc->Done()) { - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitSuccess, subproc->Finish()); - } -@@ -206,7 +206,7 @@ TEST_F(SubprocessTest, SetWithSingle) { - while (!subproc->Done()) { - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_NE("", subproc->GetOutput()); - -@@ -243,7 +243,7 @@ TEST_F(SubprocessTest, SetWithMulti) { - ASSERT_GT(subprocs_.running_.size(), 0u); - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(0u, subprocs_.running_.size()); - ASSERT_EQ(3u, subprocs_.finished_.size()); - -@@ -278,7 +278,7 @@ TEST_F(SubprocessTest, SetWithLots) { - subprocs_.ResetTokenAvailable(); - while (!subprocs_.running_.empty()) - subprocs_.DoWork(NULL); -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - for (size_t i = 0; i < procs.size(); ++i) { - ASSERT_EQ(ExitSuccess, procs[i]->Finish()); - ASSERT_NE("", procs[i]->GetOutput()); -@@ -298,7 +298,7 @@ TEST_F(SubprocessTest, ReadStdin) { - while (!subproc->Done()) { - subprocs_.DoWork(NULL); - } -- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_EQ(1u, subprocs_.finished_.size()); - } -@@ -322,7 +322,7 @@ TEST_F(SubprocessTest, TokenAvailable) { - subprocs_.DoWork(&tokens_); - #ifdef _WIN32 - tokens_._token_available = false; -- // we need to loop here as we have no conrol where the token -+ // we need to loop here as we have no control where the token - // I/O completion post ends up in the queue - while (!subproc->Done() && !subprocs_.IsTokenAvailable()) { - subprocs_.DoWork(&tokens_); -diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc -index 70d84bfff7..353bda226a 100644 ---- a/src/tokenpool-gnu-make-posix.cc -+++ b/src/tokenpool-gnu-make-posix.cc -@@ -32,8 +32,8 @@ struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool { - - virtual int GetMonitorFd(); - -- virtual const char *GetEnv(const char *name) { return getenv(name); }; -- virtual bool ParseAuth(const char *jobserver); -+ virtual const char* GetEnv(const char* name) { return getenv(name); }; -+ virtual bool ParseAuth(const char* jobserver); - virtual bool AcquireToken(); - virtual bool ReturnToken(); - -@@ -64,9 +64,7 @@ bool GNUmakeTokenPoolPosix::CheckFd(int fd) { - if (fd < 0) - return false; - int ret = fcntl(fd, F_GETFD); -- if (ret < 0) -- return false; -- return true; -+ return ret >= 0; - } - - int GNUmakeTokenPoolPosix::dup_rfd_ = -1; -@@ -82,14 +80,13 @@ bool GNUmakeTokenPoolPosix::SetAlarmHandler() { - act.sa_handler = CloseDupRfd; - if (sigaction(SIGALRM, &act, &old_act_) < 0) { - perror("sigaction:"); -- return(false); -- } else { -- restore_ = true; -- return(true); -+ return false; - } -+ restore_ = true; -+ return true; - } - --bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) { -+bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) { - int rfd = -1; - int wfd = -1; - if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && -@@ -195,9 +192,9 @@ bool GNUmakeTokenPoolPosix::ReturnToken() { - } - - int GNUmakeTokenPoolPosix::GetMonitorFd() { -- return(rfd_); -+ return rfd_; - } - --struct TokenPool *TokenPool::Get() { -+TokenPool* TokenPool::Get() { - return new GNUmakeTokenPoolPosix; - } -diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc -index 2719f2c1fc..b2bb52fadb 100644 ---- a/src/tokenpool-gnu-make-win32.cc -+++ b/src/tokenpool-gnu-make-win32.cc -@@ -14,7 +14,8 @@ - - #include "tokenpool-gnu-make.h" - --// always include first to make sure other headers do the correct thing... -+// Always include this first. -+// Otherwise the other system headers don't work correctly under Win32 - #include - - #include -@@ -32,8 +33,8 @@ struct GNUmakeTokenPoolWin32 : public GNUmakeTokenPool { - virtual void WaitForTokenAvailability(HANDLE ioport); - virtual bool TokenIsAvailable(ULONG_PTR key); - -- virtual const char *GetEnv(const char *name); -- virtual bool ParseAuth(const char *jobserver); -+ virtual const char* GetEnv(const char* name); -+ virtual bool ParseAuth(const char* jobserver); - virtual bool AcquireToken(); - virtual bool ReturnToken(); - -@@ -98,19 +99,19 @@ GNUmakeTokenPoolWin32::~GNUmakeTokenPoolWin32() { - } - } - --const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) { -+const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) { - // getenv() does not work correctly together with tokenpool_tests.cc - static char buffer[MAX_PATH + 1]; -- if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0) -+ if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0) - return NULL; -- return(buffer); -+ return buffer; - } - --bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { -+bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) { - // match "--jobserver-auth=gmake_semaphore_..." -- const char *start = strchr(jobserver, '='); -+ const char* start = strchr(jobserver, '='); - if (start) { -- const char *end = start; -+ const char* end = start; - unsigned int len; - char c, *auth; - -@@ -119,14 +120,15 @@ bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { - break; - len = end - start; // includes string terminator in count - -- if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) { -+ if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) { - strncpy(auth, start + 1, len - 1); - auth[len - 1] = '\0'; - -- if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ -- FALSE, /* Child processes DON'T inherit */ -- auth /* Semaphore name */ -- )) != NULL) { -+ if ((semaphore_jobserver_ = -+ OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ -+ FALSE, /* Child processes DON'T inherit */ -+ auth /* Semaphore name */ -+ )) != NULL) { - free(auth); - return true; - } -@@ -171,7 +173,7 @@ DWORD GNUmakeTokenPoolWin32::SemaphoreThread() { - } - - DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) { -- GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param; -+ GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param; - return This->SemaphoreThread(); - } - -@@ -216,7 +218,7 @@ void GNUmakeTokenPoolWin32::WaitForTokenAvailability(HANDLE ioport) { - - bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) { - // alert child thread to break wait on token semaphore -- QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL); -+ QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL); - - // return true when GetQueuedCompletionStatus() returned our key - return key == (ULONG_PTR) this; -@@ -232,6 +234,6 @@ void GNUmakeTokenPoolWin32::WaitForObject(HANDLE object) { - Win32Fatal("WaitForSingleObject"); - } - --struct TokenPool *TokenPool::Get() { -+TokenPool* TokenPool::Get() { - return new GNUmakeTokenPoolWin32; - } -diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc -index 92ff611721..60e0552924 100644 ---- a/src/tokenpool-gnu-make.cc -+++ b/src/tokenpool-gnu-make.cc -@@ -31,36 +31,37 @@ GNUmakeTokenPool::~GNUmakeTokenPool() { - bool GNUmakeTokenPool::Setup(bool ignore, - bool verbose, - double& max_load_average) { -- const char *value = GetEnv("MAKEFLAGS"); -- if (value) { -- // GNU make <= 4.1 -- const char *jobserver = strstr(value, "--jobserver-fds="); -+ const char* value = GetEnv("MAKEFLAGS"); -+ if (!value) -+ return false; -+ -+ // GNU make <= 4.1 -+ const char* jobserver = strstr(value, "--jobserver-fds="); -+ if (!jobserver) - // GNU make => 4.2 -- if (!jobserver) -- jobserver = strstr(value, "--jobserver-auth="); -- if (jobserver) { -- LinePrinter printer; -- -- if (ignore) { -- printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); -- } else { -- if (ParseAuth(jobserver)) { -- const char *l_arg = strstr(value, " -l"); -- int load_limit = -1; -- -- if (verbose) { -- printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); -- } -- -- // translate GNU make -lN to ninja -lN -- if (l_arg && -- (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && -- (load_limit > 0)) { -- max_load_average = load_limit; -- } -- -- return true; -+ jobserver = strstr(value, "--jobserver-auth="); -+ if (jobserver) { -+ LinePrinter printer; -+ -+ if (ignore) { -+ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); -+ } else { -+ if (ParseAuth(jobserver)) { -+ const char* l_arg = strstr(value, " -l"); -+ int load_limit = -1; -+ -+ if (verbose) { -+ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); -+ } -+ -+ // translate GNU make -lN to ninja -lN -+ if (l_arg && -+ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && -+ (load_limit > 0)) { -+ max_load_average = load_limit; - } -+ -+ return true; - } - } - } -@@ -76,10 +77,10 @@ bool GNUmakeTokenPool::Acquire() { - // token acquired - available_++; - return true; -- } else { -- // no token available -- return false; - } -+ -+ // no token available -+ return false; - } - - void GNUmakeTokenPool::Reserve() { -diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h -index d3852088e2..c94cca5e2d 100644 ---- a/src/tokenpool-gnu-make.h -+++ b/src/tokenpool-gnu-make.h -@@ -17,7 +17,7 @@ - // interface to GNU make token pool - struct GNUmakeTokenPool : public TokenPool { - GNUmakeTokenPool(); -- virtual ~GNUmakeTokenPool(); -+ ~GNUmakeTokenPool(); - - // token pool implementation - virtual bool Acquire(); -@@ -27,8 +27,8 @@ struct GNUmakeTokenPool : public TokenPool { - virtual bool Setup(bool ignore, bool verbose, double& max_load_average); - - // platform specific implementation -- virtual const char *GetEnv(const char *name) = 0; -- virtual bool ParseAuth(const char *jobserver) = 0; -+ virtual const char* GetEnv(const char* name) = 0; -+ virtual bool ParseAuth(const char* jobserver) = 0; - virtual bool AcquireToken() = 0; - virtual bool ReturnToken() = 0; - -diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc -deleted file mode 100644 -index 613d16882d..0000000000 ---- a/src/tokenpool-none.cc -+++ /dev/null -@@ -1,22 +0,0 @@ --// Copyright 2016-2018 Google Inc. All Rights Reserved. --// --// Licensed under the Apache License, Version 2.0 (the "License"); --// you may not use this file except in compliance with the License. --// You may obtain a copy of the License at --// --// http://www.apache.org/licenses/LICENSE-2.0 --// --// Unless required by applicable law or agreed to in writing, software --// distributed under the License is distributed on an "AS IS" BASIS, --// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --// See the License for the specific language governing permissions and --// limitations under the License. -- --#include "tokenpool.h" -- --#include -- --// No-op TokenPool implementation --struct TokenPool *TokenPool::Get() { -- return NULL; --} -diff --git a/src/tokenpool.h b/src/tokenpool.h -index 1be8e1d5ce..931c22754d 100644 ---- a/src/tokenpool.h -+++ b/src/tokenpool.h -@@ -38,5 +38,5 @@ struct TokenPool { - #endif - - // returns NULL if token pool is not available -- static struct TokenPool *Get(); -+ static TokenPool* Get(); - }; -diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc -index 8d4fd7d33a..8d3061cb30 100644 ---- a/src/tokenpool_test.cc -+++ b/src/tokenpool_test.cc -@@ -43,10 +43,10 @@ const double kLoadAverageDefault = -1.23456789; - - struct TokenPoolTest : public testing::Test { - double load_avg_; -- TokenPool *tokens_; -+ TokenPool* tokens_; - char buf_[1024]; - #ifdef _WIN32 -- const char *semaphore_name_; -+ const char* semaphore_name_; - HANDLE semaphore_; - #else - int fds_[2]; -@@ -65,7 +65,7 @@ struct TokenPoolTest : public testing::Test { - ASSERT_TRUE(false); - } - -- void CreatePool(const char *format, bool ignore_jobserver = false) { -+ void CreatePool(const char* format, bool ignore_jobserver = false) { - if (format) { - sprintf(buf_, format, - #ifdef _WIN32 -@@ -209,7 +209,7 @@ TEST_F(TokenPoolTest, TwoTokens) { - tokens_->Release(); - EXPECT_TRUE(tokens_->Acquire()); - -- // release implict token - must return 2nd token back to jobserver -+ // release implicit token - must return 2nd token back to jobserver - tokens_->Release(); - EXPECT_TRUE(tokens_->Acquire()); -