From 6ae2a8bbd84980955d843ecc3e771539bc378528 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 29 May 2022 17:54:31 +0800 Subject: [PATCH 01/31] firewall: dos2unix Signed-off-by: Tianling Shen (cherry picked from commit dccc926e3220af620242c441a304db91b2f12b5a) --- .../001-firewall3-fix-locking-issue.patch | 76 +++++++++---------- ...ullconenat.patch => 100-fullconenat.patch} | 0 2 files changed, 38 insertions(+), 38 deletions(-) rename package/network/config/firewall/patches/{0100-fullconenat.patch => 100-fullconenat.patch} (100%) diff --git a/package/network/config/firewall/patches/001-firewall3-fix-locking-issue.patch b/package/network/config/firewall/patches/001-firewall3-fix-locking-issue.patch index 8ce18407ca..8657b5c710 100644 --- a/package/network/config/firewall/patches/001-firewall3-fix-locking-issue.patch +++ b/package/network/config/firewall/patches/001-firewall3-fix-locking-issue.patch @@ -1,38 +1,38 @@ -From df1306a96127e91ff2d513a0a67345baaf61d113 Mon Sep 17 00:00:00 2001 -From: Florian Eckert -Date: Fri, 19 Nov 2021 09:51:02 +0100 -Subject: [PATCH] firewall3: fix locking issue - -By calling the command 'fw3 reload' several times at the same time, I -noticed that the locking was not working properly. It happened from time -to time that some firewall rules were present twice in the system! - -By removing the 'unlink' systemcall, this error no longer occurred on my -systems. - -Since fw3 does not run as a service, it makes no sense to delete this -lock file every time a filehandler is no longer open on this lock file, -because fw3 binary is not running. - -If fw3 does run as a service then we can remove this lock file on -service stop. But this is not the case for fw3. - -Signed-off-by: Florian Eckert ---- - utils.c | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/utils.c b/utils.c -index 17d5bf9..92e966c 100644 ---- a/utils.c -+++ b/utils.c -@@ -397,7 +397,6 @@ fw3_unlock_path(int *fd, const char *lockpath) - warn("Cannot release exclusive lock: %s", strerror(errno)); - - close(*fd); -- unlink(FW3_LOCKFILE); - - *fd = -1; - } --- -2.30.2 +From df1306a96127e91ff2d513a0a67345baaf61d113 Mon Sep 17 00:00:00 2001 +From: Florian Eckert +Date: Fri, 19 Nov 2021 09:51:02 +0100 +Subject: [PATCH] firewall3: fix locking issue + +By calling the command 'fw3 reload' several times at the same time, I +noticed that the locking was not working properly. It happened from time +to time that some firewall rules were present twice in the system! + +By removing the 'unlink' systemcall, this error no longer occurred on my +systems. + +Since fw3 does not run as a service, it makes no sense to delete this +lock file every time a filehandler is no longer open on this lock file, +because fw3 binary is not running. + +If fw3 does run as a service then we can remove this lock file on +service stop. But this is not the case for fw3. + +Signed-off-by: Florian Eckert +--- + utils.c | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/utils.c b/utils.c +index 17d5bf9..92e966c 100644 +--- a/utils.c ++++ b/utils.c +@@ -397,7 +397,6 @@ fw3_unlock_path(int *fd, const char *lockpath) + warn("Cannot release exclusive lock: %s", strerror(errno)); + + close(*fd); +- unlink(FW3_LOCKFILE); + + *fd = -1; + } +-- +2.30.2 diff --git a/package/network/config/firewall/patches/0100-fullconenat.patch b/package/network/config/firewall/patches/100-fullconenat.patch similarity index 100% rename from package/network/config/firewall/patches/0100-fullconenat.patch rename to package/network/config/firewall/patches/100-fullconenat.patch From c41c33dd59e69dcb1338f54dbab03974979747a2 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 29 May 2022 19:28:09 +0800 Subject: [PATCH 02/31] kernel: refresh sfp patch Signed-off-by: Tianling Shen --- ...ve-fwnode-parsing-into-sfp-bus-layer.patch | 7 +++- ....17-net-sfp-ignore-disabled-SFP-node.patch | 38 ------------------- 2 files changed, 6 insertions(+), 39 deletions(-) delete mode 100644 target/linux/generic/backport-4.19/745-v5.17-net-sfp-ignore-disabled-SFP-node.patch diff --git a/target/linux/generic/backport-4.19/716-v5.4-net-sfp-move-fwnode-parsing-into-sfp-bus-layer.patch b/target/linux/generic/backport-4.19/716-v5.4-net-sfp-move-fwnode-parsing-into-sfp-bus-layer.patch index 4825227ee4..ec18ac9858 100644 --- a/target/linux/generic/backport-4.19/716-v5.4-net-sfp-move-fwnode-parsing-into-sfp-bus-layer.patch +++ b/target/linux/generic/backport-4.19/716-v5.4-net-sfp-move-fwnode-parsing-into-sfp-bus-layer.patch @@ -64,7 +64,7 @@ Signed-off-by: Russell King #include #include -@@ -519,45 +520,63 @@ static void sfp_upstream_clear(struct sf +@@ -519,45 +520,68 @@ static void sfp_upstream_clear(struct sf } /** @@ -122,6 +122,11 @@ Signed-off-by: Russell King + else if (ret < 0) + return ERR_PTR(ret); + ++ if (!fwnode_device_is_available(ref.fwnode)) { ++ fwnode_handle_put(ref.fwnode); ++ return NULL; ++ } ++ + bus = sfp_bus_get(ref.fwnode); + fwnode_handle_put(ref.fwnode); + if (!bus) diff --git a/target/linux/generic/backport-4.19/745-v5.17-net-sfp-ignore-disabled-SFP-node.patch b/target/linux/generic/backport-4.19/745-v5.17-net-sfp-ignore-disabled-SFP-node.patch deleted file mode 100644 index 0a7af3fa93..0000000000 --- a/target/linux/generic/backport-4.19/745-v5.17-net-sfp-ignore-disabled-SFP-node.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 2148927e6ed43a1667baf7c2ae3e0e05a44b51a0 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Marek=20Beh=C3=BAn?= -Date: Wed, 19 Jan 2022 17:44:55 +0100 -Subject: net: sfp: ignore disabled SFP node -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -Commit ce0aa27ff3f6 ("sfp: add sfp-bus to bridge between network devices -and sfp cages") added code which finds SFP bus DT node even if the node -is disabled with status = "disabled". Because of this, when phylink is -created, it ends with non-null .sfp_bus member, even though the SFP -module is not probed (because the node is disabled). - -We need to ignore disabled SFP bus node. - -Fixes: ce0aa27ff3f6 ("sfp: add sfp-bus to bridge between network devices and sfp cages") -Signed-off-by: Marek Behún -Cc: stable@vger.kernel.org # 2203cbf2c8b5 ("net: sfp: move fwnode parsing into sfp-bus layer") -Signed-off-by: David S. Miller ---- - drivers/net/phy/sfp-bus.c | 5 +++++ - 1 file changed, 5 insertions(+) - ---- a/drivers/net/phy/sfp-bus.c -+++ b/drivers/net/phy/sfp-bus.c -@@ -557,6 +557,11 @@ struct sfp_bus *sfp_bus_find_fwnode(stru - else if (ret < 0) - return ERR_PTR(ret); - -+ if (!fwnode_device_is_available(ref.fwnode)) { -+ fwnode_handle_put(ref.fwnode); -+ return NULL; -+ } -+ - bus = sfp_bus_get(ref.fwnode); - fwnode_handle_put(ref.fwnode); - if (!bus) From c0f630a98f37e10581df50754443cb68dee4eea9 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 13 May 2022 22:07:59 +0200 Subject: [PATCH 03/31] linux-firmware: take linux-firmware.git's qca99x0 boardfile Kalle Valo managed to add the qca9980's boardfile in the upstream repository. Sourcing the file from his repository is no longer needed. Signed-off-by: Christian Lamparter (cherry picked from commit 4ba7f6d9cbd74adc82ab3064cc4c9f6ec5eb51a6) --- package/firmware/linux-firmware/Makefile | 1 + package/firmware/linux-firmware/qca_ath10k.mk | 16 +--------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/package/firmware/linux-firmware/Makefile b/package/firmware/linux-firmware/Makefile index 6d474b6452..7b3b8ee721 100644 --- a/package/firmware/linux-firmware/Makefile +++ b/package/firmware/linux-firmware/Makefile @@ -26,6 +26,7 @@ define Package/firmware-default CATEGORY:=Firmware URL:=http://git.kernel.org/cgit/linux/kernel/git/firmware/linux-firmware.git TITLE:=$(1) + DEPENDS:=$(2) endef define Build/Compile diff --git a/package/firmware/linux-firmware/qca_ath10k.mk b/package/firmware/linux-firmware/qca_ath10k.mk index a7c5b13b84..2296ba1a6a 100644 --- a/package/firmware/linux-firmware/qca_ath10k.mk +++ b/package/firmware/linux-firmware/qca_ath10k.mk @@ -1,14 +1,3 @@ -QCA99X0_BOARD_REV:=ddcec9efd245da9365c474f513a855a55f3ac7fe -QCA99X0_BOARD_FILE:=board-2.bin.$(QCA99X0_BOARD_REV) - -define Download/qca99x0-board - URL:=https://source.codeaurora.org/quic/qsdk/oss/firmware/ath10k-firmware/plain/ath10k/QCA99X0/hw2.0 - URL_FILE:=board-2.bin?id=$(QCA99X0_BOARD_REV) - FILE:=$(QCA99X0_BOARD_FILE) - HASH:=03711ac21e60ef59d3815e235eb721c0c22851b5410299411085aa6f2af45401 -endef -$(eval $(call Download,qca99x0-board)) - Package/ath10k-firmware-qca4019 = $(call Package/firmware-default,ath10k qca4019 firmware) define Package/ath10k-firmware-qca4019/install $(INSTALL_DIR) $(1)/lib/firmware/ath10k/QCA4019/hw1.0 @@ -80,11 +69,8 @@ Package/ath10k-firmware-qca99x0 = $(call Package/firmware-default,ath10k qca99x0 define Package/ath10k-firmware-qca99x0/install $(INSTALL_DIR) $(1)/lib/firmware/ath10k/QCA99X0/hw2.0 $(INSTALL_DATA) \ - $(DL_DIR)/$(QCA99X0_BOARD_FILE) \ + $(PKG_BUILD_DIR)/ath10k/QCA99X0/hw2.0/board-2.bin \ $(1)/lib/firmware/ath10k/QCA99X0/hw2.0/board-2.bin - $(INSTALL_DATA) \ - $(PKG_BUILD_DIR)/ath10k/QCA99X0/hw2.0/board.bin \ - $(1)/lib/firmware/ath10k/QCA99X0/hw2.0/board.bin $(INSTALL_DATA) \ $(PKG_BUILD_DIR)/ath10k/QCA99X0/hw2.0/firmware-5.bin \ $(1)/lib/firmware/ath10k/QCA99X0/hw2.0/firmware-5.bin From 367dacfe9cd42dcb4c7c1a13693569cec9e80045 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sat, 9 Apr 2022 06:32:21 +0800 Subject: [PATCH 04/31] tools: drop ucl and upx Signed-off-by: Tianling Shen (cherry picked from commit f92fd54dbbec56d0f0be9c5b70d31d6c6b4de92c) --- README.md | 1 - tools/Makefile | 5 ++--- tools/ucl/Makefile | 49 ---------------------------------------------- tools/upx/Makefile | 35 --------------------------------- 4 files changed, 2 insertions(+), 88 deletions(-) delete mode 100644 tools/ucl/Makefile delete mode 100644 tools/upx/Makefile diff --git a/README.md b/README.md index 71b6762e00..51915516ec 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ To build your own firmware you need a GNU/Linux, BSD or MacOSX system (case sens - You must __not__ have spaces in PATH or in the work folders on the drive. - If you're using Windows Subsystem for Linux (or WSL), removing Windows folders from PATH is required, please see [Build system setup WSL](https://openwrt.org/docs/guide-developer/build-system/wsl) documentation. - Using macOS as the host build OS is __not__ recommended. No warranty at all. You can get tips from [Build system setup macOS](https://openwrt.org/docs/guide-developer/build-system/buildroot.exigence.macosx) documentation. - - As you're building ImmortalWrt, patching or disabling UPX tools is also required. - For more details, please see [Build system setup](https://openwrt.org/docs/guide-developer/build-system/install-buildsystem) documentation. ### Quickstart diff --git a/tools/Makefile b/tools/Makefile index 960fba137a..ea64411922 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -24,8 +24,8 @@ endif tools-y += autoconf autoconf-archive automake bc bison cmake cpio dosfstools tools-y += e2fsprogs fakeroot findutils firmware-utils flex gengetopt tools-y += libressl libtool lzma m4 make-ext4fs meson missing-macros mkimage -tools-y += mklibs mtd-utils mtools ninja padjffs2 patch-image patchelf -tools-y += pkgconf quilt squashfskit4 sstrip ucl upx xxd zip zlib zstd +tools-y += mklibs mtd-utils mtools ninja padjffs2 patch-image +tools-y += patchelf pkgconf quilt squashfskit4 sstrip xxd zip zlib zstd tools-$(BUILD_B43_TOOLS) += b43-tools tools-$(BUILD_ISL) += isl tools-$(BUILD_TOOLCHAIN) += expat gmp mpc mpfr @@ -74,7 +74,6 @@ $(curdir)/quilt/compile := $(curdir)/autoconf/compile $(curdir)/findutils/compil $(curdir)/sdcc/compile := $(curdir)/bison/compile $(curdir)/squashfs/compile := $(curdir)/lzma-old/compile $(curdir)/squashfskit4/compile := $(curdir)/xz/compile $(curdir)/zlib/compile -$(curdir)/upx/compile := $(curdir)/ucl/compile $(curdir)/wrt350nv2-builder/compile := $(curdir)/zlib/compile $(curdir)/zlib/compile := $(curdir)/cmake/compile $(curdir)/zstd/compile := $(curdir)/meson/compile diff --git a/tools/ucl/Makefile b/tools/ucl/Makefile deleted file mode 100644 index cf1c25bd8d..0000000000 --- a/tools/ucl/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (C) 2019 OpenWrt.org -# -# This is free software, licensed under the GNU General Public License v2. -# See /LICENSE for more information. -# -include $(TOPDIR)/rules.mk - -PKG_NAME:=ucl -PKG_VERSION:=1.03 - -PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz -PKG_SOURCE_URL:=http://www.oberhumer.com/opensource/ucl/download/$(PKG_SOURCE) -PKG_HASH:=b865299ffd45d73412293369c9754b07637680e5c826915f097577cd27350348 - -HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/$(PKG_NAME)-$(PKG_VERSION) - -include $(INCLUDE_DIR)/host-build.mk - -HOSTCC :=gcc -HOST_CFLAGS +=-std=gnu89 - -define Host/Prepare - $(Host/Prepare/Default) - mkdir -p $(STAGING_DIR_HOST)/include/ucl -endef - -define Host/Configure - (cd $(HOST_BUILD_DIR); \ - CC="$(HOSTCC)" \ - CFLAGS="$(HOST_CFLAGS)" \ - ./configure --prefix=$(STAGING_DIR_HOST) \ - ); - $(call Host/Configure/Default) -endef - -define Host/Compile - $(MAKE) -C $(HOST_BUILD_DIR) -endef - -define Host/Install - $(MAKE) -C $(HOST_BUILD_DIR) install -endef - -define Host/Clean - rm -rf $(STAGING_DIR_HOST)/include/ucl -endef - -$(eval $(call HostBuild)) diff --git a/tools/upx/Makefile b/tools/upx/Makefile deleted file mode 100644 index fbf1dfbd2e..0000000000 --- a/tools/upx/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (C) 2011-2020 OpenWrt.org -# -# This is free software, licensed under the GNU General Public License v2. -# See /LICENSE for more information. -# -include $(TOPDIR)/rules.mk - -PKG_NAME:=upx -PKG_VERSION:=3.95 - -PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION)-src.tar.xz -PKG_SOURCE_URL:=https://github.com/upx/upx/releases/download/v$(PKG_VERSION) -PKG_HASH:=3b0f55468d285c760fcf5ea865a070b27696393002712054c69ff40d8f7f5592 - -HOST_BUILD_DIR:=$(BUILD_DIR_HOST)/$(PKG_NAME)-$(PKG_VERSION)-src - -include $(INCLUDE_DIR)/host-build.mk - -define Host/Compile - UPX_UCLDIR=$(STAGING_DIR_HOST) \ - $(MAKE) -C $(HOST_BUILD_DIR)/src \ - CXXFLAGS_WERROR="" LDFLAGS="$(HOST_LDFLAGS)" \ - CXX="$(HOSTCXX)" -endef - -define Host/Install - $(CP) $(HOST_BUILD_DIR)/src/upx.out $(STAGING_DIR_HOST)/bin/upx -endef - -define Host/Clean - rm -f $(STAGING_DIR_HOST)/bin/upx -endef - -$(eval $(call HostBuild)) From 5f06907f917583aca184557e5933f29276943bc6 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Tue, 31 May 2022 01:31:11 +0800 Subject: [PATCH 05/31] qos-gargoyle: fix missing ldflags Signed-off-by: Tianling Shen --- package/emortal/qos-gargoyle/Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/package/emortal/qos-gargoyle/Makefile b/package/emortal/qos-gargoyle/Makefile index 11f3796f72..91267a1ef5 100644 --- a/package/emortal/qos-gargoyle/Makefile +++ b/package/emortal/qos-gargoyle/Makefile @@ -24,6 +24,8 @@ endef define Build/Configure endef +TARGET_LDFLAGS+= -lmnl + define Build/Compile -$(MAKE) -C $(PKG_BUILD_DIR) clean $(MAKE) -C $(PKG_BUILD_DIR) \ @@ -48,9 +50,11 @@ define Package/qos-gargoyle/install endef define Package/qos-gargoyle/prerm - $${IPKG_INSTROOT}/etc/init.d/qos_gargoyle stop 2>/dev/null - $${IPKG_INSTROOT}/etc/init.d/qos_gargoyle disable 2>/dev/null - ls >/dev/null 2>&1 +[ -n $${PKG_INSTROOT} ] || { + /etc/init.d/qos_gargoyle stop 2>/dev/null + /etc/init.d/qos_gargoyle disable 2>/dev/null + exit 0 +} endef $(eval $(call BuildPackage,qos-gargoyle)) From 816b83e4ef3158ddf38dcdc7ac2b052397a611d7 Mon Sep 17 00:00:00 2001 From: Dominick Grift Date: Thu, 19 May 2022 18:50:16 +0200 Subject: [PATCH 06/31] selinux-policy: update to version 1.2.3 86ca9c6 devstatus: prints to terminal 95de949 deal with /rom/dev/console label inconsistencies ab6b6ee uci: hack to deal with potentially mislabeled char files acf9172 dnsmasq this can't be right 021db5b luci-app-tinyproxy cf3a9c4 support/secmark: removes duplicate loopback rules eeb2610 dhcp servers: recv dhcp client packets d5a5fc3 more support/secmark "fixes" 35d8604 update support secmark 4c155c0 packets these were caused by labeling issues with loopback fad35a5 nftables reads routing table f9c5a04 umurmur: kill an mumur instance that does not run as root 10a10c6 mmc stordev make this consistent ab3ec5b Makefile: sort with LC_ALL=C b34eaa5 fwenv rules 8c2960f adds rfkill nodedev and some mmc partitions to stordev 5a9ffe9 rcboot runs fwenv with a transition 9954bf6 dnsmasq in case of tcp ab66468 dnsmasq try this 5bfcb88 dnsmasq stubby not sure why this is happening 863f549 luci not sure why it recv and send server packets d5cddb0 uhttpd sends sigkill luci cgi 44cc04d stubby: it does not maintain anything in there db730b4 Adds stubby ccbcf0e tor simplify network access a308065 tor basic a9c0163 znc loose ends 327a9af acme: allow acme_cleanup.sh to restart znc 4015614 basic znc 7ef14a2 support/secmark: clarify some things 3107afe README: todo qrencode 943035a README and secmark doc 4c90937 ttyd: fix that socket leak again 3239adf dnsmasq icmp packets and fix a tty leak issue b41d38f Makefile: optimize 95d05b1 sandbox dontaudit ttyd leak 0b7d670 rpcd: reads mtu e754bf1 opkg-lists try this 35fb530 opkg-lists: custom 4328754 opkg try to address mislabeled /tmp/opkg-lists 3e2385c rcnftqos 95eae2d ucode c86d366 luci diagnostics e10b443 rpcd packets and wireguard/luci a25e020 igmpproxt packets 0106f00 luci dcef79c nftqos related 3c9bc90 related to nft-qos and luci f8502d4 dnsmasq more related to /usr/lib/dnsmasq/dhcp-script.sh 29a4271 dnsmasq: related to /usr/lib/dnsmasq/dhcp-script.sh 0c5805a some nft-qos 1100b41 adds a label for /tmp/.ujailnoafile e141a83 initscript: i labeled ujail procd.execfile a3b0302 Makefile: adds a default target + packets target 6a3f8ef label usign as opkg and label fwtool and sysupgrade 04d1cc7 sysupgrade: i meant don't do the fc spec 763bec0 sysupgrade: dont do /tmp/sysupgrade.img af2306f adds a failsafe.tmpfile and labels validate_firmware_image 5b15760 fwenv: comment doesnt make sense 370ac3b fwenv: executes shell 67e3fcb fwenv: adds fw_setsys 544d211 adds procd execfile module to label procd related exec files 99d5f13 rclocalconffile: treat /etc/rc.button like /etc/rc.local 4dfd662 label uclient-fetch the same as wget 75d8212 osreleasemiscfile: adds /etc/device_info 0c1f116 adds a rcbuttonconffile for /etc/rc.button (base-files) ccd23f8 adds a syslog.conffile for /etc/syslog.conf (busybox) f790600 adds a libattr.conffile for /etc/xattr.conf fcc028e fwenv: adds fwsys 1255470 xtables: various iptables alternatives a7c4035 Revert "sqm: runs xtables, so also allow nftables" 0d331c3 sqm: runs xtables, so also allow nftables f34076b acme: will run nftables in the near future 6217046 allow ssl.read types to read /tmp/etc/ssl/engines.cnf d0deea3 fixes dns packets 8399efc Revert "sandbox: see if dontauditing this affects things" 73d716a sandbox: see if dontauditing this affects things b5ee097 sandbox: also allow readinherited dropbear pipes 12ee46b iwinfo traverses /tmp/run/wpa_supplicant 4a4d724 agent.cil: also reads inherited dropbear pipes d48013f support/secmark: i tightened my dns packet policy 645ad9e dns packets redone 4790b25 dnsnetpacket: fix obj macro template d9fafff redo dns packets 0a68498 ttyd: leaks a netlink route socket 1d2e6be .gitattributes: remove todo e1bb954 usbutil: reads bus sysfile symlinks d275a32 support/secmark: clean it up a little af5ce12 Makefile: exclude packet types in default make target 3caacdf support/secmark: document tunable/boolean e3dd3e6 invalidpacketselinuxbool: make it build-time again 54f0ccf odhcpd packet fix 4a864ba contrib/secmark: add a big FAT warning bead937 contrib/secmark: adds note about secmark support 146ae16 netpacket remove test 2ce9899 dns packets, odhcp6c raw packet, 4123 ntpnts for netnod 070a45f chrony and unbound packets eba894f rawip socket packets cannot be labeled 656ae0b adds isakmp (500), ipsec-nat-t (4500) and rawip packet types 35325db adds igmp packet type 5cf444c adds icmp packet type 2e41304 sandbox some more packet access for sandbox net 12caad6 packet accesses b8eb9a8 adds a trunkload of packet types a42a336 move rules related to invalid netpeers and ipsec associations a9e40e0 xtables/nftables allow relabelto all packet types aa5a52c README: adds item to wish list 3a96eec experiment: simple label based packet filtering 26d6f95 nftables reads/writes fw pipes Signed-off-by: Dominick Grift (cherry picked from commit e01b1c22dfb669abb0ad14c83ec9b3e35ff3d15c) --- package/system/selinux-policy/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/system/selinux-policy/Makefile b/package/system/selinux-policy/Makefile index 10eff7be57..bcf6b4a3c2 100644 --- a/package/system/selinux-policy/Makefile +++ b/package/system/selinux-policy/Makefile @@ -8,8 +8,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=selinux-policy PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://git.defensec.nl/selinux-policy.git -PKG_VERSION:=1.1 -PKG_MIRROR_HASH:=657ec1ff51ab946753fb3559384511a536ac1e018691f3e49cbab21c55d23e08 +PKG_VERSION:=1.2.3 +PKG_MIRROR_HASH:=ff1ddca168a6631aeac34352657f424bc4acf5d50b8aa7ff8dfa8c9663ba8538 PKG_SOURCE_VERSION:=v$(PKG_VERSION) PKG_BUILD_DEPENDS:=secilc/host policycoreutils/host From 01cbb4354badc73e308363ebf49dd7cdb19eb76c Mon Sep 17 00:00:00 2001 From: Hauke Mehrtens Date: Sat, 28 May 2022 14:32:08 +0200 Subject: [PATCH 07/31] malta: use default OpenWrt network configuration Currently malta configures the first Ethernet device as WAN interface. If it finds a second one it will configure it as LAN. This commit reverses it to match armvirt and x86. If there is only one network device it will be configured as LAN device now. If we find two network devices the 2. one will be WAN. If no board.d network configuration is given it will be configured in package/base-files/files/etc/board.d/99-default_network Signed-off-by: Hauke Mehrtens [minor typos] Signed-off-by: Paul Spooren (cherry picked from commit fb1ba922026a4b7d04ef5c8058f7890ed7b790e6) --- .../linux/malta/base-files/etc/board.d/02_network | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100755 target/linux/malta/base-files/etc/board.d/02_network diff --git a/target/linux/malta/base-files/etc/board.d/02_network b/target/linux/malta/base-files/etc/board.d/02_network deleted file mode 100755 index 8f01108938..0000000000 --- a/target/linux/malta/base-files/etc/board.d/02_network +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -. /lib/functions/uci-defaults.sh - -board_config_update - -ucidef_set_interface_wan "eth0" -if [ -d "/sys/class/net/eth1" ]; then - ucidef_set_interface_lan "eth1" -fi - -board_config_flush - -exit 0 From 3be2c87a96f4eda74a0116696b0f6067cf52f9bf Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Tue, 31 May 2022 18:56:11 +0800 Subject: [PATCH 08/31] feeds: fix routing and telephony build Signed-off-by: Tianling Shen --- feeds.conf.default | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feeds.conf.default b/feeds.conf.default index 252c69b33b..17689e7930 100644 --- a/feeds.conf.default +++ b/feeds.conf.default @@ -1,4 +1,4 @@ src-git packages https://github.com/immortalwrt/packages.git;openwrt-18.06 src-git luci https://github.com/immortalwrt/luci.git;openwrt-18.06 -src-git routing https://github.com/immortalwrt/routing.git;openwrt-18.06 -src-git telephony https://github.com/openwrt/telephony.git +src-git routing https://github.com/openwrt/routing.git;openwrt-19.07 +src-git telephony https://github.com/openwrt/telephony.git;openwrt-21.02 From 9b481b6bb6b385601e5de77e0b2f6359b39497fd Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Fri, 3 Jun 2022 23:13:26 +0800 Subject: [PATCH 09/31] dnsmasq: enable cache by default Signed-off-by: Tianling Shen (cherry picked from commit 3272539aaa6b0b284c01594c188adf8a13ed679b) --- package/network/services/dnsmasq/files/dhcp.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package/network/services/dnsmasq/files/dhcp.conf b/package/network/services/dnsmasq/files/dhcp.conf index 57d6c9576d..a5f833e470 100644 --- a/package/network/services/dnsmasq/files/dhcp.conf +++ b/package/network/services/dnsmasq/files/dhcp.conf @@ -21,6 +21,8 @@ config dnsmasq #list bogusnxdomain '64.94.110.11' option localservice 1 # disable to allow DNS requests from non-local subnets option filter_aaaa 0 + option cachesize 8000 + option mini_ttl 3600 option dns_redirect 1 option ednspacket_max 1232 From caac023d9a2683d9125797bb70284fc1b26b20b3 Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Fri, 27 May 2022 22:21:43 +0000 Subject: [PATCH 10/31] kernel: crypto: add kmod-crypto-chacha20poly1305 Needed by strongSwan IPsec VPN for strongswan-mod-chapoly. Not to be confused with kmod-crypto-LIB-chacha20poly1305, which is an 8-byte nonce version used by wireguard. Signed-off-by: Xu Wang (cherry picked from commit 197b672c40613a53a78a568a1957f2c23c343c1f) --- package/kernel/linux/modules/crypto.mk | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/package/kernel/linux/modules/crypto.mk b/package/kernel/linux/modules/crypto.mk index 74d8caa3d3..6058acf93f 100644 --- a/package/kernel/linux/modules/crypto.mk +++ b/package/kernel/linux/modules/crypto.mk @@ -93,6 +93,18 @@ endef $(eval $(call KernelPackage,crypto-ccm)) +define KernelPackage/crypto-chacha20poly1305 + TITLE:=ChaCha20-Poly1305 AEAD support, RFC7539 (used by strongSwan IPsec VPN) + DEPENDS:=+kmod-crypto-aead +kmod-crypto-manager + KCONFIG:=CONFIG_CRYPTO_CHACHA20POLY1305 + FILES:=$(LINUX_DIR)/crypto/chacha20poly1305.ko + AUTOLOAD:=$(call AutoLoad,09,chacha20poly1305) + $(call AddDepends/crypto) +endef + +$(eval $(call KernelPackage,crypto-chacha20poly1305)) + + define KernelPackage/crypto-cmac TITLE:=Support for Cipher-based Message Authentication Code (CMAC) DEPENDS:=+kmod-crypto-hash From ad602514ccd53e8ff6566091cfe32082c67057bd Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sat, 11 Jun 2022 00:25:22 +0800 Subject: [PATCH 11/31] mbedtls: mark as nonshared Signed-off-by: Tianling Shen (cherry picked from commit 48383c2847dae61d81069315bcfbbc468a61c4cd) --- package/libs/mbedtls/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package/libs/mbedtls/Makefile b/package/libs/mbedtls/Makefile index ab59ea580e..c0da59cba7 100644 --- a/package/libs/mbedtls/Makefile +++ b/package/libs/mbedtls/Makefile @@ -45,6 +45,7 @@ $(call Package/mbedtls/Default) CATEGORY:=Libraries SUBMENU:=SSL TITLE+= (library) + PKGFLAGS:=nonshared ABI_VERSION:=12 endef @@ -71,8 +72,6 @@ config LIBMBEDTLS_HAVE_ARMV8CE_AES Use of the ARMv8 Crypto Extensions greatly increase performance (up to 4x faster on AES-GCM while 10x faster on raw AES). - Related instructions should be included in all modern Aarch64 - devices, except some wastes like Broadcom. If you don't sure, say Y here. config LIBMBEDTLS_HAVE_SSE2 From 2aa9fdb63fad7e8c6e85740863e182c782a010b8 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sat, 11 Jun 2022 05:42:50 +0800 Subject: [PATCH 12/31] exfat: Update to 5.19.1 Signed-off-by: Tianling Shen (cherry picked from commit e72122a460e5e05bc548cd6846f018ad148d938c) [Removed upstreamed patch] Signed-off-by: Tianling Shen --- package/kernel/exfat/Makefile | 4 ++-- .../patches/001-fix-build-on-kernel-4.x.patch | 14 -------------- 2 files changed, 2 insertions(+), 16 deletions(-) delete mode 100644 package/kernel/exfat/patches/001-fix-build-on-kernel-4.x.patch diff --git a/package/kernel/exfat/Makefile b/package/kernel/exfat/Makefile index 8b1d22211d..84ed240e0d 100644 --- a/package/kernel/exfat/Makefile +++ b/package/kernel/exfat/Makefile @@ -7,12 +7,12 @@ include $(TOPDIR)/rules.mk include $(INCLUDE_DIR)/kernel.mk PKG_NAME:=exfat -PKG_VERSION:=5.14.1 +PKG_VERSION:=5.19.1 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/namjaejeon/linux-exfat-oot/tar.gz/$(PKG_VERSION)? -PKG_HASH:=5452a7c48ff7d19e258d1521213c84f5e486ffeec65b8af913bd43c85b010e86 +PKG_HASH:=80750bfa3bcdf743ca0d027be8244cc7b6ccd78f20304c2cabbb4011c88e4f0a PKG_BUILD_DIR:=$(KERNEL_BUILD_DIR)/linux-exfat-oot-$(PKG_VERSION) PKG_MAINTAINER:= diff --git a/package/kernel/exfat/patches/001-fix-build-on-kernel-4.x.patch b/package/kernel/exfat/patches/001-fix-build-on-kernel-4.x.patch deleted file mode 100644 index 391d27355e..0000000000 --- a/package/kernel/exfat/patches/001-fix-build-on-kernel-4.x.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- a/balloc.c -+++ b/balloc.c -@@ -7,8 +7,10 @@ - #include - #include - #include --#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0) -+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) - #include -+#else -+#include - #endif - - #include "exfat_raw.h" From 4c329354bf16d0bcd6a1365be1c9ef3d159f1ffe Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Mon, 16 May 2022 17:21:48 -0700 Subject: [PATCH 13/31] tools/ninja: update to 1.11.0 Updated patchset to latest. Signed-off-by: Rosen Penev (cherry picked from commit a7be143646db9365f6ac8d5749a2dfef805789cb) --- tools/ninja/Makefile | 4 +- .../patches/100-make_jobserver_support.patch | 4147 ++++++++++++----- 2 files changed, 3081 insertions(+), 1070 deletions(-) diff --git a/tools/ninja/Makefile b/tools/ninja/Makefile index 0ff642a740..c5c83d9b14 100644 --- a/tools/ninja/Makefile +++ b/tools/ninja/Makefile @@ -1,12 +1,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ninja -PKG_VERSION:=1.10.2 +PKG_VERSION:=1.11.0 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/ninja-build/ninja/tar.gz/v$(PKG_VERSION)? -PKG_HASH:=ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed +PKG_HASH:=3c6ba2e66400fe3f1ae83deb4b235faf3137ec20bd5b08c29bfc368db143e4c6 include $(INCLUDE_DIR)/host-build.mk diff --git a/tools/ninja/patches/100-make_jobserver_support.patch b/tools/ninja/patches/100-make_jobserver_support.patch index ecceaf23ca..7dac8ef814 100644 --- a/tools/ninja/patches/100-make_jobserver_support.patch +++ b/tools/ninja/patches/100-make_jobserver_support.patch @@ -1,7 +1,7 @@ -From c1a081c00f803fc28e51f155f25abe8346ce5f13 Mon Sep 17 00:00:00 2001 +From 17d13fd7881fd3ce9f9b9d44ce435d6caf4b8f28 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Tue, 22 Mar 2016 13:48:07 +0200 -Subject: [PATCH] Add GNU make jobserver client support +Subject: [PATCH 01/11] Add GNU make jobserver client support - add new TokenPool interface - GNU make implementation for TokenPool parses and verifies the magic @@ -19,263 +19,60 @@ Documentation for GNU make jobserver http://make.mad-scientist.net/papers/jobserver-implementation/ Fixes https://github.com/ninja-build/ninja/issues/1139 - -Add TokenPool monitoring to SubprocessSet::DoWork() - -Improve on the original jobserver client implementation. This makes -ninja a more aggressive GNU make jobserver client. - -- add monitor interface to TokenPool -- TokenPool is passed down when main loop indicates that more work is - ready and would be allowed to start if a token becomes available -- posix: update DoWork() to monitor TokenPool read file descriptor -- WaitForCommand() exits when DoWork() sets token flag -- Main loop starts over when WaitForCommand() sets token exit status - -Ignore jobserver when -jN is forced on command line - -This emulates the behaviour of GNU make. - -- add parallelism_from_cmdline flag to build configuration -- set the flag when -jN is given on command line -- pass the flag to TokenPool::Get() -- GNUmakeTokenPool::Setup() - * prints a warning when the flag is true and jobserver was detected - * returns false, i.e. jobserver will be ignored -- ignore config.parallelism in CanRunMore() when we have a valid - TokenPool, because it gets always initialized to a default when not - given on the command line - -Honor -lN from MAKEFLAGS - -This emulates the behaviour of GNU make. - -- build: make a copy of max_load_average and pass it to TokenPool. -- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in - MAKEFLAGS then set max_load_average to N. - -Use LinePrinter for TokenPool messages - -- replace printf() with calls to LinePrinter -- print GNU make jobserver message only when verbose build is requested - -Prepare PR for merging - -- fix Windows build error in no-op TokenPool implementation -- improve Acquire() to block for a maximum of 100ms -- address review comments - -Add tests for TokenPool - -- TokenPool setup -- GetMonitorFd() API -- implicit token and tokens in jobserver pipe -- Acquire() / Reserve() / Release() protocol -- Clear() API - -Add tests for subprocess module - -- add TokenPoolTest stub to provide TokenPool::GetMonitorFd() -- add two tests - * both tests set up a dummy GNUmake jobserver pipe - * both tests call DoWork() with TokenPoolTest - * test 1: verify that DoWork() detects when a token is available - * test 2: verify that DoWork() works as before without a token -- the tests are not compiled in under Windows - -Add tests for build module - -Add tests that verify the token functionality of the builder main loop. -We replace the default fake command runner with a special version where -the tests can control each call to AcquireToken(), CanRunMore() and -WaitForCommand(). - -Add Win32 implementation for GNUmakeTokenPool - -GNU make uses a semaphore as jobserver protocol on Win32. See also - - https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html - -Usage is pretty simple and straightforward, i.e. WaitForSingleObject() -to obtain a token and ReleaseSemaphore() to return it. - -Unfortunately subprocess-win32.cc uses an I/O completion port (IOCP). -IOCPs aren't waitable objects, i.e. we can't use WaitForMultipleObjects() -to wait on the IOCP and the token semaphore at the same time. - -Therefore GNUmakeTokenPoolWin32 creates a child thread that waits on the -token semaphore and posts a dummy I/O completion status on the IOCP when -it was able to obtain a token. That unblocks SubprocessSet::DoWork() and -it can then check if a token became available or not. - -- split existing GNUmakeTokenPool into common and platform bits -- add GNUmakeTokenPool interface -- move the Posix bits to GNUmakeTokenPoolPosix -- add the Win32 bits as GNUmakeTokenPoolWin32 -- move Setup() method up to TokenPool interface -- update Subprocess & TokenPool tests accordingly - -Prepare PR for merging - part II - -- remove unnecessary "struct" from TokenPool -- add PAPCFUNC cast to QueryUserAPC() -- remove hard-coded MAKEFLAGS string from win32 -- remove useless build test CompleteNoWork -- rename TokenPoolTest to TestTokenPool -- add tokenpool modules to CMake build -- remove unused no-op TokenPool implementation -- address review comments from - -https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-195195803 -https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-185089255 -https://github.com/ninja-build/ninja/pull/1140#issuecomment-473898963 -https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 --- - CMakeLists.txt | 8 +- - configure.py | 7 +- - src/build.cc | 127 ++++++++--- - src/build.h | 12 +- - src/build_test.cc | 363 +++++++++++++++++++++++++++++++- - src/exit_status.h | 3 +- - src/ninja.cc | 1 + - src/subprocess-posix.cc | 33 ++- - src/subprocess-win32.cc | 11 +- - src/subprocess.h | 8 +- - src/subprocess_test.cc | 149 +++++++++++-- - src/tokenpool-gnu-make-posix.cc | 202 ++++++++++++++++++ - src/tokenpool-gnu-make-win32.cc | 239 +++++++++++++++++++++ - src/tokenpool-gnu-make.cc | 108 ++++++++++ - src/tokenpool-gnu-make.h | 40 ++++ - src/tokenpool.h | 42 ++++ - src/tokenpool_test.cc | 269 +++++++++++++++++++++++ - 17 files changed, 1562 insertions(+), 60 deletions(-) - create mode 100644 src/tokenpool-gnu-make-posix.cc - create mode 100644 src/tokenpool-gnu-make-win32.cc + configure.py | 2 + + src/build.cc | 63 ++++++++---- + src/build.h | 3 + + src/tokenpool-gnu-make.cc | 211 ++++++++++++++++++++++++++++++++++++++ + src/tokenpool-none.cc | 27 +++++ + src/tokenpool.h | 26 +++++ + 6 files changed, 310 insertions(+), 22 deletions(-) create mode 100644 src/tokenpool-gnu-make.cc - create mode 100644 src/tokenpool-gnu-make.h + create mode 100644 src/tokenpool-none.cc create mode 100644 src/tokenpool.h - create mode 100644 src/tokenpool_test.cc ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -94,6 +94,7 @@ add_library(libninja OBJECT - src/parser.cc - src/state.cc - src/string_piece_util.cc -+ src/tokenpool-gnu-make.cc - src/util.cc - src/version.cc - ) -@@ -104,12 +105,16 @@ if(WIN32) - src/msvc_helper-win32.cc - src/msvc_helper_main-win32.cc - src/getopt.c -+ src/tokenpool-gnu-make-win32.cc - ) - if(MSVC) - target_sources(libninja PRIVATE src/minidump-win32.cc) - endif() - else() -- target_sources(libninja PRIVATE src/subprocess-posix.cc) -+ target_sources(libninja PRIVATE -+ src/subprocess-posix.cc -+ src/tokenpool-gnu-make-posix.cc -+ ) - if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") - target_sources(libninja PRIVATE src/getopt.c) - endif() -@@ -182,6 +187,7 @@ if(BUILD_TESTING) - src/string_piece_util_test.cc - src/subprocess_test.cc - src/test.cc -+ src/tokenpool_test.cc - src/util_test.cc - ) - if(WIN32) +diff --git a/configure.py b/configure.py +index 43904349a8..db3492c93c 100755 --- a/configure.py +++ b/configure.py -@@ -514,11 +514,13 @@ for name in ['build', - 'parser', - 'state', - 'string_piece_util', -+ 'tokenpool-gnu-make', - 'util', - 'version']: +@@ -522,6 +522,7 @@ def has_re2c(): objs += cxx(name, variables=cxxvariables) if platform.is_windows(): for name in ['subprocess-win32', -+ 'tokenpool-gnu-make-win32', ++ 'tokenpool-none', 'includes_normalize-win32', 'msvc_helper-win32', 'msvc_helper_main-win32']: -@@ -527,7 +529,9 @@ if platform.is_windows(): - objs += cxx('minidump-win32', variables=cxxvariables) +@@ -531,6 +532,7 @@ def has_re2c(): objs += cc('getopt') else: -- objs += cxx('subprocess-posix') -+ for name in ['subprocess-posix', -+ 'tokenpool-gnu-make-posix']: -+ objs += cxx(name) + objs += cxx('subprocess-posix') ++ objs += cxx('tokenpool-gnu-make') if platform.is_aix(): objs += cc('getopt') if platform.is_msvc(): -@@ -582,6 +586,7 @@ for name in ['build_log_test', - 'string_piece_util_test', - 'subprocess_test', - 'test', -+ 'tokenpool_test', - 'util_test']: - objs += cxx(name, variables=cxxvariables) - if platform.is_windows(): +diff --git a/src/build.cc b/src/build.cc +index 6f11ed7a3c..fa096eac33 100644 --- a/src/build.cc +++ b/src/build.cc -@@ -38,6 +38,7 @@ - #include "graph.h" +@@ -35,6 +35,7 @@ #include "state.h" + #include "status.h" #include "subprocess.h" +#include "tokenpool.h" #include "util.h" using namespace std; -@@ -50,8 +51,9 @@ struct DryRunCommandRunner : public Comm - - // Overridden from CommandRunner: - virtual bool CanRunMore() const; -+ virtual bool AcquireToken(); - virtual bool StartCommand(Edge* edge); -- virtual bool WaitForCommand(Result* result); -+ virtual bool WaitForCommand(Result* result, bool more_ready); - - private: - queue finished_; -@@ -61,12 +63,16 @@ bool DryRunCommandRunner::CanRunMore() c - return true; - } - -+bool DryRunCommandRunner::AcquireToken() { -+ return true; -+} -+ - bool DryRunCommandRunner::StartCommand(Edge* edge) { - finished_.push(edge); - return true; - } - --bool DryRunCommandRunner::WaitForCommand(Result* result) { -+bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) { - if (finished_.empty()) - return false; - -@@ -379,7 +385,7 @@ void Plan::EdgeWanted(const Edge* edge) +@@ -149,7 +150,7 @@ void Plan::EdgeWanted(const Edge* edge) { } Edge* Plan::FindWork() { - if (ready_.empty()) + if (!more_ready()) return NULL; - set::iterator e = ready_.begin(); + EdgeSet::iterator e = ready_.begin(); Edge* edge = *e; -@@ -665,19 +671,39 @@ void Plan::Dump() const { +@@ -448,8 +449,8 @@ void Plan::Dump() const { } struct RealCommandRunner : public CommandRunner { @@ -284,31 +81,18 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + explicit RealCommandRunner(const BuildConfig& config); + virtual ~RealCommandRunner(); virtual bool CanRunMore() const; -+ virtual bool AcquireToken(); virtual bool StartCommand(Edge* edge); -- virtual bool WaitForCommand(Result* result); -+ virtual bool WaitForCommand(Result* result, bool more_ready); - virtual vector GetActiveEdges(); - virtual void Abort(); + virtual bool WaitForCommand(Result* result); +@@ -458,9 +459,18 @@ struct RealCommandRunner : public CommandRunner { const BuildConfig& config_; -+ // copy of config_.max_load_average; can be modified by TokenPool setup -+ double max_load_average_; SubprocessSet subprocs_; -+ TokenPool* tokens_; ++ TokenPool *tokens_; map subproc_to_edge_; }; +RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { -+ max_load_average_ = config.max_load_average; -+ if ((tokens_ = TokenPool::Get()) != NULL) { -+ if (!tokens_->Setup(config_.parallelism_from_cmdline, -+ config_.verbosity == BuildConfig::VERBOSE, -+ max_load_average_)) { -+ delete tokens_; -+ tokens_ = NULL; -+ } -+ } ++ tokens_ = TokenPool::Get(); +} + +RealCommandRunner::~RealCommandRunner() { @@ -318,7 +102,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 vector RealCommandRunner::GetActiveEdges() { vector edges; for (map::iterator e = subproc_to_edge_.begin(); -@@ -688,14 +714,23 @@ vector RealCommandRunner::GetActi +@@ -471,14 +481,18 @@ vector RealCommandRunner::GetActiveEdges() { void RealCommandRunner::Abort() { subprocs_.Clear(); @@ -327,27 +111,19 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 } bool RealCommandRunner::CanRunMore() const { -- size_t subproc_number = -- subprocs_.running_.size() + subprocs_.finished_.size(); -- return (int)subproc_number < config_.parallelism + size_t subproc_number = + subprocs_.running_.size() + subprocs_.finished_.size(); + return (int)subproc_number < config_.parallelism - && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) - || GetLoadAverage() < config_.max_load_average); -+ bool parallelism_limit_not_reached = -+ tokens_ || // ignore config_.parallelism -+ ((int) (subprocs_.running_.size() + -+ subprocs_.finished_.size()) < config_.parallelism); -+ return parallelism_limit_not_reached + && (subprocs_.running_.empty() || -+ (max_load_average_ <= 0.0f || -+ GetLoadAverage() < max_load_average_)); -+} -+ -+bool RealCommandRunner::AcquireToken() { -+ return (!tokens_ || tokens_->Acquire()); ++ ((config_.max_load_average <= 0.0f || ++ GetLoadAverage() < config_.max_load_average) ++ && (!tokens_ || tokens_->Acquire()))); } bool RealCommandRunner::StartCommand(Edge* edge) { -@@ -703,19 +738,33 @@ bool RealCommandRunner::StartCommand(Edg +@@ -486,6 +500,8 @@ bool RealCommandRunner::StartCommand(Edge* edge) { Subprocess* subproc = subprocs_.Add(command, edge->use_console()); if (!subproc) return false; @@ -356,63 +132,37 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 subproc_to_edge_.insert(make_pair(subproc, edge)); return true; - } - --bool RealCommandRunner::WaitForCommand(Result* result) { -+bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) { - Subprocess* subproc; -- while ((subproc = subprocs_.NextFinished()) == NULL) { -- bool interrupted = subprocs_.DoWork(); -+ subprocs_.ResetTokenAvailable(); -+ while (((subproc = subprocs_.NextFinished()) == NULL) && -+ !subprocs_.IsTokenAvailable()) { -+ bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL); - if (interrupted) +@@ -499,6 +515,9 @@ bool RealCommandRunner::WaitForCommand(Result* result) { return false; } -+ // token became available -+ if (subproc == NULL) { -+ result->status = ExitTokenAvailable; -+ return true; -+ } -+ -+ // command completed + if (tokens_) + tokens_->Release(); + result->status = subproc->Finish(); result->output = subproc->GetOutput(); -@@ -825,38 +874,42 @@ bool Builder::Build(string* err) { - // command runner. +@@ -621,31 +640,31 @@ bool Builder::Build(string* err) { // Second, we attempt to wait for / reap the next finished command. while (plan_.more_to_do()) { -- // See if we can start any more commands. + // See if we can start any more commands. - if (failures_allowed && command_runner_->CanRunMore()) { - if (Edge* edge = plan_.FindWork()) { - if (edge->GetBindingBool("generator")) { -- scan_.build_log()->Close(); -- } -+ // See if we can start any more commands... -+ bool can_run_more = -+ failures_allowed && -+ plan_.more_ready() && -+ command_runner_->CanRunMore(); -+ -+ // ... but we also need a token to do that. -+ if (can_run_more && command_runner_->AcquireToken()) { ++ if (failures_allowed && plan_.more_ready() && ++ command_runner_->CanRunMore()) { + Edge* edge = plan_.FindWork(); + if (edge->GetBindingBool("generator")) { -+ scan_.build_log()->Close(); -+ } + scan_.build_log()->Close(); + } + +- if (!StartEdge(edge, err)) { + if (!StartEdge(edge, err)) { + Cleanup(); + status_->BuildFinished(); + return false; + } - -- if (!StartEdge(edge, err)) { ++ + if (edge->is_phony()) { + if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) { Cleanup(); @@ -441,6 +191,437 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 } // See if we can reap any finished commands. +diff --git a/src/build.h b/src/build.h +index d697dfb89e..7dcd111e61 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -52,6 +52,9 @@ struct Plan { + /// Returns true if there's more work to be done. + bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; } + ++ /// Returns true if there's more edges ready to start ++ bool more_ready() const { return !ready_.empty(); } ++ + /// Dumps the current state of the plan. + void Dump() const; + +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +new file mode 100644 +index 0000000000..a8f9b7139d +--- /dev/null ++++ b/src/tokenpool-gnu-make.cc +@@ -0,0 +1,211 @@ ++// Copyright 2016 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "tokenpool.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TokenPool implementation for GNU make jobserver ++// (http://make.mad-scientist.net/papers/jobserver-implementation/) ++struct GNUmakeTokenPool : public TokenPool { ++ GNUmakeTokenPool(); ++ virtual ~GNUmakeTokenPool(); ++ ++ virtual bool Acquire(); ++ virtual void Reserve(); ++ virtual void Release(); ++ virtual void Clear(); ++ ++ bool Setup(); ++ ++ private: ++ int available_; ++ int used_; ++ ++#ifdef _WIN32 ++ // @TODO ++#else ++ int rfd_; ++ int wfd_; ++ ++ struct sigaction old_act_; ++ bool restore_; ++ ++ static int dup_rfd_; ++ static void CloseDupRfd(int signum); ++ ++ bool CheckFd(int fd); ++ bool SetAlarmHandler(); ++#endif ++ ++ void Return(); ++}; ++ ++// every instance owns an implicit token -> available_ == 1 ++GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0), ++ rfd_(-1), wfd_(-1), restore_(false) { ++} ++ ++GNUmakeTokenPool::~GNUmakeTokenPool() { ++ Clear(); ++ if (restore_) ++ sigaction(SIGALRM, &old_act_, NULL); ++} ++ ++bool GNUmakeTokenPool::CheckFd(int fd) { ++ if (fd < 0) ++ return false; ++ int ret = fcntl(fd, F_GETFD); ++ if (ret < 0) ++ return false; ++ return true; ++} ++ ++int GNUmakeTokenPool::dup_rfd_ = -1; ++ ++void GNUmakeTokenPool::CloseDupRfd(int signum) { ++ close(dup_rfd_); ++ dup_rfd_ = -1; ++} ++ ++bool GNUmakeTokenPool::SetAlarmHandler() { ++ struct sigaction act; ++ memset(&act, 0, sizeof(act)); ++ act.sa_handler = CloseDupRfd; ++ if (sigaction(SIGALRM, &act, &old_act_) < 0) { ++ perror("sigaction:"); ++ return(false); ++ } else { ++ restore_ = true; ++ return(true); ++ } ++} ++ ++bool GNUmakeTokenPool::Setup() { ++ const char *value = getenv("MAKEFLAGS"); ++ if (value) { ++ // GNU make <= 4.1 ++ const char *jobserver = strstr(value, "--jobserver-fds="); ++ // GNU make => 4.2 ++ if (!jobserver) ++ jobserver = strstr(value, "--jobserver-auth="); ++ if (jobserver) { ++ int rfd = -1; ++ int wfd = -1; ++ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && ++ CheckFd(rfd) && ++ CheckFd(wfd) && ++ SetAlarmHandler()) { ++ printf("ninja: using GNU make jobserver.\n"); ++ rfd_ = rfd; ++ wfd_ = wfd; ++ return true; ++ } ++ } ++ } ++ ++ return false; ++} ++ ++bool GNUmakeTokenPool::Acquire() { ++ if (available_ > 0) ++ return true; ++ ++#ifdef USE_PPOLL ++ pollfd pollfds[] = {{rfd_, POLLIN, 0}}; ++ int ret = poll(pollfds, 1, 0); ++#else ++ fd_set set; ++ struct timeval timeout = { 0, 0 }; ++ FD_ZERO(&set); ++ FD_SET(rfd_, &set); ++ int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); ++#endif ++ if (ret > 0) { ++ dup_rfd_ = dup(rfd_); ++ ++ if (dup_rfd_ != -1) { ++ struct sigaction act, old_act; ++ int ret = 0; ++ ++ memset(&act, 0, sizeof(act)); ++ act.sa_handler = CloseDupRfd; ++ if (sigaction(SIGCHLD, &act, &old_act) == 0) { ++ char buf; ++ ++ // block until token read, child exits or timeout ++ alarm(1); ++ ret = read(dup_rfd_, &buf, 1); ++ alarm(0); ++ ++ sigaction(SIGCHLD, &old_act, NULL); ++ } ++ ++ CloseDupRfd(0); ++ ++ if (ret > 0) { ++ available_++; ++ return true; ++ } ++ } ++ } ++ return false; ++} ++ ++void GNUmakeTokenPool::Reserve() { ++ available_--; ++ used_++; ++} ++ ++void GNUmakeTokenPool::Return() { ++ const char buf = '+'; ++ while (1) { ++ int ret = write(wfd_, &buf, 1); ++ if (ret > 0) ++ available_--; ++ if ((ret != -1) || (errno != EINTR)) ++ return; ++ // write got interrupted - retry ++ } ++} ++ ++void GNUmakeTokenPool::Release() { ++ available_++; ++ used_--; ++ if (available_ > 1) ++ Return(); ++} ++ ++void GNUmakeTokenPool::Clear() { ++ while (used_ > 0) ++ Release(); ++ while (available_ > 1) ++ Return(); ++} ++ ++struct TokenPool *TokenPool::Get(void) { ++ GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; ++ if (tokenpool->Setup()) ++ return tokenpool; ++ else ++ delete tokenpool; ++ return NULL; ++} +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +new file mode 100644 +index 0000000000..602b3316f5 +--- /dev/null ++++ b/src/tokenpool-none.cc +@@ -0,0 +1,27 @@ ++// Copyright 2016 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "tokenpool.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// No-op TokenPool implementation ++struct TokenPool *TokenPool::Get(void) { ++ return NULL; ++} +diff --git a/src/tokenpool.h b/src/tokenpool.h +new file mode 100644 +index 0000000000..f560b1083b +--- /dev/null ++++ b/src/tokenpool.h +@@ -0,0 +1,26 @@ ++// Copyright 2016 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++// interface to token pool ++struct TokenPool { ++ virtual ~TokenPool() {} ++ ++ virtual bool Acquire() = 0; ++ virtual void Reserve() = 0; ++ virtual void Release() = 0; ++ virtual void Clear() = 0; ++ ++ // returns NULL if token pool is not available ++ static struct TokenPool *Get(void); ++}; + +From ccaccc610cd456f6068758f82e72006364c7380b Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Fri, 27 May 2016 16:47:10 +0300 +Subject: [PATCH 02/11] Add TokenPool monitoring to SubprocessSet::DoWork() + +Improve on the original jobserver client implementation. This makes +ninja a more aggressive GNU make jobserver client. + +- add monitor interface to TokenPool +- TokenPool is passed down when main loop indicates that more work is + ready and would be allowed to start if a token becomes available +- posix: update DoWork() to monitor TokenPool read file descriptor +- WaitForCommand() exits when DoWork() sets token flag +- Main loop starts over when WaitForCommand() sets token exit status +--- + src/build.cc | 53 +++++++++++++++++++++++++++++---------- + src/build.h | 3 ++- + src/build_test.cc | 9 +++++-- + src/exit_status.h | 3 ++- + src/subprocess-posix.cc | 33 ++++++++++++++++++++++-- + src/subprocess-win32.cc | 2 +- + src/subprocess.h | 8 +++++- + src/subprocess_test.cc | 47 +++++++++++++++++++++++----------- + src/tokenpool-gnu-make.cc | 5 ++++ + src/tokenpool.h | 6 +++++ + 10 files changed, 134 insertions(+), 35 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index fa096eac33..a25c349050 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -48,8 +48,9 @@ struct DryRunCommandRunner : public CommandRunner { + + // Overridden from CommandRunner: + virtual bool CanRunMore() const; ++ virtual bool AcquireToken(); + virtual bool StartCommand(Edge* edge); +- virtual bool WaitForCommand(Result* result); ++ virtual bool WaitForCommand(Result* result, bool more_ready); + + private: + queue finished_; +@@ -59,12 +60,16 @@ bool DryRunCommandRunner::CanRunMore() const { + return true; + } + ++bool DryRunCommandRunner::AcquireToken() { ++ return true; ++} ++ + bool DryRunCommandRunner::StartCommand(Edge* edge) { + finished_.push(edge); + return true; + } + +-bool DryRunCommandRunner::WaitForCommand(Result* result) { ++bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) { + if (finished_.empty()) + return false; + +@@ -452,8 +457,9 @@ struct RealCommandRunner : public CommandRunner { + explicit RealCommandRunner(const BuildConfig& config); + virtual ~RealCommandRunner(); + virtual bool CanRunMore() const; ++ virtual bool AcquireToken(); + virtual bool StartCommand(Edge* edge); +- virtual bool WaitForCommand(Result* result); ++ virtual bool WaitForCommand(Result* result, bool more_ready); + virtual vector GetActiveEdges(); + virtual void Abort(); + +@@ -490,9 +496,12 @@ bool RealCommandRunner::CanRunMore() const { + subprocs_.running_.size() + subprocs_.finished_.size(); + return (int)subproc_number < config_.parallelism + && (subprocs_.running_.empty() || +- ((config_.max_load_average <= 0.0f || +- GetLoadAverage() < config_.max_load_average) +- && (!tokens_ || tokens_->Acquire()))); ++ (config_.max_load_average <= 0.0f || ++ GetLoadAverage() < config_.max_load_average)); ++} ++ ++bool RealCommandRunner::AcquireToken() { ++ return (!tokens_ || tokens_->Acquire()); + } + + bool RealCommandRunner::StartCommand(Edge* edge) { +@@ -507,14 +516,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) { + return true; + } + +-bool RealCommandRunner::WaitForCommand(Result* result) { ++bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) { + Subprocess* subproc; +- while ((subproc = subprocs_.NextFinished()) == NULL) { +- bool interrupted = subprocs_.DoWork(); ++ subprocs_.ResetTokenAvailable(); ++ while (((subproc = subprocs_.NextFinished()) == NULL) && ++ !subprocs_.IsTokenAvailable()) { ++ bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL); + if (interrupted) + return false; + } + ++ // token became available ++ if (subproc == NULL) { ++ result->status = ExitTokenAvailable; ++ return true; ++ } ++ ++ // command completed + if (tokens_) + tokens_->Release(); + +@@ -639,9 +657,14 @@ bool Builder::Build(string* err) { + // command runner. + // Second, we attempt to wait for / reap the next finished command. + while (plan_.more_to_do()) { +- // See if we can start any more commands. +- if (failures_allowed && plan_.more_ready() && +- command_runner_->CanRunMore()) { ++ // See if we can start any more commands... ++ bool can_run_more = ++ failures_allowed && ++ plan_.more_ready() && ++ command_runner_->CanRunMore(); ++ ++ // ... but we also need a token to do that. ++ if (can_run_more && command_runner_->AcquireToken()) { + Edge* edge = plan_.FindWork(); + if (edge->GetBindingBool("generator")) { + scan_.build_log()->Close(); +@@ -670,7 +693,7 @@ bool Builder::Build(string* err) { + // See if we can reap any finished commands. if (pending_commands) { CommandRunner::Result result; - if (!command_runner_->WaitForCommand(&result) || @@ -448,7 +629,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 result.status == ExitInterrupted) { Cleanup(); status_->BuildFinished(); -@@ -864,6 +917,10 @@ bool Builder::Build(string* err) { +@@ -678,6 +701,10 @@ bool Builder::Build(string* err) { return false; } @@ -459,19 +640,11 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 --pending_commands; if (!FinishCommand(&result, err)) { Cleanup(); +diff --git a/src/build.h b/src/build.h +index 7dcd111e61..35c7b97d12 100644 --- a/src/build.h +++ b/src/build.h -@@ -55,6 +55,9 @@ struct Plan { - /// Returns true if there's more work to be done. - bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; } - -+ /// Returns true if there's more edges ready to start -+ bool more_ready() const { return !ready_.empty(); } -+ - /// Dumps the current state of the plan. - void Dump() const; - -@@ -139,6 +142,7 @@ private: +@@ -139,6 +139,7 @@ struct Plan { struct CommandRunner { virtual ~CommandRunner() {} virtual bool CanRunMore() const = 0; @@ -479,46 +652,20 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 virtual bool StartCommand(Edge* edge) = 0; /// The result of waiting for a command. -@@ -150,7 +154,9 @@ struct CommandRunner { +@@ -150,7 +151,7 @@ struct CommandRunner { bool success() const { return status == ExitSuccess; } }; /// Wait for a command to complete, or return false if interrupted. - virtual bool WaitForCommand(Result* result) = 0; -+ /// If more_ready is true then the optional TokenPool is monitored too -+ /// and we return when a token becomes available. + virtual bool WaitForCommand(Result* result, bool more_ready) = 0; virtual std::vector GetActiveEdges() { return std::vector(); } virtual void Abort() {} -@@ -158,7 +164,8 @@ struct CommandRunner { - - /// Options (e.g. verbosity, parallelism) passed to a build. - struct BuildConfig { -- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), -+ BuildConfig() : verbosity(NORMAL), dry_run(false), -+ parallelism(1), parallelism_from_cmdline(false), - failures_allowed(1), max_load_average(-0.0f) {} - - enum Verbosity { -@@ -169,6 +176,7 @@ struct BuildConfig { - Verbosity verbosity; - bool dry_run; - int parallelism; -+ bool parallelism_from_cmdline; - int failures_allowed; - /// The maximum load average we must not exceed. A negative value - /// means that we do not have any limit. +diff --git a/src/build_test.cc b/src/build_test.cc +index 4ef62b2113..7a5ff4015a 100644 --- a/src/build_test.cc +++ b/src/build_test.cc -@@ -15,6 +15,7 @@ - #include "build.h" - - #include -+#include - - #include "build_log.h" - #include "deps_log.h" -@@ -473,8 +474,9 @@ struct FakeCommandRunner : public Comman +@@ -474,8 +474,9 @@ struct FakeCommandRunner : public CommandRunner { // CommandRunner impl virtual bool CanRunMore() const; @@ -529,7 +676,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 virtual vector GetActiveEdges(); virtual void Abort(); -@@ -580,6 +582,10 @@ bool FakeCommandRunner::CanRunMore() con +@@ -578,6 +579,10 @@ bool FakeCommandRunner::CanRunMore() const { return active_edges_.size() < max_active_edges_; } @@ -540,7 +687,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 bool FakeCommandRunner::StartCommand(Edge* edge) { assert(active_edges_.size() < max_active_edges_); assert(find(active_edges_.begin(), active_edges_.end(), edge) -@@ -625,7 +631,7 @@ bool FakeCommandRunner::StartCommand(Edg +@@ -649,7 +654,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { return true; } @@ -549,9 +696,1333 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 if (active_edges_.empty()) return false; -@@ -3302,3 +3308,356 @@ TEST_F(BuildTest, DyndepTwoLevelDiscover - EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]); - EXPECT_EQ("touch out", command_runner_.commands_ran_[4]); +diff --git a/src/exit_status.h b/src/exit_status.h +index a714ece791..75ebf6a7a0 100644 +--- a/src/exit_status.h ++++ b/src/exit_status.h +@@ -18,7 +18,8 @@ + enum ExitStatus { + ExitSuccess, + ExitFailure, +- ExitInterrupted ++ ExitTokenAvailable, ++ ExitInterrupted, + }; + + #endif // NINJA_EXIT_STATUS_H_ +diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc +index 8e785406c9..74451b0be2 100644 +--- a/src/subprocess-posix.cc ++++ b/src/subprocess-posix.cc +@@ -13,6 +13,7 @@ + // limitations under the License. + + #include "subprocess.h" ++#include "tokenpool.h" + + #include + #include +@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + } + + #ifdef USE_PPOLL +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(struct TokenPool* tokens) { + vector fds; + nfds_t nfds = 0; + +@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() { + ++nfds; + } + ++ if (tokens) { ++ pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 }; ++ fds.push_back(pfd); ++ ++nfds; ++ } ++ + interrupted_ = 0; + int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); + if (ret == -1) { +@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() { + ++i; + } + ++ if (tokens) { ++ pollfd *pfd = &fds[nfds - 1]; ++ if (pfd->fd >= 0) { ++ assert(pfd->fd == tokens->GetMonitorFd()); ++ if (pfd->revents != 0) ++ token_available_ = true; ++ } ++ } ++ + return IsInterrupted(); + } + + #else // !defined(USE_PPOLL) +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(struct TokenPool* tokens) { + fd_set set; + int nfds = 0; + FD_ZERO(&set); +@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() { + } + } + ++ if (tokens) { ++ int fd = tokens->GetMonitorFd(); ++ FD_SET(fd, &set); ++ if (nfds < fd+1) ++ nfds = fd+1; ++ } ++ + interrupted_ = 0; + int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); + if (ret == -1) { +@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() { + ++i; + } + ++ if (tokens) { ++ int fd = tokens->GetMonitorFd(); ++ if ((fd >= 0) && FD_ISSET(fd, &set)) ++ token_available_ = true; ++ } ++ + return IsInterrupted(); + } + #endif // !defined(USE_PPOLL) +diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc +index ff3baaca7f..66d2c2c430 100644 +--- a/src/subprocess-win32.cc ++++ b/src/subprocess-win32.cc +@@ -251,7 +251,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + return subprocess; + } + +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(struct TokenPool* tokens) { + DWORD bytes_read; + Subprocess* subproc; + OVERLAPPED* overlapped; +diff --git a/src/subprocess.h b/src/subprocess.h +index 9e3d2ee98f..9ea67ea477 100644 +--- a/src/subprocess.h ++++ b/src/subprocess.h +@@ -76,6 +76,8 @@ struct Subprocess { + friend struct SubprocessSet; + }; + ++struct TokenPool; ++ + /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses. + /// DoWork() waits for any state change in subprocesses; finished_ + /// is a queue of subprocesses as they finish. +@@ -84,13 +86,17 @@ struct SubprocessSet { + ~SubprocessSet(); + + Subprocess* Add(const std::string& command, bool use_console = false); +- bool DoWork(); ++ bool DoWork(struct TokenPool* tokens); + Subprocess* NextFinished(); + void Clear(); + + std::vector running_; + std::queue finished_; + ++ bool token_available_; ++ bool IsTokenAvailable() { return token_available_; } ++ void ResetTokenAvailable() { token_available_ = false; } ++ + #ifdef _WIN32 + static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType); + static HANDLE ioport_; +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index 073fe86931..4bc8083e26 100644 +--- a/src/subprocess_test.cc ++++ b/src/subprocess_test.cc +@@ -45,10 +45,12 @@ TEST_F(SubprocessTest, BadCommandStderr) { + Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + // Pretend we discovered that stderr was ready for writing. +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -59,10 +61,12 @@ TEST_F(SubprocessTest, NoSuchCommand) { + Subprocess* subproc = subprocs_.Add("ninja_no_such_command"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + // Pretend we discovered that stderr was ready for writing. +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -78,9 +82,11 @@ TEST_F(SubprocessTest, InterruptChild) { + Subprocess* subproc = subprocs_.Add("kill -INT $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -90,7 +96,7 @@ TEST_F(SubprocessTest, InterruptParent) { + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -102,9 +108,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) { + Subprocess* subproc = subprocs_.Add("kill -TERM $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -114,7 +122,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigTerm) { + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -126,9 +134,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) { + Subprocess* subproc = subprocs_.Add("kill -HUP $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -138,7 +148,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigHup) { + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -153,9 +163,11 @@ TEST_F(SubprocessTest, Console) { + subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true); + ASSERT_NE((Subprocess*)0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } +@@ -167,9 +179,11 @@ TEST_F(SubprocessTest, SetWithSingle) { + Subprocess* subproc = subprocs_.Add(kSimpleCommand); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_NE("", subproc->GetOutput()); + +@@ -200,12 +214,13 @@ TEST_F(SubprocessTest, SetWithMulti) { + ASSERT_EQ("", processes[i]->GetOutput()); + } + ++ subprocs_.ResetTokenAvailable(); + while (!processes[0]->Done() || !processes[1]->Done() || + !processes[2]->Done()) { + ASSERT_GT(subprocs_.running_.size(), 0u); +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } +- ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + ASSERT_EQ(0u, subprocs_.running_.size()); + ASSERT_EQ(3u, subprocs_.finished_.size()); + +@@ -237,8 +252,10 @@ TEST_F(SubprocessTest, SetWithLots) { + ASSERT_NE((Subprocess *) 0, subproc); + procs.push_back(subproc); + } ++ subprocs_.ResetTokenAvailable(); + while (!subprocs_.running_.empty()) +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + for (size_t i = 0; i < procs.size(); ++i) { + ASSERT_EQ(ExitSuccess, procs[i]->Finish()); + ASSERT_NE("", procs[i]->GetOutput()); +@@ -254,9 +271,11 @@ TEST_F(SubprocessTest, SetWithLots) { + // that stdin is closed. + TEST_F(SubprocessTest, ReadStdin) { + Subprocess* subproc = subprocs_.Add("cat -"); ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_EQ(1u, subprocs_.finished_.size()); + } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index a8f9b7139d..396bb7d874 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -33,6 +33,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Reserve(); + virtual void Release(); + virtual void Clear(); ++ virtual int GetMonitorFd(); + + bool Setup(); + +@@ -201,6 +202,10 @@ void GNUmakeTokenPool::Clear() { + Return(); + } + ++int GNUmakeTokenPool::GetMonitorFd() { ++ return(rfd_); ++} ++ + struct TokenPool *TokenPool::Get(void) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; + if (tokenpool->Setup()) +diff --git a/src/tokenpool.h b/src/tokenpool.h +index f560b1083b..301e1998ee 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -21,6 +21,12 @@ struct TokenPool { + virtual void Release() = 0; + virtual void Clear() = 0; + ++#ifdef _WIN32 ++ // @TODO ++#else ++ virtual int GetMonitorFd() = 0; ++#endif ++ + // returns NULL if token pool is not available + static struct TokenPool *Get(void); + }; + +From d09f3d77821b3b1fdf09fc0ef8e814907675eafb Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sun, 12 Nov 2017 16:58:55 +0200 +Subject: [PATCH 03/11] Ignore jobserver when -jN is forced on command line + +This emulates the behaviour of GNU make. + +- add parallelism_from_cmdline flag to build configuration +- set the flag when -jN is given on command line +- pass the flag to TokenPool::Get() +- GNUmakeTokenPool::Setup() + * prints a warning when the flag is true and jobserver was detected + * returns false, i.e. jobserver will be ignored +- ignore config.parallelism in CanRunMore() when we have a valid + TokenPool, because it gets always initialized to a default when not + given on the command line +--- + src/build.cc | 10 ++++++---- + src/build.h | 4 +++- + src/ninja.cc | 1 + + src/tokenpool-gnu-make.cc | 34 +++++++++++++++++++--------------- + src/tokenpool-none.cc | 4 ++-- + src/tokenpool.h | 4 ++-- + 6 files changed, 33 insertions(+), 24 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index a25c349050..406a84ec39 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -470,7 +470,7 @@ struct RealCommandRunner : public CommandRunner { + }; + + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { +- tokens_ = TokenPool::Get(); ++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); + } + + RealCommandRunner::~RealCommandRunner() { +@@ -492,9 +492,11 @@ void RealCommandRunner::Abort() { + } + + bool RealCommandRunner::CanRunMore() const { +- size_t subproc_number = +- subprocs_.running_.size() + subprocs_.finished_.size(); +- return (int)subproc_number < config_.parallelism ++ bool parallelism_limit_not_reached = ++ tokens_ || // ignore config_.parallelism ++ ((int) (subprocs_.running_.size() + ++ subprocs_.finished_.size()) < config_.parallelism); ++ return parallelism_limit_not_reached + && (subprocs_.running_.empty() || + (config_.max_load_average <= 0.0f || + GetLoadAverage() < config_.max_load_average)); +diff --git a/src/build.h b/src/build.h +index 35c7b97d12..dfde576573 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -159,7 +159,8 @@ struct CommandRunner { + + /// Options (e.g. verbosity, parallelism) passed to a build. + struct BuildConfig { +- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), ++ BuildConfig() : verbosity(NORMAL), dry_run(false), ++ parallelism(1), parallelism_from_cmdline(false), + failures_allowed(1), max_load_average(-0.0f) {} + + enum Verbosity { +@@ -171,6 +172,7 @@ struct BuildConfig { + Verbosity verbosity; + bool dry_run; + int parallelism; ++ bool parallelism_from_cmdline; + int failures_allowed; + /// The maximum load average we must not exceed. A negative value + /// means that we do not have any limit. +diff --git a/src/ninja.cc b/src/ninja.cc +index df39ba92d1..d904c56c4e 100644 +--- a/src/ninja.cc ++++ b/src/ninja.cc +@@ -1447,6 +1447,7 @@ int ReadFlags(int* argc, char*** argv, + // We want to run N jobs in parallel. For N = 0, INT_MAX + // is close enough to infinite for most sane builds. + config->parallelism = value > 0 ? value : INT_MAX; ++ config->parallelism_from_cmdline = true; + deferGuessParallelism.needGuess = false; + break; + } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index 396bb7d874..af4be05a31 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(); ++ bool Setup(bool ignore); + + private: + int available_; +@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup() { ++bool GNUmakeTokenPool::Setup(bool ignore) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -109,16 +109,20 @@ bool GNUmakeTokenPool::Setup() { + if (!jobserver) + jobserver = strstr(value, "--jobserver-auth="); + if (jobserver) { +- int rfd = -1; +- int wfd = -1; +- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && +- CheckFd(rfd) && +- CheckFd(wfd) && +- SetAlarmHandler()) { +- printf("ninja: using GNU make jobserver.\n"); +- rfd_ = rfd; +- wfd_ = wfd; +- return true; ++ if (ignore) { ++ printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ } else { ++ int rfd = -1; ++ int wfd = -1; ++ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && ++ CheckFd(rfd) && ++ CheckFd(wfd) && ++ SetAlarmHandler()) { ++ printf("ninja: using GNU make jobserver.\n"); ++ rfd_ = rfd; ++ wfd_ = wfd; ++ return true; ++ } + } + } + } +@@ -206,9 +210,9 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(void) { ++struct TokenPool *TokenPool::Get(bool ignore) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup()) ++ if (tokenpool->Setup(ignore)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 602b3316f5..199b22264b 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -22,6 +22,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(void) { ++struct TokenPool *TokenPool::Get(bool ignore) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 301e1998ee..878a0933c2 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -28,5 +28,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(void); ++ static struct TokenPool *Get(bool ignore); + }; + +From dfe4ca753caee65bf9041e2b4e883dfa172a5c6a Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sun, 12 Nov 2017 18:04:12 +0200 +Subject: [PATCH 04/11] Honor -lN from MAKEFLAGS + +This emulates the behaviour of GNU make. + +- build: make a copy of max_load_average and pass it to TokenPool. +- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in + MAKEFLAGS then set max_load_average to N. +--- + src/build.cc | 10 +++++++--- + src/tokenpool-gnu-make.cc | 19 +++++++++++++++---- + src/tokenpool-none.cc | 2 +- + src/tokenpool.h | 2 +- + 4 files changed, 24 insertions(+), 9 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index 406a84ec39..9e6272d035 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -464,13 +464,17 @@ struct RealCommandRunner : public CommandRunner { + virtual void Abort(); + + const BuildConfig& config_; ++ // copy of config_.max_load_average; can be modified by TokenPool setup ++ double max_load_average_; + SubprocessSet subprocs_; + TokenPool *tokens_; + map subproc_to_edge_; + }; + + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { +- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); ++ max_load_average_ = config.max_load_average; ++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, ++ max_load_average_); + } + + RealCommandRunner::~RealCommandRunner() { +@@ -498,8 +502,8 @@ bool RealCommandRunner::CanRunMore() const { + subprocs_.finished_.size()) < config_.parallelism); + return parallelism_limit_not_reached + && (subprocs_.running_.empty() || +- (config_.max_load_average <= 0.0f || +- GetLoadAverage() < config_.max_load_average)); ++ (max_load_average_ <= 0.0f || ++ GetLoadAverage() < max_load_average_)); + } + + bool RealCommandRunner::AcquireToken() { +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index af4be05a31..fb654c4d88 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(bool ignore); ++ bool Setup(bool ignore, double& max_load_average); + + private: + int available_; +@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup(bool ignore) { ++bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -118,9 +118,20 @@ bool GNUmakeTokenPool::Setup(bool ignore) { + CheckFd(rfd) && + CheckFd(wfd) && + SetAlarmHandler()) { ++ const char *l_arg = strstr(value, " -l"); ++ int load_limit = -1; ++ + printf("ninja: using GNU make jobserver.\n"); + rfd_ = rfd; + wfd_ = wfd; ++ ++ // translate GNU make -lN to ninja -lN ++ if (l_arg && ++ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && ++ (load_limit > 0)) { ++ max_load_average = load_limit; ++ } ++ + return true; + } + } +@@ -210,9 +221,9 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(bool ignore) { ++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore)) ++ if (tokenpool->Setup(ignore, max_load_average)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 199b22264b..e8e25426c3 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -22,6 +22,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore) { ++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 878a0933c2..f9e8cc2ee0 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -28,5 +28,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore); ++ static struct TokenPool *Get(bool ignore, double& max_load_average); + }; + +From 1c10047fc6a3269ba42839da19361e09cbc06ff0 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Wed, 6 Dec 2017 22:14:21 +0200 +Subject: [PATCH 05/11] Use LinePrinter for TokenPool messages + +- replace printf() with calls to LinePrinter +- print GNU make jobserver message only when verbose build is requested +--- + src/build.cc | 1 + + src/tokenpool-gnu-make.cc | 22 ++++++++++++++++------ + src/tokenpool-none.cc | 4 +++- + src/tokenpool.h | 4 +++- + 4 files changed, 23 insertions(+), 8 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index 9e6272d035..662e4bd7be 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -474,6 +474,7 @@ struct RealCommandRunner : public CommandRunner { + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { + max_load_average_ = config.max_load_average; + tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, ++ config_.verbosity == BuildConfig::VERBOSE, + max_load_average_); + } + +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index fb654c4d88..b0d3e6ebc4 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -23,6 +23,8 @@ + #include + #include + ++#include "line_printer.h" ++ + // TokenPool implementation for GNU make jobserver + // (http://make.mad-scientist.net/papers/jobserver-implementation/) + struct GNUmakeTokenPool : public TokenPool { +@@ -35,7 +37,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(bool ignore, double& max_load_average); ++ bool Setup(bool ignore, bool verbose, double& max_load_average); + + private: + int available_; +@@ -100,7 +102,9 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { ++bool GNUmakeTokenPool::Setup(bool ignore, ++ bool verbose, ++ double& max_load_average) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -109,8 +113,10 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + if (!jobserver) + jobserver = strstr(value, "--jobserver-auth="); + if (jobserver) { ++ LinePrinter printer; ++ + if (ignore) { +- printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); + } else { + int rfd = -1; + int wfd = -1; +@@ -121,7 +127,9 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + const char *l_arg = strstr(value, " -l"); + int load_limit = -1; + +- printf("ninja: using GNU make jobserver.\n"); ++ if (verbose) { ++ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); ++ } + rfd_ = rfd; + wfd_ = wfd; + +@@ -221,9 +229,11 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { ++struct TokenPool *TokenPool::Get(bool ignore, ++ bool verbose, ++ double& max_load_average) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore, max_load_average)) ++ if (tokenpool->Setup(ignore, verbose, max_load_average)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index e8e25426c3..1c1c499c8d 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -22,6 +22,8 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { ++struct TokenPool *TokenPool::Get(bool ignore, ++ bool verbose, ++ double& max_load_average) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index f9e8cc2ee0..4bf477f20c 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -28,5 +28,7 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore, double& max_load_average); ++ static struct TokenPool *Get(bool ignore, ++ bool verbose, ++ double& max_load_average); + }; + +From fdbf68416e3574add3bffd0b637d0694fbaba320 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sat, 7 Apr 2018 17:11:21 +0300 +Subject: [PATCH 06/11] Prepare PR for merging + +- fix Windows build error in no-op TokenPool implementation +- improve Acquire() to block for a maximum of 100ms +- address review comments +--- + src/build.h | 2 ++ + src/tokenpool-gnu-make.cc | 53 +++++++++++++++++++++++++++++++++------ + src/tokenpool-none.cc | 7 +----- + 3 files changed, 49 insertions(+), 13 deletions(-) + +diff --git a/src/build.h b/src/build.h +index dfde576573..66ddefb888 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -151,6 +151,8 @@ struct CommandRunner { + bool success() const { return status == ExitSuccess; } + }; + /// Wait for a command to complete, or return false if interrupted. ++ /// If more_ready is true then the optional TokenPool is monitored too ++ /// and we return when a token becomes available. + virtual bool WaitForCommand(Result* result, bool more_ready) = 0; + + virtual std::vector GetActiveEdges() { return std::vector(); } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index b0d3e6ebc4..4132bb06d9 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. ++// Copyright 2016-2018 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -153,6 +154,15 @@ bool GNUmakeTokenPool::Acquire() { + if (available_ > 0) + return true; + ++ // Please read ++ // ++ // http://make.mad-scientist.net/papers/jobserver-implementation/ ++ // ++ // for the reasoning behind the following code. ++ // ++ // Try to read one character from the pipe. Returns true on success. ++ // ++ // First check if read() would succeed without blocking. + #ifdef USE_PPOLL + pollfd pollfds[] = {{rfd_, POLLIN, 0}}; + int ret = poll(pollfds, 1, 0); +@@ -164,33 +174,62 @@ bool GNUmakeTokenPool::Acquire() { + int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); + #endif + if (ret > 0) { ++ // Handle potential race condition: ++ // - the above check succeeded, i.e. read() should not block ++ // - the character disappears before we call read() ++ // ++ // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_ ++ // can safely be closed by signal handlers without affecting rfd_. + dup_rfd_ = dup(rfd_); + + if (dup_rfd_ != -1) { + struct sigaction act, old_act; + int ret = 0; + ++ // Temporarily replace SIGCHLD handler with our own + memset(&act, 0, sizeof(act)); + act.sa_handler = CloseDupRfd; + if (sigaction(SIGCHLD, &act, &old_act) == 0) { +- char buf; +- +- // block until token read, child exits or timeout +- alarm(1); +- ret = read(dup_rfd_, &buf, 1); +- alarm(0); ++ struct itimerval timeout; ++ ++ // install a 100ms timeout that generates SIGALARM on expiration ++ memset(&timeout, 0, sizeof(timeout)); ++ timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] ++ if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { ++ char buf; ++ ++ // Now try to read() from dup_rfd_. Return values from read(): ++ // ++ // 1. token read -> 1 ++ // 2. pipe closed -> 0 ++ // 3. alarm expires -> -1 (EINTR) ++ // 4. child exits -> -1 (EINTR) ++ // 5. alarm expired before entering read() -> -1 (EBADF) ++ // 6. child exited before entering read() -> -1 (EBADF) ++ // 7. child exited before handler is installed -> go to 1 - 3 ++ ret = read(dup_rfd_, &buf, 1); ++ ++ // disarm timer ++ memset(&timeout, 0, sizeof(timeout)); ++ setitimer(ITIMER_REAL, &timeout, NULL); ++ } + + sigaction(SIGCHLD, &old_act, NULL); + } + + CloseDupRfd(0); + ++ // Case 1 from above list + if (ret > 0) { + available_++; + return true; + } + } + } ++ ++ // read() would block, i.e. no token available, ++ // cases 2-6 from above list or ++ // select() / poll() / dup() / sigaction() / setitimer() failed + return false; + } + +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 1c1c499c8d..4c592875b4 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. ++// Copyright 2016-2018 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -14,11 +14,6 @@ + + #include "tokenpool.h" + +-#include +-#include +-#include +-#include +-#include + #include + + // No-op TokenPool implementation + +From ec6220a0baf7d3a6eaf1a2b75bf8960ddfe24c2f Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Fri, 25 May 2018 00:17:07 +0300 +Subject: [PATCH 07/11] Add tests for TokenPool + +- TokenPool setup +- GetMonitorFd() API +- implicit token and tokens in jobserver pipe +- Acquire() / Reserve() / Release() protocol +- Clear() API +--- + configure.py | 1 + + src/tokenpool_test.cc | 198 ++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 199 insertions(+) + create mode 100644 src/tokenpool_test.cc + +diff --git a/configure.py b/configure.py +index db3492c93c..dc8a0066b7 100755 +--- a/configure.py ++++ b/configure.py +@@ -590,6 +590,7 @@ def has_re2c(): + 'string_piece_util_test', + 'subprocess_test', + 'test', ++ 'tokenpool_test', + 'util_test']: + objs += cxx(name, variables=cxxvariables) + if platform.is_windows(): +diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc +new file mode 100644 +index 0000000000..6c89064ca4 +--- /dev/null ++++ b/src/tokenpool_test.cc +@@ -0,0 +1,198 @@ ++// Copyright 2018 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "tokenpool.h" ++ ++#include "test.h" ++ ++#ifndef _WIN32 ++#include ++#include ++#include ++ ++#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") ++#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true); ++#endif ++ ++namespace { ++ ++const double kLoadAverageDefault = -1.23456789; ++ ++struct TokenPoolTest : public testing::Test { ++ double load_avg_; ++ TokenPool *tokens_; ++#ifndef _WIN32 ++ char buf_[1024]; ++ int fds_[2]; ++#endif ++ ++ virtual void SetUp() { ++ load_avg_ = kLoadAverageDefault; ++ tokens_ = NULL; ++#ifndef _WIN32 ++ ENVIRONMENT_CLEAR(); ++ if (pipe(fds_) < 0) ++ ASSERT_TRUE(false); ++#endif ++ } ++ ++ void CreatePool(const char *format, bool ignore_jobserver) { ++#ifndef _WIN32 ++ if (format) { ++ sprintf(buf_, format, fds_[0], fds_[1]); ++ ENVIRONMENT_INIT(buf_); ++ } ++#endif ++ tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_); ++ } ++ ++ void CreateDefaultPool() { ++ CreatePool("foo --jobserver-auth=%d,%d bar", false); ++ } ++ ++ virtual void TearDown() { ++ if (tokens_) ++ delete tokens_; ++#ifndef _WIN32 ++ close(fds_[0]); ++ close(fds_[1]); ++ ENVIRONMENT_CLEAR(); ++#endif ++ } ++}; ++ ++} // anonymous namespace ++ ++// verifies none implementation ++TEST_F(TokenPoolTest, NoTokenPool) { ++ CreatePool(NULL, false); ++ ++ EXPECT_EQ(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++#ifndef _WIN32 ++TEST_F(TokenPoolTest, SuccessfulOldSetup) { ++ // GNUmake <= 4.1 ++ CreatePool("foo --jobserver-fds=%d,%d bar", false); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, SuccessfulNewSetup) { ++ // GNUmake => 4.2 ++ CreateDefaultPool(); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, IgnoreWithJN) { ++ CreatePool("foo --jobserver-auth=%d,%d bar", true); ++ ++ EXPECT_EQ(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, HonorLN) { ++ CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(9.0, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, MonitorFD) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); ++} ++ ++TEST_F(TokenPoolTest, ImplicitToken) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++ ++TEST_F(TokenPoolTest, TwoTokens) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // jobserver offers 2nd token ++ ASSERT_EQ(1u, write(fds_[1], "T", 1)); ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // release 2nd token ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // release implict token - must return 2nd token back to jobserver ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // there must be one token in the pipe ++ EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++ ++TEST_F(TokenPoolTest, Clear) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // jobserver offers 2nd & 3rd token ++ ASSERT_EQ(2u, write(fds_[1], "TT", 2)); ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ tokens_->Clear(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // there must be two tokens in the pipe ++ EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++#endif + +From e59d8858327126d1593fd0b8e607975a79072e92 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Thu, 24 May 2018 18:52:45 +0300 +Subject: [PATCH 08/11] Add tests for subprocess module + +- add TokenPoolTest stub to provide TokenPool::GetMonitorFd() +- add two tests + * both tests set up a dummy GNUmake jobserver pipe + * both tests call DoWork() with TokenPoolTest + * test 1: verify that DoWork() detects when a token is available + * test 2: verify that DoWork() works as before without a token +- the tests are not compiled in under Windows +--- + src/subprocess_test.cc | 76 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 76 insertions(+) + +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index 4bc8083e26..6264c8bf11 100644 +--- a/src/subprocess_test.cc ++++ b/src/subprocess_test.cc +@@ -13,6 +13,7 @@ + // limitations under the License. + + #include "subprocess.h" ++#include "tokenpool.h" + + #include "test.h" + +@@ -34,8 +35,23 @@ const char* kSimpleCommand = "cmd /c dir \\"; + const char* kSimpleCommand = "ls /"; + #endif + ++struct TokenPoolTest : public TokenPool { ++ bool Acquire() { return false; } ++ void Reserve() {} ++ void Release() {} ++ void Clear() {} ++ ++#ifdef _WIN32 ++ // @TODO ++#else ++ int _fd; ++ int GetMonitorFd() { return _fd; } ++#endif ++}; ++ + struct SubprocessTest : public testing::Test { + SubprocessSet subprocs_; ++ TokenPoolTest tokens_; + }; + + } // anonymous namespace +@@ -280,3 +296,63 @@ TEST_F(SubprocessTest, ReadStdin) { + ASSERT_EQ(1u, subprocs_.finished_.size()); + } + #endif // _WIN32 ++ ++// @TODO: remove once TokenPool implementation for Windows is available ++#ifndef _WIN32 ++TEST_F(SubprocessTest, TokenAvailable) { ++ Subprocess* subproc = subprocs_.Add(kSimpleCommand); ++ ASSERT_NE((Subprocess *) 0, subproc); ++ ++ // simulate GNUmake jobserver pipe with 1 token ++ int fds[2]; ++ ASSERT_EQ(0u, pipe(fds)); ++ tokens_._fd = fds[0]; ++ ASSERT_EQ(1u, write(fds[1], "T", 1)); ++ ++ subprocs_.ResetTokenAvailable(); ++ subprocs_.DoWork(&tokens_); ++ ++ EXPECT_TRUE(subprocs_.IsTokenAvailable()); ++ EXPECT_EQ(0u, subprocs_.finished_.size()); ++ ++ // remove token to let DoWork() wait for command again ++ char token; ++ ASSERT_EQ(1u, read(fds[0], &token, 1)); ++ ++ while (!subproc->Done()) { ++ subprocs_.DoWork(&tokens_); ++ } ++ ++ close(fds[1]); ++ close(fds[0]); ++ ++ EXPECT_EQ(ExitSuccess, subproc->Finish()); ++ EXPECT_NE("", subproc->GetOutput()); ++ ++ EXPECT_EQ(1u, subprocs_.finished_.size()); ++} ++ ++TEST_F(SubprocessTest, TokenNotAvailable) { ++ Subprocess* subproc = subprocs_.Add(kSimpleCommand); ++ ASSERT_NE((Subprocess *) 0, subproc); ++ ++ // simulate GNUmake jobserver pipe with 0 tokens ++ int fds[2]; ++ ASSERT_EQ(0u, pipe(fds)); ++ tokens_._fd = fds[0]; ++ ++ subprocs_.ResetTokenAvailable(); ++ while (!subproc->Done()) { ++ subprocs_.DoWork(&tokens_); ++ } ++ ++ close(fds[1]); ++ close(fds[0]); ++ ++ EXPECT_FALSE(subprocs_.IsTokenAvailable()); ++ EXPECT_EQ(ExitSuccess, subproc->Finish()); ++ EXPECT_NE("", subproc->GetOutput()); ++ ++ EXPECT_EQ(1u, subprocs_.finished_.size()); ++} ++#endif // _WIN32 + +From 0145e2d4db64ea6c21aeb371928e4071f65164eb Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sat, 26 May 2018 23:17:51 +0300 +Subject: [PATCH 09/11] Add tests for build module + +Add tests that verify the token functionality of the builder main loop. +We replace the default fake command runner with a special version where +the tests can control each call to AcquireToken(), CanRunMore() and +WaitForCommand(). +--- + src/build_test.cc | 364 ++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 364 insertions(+) + +diff --git a/src/build_test.cc b/src/build_test.cc +index 7a5ff4015a..dd41dfbe1d 100644 +--- a/src/build_test.cc ++++ b/src/build_test.cc +@@ -15,6 +15,7 @@ + #include "build.h" + + #include ++#include + + #include "build_log.h" + #include "deps_log.h" +@@ -3990,3 +3991,366 @@ TEST_F(BuildTest, ValidationWithCircularDependency) { + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err); } + +/// The token tests are concerned with the main loop functionality when @@ -706,6 +2177,16 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + } +} + ++TEST_F(BuildTokenTest, CompleteNoWork) { ++ // plan should not execute anything ++ string err; ++ ++ EXPECT_TRUE(builder_.Build(&err)); ++ EXPECT_EQ("", err); ++ ++ EXPECT_EQ(0u, token_command_runner_.commands_ran_.size()); ++} ++ +TEST_F(BuildTokenTest, DoNotAquireToken) { + // plan should execute one command + string err; @@ -906,109 +2387,106 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + token_command_runner_.commands_ran_[1] == "cat in1 > out1")); + EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat out1 out2 > out12"); +} ---- a/src/exit_status.h -+++ b/src/exit_status.h -@@ -18,7 +18,8 @@ - enum ExitStatus { - ExitSuccess, - ExitFailure, -- ExitInterrupted -+ ExitTokenAvailable, -+ ExitInterrupted, - }; + +From f016e5430c9123d34a73ea7ad28693b20ee59d6d Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Mon, 8 Oct 2018 17:47:50 +0300 +Subject: [PATCH 10/11] Add Win32 implementation for GNUmakeTokenPool + +GNU make uses a semaphore as jobserver protocol on Win32. See also + + https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html + +Usage is pretty simple and straightforward, i.e. WaitForSingleObject() +to obtain a token and ReleaseSemaphore() to return it. + +Unfortunately subprocess-win32.cc uses an I/O completion port (IOCP). +IOCPs aren't waitable objects, i.e. we can't use WaitForMultipleObjects() +to wait on the IOCP and the token semaphore at the same time. + +Therefore GNUmakeTokenPoolWin32 creates a child thread that waits on the +token semaphore and posts a dummy I/O completion status on the IOCP when +it was able to obtain a token. That unblocks SubprocessSet::DoWork() and +it can then check if a token became available or not. + +- split existing GNUmakeTokenPool into common and platform bits +- add GNUmakeTokenPool interface +- move the Posix bits to GNUmakeTokenPoolPosix +- add the Win32 bits as GNUmakeTokenPoolWin32 +- move Setup() method up to TokenPool interface +- update Subprocess & TokenPool tests accordingly +--- + configure.py | 8 +- + src/build.cc | 11 +- + src/subprocess-win32.cc | 9 ++ + src/subprocess_test.cc | 34 ++++- + src/tokenpool-gnu-make-posix.cc | 203 +++++++++++++++++++++++++++ + src/tokenpool-gnu-make-win32.cc | 237 ++++++++++++++++++++++++++++++++ + src/tokenpool-gnu-make.cc | 203 ++------------------------- + src/tokenpool-gnu-make.h | 40 ++++++ + src/tokenpool-none.cc | 4 +- + src/tokenpool.h | 18 ++- + src/tokenpool_test.cc | 113 ++++++++++++--- + 11 files changed, 653 insertions(+), 227 deletions(-) + create mode 100644 src/tokenpool-gnu-make-posix.cc + create mode 100644 src/tokenpool-gnu-make-win32.cc + create mode 100644 src/tokenpool-gnu-make.h + +diff --git a/configure.py b/configure.py +index dc8a0066b7..a239b90eef 100755 +--- a/configure.py ++++ b/configure.py +@@ -517,12 +517,13 @@ def has_re2c(): + 'state', + 'status', + 'string_piece_util', ++ 'tokenpool-gnu-make', + 'util', + 'version']: + objs += cxx(name, variables=cxxvariables) + if platform.is_windows(): + for name in ['subprocess-win32', +- 'tokenpool-none', ++ 'tokenpool-gnu-make-win32', + 'includes_normalize-win32', + 'msvc_helper-win32', + 'msvc_helper_main-win32']: +@@ -531,8 +532,9 @@ def has_re2c(): + objs += cxx('minidump-win32', variables=cxxvariables) + objs += cc('getopt') + else: +- objs += cxx('subprocess-posix') +- objs += cxx('tokenpool-gnu-make') ++ for name in ['subprocess-posix', ++ 'tokenpool-gnu-make-posix']: ++ objs += cxx(name) + if platform.is_aix(): + objs += cc('getopt') + if platform.is_msvc(): +diff --git a/src/build.cc b/src/build.cc +index 662e4bd7be..20c3bdc2a0 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -473,9 +473,14 @@ struct RealCommandRunner : public CommandRunner { - #endif // NINJA_EXIT_STATUS_H_ ---- a/src/ninja.cc -+++ b/src/ninja.cc -@@ -1289,6 +1289,7 @@ int ReadFlags(int* argc, char*** argv, - // We want to run N jobs in parallel. For N = 0, INT_MAX - // is close enough to infinite for most sane builds. - config->parallelism = value > 0 ? value : INT_MAX; -+ config->parallelism_from_cmdline = true; - break; - } - case 'k': { ---- a/src/subprocess-posix.cc -+++ b/src/subprocess-posix.cc -@@ -13,6 +13,7 @@ - // limitations under the License. - - #include "subprocess.h" -+#include "tokenpool.h" - - #include - #include -@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const str - } - - #ifdef USE_PPOLL --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - vector fds; - nfds_t nfds = 0; - -@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() { - ++nfds; - } - -+ if (tokens) { -+ pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 }; -+ fds.push_back(pfd); -+ ++nfds; -+ } -+ - interrupted_ = 0; - int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); - if (ret == -1) { -@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() { - ++i; - } - -+ if (tokens) { -+ pollfd *pfd = &fds[nfds - 1]; -+ if (pfd->fd >= 0) { -+ assert(pfd->fd == tokens->GetMonitorFd()); -+ if (pfd->revents != 0) -+ token_available_ = true; + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { + max_load_average_ = config.max_load_average; +- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, +- config_.verbosity == BuildConfig::VERBOSE, +- max_load_average_); ++ if ((tokens_ = TokenPool::Get()) != NULL) { ++ if (!tokens_->Setup(config_.parallelism_from_cmdline, ++ config_.verbosity == BuildConfig::VERBOSE, ++ max_load_average_)) { ++ delete tokens_; ++ tokens_ = NULL; + } + } -+ - return IsInterrupted(); } - #else // !defined(USE_PPOLL) --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - fd_set set; - int nfds = 0; - FD_ZERO(&set); -@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() { - } - } - -+ if (tokens) { -+ int fd = tokens->GetMonitorFd(); -+ FD_SET(fd, &set); -+ if (nfds < fd+1) -+ nfds = fd+1; -+ } -+ - interrupted_ = 0; - int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); - if (ret == -1) { -@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() { - ++i; - } - -+ if (tokens) { -+ int fd = tokens->GetMonitorFd(); -+ if ((fd >= 0) && FD_ISSET(fd, &set)) -+ token_available_ = true; -+ } -+ - return IsInterrupted(); - } - #endif // !defined(USE_PPOLL) + RealCommandRunner::~RealCommandRunner() { +diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc +index 66d2c2c430..ce3e2c20a4 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -13,6 +13,7 @@ @@ -1019,13 +2497,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 #include #include -@@ -251,11 +252,14 @@ Subprocess *SubprocessSet::Add(const str - return subprocess; - } - --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - DWORD bytes_read; +@@ -256,6 +257,9 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { Subprocess* subproc; OVERLAPPED* overlapped; @@ -1035,7 +2507,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 if (!GetQueuedCompletionStatus(ioport_, &bytes_read, (PULONG_PTR)&subproc, &overlapped, INFINITE)) { if (GetLastError() != ERROR_BROKEN_PIPE) -@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork() { +@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { // delivered by NotifyInterrupted above. return true; @@ -1047,58 +2519,18 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 subproc->OnPipeReady(); if (subproc->Done()) { ---- a/src/subprocess.h -+++ b/src/subprocess.h -@@ -76,6 +76,8 @@ struct Subprocess { - friend struct SubprocessSet; - }; - -+struct TokenPool; -+ - /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses. - /// DoWork() waits for any state change in subprocesses; finished_ - /// is a queue of subprocesses as they finish. -@@ -84,13 +86,17 @@ struct SubprocessSet { - ~SubprocessSet(); - - Subprocess* Add(const std::string& command, bool use_console = false); -- bool DoWork(); -+ bool DoWork(struct TokenPool* tokens); - Subprocess* NextFinished(); - void Clear(); - - std::vector running_; - std::queue finished_; - -+ bool token_available_; -+ bool IsTokenAvailable() { return token_available_; } -+ void ResetTokenAvailable() { token_available_ = false; } -+ - #ifdef _WIN32 - static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType); - static HANDLE ioport_; +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index 6264c8bf11..f625963462 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc -@@ -13,6 +13,7 @@ - // limitations under the License. - - #include "subprocess.h" -+#include "tokenpool.h" - - #include "test.h" - -@@ -34,8 +35,30 @@ const char* kSimpleCommand = "cmd /c dir - const char* kSimpleCommand = "ls /"; - #endif - -+struct TestTokenPool : public TokenPool { -+ bool Acquire() { return false; } -+ void Reserve() {} -+ void Release() {} -+ void Clear() {} +@@ -40,9 +40,16 @@ struct TokenPoolTest : public TokenPool { + void Reserve() {} + void Release() {} + void Clear() {} + bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; } -+ -+#ifdef _WIN32 + + #ifdef _WIN32 +- // @TODO + bool _token_available; + void WaitForTokenAvailability(HANDLE ioport) { + if (_token_available) @@ -1106,197 +2538,31 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + PostQueuedCompletionStatus(ioport, 0, (ULONG_PTR) this, NULL); + } + bool TokenIsAvailable(ULONG_PTR key) { return key == (ULONG_PTR) this; } -+#else -+ int _fd; -+ int GetMonitorFd() { return _fd; } -+#endif -+}; -+ - struct SubprocessTest : public testing::Test { - SubprocessSet subprocs_; -+ TestTokenPool tokens_; - }; - - } // anonymous namespace -@@ -45,10 +68,12 @@ TEST_F(SubprocessTest, BadCommandStderr) - Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { - // Pretend we discovered that stderr was ready for writing. -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -59,10 +84,12 @@ TEST_F(SubprocessTest, NoSuchCommand) { - Subprocess* subproc = subprocs_.Add("ninja_no_such_command"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { - // Pretend we discovered that stderr was ready for writing. -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -78,9 +105,11 @@ TEST_F(SubprocessTest, InterruptChild) { - Subprocess* subproc = subprocs_.Add("kill -INT $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); + #else + int _fd; + int GetMonitorFd() { return _fd; } +@@ -297,34 +304,48 @@ TEST_F(SubprocessTest, ReadStdin) { } -@@ -90,7 +119,7 @@ TEST_F(SubprocessTest, InterruptParent) - ASSERT_NE((Subprocess *) 0, subproc); + #endif // _WIN32 - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -102,9 +131,11 @@ TEST_F(SubprocessTest, InterruptChildWit - Subprocess* subproc = subprocs_.Add("kill -TERM $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -114,7 +145,7 @@ TEST_F(SubprocessTest, InterruptParentWi - ASSERT_NE((Subprocess *) 0, subproc); - - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -126,9 +157,11 @@ TEST_F(SubprocessTest, InterruptChildWit - Subprocess* subproc = subprocs_.Add("kill -HUP $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -138,7 +171,7 @@ TEST_F(SubprocessTest, InterruptParentWi - ASSERT_NE((Subprocess *) 0, subproc); - - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -153,9 +186,11 @@ TEST_F(SubprocessTest, Console) { - subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true); - ASSERT_NE((Subprocess*)0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitSuccess, subproc->Finish()); - } -@@ -167,9 +202,11 @@ TEST_F(SubprocessTest, SetWithSingle) { +-// @TODO: remove once TokenPool implementation for Windows is available +-#ifndef _WIN32 + TEST_F(SubprocessTest, TokenAvailable) { Subprocess* subproc = subprocs_.Add(kSimpleCommand); ASSERT_NE((Subprocess *) 0, subproc); -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_NE("", subproc->GetOutput()); - -@@ -200,12 +237,13 @@ TEST_F(SubprocessTest, SetWithMulti) { - ASSERT_EQ("", processes[i]->GetOutput()); - } - -+ subprocs_.ResetTokenAvailable(); - while (!processes[0]->Done() || !processes[1]->Done() || - !processes[2]->Done()) { - ASSERT_GT(subprocs_.running_.size(), 0u); -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -- -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(0u, subprocs_.running_.size()); - ASSERT_EQ(3u, subprocs_.finished_.size()); - -@@ -237,8 +275,10 @@ TEST_F(SubprocessTest, SetWithLots) { - ASSERT_NE((Subprocess *) 0, subproc); - procs.push_back(subproc); - } -+ subprocs_.ResetTokenAvailable(); - while (!subprocs_.running_.empty()) -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - for (size_t i = 0; i < procs.size(); ++i) { - ASSERT_EQ(ExitSuccess, procs[i]->Finish()); - ASSERT_NE("", procs[i]->GetOutput()); -@@ -254,10 +294,91 @@ TEST_F(SubprocessTest, SetWithLots) { - // that stdin is closed. - TEST_F(SubprocessTest, ReadStdin) { - Subprocess* subproc = subprocs_.Add("cat -"); -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_EQ(1u, subprocs_.finished_.size()); - } - #endif // _WIN32 -+ -+TEST_F(SubprocessTest, TokenAvailable) { -+ Subprocess* subproc = subprocs_.Add(kSimpleCommand); -+ ASSERT_NE((Subprocess *) 0, subproc); -+ -+ // simulate GNUmake jobserver pipe with 1 token + // simulate GNUmake jobserver pipe with 1 token +#ifdef _WIN32 + tokens_._token_available = true; +#else -+ int fds[2]; -+ ASSERT_EQ(0u, pipe(fds)); -+ tokens_._fd = fds[0]; -+ ASSERT_EQ(1u, write(fds[1], "T", 1)); + int fds[2]; + ASSERT_EQ(0u, pipe(fds)); + tokens_._fd = fds[0]; + ASSERT_EQ(1u, write(fds[1], "T", 1)); +#endif -+ -+ subprocs_.ResetTokenAvailable(); -+ subprocs_.DoWork(&tokens_); + + subprocs_.ResetTokenAvailable(); + subprocs_.DoWork(&tokens_); +#ifdef _WIN32 + tokens_._token_available = false; + // we need to loop here as we have no conrol where the token @@ -1305,63 +2571,62 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + subprocs_.DoWork(&tokens_); + } +#endif -+ -+ EXPECT_TRUE(subprocs_.IsTokenAvailable()); -+ EXPECT_EQ(0u, subprocs_.finished_.size()); -+ -+ // remove token to let DoWork() wait for command again + + EXPECT_TRUE(subprocs_.IsTokenAvailable()); + EXPECT_EQ(0u, subprocs_.finished_.size()); + + // remove token to let DoWork() wait for command again +#ifndef _WIN32 -+ char token; -+ ASSERT_EQ(1u, read(fds[0], &token, 1)); + char token; + ASSERT_EQ(1u, read(fds[0], &token, 1)); +#endif -+ -+ while (!subproc->Done()) { -+ subprocs_.DoWork(&tokens_); -+ } -+ + + while (!subproc->Done()) { + subprocs_.DoWork(&tokens_); + } + +#ifndef _WIN32 -+ close(fds[1]); -+ close(fds[0]); + close(fds[1]); + close(fds[0]); +#endif -+ -+ EXPECT_EQ(ExitSuccess, subproc->Finish()); -+ EXPECT_NE("", subproc->GetOutput()); -+ -+ EXPECT_EQ(1u, subprocs_.finished_.size()); -+} -+ -+TEST_F(SubprocessTest, TokenNotAvailable) { -+ Subprocess* subproc = subprocs_.Add(kSimpleCommand); -+ ASSERT_NE((Subprocess *) 0, subproc); -+ -+ // simulate GNUmake jobserver pipe with 0 tokens + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -337,17 +358,23 @@ TEST_F(SubprocessTest, TokenNotAvailable) { + ASSERT_NE((Subprocess *) 0, subproc); + + // simulate GNUmake jobserver pipe with 0 tokens +#ifdef _WIN32 + tokens_._token_available = false; +#else -+ int fds[2]; -+ ASSERT_EQ(0u, pipe(fds)); -+ tokens_._fd = fds[0]; + int fds[2]; + ASSERT_EQ(0u, pipe(fds)); + tokens_._fd = fds[0]; +#endif -+ -+ subprocs_.ResetTokenAvailable(); -+ while (!subproc->Done()) { -+ subprocs_.DoWork(&tokens_); -+ } -+ + + subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + subprocs_.DoWork(&tokens_); + } + +#ifndef _WIN32 -+ close(fds[1]); -+ close(fds[0]); + close(fds[1]); + close(fds[0]); +#endif -+ -+ EXPECT_FALSE(subprocs_.IsTokenAvailable()); -+ EXPECT_EQ(ExitSuccess, subproc->Finish()); -+ EXPECT_NE("", subproc->GetOutput()); -+ -+ EXPECT_EQ(1u, subprocs_.finished_.size()); -+} + + EXPECT_FALSE(subprocs_.IsTokenAvailable()); + EXPECT_EQ(ExitSuccess, subproc->Finish()); +@@ -355,4 +382,3 @@ TEST_F(SubprocessTest, TokenNotAvailable) { + + EXPECT_EQ(1u, subprocs_.finished_.size()); + } +-#endif // _WIN32 +diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc +new file mode 100644 +index 0000000000..70d84bfff7 --- /dev/null +++ b/src/tokenpool-gnu-make-posix.cc -@@ -0,0 +1,202 @@ +@@ -0,0 +1,203 @@ +// Copyright 2016-2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); @@ -1396,8 +2661,8 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + + virtual int GetMonitorFd(); + -+ virtual const char* GetEnv(const char* name) { return getenv(name); }; -+ virtual bool ParseAuth(const char* jobserver); ++ virtual const char *GetEnv(const char *name) { return getenv(name); }; ++ virtual bool ParseAuth(const char *jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + @@ -1446,13 +2711,14 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + act.sa_handler = CloseDupRfd; + if (sigaction(SIGALRM, &act, &old_act_) < 0) { + perror("sigaction:"); -+ return false; ++ return(false); ++ } else { ++ restore_ = true; ++ return(true); + } -+ restore_ = true; -+ return true; +} + -+bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) { ++bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) { + int rfd = -1; + int wfd = -1; + if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && @@ -1558,15 +2824,18 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +} + +int GNUmakeTokenPoolPosix::GetMonitorFd() { -+ return rfd_; ++ return(rfd_); +} + -+TokenPool* TokenPool::Get() { ++struct TokenPool *TokenPool::Get() { + return new GNUmakeTokenPoolPosix; +} +diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc +new file mode 100644 +index 0000000000..2719f2c1fc --- /dev/null +++ b/src/tokenpool-gnu-make-win32.cc -@@ -0,0 +1,239 @@ +@@ -0,0 +1,237 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); @@ -1583,8 +2852,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + +#include "tokenpool-gnu-make.h" + -+// Always include this first. -+// Otherwise the other system headers don't work correctly under Win32 ++// always include first to make sure other headers do the correct thing... +#include + +#include @@ -1602,8 +2870,8 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + virtual void WaitForTokenAvailability(HANDLE ioport); + virtual bool TokenIsAvailable(ULONG_PTR key); + -+ virtual const char* GetEnv(const char* name); -+ virtual bool ParseAuth(const char* jobserver); ++ virtual const char *GetEnv(const char *name); ++ virtual bool ParseAuth(const char *jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + @@ -1668,19 +2936,19 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + } +} + -+const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) { ++const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) { + // getenv() does not work correctly together with tokenpool_tests.cc + static char buffer[MAX_PATH + 1]; -+ if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0) ++ if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0) + return NULL; -+ return buffer; ++ return(buffer); +} + -+bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) { ++bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { + // match "--jobserver-auth=gmake_semaphore_..." -+ const char* start = strchr(jobserver, '='); ++ const char *start = strchr(jobserver, '='); + if (start) { -+ const char* end = start; ++ const char *end = start; + unsigned int len; + char c, *auth; + @@ -1689,15 +2957,14 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + break; + len = end - start; // includes string terminator in count + -+ if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) { ++ if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) { + strncpy(auth, start + 1, len - 1); + auth[len - 1] = '\0'; + -+ if ((semaphore_jobserver_ = -+ OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ -+ FALSE, /* Child processes DON'T inherit */ -+ auth /* Semaphore name */ -+ )) != NULL) { ++ if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ ++ FALSE, /* Child processes DON'T inherit */ ++ auth /* Semaphore name */ ++ )) != NULL) { + free(auth); + return true; + } @@ -1742,7 +3009,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +} + +DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) { -+ GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param; ++ GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param; + return This->SemaphoreThread(); +} + @@ -1787,7 +3054,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + +bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) { + // alert child thread to break wait on token semaphore -+ QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL); ++ QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL); + + // return true when GetQueuedCompletionStatus() returned our key + return key == (ULONG_PTR) this; @@ -1803,120 +3070,273 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + Win32Fatal("WaitForSingleObject"); +} + -+TokenPool* TokenPool::Get() { ++struct TokenPool *TokenPool::Get() { + return new GNUmakeTokenPoolWin32; +} ---- /dev/null +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index 4132bb06d9..92ff611721 100644 +--- a/src/tokenpool-gnu-make.cc +++ b/src/tokenpool-gnu-make.cc -@@ -0,0 +1,108 @@ -+// Copyright 2016-2018 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ +@@ -12,101 +12,26 @@ + // See the License for the specific language governing permissions and + // limitations under the License. + +-#include "tokenpool.h" +#include "tokenpool-gnu-make.h" -+ + +-#include +-#include +-#include +-#include +-#include +-#include +#include -+#include -+#include -+ -+#include "line_printer.h" -+ + #include + #include +-#include + + #include "line_printer.h" + +-// TokenPool implementation for GNU make jobserver +-// (http://make.mad-scientist.net/papers/jobserver-implementation/) +-struct GNUmakeTokenPool : public TokenPool { +- GNUmakeTokenPool(); +- virtual ~GNUmakeTokenPool(); +- +- virtual bool Acquire(); +- virtual void Reserve(); +- virtual void Release(); +- virtual void Clear(); +- virtual int GetMonitorFd(); +- +- bool Setup(bool ignore, bool verbose, double& max_load_average); +- +- private: +- int available_; +- int used_; +- +-#ifdef _WIN32 +- // @TODO +-#else +- int rfd_; +- int wfd_; +- +- struct sigaction old_act_; +- bool restore_; +- +- static int dup_rfd_; +- static void CloseDupRfd(int signum); +- +- bool CheckFd(int fd); +- bool SetAlarmHandler(); +-#endif +- +- void Return(); +-}; +- +// TokenPool implementation for GNU make jobserver - common bits -+// every instance owns an implicit token -> available_ == 1 + // every instance owns an implicit token -> available_ == 1 +-GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0), +- rfd_(-1), wfd_(-1), restore_(false) { +GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) { -+} -+ -+GNUmakeTokenPool::~GNUmakeTokenPool() { -+} -+ -+bool GNUmakeTokenPool::Setup(bool ignore, -+ bool verbose, -+ double& max_load_average) { -+ const char* value = GetEnv("MAKEFLAGS"); -+ if (!value) -+ return false; -+ -+ // GNU make <= 4.1 -+ const char* jobserver = strstr(value, "--jobserver-fds="); -+ if (!jobserver) -+ // GNU make => 4.2 -+ jobserver = strstr(value, "--jobserver-auth="); -+ if (jobserver) { -+ LinePrinter printer; -+ -+ if (ignore) { -+ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); -+ } else { -+ if (ParseAuth(jobserver)) { -+ const char* l_arg = strstr(value, " -l"); -+ int load_limit = -1; -+ -+ if (verbose) { -+ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); -+ } -+ -+ // translate GNU make -lN to ninja -lN -+ if (l_arg && -+ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && -+ (load_limit > 0)) { -+ max_load_average = load_limit; -+ } -+ -+ return true; -+ } -+ } -+ } -+ -+ return false; -+} -+ -+bool GNUmakeTokenPool::Acquire() { -+ if (available_ > 0) -+ return true; -+ + } + + GNUmakeTokenPool::~GNUmakeTokenPool() { +- Clear(); +- if (restore_) +- sigaction(SIGALRM, &old_act_, NULL); +-} +- +-bool GNUmakeTokenPool::CheckFd(int fd) { +- if (fd < 0) +- return false; +- int ret = fcntl(fd, F_GETFD); +- if (ret < 0) +- return false; +- return true; +-} +- +-int GNUmakeTokenPool::dup_rfd_ = -1; +- +-void GNUmakeTokenPool::CloseDupRfd(int signum) { +- close(dup_rfd_); +- dup_rfd_ = -1; +-} +- +-bool GNUmakeTokenPool::SetAlarmHandler() { +- struct sigaction act; +- memset(&act, 0, sizeof(act)); +- act.sa_handler = CloseDupRfd; +- if (sigaction(SIGALRM, &act, &old_act_) < 0) { +- perror("sigaction:"); +- return(false); +- } else { +- restore_ = true; +- return(true); +- } + } + + bool GNUmakeTokenPool::Setup(bool ignore, + bool verbose, + double& max_load_average) { +- const char *value = getenv("MAKEFLAGS"); ++ const char *value = GetEnv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 + const char *jobserver = strstr(value, "--jobserver-fds="); +@@ -119,20 +44,13 @@ bool GNUmakeTokenPool::Setup(bool ignore, + if (ignore) { + printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); + } else { +- int rfd = -1; +- int wfd = -1; +- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && +- CheckFd(rfd) && +- CheckFd(wfd) && +- SetAlarmHandler()) { ++ if (ParseAuth(jobserver)) { + const char *l_arg = strstr(value, " -l"); + int load_limit = -1; + + if (verbose) { + printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); + } +- rfd_ = rfd; +- wfd_ = wfd; + + // translate GNU make -lN to ninja -lN + if (l_arg && +@@ -154,83 +72,14 @@ bool GNUmakeTokenPool::Acquire() { + if (available_ > 0) + return true; + +- // Please read +- // +- // http://make.mad-scientist.net/papers/jobserver-implementation/ +- // +- // for the reasoning behind the following code. +- // +- // Try to read one character from the pipe. Returns true on success. +- // +- // First check if read() would succeed without blocking. +-#ifdef USE_PPOLL +- pollfd pollfds[] = {{rfd_, POLLIN, 0}}; +- int ret = poll(pollfds, 1, 0); +-#else +- fd_set set; +- struct timeval timeout = { 0, 0 }; +- FD_ZERO(&set); +- FD_SET(rfd_, &set); +- int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); +-#endif +- if (ret > 0) { +- // Handle potential race condition: +- // - the above check succeeded, i.e. read() should not block +- // - the character disappears before we call read() +- // +- // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_ +- // can safely be closed by signal handlers without affecting rfd_. +- dup_rfd_ = dup(rfd_); +- +- if (dup_rfd_ != -1) { +- struct sigaction act, old_act; +- int ret = 0; +- +- // Temporarily replace SIGCHLD handler with our own +- memset(&act, 0, sizeof(act)); +- act.sa_handler = CloseDupRfd; +- if (sigaction(SIGCHLD, &act, &old_act) == 0) { +- struct itimerval timeout; +- +- // install a 100ms timeout that generates SIGALARM on expiration +- memset(&timeout, 0, sizeof(timeout)); +- timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] +- if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { +- char buf; +- +- // Now try to read() from dup_rfd_. Return values from read(): +- // +- // 1. token read -> 1 +- // 2. pipe closed -> 0 +- // 3. alarm expires -> -1 (EINTR) +- // 4. child exits -> -1 (EINTR) +- // 5. alarm expired before entering read() -> -1 (EBADF) +- // 6. child exited before entering read() -> -1 (EBADF) +- // 7. child exited before handler is installed -> go to 1 - 3 +- ret = read(dup_rfd_, &buf, 1); +- +- // disarm timer +- memset(&timeout, 0, sizeof(timeout)); +- setitimer(ITIMER_REAL, &timeout, NULL); +- } +- +- sigaction(SIGCHLD, &old_act, NULL); +- } +- +- CloseDupRfd(0); +- +- // Case 1 from above list +- if (ret > 0) { +- available_++; +- return true; +- } +- } + if (AcquireToken()) { + // token acquired + available_++; + return true; -+ } -+ -+ // no token available -+ return false; -+} -+ -+void GNUmakeTokenPool::Reserve() { -+ available_--; -+ used_++; -+} -+ -+void GNUmakeTokenPool::Return() { ++ } else { ++ // no token available ++ return false; + } +- +- // read() would block, i.e. no token available, +- // cases 2-6 from above list or +- // select() / poll() / dup() / sigaction() / setitimer() failed +- return false; + } + + void GNUmakeTokenPool::Reserve() { +@@ -239,15 +88,8 @@ void GNUmakeTokenPool::Reserve() { + } + + void GNUmakeTokenPool::Return() { +- const char buf = '+'; +- while (1) { +- int ret = write(wfd_, &buf, 1); +- if (ret > 0) +- available_--; +- if ((ret != -1) || (errno != EINTR)) +- return; +- // write got interrupted - retry +- } + if (ReturnToken()) + available_--; -+} -+ -+void GNUmakeTokenPool::Release() { -+ available_++; -+ used_--; -+ if (available_ > 1) -+ Return(); -+} -+ -+void GNUmakeTokenPool::Clear() { -+ while (used_ > 0) -+ Release(); -+ while (available_ > 1) -+ Return(); -+} + } + + void GNUmakeTokenPool::Release() { +@@ -263,18 +105,3 @@ void GNUmakeTokenPool::Clear() { + while (available_ > 1) + Return(); + } +- +-int GNUmakeTokenPool::GetMonitorFd() { +- return(rfd_); +-} +- +-struct TokenPool *TokenPool::Get(bool ignore, +- bool verbose, +- double& max_load_average) { +- GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore, verbose, max_load_average)) +- return tokenpool; +- else +- delete tokenpool; +- return NULL; +-} +diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h +new file mode 100644 +index 0000000000..d3852088e2 --- /dev/null +++ b/src/tokenpool-gnu-make.h @@ -0,0 +1,40 @@ @@ -1939,7 +3359,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +// interface to GNU make token pool +struct GNUmakeTokenPool : public TokenPool { + GNUmakeTokenPool(); -+ ~GNUmakeTokenPool(); ++ virtual ~GNUmakeTokenPool(); + + // token pool implementation + virtual bool Acquire(); @@ -1949,8 +3369,8 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + virtual bool Setup(bool ignore, bool verbose, double& max_load_average); + + // platform specific implementation -+ virtual const char* GetEnv(const char* name) = 0; -+ virtual bool ParseAuth(const char* jobserver) = 0; ++ virtual const char *GetEnv(const char *name) = 0; ++ virtual bool ParseAuth(const char *jobserver) = 0; + virtual bool AcquireToken() = 0; + virtual bool ReturnToken() = 0; + @@ -1960,81 +3380,83 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + + void Return(); +}; ---- /dev/null +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 4c592875b4..613d16882d 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -17,8 +17,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore, +- bool verbose, +- double& max_load_average) { ++struct TokenPool *TokenPool::Get() { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 4bf477f20c..1be8e1d5ce 100644 +--- a/src/tokenpool.h +++ b/src/tokenpool.h -@@ -0,0 +1,42 @@ +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. +// Copyright 2016-2018 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -12,6 +12,10 @@ + // See the License for the specific language governing permissions and + // limitations under the License. + +#ifdef _WIN32 +#include +#endif + -+// interface to token pool -+struct TokenPool { -+ virtual ~TokenPool() {} -+ -+ virtual bool Acquire() = 0; -+ virtual void Reserve() = 0; -+ virtual void Release() = 0; -+ virtual void Clear() = 0; -+ + // interface to token pool + struct TokenPool { + virtual ~TokenPool() {} +@@ -21,14 +25,18 @@ struct TokenPool { + virtual void Release() = 0; + virtual void Clear() = 0; + + // returns false if token pool setup failed + virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0; + -+#ifdef _WIN32 + #ifdef _WIN32 +- // @TODO + virtual void WaitForTokenAvailability(HANDLE ioport) = 0; + // returns true if a token has become available + // key is result from GetQueuedCompletionStatus() + virtual bool TokenIsAvailable(ULONG_PTR key) = 0; -+#else -+ virtual int GetMonitorFd() = 0; -+#endif -+ -+ // returns NULL if token pool is not available -+ static TokenPool* Get(); -+}; ---- /dev/null + #else + virtual int GetMonitorFd() = 0; + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore, +- bool verbose, +- double& max_load_average); ++ static struct TokenPool *Get(); + }; +diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc +index 6c89064ca4..8d4fd7d33a 100644 +--- a/src/tokenpool_test.cc +++ b/src/tokenpool_test.cc -@@ -0,0 +1,269 @@ -+// Copyright 2018 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ -+#include "tokenpool.h" -+ -+#include "test.h" -+ +@@ -16,13 +16,25 @@ + + #include "test.h" + +-#ifndef _WIN32 +#ifdef _WIN32 +#include +#else +#include +#endif + -+#include -+#include -+ + #include + #include +-#include + +#ifdef _WIN32 +// should contain all valid characters +#define SEMAPHORE_NAME "abcdefghijklmnopqrstwxyz01234567890_" @@ -2043,40 +3465,45 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +#define ENVIRONMENT_INIT(v) SetEnvironmentVariable("MAKEFLAGS", v) +#else +#define AUTH_FORMAT(tmpl) "foo " tmpl "=%d,%d bar" -+#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") + #define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") +-#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true); +#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true) -+#endif -+ -+namespace { -+ -+const double kLoadAverageDefault = -1.23456789; -+ -+struct TokenPoolTest : public testing::Test { -+ double load_avg_; -+ TokenPool* tokens_; -+ char buf_[1024]; + #endif + + namespace { +@@ -32,43 +44,60 @@ const double kLoadAverageDefault = -1.23456789; + struct TokenPoolTest : public testing::Test { + double load_avg_; + TokenPool *tokens_; +-#ifndef _WIN32 + char buf_[1024]; +#ifdef _WIN32 -+ const char* semaphore_name_; ++ const char *semaphore_name_; + HANDLE semaphore_; +#else -+ int fds_[2]; -+#endif -+ -+ virtual void SetUp() { -+ load_avg_ = kLoadAverageDefault; -+ tokens_ = NULL; -+ ENVIRONMENT_CLEAR(); + int fds_[2]; + #endif + + virtual void SetUp() { + load_avg_ = kLoadAverageDefault; + tokens_ = NULL; +-#ifndef _WIN32 + ENVIRONMENT_CLEAR(); +#ifdef _WIN32 + semaphore_name_ = SEMAPHORE_NAME; + if ((semaphore_ = CreateSemaphore(0, 0, 2, SEMAPHORE_NAME)) == NULL) +#else -+ if (pipe(fds_) < 0) -+#endif + if (pipe(fds_) < 0) +- ASSERT_TRUE(false); + #endif + ASSERT_TRUE(false); -+ } -+ -+ void CreatePool(const char* format, bool ignore_jobserver = false) { -+ if (format) { + } + +- void CreatePool(const char *format, bool ignore_jobserver) { +-#ifndef _WIN32 ++ void CreatePool(const char *format, bool ignore_jobserver = false) { + if (format) { +- sprintf(buf_, format, fds_[0], fds_[1]); + sprintf(buf_, format, +#ifdef _WIN32 + semaphore_name_ @@ -2084,73 +3511,69 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + fds_[0], fds_[1] +#endif + ); -+ ENVIRONMENT_INIT(buf_); -+ } + ENVIRONMENT_INIT(buf_); + } +-#endif +- tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_); + if ((tokens_ = TokenPool::Get()) != NULL) { + if (!tokens_->Setup(ignore_jobserver, false, load_avg_)) { + delete tokens_; + tokens_ = NULL; + } + } -+ } -+ -+ void CreateDefaultPool() { + } + + void CreateDefaultPool() { +- CreatePool("foo --jobserver-auth=%d,%d bar", false); + CreatePool(AUTH_FORMAT("--jobserver-auth")); -+ } -+ -+ virtual void TearDown() { -+ if (tokens_) -+ delete tokens_; + } + + virtual void TearDown() { + if (tokens_) + delete tokens_; +-#ifndef _WIN32 +#ifdef _WIN32 + CloseHandle(semaphore_); +#else -+ close(fds_[0]); -+ close(fds_[1]); -+#endif + close(fds_[0]); + close(fds_[1]); +- ENVIRONMENT_CLEAR(); + #endif + ENVIRONMENT_CLEAR(); -+ } -+}; -+ -+} // anonymous namespace -+ -+// verifies none implementation -+TEST_F(TokenPoolTest, NoTokenPool) { -+ CreatePool(NULL, false); -+ -+ EXPECT_EQ(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, SuccessfulOldSetup) { -+ // GNUmake <= 4.1 + } + }; + +@@ -82,10 +111,9 @@ TEST_F(TokenPoolTest, NoTokenPool) { + EXPECT_EQ(kLoadAverageDefault, load_avg_); + } + +-#ifndef _WIN32 + TEST_F(TokenPoolTest, SuccessfulOldSetup) { + // GNUmake <= 4.1 +- CreatePool("foo --jobserver-fds=%d,%d bar", false); + CreatePool(AUTH_FORMAT("--jobserver-fds")); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, SuccessfulNewSetup) { -+ // GNUmake => 4.2 -+ CreateDefaultPool(); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, IgnoreWithJN) { + + EXPECT_NE(NULL, tokens_); + EXPECT_EQ(kLoadAverageDefault, load_avg_); +@@ -100,19 +128,37 @@ TEST_F(TokenPoolTest, SuccessfulNewSetup) { + } + + TEST_F(TokenPoolTest, IgnoreWithJN) { +- CreatePool("foo --jobserver-auth=%d,%d bar", true); + CreatePool(AUTH_FORMAT("--jobserver-auth"), true); -+ -+ EXPECT_EQ(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, HonorLN) { + + EXPECT_EQ(NULL, tokens_); + EXPECT_EQ(kLoadAverageDefault, load_avg_); + } + + TEST_F(TokenPoolTest, HonorLN) { +- CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false); + CreatePool(AUTH_FORMAT("-l9 --jobserver-auth")); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(9.0, load_avg_); -+} -+ + + EXPECT_NE(NULL, tokens_); + EXPECT_EQ(9.0, load_avg_); + } + +#ifdef _WIN32 +TEST_F(TokenPoolTest, SemaphoreNotFound) { + semaphore_name_ = SEMAPHORE_NAME "_foobar"; @@ -2169,101 +3592,66 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + EXPECT_TRUE(tokens_->TokenIsAvailable((ULONG_PTR)tokens_)); +} +#else -+TEST_F(TokenPoolTest, MonitorFD) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); -+} + TEST_F(TokenPoolTest, MonitorFD) { + CreateDefaultPool(); + +@@ -121,6 +167,7 @@ TEST_F(TokenPoolTest, MonitorFD) { + + EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); + } +#endif -+ -+TEST_F(TokenPoolTest, ImplicitToken) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+} -+ -+TEST_F(TokenPoolTest, TwoTokens) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // jobserver offers 2nd token + + TEST_F(TokenPoolTest, ImplicitToken) { + CreateDefaultPool(); +@@ -147,7 +194,13 @@ TEST_F(TokenPoolTest, TwoTokens) { + EXPECT_FALSE(tokens_->Acquire()); + + // jobserver offers 2nd token +#ifdef _WIN32 + LONG previous; + ASSERT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); + ASSERT_EQ(0, previous); +#else -+ ASSERT_EQ(1u, write(fds_[1], "T", 1)); + ASSERT_EQ(1u, write(fds_[1], "T", 1)); +#endif -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // release 2nd token -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ -+ // release implict token - must return 2nd token back to jobserver -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ + EXPECT_TRUE(tokens_->Acquire()); + tokens_->Reserve(); + EXPECT_FALSE(tokens_->Acquire()); +@@ -160,8 +213,14 @@ TEST_F(TokenPoolTest, TwoTokens) { + tokens_->Release(); + EXPECT_TRUE(tokens_->Acquire()); + +- // there must be one token in the pipe + // there must be one token available +#ifdef _WIN32 + EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); + EXPECT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); + EXPECT_EQ(0, previous); +#else -+ EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); + EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); +#endif -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+} -+ -+TEST_F(TokenPoolTest, Clear) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // jobserver offers 2nd & 3rd token + + // implicit token + EXPECT_TRUE(tokens_->Acquire()); +@@ -179,7 +238,13 @@ TEST_F(TokenPoolTest, Clear) { + EXPECT_FALSE(tokens_->Acquire()); + + // jobserver offers 2nd & 3rd token +#ifdef _WIN32 + LONG previous; + ASSERT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); + ASSERT_EQ(0, previous); +#else -+ ASSERT_EQ(2u, write(fds_[1], "TT", 2)); + ASSERT_EQ(2u, write(fds_[1], "TT", 2)); +#endif -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ tokens_->Clear(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ + EXPECT_TRUE(tokens_->Acquire()); + tokens_->Reserve(); + EXPECT_TRUE(tokens_->Acquire()); +@@ -189,10 +254,16 @@ TEST_F(TokenPoolTest, Clear) { + tokens_->Clear(); + EXPECT_TRUE(tokens_->Acquire()); + +- // there must be two tokens in the pipe + // there must be two tokens available +#ifdef _WIN32 + EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); @@ -2271,9 +3659,632 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + EXPECT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); + EXPECT_EQ(0, previous); +#else -+ EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); + EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); +#endif + + // implicit token + EXPECT_TRUE(tokens_->Acquire()); + } +-#endif + +From 2b9c81c0ec1226d8795e7725529f13be41eaa385 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Fri, 14 Dec 2018 13:27:11 +0200 +Subject: [PATCH 11/11] Prepare PR for merging - part II + +- remove unnecessary "struct" from TokenPool +- add PAPCFUNC cast to QueryUserAPC() +- remove hard-coded MAKEFLAGS string from win32 +- remove useless build test CompleteNoWork +- rename TokenPoolTest to TestTokenPool +- add tokenpool modules to CMake build +- remove unused no-op TokenPool implementation +- fix errors flagged by codespell & clang-tidy +- address review comments from + +https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-195195803 +https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-185089255 +https://github.com/ninja-build/ninja/pull/1140#issuecomment-473898963 +https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +--- + CMakeLists.txt | 8 ++++- + src/build.cc | 2 +- + src/build_test.cc | 12 +------ + src/subprocess-posix.cc | 4 +-- + src/subprocess-win32.cc | 2 +- + src/subprocess.h | 2 +- + src/subprocess_test.cc | 26 +++++++------- + src/tokenpool-gnu-make-posix.cc | 21 +++++------ + src/tokenpool-gnu-make-win32.cc | 36 ++++++++++--------- + src/tokenpool-gnu-make.cc | 63 +++++++++++++++++---------------- + src/tokenpool-gnu-make.h | 6 ++-- + src/tokenpool-none.cc | 22 ------------ + src/tokenpool.h | 2 +- + src/tokenpool_test.cc | 8 ++--- + 14 files changed, 94 insertions(+), 120 deletions(-) + delete mode 100644 src/tokenpool-none.cc + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 57ae548f5b..e2876fe413 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -112,6 +112,7 @@ add_library(libninja OBJECT + src/state.cc + src/status.cc + src/string_piece_util.cc ++ src/tokenpool-gnu-make.cc + src/util.cc + src/version.cc + ) +@@ -123,9 +124,13 @@ if(WIN32) + src/msvc_helper_main-win32.cc + src/getopt.c + src/minidump-win32.cc ++ src/tokenpool-gnu-make-win32.cc + ) + else() +- target_sources(libninja PRIVATE src/subprocess-posix.cc) ++ target_sources(libninja PRIVATE ++ src/subprocess-posix.cc ++ src/tokenpool-gnu-make-posix.cc ++ ) + if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") + target_sources(libninja PRIVATE src/getopt.c) + endif() +@@ -204,6 +209,7 @@ if(BUILD_TESTING) + src/string_piece_util_test.cc + src/subprocess_test.cc + src/test.cc ++ src/tokenpool_test.cc + src/util_test.cc + ) + if(WIN32) +diff --git a/src/build.cc b/src/build.cc +index 20c3bdc2a0..854df08c2a 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -467,7 +467,7 @@ struct RealCommandRunner : public CommandRunner { + // copy of config_.max_load_average; can be modified by TokenPool setup + double max_load_average_; + SubprocessSet subprocs_; +- TokenPool *tokens_; ++ TokenPool* tokens_; + map subproc_to_edge_; + }; + +diff --git a/src/build_test.cc b/src/build_test.cc +index dd41dfbe1d..8901c9518f 100644 +--- a/src/build_test.cc ++++ b/src/build_test.cc +@@ -4098,7 +4098,7 @@ struct BuildTokenTest : public BuildTest { + void ExpectWaitForCommand(int count, ...); + + private: +- void EnqueueBooleans(vector& booleans, int count, va_list ao); ++ void EnqueueBooleans(vector& booleans, int count, va_list ap); + }; + + void BuildTokenTest::SetUp() { +@@ -4144,16 +4144,6 @@ void BuildTokenTest::EnqueueBooleans(vector& booleans, int count, va_list + } + } + +-TEST_F(BuildTokenTest, CompleteNoWork) { +- // plan should not execute anything +- string err; +- +- EXPECT_TRUE(builder_.Build(&err)); +- EXPECT_EQ("", err); +- +- EXPECT_EQ(0u, token_command_runner_.commands_ran_.size()); +-} +- + TEST_F(BuildTokenTest, DoNotAquireToken) { + // plan should execute one command + string err; +diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc +index 74451b0be2..31839276c4 100644 +--- a/src/subprocess-posix.cc ++++ b/src/subprocess-posix.cc +@@ -250,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + } + + #ifdef USE_PPOLL +-bool SubprocessSet::DoWork(struct TokenPool* tokens) { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + vector fds; + nfds_t nfds = 0; + +@@ -315,7 +315,7 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { + } + + #else // !defined(USE_PPOLL) +-bool SubprocessSet::DoWork(struct TokenPool* tokens) { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + fd_set set; + int nfds = 0; + FD_ZERO(&set); +diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc +index ce3e2c20a4..2926e9a221 100644 +--- a/src/subprocess-win32.cc ++++ b/src/subprocess-win32.cc +@@ -252,7 +252,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + return subprocess; + } + +-bool SubprocessSet::DoWork(struct TokenPool* tokens) { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + DWORD bytes_read; + Subprocess* subproc; + OVERLAPPED* overlapped; +diff --git a/src/subprocess.h b/src/subprocess.h +index 9ea67ea477..1ec78171e8 100644 +--- a/src/subprocess.h ++++ b/src/subprocess.h +@@ -86,7 +86,7 @@ struct SubprocessSet { + ~SubprocessSet(); + + Subprocess* Add(const std::string& command, bool use_console = false); +- bool DoWork(struct TokenPool* tokens); ++ bool DoWork(TokenPool* tokens); + Subprocess* NextFinished(); + void Clear(); + +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index f625963462..7b146f31be 100644 +--- a/src/subprocess_test.cc ++++ b/src/subprocess_test.cc +@@ -35,7 +35,7 @@ const char* kSimpleCommand = "cmd /c dir \\"; + const char* kSimpleCommand = "ls /"; + #endif + +-struct TokenPoolTest : public TokenPool { ++struct TestTokenPool : public TokenPool { + bool Acquire() { return false; } + void Reserve() {} + void Release() {} +@@ -58,7 +58,7 @@ struct TokenPoolTest : public TokenPool { + + struct SubprocessTest : public testing::Test { + SubprocessSet subprocs_; +- TokenPoolTest tokens_; ++ TestTokenPool tokens_; + }; + + } // anonymous namespace +@@ -73,7 +73,7 @@ TEST_F(SubprocessTest, BadCommandStderr) { + // Pretend we discovered that stderr was ready for writing. + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -89,7 +89,7 @@ TEST_F(SubprocessTest, NoSuchCommand) { + // Pretend we discovered that stderr was ready for writing. + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -109,7 +109,7 @@ TEST_F(SubprocessTest, InterruptChild) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -135,7 +135,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -161,7 +161,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -190,7 +190,7 @@ TEST_F(SubprocessTest, Console) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } +@@ -206,7 +206,7 @@ TEST_F(SubprocessTest, SetWithSingle) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_NE("", subproc->GetOutput()); + +@@ -243,7 +243,7 @@ TEST_F(SubprocessTest, SetWithMulti) { + ASSERT_GT(subprocs_.running_.size(), 0u); + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(0u, subprocs_.running_.size()); + ASSERT_EQ(3u, subprocs_.finished_.size()); + +@@ -278,7 +278,7 @@ TEST_F(SubprocessTest, SetWithLots) { + subprocs_.ResetTokenAvailable(); + while (!subprocs_.running_.empty()) + subprocs_.DoWork(NULL); +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + for (size_t i = 0; i < procs.size(); ++i) { + ASSERT_EQ(ExitSuccess, procs[i]->Finish()); + ASSERT_NE("", procs[i]->GetOutput()); +@@ -298,7 +298,7 @@ TEST_F(SubprocessTest, ReadStdin) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_EQ(1u, subprocs_.finished_.size()); + } +@@ -322,7 +322,7 @@ TEST_F(SubprocessTest, TokenAvailable) { + subprocs_.DoWork(&tokens_); + #ifdef _WIN32 + tokens_._token_available = false; +- // we need to loop here as we have no conrol where the token ++ // we need to loop here as we have no control where the token + // I/O completion post ends up in the queue + while (!subproc->Done() && !subprocs_.IsTokenAvailable()) { + subprocs_.DoWork(&tokens_); +diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc +index 70d84bfff7..353bda226a 100644 +--- a/src/tokenpool-gnu-make-posix.cc ++++ b/src/tokenpool-gnu-make-posix.cc +@@ -32,8 +32,8 @@ struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool { + + virtual int GetMonitorFd(); + +- virtual const char *GetEnv(const char *name) { return getenv(name); }; +- virtual bool ParseAuth(const char *jobserver); ++ virtual const char* GetEnv(const char* name) { return getenv(name); }; ++ virtual bool ParseAuth(const char* jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + +@@ -64,9 +64,7 @@ bool GNUmakeTokenPoolPosix::CheckFd(int fd) { + if (fd < 0) + return false; + int ret = fcntl(fd, F_GETFD); +- if (ret < 0) +- return false; +- return true; ++ return ret >= 0; + } + + int GNUmakeTokenPoolPosix::dup_rfd_ = -1; +@@ -82,14 +80,13 @@ bool GNUmakeTokenPoolPosix::SetAlarmHandler() { + act.sa_handler = CloseDupRfd; + if (sigaction(SIGALRM, &act, &old_act_) < 0) { + perror("sigaction:"); +- return(false); +- } else { +- restore_ = true; +- return(true); ++ return false; + } ++ restore_ = true; ++ return true; + } + +-bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) { ++bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) { + int rfd = -1; + int wfd = -1; + if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && +@@ -195,9 +192,9 @@ bool GNUmakeTokenPoolPosix::ReturnToken() { + } + + int GNUmakeTokenPoolPosix::GetMonitorFd() { +- return(rfd_); ++ return rfd_; + } + +-struct TokenPool *TokenPool::Get() { ++TokenPool* TokenPool::Get() { + return new GNUmakeTokenPoolPosix; + } +diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc +index 2719f2c1fc..b2bb52fadb 100644 +--- a/src/tokenpool-gnu-make-win32.cc ++++ b/src/tokenpool-gnu-make-win32.cc +@@ -14,7 +14,8 @@ + + #include "tokenpool-gnu-make.h" + +-// always include first to make sure other headers do the correct thing... ++// Always include this first. ++// Otherwise the other system headers don't work correctly under Win32 + #include + + #include +@@ -32,8 +33,8 @@ struct GNUmakeTokenPoolWin32 : public GNUmakeTokenPool { + virtual void WaitForTokenAvailability(HANDLE ioport); + virtual bool TokenIsAvailable(ULONG_PTR key); + +- virtual const char *GetEnv(const char *name); +- virtual bool ParseAuth(const char *jobserver); ++ virtual const char* GetEnv(const char* name); ++ virtual bool ParseAuth(const char* jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + +@@ -98,19 +99,19 @@ GNUmakeTokenPoolWin32::~GNUmakeTokenPoolWin32() { + } + } + +-const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) { ++const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) { + // getenv() does not work correctly together with tokenpool_tests.cc + static char buffer[MAX_PATH + 1]; +- if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0) ++ if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0) + return NULL; +- return(buffer); ++ return buffer; + } + +-bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { ++bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) { + // match "--jobserver-auth=gmake_semaphore_..." +- const char *start = strchr(jobserver, '='); ++ const char* start = strchr(jobserver, '='); + if (start) { +- const char *end = start; ++ const char* end = start; + unsigned int len; + char c, *auth; + +@@ -119,14 +120,15 @@ bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { + break; + len = end - start; // includes string terminator in count + +- if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) { ++ if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) { + strncpy(auth, start + 1, len - 1); + auth[len - 1] = '\0'; + +- if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ +- FALSE, /* Child processes DON'T inherit */ +- auth /* Semaphore name */ +- )) != NULL) { ++ if ((semaphore_jobserver_ = ++ OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ ++ FALSE, /* Child processes DON'T inherit */ ++ auth /* Semaphore name */ ++ )) != NULL) { + free(auth); + return true; + } +@@ -171,7 +173,7 @@ DWORD GNUmakeTokenPoolWin32::SemaphoreThread() { + } + + DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) { +- GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param; ++ GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param; + return This->SemaphoreThread(); + } + +@@ -216,7 +218,7 @@ void GNUmakeTokenPoolWin32::WaitForTokenAvailability(HANDLE ioport) { + + bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) { + // alert child thread to break wait on token semaphore +- QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL); ++ QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL); + + // return true when GetQueuedCompletionStatus() returned our key + return key == (ULONG_PTR) this; +@@ -232,6 +234,6 @@ void GNUmakeTokenPoolWin32::WaitForObject(HANDLE object) { + Win32Fatal("WaitForSingleObject"); + } + +-struct TokenPool *TokenPool::Get() { ++TokenPool* TokenPool::Get() { + return new GNUmakeTokenPoolWin32; + } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index 92ff611721..60e0552924 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -31,36 +31,37 @@ GNUmakeTokenPool::~GNUmakeTokenPool() { + bool GNUmakeTokenPool::Setup(bool ignore, + bool verbose, + double& max_load_average) { +- const char *value = GetEnv("MAKEFLAGS"); +- if (value) { +- // GNU make <= 4.1 +- const char *jobserver = strstr(value, "--jobserver-fds="); ++ const char* value = GetEnv("MAKEFLAGS"); ++ if (!value) ++ return false; + -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+} ++ // GNU make <= 4.1 ++ const char* jobserver = strstr(value, "--jobserver-fds="); ++ if (!jobserver) + // GNU make => 4.2 +- if (!jobserver) +- jobserver = strstr(value, "--jobserver-auth="); +- if (jobserver) { +- LinePrinter printer; +- +- if (ignore) { +- printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); +- } else { +- if (ParseAuth(jobserver)) { +- const char *l_arg = strstr(value, " -l"); +- int load_limit = -1; +- +- if (verbose) { +- printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); +- } +- +- // translate GNU make -lN to ninja -lN +- if (l_arg && +- (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && +- (load_limit > 0)) { +- max_load_average = load_limit; +- } +- +- return true; ++ jobserver = strstr(value, "--jobserver-auth="); ++ if (jobserver) { ++ LinePrinter printer; ++ ++ if (ignore) { ++ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ } else { ++ if (ParseAuth(jobserver)) { ++ const char* l_arg = strstr(value, " -l"); ++ int load_limit = -1; ++ ++ if (verbose) { ++ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); ++ } ++ ++ // translate GNU make -lN to ninja -lN ++ if (l_arg && ++ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && ++ (load_limit > 0)) { ++ max_load_average = load_limit; + } ++ ++ return true; + } + } + } +@@ -76,10 +77,10 @@ bool GNUmakeTokenPool::Acquire() { + // token acquired + available_++; + return true; +- } else { +- // no token available +- return false; + } ++ ++ // no token available ++ return false; + } + + void GNUmakeTokenPool::Reserve() { +diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h +index d3852088e2..c94cca5e2d 100644 +--- a/src/tokenpool-gnu-make.h ++++ b/src/tokenpool-gnu-make.h +@@ -17,7 +17,7 @@ + // interface to GNU make token pool + struct GNUmakeTokenPool : public TokenPool { + GNUmakeTokenPool(); +- virtual ~GNUmakeTokenPool(); ++ ~GNUmakeTokenPool(); + + // token pool implementation + virtual bool Acquire(); +@@ -27,8 +27,8 @@ struct GNUmakeTokenPool : public TokenPool { + virtual bool Setup(bool ignore, bool verbose, double& max_load_average); + + // platform specific implementation +- virtual const char *GetEnv(const char *name) = 0; +- virtual bool ParseAuth(const char *jobserver) = 0; ++ virtual const char* GetEnv(const char* name) = 0; ++ virtual bool ParseAuth(const char* jobserver) = 0; + virtual bool AcquireToken() = 0; + virtual bool ReturnToken() = 0; + +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +deleted file mode 100644 +index 613d16882d..0000000000 +--- a/src/tokenpool-none.cc ++++ /dev/null +@@ -1,22 +0,0 @@ +-// Copyright 2016-2018 Google Inc. All Rights Reserved. +-// +-// Licensed under the Apache License, Version 2.0 (the "License"); +-// you may not use this file except in compliance with the License. +-// You may obtain a copy of the License at +-// +-// http://www.apache.org/licenses/LICENSE-2.0 +-// +-// Unless required by applicable law or agreed to in writing, software +-// distributed under the License is distributed on an "AS IS" BASIS, +-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-// See the License for the specific language governing permissions and +-// limitations under the License. +- +-#include "tokenpool.h" +- +-#include +- +-// No-op TokenPool implementation +-struct TokenPool *TokenPool::Get() { +- return NULL; +-} +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 1be8e1d5ce..931c22754d 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -38,5 +38,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(); ++ static TokenPool* Get(); + }; +diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc +index 8d4fd7d33a..8d3061cb30 100644 +--- a/src/tokenpool_test.cc ++++ b/src/tokenpool_test.cc +@@ -43,10 +43,10 @@ const double kLoadAverageDefault = -1.23456789; + + struct TokenPoolTest : public testing::Test { + double load_avg_; +- TokenPool *tokens_; ++ TokenPool* tokens_; + char buf_[1024]; + #ifdef _WIN32 +- const char *semaphore_name_; ++ const char* semaphore_name_; + HANDLE semaphore_; + #else + int fds_[2]; +@@ -65,7 +65,7 @@ struct TokenPoolTest : public testing::Test { + ASSERT_TRUE(false); + } + +- void CreatePool(const char *format, bool ignore_jobserver = false) { ++ void CreatePool(const char* format, bool ignore_jobserver = false) { + if (format) { + sprintf(buf_, format, + #ifdef _WIN32 +@@ -209,7 +209,7 @@ TEST_F(TokenPoolTest, TwoTokens) { + tokens_->Release(); + EXPECT_TRUE(tokens_->Acquire()); + +- // release implict token - must return 2nd token back to jobserver ++ // release implicit token - must return 2nd token back to jobserver + tokens_->Release(); + EXPECT_TRUE(tokens_->Acquire()); + From bd0ccd37b9da09ee4d284f32aa361dd00292b883 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Mon, 25 Apr 2022 09:09:23 -0300 Subject: [PATCH 14/31] wolfssl: don't change ABI because of hw crypto Enabling different hardware crypto acceleration should not change the library ABI. Add them to PKG_CONFIG_DEPENDS after the ABI version hash has been computed. Signed-off-by: Eneas U de Queiroz (cherry picked from commit 677774d445ced1a56e73fe62df47b4eb66441721) --- package/libs/wolfssl/Makefile | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/package/libs/wolfssl/Makefile b/package/libs/wolfssl/Makefile index 472a25cf6f..46bf52795b 100644 --- a/package/libs/wolfssl/Makefile +++ b/package/libs/wolfssl/Makefile @@ -15,7 +15,7 @@ PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://github.com/wolfSSL/wolfssl/archive/v$(PKG_VERSION) PKG_HASH:=1a3bb310dc01d3e73d9ad91b6ea8249d081016f8eef4ae8f21d3421f91ef1de9 -PKG_FIXUP:=libtool +PKG_FIXUP:=libtool libtool-abiver PKG_INSTALL:=1 PKG_USE_MIPS16:=0 PKG_BUILD_PARALLEL:=1 @@ -25,14 +25,27 @@ PKG_MAINTAINER:=Eneas U de Queiroz PKG_CPE_ID:=cpe:/a:wolfssl:wolfssl PKG_CONFIG_DEPENDS:=\ - CONFIG_WOLFSSL_HAS_AES_CCM CONFIG_WOLFSSL_HAS_AFALG \ - CONFIG_WOLFSSL_HAS_ARC4 CONFIG_WOLFSSL_HAS_CHACHA_POLY \ - CONFIG_WOLFSSL_HAS_DEVCRYPTO_AES CONFIG_WOLFSSL_HAS_DEVCRYPTO_FULL \ - CONFIG_WOLFSSL_HAS_DH CONFIG_WOLFSSL_HAS_DTLS \ - CONFIG_WOLFSSL_HAS_ECC25519 CONFIG_WOLFSSL_HAS_OCSP \ - CONFIG_WOLFSSL_HAS_SESSION_TICKET CONFIG_WOLFSSL_HAS_TLSV10 \ - CONFIG_WOLFSSL_HAS_TLSV13 CONFIG_WOLFSSL_HAS_WPAS CONFIG_WOLFSSL_HAS_CERTGEN \ - CONFIG_WOLFSSL_HAS_OPENVPN CONFIG_WOLFSSL_ALT_NAMES + CONFIG_WOLFSSL_HAS_AES_CCM \ + CONFIG_WOLFSSL_HAS_ARC4 \ + CONFIG_WOLFSSL_HAS_CERTGEN \ + CONFIG_WOLFSSL_HAS_CHACHA_POLY \ + CONFIG_WOLFSSL_HAS_DH \ + CONFIG_WOLFSSL_HAS_DTLS \ + CONFIG_WOLFSSL_HAS_ECC25519 \ + CONFIG_WOLFSSL_HAS_OCSP \ + CONFIG_WOLFSSL_HAS_OPENVPN CONFIG_WOLFSSL_ALT_NAMES \ + CONFIG_WOLFSSL_HAS_SESSION_TICKET \ + CONFIG_WOLFSSL_HAS_TLSV10 \ + CONFIG_WOLFSSL_HAS_TLSV13 \ + CONFIG_WOLFSSL_HAS_WPAS + +PKG_ABI_VERSION:=$(patsubst %-stable,%,$(PKG_VERSION)).$(call version_abbrev,$(call confvar,$(PKG_CONFIG_DEPENDS))) + +PKG_CONFIG_DEPENDS+=\ + CONFIG_WOLFSSL_HAS_AFALG \ + CONFIG_WOLFSSL_HAS_DEVCRYPTO_AES \ + CONFIG_WOLFSSL_HAS_DEVCRYPTO_CBC \ + CONFIG_WOLFSSL_HAS_DEVCRYPTO_FULL include $(INCLUDE_DIR)/package.mk @@ -45,7 +58,7 @@ define Package/libwolfssl MENU:=1 PROVIDES:=libcyassl DEPENDS:=+WOLFSSL_HAS_DEVCRYPTO:kmod-cryptodev +WOLFSSL_HAS_AFALG:kmod-crypto-user - ABI_VERSION:=24 + ABI_VERSION:=$(PKG_ABI_VERSION) endef define Package/libwolfssl/description From ff489166f01e0e8aa0c3566dbd3be9d857e51a48 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Tue, 19 Apr 2022 18:23:05 -0300 Subject: [PATCH 15/31] wolfssl: add benchmark utility This packages the wolfssl benchmark utility. Signed-off-by: Eneas U de Queiroz (cherry picked from commit 18fd12edb810f9dfbf8410bb81f639df052134cb) --- package/libs/wolfssl/Makefile | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/package/libs/wolfssl/Makefile b/package/libs/wolfssl/Makefile index 46bf52795b..9283737542 100644 --- a/package/libs/wolfssl/Makefile +++ b/package/libs/wolfssl/Makefile @@ -49,12 +49,16 @@ PKG_CONFIG_DEPENDS+=\ include $(INCLUDE_DIR)/package.mk -define Package/libwolfssl +define Package/libwolfssl/Default SECTION:=libs SUBMENU:=SSL CATEGORY:=Libraries - TITLE:=wolfSSL library URL:=http://www.wolfssl.com/ +endef + +define Package/libwolfssl +$(call Package/libwolfssl/Default) + TITLE:=wolfSSL library MENU:=1 PROVIDES:=libcyassl DEPENDS:=+WOLFSSL_HAS_DEVCRYPTO:kmod-cryptodev +WOLFSSL_HAS_AFALG:kmod-crypto-user @@ -70,6 +74,16 @@ define Package/libwolfssl/config source "$(SOURCE)/Config.in" endef +define Package/libwolfssl-benchmark +$(call Package/libwolfssl/Default) + TITLE:=wolfSSL Benchmark Utility + DEPENDS:=libwolfssl +endef + +define Package/libwolfssl-benchmark/description +This is the wolfssl benchmark utility. +endef + TARGET_CFLAGS += \ $(FPIC) \ -fomit-frame-pointer \ @@ -88,7 +102,7 @@ CONFIGURE_ARGS += \ --enable-sni \ --enable-stunnel \ --enable-altcertchains \ - --disable-crypttests \ + --$(if $(CONFIG_PACKAGE_libwolfssl-benchmark),enable,disable)-crypttests \ --disable-examples \ --disable-jobserver \ --$(if $(CONFIG_IPV6),enable,disable)-ipv6 \ @@ -136,4 +150,10 @@ define Package/libwolfssl/install $(CP) $(PKG_INSTALL_DIR)/usr/lib/libwolfssl.so.* $(1)/usr/lib/ endef +define Package/libwolfssl-benchmark/install + $(INSTALL_DIR) $(1)/usr/bin + $(CP) $(PKG_BUILD_DIR)/wolfcrypt/benchmark/.libs/benchmark $(1)/usr/bin/wolfssl-benchmark +endef + $(eval $(call BuildPackage,libwolfssl)) +$(eval $(call BuildPackage,libwolfssl-benchmark)) From 2ea2702e5e2fc58b4be2d96c4962155f241440f7 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Tue, 19 Apr 2022 12:02:09 -0300 Subject: [PATCH 16/31] wolfssl: enable CPU crypto instructions This enables AES & SHA CPU instructions for compatible armv8, and x86_64 architectures. Add this to the hardware acceleration choice, since they can't be enabled at the same time. The package was marked non-shared, since the arm CPUs may or may not have crypto extensions enabled based on licensing; bcm27xx does not enable them. There is no run-time detection of this for arm. NOTE: Should this be backported to a release branch, it must be done shortly before a new minor release, because the change to nonshared will remove libwolfssl from the shared packages, but the nonshared are only built in a subsequent release! Signed-off-by: Eneas U de Queiroz (cherry picked from commit 0a2edc2714dcda10be902c32525723ce2cbcb138) --- package/libs/wolfssl/Config.in | 12 ++++++++++++ package/libs/wolfssl/Makefile | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/package/libs/wolfssl/Config.in b/package/libs/wolfssl/Config.in index b32d5ab6cb..f495a90ff6 100644 --- a/package/libs/wolfssl/Config.in +++ b/package/libs/wolfssl/Config.in @@ -62,13 +62,25 @@ config WOLFSSL_ALT_NAMES config WOLFSSL_HAS_DEVCRYPTO bool +config WOLFSSL_ASM_CAPABLE + bool + default x86_64 || (aarch64 && !TARGET_bcm27xx) + choice prompt "Hardware Acceleration" + default WOLFSSL_HAS_CPU_CRYPTO if WOLFSSL_ASM_CAPABLE default WOLFSSL_HAS_NO_HW config WOLFSSL_HAS_NO_HW bool "None" + config WOLFSSL_HAS_CPU_CRYPTO + bool "Use CPU crypto instructions" + depends on WOLFSSL_ASM_CAPABLE + help + This will use Intel AESNI insturctions or armv8 Crypto Extensions. + Either of them should easily outperform hardware crypto in WolfSSL. + config WOLFSSL_HAS_AFALG bool "AF_ALG" diff --git a/package/libs/wolfssl/Makefile b/package/libs/wolfssl/Makefile index 9283737542..3edd526364 100644 --- a/package/libs/wolfssl/Makefile +++ b/package/libs/wolfssl/Makefile @@ -43,6 +43,7 @@ PKG_ABI_VERSION:=$(patsubst %-stable,%,$(PKG_VERSION)).$(call version_abbrev,$(c PKG_CONFIG_DEPENDS+=\ CONFIG_WOLFSSL_HAS_AFALG \ + CONFIG_WOLFSSL_HAS_CPU_CRYPTO \ CONFIG_WOLFSSL_HAS_DEVCRYPTO_AES \ CONFIG_WOLFSSL_HAS_DEVCRYPTO_CBC \ CONFIG_WOLFSSL_HAS_DEVCRYPTO_FULL @@ -59,6 +60,7 @@ endef define Package/libwolfssl $(call Package/libwolfssl/Default) TITLE:=wolfSSL library + PKGFLAGS:=nonshared MENU:=1 PROVIDES:=libcyassl DEPENDS:=+WOLFSSL_HAS_DEVCRYPTO:kmod-cryptodev +WOLFSSL_HAS_AFALG:kmod-crypto-user @@ -133,6 +135,15 @@ CONFIGURE_ARGS += \ --enable-wpas --enable-fortress --enable-fastmath endif +ifdef CONFIG_WOLFSSL_HAS_CPU_CRYPTO + ifdef CONFIG_aarch64 + CONFIGURE_ARGS += --enable-armasm + TARGET_CFLAGS:=$(TARGET_CFLAGS:-mcpu%=-mcpu%+crypto) + else ifdef CONFIG_TARGET_x86_64 + CONFIGURE_ARGS += --enable-intelasm + endif +endif + define Build/InstallDev $(INSTALL_DIR) $(1)/usr/include $(1)/usr/lib/pkgconfig $(CP) $(PKG_INSTALL_DIR)/usr/include/* $(1)/usr/include/ From c0dbdac03817636baa258645a1629011013c0d6b Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Tue, 17 May 2022 12:00:41 -0300 Subject: [PATCH 17/31] bcm27xx/bcm2710: enable asm crypto algorithms This enables arm64/neon version of AES, SHA256 and SHA512 algorithms in the kernel. bcm2710 does not support armv8 crypto extensions, so they are not included. Signed-off-by: Eneas U de Queiroz (cherry picked from commit 38ebb210a9f4895cfade3580815d5c9a3eb1b7e4) --- target/linux/bcm27xx/bcm2710/config-4.19 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/target/linux/bcm27xx/bcm2710/config-4.19 b/target/linux/bcm27xx/bcm2710/config-4.19 index 80cd167283..4ba891e859 100644 --- a/target/linux/bcm27xx/bcm2710/config-4.19 +++ b/target/linux/bcm27xx/bcm2710/config-4.19 @@ -74,7 +74,7 @@ CONFIG_ARM64=y CONFIG_ARM64_4K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set CONFIG_ARM64_CONT_SHIFT=4 -# CONFIG_ARM64_CRYPTO is not set +CONFIG_ARM64_CRYPTO=y CONFIG_ARM64_ERRATUM_819472=y CONFIG_ARM64_ERRATUM_824069=y CONFIG_ARM64_ERRATUM_826319=y @@ -186,9 +186,13 @@ CONFIG_CPU_RMAP=y CONFIG_CRC16=y CONFIG_CRYPTO_AEAD=y CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_BS=y +CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_CRC32=y CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_CRYPTD=y CONFIG_CRYPTO_CTR=y CONFIG_CRYPTO_CTS=y CONFIG_CRYPTO_DRBG=y @@ -208,6 +212,10 @@ CONFIG_CRYPTO_RNG2=y CONFIG_CRYPTO_RNG_DEFAULT=y CONFIG_CRYPTO_SEQIV=y CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA256_ARM64=y +CONFIG_CRYPTO_SHA512=y +CONFIG_CRYPTO_SHA512_ARM64=y +CONFIG_CRYPTO_SIMD=y CONFIG_CRYPTO_WORKQUEUE=y CONFIG_CRYPTO_XTS=y CONFIG_DCACHE_WORD_ACCESS=y From 86fc1c53e9fc09b91119f0e04fb7cfffd62d9e12 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Tue, 17 May 2022 12:06:12 -0300 Subject: [PATCH 18/31] bcm27xx/bcm2711: enable asm crypto algorithms This enables arm64/neon version of AES, SHA256 and SHA512 algorithms in the kernel. bcm2711 does not support armv8 crypto extensions, so they are not included. Signed-off-by: Eneas U de Queiroz (cherry picked from commit 7b6beb7489c750c0613153822ec1d5ba8a9ab388) --- target/linux/bcm27xx/bcm2711/config-4.19 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/target/linux/bcm27xx/bcm2711/config-4.19 b/target/linux/bcm27xx/bcm2711/config-4.19 index 6aa897a2e1..491fd70c70 100644 --- a/target/linux/bcm27xx/bcm2711/config-4.19 +++ b/target/linux/bcm27xx/bcm2711/config-4.19 @@ -74,7 +74,7 @@ CONFIG_ARM64=y CONFIG_ARM64_4K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set CONFIG_ARM64_CONT_SHIFT=4 -# CONFIG_ARM64_CRYPTO is not set +CONFIG_ARM64_CRYPTO=y CONFIG_ARM64_ERRATUM_819472=y CONFIG_ARM64_ERRATUM_824069=y CONFIG_ARM64_ERRATUM_826319=y @@ -190,9 +190,13 @@ CONFIG_CPU_RMAP=y CONFIG_CRC16=y CONFIG_CRYPTO_AEAD=y CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_BS=y +CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_CRC32=y CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_CRYPTD=y CONFIG_CRYPTO_CTR=y CONFIG_CRYPTO_CTS=y CONFIG_CRYPTO_DRBG=y @@ -212,6 +216,10 @@ CONFIG_CRYPTO_RNG2=y CONFIG_CRYPTO_RNG_DEFAULT=y CONFIG_CRYPTO_SEQIV=y CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA256_ARM64=y +CONFIG_CRYPTO_SHA512=y +CONFIG_CRYPTO_SHA512_ARM64=y +CONFIG_CRYPTO_SIMD=y CONFIG_CRYPTO_WORKQUEUE=y CONFIG_CRYPTO_XTS=y CONFIG_DCACHE_WORD_ACCESS=y From 18dade892048a84804eefbbefa32293c78feb3a9 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Wed, 20 Apr 2022 15:26:32 -0300 Subject: [PATCH 19/31] layerscape/armv8_64b: enable armv8-CE crypto algos This enables armv8 crypto extensions version of AES, GHASH, SHA256 and CRC T10 algorithms in the kernel. Signed-off-by: Eneas U de Queiroz (cherry picked from commit eb33232420ea2537d8302d5ec121eed03db474d1) --- target/linux/layerscape/armv8_64b/config-4.14 | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/target/linux/layerscape/armv8_64b/config-4.14 b/target/linux/layerscape/armv8_64b/config-4.14 index c981c0119e..6c8b40058a 100644 --- a/target/linux/layerscape/armv8_64b/config-4.14 +++ b/target/linux/layerscape/armv8_64b/config-4.14 @@ -45,8 +45,7 @@ CONFIG_ARM64=y # CONFIG_ARM64_16K_PAGES is not set CONFIG_ARM64_4K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set -CONFIG_ARM64_CONT_SHIFT=4 -# CONFIG_ARM64_CRYPTO is not set +CONFIG_ARM64_CRYPTO=y CONFIG_ARM64_ERRATUM_819472=y CONFIG_ARM64_ERRATUM_824069=y CONFIG_ARM64_ERRATUM_826319=y @@ -184,20 +183,57 @@ CONFIG_CROSS_MEMORY_ATTACH=y CONFIG_CRYPTO_ACOMP2=y CONFIG_CRYPTO_AEAD=y CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_CE=y +CONFIG_CRYPTO_AES_ARM64_CE_BLK=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_BLAKE2B=y +CONFIG_CRYPTO_CRC32=y CONFIG_CRYPTO_CRC32C=y CONFIG_CRYPTO_CRCT10DIF=y +CONFIG_CRYPTO_CRCT10DIF_ARM64_CE=y +CONFIG_CRYPTO_CRYPTD=y CONFIG_CRYPTO_DEFLATE=y -# CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM is not set +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_DEV_FSL_CAAM=y +CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API=y +CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC=y +CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON=y +CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API=y +CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC=y +CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI=y +# CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG is not set +# CONFIG_CRYPTO_DEV_FSL_CAAM_INTC is not set +CONFIG_CRYPTO_DEV_FSL_CAAM_JR=y +CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API=y +CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE=9 +CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API=y +CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=y +CONFIG_CRYPTO_ECB=y +CONFIG_CRYPTO_ENGINE=y +CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_GF128MUL=y CONFIG_CRYPTO_HASH=y CONFIG_CRYPTO_HASH2=y CONFIG_CRYPTO_HW=y +CONFIG_CRYPTO_LIB_DES=y +CONFIG_CRYPTO_LIB_SHA256=y CONFIG_CRYPTO_LZO=y CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_MANAGER2=y CONFIG_CRYPTO_NULL2=y +CONFIG_CRYPTO_RNG=y CONFIG_CRYPTO_RNG2=y -# CONFIG_CRYPTO_TLS is not set +CONFIG_CRYPTO_RSA=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA256_ARM64=y +CONFIG_CRYPTO_SHA2_ARM64_CE=y +CONFIG_CRYPTO_SIMD=y CONFIG_CRYPTO_WORKQUEUE=y +CONFIG_CRYPTO_XTS=y +CONFIG_CRYPTO_XXHASH=y +CONFIG_CRYPTO_ZSTD=y CONFIG_DCACHE_WORD_ACCESS=y CONFIG_DEBUG_BUGVERBOSE=y # CONFIG_DEBUG_EFI is not set From 9908f7a9e93380f713790755289270d5454dccf3 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Wed, 20 Apr 2022 15:26:32 -0300 Subject: [PATCH 20/31] mvebu/cortexa53: enable armv8-CE crypto algos This enables armv8 crypto extensions version of AES, GHASH, SHA1, SHA256, and SHA512 algorithms in the kernel. The choice of algorithms match the 32-bit versions that are enabled in the target config-5.10 file, but were only used by the cortexa9 subtarget. Signed-off-by: Eneas U de Queiroz (cherry picked from commit f5167e11bf7e0a1a3675f0563423254005d0eb2d) (cherry picked from commit ac328e7526e746daefc31ca67090b6d9b3a56ff6) --- target/linux/mvebu/cortexa53/config-4.19 | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/target/linux/mvebu/cortexa53/config-4.19 b/target/linux/mvebu/cortexa53/config-4.19 index 7f3c2b21de..c46f1b22b7 100644 --- a/target/linux/mvebu/cortexa53/config-4.19 +++ b/target/linux/mvebu/cortexa53/config-4.19 @@ -54,7 +54,7 @@ CONFIG_ARM64=y CONFIG_ARM64_4K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set CONFIG_ARM64_CONT_SHIFT=4 -# CONFIG_ARM64_CRYPTO is not set +CONFIG_ARM64_CRYPTO=y # CONFIG_ARM64_HW_AFDBM is not set # CONFIG_ARM64_LSE_ATOMICS is not set CONFIG_ARM64_PAGE_SHIFT=12 @@ -86,6 +86,17 @@ CONFIG_ARM_GIC_V3_ITS_PCI=y CONFIG_ARM_PSCI_FW=y # CONFIG_ARM_SP805_WATCHDOG is not set CONFIG_AUDIT_ARCH_COMPAT_GENERIC=y +CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_CE=y +CONFIG_CRYPTO_AES_ARM64_CE_BLK=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y +CONFIG_CRYPTO_SHA256_ARM64=y +CONFIG_CRYPTO_SHA2_ARM64_CE=y +CONFIG_CRYPTO_SHA512_ARM64=y +CONFIG_CRYPTO_SHA512_ARM64_CE=y +CONFIG_CRYPTO_ZSTD=y # CONFIG_DEBUG_ALIGN_RODATA is not set CONFIG_DMA_DIRECT_OPS=y # CONFIG_FLATMEM_MANUAL is not set From c4854079a1495c86cf07a2cc169f9aa909bc115c Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Wed, 20 Apr 2022 15:26:32 -0300 Subject: [PATCH 21/31] mvebu/cortexa72: enable armv8-CE crypto algos This enables armv8 crypto extensions version of AES, GHASH, SHA1, SHA256, and SHA512 algorithms in the kernel. The choice of algorithms match the 32-bit versions that are enabled in the target config-5.10 file, but were only used by the cortexa9 subtarget. Signed-off-by: Eneas U de Queiroz (cherry picked from commit 06bb5ac1f2b62c3e10f24d7096e86f6368aaf41d) --- target/linux/mvebu/cortexa72/config-4.19 | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/target/linux/mvebu/cortexa72/config-4.19 b/target/linux/mvebu/cortexa72/config-4.19 index c78eb84372..474f3858b8 100644 --- a/target/linux/mvebu/cortexa72/config-4.19 +++ b/target/linux/mvebu/cortexa72/config-4.19 @@ -54,7 +54,7 @@ CONFIG_ARM64=y CONFIG_ARM64_4K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set CONFIG_ARM64_CONT_SHIFT=4 -# CONFIG_ARM64_CRYPTO is not set +CONFIG_ARM64_CRYPTO=y # CONFIG_ARM64_HW_AFDBM is not set # CONFIG_ARM64_LSE_ATOMICS is not set CONFIG_ARM64_PAGE_SHIFT=12 @@ -85,6 +85,17 @@ CONFIG_ARM_GIC_V3_ITS_PCI=y CONFIG_ARM_PSCI_FW=y # CONFIG_ARM_SP805_WATCHDOG is not set CONFIG_AUDIT_ARCH_COMPAT_GENERIC=y +CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_CE=y +CONFIG_CRYPTO_AES_ARM64_CE_BLK=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y +CONFIG_CRYPTO_SHA256_ARM64=y +CONFIG_CRYPTO_SHA2_ARM64_CE=y +CONFIG_CRYPTO_SHA512_ARM64=y +CONFIG_CRYPTO_SHA512_ARM64_CE=y +CONFIG_CRYPTO_ZSTD=y # CONFIG_DEBUG_ALIGN_RODATA is not set CONFIG_DMA_DIRECT_OPS=y # CONFIG_FLATMEM_MANUAL is not set From 3de5688c88c911cf23384d9fed6659de6f69caf4 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Wed, 20 Apr 2022 16:23:47 -0300 Subject: [PATCH 22/31] octeontx: add armv8-CE version of CRC T10 Adds the crypto extensions version of the CRC T10 algorithm that is already built into the kernel. Signed-off-by: Eneas U de Queiroz (cherry picked from commit 1b94e4aab8ddbe5719f1e859e064c1c5dfa4587f) --- target/linux/octeontx/config-4.14 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/linux/octeontx/config-4.14 b/target/linux/octeontx/config-4.14 index b5a4a48dc2..bf5576e7c5 100644 --- a/target/linux/octeontx/config-4.14 +++ b/target/linux/octeontx/config-4.14 @@ -155,7 +155,7 @@ CONFIG_CRYPTO_CRC32=y CONFIG_CRYPTO_CRC32C=y # CONFIG_CRYPTO_CRC32_ARM64_CE is not set CONFIG_CRYPTO_CRCT10DIF=y -# CONFIG_CRYPTO_CRCT10DIF_ARM64_CE is not set +CONFIG_CRYPTO_CRCT10DIF_ARM64_CE=y CONFIG_CRYPTO_CRYPTD=y CONFIG_CRYPTO_DRBG=y CONFIG_CRYPTO_DRBG_HMAC=y From d486dd89596bd789af16c4e8fd1c0e4ab7a15e12 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Wed, 20 Apr 2022 15:26:32 -0300 Subject: [PATCH 23/31] sunxi/cortexa53: enable armv8-CE crypto algorithms This enables armv8 crypto extensions version of AES, GHASH, SHA1, and CRC T10 algorithms in the kernel. Signed-off-by: Eneas U de Queiroz (cherry picked from commit 9be35180f43a4916f53430d8c93437d33896e860) --- target/linux/sunxi/cortexa53/config-4.14 | 11 ++++++++++- target/linux/sunxi/cortexa53/config-4.19 | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/target/linux/sunxi/cortexa53/config-4.14 b/target/linux/sunxi/cortexa53/config-4.14 index f58d687e65..63b236391d 100644 --- a/target/linux/sunxi/cortexa53/config-4.14 +++ b/target/linux/sunxi/cortexa53/config-4.14 @@ -25,7 +25,7 @@ CONFIG_ARM64=y CONFIG_ARM64_4K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set CONFIG_ARM64_CONT_SHIFT=4 -# CONFIG_ARM64_CRYPTO is not set +CONFIG_ARM64_CRYPTO=y # CONFIG_ARM64_HW_AFDBM is not set # CONFIG_ARM64_LSE_ATOMICS is not set CONFIG_ARM64_PAGE_SHIFT=12 @@ -44,6 +44,15 @@ CONFIG_ARM_AMBA=y CONFIG_ARM_GIC_V3=y # CONFIG_ARM_SP805_WATCHDOG is not set CONFIG_AUDIT_ARCH_COMPAT_GENERIC=y +CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_CE=y +CONFIG_CRYPTO_AES_ARM64_CE_BLK=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_CRCT10DIF_ARM64_CE=y +CONFIG_CRYPTO_CRYPTD=y +CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y +CONFIG_CRYPTO_SIMD=y CONFIG_DWMAC_SUN8I=y CONFIG_EEPROM_AT24=y CONFIG_FRAME_POINTER=y diff --git a/target/linux/sunxi/cortexa53/config-4.19 b/target/linux/sunxi/cortexa53/config-4.19 index 4103862716..925de5959b 100644 --- a/target/linux/sunxi/cortexa53/config-4.19 +++ b/target/linux/sunxi/cortexa53/config-4.19 @@ -26,7 +26,7 @@ CONFIG_ARM64=y CONFIG_ARM64_4K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set CONFIG_ARM64_CONT_SHIFT=4 -# CONFIG_ARM64_CRYPTO is not set +CONFIG_ARM64_CRYPTO=y # CONFIG_ARM64_HW_AFDBM is not set # CONFIG_ARM64_LSE_ATOMICS is not set CONFIG_ARM64_PAGE_SHIFT=12 @@ -45,6 +45,15 @@ CONFIG_ARM64_VA_BITS_39=y CONFIG_ARM_AMBA=y # CONFIG_ARM_SP805_WATCHDOG is not set CONFIG_AUDIT_ARCH_COMPAT_GENERIC=y +CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_CE=y +CONFIG_CRYPTO_AES_ARM64_CE_BLK=y +CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_CRCT10DIF_ARM64_CE=y +CONFIG_CRYPTO_CRYPTD=y +CONFIG_CRYPTO_GHASH_ARM64_CE=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y +CONFIG_CRYPTO_SIMD=y CONFIG_DMA_DIRECT_OPS=y CONFIG_DWMAC_SUN8I=y CONFIG_EEPROM_AT24=y From 602d81c1dacbd7200e596ee8bfde429b96901622 Mon Sep 17 00:00:00 2001 From: Eneas U de Queiroz Date: Wed, 8 Jun 2022 20:30:39 -0300 Subject: [PATCH 24/31] wolfssl: make WOLFSSL_HAS_OPENVPN default to y Openvpn forces CONFIG_WOLFSSL_HAS_OPENVPN=y. When the phase1 bots build the now non-shared package, openvpn will not be selected, and WolfSSL will be built without it. Then phase2 bots have CONFIG_ALL=y, which will select openvpn and force CONFIG_WOLFSSL_HAS_OPENVPN=y. This changes the version hash, causing dependency failures, as shared packages expect the phase2 hash. Fixes: #9738 Signed-off-by: Eneas U de Queiroz (cherry picked from commit d55f12cc7925b7d84af80015cbeb0422dd7e0b36) --- package/libs/wolfssl/Config.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/libs/wolfssl/Config.in b/package/libs/wolfssl/Config.in index f495a90ff6..3d264e7743 100644 --- a/package/libs/wolfssl/Config.in +++ b/package/libs/wolfssl/Config.in @@ -53,7 +53,7 @@ config WOLFSSL_HAS_ECC25519 config WOLFSSL_HAS_OPENVPN bool "Include OpenVPN support" - default n + default y config WOLFSSL_ALT_NAMES bool "Include SAN (Subject Alternative Name) support" From 0f30a099006acddc086be37dd9510cad2dac30a7 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Tue, 14 Jun 2022 04:58:05 +0800 Subject: [PATCH 25/31] kernel: bump to 4.9.317, 4.14.282, 4.19.246 Refreshed all patches. Signed-off-by: Tianling Shen --- include/kernel-version.mk | 12 ++++++------ .../patches-4.14/910-unaligned_access_hacks.patch | 4 ++-- .../patches-4.14/910-unaligned_access_hacks.patch | 4 ++-- .../patches-4.19/910-unaligned_access_hacks.patch | 4 ++-- .../717-v5.5-net-sfp-rework-upstream-interface.patch | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/kernel-version.mk b/include/kernel-version.mk index cf78487424..0c25f72d38 100644 --- a/include/kernel-version.mk +++ b/include/kernel-version.mk @@ -6,13 +6,13 @@ ifdef CONFIG_TESTING_KERNEL KERNEL_PATCHVER:=$(KERNEL_TESTING_PATCHVER) endif -LINUX_VERSION-4.9 = .316 -LINUX_VERSION-4.14 = .281 -LINUX_VERSION-4.19 = .245 +LINUX_VERSION-4.9 = .317 +LINUX_VERSION-4.14 = .282 +LINUX_VERSION-4.19 = .246 -LINUX_KERNEL_HASH-4.9.316 = 57b698920b2c66b2615d0aa3fd551084560fee901552aff6d89d3553653bcd17 -LINUX_KERNEL_HASH-4.14.281 = 3ae2f9c6966b9b19d27de7d84a4b1697df84a9658c34b850a51673266a583b5e -LINUX_KERNEL_HASH-4.19.245 = 99fddd6027ce924abe9f0435027fc517983b0399787efa44d0bac26e22c3a8e8 +LINUX_KERNEL_HASH-4.9.317 = 7ea3369b915c99b38528dbf68c491f3f9324b2f4e56eac980dd7524d2fc30d1b +LINUX_KERNEL_HASH-4.14.282 = 6637c8470c3093a477d2de6fa9e0f63c55810438a411c73bcd723634371657a3 +LINUX_KERNEL_HASH-4.19.246 = 00ad2f5a36c91221a2ade0078b93bf84b60d494bd1ef51eaccb5bdb6277dba3a remove_uri_prefix=$(subst git://,,$(subst http://,,$(subst https://,,$(1)))) sanitize_uri=$(call qstrip,$(subst @,_,$(subst :,_,$(subst .,_,$(subst -,_,$(subst /,_,$(1))))))) diff --git a/target/linux/ar71xx/patches-4.14/910-unaligned_access_hacks.patch b/target/linux/ar71xx/patches-4.14/910-unaligned_access_hacks.patch index 6e379cfa9e..f0305e26c5 100644 --- a/target/linux/ar71xx/patches-4.14/910-unaligned_access_hacks.patch +++ b/target/linux/ar71xx/patches-4.14/910-unaligned_access_hacks.patch @@ -707,8 +707,8 @@ #include +#include - u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); - u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, + u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); + u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, --- a/include/uapi/linux/in.h +++ b/include/uapi/linux/in.h @@ -84,7 +84,7 @@ enum { diff --git a/target/linux/ath79/patches-4.14/910-unaligned_access_hacks.patch b/target/linux/ath79/patches-4.14/910-unaligned_access_hacks.patch index eaf474eb09..f134304a28 100644 --- a/target/linux/ath79/patches-4.14/910-unaligned_access_hacks.patch +++ b/target/linux/ath79/patches-4.14/910-unaligned_access_hacks.patch @@ -687,8 +687,8 @@ #include +#include - u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); - u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, + u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); + u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, --- a/include/uapi/linux/in.h +++ b/include/uapi/linux/in.h @@ -84,7 +84,7 @@ enum { diff --git a/target/linux/ath79/patches-4.19/910-unaligned_access_hacks.patch b/target/linux/ath79/patches-4.19/910-unaligned_access_hacks.patch index 6697434276..d9dc12fe5b 100644 --- a/target/linux/ath79/patches-4.19/910-unaligned_access_hacks.patch +++ b/target/linux/ath79/patches-4.19/910-unaligned_access_hacks.patch @@ -678,8 +678,8 @@ #include +#include - u32 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); - u32 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, + u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); + u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, --- a/include/uapi/linux/in.h +++ b/include/uapi/linux/in.h @@ -84,7 +84,7 @@ enum { diff --git a/target/linux/generic/backport-4.19/717-v5.5-net-sfp-rework-upstream-interface.patch b/target/linux/generic/backport-4.19/717-v5.5-net-sfp-rework-upstream-interface.patch index a901b15aaf..b28ebe6f79 100644 --- a/target/linux/generic/backport-4.19/717-v5.5-net-sfp-rework-upstream-interface.patch +++ b/target/linux/generic/backport-4.19/717-v5.5-net-sfp-rework-upstream-interface.patch @@ -117,7 +117,7 @@ Signed-off-by: Russell King { struct fwnode_reference_args ref; struct sfp_bus *bus; -@@ -556,7 +562,39 @@ struct sfp_bus *sfp_register_upstream_no +@@ -561,7 +567,39 @@ struct sfp_bus *sfp_register_upstream_no if (!bus) return ERR_PTR(-ENOMEM); @@ -157,7 +157,7 @@ Signed-off-by: Russell King bus->upstream_ops = ops; bus->upstream = upstream; -@@ -569,33 +607,33 @@ struct sfp_bus *sfp_register_upstream_no +@@ -574,33 +612,33 @@ struct sfp_bus *sfp_register_upstream_no } rtnl_unlock(); From 1ceaf460bef66757ee0638806b9dfff0c1abf0c0 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Wed, 15 Jun 2022 19:32:03 +0800 Subject: [PATCH 26/31] layerscape: armv8_64b: add missing Kconfig Signed-off-by: Tianling Shen --- target/linux/layerscape/armv8_64b/config-4.14 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/target/linux/layerscape/armv8_64b/config-4.14 b/target/linux/layerscape/armv8_64b/config-4.14 index 6c8b40058a..f3345f73a1 100644 --- a/target/linux/layerscape/armv8_64b/config-4.14 +++ b/target/linux/layerscape/armv8_64b/config-4.14 @@ -184,13 +184,16 @@ CONFIG_CRYPTO_ACOMP2=y CONFIG_CRYPTO_AEAD=y CONFIG_CRYPTO_AEAD2=y CONFIG_CRYPTO_AES_ARM64=y +CONFIG_CRYPTO_AES_ARM64_BS=y CONFIG_CRYPTO_AES_ARM64_CE=y CONFIG_CRYPTO_AES_ARM64_CE_BLK=y CONFIG_CRYPTO_AES_ARM64_CE_CCM=y +CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y CONFIG_CRYPTO_AUTHENC=y CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_CRC32=y CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_CRC32_ARM64_CE=y CONFIG_CRYPTO_CRCT10DIF=y CONFIG_CRYPTO_CRCT10DIF_ARM64_CE=y CONFIG_CRYPTO_CRYPTD=y @@ -204,6 +207,7 @@ CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API=y CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC=y CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI=y # CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG is not set +CONFIG_CRYPTO_DEV_FSL_CAAM_DMA=y # CONFIG_CRYPTO_DEV_FSL_CAAM_INTC is not set CONFIG_CRYPTO_DEV_FSL_CAAM_JR=y CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API=y @@ -226,10 +230,14 @@ CONFIG_CRYPTO_NULL2=y CONFIG_CRYPTO_RNG=y CONFIG_CRYPTO_RNG2=y CONFIG_CRYPTO_RSA=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_SHA1_ARM64_CE=y CONFIG_CRYPTO_SHA256=y CONFIG_CRYPTO_SHA256_ARM64=y CONFIG_CRYPTO_SHA2_ARM64_CE=y +CONFIG_CRYPTO_SHA512_ARM64=y CONFIG_CRYPTO_SIMD=y +# CONFIG_CRYPTO_TLS is not set CONFIG_CRYPTO_WORKQUEUE=y CONFIG_CRYPTO_XTS=y CONFIG_CRYPTO_XXHASH=y @@ -839,6 +847,7 @@ CONFIG_SCSI_SAS_HOST_SMP=y CONFIG_SCSI_SAS_LIBSAS=y CONFIG_SECCOMP=y CONFIG_SECCOMP_FILTER=y +# CONFIG_SECURE_KEYS is not set CONFIG_SERIAL_8250_DEPRECATED_OPTIONS=y CONFIG_SERIAL_8250_DW=y CONFIG_SERIAL_8250_EXTENDED=y From 516798ace3d25ad1697607de1780c1bfcbfa94e6 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 13 Jan 2021 00:54:08 +0100 Subject: [PATCH 27/31] nf-conntrack: allow querying conntrack info in nfqueue This allows libnetfilter_queue to access connection tracking information by requesting NFQA_CFG_F_CONNTRACK. Connection tracking information is provided in the NFQA_CT attribute. CONFIG_NETFILTER_NETLINK_GLUE_CT enables the interaction between nf_queue and nf_conntrack_netlink. Without this option, trying to access connection tracking information results in "Operation not supported". Signed-off-by: Etan Kissling [Backport to kernel 4.9, 4.14 and 4.19] Signed-off-by: AmadeusGhost Signed-off-by: Tianling Shen --- package/kernel/linux/modules/netfilter.mk | 2 +- target/linux/generic/config-4.14 | 1 + target/linux/generic/config-4.19 | 1 + target/linux/generic/config-4.9 | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/package/kernel/linux/modules/netfilter.mk b/package/kernel/linux/modules/netfilter.mk index cd58b73275..31b5698f93 100644 --- a/package/kernel/linux/modules/netfilter.mk +++ b/package/kernel/linux/modules/netfilter.mk @@ -1067,7 +1067,7 @@ $(eval $(call KernelPackage,nfnetlink-queue)) define KernelPackage/nf-conntrack-netlink TITLE:=Connection tracking netlink interface FILES:=$(LINUX_DIR)/net/netfilter/nf_conntrack_netlink.ko - KCONFIG:=CONFIG_NF_CT_NETLINK CONFIG_NF_CONNTRACK_EVENTS=y + KCONFIG:=CONFIG_NF_CT_NETLINK CONFIG_NF_CONNTRACK_EVENTS=y CONFIG_NETFILTER_NETLINK_GLUE_CT=y AUTOLOAD:=$(call AutoProbe,nf_conntrack_netlink) $(call AddDepends/nfnetlink,+kmod-ipt-conntrack) endef diff --git a/target/linux/generic/config-4.14 b/target/linux/generic/config-4.14 index 4e4df6ec16..5a8e3781ea 100644 --- a/target/linux/generic/config-4.14 +++ b/target/linux/generic/config-4.14 @@ -3335,6 +3335,7 @@ CONFIG_NF_CONNTRACK_PROCFS=y # CONFIG_NF_CONNTRACK_TIMESTAMP is not set # CONFIG_NF_CONNTRACK_ZONES is not set # CONFIG_NF_CT_NETLINK is not set +# CONFIG_NF_CT_NETLINK_HELPER is not set # CONFIG_NF_CT_NETLINK_TIMEOUT is not set # CONFIG_NF_CT_PROTO_DCCP is not set # CONFIG_NF_CT_PROTO_GRE is not set diff --git a/target/linux/generic/config-4.19 b/target/linux/generic/config-4.19 index 658bcb09db..63813be3c9 100644 --- a/target/linux/generic/config-4.19 +++ b/target/linux/generic/config-4.19 @@ -3495,6 +3495,7 @@ CONFIG_NF_CONNTRACK_PROCFS=y # CONFIG_NF_CONNTRACK_TIMESTAMP is not set # CONFIG_NF_CONNTRACK_ZONES is not set # CONFIG_NF_CT_NETLINK is not set +# CONFIG_NF_CT_NETLINK_HELPER is not set # CONFIG_NF_CT_NETLINK_TIMEOUT is not set # CONFIG_NF_CT_PROTO_DCCP is not set # CONFIG_NF_CT_PROTO_GRE is not set diff --git a/target/linux/generic/config-4.9 b/target/linux/generic/config-4.9 index 8eae9b4e7b..5a5e982534 100644 --- a/target/linux/generic/config-4.9 +++ b/target/linux/generic/config-4.9 @@ -3026,6 +3026,7 @@ CONFIG_NF_CONNTRACK_PROCFS=y # CONFIG_NF_CONNTRACK_TIMESTAMP is not set # CONFIG_NF_CONNTRACK_ZONES is not set # CONFIG_NF_CT_NETLINK is not set +# CONFIG_NF_CT_NETLINK_HELPER is not set # CONFIG_NF_CT_NETLINK_TIMEOUT is not set # CONFIG_NF_CT_PROTO_DCCP is not set # CONFIG_NF_CT_PROTO_GRE is not set From 56e17e1fbc60bbe0d2509ba436e974320ccdd18e Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sat, 18 Jun 2022 01:03:08 +0800 Subject: [PATCH 28/31] r8168: bump to 8.050.03 Signed-off-by: Tianling Shen (cherry picked from commit f3002993d248a8b5d50c71d3e4a0173cd4266e64) --- package/kernel/r8168/Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/package/kernel/r8168/Makefile b/package/kernel/r8168/Makefile index a68b74e225..041fcc5d7a 100644 --- a/package/kernel/r8168/Makefile +++ b/package/kernel/r8168/Makefile @@ -7,14 +7,13 @@ include $(TOPDIR)/rules.mk include $(INCLUDE_DIR)/kernel.mk PKG_NAME:=r8168 -PKG_VERSION:=8.050.00 +PKG_VERSION:=8.050.03 PKG_RELEASE:=$(AUTORELEAE) PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/BROBIRD/openwrt-r8168.git -PKG_SOURCE_DATE:=2022-05-03 -PKG_SOURCE_VERSION:=1c18465e76ed6872f001d08e2fe0c139fc80ef7e -PKG_MIRROR_HASH:=5ca8242bd209079a5abac608a865f86e727cc8de15a460d7d78e611d0c443f15 +PKG_SOURCE_VERSION:=ddfaceacd1b7ed2857fb995642a8ffb1fc37e989 +PKG_MIRROR_HASH:=5428f60dc33e9503c6cfdf690c00077149dce24cbb0591129d905b9f1aad9202 PKG_BUILD_DIR:=$(KERNEL_BUILD_DIR)/$(PKG_NAME)-$(PKG_VERSION) From 3253d89cd540f9974a0d7789e4f60c2388751d8d Mon Sep 17 00:00:00 2001 From: MkQtS <81752398+MkQtS@users.noreply.github.com> Date: Sat, 18 Jun 2022 23:15:06 +0800 Subject: [PATCH 29/31] default-settings: update tencent ntp server see tencentyun's doc [1] 1. https://github.com/tencentyun/qcloud-documents/blob/master/product/%E8%AE%A1%E7%AE%97%E4%B8%8E%E7%BD%91%E7%BB%9C/%E4%BA%91%E6%9C%8D%E5%8A%A1%E5%99%A8/%E6%9C%80%E4%BD%B3%E5%AE%9E%E8%B7%B5/NTP%20%E6%9C%8D%E5%8A%A1/%E8%85%BE%E8%AE%AF%E4%BA%91%20NTP%20%E6%9C%8D%E5%8A%A1.md (cherry picked from commit 0688e346d06452508345ec92e88bf6d2876e7b52) --- .../emortal/default-settings/files/99-default-settings-chinese | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/emortal/default-settings/files/99-default-settings-chinese b/package/emortal/default-settings/files/99-default-settings-chinese index 30a9167bca..d7c4d9d4d9 100755 --- a/package/emortal/default-settings/files/99-default-settings-chinese +++ b/package/emortal/default-settings/files/99-default-settings-chinese @@ -5,7 +5,7 @@ uci -q batch <<-EOF set system.@system[0].zonename='Asia/Shanghai' delete system.ntp.server - add_list system.ntp.server='time1.cloud.tencent.com' + add_list system.ntp.server='ntp.tencent.com' add_list system.ntp.server='ntp1.aliyun.com' add_list system.ntp.server='ntp.ntsc.ac.cn' add_list system.ntp.server='cn.ntp.org.cn' From c77692e91d97552ca72d50b466199c62ed7404d3 Mon Sep 17 00:00:00 2001 From: Josef Schlehofer Date: Mon, 6 Jun 2022 22:08:42 +0200 Subject: [PATCH 30/31] tools/libressl: update to version 3.4.3 Release notes: https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-3.4.3-relnotes.txt ``` It includes the following security fix: * A malicious certificate can cause an infinite loop. Reported by and fix from Tavis Ormandy and David Benjamin, Google. ``` Signed-off-by: Josef Schlehofer (cherry picked from commit 25534d5cc20a807ff776fdb18847344167ce081d) --- tools/libressl/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/libressl/Makefile b/tools/libressl/Makefile index da7cd6201b..0679961d17 100644 --- a/tools/libressl/Makefile +++ b/tools/libressl/Makefile @@ -8,8 +8,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=libressl -PKG_VERSION:=3.4.2 -PKG_HASH:=cb82ca7d547336917352fbd23db2fc483c6c44d35157b32780214ec74197b3ce +PKG_VERSION:=3.4.3 +PKG_HASH:=ff88bffe354818b3ccf545e3cafe454c5031c7a77217074f533271d63c37f08d PKG_RELEASE:=1 PKG_CPE_ID:=cpe:/a:openbsd:libressl From 186be591ff1f80bb701abda1965b4cc41f70b619 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Tue, 21 Jun 2022 23:51:16 +0800 Subject: [PATCH 31/31] r8152: bump to 2.16.1 Signed-off-by: Tianling Shen (cherry picked from commit 51333e4cf5be36b5936e48add71a60423906c19a) [removed upstreamed patch] Signed-off-by: Tianling Shen --- package/kernel/r8152/Makefile | 24 ++++----------- ...-compilation-issue-under-kernel-4.14.patch | 30 ------------------- .../100-add-LED-configuration-from-OF.patch | 10 +++---- 3 files changed, 10 insertions(+), 54 deletions(-) delete mode 100644 package/kernel/r8152/patches/010-Fix-compilation-issue-under-kernel-4.14.patch diff --git a/package/kernel/r8152/Makefile b/package/kernel/r8152/Makefile index 76fe25f3fe..435e62dbfa 100644 --- a/package/kernel/r8152/Makefile +++ b/package/kernel/r8152/Makefile @@ -8,19 +8,17 @@ include $(TOPDIR)/rules.mk include $(INCLUDE_DIR)/kernel.mk PKG_NAME:=r8152 -PKG_VERSION:=2.15.20211119 -PKG_RELEASE:=1 +PKG_VERSION:=2.16.1 +PKG_RELEASE:=$(AUTORELEASE) PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/wget/realtek-r8152-linux/tar.gz/v$(PKG_VERSION)? -PKG_HASH:=b7926db3b4ca71d453ac1cf875d7a8ab409ece108edc6913e8bc1c0c3b99179d - +PKG_HASH:=2be6a02f6e29485efd107bb7e777ad3c482d9db0ff7e5e6c5ef034a1557a395b PKG_BUILD_DIR:=$(KERNEL_BUILD_DIR)/realtek-$(PKG_NAME)-linux-$(PKG_VERSION) +PKG_LICENSE:=GPL-2.0-only PKG_MAINTAINER:=Tianling Shen -PKG_BUILD_PARALLEL:=1 - include $(INCLUDE_DIR)/package.mk define KernelPackage/usb-net-rtl8152-vendor @@ -37,20 +35,8 @@ define KernelPackage/usb-net-rtl8152-vendor/description Kernel module for Realtek RTL8152/RTL8153 Based USB Ethernet Adapters endef -R8152_MAKEOPTS= -C $(PKG_BUILD_DIR) \ - PATH="$(TARGET_PATH)" \ - ARCH="$(LINUX_KARCH)" \ - CROSS_COMPILE="$(TARGET_CROSS)" \ - TARGET="$(HAL_TARGET)" \ - TOOLPREFIX="$(KERNEL_CROSS)" \ - TOOLPATH="$(KERNEL_CROSS)" \ - KERNELPATH="$(LINUX_DIR)" \ - KERNELDIR="$(LINUX_DIR)" \ - LDOPTS=" " \ - DOMULTI=1 - define Build/Compile - +$(MAKE) $(PKG_JOBS) $(R8152_MAKEOPTS) modules + +$(KERNEL_MAKE) M=$(PKG_BUILD_DIR) modules endef $(eval $(call KernelPackage,usb-net-rtl8152-vendor)) diff --git a/package/kernel/r8152/patches/010-Fix-compilation-issue-under-kernel-4.14.patch b/package/kernel/r8152/patches/010-Fix-compilation-issue-under-kernel-4.14.patch deleted file mode 100644 index 6e37ced067..0000000000 --- a/package/kernel/r8152/patches/010-Fix-compilation-issue-under-kernel-4.14.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 3295cef0552f0fa64b753f1e5a8ac81307260ae5 Mon Sep 17 00:00:00 2001 -From: AmadeusGhost <42570690+AmadeusGhost@users.noreply.github.com> -Date: Sun, 3 Apr 2022 23:50:38 +0800 -Subject: [PATCH] Fix compilation issue under kernel 4.14 - ---- - compatibility.h | 6 ++++-- - 1 file changed, 4 insertions(+), 2 deletions(-) - -diff --git a/compatibility.h b/compatibility.h -index 1e2d3d8..fb981a0 100644 ---- a/compatibility.h -+++ b/compatibility.h -@@ -549,12 +549,14 @@ - linkmode_clear_bit(nr, addr); - } - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,217) || LINUX_VERSION_CODE < KERNEL_VERSION(4,19,10) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,19,10) -+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,217) || LINUX_VERSION_CODE >= KERNEL_VERSION(4,15,0) - static inline void skb_mark_not_on_list(struct sk_buff *skb) - { - skb->next = NULL; - } --#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,14,217) || LINUX_VERSION_CODE < KERNEL_VERSION(4,19,10) */ -+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,14,217) || LINUX_VERSION_CODE >= KERNEL_VERSION(4,15,0) */ -+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,19,10) */ - #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,0,0) */ - #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,1,0) */ - #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) */ diff --git a/package/kernel/r8152/patches/100-add-LED-configuration-from-OF.patch b/package/kernel/r8152/patches/100-add-LED-configuration-from-OF.patch index 38d885d72d..1c519e686d 100644 --- a/package/kernel/r8152/patches/100-add-LED-configuration-from-OF.patch +++ b/package/kernel/r8152/patches/100-add-LED-configuration-from-OF.patch @@ -22,8 +22,8 @@ Signed-off-by: David Bauer #include #include #include -@@ -9804,6 +9805,22 @@ static void rtl_tally_reset(struct r8152 - ocp_write_word(tp, MCU_TYPE_PLA, PLA_RSTTALLY, ocp_data); +@@ -10107,6 +10108,22 @@ static void rtl_disable_spi(struct r8152 + ocp_write_word(tp, MCU_TYPE_USB, 0xcbf0, ocp_data); } +static int r8152_led_configuration(struct r8152 *tp) @@ -45,7 +45,7 @@ Signed-off-by: David Bauer static void r8152b_init(struct r8152 *tp) { u32 ocp_data; -@@ -9865,6 +9882,8 @@ static void r8152b_init(struct r8152 *tp +@@ -10168,6 +10185,8 @@ static void r8152b_init(struct r8152 *tp ocp_data = ocp_read_word(tp, MCU_TYPE_USB, USB_USB_CTRL); ocp_data &= ~(RX_AGG_DISABLE | RX_ZERO_EN); ocp_write_word(tp, MCU_TYPE_USB, USB_USB_CTRL, ocp_data); @@ -54,7 +54,7 @@ Signed-off-by: David Bauer } static void r8153_init(struct r8152 *tp) -@@ -10008,6 +10027,8 @@ static void r8153_init(struct r8152 *tp) +@@ -10311,6 +10330,8 @@ static void r8153_init(struct r8152 *tp) tp->coalesce = COALESCE_SLOW; break; } @@ -63,7 +63,7 @@ Signed-off-by: David Bauer } static void r8153b_init(struct r8152 *tp) -@@ -10098,6 +10119,8 @@ static void r8153b_init(struct r8152 *tp +@@ -10413,6 +10434,8 @@ static void r8153b_init(struct r8152 *tp rtl_tally_reset(tp); tp->coalesce = 15000; /* 15 us */