diff --git a/package/boot/uboot-envtools/files/ramips b/package/boot/uboot-envtools/files/ramips index d97c812881..eebc08d65d 100644 --- a/package/boot/uboot-envtools/files/ramips +++ b/package/boot/uboot-envtools/files/ramips @@ -60,6 +60,7 @@ linksys,ea7300-v2|\ linksys,ea7500-v2|\ linksys,ea8100-v1|\ linksys,ea8100-v2|\ +mts,wg430223|\ xiaomi,mi-router-3g|\ xiaomi,mi-router-3-pro|\ xiaomi,mi-router-4|\ diff --git a/package/firmware/wireless-regdb/Makefile b/package/firmware/wireless-regdb/Makefile index 49b1ad5cab..006974b3d1 100644 --- a/package/firmware/wireless-regdb/Makefile +++ b/package/firmware/wireless-regdb/Makefile @@ -1,12 +1,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=wireless-regdb -PKG_VERSION:=2022.02.18 +PKG_VERSION:=2022.06.06 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=@KERNEL/software/network/wireless-regdb/ -PKG_HASH:=8828c25a4ee25020044004f57374bb9deac852809fad70f8d3d01770bf9ac97f +PKG_HASH:=ac00f97efecce5046ed069d1d93f3365fdf994c7c7854a8fc50831e959537230 PKG_MAINTAINER:=Felix Fietkau diff --git a/package/firmware/wireless-regdb/patches/500-world-regd-5GHz.patch b/package/firmware/wireless-regdb/patches/500-world-regd-5GHz.patch index ae38f93537..3f6d4c7e8d 100644 --- a/package/firmware/wireless-regdb/patches/500-world-regd-5GHz.patch +++ b/package/firmware/wireless-regdb/patches/500-world-regd-5GHz.patch @@ -5,7 +5,7 @@ Signed-off-by: Felix Fietkau --- --- a/db.txt +++ b/db.txt -@@ -16,7 +16,7 @@ country 00: +@@ -19,7 +19,7 @@ country 00: # Channel 14. Only JP enables this and for 802.11b only (2474 - 2494 @ 20), (20), NO-IR, NO-OFDM # Channel 36 - 48 diff --git a/package/kernel/mac80211/patches/subsys/783-sync-nl80211.patch b/package/kernel/mac80211/patches/subsys/783-sync-nl80211.patch new file mode 100644 index 0000000000..dc2b05b1a3 --- /dev/null +++ b/package/kernel/mac80211/patches/subsys/783-sync-nl80211.patch @@ -0,0 +1,22 @@ +--- a/include/uapi/linux/nl80211.h ++++ b/include/uapi/linux/nl80211.h +@@ -6027,6 +6027,11 @@ enum nl80211_feature_flags { + * @NL80211_EXT_FEATURE_BSS_COLOR: The driver supports BSS color collision + * detection and change announcemnts. + * ++ * @NL80211_EXT_FEATURE_FILS_CRYPTO_OFFLOAD: Driver running in AP mode supports ++ * FILS encryption and decryption for (Re)Association Request and Response ++ * frames. Userspace has to share FILS AAD details to the driver by using ++ * @NL80211_CMD_SET_FILS_AAD. ++ * + * @NL80211_EXT_FEATURE_RADAR_BACKGROUND: Device supports background radar/CAC + * detection. + * +@@ -6095,6 +6100,7 @@ enum nl80211_ext_feature_index { + NL80211_EXT_FEATURE_SECURE_RTT, + NL80211_EXT_FEATURE_PROT_RANGE_NEGO_AND_MEASURE, + NL80211_EXT_FEATURE_BSS_COLOR, ++ NL80211_EXT_FEATURE_FILS_CRYPTO_OFFLOAD, + NL80211_EXT_FEATURE_RADAR_BACKGROUND, + + /* add new features before the definition below */ diff --git a/package/network/services/hostapd/patches/420-indicate-features.patch b/package/network/services/hostapd/patches/420-indicate-features.patch index 80d945ca16..12edb6bacc 100644 --- a/package/network/services/hostapd/patches/420-indicate-features.patch +++ b/package/network/services/hostapd/patches/420-indicate-features.patch @@ -14,7 +14,7 @@ wpa_supplicant_event_global = hostapd_wpa_event_global; for (;;) { - c = getopt(argc, argv, "b:Bde:f:hi:KP:sSTtu:vg:G:q"); -+ c = getopt(argc, argv, "b:Bde:f:hi:KP:sSTtu:vg:G:qv::"); ++ c = getopt(argc, argv, "b:Bde:f:hi:KP:sSTtu:g:G:qv::"); if (c < 0) break; switch (c) { diff --git a/package/network/utils/iw/Makefile b/package/network/utils/iw/Makefile index da2b556416..eb3a9ba091 100644 --- a/package/network/utils/iw/Makefile +++ b/package/network/utils/iw/Makefile @@ -8,12 +8,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=iw -PKG_VERSION:=5.16 +PKG_VERSION:=5.19 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.xz PKG_SOURCE_URL:=@KERNEL/software/network/iw -PKG_HASH:=4c44e42762f903f9094ba5a598998c800a97a62afd6fd31ec1e0a799e308659c +PKG_HASH:=f167bbe947dd53bb9ebc0c1dcef5db6ad73ac1d6084f2c6f9376c5c360cc4d4e PKG_MAINTAINER:=Felix Fietkau PKG_LICENSE:=GPL-2.0 diff --git a/package/network/utils/iw/patches/001-nl80211_h_sync.patch b/package/network/utils/iw/patches/001-nl80211_h_sync.patch index 3fb5ca255c..49d08d9b01 100644 --- a/package/network/utils/iw/patches/001-nl80211_h_sync.patch +++ b/package/network/utils/iw/patches/001-nl80211_h_sync.patch @@ -1,5 +1,14 @@ --- a/nl80211.h +++ b/nl80211.h +@@ -11,7 +11,7 @@ + * Copyright 2008 Jouni Malinen + * Copyright 2008 Colin McCabe + * Copyright 2015-2017 Intel Deutschland GmbH +- * Copyright (C) 2018-2022 Intel Corporation ++ * Copyright (C) 2018-2021 Intel Corporation + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above @@ -301,29 +301,6 @@ */ @@ -30,7 +39,7 @@ * enum nl80211_commands - supported nl80211 commands * * @NL80211_CMD_UNSPEC: unspecified command to catch errors -@@ -1226,12 +1203,6 @@ +@@ -1226,17 +1203,6 @@ * @NL80211_CMD_COLOR_CHANGE_COMPLETED: Notify userland that the color change * has completed * @@ -39,39 +48,255 @@ - * &NL80211_ATTR_FILS_KEK - for KEK - * &NL80211_ATTR_FILS_NONCES - for FILS Nonces - * (STA Nonce 16 bytes followed by AP Nonce 16 bytes) +- * +- * @NL80211_CMD_ASSOC_COMEBACK: notification about an association +- * temporal rejection with comeback. The event includes %NL80211_ATTR_MAC +- * to describe the BSSID address of the AP and %NL80211_ATTR_TIMEOUT to +- * specify the timeout value. - * * @NL80211_CMD_MAX: highest used command number * @__NL80211_CMD_AFTER_LAST: internal use */ -@@ -1472,8 +1443,6 @@ enum nl80211_commands { +@@ -1477,10 +1443,6 @@ enum nl80211_commands { NL80211_CMD_COLOR_CHANGE_ABORTED, NL80211_CMD_COLOR_CHANGE_COMPLETED, - NL80211_CMD_SET_FILS_AAD, +- +- NL80211_CMD_ASSOC_COMEBACK, - /* add new commands above here */ /* used to define NL80211_CMD_MAX below */ -@@ -2639,6 +2608,9 @@ enum nl80211_commands { - * Mandatory parameter for the transmitting interface to enable MBSSID. - * Optional for the non-transmitting interfaces. +@@ -2477,9 +2439,7 @@ enum nl80211_commands { + * space supports external authentication. This attribute shall be used + * with %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP request. The driver + * may offload authentication processing to user space if this capability +- * is indicated in the respective requests from the user space. (This flag +- * attribute deprecated for %NL80211_CMD_START_AP, use +- * %NL80211_ATTR_AP_SETTINGS_FLAGS) ++ * is indicated in the respective requests from the user space. * + * @NL80211_ATTR_NSS: Station's New/updated RX_NSS value notified using this + * u8 attribute. This is used with %NL80211_CMD_STA_OPMODE_CHANGED. +@@ -2655,13 +2615,8 @@ enum nl80211_commands { + * switching on a different channel during CAC detection on the selected + * radar channel. + * +- * @NL80211_ATTR_AP_SETTINGS_FLAGS: u32 attribute contains ap settings flags, +- * enumerated in &enum nl80211_ap_settings_flags. This attribute shall be +- * used with %NL80211_CMD_START_AP request. +- * +- * @NL80211_ATTR_EHT_CAPABILITY: EHT Capability information element (from +- * association request when used with NL80211_CMD_NEW_STATION). Can be set +- * only if %NL80211_STA_FLAG_WME is set. + * @NL80211_ATTR_WIPHY_ANTENNA_GAIN: Configured antenna gain. Used to reduce + * transmit power to stay within regulatory limits. u32, dBi. -+ * + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined - * @__NL80211_ATTR_AFTER_LAST: internal use -@@ -3145,6 +3117,8 @@ enum nl80211_attrs { - NL80211_ATTR_MBSSID_CONFIG, - NL80211_ATTR_MBSSID_ELEMS, +@@ -3171,11 +3126,7 @@ enum nl80211_attrs { + NL80211_ATTR_RADAR_BACKGROUND, + +- NL80211_ATTR_AP_SETTINGS_FLAGS, +- +- NL80211_ATTR_EHT_CAPABILITY, +- +- NL80211_ATTR_DISABLE_EHT, + NL80211_ATTR_WIPHY_ANTENNA_GAIN, -+ + /* add attributes here, update the policy in nl80211.c */ - __NL80211_ATTR_AFTER_LAST, -@@ -4978,7 +4952,6 @@ enum nl80211_txrate_gi { +@@ -3232,8 +3183,6 @@ enum nl80211_attrs { + #define NL80211_HE_MAX_CAPABILITY_LEN 54 + #define NL80211_MAX_NR_CIPHER_SUITES 5 + #define NL80211_MAX_NR_AKM_SUITES 2 +-#define NL80211_EHT_MIN_CAPABILITY_LEN 13 +-#define NL80211_EHT_MAX_CAPABILITY_LEN 51 + + #define NL80211_MIN_REMAIN_ON_CHANNEL_TIME 10 + +@@ -3261,7 +3210,7 @@ enum nl80211_attrs { + * and therefore can't be created in the normal ways, use the + * %NL80211_CMD_START_P2P_DEVICE and %NL80211_CMD_STOP_P2P_DEVICE + * commands to create and destroy one +- * @NL80211_IFTYPE_OCB: Outside Context of a BSS ++ * @NL80211_IF_TYPE_OCB: Outside Context of a BSS + * This mode corresponds to the MIB variable dot11OCBActivated=true + * @NL80211_IFTYPE_NAN: NAN device interface type (not a netdev) + * @NL80211_IFTYPE_MAX: highest interface type number currently defined +@@ -3403,56 +3352,6 @@ enum nl80211_he_ru_alloc { + }; + + /** +- * enum nl80211_eht_gi - EHT guard interval +- * @NL80211_RATE_INFO_EHT_GI_0_8: 0.8 usec +- * @NL80211_RATE_INFO_EHT_GI_1_6: 1.6 usec +- * @NL80211_RATE_INFO_EHT_GI_3_2: 3.2 usec +- */ +-enum nl80211_eht_gi { +- NL80211_RATE_INFO_EHT_GI_0_8, +- NL80211_RATE_INFO_EHT_GI_1_6, +- NL80211_RATE_INFO_EHT_GI_3_2, +-}; +- +-/** +- * enum nl80211_eht_ru_alloc - EHT RU allocation values +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_26: 26-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_52: 52-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_52P26: 52+26-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_106: 106-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_106P26: 106+26 tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_242: 242-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_484: 484-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_484P242: 484+242 tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_996: 996-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_996P484: 996+484 tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_996P484P242: 996+484+242 tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_2x996: 2x996-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_2x996P484: 2x996+484 tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_3x996: 3x996-tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_3x996P484: 3x996+484 tone RU allocation +- * @NL80211_RATE_INFO_EHT_RU_ALLOC_4x996: 4x996-tone RU allocation +- */ +-enum nl80211_eht_ru_alloc { +- NL80211_RATE_INFO_EHT_RU_ALLOC_26, +- NL80211_RATE_INFO_EHT_RU_ALLOC_52, +- NL80211_RATE_INFO_EHT_RU_ALLOC_52P26, +- NL80211_RATE_INFO_EHT_RU_ALLOC_106, +- NL80211_RATE_INFO_EHT_RU_ALLOC_106P26, +- NL80211_RATE_INFO_EHT_RU_ALLOC_242, +- NL80211_RATE_INFO_EHT_RU_ALLOC_484, +- NL80211_RATE_INFO_EHT_RU_ALLOC_484P242, +- NL80211_RATE_INFO_EHT_RU_ALLOC_996, +- NL80211_RATE_INFO_EHT_RU_ALLOC_996P484, +- NL80211_RATE_INFO_EHT_RU_ALLOC_996P484P242, +- NL80211_RATE_INFO_EHT_RU_ALLOC_2x996, +- NL80211_RATE_INFO_EHT_RU_ALLOC_2x996P484, +- NL80211_RATE_INFO_EHT_RU_ALLOC_3x996, +- NL80211_RATE_INFO_EHT_RU_ALLOC_3x996P484, +- NL80211_RATE_INFO_EHT_RU_ALLOC_4x996, +-}; +- +-/** + * enum nl80211_rate_info - bitrate information + * + * These attribute types are used with %NL80211_STA_INFO_TXRATE +@@ -3491,13 +3390,6 @@ enum nl80211_eht_ru_alloc { + * @NL80211_RATE_INFO_HE_DCM: HE DCM value (u8, 0/1) + * @NL80211_RATE_INFO_RU_ALLOC: HE RU allocation, if not present then + * non-OFDMA was used (u8, see &enum nl80211_he_ru_alloc) +- * @NL80211_RATE_INFO_320_MHZ_WIDTH: 320 MHz bitrate +- * @NL80211_RATE_INFO_EHT_MCS: EHT MCS index (u8, 0-15) +- * @NL80211_RATE_INFO_EHT_NSS: EHT NSS value (u8, 1-8) +- * @NL80211_RATE_INFO_EHT_GI: EHT guard interval identifier +- * (u8, see &enum nl80211_eht_gi) +- * @NL80211_RATE_INFO_EHT_RU_ALLOC: EHT RU allocation, if not present then +- * non-OFDMA was used (u8, see &enum nl80211_eht_ru_alloc) + * @__NL80211_RATE_INFO_AFTER_LAST: internal use + */ + enum nl80211_rate_info { +@@ -3519,11 +3411,6 @@ enum nl80211_rate_info { + NL80211_RATE_INFO_HE_GI, + NL80211_RATE_INFO_HE_DCM, + NL80211_RATE_INFO_HE_RU_ALLOC, +- NL80211_RATE_INFO_320_MHZ_WIDTH, +- NL80211_RATE_INFO_EHT_MCS, +- NL80211_RATE_INFO_EHT_NSS, +- NL80211_RATE_INFO_EHT_GI, +- NL80211_RATE_INFO_EHT_RU_ALLOC, + + /* keep last */ + __NL80211_RATE_INFO_AFTER_LAST, +@@ -3834,20 +3721,13 @@ enum nl80211_mpath_info { + * capabilities IE + * @NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE: HE PPE thresholds information as + * defined in HE capabilities IE ++ * @NL80211_BAND_IFTYPE_ATTR_MAX: highest band HE capability attribute currently ++ * defined + * @NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA: HE 6GHz band capabilities (__le16), + * given for all 6 GHz band channels + * @NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS: vendor element capabilities that are + * advertised on this band/for this iftype (binary) +- * @NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC: EHT MAC capabilities as in EHT +- * capabilities element +- * @NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY: EHT PHY capabilities as in EHT +- * capabilities element +- * @NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET: EHT supported NSS/MCS as in EHT +- * capabilities element +- * @NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE: EHT PPE thresholds information as +- * defined in EHT capabilities element + * @__NL80211_BAND_IFTYPE_ATTR_AFTER_LAST: internal use +- * @NL80211_BAND_IFTYPE_ATTR_MAX: highest band attribute currently defined + */ + enum nl80211_band_iftype_attr { + __NL80211_BAND_IFTYPE_ATTR_INVALID, +@@ -3859,10 +3739,6 @@ enum nl80211_band_iftype_attr { + NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE, + NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA, + NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS, +- NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC, +- NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY, +- NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET, +- NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE, + + /* keep last */ + __NL80211_BAND_IFTYPE_ATTR_AFTER_LAST, +@@ -4007,10 +3883,6 @@ enum nl80211_wmm_rule { + * on this channel in current regulatory domain. + * @NL80211_FREQUENCY_ATTR_16MHZ: 16 MHz operation is allowed + * on this channel in current regulatory domain. +- * @NL80211_FREQUENCY_ATTR_NO_320MHZ: any 320 MHz channel using this channel +- * as the primary or any of the secondary channels isn't possible +- * @NL80211_FREQUENCY_ATTR_NO_EHT: EHT operation is not allowed on this channel +- * in current regulatory domain. + * @NL80211_FREQUENCY_ATTR_MAX: highest frequency attribute number + * currently defined + * @__NL80211_FREQUENCY_ATTR_AFTER_LAST: internal use +@@ -4047,8 +3919,6 @@ enum nl80211_frequency_attr { + NL80211_FREQUENCY_ATTR_4MHZ, + NL80211_FREQUENCY_ATTR_8MHZ, + NL80211_FREQUENCY_ATTR_16MHZ, +- NL80211_FREQUENCY_ATTR_NO_320MHZ, +- NL80211_FREQUENCY_ATTR_NO_EHT, + + /* keep last */ + __NL80211_FREQUENCY_ATTR_AFTER_LAST, +@@ -4247,7 +4117,6 @@ enum nl80211_sched_scan_match_attr { + * @NL80211_RRF_NO_80MHZ: 80MHz operation not allowed + * @NL80211_RRF_NO_160MHZ: 160MHz operation not allowed + * @NL80211_RRF_NO_HE: HE operation not allowed +- * @NL80211_RRF_NO_320MHZ: 320MHz operation not allowed + */ + enum nl80211_reg_rule_flags { + NL80211_RRF_NO_OFDM = 1<<0, +@@ -4266,7 +4135,6 @@ enum nl80211_reg_rule_flags { + NL80211_RRF_NO_80MHZ = 1<<15, + NL80211_RRF_NO_160MHZ = 1<<16, + NL80211_RRF_NO_HE = 1<<17, +- NL80211_RRF_NO_320MHZ = 1<<18, + }; + + #define NL80211_RRF_PASSIVE_SCAN NL80211_RRF_NO_IR +@@ -4764,8 +4632,6 @@ enum nl80211_key_mode { + * @NL80211_CHAN_WIDTH_4: 4 MHz OFDM channel + * @NL80211_CHAN_WIDTH_8: 8 MHz OFDM channel + * @NL80211_CHAN_WIDTH_16: 16 MHz OFDM channel +- * @NL80211_CHAN_WIDTH_320: 320 MHz channel, the %NL80211_ATTR_CENTER_FREQ1 +- * attribute must be provided as well + */ + enum nl80211_chan_width { + NL80211_CHAN_WIDTH_20_NOHT, +@@ -4781,7 +4647,6 @@ enum nl80211_chan_width { + NL80211_CHAN_WIDTH_4, + NL80211_CHAN_WIDTH_8, + NL80211_CHAN_WIDTH_16, +- NL80211_CHAN_WIDTH_320, + }; + + /** +@@ -5096,7 +4961,6 @@ enum nl80211_txrate_gi { * @NL80211_BAND_60GHZ: around 60 GHz band (58.32 - 69.12 GHz) * @NL80211_BAND_6GHZ: around 6 GHz band (5.9 - 7.2 GHz) * @NL80211_BAND_S1GHZ: around 900MHz, supported by S1G PHYs @@ -79,7 +304,7 @@ * @NUM_NL80211_BANDS: number of bands, avoid using this in userspace * since newer kernel versions may support more bands */ -@@ -4988,7 +4961,6 @@ enum nl80211_band { +@@ -5106,7 +4970,6 @@ enum nl80211_band { NL80211_BAND_60GHZ, NL80211_BAND_6GHZ, NL80211_BAND_S1GHZ, @@ -87,7 +312,45 @@ NUM_NL80211_BANDS, }; -@@ -6046,11 +6018,6 @@ enum nl80211_feature_flags { +@@ -5673,7 +5536,7 @@ enum nl80211_iface_limit_attrs { + * => allows 8 of AP/GO that can have BI gcd >= min gcd + * + * numbers = [ #{STA} <= 2 ], channels = 2, max = 2 +- * => allows two STAs on the same or on different channels ++ * => allows two STAs on different channels + * + * numbers = [ #{STA} <= 1, #{P2P-client,P2P-GO} <= 3 ], max = 4 + * => allows a STA plus three P2P interfaces +@@ -5718,7 +5581,7 @@ enum nl80211_if_combination_attrs { + * @NL80211_PLINK_ESTAB: mesh peer link is established + * @NL80211_PLINK_HOLDING: mesh peer link is being closed or cancelled + * @NL80211_PLINK_BLOCKED: all frames transmitted from this mesh +- * plink are discarded, except for authentication frames ++ * plink are discarded + * @NUM_NL80211_PLINK_STATES: number of peer link states + * @MAX_NL80211_PLINK_STATES: highest numerical value of plink states + */ +@@ -5855,15 +5718,13 @@ enum nl80211_tdls_operation { + NL80211_TDLS_DISABLE_LINK, + }; + +-/** ++/* + * enum nl80211_ap_sme_features - device-integrated AP features +- * @NL80211_AP_SME_SA_QUERY_OFFLOAD: SA Query procedures offloaded to driver +- * when user space indicates support for SA Query procedures offload during +- * "start ap" with %NL80211_AP_SETTINGS_SA_QUERY_OFFLOAD_SUPPORT. +- */ ++ * Reserved for future use, no bits are defined in ++ * NL80211_ATTR_DEVICE_AP_SME yet. + enum nl80211_ap_sme_features { +- NL80211_AP_SME_SA_QUERY_OFFLOAD = 1 << 0, + }; ++ */ + + /** + * enum nl80211_feature_flags - device/driver features +@@ -6166,11 +6027,6 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_BSS_COLOR: The driver supports BSS color collision * detection and change announcemnts. * @@ -96,18 +359,18 @@ - * frames. Userspace has to share FILS AAD details to the driver by using - * @NL80211_CMD_SET_FILS_AAD. - * - * @NUM_NL80211_EXT_FEATURES: number of extended features. - * @MAX_NL80211_EXT_FEATURES: highest extended feature index. - */ -@@ -6116,7 +6083,6 @@ enum nl80211_ext_feature_index { + * @NL80211_EXT_FEATURE_RADAR_BACKGROUND: Device supports background radar/CAC + * detection. + * +@@ -6239,7 +6095,6 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_SECURE_RTT, NL80211_EXT_FEATURE_PROT_RANGE_NEGO_AND_MEASURE, NL80211_EXT_FEATURE_BSS_COLOR, - NL80211_EXT_FEATURE_FILS_CRYPTO_OFFLOAD, + NL80211_EXT_FEATURE_RADAR_BACKGROUND, /* add new features before the definition below */ - NUM_NL80211_EXT_FEATURES, -@@ -7424,7 +7390,7 @@ enum nl80211_sar_specs_attrs { +@@ -7548,7 +7403,7 @@ enum nl80211_sar_specs_attrs { * @NL80211_MBSSID_CONFIG_ATTR_MAX_EMA_PROFILE_PERIODICITY: Used by the kernel * to advertise the maximum profile periodicity supported by the driver * if EMA is enabled. Driver should indicate EMA support to the userspace @@ -116,7 +379,7 @@ * a non-zero value. * * @NL80211_MBSSID_CONFIG_ATTR_INDEX: Mandatory parameter to pass the index of -@@ -7443,7 +7409,7 @@ enum nl80211_sar_specs_attrs { +@@ -7567,7 +7422,7 @@ enum nl80211_sar_specs_attrs { * * @NL80211_MBSSID_CONFIG_ATTR_EMA: Flag used to enable EMA AP feature. * Setting this flag is permitted only if the driver advertises EMA support @@ -125,13 +388,167 @@ * * @__NL80211_MBSSID_CONFIG_ATTR_LAST: Internal * @NL80211_MBSSID_CONFIG_ATTR_MAX: highest attribute +@@ -7586,20 +7441,4 @@ enum nl80211_mbssid_config_attributes { + NL80211_MBSSID_CONFIG_ATTR_MAX = __NL80211_MBSSID_CONFIG_ATTR_LAST - 1, + }; + +-/** +- * enum nl80211_ap_settings_flags - AP settings flags +- * +- * @NL80211_AP_SETTINGS_EXTERNAL_AUTH_SUPPORT: AP supports external +- * authentication. +- * @NL80211_AP_SETTINGS_SA_QUERY_OFFLOAD_SUPPORT: Userspace supports SA Query +- * procedures offload to driver. If driver advertises +- * %NL80211_AP_SME_SA_QUERY_OFFLOAD in AP SME features, userspace shall +- * ignore SA Query procedures and validations when this flag is set by +- * userspace. +- */ +-enum nl80211_ap_settings_flags { +- NL80211_AP_SETTINGS_EXTERNAL_AUTH_SUPPORT = 1 << 0, +- NL80211_AP_SETTINGS_SA_QUERY_OFFLOAD_SUPPORT = 1 << 1, +-}; +- + #endif /* __LINUX_NL80211_H */ +--- a/event.c ++++ b/event.c +@@ -1292,9 +1292,6 @@ static int print_event(struct nl_msg *ms + case NL80211_CMD_CH_SWITCH_NOTIFY: + parse_ch_switch_notify(tb, gnlh->cmd); + break; +- case NL80211_CMD_ASSOC_COMEBACK: /* 147 */ +- parse_assoc_comeback(tb, gnlh->cmd); +- break; + default: + printf("unknown event %d (%s)\n", + gnlh->cmd, command_name(gnlh->cmd)); --- a/info.c +++ b/info.c -@@ -701,7 +701,6 @@ broken_combination: - ext_feat_print(tb, OPERATING_CHANNEL_VALIDATION, "Operating Channel Validation (OCV) support"); - ext_feat_print(tb, 4WAY_HANDSHAKE_AP_PSK, "AP mode PSK offload support"); - ext_feat_print(tb, BSS_COLOR, "BSS coloring support"); -- ext_feat_print(tb, FILS_CRYPTO_OFFLOAD, "FILS crypto offload"); +@@ -164,7 +164,6 @@ static void ext_feat_print(enum nl80211_ + ext_feat_case(PROT_RANGE_NEGO_AND_MEASURE, + "support for MFP in range measurement negotiation/procedure"); + ext_feat_case(BSS_COLOR, "BSS coloring support"); +- ext_feat_case(FILS_CRYPTO_OFFLOAD, "FILS crypto offload"); + ext_feat_case(RADAR_BACKGROUND, "Radar background support"); } + } +--- a/interface.c ++++ b/interface.c +@@ -362,8 +362,6 @@ char *channel_width_name(enum nl80211_ch + return "5 MHz"; + case NL80211_CHAN_WIDTH_10: + return "10 MHz"; +- case NL80211_CHAN_WIDTH_320: +- return "320 MHz"; + default: + return "unknown"; + } +--- a/util.c ++++ b/util.c +@@ -508,7 +508,6 @@ static int parse_freqs(struct chandef *c + case NL80211_CHAN_WIDTH_40: + case NL80211_CHAN_WIDTH_80: + case NL80211_CHAN_WIDTH_160: +- case NL80211_CHAN_WIDTH_320: + need_cf1 = true; + break; + case NL80211_CHAN_WIDTH_1: +@@ -626,10 +625,6 @@ int parse_freqchan(struct chandef *chand + .width = NL80211_CHAN_WIDTH_160, + .freq1_diff = 0, + .chantype = -1 }, +- { .name = "320MHz", +- .width = NL80211_CHAN_WIDTH_320, +- .freq1_diff = 0, +- .chantype = -1 }, + }; + const struct chanmode *chanmode_selected = NULL; + unsigned int freq; +@@ -1599,48 +1594,6 @@ void print_eht_info(struct nlattr *nl_if + print_iftype_line(tb[NL80211_BAND_IFTYPE_ATTR_IFTYPES]); + printf("\n"); - if (tb_msg[NL80211_ATTR_COALESCE_RULE]) { +- if (tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC]) { +- len = nla_len(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC]); +- if (len > sizeof(mac_cap)) +- len = sizeof(mac_cap); +- memcpy(mac_cap, +- nla_data(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC]), +- len); +- } +- +- if (tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY]) { +- len = nla_len(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY]); +- +- if (len > sizeof(phy_cap)) +- len = sizeof(phy_cap); +- +- memcpy(phy_cap, +- nla_data(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY]), +- len); +- } +- +- if (tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET]) { +- len = nla_len(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET]); +- if (len > sizeof(mcs_set)) +- len = sizeof(mcs_set); +- memcpy(mcs_set, +- nla_data(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET]), +- len); +- +- // Assume that all parts of the MCS set are present +- mcs_len = sizeof(mcs_set); +- } +- +- if (tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE]) { +- len = nla_len(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE]); +- if (len > sizeof(ppet)) +- len = sizeof(ppet); +- memcpy(ppet, +- nla_data(tb[NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE]), +- len); +- ppet_len = len; +- } +- + if (tb[NL80211_BAND_IFTYPE_ATTR_HE_CAP_PHY]) { + len = nla_len(tb[NL80211_BAND_IFTYPE_ATTR_HE_CAP_PHY]); + +--- a/reg.c ++++ b/reg.c +@@ -210,7 +210,6 @@ static int print_reg_handler(struct nl_m + PARSE_FLAG(NL80211_RRF_NO_80MHZ, "NO-80MHZ"); + PARSE_FLAG(NL80211_RRF_NO_160MHZ, "NO-160MHZ"); + PARSE_FLAG(NL80211_RRF_NO_HE, "NO-HE"); +- PARSE_FLAG(NL80211_RRF_NO_320MHZ, "NO-320MHZ"); + + /* Kernels that support NO_IR always turn on both flags */ + if ((flags & NL80211_RRF_NO_IR) && (flags & __NL80211_RRF_NO_IBSS)) { +--- a/station.c ++++ b/station.c +@@ -239,8 +239,6 @@ void parse_bitrate(struct nlattr *bitrat + pos += snprintf(pos, buflen - (pos - buf), " 80P80MHz"); + if (rinfo[NL80211_RATE_INFO_160_MHZ_WIDTH]) + pos += snprintf(pos, buflen - (pos - buf), " 160MHz"); +- if (rinfo[NL80211_RATE_INFO_320_MHZ_WIDTH]) +- pos += snprintf(pos, buflen - (pos - buf), " 320MHz"); + if (rinfo[NL80211_RATE_INFO_SHORT_GI]) + pos += snprintf(pos, buflen - (pos - buf), " short GI"); + if (rinfo[NL80211_RATE_INFO_VHT_NSS]) +@@ -261,18 +259,6 @@ void parse_bitrate(struct nlattr *bitrat + if (rinfo[NL80211_RATE_INFO_HE_RU_ALLOC]) + pos += snprintf(pos, buflen - (pos - buf), + " HE-RU-ALLOC %d", nla_get_u8(rinfo[NL80211_RATE_INFO_HE_RU_ALLOC])); +- if (rinfo[NL80211_RATE_INFO_EHT_MCS]) +- pos += snprintf(pos, buflen - (pos - buf), +- " EHT-MCS %d", nla_get_u8(rinfo[NL80211_RATE_INFO_EHT_MCS])); +- if (rinfo[NL80211_RATE_INFO_EHT_NSS]) +- pos += snprintf(pos, buflen - (pos - buf), +- " EHT-NSS %d", nla_get_u8(rinfo[NL80211_RATE_INFO_EHT_NSS])); +- if (rinfo[NL80211_RATE_INFO_EHT_GI]) +- pos += snprintf(pos, buflen - (pos - buf), +- " EHT-GI %d", nla_get_u8(rinfo[NL80211_RATE_INFO_EHT_GI])); +- if (rinfo[NL80211_RATE_INFO_EHT_RU_ALLOC]) +- pos += snprintf(pos, buflen - (pos - buf), +- " EHT-RU-ALLOC %d", nla_get_u8(rinfo[NL80211_RATE_INFO_EHT_RU_ALLOC])); + } + + static char *get_chain_signal(struct nlattr *attr_list) diff --git a/package/network/utils/iw/patches/200-reduce_size.patch b/package/network/utils/iw/patches/200-reduce_size.patch index d0e837c695..88df56b385 100644 --- a/package/network/utils/iw/patches/200-reduce_size.patch +++ b/package/network/utils/iw/patches/200-reduce_size.patch @@ -1,6 +1,6 @@ --- a/event.c +++ b/event.c -@@ -956,6 +956,7 @@ static int print_event(struct nl_msg *ms +@@ -971,6 +971,7 @@ static int print_event(struct nl_msg *ms } switch (gnlh->cmd) { @@ -8,7 +8,7 @@ case NL80211_CMD_NEW_WIPHY: printf("renamed to %s\n", nla_get_string(tb[NL80211_ATTR_WIPHY_NAME])); break; -@@ -991,6 +992,7 @@ static int print_event(struct nl_msg *ms +@@ -1006,6 +1007,7 @@ static int print_event(struct nl_msg *ms case NL80211_CMD_SCHED_SCAN_RESULTS: printf("got scheduled scan results\n"); break; @@ -16,7 +16,7 @@ case NL80211_CMD_WIPHY_REG_CHANGE: case NL80211_CMD_REG_CHANGE: if (gnlh->cmd == NL80211_CMD_WIPHY_REG_CHANGE) -@@ -1073,6 +1075,7 @@ static int print_event(struct nl_msg *ms +@@ -1088,6 +1090,7 @@ static int print_event(struct nl_msg *ms mac_addr_n2a(macbuf, nla_data(tb[NL80211_ATTR_MAC])); printf("del station %s\n", macbuf); break; @@ -24,7 +24,7 @@ case NL80211_CMD_JOIN_IBSS: mac_addr_n2a(macbuf, nla_data(tb[NL80211_ATTR_MAC])); printf("IBSS %s joined\n", macbuf); -@@ -1271,9 +1274,9 @@ static int print_event(struct nl_msg *ms +@@ -1292,9 +1295,9 @@ static int print_event(struct nl_msg *ms case NL80211_CMD_CH_SWITCH_NOTIFY: parse_ch_switch_notify(tb, gnlh->cmd); break; @@ -38,7 +38,7 @@ --- a/info.c +++ b/info.c -@@ -215,6 +215,7 @@ next: +@@ -308,6 +308,7 @@ next: } } @@ -46,7 +46,7 @@ if (tb_band[NL80211_BAND_ATTR_RATES]) { printf("\t\tBitrates (non-HT):\n"); nla_for_each_nested(nl_rate, tb_band[NL80211_BAND_ATTR_RATES], rem_rate) { -@@ -231,6 +232,7 @@ next: +@@ -324,6 +325,7 @@ next: printf("\n"); } } @@ -54,7 +54,7 @@ } } -@@ -296,6 +298,7 @@ next: +@@ -389,6 +391,7 @@ next: printf("\tCoverage class: %d (up to %dm)\n", coverage, 450 * coverage); } @@ -62,7 +62,7 @@ if (tb_msg[NL80211_ATTR_CIPHER_SUITES]) { int num = nla_len(tb_msg[NL80211_ATTR_CIPHER_SUITES]) / sizeof(__u32); int i; -@@ -307,6 +310,7 @@ next: +@@ -400,6 +403,7 @@ next: cipher_name(ciphers[i])); } } @@ -70,7 +70,7 @@ if (tb_msg[NL80211_ATTR_WIPHY_ANTENNA_AVAIL_TX] && tb_msg[NL80211_ATTR_WIPHY_ANTENNA_AVAIL_RX]) -@@ -324,9 +328,11 @@ next: +@@ -417,9 +421,11 @@ next: print_iftype_list("\tSupported interface modes", "\t\t", tb_msg[NL80211_ATTR_SUPPORTED_IFTYPES]); @@ -82,7 +82,7 @@ if (tb_msg[NL80211_ATTR_INTERFACE_COMBINATIONS]) { struct nlattr *nl_combi; -@@ -416,6 +422,7 @@ broken_combination: +@@ -509,6 +515,7 @@ broken_combination: printf("\tinterface combinations are not supported\n"); } @@ -90,7 +90,7 @@ if (tb_msg[NL80211_ATTR_SUPPORTED_COMMANDS]) { printf("\tSupported commands:\n"); nla_for_each_nested(nl_cmd, tb_msg[NL80211_ATTR_SUPPORTED_COMMANDS], rem_cmd) -@@ -513,6 +520,7 @@ broken_combination: +@@ -606,6 +613,7 @@ broken_combination: printf("\t\t * wake up on TCP connection\n"); } } @@ -98,7 +98,7 @@ if (tb_msg[NL80211_ATTR_ROAM_SUPPORT]) printf("\tDevice supports roaming.\n"); -@@ -551,6 +559,7 @@ broken_combination: +@@ -644,6 +652,7 @@ broken_combination: } } @@ -106,7 +106,7 @@ if (tb_msg[NL80211_ATTR_FEATURE_FLAGS]) { unsigned int features = nla_get_u32(tb_msg[NL80211_ATTR_FEATURE_FLAGS]); -@@ -615,6 +624,7 @@ broken_combination: +@@ -708,6 +717,7 @@ broken_combination: if (features & NL80211_FEATURE_ND_RANDOM_MAC_ADDR) printf("\tDevice supports randomizing MAC-addr in net-detect scans.\n"); } @@ -114,7 +114,7 @@ if (tb_msg[NL80211_ATTR_TDLS_SUPPORT]) printf("\tDevice supports T-DLS.\n"); -@@ -751,6 +761,7 @@ TOPLEVEL(list, NULL, NL80211_CMD_GET_WIP +@@ -773,6 +783,7 @@ TOPLEVEL(list, NULL, NL80211_CMD_GET_WIP "List all wireless devices and their capabilities."); TOPLEVEL(phy, NULL, NL80211_CMD_GET_WIPHY, NLM_F_DUMP, CIB_NONE, handle_info, NULL); @@ -122,7 +122,7 @@ static int handle_commands(struct nl80211_state *state, struct nl_msg *msg, int argc, char **argv, enum id_input id) { -@@ -762,6 +773,7 @@ static int handle_commands(struct nl8021 +@@ -784,6 +795,7 @@ static int handle_commands(struct nl8021 } TOPLEVEL(commands, NULL, NL80211_CMD_GET_WIPHY, 0, CIB_NONE, handle_commands, "list all known commands and their decimal & hex value"); diff --git a/target/linux/ath79/generic/base-files/etc/board.d/01_leds b/target/linux/ath79/generic/base-files/etc/board.d/01_leds index 24beb7a361..93438b1498 100644 --- a/target/linux/ath79/generic/base-files/etc/board.d/01_leds +++ b/target/linux/ath79/generic/base-files/etc/board.d/01_leds @@ -436,19 +436,11 @@ ubnt,powerbeam-5ac-gen2) ucidef_set_led_rssi "rssimediumhigh" "RSSIMEDIUMHIGH" "blue:rssi2" "wlan0" "51" "100" ucidef_set_led_rssi "rssihigh" "RSSIHIGH" "blue:rssi3" "wlan0" "76" "100" ;; -ubnt,bullet-m-ar7240|\ -ubnt,bullet-m-ar7241|\ ubnt,bullet-m-xw|\ -ubnt,nanobridge-m|\ -ubnt,nanostation-loco-m|\ ubnt,nanostation-loco-m-xw|\ -ubnt,nanostation-m|\ ubnt,nanostation-m-xw|\ -ubnt,picostation-m|\ ubnt,powerbeam-m2-xw|\ -ubnt,powerbeam-m5-xw|\ -ubnt,powerbridge-m|\ -ubnt,rocket-m) +ubnt,powerbeam-m5-xw) ucidef_set_rssimon "wlan0" "200000" "1" ucidef_set_led_rssi "rssilow" "RSSILOW" "red:link1" "wlan0" "1" "100" ucidef_set_led_rssi "rssimediumlow" "RSSIMEDIUMLOW" "orange:link2" "wlan0" "26" "100" diff --git a/target/linux/ath79/generic/base-files/etc/board.d/02_network b/target/linux/ath79/generic/base-files/etc/board.d/02_network index 48cd81fb21..d4e956c406 100644 --- a/target/linux/ath79/generic/base-files/etc/board.d/02_network +++ b/target/linux/ath79/generic/base-files/etc/board.d/02_network @@ -88,25 +88,18 @@ ath79_setup_interfaces() tplink,tl-wa1201-v2|\ tplink,tl-wr902ac-v1|\ ubnt,bullet-ac|\ - ubnt,bullet-m-ar7240|\ - ubnt,bullet-m-ar7241|\ ubnt,bullet-m-xw|\ ubnt,lap-120|\ ubnt,litebeam-ac-gen2|\ ubnt,nanobeam-ac|\ ubnt,nanobeam-ac-xc|\ - ubnt,nanobridge-m|\ ubnt,nanostation-ac-loco|\ - ubnt,nanostation-loco-m|\ ubnt,nanostation-loco-m-xw|\ - ubnt,picostation-m|\ ubnt,powerbeam-5ac-500|\ ubnt,powerbeam-5ac-gen2|\ ubnt,powerbeam-m2-xw|\ ubnt,powerbeam-m5-xw|\ - ubnt,powerbridge-m|\ ubnt,rocket-5ac-lite|\ - ubnt,rocket-m|\ ubnt,unifiac-lite|\ ubnt,unifiac-lr|\ ubnt,unifiac-mesh|\ @@ -176,8 +169,7 @@ ath79_setup_interfaces() tplink,archer-c60-v3|\ tplink,tl-wdr3500-v1|\ tplink,tl-wr842n-v1|\ - tplink,tl-wr842n-v3|\ - ubnt,airrouter) + tplink,tl-wr842n-v3) ucidef_set_interface_wan "eth1" ucidef_add_switch "switch0" \ "0@eth0" "1:lan:4" "2:lan:3" "3:lan:2" "4:lan:1" @@ -212,7 +204,6 @@ ath79_setup_interfaces() tplink,wbs210-v2|\ tplink,wbs510-v1|\ tplink,wbs510-v2|\ - ubnt,nanostation-m|\ ubnt,routerstation) ucidef_set_interfaces_lan_wan "eth1" "eth0" ;; @@ -717,15 +708,6 @@ ath79_setup_macs() wan_mac=$(mtd_get_mac_text mac 0x18) label_mac=$wan_mac ;; - ubnt,airrouter|\ - ubnt,bullet-m-ar7240|\ - ubnt,bullet-m-ar7241|\ - ubnt,nanobridge-m|\ - ubnt,nanostation-loco-m|\ - ubnt,nanostation-m|\ - ubnt,picostation-m|\ - ubnt,powerbridge-m|\ - ubnt,rocket-m|\ ubnt,unifi) label_mac=$(cat /sys/class/ieee80211/phy0/macaddress) ;; diff --git a/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom b/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom index a19d2e95dc..8631984ee8 100644 --- a/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom +++ b/target/linux/ath79/generic/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom @@ -115,16 +115,7 @@ case "$FIRMWARE" in netgear,wnr2200-16m|\ pcs,cap324|\ tplink,tl-wr2543-v1|\ - tplink,tl-wr842n-v1|\ - ubnt,airrouter|\ - ubnt,bullet-m-ar7240|\ - ubnt,bullet-m-ar7241|\ - ubnt,nanobridge-m|\ - ubnt,nanostation-loco-m|\ - ubnt,nanostation-m|\ - ubnt,picostation-m|\ - ubnt,powerbridge-m|\ - ubnt,rocket-m) + tplink,tl-wr842n-v1) caldata_extract "art" 0x1000 0x1000 ;; openmesh,mr600-v1|\ diff --git a/target/linux/ath79/image/Makefile b/target/linux/ath79/image/Makefile index 768ec57044..2a7a0abd50 100644 --- a/target/linux/ath79/image/Makefile +++ b/target/linux/ath79/image/Makefile @@ -105,6 +105,7 @@ endif ifeq ($(SUBTARGET),tiny) include tiny-netgear.mk include tiny-tp-link.mk +include tiny-ubnt.mk endif $(eval $(call BuildImage)) diff --git a/target/linux/ath79/image/common-ubnt.mk b/target/linux/ath79/image/common-ubnt.mk new file mode 100644 index 0000000000..625f0b4f3f --- /dev/null +++ b/target/linux/ath79/image/common-ubnt.mk @@ -0,0 +1,142 @@ +DEVICE_VARS += UBNT_BOARD UBNT_CHIP UBNT_TYPE UBNT_VERSION UBNT_REVISION + +# On M (XW) devices the U-Boot as of version 1.1.4-s1039 doesn't like +# VERSION_DIST being on the place of major(?) version number, so we need to +# use some number. +UBNT_REVISION := $(VERSION_DIST)-$(REVISION) + +# mkubntimage is using the kernel image direct +# routerboard creates partitions out of the ubnt header +define Build/mkubntimage + -$(STAGING_DIR_HOST)/bin/mkfwimage -B $(UBNT_BOARD) \ + -v $(UBNT_TYPE).$(UBNT_CHIP).v6.0.0-$(VERSION_DIST)-$(REVISION) \ + -k $(IMAGE_KERNEL) -r $@ -o $@ +endef + +define Build/mkubntimage2 + -$(STAGING_DIR_HOST)/bin/mkfwimage2 -f 0x9f000000 \ + -v $(UBNT_TYPE).$(UBNT_CHIP).v6.0.0-$(VERSION_DIST)-$(REVISION) \ + -p jffs2:0x50000:0xf60000:0:0:$@ \ + -o $@.new + @mv $@.new $@ +endef + +# all UBNT XM/WA devices expect the kernel image to have 1024k while flash, when +# booting the image, the size doesn't matter. +define Build/mkubntimage-split + -[ -f $@ ] && ( \ + dd if=$@ of=$@.old1 bs=1024k count=1; \ + dd if=$@ of=$@.old2 bs=1024k skip=1; \ + $(STAGING_DIR_HOST)/bin/mkfwimage -B $(UBNT_BOARD) \ + -v $(UBNT_TYPE).$(UBNT_CHIP).v$(UBNT_VERSION)-$(UBNT_REVISION) \ + -k $@.old1 -r $@.old2 -o $@; \ + rm $@.old1 $@.old2 ) +endef + +# UBNT_BOARD e.g. one of (XS2, XS5, RS, XM) +# UBNT_TYPE e.g. one of (BZ, XM, XW) +# UBNT_CHIP e.g. one of (ar7240, ar933x, ar934x) +# UBNT_VERSION e.g. one of (6.0.0, 8.5.3) +define Device/ubnt + DEVICE_VENDOR := Ubiquiti + DEVICE_PACKAGES := kmod-usb2 + IMAGES += factory.bin + IMAGE/factory.bin := append-kernel | pad-to $$$$(BLOCKSIZE) | \ + append-rootfs | pad-rootfs | check-size | mkubntimage-split +endef + +define Device/ubnt-bz + $(Device/ubnt) + SOC := ar7241 + IMAGE_SIZE := 7448k + UBNT_BOARD := XM + UBNT_CHIP := ar7240 + UBNT_TYPE := BZ + UBNT_VERSION := 6.0.0 +endef + +define Device/ubnt-sw + $(Device/ubnt) + SOC := ar7242 + DEVICE_PACKAGES += kmod-usb-ohci + IMAGE_SIZE := 7552k + UBNT_BOARD := SW + UBNT_CHIP := ar7240 + UBNT_TYPE := SW + UBNT_VERSION := 1.4.1 + KERNEL := kernel-bin | append-dtb | relocate-kernel | lzma | uImage lzma +endef + +define Device/ubnt-2wa + $(Device/ubnt) + SOC := ar9342 + IMAGE_SIZE := 15744k + UBNT_BOARD := WA + UBNT_CHIP := ar934x + UBNT_TYPE := 2WA + UBNT_VERSION := 8.5.3 +endef + +define Device/ubnt-wa + $(Device/ubnt) + SOC := ar9342 + IMAGE_SIZE := 15744k + UBNT_BOARD := WA + UBNT_CHIP := ar934x + UBNT_TYPE := WA + UBNT_VERSION := 8.5.3 +endef + +define Device/ubnt-xc + $(Device/ubnt) + IMAGE_SIZE := 15744k + UBNT_BOARD := XC + UBNT_CHIP := qca955x + UBNT_TYPE := XC + UBNT_VERSION := 8.5.3 +endef + +define Device/ubnt-xm + $(Device/ubnt) + DEVICE_VARIANT := XM + DEVICE_PACKAGES += kmod-usb-ohci + IMAGE_SIZE := 7448k + UBNT_BOARD := XM + UBNT_CHIP := ar7240 + UBNT_REVISION := 42.$(UBNT_REVISION) + UBNT_TYPE := XM + UBNT_VERSION := 6.0.0 + KERNEL := kernel-bin | append-dtb | relocate-kernel | lzma | uImage lzma +endef + +define Device/ubnt-xw + $(Device/ubnt) + SOC := ar9342 + DEVICE_VARIANT := XW + IMAGE_SIZE := 7552k + UBNT_BOARD := XM + UBNT_CHIP := ar934x + UBNT_REVISION := 42.$(UBNT_REVISION) + UBNT_TYPE := XW + UBNT_VERSION := 6.0.4 +endef + +define Device/ubnt-unifi-jffs2 + $(Device/ubnt) + KERNEL_SIZE := 3072k + IMAGE_SIZE := 15744k + UBNT_TYPE := BZ + KERNEL := kernel-bin | append-dtb | lzma | uImage lzma | jffs2 kernel0 + IMAGES := sysupgrade.bin factory.bin + IMAGE/sysupgrade.bin := append-kernel | pad-to $$$$(KERNEL_SIZE) | append-rootfs |\ + pad-rootfs | check-size | append-metadata + IMAGE/factory.bin := $$(IMAGE/sysupgrade.bin) | mkubntimage2 +endef + +define Device/ubnt-acb + $(Device/ubnt) + IMAGE_SIZE := 15744k + UBNT_BOARD := ACB + UBNT_TYPE := ACB + UBNT_VERSION := 2.5.0 +endef diff --git a/target/linux/ath79/image/generic-ubnt.mk b/target/linux/ath79/image/generic-ubnt.mk index 2381d92567..5259a339df 100644 --- a/target/linux/ath79/image/generic-ubnt.mk +++ b/target/linux/ath79/image/generic-ubnt.mk @@ -1,145 +1,4 @@ -DEVICE_VARS += UBNT_BOARD UBNT_CHIP UBNT_TYPE UBNT_VERSION UBNT_REVISION - -# On M (XW) devices the U-Boot as of version 1.1.4-s1039 doesn't like -# VERSION_DIST being on the place of major(?) version number, so we need to -# use some number. -UBNT_REVISION := $(VERSION_DIST)-$(REVISION) - -# mkubntimage is using the kernel image direct -# routerboard creates partitions out of the ubnt header -define Build/mkubntimage - -$(STAGING_DIR_HOST)/bin/mkfwimage -B $(UBNT_BOARD) \ - -v $(UBNT_TYPE).$(UBNT_CHIP).v6.0.0-$(VERSION_DIST)-$(REVISION) \ - -k $(IMAGE_KERNEL) -r $@ -o $@ -endef - -define Build/mkubntimage2 - -$(STAGING_DIR_HOST)/bin/mkfwimage2 -f 0x9f000000 \ - -v $(UBNT_TYPE).$(UBNT_CHIP).v6.0.0-$(VERSION_DIST)-$(REVISION) \ - -p jffs2:0x50000:0xf60000:0:0:$@ \ - -o $@.new - @mv $@.new $@ -endef - -# all UBNT XM/WA devices expect the kernel image to have 1024k while flash, when -# booting the image, the size doesn't matter. -define Build/mkubntimage-split - -[ -f $@ ] && ( \ - dd if=$@ of=$@.old1 bs=1024k count=1; \ - dd if=$@ of=$@.old2 bs=1024k skip=1; \ - $(STAGING_DIR_HOST)/bin/mkfwimage -B $(UBNT_BOARD) \ - -v $(UBNT_TYPE).$(UBNT_CHIP).v$(UBNT_VERSION)-$(UBNT_REVISION) \ - -k $@.old1 -r $@.old2 -o $@; \ - rm $@.old1 $@.old2 ) -endef - -# UBNT_BOARD e.g. one of (XS2, XS5, RS, XM) -# UBNT_TYPE e.g. one of (BZ, XM, XW) -# UBNT_CHIP e.g. one of (ar7240, ar933x, ar934x) -# UBNT_VERSION e.g. one of (6.0.0, 8.5.3) -define Device/ubnt - DEVICE_VENDOR := Ubiquiti - DEVICE_PACKAGES := kmod-usb2 - IMAGES += factory.bin - IMAGE/factory.bin := append-kernel | pad-to $$$$(BLOCKSIZE) | \ - append-rootfs | pad-rootfs | check-size | mkubntimage-split -endef - -define Device/ubnt-bz - $(Device/ubnt) - SOC := ar7241 - IMAGE_SIZE := 7448k - UBNT_BOARD := XM - UBNT_CHIP := ar7240 - UBNT_TYPE := BZ - UBNT_VERSION := 6.0.0 -endef - -define Device/ubnt-sw - $(Device/ubnt) - SOC := ar7242 - DEVICE_PACKAGES += kmod-usb-ohci - IMAGE_SIZE := 7552k - UBNT_BOARD := SW - UBNT_CHIP := ar7240 - UBNT_TYPE := SW - UBNT_VERSION := 1.4.1 - KERNEL := kernel-bin | append-dtb | relocate-kernel | lzma | uImage lzma -endef - -define Device/ubnt-2wa - $(Device/ubnt) - SOC := ar9342 - IMAGE_SIZE := 15744k - UBNT_BOARD := WA - UBNT_CHIP := ar934x - UBNT_TYPE := 2WA - UBNT_VERSION := 8.5.3 -endef - -define Device/ubnt-wa - $(Device/ubnt) - SOC := ar9342 - IMAGE_SIZE := 15744k - UBNT_BOARD := WA - UBNT_CHIP := ar934x - UBNT_TYPE := WA - UBNT_VERSION := 8.5.3 -endef - -define Device/ubnt-xc - $(Device/ubnt) - IMAGE_SIZE := 15744k - UBNT_BOARD := XC - UBNT_CHIP := qca955x - UBNT_TYPE := XC - UBNT_VERSION := 8.5.3 -endef - -define Device/ubnt-xm - $(Device/ubnt) - DEVICE_VARIANT := XM - DEVICE_PACKAGES += kmod-usb-ohci - IMAGE_SIZE := 7448k - UBNT_BOARD := XM - UBNT_CHIP := ar7240 - UBNT_REVISION := 42.$(UBNT_REVISION) - UBNT_TYPE := XM - UBNT_VERSION := 6.0.0 - KERNEL := kernel-bin | append-dtb | relocate-kernel | lzma | uImage lzma -endef - -define Device/ubnt-xw - $(Device/ubnt) - SOC := ar9342 - DEVICE_VARIANT := XW - IMAGE_SIZE := 7552k - UBNT_BOARD := XM - UBNT_CHIP := ar934x - UBNT_REVISION := 42.$(UBNT_REVISION) - UBNT_TYPE := XW - UBNT_VERSION := 6.0.4 -endef - -define Device/ubnt-unifi-jffs2 - $(Device/ubnt) - KERNEL_SIZE := 3072k - IMAGE_SIZE := 15744k - UBNT_TYPE := BZ - KERNEL := kernel-bin | append-dtb | lzma | uImage lzma | jffs2 kernel0 - IMAGES := sysupgrade.bin factory.bin - IMAGE/sysupgrade.bin := append-kernel | pad-to $$$$(KERNEL_SIZE) | append-rootfs |\ - pad-rootfs | check-size | append-metadata - IMAGE/factory.bin := $$(IMAGE/sysupgrade.bin) | mkubntimage2 -endef - -define Device/ubnt-acb - $(Device/ubnt) - IMAGE_SIZE := 15744k - UBNT_BOARD := ACB - UBNT_TYPE := ACB - UBNT_VERSION := 2.5.0 -endef +include ./common-ubnt.mk define Device/ubnt_aircube-ac $(Device/ubnt-acb) @@ -159,14 +18,6 @@ define Device/ubnt_aircube-isp endef TARGET_DEVICES += ubnt_aircube-isp -define Device/ubnt_airrouter - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := AirRouter - SUPPORTED_DEVICES += airrouter -endef -TARGET_DEVICES += ubnt_airrouter - define Device/ubnt_bullet-ac $(Device/ubnt-2wa) DEVICE_MODEL := Bullet AC @@ -174,26 +25,6 @@ define Device/ubnt_bullet-ac endef TARGET_DEVICES += ubnt_bullet-ac -define Device/ubnt_bullet-m-ar7240 - $(Device/ubnt-xm) - SOC := ar7240 - DEVICE_MODEL := Bullet M - DEVICE_VARIANT := XM (AR7240) - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += bullet-m -endef -TARGET_DEVICES += ubnt_bullet-m-ar7240 - -define Device/ubnt_bullet-m-ar7241 - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := Bullet M - DEVICE_VARIANT := XM (AR7241) - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += bullet-m ubnt,bullet-m -endef -TARGET_DEVICES += ubnt_bullet-m-ar7241 - define Device/ubnt_bullet-m-xw $(Device/ubnt-xw) DEVICE_MODEL := Bullet M @@ -263,15 +94,6 @@ define Device/ubnt_nanobeam-m5-xw endef TARGET_DEVICES += ubnt_nanobeam-m5-xw -define Device/ubnt_nanobridge-m - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := NanoBridge M - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += bullet-m -endef -TARGET_DEVICES += ubnt_nanobridge-m - define Device/ubnt_nanostation-ac $(Device/ubnt-wa) DEVICE_MODEL := Nanostation AC @@ -286,15 +108,6 @@ define Device/ubnt_nanostation-ac-loco endef TARGET_DEVICES += ubnt_nanostation-ac-loco -define Device/ubnt_nanostation-loco-m - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := Nanostation Loco M - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += bullet-m -endef -TARGET_DEVICES += ubnt_nanostation-loco-m - define Device/ubnt_nanostation-loco-m-xw $(Device/ubnt-xw) DEVICE_MODEL := Nanostation Loco M @@ -303,15 +116,6 @@ define Device/ubnt_nanostation-loco-m-xw endef TARGET_DEVICES += ubnt_nanostation-loco-m-xw -define Device/ubnt_nanostation-m - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := Nanostation M - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += nanostation-m -endef -TARGET_DEVICES += ubnt_nanostation-m - define Device/ubnt_nanostation-m-xw $(Device/ubnt-xw) DEVICE_MODEL := Nanostation M @@ -320,15 +124,6 @@ define Device/ubnt_nanostation-m-xw endef TARGET_DEVICES += ubnt_nanostation-m-xw -define Device/ubnt_picostation-m - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := Picostation M - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += bullet-m -endef -TARGET_DEVICES += ubnt_picostation-m - define Device/ubnt_powerbeam-5ac-500 $(Device/ubnt-xc) SOC := qca9558 @@ -362,15 +157,6 @@ define Device/ubnt_powerbeam-m5-xw endef TARGET_DEVICES += ubnt_powerbeam-m5-xw -define Device/ubnt_powerbridge-m - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := PowerBridge M - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += bullet-m -endef -TARGET_DEVICES += ubnt_powerbridge-m - define Device/ubnt_rocket-5ac-lite $(Device/ubnt-xc) SOC := qca9558 @@ -380,15 +166,6 @@ define Device/ubnt_rocket-5ac-lite endef TARGET_DEVICES += ubnt_rocket-5ac-lite -define Device/ubnt_rocket-m - $(Device/ubnt-xm) - SOC := ar7241 - DEVICE_MODEL := Rocket M - DEVICE_PACKAGES += rssileds - SUPPORTED_DEVICES += rocket-m -endef -TARGET_DEVICES += ubnt_rocket-m - define Device/ubnt_routerstation_common DEVICE_PACKAGES := -kmod-ath9k -wpad-basic-wolfssl -uboot-envtools kmod-usb-ohci \ kmod-usb2 fconfig diff --git a/target/linux/ath79/image/generic.mk b/target/linux/ath79/image/generic.mk index 2fa68ef3cf..7deaefda0a 100644 --- a/target/linux/ath79/image/generic.mk +++ b/target/linux/ath79/image/generic.mk @@ -3,6 +3,7 @@ include ./common-netgear.mk include ./common-senao.mk include ./common-tp-link.mk include ./common-yuncore.mk +include ./common-ubnt.mk DEVICE_VARS += ADDPATTERN_ID ADDPATTERN_VERSION DEVICE_VARS += SEAMA_SIGNATURE SEAMA_MTDBLOCK @@ -896,7 +897,7 @@ define Device/dlink_dap-2680-a1 DEVICE_VENDOR := D-Link DEVICE_MODEL := DAP-2680 DEVICE_VARIANT := A1 - DEVICE_PACKAGES := ath10k-firmware-qca99x0-ct kmod-ath10k-ct + DEVICE_PACKAGES := ath10k-firmware-qca9984-ct kmod-ath10k-ct IMAGE_SIZE := 15232k DAP_SIGNATURE := wapac36_dkbs_dap2680 endef diff --git a/target/linux/ath79/image/tiny-ubnt.mk b/target/linux/ath79/image/tiny-ubnt.mk new file mode 100644 index 0000000000..805d193d45 --- /dev/null +++ b/target/linux/ath79/image/tiny-ubnt.mk @@ -0,0 +1,83 @@ +include ./common-ubnt.mk + +define Device/ubnt_airrouter + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := AirRouter + SUPPORTED_DEVICES += airrouter +endef +TARGET_DEVICES += ubnt_airrouter + +define Device/ubnt_nanobridge-m + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := NanoBridge M + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += bullet-m +endef +TARGET_DEVICES += ubnt_nanobridge-m + +define Device/ubnt_bullet-m-ar7240 + $(Device/ubnt-xm) + SOC := ar7240 + DEVICE_MODEL := Bullet M + DEVICE_VARIANT := XM (AR7240) + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += bullet-m +endef +TARGET_DEVICES += ubnt_bullet-m-ar7240 + +define Device/ubnt_bullet-m-ar7241 + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := Bullet M + DEVICE_VARIANT := XM (AR7241) + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += bullet-m ubnt,bullet-m +endef +TARGET_DEVICES += ubnt_bullet-m-ar7241 + +define Device/ubnt_picostation-m + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := Picostation M + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += bullet-m +endef +TARGET_DEVICES += ubnt_picostation-m + +define Device/ubnt_nanostation-m + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := Nanostation M + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += nanostation-m +endef +TARGET_DEVICES += ubnt_nanostation-m + +define Device/ubnt_nanostation-loco-m + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := Nanostation Loco M + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += bullet-m +endef +TARGET_DEVICES += ubnt_nanostation-loco-m + +define Device/ubnt_powerbridge-m + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := PowerBridge M + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += bullet-m +endef +TARGET_DEVICES += ubnt_powerbridge-m + +define Device/ubnt_rocket-m + $(Device/ubnt-xm) + SOC := ar7241 + DEVICE_MODEL := Rocket M + DEVICE_PACKAGES += rssileds + SUPPORTED_DEVICES += rocket-m +endef +TARGET_DEVICES += ubnt_rocket-m diff --git a/target/linux/ath79/tiny/base-files/etc/board.d/01_leds b/target/linux/ath79/tiny/base-files/etc/board.d/01_leds index f705c5c718..7ba2c2c897 100644 --- a/target/linux/ath79/tiny/base-files/etc/board.d/01_leds +++ b/target/linux/ath79/tiny/base-files/etc/board.d/01_leds @@ -110,6 +110,20 @@ tplink,tl-wr941nd-v6) tplink,tl-wr940n-v6) ucidef_set_led_netdev "wan" "WAN" "blue:wan" "eth1" ;; +ubnt,bullet-m-ar7240|\ +ubnt,bullet-m-ar7241|\ +ubnt,nanobridge-m|\ +ubnt,nanostation-loco-m|\ +ubnt,nanostation-m|\ +ubnt,picostation-m|\ +ubnt,powerbridge-m|\ +ubnt,rocket-m) + ucidef_set_rssimon "wlan0" "200000" "1" + ucidef_set_led_rssi "rssilow" "RSSILOW" "red:link1" "wlan0" "1" "100" + ucidef_set_led_rssi "rssimediumlow" "RSSIMEDIUMLOW" "orange:link2" "wlan0" "26" "100" + ucidef_set_led_rssi "rssimediumhigh" "RSSIMEDIUMHIGH" "green:link3" "wlan0" "51" "100" + ucidef_set_led_rssi "rssihigh" "RSSIHIGH" "green:link4" "wlan0" "76" "100" + ;; esac board_config_flush diff --git a/target/linux/ath79/tiny/base-files/etc/board.d/02_network b/target/linux/ath79/tiny/base-files/etc/board.d/02_network index 77b2444819..664359ce9a 100644 --- a/target/linux/ath79/tiny/base-files/etc/board.d/02_network +++ b/target/linux/ath79/tiny/base-files/etc/board.d/02_network @@ -49,7 +49,14 @@ ath79_setup_interfaces() tplink,tl-wa901nd-v5|\ tplink,tl-wr703n|\ tplink,tl-wr802n-v1|\ - tplink,tl-wr802n-v2) + tplink,tl-wr802n-v2|\ + ubnt,bullet-m-ar7240|\ + ubnt,bullet-m-ar7241|\ + ubnt,nanobridge-m|\ + ubnt,picostation-m|\ + ubnt,nanostation-loco-m|\ + ubnt,powerbridge-m|\ + ubnt,rocket-m) ucidef_set_interface_lan "eth0" ;; engenius,enh202-v1) @@ -96,6 +103,14 @@ ath79_setup_interfaces() ucidef_add_switch "switch0" \ "0@eth0" "2:lan:1" "3:lan:2" "4:lan:3" "5:lan:4" "1:wan" ;; + ubnt,airrouter) + ucidef_set_interface_wan "eth1" + ucidef_add_switch "switch0" \ + "0@eth0" "1:lan:4" "2:lan:3" "3:lan:2" "4:lan:1" + ;; + ubnt,nanostation-m) + ucidef_set_interfaces_lan_wan "eth1" "eth0" + ;; *) ucidef_set_interfaces_lan_wan "eth0" "eth1" ;; @@ -112,7 +127,16 @@ ath79_setup_macs() wan_mac=$(mtd_get_mac_ascii "nvram" "wan_mac") label_mac=$wan_mac ;; - engenius,enh202-v1) + engenius,enh202-v1|\ + ubnt,airrouter|\ + ubnt,bullet-m-ar7240|\ + ubnt,bullet-m-ar7241|\ + ubnt,nanobridge-m|\ + ubnt,nanostation-loco-m|\ + ubnt,nanostation-m|\ + ubnt,picostation-m|\ + ubnt,powerbridge-m|\ + ubnt,rocket-m) label_mac=$(cat /sys/class/ieee80211/phy0/macaddress) ;; tplink,tl-wr941-v2|\ diff --git a/target/linux/ath79/tiny/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom b/target/linux/ath79/tiny/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom index ffbc46afd0..792c503ab7 100644 --- a/target/linux/ath79/tiny/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom +++ b/target/linux/ath79/tiny/base-files/etc/hotplug.d/firmware/10-ath9k-eeprom @@ -36,7 +36,16 @@ case "$FIRMWARE" in tplink,tl-wr740n-v3|\ tplink,tl-wr741-v1|\ tplink,tl-wr743nd-v1|\ - tplink,tl-wr841-v7) + tplink,tl-wr841-v7|\ + ubnt,airrouter|\ + ubnt,bullet-m-ar7240|\ + ubnt,bullet-m-ar7241|\ + ubnt,nanobridge-m|\ + ubnt,nanostation-loco-m|\ + ubnt,nanostation-m|\ + ubnt,picostation-m|\ + ubnt,powerbridge-m|\ + ubnt,rocket-m) caldata_extract "art" 0x1000 0x1000 ;; pqi,air-pen) diff --git a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-ubootmod.dts b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1-ubootmod.dts similarity index 93% rename from target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-ubootmod.dts rename to target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1-ubootmod.dts index e573d30912..a22bc70180 100644 --- a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-ubootmod.dts +++ b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1-ubootmod.dts @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later OR MIT -#include "mt7622-ubnt-unifi-6-lr.dtsi" +#include "mt7622-ubnt-unifi-6-lr-v1.dtsi" / { - model = "Ubiquiti UniFi 6 LR (U-Boot mod)"; + model = "Ubiquiti UniFi 6 LR v1 (U-Boot mod)"; compatible = "ubnt,unifi-6-lr-ubootmod", "mediatek,mt7622"; }; diff --git a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr.dts b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1.dts similarity index 95% rename from target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr.dts rename to target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1.dts index 9b676b43c1..d0048d4715 100644 --- a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr.dts +++ b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1.dts @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later OR MIT -#include "mt7622-ubnt-unifi-6-lr.dtsi" +#include "mt7622-ubnt-unifi-6-lr-v1.dtsi" / { - model = "Ubiquiti UniFi 6 LR"; + model = "Ubiquiti UniFi 6 LR v1"; compatible = "ubnt,unifi-6-lr", "mediatek,mt7622"; }; diff --git a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1.dtsi b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1.dtsi new file mode 100644 index 0000000000..8f15d3f110 --- /dev/null +++ b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v1.dtsi @@ -0,0 +1,27 @@ +#include "mt7622-ubnt-unifi-6-lr.dtsi" + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + status = "okay"; + + led-controller@30 { + compatible = "ubnt,ledbar"; + reg = <0x30>; + + enable-gpio = <&pio 59 0>; + + red { + label = "red"; + }; + + green { + label = "green"; + }; + + led_blue: blue { + label = "blue"; + }; + }; +}; + diff --git a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2-ubootmod.dts b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2-ubootmod.dts new file mode 100644 index 0000000000..711d58d0e5 --- /dev/null +++ b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2-ubootmod.dts @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR MIT + +#include "mt7622-ubnt-unifi-6-lr-v2.dtsi" + +/ { + model = "Ubiquiti UniFi 6 LR v2 (U-Boot mod)"; + compatible = "ubnt,unifi-6-lr-ubootmod-v2", "mediatek,mt7622"; +}; + +&nor_partitions { + partition@0 { + label = "bl2"; + reg = <0x0 0x20000>; + }; + + partition@20000 { + label = "fip"; + reg = <0x20000 0xa0000>; + }; + + partition@c0000 { + label = "u-boot-env"; + reg = <0xc0000 0x10000>; + }; + + factory: partition@d0000 { + label = "factory"; + reg = <0xd0000 0x40000>; + read-only; + }; + + eeprom: partition@110000 { + label = "eeprom"; + reg = <0x110000 0x10000>; + read-only; + }; + + partition@120000 { + label = "recovery"; + reg = <0x120000 0xee0000>; + }; + + partition@1000000 { + compatible = "denx,fit"; + label = "firmware"; + reg = <0x1000000 0x3000000>; + }; +}; + +&wmac { + mediatek,mtd-eeprom = <&factory 0x0>; + nvmem-cells = <&macaddr_eeprom_0>; + nvmem-cell-names = "mac-address"; + status = "okay"; +}; + +&slot0 { + wifi@0,0 { + reg = <0x0 0 0 0 0>; + mediatek,mtd-eeprom = <&factory 0x20000>; + nvmem-cells = <&macaddr_eeprom_6>; + nvmem-cell-names = "mac-address"; + ieee80211-freq-limit = <5000000 6000000>; + }; +}; + +&gmac0 { + nvmem-cells = <&macaddr_eeprom_0>; + nvmem-cell-names = "mac-address"; +}; + +&eeprom { + compatible = "nvmem-cells"; + #address-cells = <1>; + #size-cells = <1>; + + macaddr_eeprom_0: macaddr@0 { + reg = <0x0 0x6>; + }; + + macaddr_eeprom_6: macaddr@6 { + reg = <0x6 0x6>; + }; +}; diff --git a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2.dts b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2.dts new file mode 100644 index 0000000000..f40e8e632b --- /dev/null +++ b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2.dts @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR MIT + +#include "mt7622-ubnt-unifi-6-lr-v2.dtsi" + +/ { + model = "Ubiquiti UniFi 6 LR v2"; + compatible = "ubnt,unifi-6-lr-v2", "mediatek,mt7622"; +}; + +&nor_partitions { + partition@0 { + label = "preloader"; + reg = <0x0 0x40000>; + }; + + partition@40000 { + label = "atf"; + reg = <0x40000 0x20000>; + }; + + partition@60000 { + label = "u-boot"; + reg = <0x60000 0x60000>; + }; + + partition@c0000 { + label = "u-boot-env"; + reg = <0xc0000 0x10000>; + }; + + factory: partition@d0000 { + label = "factory"; + reg = <0xd0000 0x40000>; + read-only; + }; + + eeprom: partition@110000 { + label = "eeprom"; + reg = <0x110000 0x10000>; + read-only; + }; + + partition@120000 { + label = "bs"; + reg = <0x120000 0x10000>; + }; + + partition@130000 { + label = "cfg"; + reg = <0x130000 0x100000>; + read-only; + }; + + partition@230000 { + compatible = "denx,fit"; + label = "firmware"; + reg = <0x230000 0x1ee0000>; + }; + + partition@2110000 { + label = "kernel1"; + reg = <0x2110000 0x1ee0000>; + }; +}; + +&wmac { + mediatek,mtd-eeprom = <&factory 0x0>; + nvmem-cells = <&macaddr_eeprom_0>; + nvmem-cell-names = "mac-address"; + status = "okay"; +}; + +&slot0 { + wifi@0,0 { + reg = <0x0 0 0 0 0>; + mediatek,mtd-eeprom = <&factory 0x20000>; + nvmem-cells = <&macaddr_eeprom_6>; + nvmem-cell-names = "mac-address"; + ieee80211-freq-limit = <5000000 6000000>; + }; +}; + +&gmac0 { + nvmem-cells = <&macaddr_eeprom_0>; + nvmem-cell-names = "mac-address"; +}; + +&eeprom { + compatible = "nvmem-cells"; + #address-cells = <1>; + #size-cells = <1>; + + macaddr_eeprom_0: macaddr@0 { + reg = <0x0 0x6>; + }; + + macaddr_eeprom_6: macaddr@6 { + reg = <0x6 0x6>; + }; +}; diff --git a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2.dtsi b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2.dtsi new file mode 100644 index 0000000000..3e9cea7285 --- /dev/null +++ b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr-v2.dtsi @@ -0,0 +1,26 @@ +#include "mt7622-ubnt-unifi-6-lr.dtsi" + +/ { + aliases { + led-boot = &led_white; + led-failsafe = &led_white; + led-running = &led_blue; + led-upgrade = &led_blue; + }; + + gpio-leds { + compatible = "gpio-leds"; + + led_white: dome_white { + label = "white:dome"; + linux,default-trigger = "default-on"; + gpios = <&pio 0x43 GPIO_ACTIVE_HIGH>; + }; + + led_blue: dome_blue { + label = "blue:dome"; + default-state = "off"; + gpios = <&pio 0x44 GPIO_ACTIVE_HIGH>; + }; + }; +}; diff --git a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr.dtsi b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr.dtsi index ae587a11a0..0827f18e48 100644 --- a/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr.dtsi +++ b/target/linux/mediatek/dts/mt7622-ubnt-unifi-6-lr.dtsi @@ -217,31 +217,6 @@ /* MT7915 Bluetooth */ }; -&i2c0 { - pinctrl-names = "default"; - pinctrl-0 = <&i2c0_pins>; - status = "okay"; - - led-controller@30 { - compatible = "ubnt,ledbar"; - reg = <0x30>; - - enable-gpio = <&pio 59 0>; - - red { - label = "red"; - }; - - green { - label = "green"; - }; - - led_blue: blue { - label = "blue"; - }; - }; -}; - &watchdog { pinctrl-names = "default"; pinctrl-0 = <&watchdog_pins>; diff --git a/target/linux/mediatek/image/mt7622.mk b/target/linux/mediatek/image/mt7622.mk index aba8bdfe0b..810645067a 100644 --- a/target/linux/mediatek/image/mt7622.mk +++ b/target/linux/mediatek/image/mt7622.mk @@ -242,21 +242,23 @@ define Device/totolink_a8000ru endef TARGET_DEVICES += totolink_a8000ru -define Device/ubnt_unifi-6-lr +define Device/ubnt_unifi-6-lr-v1 DEVICE_VENDOR := Ubiquiti DEVICE_MODEL := UniFi 6 LR + DEVICE_VARIANT := v1 DEVICE_DTS_CONFIG := config@1 - DEVICE_DTS := mt7622-ubnt-unifi-6-lr + DEVICE_DTS := mt7622-ubnt-unifi-6-lr-v1 DEVICE_DTS_DIR := ../dts DEVICE_PACKAGES := kmod-mt7915e kmod-leds-ubnt-ledbar + SUPPORTED_DEVICES += ubnt,unifi-6-lr endef -TARGET_DEVICES += ubnt_unifi-6-lr +TARGET_DEVICES += ubnt_unifi-6-lr-v1 -define Device/ubnt_unifi-6-lr-ubootmod +define Device/ubnt_unifi-6-lr-v1-ubootmod DEVICE_VENDOR := Ubiquiti DEVICE_MODEL := UniFi 6 LR - DEVICE_VARIANT := U-Boot mod - DEVICE_DTS := mt7622-ubnt-unifi-6-lr-ubootmod + DEVICE_VARIANT := v1 U-Boot mod + DEVICE_DTS := mt7622-ubnt-unifi-6-lr-v1-ubootmod DEVICE_DTS_DIR := ../dts DEVICE_PACKAGES := kmod-mt7915e kmod-leds-ubnt-ledbar KERNEL := kernel-bin | lzma @@ -267,8 +269,38 @@ define Device/ubnt_unifi-6-lr-ubootmod ARTIFACTS := preloader.bin bl31-uboot.fip ARTIFACT/preloader.bin := bl2 nor-2ddr ARTIFACT/bl31-uboot.fip := bl31-uboot ubnt_unifi-6-lr + SUPPORTED_DEVICES += ubnt,unifi-6-lr-ubootmod endef -TARGET_DEVICES += ubnt_unifi-6-lr-ubootmod +TARGET_DEVICES += ubnt_unifi-6-lr-v1-ubootmod + +define Device/ubnt_unifi-6-lr-v2 + DEVICE_VENDOR := Ubiquiti + DEVICE_MODEL := UniFi 6 LR + DEVICE_VARIANT := v2 + DEVICE_DTS_CONFIG := config@1 + DEVICE_DTS := mt7622-ubnt-unifi-6-lr-v2 + DEVICE_DTS_DIR := ../dts + DEVICE_PACKAGES := kmod-mt7915e +endef +TARGET_DEVICES += ubnt_unifi-6-lr-v2 + +define Device/ubnt_unifi-6-lr-v2-ubootmod + DEVICE_VENDOR := Ubiquiti + DEVICE_MODEL := UniFi 6 LR + DEVICE_VARIANT := v2 U-Boot mod + DEVICE_DTS := mt7622-ubnt-unifi-6-lr-v2-ubootmod + DEVICE_DTS_DIR := ../dts + DEVICE_PACKAGES := kmod-mt7915e + KERNEL := kernel-bin | lzma + KERNEL_INITRAMFS_SUFFIX := -recovery.itb + KERNEL_INITRAMFS := kernel-bin | lzma | fit lzma $$(KDIR)/image-$$(firstword $$(DEVICE_DTS)).dtb with-initrd | pad-to 64k + IMAGES := sysupgrade.itb + IMAGE/sysupgrade.itb := append-kernel | fit lzma $$(KDIR)/image-$$(firstword $$(DEVICE_DTS)).dtb external-static-with-rootfs | pad-rootfs | append-metadata + ARTIFACTS := preloader.bin bl31-uboot.fip + ARTIFACT/preloader.bin := bl2 nor-2ddr + ARTIFACT/bl31-uboot.fip := bl31-uboot ubnt_unifi-6-lr +endef +TARGET_DEVICES += ubnt_unifi-6-lr-v2-ubootmod define Device/xiaomi_redmi-router-ax6s DEVICE_VENDOR := Xiaomi diff --git a/target/linux/ramips/dts/mt7621_arcadyan_wg4xx223.dtsi b/target/linux/ramips/dts/mt7621_arcadyan_wg4xx223.dtsi new file mode 100644 index 0000000000..5a845c4e9b --- /dev/null +++ b/target/linux/ramips/dts/mt7621_arcadyan_wg4xx223.dtsi @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR MIT + +#include "mt7621.dtsi" + +#include +#include +#include + +/ { + aliases { + led-boot = &led_status_red; + led-failsafe = &led_status_red; + led-running = &led_status_green; + led-upgrade = &led_status_red; + }; + + leds { + compatible = "gpio-leds"; + + led_status_green: led-1 { + label = "green:status"; + color = ; + function = LED_FUNCTION_STATUS; + gpios = <&gpio 15 GPIO_ACTIVE_LOW>; + }; + + led_status_red: led-2 { + label = "red:status"; + color = ; + function = LED_FUNCTION_STATUS; + gpios = <&gpio 16 GPIO_ACTIVE_LOW>; + }; + }; + + keys { + compatible = "gpio-keys"; + + reset { + label = "reset"; + gpios = <&gpio 3 GPIO_ACTIVE_LOW>; + linux,code = ; + }; + }; + + ubi-concat { + compatible = "mtd-concat"; + devices = <&ubiconcat0 &ubiconcat1>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "ubi"; + reg = <0x0 0x5240000>; + }; + }; + }; +}; + +&nand { + status = "okay"; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "u-boot"; + reg = <0x0 0x100000>; + read-only; + }; + + partition@100000 { + label = "u-boot-env"; + reg = <0x100000 0x100000>; + }; + + factory: partition@200000 { + label = "Factory"; + reg = <0x200000 0x100000>; + read-only; + + compatible = "nvmem-cells"; + #address-cells = <1>; + #size-cells = <1>; + + /* We keep the block below to prevent eth0 MAC + * from randomization. Unique WAN, LAN, WLAN MACs + * are stored in u-boot-env. + */ + + /* Default Ralink MAC (00:0c:43:28:80:xx) */ + macaddr_factory_fff0: macaddr@fff0 { + reg = <0xfff0 0x6>; + }; + }; + + partition@300000 { + label = "firmware"; + reg = <0x300000 0x2000000>; + + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "kernel"; + reg = <0x0 0x440000>; + }; + + ubiconcat0: partition@400000 { + label = "ubiconcat0"; + reg = <0x440000 0x1bc0000>; + }; + }; + + partition@2300000 { + label = "Firmware2"; + reg = <0x2300000 0x2000000>; + read-only; + }; + + partition@4300000 { + label = "glbcfg"; + reg = <0x4300000 0x200000>; + read-only; + }; + + partition@4500000 { + label = "board_data"; + reg = <0x4500000 0x100000>; + read-only; + }; + + partition@4600000 { + label = "glbcfg2"; + reg = <0x4600000 0x200000>; + read-only; + }; + + partition@4800000 { + label = "board_data2"; + reg = <0x4800000 0x100000>; + read-only; + }; + + ubiconcat1: partition@4900000 { + label = "ubiconcat1"; + reg = <0x4900000 0x3680000>; + }; + }; +}; + +&pcie { + status = "okay"; +}; + +&pcie0 { + wifi@0,0 { + compatible = "mediatek,mt76"; + reg = <0x0000 0 0 0 0>; + mediatek,mtd-eeprom = <&factory 0x0>; + }; +}; + +&gmac0 { + nvmem-cells = <&macaddr_factory_fff0>; + nvmem-cell-names = "mac-address"; +}; + +&switch0 { + ports { + port@2 { + status = "okay"; + label = "lan2"; + }; + + port@3 { + status = "okay"; + label = "lan1"; + }; + + port@4 { + status = "okay"; + label = "wan"; + }; + }; +}; + +&state_default { + gpio { + groups = "i2c", "jtag"; + function = "gpio"; + }; +}; diff --git a/target/linux/ramips/dts/mt7621_beeline_smartbox-flash.dts b/target/linux/ramips/dts/mt7621_beeline_smartbox-flash.dts index 45feb13756..d373a801d2 100644 --- a/target/linux/ramips/dts/mt7621_beeline_smartbox-flash.dts +++ b/target/linux/ramips/dts/mt7621_beeline_smartbox-flash.dts @@ -1,202 +1,17 @@ // SPDX-License-Identifier: GPL-2.0-or-later OR MIT -#include "mt7621.dtsi" - -#include -#include -#include +#include "mt7621_arcadyan_wg4xx223.dtsi" / { compatible = "beeline,smartbox-flash", "mediatek,mt7621-soc"; model = "Beeline SmartBox Flash"; - aliases { - led-boot = &led_status_red; - led-failsafe = &led_status_red; - led-running = &led_status_green; - led-upgrade = &led_status_red; - }; - leds { - compatible = "gpio-leds"; - led-0 { label = "blue:wan"; color = ; function = LED_FUNCTION_WAN; gpios = <&gpio 14 GPIO_ACTIVE_LOW>; }; - - led_status_green: led-1 { - label = "green:status"; - color = ; - function = LED_FUNCTION_STATUS; - gpios = <&gpio 15 GPIO_ACTIVE_LOW>; - }; - - led_status_red: led-2 { - label = "red:status"; - color = ; - function = LED_FUNCTION_STATUS; - gpios = <&gpio 16 GPIO_ACTIVE_LOW>; - }; - }; - - keys { - compatible = "gpio-keys"; - - reset { - label = "reset"; - gpios = <&gpio 3 GPIO_ACTIVE_LOW>; - linux,code = ; - }; - }; - - ubi-concat { - compatible = "mtd-concat"; - devices = <&ubiconcat0 &ubiconcat1>; - - partitions { - compatible = "fixed-partitions"; - #address-cells = <1>; - #size-cells = <1>; - - partition@0 { - label = "ubi"; - reg = <0x0 0x5240000>; - }; - }; - }; -}; - -&nand { - status = "okay"; - - partitions { - compatible = "fixed-partitions"; - #address-cells = <1>; - #size-cells = <1>; - - partition@0 { - label = "u-boot"; - reg = <0x0 0x100000>; - read-only; - }; - - partition@100000 { - label = "u-boot-env"; - reg = <0x100000 0x100000>; - }; - - factory: partition@200000 { - label = "Factory"; - reg = <0x200000 0x100000>; - read-only; - - compatible = "nvmem-cells"; - #address-cells = <1>; - #size-cells = <1>; - - macaddr_factory_fff0: macaddr@fff0 { - reg = <0xfff0 0x6>; - }; - }; - - partition@300000 { - label = "firmware"; - reg = <0x300000 0x2000000>; - - compatible = "fixed-partitions"; - #address-cells = <1>; - #size-cells = <1>; - - partition@0 { - label = "kernel"; - reg = <0x0 0x440000>; - }; - - ubiconcat0: partition@400000 { - label = "ubiconcat0"; - reg = <0x440000 0x1bc0000>; - }; - }; - - partition@2300000 { - label = "Firmware2"; - reg = <0x2300000 0x2000000>; - read-only; - }; - - partition@4300000 { - label = "glbcfg"; - reg = <0x4300000 0x200000>; - read-only; - }; - - partition@4500000 { - label = "board_data"; - reg = <0x4500000 0x100000>; - read-only; - }; - - partition@4600000 { - label = "glbcfg2"; - reg = <0x4600000 0x200000>; - read-only; - }; - - partition@4800000 { - label = "board_data2"; - reg = <0x4800000 0x100000>; - read-only; - }; - - ubiconcat1: partition@4900000 { - label = "ubiconcat1"; - reg = <0x4900000 0x3680000>; - }; - }; -}; - -&pcie { - status = "okay"; -}; - -&pcie0 { - wifi@0,0 { - compatible = "mediatek,mt76"; - reg = <0x0000 0 0 0 0>; - mediatek,mtd-eeprom = <&factory 0x0>; - }; -}; - -&gmac0 { - nvmem-cells = <&macaddr_factory_fff0>; - nvmem-cell-names = "mac-address"; -}; - -&switch0 { - ports { - port@2 { - status = "okay"; - label = "lan2"; - }; - - port@3 { - status = "okay"; - label = "lan1"; - }; - - port@4 { - status = "okay"; - label = "wan"; - }; - }; -}; - -&state_default { - gpio { - groups = "i2c", "jtag"; - function = "gpio"; }; }; diff --git a/target/linux/ramips/dts/mt7621_mts_wg430223.dts b/target/linux/ramips/dts/mt7621_mts_wg430223.dts new file mode 100644 index 0000000000..9ddb1d8dc5 --- /dev/null +++ b/target/linux/ramips/dts/mt7621_mts_wg430223.dts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR MIT + +#include "mt7621_arcadyan_wg4xx223.dtsi" + +/ { + compatible = "mts,wg430223", "mediatek,mt7621-soc"; + model = "MTS WG430223"; +}; diff --git a/target/linux/ramips/dts/mt7621_phicomm_k2p.dts b/target/linux/ramips/dts/mt7621_phicomm_k2p.dts index a0bdab2226..078b76a50b 100644 --- a/target/linux/ramips/dts/mt7621_phicomm_k2p.dts +++ b/target/linux/ramips/dts/mt7621_phicomm_k2p.dts @@ -50,8 +50,7 @@ flash@0 { compatible = "jedec,spi-nor"; reg = <0>; - spi-max-frequency = <80000000>; - m25p,fast-read; + spi-max-frequency = <50000000>; partitions { compatible = "fixed-partitions"; diff --git a/target/linux/ramips/image/mt7621.mk b/target/linux/ramips/image/mt7621.mk index 59dffc78ff..f73c900400 100644 --- a/target/linux/ramips/image/mt7621.mk +++ b/target/linux/ramips/image/mt7621.mk @@ -13,9 +13,10 @@ ifdef CONFIG_LINUX_5_10 DTS_CPPFLAGS += -DDTS_LEGACY endif -define Build/beeline-trx +define Build/arcadyan-trx echo -ne "hsqs" > $@.hsqs - $(STAGING_DIR_HOST)/bin/otrx create $@.trx -M 0x746f435d -f $@ \ + $(eval trx_magic=$(word 1,$(1))) + $(STAGING_DIR_HOST)/bin/otrx create $@.trx -M $(trx_magic) -f $@ \ -a 0x20000 -b 0x420000 -f $@.hsqs -a 1000 mv $@.trx $@ dd if=/dev/zero bs=1024 count=1 >> $@.tail @@ -242,7 +243,7 @@ define Device/beeline_smartbox-flash BLOCKSIZE := 128k PAGESIZE := 2048 KERNEL := kernel-bin | append-dtb | lzma | loader-kernel | \ - uImage none | beeline-trx | pad-to $$(KERNEL_SIZE) + uImage none | arcadyan-trx 0x746f435d | pad-to $$(KERNEL_SIZE) KERNEL_INITRAMFS := kernel-bin | append-dtb | lzma | loader-kernel | \ uImage none IMAGES += factory.trx @@ -1189,6 +1190,27 @@ define Device/mtc_wr1201 endef TARGET_DEVICES += mtc_wr1201 +define Device/mts_wg430223 + $(Device/dsa-migration) + $(Device/uimage-lzma-loader) + DEVICE_VENDOR := MTS + DEVICE_MODEL := WG430223 + IMAGE_SIZE := 32768k + KERNEL_SIZE := 4352k + UBINIZE_OPTS := -E 5 + BLOCKSIZE := 128k + PAGESIZE := 2048 + KERNEL := kernel-bin | append-dtb | lzma | loader-kernel | \ + uImage none | arcadyan-trx 0x53485231 | pad-to $$(KERNEL_SIZE) + KERNEL_INITRAMFS := kernel-bin | append-dtb | lzma | loader-kernel | \ + uImage none + IMAGES += factory.trx + IMAGE/factory.trx := append-kernel | append-ubi | check-size + IMAGE/sysupgrade.bin := sysupgrade-tar | append-metadata + DEVICE_PACKAGES := kmod-mt7615e kmod-mt7615-firmware uboot-envtools +endef +TARGET_DEVICES += mts_wg430223 + define Device/netgear_ex6150 $(Device/dsa-migration) DEVICE_VENDOR := NETGEAR diff --git a/target/linux/ramips/mt7621/base-files/etc/board.d/02_network b/target/linux/ramips/mt7621/base-files/etc/board.d/02_network index a8d8b8eef2..0a139e0113 100644 --- a/target/linux/ramips/mt7621/base-files/etc/board.d/02_network +++ b/target/linux/ramips/mt7621/base-files/etc/board.d/02_network @@ -46,6 +46,7 @@ ramips_setup_interfaces() jcg,q20|\ lenovo,newifi-d1|\ mikrotik,routerboard-m33g|\ + mts,wg430223|\ oraybox,x3a|\ renkforce,ws-wn530hp3-a|\ xiaomi,mi-router-3g|\ @@ -127,7 +128,8 @@ ramips_setup_macs() wan_mac=$(mtd_get_mac_ascii u-boot-env et1macaddr) label_mac=$(mtd_get_mac_binary factory 0x4) ;; - beeline,smartbox-flash) + beeline,smartbox-flash|\ + mts,wg430223) lan_mac=$(mtd_get_mac_ascii u-boot-env eth2macaddr) wan_mac=$(mtd_get_mac_ascii u-boot-env eth3macaddr) label_mac=$lan_mac diff --git a/target/linux/ramips/mt7621/base-files/etc/hotplug.d/ieee80211/10_fix_wifi_mac b/target/linux/ramips/mt7621/base-files/etc/hotplug.d/ieee80211/10_fix_wifi_mac index 5b12416872..3390104e80 100644 --- a/target/linux/ramips/mt7621/base-files/etc/hotplug.d/ieee80211/10_fix_wifi_mac +++ b/target/linux/ramips/mt7621/base-files/etc/hotplug.d/ieee80211/10_fix_wifi_mac @@ -10,7 +10,8 @@ PHYNBR=${DEVPATH##*/phy} board=$(board_name) case "$board" in - beeline,smartbox-flash) + beeline,smartbox-flash|\ + mts,wg430223) hw_mac_addr_ra0="$(mtd_get_mac_ascii u-boot-env ra0macaddr)" hw_mac_addr_rax0="$(mtd_get_mac_ascii u-boot-env rax0macaddr)" [ "$PHYNBR" = "0" ] && echo -n $hw_mac_addr_ra0 > /sys${DEVPATH}/macaddress diff --git a/target/linux/ramips/mt7621/base-files/lib/upgrade/platform.sh b/target/linux/ramips/mt7621/base-files/lib/upgrade/platform.sh index 407611a6a2..9d5e7576ec 100755 --- a/target/linux/ramips/mt7621/base-files/lib/upgrade/platform.sh +++ b/target/linux/ramips/mt7621/base-files/lib/upgrade/platform.sh @@ -70,6 +70,7 @@ platform_do_upgrade() { linksys,ea7500-v2|\ linksys,ea8100-v1|\ linksys,ea8100-v2|\ + mts,wg430223|\ netgear,r6220|\ netgear,r6260|\ netgear,r6350|\ diff --git a/tools/ninja/Makefile b/tools/ninja/Makefile index 0ff642a740..c5c83d9b14 100644 --- a/tools/ninja/Makefile +++ b/tools/ninja/Makefile @@ -1,12 +1,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ninja -PKG_VERSION:=1.10.2 +PKG_VERSION:=1.11.0 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/ninja-build/ninja/tar.gz/v$(PKG_VERSION)? -PKG_HASH:=ce35865411f0490368a8fc383f29071de6690cbadc27704734978221f25e2bed +PKG_HASH:=3c6ba2e66400fe3f1ae83deb4b235faf3137ec20bd5b08c29bfc368db143e4c6 include $(INCLUDE_DIR)/host-build.mk diff --git a/tools/ninja/patches/100-make_jobserver_support.patch b/tools/ninja/patches/100-make_jobserver_support.patch index ecceaf23ca..7dac8ef814 100644 --- a/tools/ninja/patches/100-make_jobserver_support.patch +++ b/tools/ninja/patches/100-make_jobserver_support.patch @@ -1,7 +1,7 @@ -From c1a081c00f803fc28e51f155f25abe8346ce5f13 Mon Sep 17 00:00:00 2001 +From 17d13fd7881fd3ce9f9b9d44ce435d6caf4b8f28 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Tue, 22 Mar 2016 13:48:07 +0200 -Subject: [PATCH] Add GNU make jobserver client support +Subject: [PATCH 01/11] Add GNU make jobserver client support - add new TokenPool interface - GNU make implementation for TokenPool parses and verifies the magic @@ -19,263 +19,60 @@ Documentation for GNU make jobserver http://make.mad-scientist.net/papers/jobserver-implementation/ Fixes https://github.com/ninja-build/ninja/issues/1139 - -Add TokenPool monitoring to SubprocessSet::DoWork() - -Improve on the original jobserver client implementation. This makes -ninja a more aggressive GNU make jobserver client. - -- add monitor interface to TokenPool -- TokenPool is passed down when main loop indicates that more work is - ready and would be allowed to start if a token becomes available -- posix: update DoWork() to monitor TokenPool read file descriptor -- WaitForCommand() exits when DoWork() sets token flag -- Main loop starts over when WaitForCommand() sets token exit status - -Ignore jobserver when -jN is forced on command line - -This emulates the behaviour of GNU make. - -- add parallelism_from_cmdline flag to build configuration -- set the flag when -jN is given on command line -- pass the flag to TokenPool::Get() -- GNUmakeTokenPool::Setup() - * prints a warning when the flag is true and jobserver was detected - * returns false, i.e. jobserver will be ignored -- ignore config.parallelism in CanRunMore() when we have a valid - TokenPool, because it gets always initialized to a default when not - given on the command line - -Honor -lN from MAKEFLAGS - -This emulates the behaviour of GNU make. - -- build: make a copy of max_load_average and pass it to TokenPool. -- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in - MAKEFLAGS then set max_load_average to N. - -Use LinePrinter for TokenPool messages - -- replace printf() with calls to LinePrinter -- print GNU make jobserver message only when verbose build is requested - -Prepare PR for merging - -- fix Windows build error in no-op TokenPool implementation -- improve Acquire() to block for a maximum of 100ms -- address review comments - -Add tests for TokenPool - -- TokenPool setup -- GetMonitorFd() API -- implicit token and tokens in jobserver pipe -- Acquire() / Reserve() / Release() protocol -- Clear() API - -Add tests for subprocess module - -- add TokenPoolTest stub to provide TokenPool::GetMonitorFd() -- add two tests - * both tests set up a dummy GNUmake jobserver pipe - * both tests call DoWork() with TokenPoolTest - * test 1: verify that DoWork() detects when a token is available - * test 2: verify that DoWork() works as before without a token -- the tests are not compiled in under Windows - -Add tests for build module - -Add tests that verify the token functionality of the builder main loop. -We replace the default fake command runner with a special version where -the tests can control each call to AcquireToken(), CanRunMore() and -WaitForCommand(). - -Add Win32 implementation for GNUmakeTokenPool - -GNU make uses a semaphore as jobserver protocol on Win32. See also - - https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html - -Usage is pretty simple and straightforward, i.e. WaitForSingleObject() -to obtain a token and ReleaseSemaphore() to return it. - -Unfortunately subprocess-win32.cc uses an I/O completion port (IOCP). -IOCPs aren't waitable objects, i.e. we can't use WaitForMultipleObjects() -to wait on the IOCP and the token semaphore at the same time. - -Therefore GNUmakeTokenPoolWin32 creates a child thread that waits on the -token semaphore and posts a dummy I/O completion status on the IOCP when -it was able to obtain a token. That unblocks SubprocessSet::DoWork() and -it can then check if a token became available or not. - -- split existing GNUmakeTokenPool into common and platform bits -- add GNUmakeTokenPool interface -- move the Posix bits to GNUmakeTokenPoolPosix -- add the Win32 bits as GNUmakeTokenPoolWin32 -- move Setup() method up to TokenPool interface -- update Subprocess & TokenPool tests accordingly - -Prepare PR for merging - part II - -- remove unnecessary "struct" from TokenPool -- add PAPCFUNC cast to QueryUserAPC() -- remove hard-coded MAKEFLAGS string from win32 -- remove useless build test CompleteNoWork -- rename TokenPoolTest to TestTokenPool -- add tokenpool modules to CMake build -- remove unused no-op TokenPool implementation -- address review comments from - -https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-195195803 -https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-185089255 -https://github.com/ninja-build/ninja/pull/1140#issuecomment-473898963 -https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 --- - CMakeLists.txt | 8 +- - configure.py | 7 +- - src/build.cc | 127 ++++++++--- - src/build.h | 12 +- - src/build_test.cc | 363 +++++++++++++++++++++++++++++++- - src/exit_status.h | 3 +- - src/ninja.cc | 1 + - src/subprocess-posix.cc | 33 ++- - src/subprocess-win32.cc | 11 +- - src/subprocess.h | 8 +- - src/subprocess_test.cc | 149 +++++++++++-- - src/tokenpool-gnu-make-posix.cc | 202 ++++++++++++++++++ - src/tokenpool-gnu-make-win32.cc | 239 +++++++++++++++++++++ - src/tokenpool-gnu-make.cc | 108 ++++++++++ - src/tokenpool-gnu-make.h | 40 ++++ - src/tokenpool.h | 42 ++++ - src/tokenpool_test.cc | 269 +++++++++++++++++++++++ - 17 files changed, 1562 insertions(+), 60 deletions(-) - create mode 100644 src/tokenpool-gnu-make-posix.cc - create mode 100644 src/tokenpool-gnu-make-win32.cc + configure.py | 2 + + src/build.cc | 63 ++++++++---- + src/build.h | 3 + + src/tokenpool-gnu-make.cc | 211 ++++++++++++++++++++++++++++++++++++++ + src/tokenpool-none.cc | 27 +++++ + src/tokenpool.h | 26 +++++ + 6 files changed, 310 insertions(+), 22 deletions(-) create mode 100644 src/tokenpool-gnu-make.cc - create mode 100644 src/tokenpool-gnu-make.h + create mode 100644 src/tokenpool-none.cc create mode 100644 src/tokenpool.h - create mode 100644 src/tokenpool_test.cc ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -94,6 +94,7 @@ add_library(libninja OBJECT - src/parser.cc - src/state.cc - src/string_piece_util.cc -+ src/tokenpool-gnu-make.cc - src/util.cc - src/version.cc - ) -@@ -104,12 +105,16 @@ if(WIN32) - src/msvc_helper-win32.cc - src/msvc_helper_main-win32.cc - src/getopt.c -+ src/tokenpool-gnu-make-win32.cc - ) - if(MSVC) - target_sources(libninja PRIVATE src/minidump-win32.cc) - endif() - else() -- target_sources(libninja PRIVATE src/subprocess-posix.cc) -+ target_sources(libninja PRIVATE -+ src/subprocess-posix.cc -+ src/tokenpool-gnu-make-posix.cc -+ ) - if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") - target_sources(libninja PRIVATE src/getopt.c) - endif() -@@ -182,6 +187,7 @@ if(BUILD_TESTING) - src/string_piece_util_test.cc - src/subprocess_test.cc - src/test.cc -+ src/tokenpool_test.cc - src/util_test.cc - ) - if(WIN32) +diff --git a/configure.py b/configure.py +index 43904349a8..db3492c93c 100755 --- a/configure.py +++ b/configure.py -@@ -514,11 +514,13 @@ for name in ['build', - 'parser', - 'state', - 'string_piece_util', -+ 'tokenpool-gnu-make', - 'util', - 'version']: +@@ -522,6 +522,7 @@ def has_re2c(): objs += cxx(name, variables=cxxvariables) if platform.is_windows(): for name in ['subprocess-win32', -+ 'tokenpool-gnu-make-win32', ++ 'tokenpool-none', 'includes_normalize-win32', 'msvc_helper-win32', 'msvc_helper_main-win32']: -@@ -527,7 +529,9 @@ if platform.is_windows(): - objs += cxx('minidump-win32', variables=cxxvariables) +@@ -531,6 +532,7 @@ def has_re2c(): objs += cc('getopt') else: -- objs += cxx('subprocess-posix') -+ for name in ['subprocess-posix', -+ 'tokenpool-gnu-make-posix']: -+ objs += cxx(name) + objs += cxx('subprocess-posix') ++ objs += cxx('tokenpool-gnu-make') if platform.is_aix(): objs += cc('getopt') if platform.is_msvc(): -@@ -582,6 +586,7 @@ for name in ['build_log_test', - 'string_piece_util_test', - 'subprocess_test', - 'test', -+ 'tokenpool_test', - 'util_test']: - objs += cxx(name, variables=cxxvariables) - if platform.is_windows(): +diff --git a/src/build.cc b/src/build.cc +index 6f11ed7a3c..fa096eac33 100644 --- a/src/build.cc +++ b/src/build.cc -@@ -38,6 +38,7 @@ - #include "graph.h" +@@ -35,6 +35,7 @@ #include "state.h" + #include "status.h" #include "subprocess.h" +#include "tokenpool.h" #include "util.h" using namespace std; -@@ -50,8 +51,9 @@ struct DryRunCommandRunner : public Comm - - // Overridden from CommandRunner: - virtual bool CanRunMore() const; -+ virtual bool AcquireToken(); - virtual bool StartCommand(Edge* edge); -- virtual bool WaitForCommand(Result* result); -+ virtual bool WaitForCommand(Result* result, bool more_ready); - - private: - queue finished_; -@@ -61,12 +63,16 @@ bool DryRunCommandRunner::CanRunMore() c - return true; - } - -+bool DryRunCommandRunner::AcquireToken() { -+ return true; -+} -+ - bool DryRunCommandRunner::StartCommand(Edge* edge) { - finished_.push(edge); - return true; - } - --bool DryRunCommandRunner::WaitForCommand(Result* result) { -+bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) { - if (finished_.empty()) - return false; - -@@ -379,7 +385,7 @@ void Plan::EdgeWanted(const Edge* edge) +@@ -149,7 +150,7 @@ void Plan::EdgeWanted(const Edge* edge) { } Edge* Plan::FindWork() { - if (ready_.empty()) + if (!more_ready()) return NULL; - set::iterator e = ready_.begin(); + EdgeSet::iterator e = ready_.begin(); Edge* edge = *e; -@@ -665,19 +671,39 @@ void Plan::Dump() const { +@@ -448,8 +449,8 @@ void Plan::Dump() const { } struct RealCommandRunner : public CommandRunner { @@ -284,31 +81,18 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + explicit RealCommandRunner(const BuildConfig& config); + virtual ~RealCommandRunner(); virtual bool CanRunMore() const; -+ virtual bool AcquireToken(); virtual bool StartCommand(Edge* edge); -- virtual bool WaitForCommand(Result* result); -+ virtual bool WaitForCommand(Result* result, bool more_ready); - virtual vector GetActiveEdges(); - virtual void Abort(); + virtual bool WaitForCommand(Result* result); +@@ -458,9 +459,18 @@ struct RealCommandRunner : public CommandRunner { const BuildConfig& config_; -+ // copy of config_.max_load_average; can be modified by TokenPool setup -+ double max_load_average_; SubprocessSet subprocs_; -+ TokenPool* tokens_; ++ TokenPool *tokens_; map subproc_to_edge_; }; +RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { -+ max_load_average_ = config.max_load_average; -+ if ((tokens_ = TokenPool::Get()) != NULL) { -+ if (!tokens_->Setup(config_.parallelism_from_cmdline, -+ config_.verbosity == BuildConfig::VERBOSE, -+ max_load_average_)) { -+ delete tokens_; -+ tokens_ = NULL; -+ } -+ } ++ tokens_ = TokenPool::Get(); +} + +RealCommandRunner::~RealCommandRunner() { @@ -318,7 +102,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 vector RealCommandRunner::GetActiveEdges() { vector edges; for (map::iterator e = subproc_to_edge_.begin(); -@@ -688,14 +714,23 @@ vector RealCommandRunner::GetActi +@@ -471,14 +481,18 @@ vector RealCommandRunner::GetActiveEdges() { void RealCommandRunner::Abort() { subprocs_.Clear(); @@ -327,27 +111,19 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 } bool RealCommandRunner::CanRunMore() const { -- size_t subproc_number = -- subprocs_.running_.size() + subprocs_.finished_.size(); -- return (int)subproc_number < config_.parallelism + size_t subproc_number = + subprocs_.running_.size() + subprocs_.finished_.size(); + return (int)subproc_number < config_.parallelism - && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) - || GetLoadAverage() < config_.max_load_average); -+ bool parallelism_limit_not_reached = -+ tokens_ || // ignore config_.parallelism -+ ((int) (subprocs_.running_.size() + -+ subprocs_.finished_.size()) < config_.parallelism); -+ return parallelism_limit_not_reached + && (subprocs_.running_.empty() || -+ (max_load_average_ <= 0.0f || -+ GetLoadAverage() < max_load_average_)); -+} -+ -+bool RealCommandRunner::AcquireToken() { -+ return (!tokens_ || tokens_->Acquire()); ++ ((config_.max_load_average <= 0.0f || ++ GetLoadAverage() < config_.max_load_average) ++ && (!tokens_ || tokens_->Acquire()))); } bool RealCommandRunner::StartCommand(Edge* edge) { -@@ -703,19 +738,33 @@ bool RealCommandRunner::StartCommand(Edg +@@ -486,6 +500,8 @@ bool RealCommandRunner::StartCommand(Edge* edge) { Subprocess* subproc = subprocs_.Add(command, edge->use_console()); if (!subproc) return false; @@ -356,63 +132,37 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 subproc_to_edge_.insert(make_pair(subproc, edge)); return true; - } - --bool RealCommandRunner::WaitForCommand(Result* result) { -+bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) { - Subprocess* subproc; -- while ((subproc = subprocs_.NextFinished()) == NULL) { -- bool interrupted = subprocs_.DoWork(); -+ subprocs_.ResetTokenAvailable(); -+ while (((subproc = subprocs_.NextFinished()) == NULL) && -+ !subprocs_.IsTokenAvailable()) { -+ bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL); - if (interrupted) +@@ -499,6 +515,9 @@ bool RealCommandRunner::WaitForCommand(Result* result) { return false; } -+ // token became available -+ if (subproc == NULL) { -+ result->status = ExitTokenAvailable; -+ return true; -+ } -+ -+ // command completed + if (tokens_) + tokens_->Release(); + result->status = subproc->Finish(); result->output = subproc->GetOutput(); -@@ -825,38 +874,42 @@ bool Builder::Build(string* err) { - // command runner. +@@ -621,31 +640,31 @@ bool Builder::Build(string* err) { // Second, we attempt to wait for / reap the next finished command. while (plan_.more_to_do()) { -- // See if we can start any more commands. + // See if we can start any more commands. - if (failures_allowed && command_runner_->CanRunMore()) { - if (Edge* edge = plan_.FindWork()) { - if (edge->GetBindingBool("generator")) { -- scan_.build_log()->Close(); -- } -+ // See if we can start any more commands... -+ bool can_run_more = -+ failures_allowed && -+ plan_.more_ready() && -+ command_runner_->CanRunMore(); -+ -+ // ... but we also need a token to do that. -+ if (can_run_more && command_runner_->AcquireToken()) { ++ if (failures_allowed && plan_.more_ready() && ++ command_runner_->CanRunMore()) { + Edge* edge = plan_.FindWork(); + if (edge->GetBindingBool("generator")) { -+ scan_.build_log()->Close(); -+ } + scan_.build_log()->Close(); + } + +- if (!StartEdge(edge, err)) { + if (!StartEdge(edge, err)) { + Cleanup(); + status_->BuildFinished(); + return false; + } - -- if (!StartEdge(edge, err)) { ++ + if (edge->is_phony()) { + if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) { Cleanup(); @@ -441,6 +191,437 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 } // See if we can reap any finished commands. +diff --git a/src/build.h b/src/build.h +index d697dfb89e..7dcd111e61 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -52,6 +52,9 @@ struct Plan { + /// Returns true if there's more work to be done. + bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; } + ++ /// Returns true if there's more edges ready to start ++ bool more_ready() const { return !ready_.empty(); } ++ + /// Dumps the current state of the plan. + void Dump() const; + +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +new file mode 100644 +index 0000000000..a8f9b7139d +--- /dev/null ++++ b/src/tokenpool-gnu-make.cc +@@ -0,0 +1,211 @@ ++// Copyright 2016 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "tokenpool.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// TokenPool implementation for GNU make jobserver ++// (http://make.mad-scientist.net/papers/jobserver-implementation/) ++struct GNUmakeTokenPool : public TokenPool { ++ GNUmakeTokenPool(); ++ virtual ~GNUmakeTokenPool(); ++ ++ virtual bool Acquire(); ++ virtual void Reserve(); ++ virtual void Release(); ++ virtual void Clear(); ++ ++ bool Setup(); ++ ++ private: ++ int available_; ++ int used_; ++ ++#ifdef _WIN32 ++ // @TODO ++#else ++ int rfd_; ++ int wfd_; ++ ++ struct sigaction old_act_; ++ bool restore_; ++ ++ static int dup_rfd_; ++ static void CloseDupRfd(int signum); ++ ++ bool CheckFd(int fd); ++ bool SetAlarmHandler(); ++#endif ++ ++ void Return(); ++}; ++ ++// every instance owns an implicit token -> available_ == 1 ++GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0), ++ rfd_(-1), wfd_(-1), restore_(false) { ++} ++ ++GNUmakeTokenPool::~GNUmakeTokenPool() { ++ Clear(); ++ if (restore_) ++ sigaction(SIGALRM, &old_act_, NULL); ++} ++ ++bool GNUmakeTokenPool::CheckFd(int fd) { ++ if (fd < 0) ++ return false; ++ int ret = fcntl(fd, F_GETFD); ++ if (ret < 0) ++ return false; ++ return true; ++} ++ ++int GNUmakeTokenPool::dup_rfd_ = -1; ++ ++void GNUmakeTokenPool::CloseDupRfd(int signum) { ++ close(dup_rfd_); ++ dup_rfd_ = -1; ++} ++ ++bool GNUmakeTokenPool::SetAlarmHandler() { ++ struct sigaction act; ++ memset(&act, 0, sizeof(act)); ++ act.sa_handler = CloseDupRfd; ++ if (sigaction(SIGALRM, &act, &old_act_) < 0) { ++ perror("sigaction:"); ++ return(false); ++ } else { ++ restore_ = true; ++ return(true); ++ } ++} ++ ++bool GNUmakeTokenPool::Setup() { ++ const char *value = getenv("MAKEFLAGS"); ++ if (value) { ++ // GNU make <= 4.1 ++ const char *jobserver = strstr(value, "--jobserver-fds="); ++ // GNU make => 4.2 ++ if (!jobserver) ++ jobserver = strstr(value, "--jobserver-auth="); ++ if (jobserver) { ++ int rfd = -1; ++ int wfd = -1; ++ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && ++ CheckFd(rfd) && ++ CheckFd(wfd) && ++ SetAlarmHandler()) { ++ printf("ninja: using GNU make jobserver.\n"); ++ rfd_ = rfd; ++ wfd_ = wfd; ++ return true; ++ } ++ } ++ } ++ ++ return false; ++} ++ ++bool GNUmakeTokenPool::Acquire() { ++ if (available_ > 0) ++ return true; ++ ++#ifdef USE_PPOLL ++ pollfd pollfds[] = {{rfd_, POLLIN, 0}}; ++ int ret = poll(pollfds, 1, 0); ++#else ++ fd_set set; ++ struct timeval timeout = { 0, 0 }; ++ FD_ZERO(&set); ++ FD_SET(rfd_, &set); ++ int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); ++#endif ++ if (ret > 0) { ++ dup_rfd_ = dup(rfd_); ++ ++ if (dup_rfd_ != -1) { ++ struct sigaction act, old_act; ++ int ret = 0; ++ ++ memset(&act, 0, sizeof(act)); ++ act.sa_handler = CloseDupRfd; ++ if (sigaction(SIGCHLD, &act, &old_act) == 0) { ++ char buf; ++ ++ // block until token read, child exits or timeout ++ alarm(1); ++ ret = read(dup_rfd_, &buf, 1); ++ alarm(0); ++ ++ sigaction(SIGCHLD, &old_act, NULL); ++ } ++ ++ CloseDupRfd(0); ++ ++ if (ret > 0) { ++ available_++; ++ return true; ++ } ++ } ++ } ++ return false; ++} ++ ++void GNUmakeTokenPool::Reserve() { ++ available_--; ++ used_++; ++} ++ ++void GNUmakeTokenPool::Return() { ++ const char buf = '+'; ++ while (1) { ++ int ret = write(wfd_, &buf, 1); ++ if (ret > 0) ++ available_--; ++ if ((ret != -1) || (errno != EINTR)) ++ return; ++ // write got interrupted - retry ++ } ++} ++ ++void GNUmakeTokenPool::Release() { ++ available_++; ++ used_--; ++ if (available_ > 1) ++ Return(); ++} ++ ++void GNUmakeTokenPool::Clear() { ++ while (used_ > 0) ++ Release(); ++ while (available_ > 1) ++ Return(); ++} ++ ++struct TokenPool *TokenPool::Get(void) { ++ GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; ++ if (tokenpool->Setup()) ++ return tokenpool; ++ else ++ delete tokenpool; ++ return NULL; ++} +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +new file mode 100644 +index 0000000000..602b3316f5 +--- /dev/null ++++ b/src/tokenpool-none.cc +@@ -0,0 +1,27 @@ ++// Copyright 2016 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "tokenpool.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++// No-op TokenPool implementation ++struct TokenPool *TokenPool::Get(void) { ++ return NULL; ++} +diff --git a/src/tokenpool.h b/src/tokenpool.h +new file mode 100644 +index 0000000000..f560b1083b +--- /dev/null ++++ b/src/tokenpool.h +@@ -0,0 +1,26 @@ ++// Copyright 2016 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++// interface to token pool ++struct TokenPool { ++ virtual ~TokenPool() {} ++ ++ virtual bool Acquire() = 0; ++ virtual void Reserve() = 0; ++ virtual void Release() = 0; ++ virtual void Clear() = 0; ++ ++ // returns NULL if token pool is not available ++ static struct TokenPool *Get(void); ++}; + +From ccaccc610cd456f6068758f82e72006364c7380b Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Fri, 27 May 2016 16:47:10 +0300 +Subject: [PATCH 02/11] Add TokenPool monitoring to SubprocessSet::DoWork() + +Improve on the original jobserver client implementation. This makes +ninja a more aggressive GNU make jobserver client. + +- add monitor interface to TokenPool +- TokenPool is passed down when main loop indicates that more work is + ready and would be allowed to start if a token becomes available +- posix: update DoWork() to monitor TokenPool read file descriptor +- WaitForCommand() exits when DoWork() sets token flag +- Main loop starts over when WaitForCommand() sets token exit status +--- + src/build.cc | 53 +++++++++++++++++++++++++++++---------- + src/build.h | 3 ++- + src/build_test.cc | 9 +++++-- + src/exit_status.h | 3 ++- + src/subprocess-posix.cc | 33 ++++++++++++++++++++++-- + src/subprocess-win32.cc | 2 +- + src/subprocess.h | 8 +++++- + src/subprocess_test.cc | 47 +++++++++++++++++++++++----------- + src/tokenpool-gnu-make.cc | 5 ++++ + src/tokenpool.h | 6 +++++ + 10 files changed, 134 insertions(+), 35 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index fa096eac33..a25c349050 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -48,8 +48,9 @@ struct DryRunCommandRunner : public CommandRunner { + + // Overridden from CommandRunner: + virtual bool CanRunMore() const; ++ virtual bool AcquireToken(); + virtual bool StartCommand(Edge* edge); +- virtual bool WaitForCommand(Result* result); ++ virtual bool WaitForCommand(Result* result, bool more_ready); + + private: + queue finished_; +@@ -59,12 +60,16 @@ bool DryRunCommandRunner::CanRunMore() const { + return true; + } + ++bool DryRunCommandRunner::AcquireToken() { ++ return true; ++} ++ + bool DryRunCommandRunner::StartCommand(Edge* edge) { + finished_.push(edge); + return true; + } + +-bool DryRunCommandRunner::WaitForCommand(Result* result) { ++bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) { + if (finished_.empty()) + return false; + +@@ -452,8 +457,9 @@ struct RealCommandRunner : public CommandRunner { + explicit RealCommandRunner(const BuildConfig& config); + virtual ~RealCommandRunner(); + virtual bool CanRunMore() const; ++ virtual bool AcquireToken(); + virtual bool StartCommand(Edge* edge); +- virtual bool WaitForCommand(Result* result); ++ virtual bool WaitForCommand(Result* result, bool more_ready); + virtual vector GetActiveEdges(); + virtual void Abort(); + +@@ -490,9 +496,12 @@ bool RealCommandRunner::CanRunMore() const { + subprocs_.running_.size() + subprocs_.finished_.size(); + return (int)subproc_number < config_.parallelism + && (subprocs_.running_.empty() || +- ((config_.max_load_average <= 0.0f || +- GetLoadAverage() < config_.max_load_average) +- && (!tokens_ || tokens_->Acquire()))); ++ (config_.max_load_average <= 0.0f || ++ GetLoadAverage() < config_.max_load_average)); ++} ++ ++bool RealCommandRunner::AcquireToken() { ++ return (!tokens_ || tokens_->Acquire()); + } + + bool RealCommandRunner::StartCommand(Edge* edge) { +@@ -507,14 +516,23 @@ bool RealCommandRunner::StartCommand(Edge* edge) { + return true; + } + +-bool RealCommandRunner::WaitForCommand(Result* result) { ++bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) { + Subprocess* subproc; +- while ((subproc = subprocs_.NextFinished()) == NULL) { +- bool interrupted = subprocs_.DoWork(); ++ subprocs_.ResetTokenAvailable(); ++ while (((subproc = subprocs_.NextFinished()) == NULL) && ++ !subprocs_.IsTokenAvailable()) { ++ bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL); + if (interrupted) + return false; + } + ++ // token became available ++ if (subproc == NULL) { ++ result->status = ExitTokenAvailable; ++ return true; ++ } ++ ++ // command completed + if (tokens_) + tokens_->Release(); + +@@ -639,9 +657,14 @@ bool Builder::Build(string* err) { + // command runner. + // Second, we attempt to wait for / reap the next finished command. + while (plan_.more_to_do()) { +- // See if we can start any more commands. +- if (failures_allowed && plan_.more_ready() && +- command_runner_->CanRunMore()) { ++ // See if we can start any more commands... ++ bool can_run_more = ++ failures_allowed && ++ plan_.more_ready() && ++ command_runner_->CanRunMore(); ++ ++ // ... but we also need a token to do that. ++ if (can_run_more && command_runner_->AcquireToken()) { + Edge* edge = plan_.FindWork(); + if (edge->GetBindingBool("generator")) { + scan_.build_log()->Close(); +@@ -670,7 +693,7 @@ bool Builder::Build(string* err) { + // See if we can reap any finished commands. if (pending_commands) { CommandRunner::Result result; - if (!command_runner_->WaitForCommand(&result) || @@ -448,7 +629,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 result.status == ExitInterrupted) { Cleanup(); status_->BuildFinished(); -@@ -864,6 +917,10 @@ bool Builder::Build(string* err) { +@@ -678,6 +701,10 @@ bool Builder::Build(string* err) { return false; } @@ -459,19 +640,11 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 --pending_commands; if (!FinishCommand(&result, err)) { Cleanup(); +diff --git a/src/build.h b/src/build.h +index 7dcd111e61..35c7b97d12 100644 --- a/src/build.h +++ b/src/build.h -@@ -55,6 +55,9 @@ struct Plan { - /// Returns true if there's more work to be done. - bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; } - -+ /// Returns true if there's more edges ready to start -+ bool more_ready() const { return !ready_.empty(); } -+ - /// Dumps the current state of the plan. - void Dump() const; - -@@ -139,6 +142,7 @@ private: +@@ -139,6 +139,7 @@ struct Plan { struct CommandRunner { virtual ~CommandRunner() {} virtual bool CanRunMore() const = 0; @@ -479,46 +652,20 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 virtual bool StartCommand(Edge* edge) = 0; /// The result of waiting for a command. -@@ -150,7 +154,9 @@ struct CommandRunner { +@@ -150,7 +151,7 @@ struct CommandRunner { bool success() const { return status == ExitSuccess; } }; /// Wait for a command to complete, or return false if interrupted. - virtual bool WaitForCommand(Result* result) = 0; -+ /// If more_ready is true then the optional TokenPool is monitored too -+ /// and we return when a token becomes available. + virtual bool WaitForCommand(Result* result, bool more_ready) = 0; virtual std::vector GetActiveEdges() { return std::vector(); } virtual void Abort() {} -@@ -158,7 +164,8 @@ struct CommandRunner { - - /// Options (e.g. verbosity, parallelism) passed to a build. - struct BuildConfig { -- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), -+ BuildConfig() : verbosity(NORMAL), dry_run(false), -+ parallelism(1), parallelism_from_cmdline(false), - failures_allowed(1), max_load_average(-0.0f) {} - - enum Verbosity { -@@ -169,6 +176,7 @@ struct BuildConfig { - Verbosity verbosity; - bool dry_run; - int parallelism; -+ bool parallelism_from_cmdline; - int failures_allowed; - /// The maximum load average we must not exceed. A negative value - /// means that we do not have any limit. +diff --git a/src/build_test.cc b/src/build_test.cc +index 4ef62b2113..7a5ff4015a 100644 --- a/src/build_test.cc +++ b/src/build_test.cc -@@ -15,6 +15,7 @@ - #include "build.h" - - #include -+#include - - #include "build_log.h" - #include "deps_log.h" -@@ -473,8 +474,9 @@ struct FakeCommandRunner : public Comman +@@ -474,8 +474,9 @@ struct FakeCommandRunner : public CommandRunner { // CommandRunner impl virtual bool CanRunMore() const; @@ -529,7 +676,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 virtual vector GetActiveEdges(); virtual void Abort(); -@@ -580,6 +582,10 @@ bool FakeCommandRunner::CanRunMore() con +@@ -578,6 +579,10 @@ bool FakeCommandRunner::CanRunMore() const { return active_edges_.size() < max_active_edges_; } @@ -540,7 +687,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 bool FakeCommandRunner::StartCommand(Edge* edge) { assert(active_edges_.size() < max_active_edges_); assert(find(active_edges_.begin(), active_edges_.end(), edge) -@@ -625,7 +631,7 @@ bool FakeCommandRunner::StartCommand(Edg +@@ -649,7 +654,7 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { return true; } @@ -549,9 +696,1333 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 if (active_edges_.empty()) return false; -@@ -3302,3 +3308,356 @@ TEST_F(BuildTest, DyndepTwoLevelDiscover - EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]); - EXPECT_EQ("touch out", command_runner_.commands_ran_[4]); +diff --git a/src/exit_status.h b/src/exit_status.h +index a714ece791..75ebf6a7a0 100644 +--- a/src/exit_status.h ++++ b/src/exit_status.h +@@ -18,7 +18,8 @@ + enum ExitStatus { + ExitSuccess, + ExitFailure, +- ExitInterrupted ++ ExitTokenAvailable, ++ ExitInterrupted, + }; + + #endif // NINJA_EXIT_STATUS_H_ +diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc +index 8e785406c9..74451b0be2 100644 +--- a/src/subprocess-posix.cc ++++ b/src/subprocess-posix.cc +@@ -13,6 +13,7 @@ + // limitations under the License. + + #include "subprocess.h" ++#include "tokenpool.h" + + #include + #include +@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + } + + #ifdef USE_PPOLL +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(struct TokenPool* tokens) { + vector fds; + nfds_t nfds = 0; + +@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() { + ++nfds; + } + ++ if (tokens) { ++ pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 }; ++ fds.push_back(pfd); ++ ++nfds; ++ } ++ + interrupted_ = 0; + int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); + if (ret == -1) { +@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() { + ++i; + } + ++ if (tokens) { ++ pollfd *pfd = &fds[nfds - 1]; ++ if (pfd->fd >= 0) { ++ assert(pfd->fd == tokens->GetMonitorFd()); ++ if (pfd->revents != 0) ++ token_available_ = true; ++ } ++ } ++ + return IsInterrupted(); + } + + #else // !defined(USE_PPOLL) +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(struct TokenPool* tokens) { + fd_set set; + int nfds = 0; + FD_ZERO(&set); +@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() { + } + } + ++ if (tokens) { ++ int fd = tokens->GetMonitorFd(); ++ FD_SET(fd, &set); ++ if (nfds < fd+1) ++ nfds = fd+1; ++ } ++ + interrupted_ = 0; + int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); + if (ret == -1) { +@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() { + ++i; + } + ++ if (tokens) { ++ int fd = tokens->GetMonitorFd(); ++ if ((fd >= 0) && FD_ISSET(fd, &set)) ++ token_available_ = true; ++ } ++ + return IsInterrupted(); + } + #endif // !defined(USE_PPOLL) +diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc +index ff3baaca7f..66d2c2c430 100644 +--- a/src/subprocess-win32.cc ++++ b/src/subprocess-win32.cc +@@ -251,7 +251,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + return subprocess; + } + +-bool SubprocessSet::DoWork() { ++bool SubprocessSet::DoWork(struct TokenPool* tokens) { + DWORD bytes_read; + Subprocess* subproc; + OVERLAPPED* overlapped; +diff --git a/src/subprocess.h b/src/subprocess.h +index 9e3d2ee98f..9ea67ea477 100644 +--- a/src/subprocess.h ++++ b/src/subprocess.h +@@ -76,6 +76,8 @@ struct Subprocess { + friend struct SubprocessSet; + }; + ++struct TokenPool; ++ + /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses. + /// DoWork() waits for any state change in subprocesses; finished_ + /// is a queue of subprocesses as they finish. +@@ -84,13 +86,17 @@ struct SubprocessSet { + ~SubprocessSet(); + + Subprocess* Add(const std::string& command, bool use_console = false); +- bool DoWork(); ++ bool DoWork(struct TokenPool* tokens); + Subprocess* NextFinished(); + void Clear(); + + std::vector running_; + std::queue finished_; + ++ bool token_available_; ++ bool IsTokenAvailable() { return token_available_; } ++ void ResetTokenAvailable() { token_available_ = false; } ++ + #ifdef _WIN32 + static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType); + static HANDLE ioport_; +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index 073fe86931..4bc8083e26 100644 +--- a/src/subprocess_test.cc ++++ b/src/subprocess_test.cc +@@ -45,10 +45,12 @@ TEST_F(SubprocessTest, BadCommandStderr) { + Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + // Pretend we discovered that stderr was ready for writing. +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -59,10 +61,12 @@ TEST_F(SubprocessTest, NoSuchCommand) { + Subprocess* subproc = subprocs_.Add("ninja_no_such_command"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + // Pretend we discovered that stderr was ready for writing. +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -78,9 +82,11 @@ TEST_F(SubprocessTest, InterruptChild) { + Subprocess* subproc = subprocs_.Add("kill -INT $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -90,7 +96,7 @@ TEST_F(SubprocessTest, InterruptParent) { + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -102,9 +108,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) { + Subprocess* subproc = subprocs_.Add("kill -TERM $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -114,7 +122,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigTerm) { + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -126,9 +134,11 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) { + Subprocess* subproc = subprocs_.Add("kill -HUP $$"); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -138,7 +148,7 @@ TEST_F(SubprocessTest, InterruptParentWithSigHup) { + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { +- bool interrupted = subprocs_.DoWork(); ++ bool interrupted = subprocs_.DoWork(NULL); + if (interrupted) + return; + } +@@ -153,9 +163,11 @@ TEST_F(SubprocessTest, Console) { + subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true); + ASSERT_NE((Subprocess*)0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } +@@ -167,9 +179,11 @@ TEST_F(SubprocessTest, SetWithSingle) { + Subprocess* subproc = subprocs_.Add(kSimpleCommand); + ASSERT_NE((Subprocess *) 0, subproc); + ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_NE("", subproc->GetOutput()); + +@@ -200,12 +214,13 @@ TEST_F(SubprocessTest, SetWithMulti) { + ASSERT_EQ("", processes[i]->GetOutput()); + } + ++ subprocs_.ResetTokenAvailable(); + while (!processes[0]->Done() || !processes[1]->Done() || + !processes[2]->Done()) { + ASSERT_GT(subprocs_.running_.size(), 0u); +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } +- ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + ASSERT_EQ(0u, subprocs_.running_.size()); + ASSERT_EQ(3u, subprocs_.finished_.size()); + +@@ -237,8 +252,10 @@ TEST_F(SubprocessTest, SetWithLots) { + ASSERT_NE((Subprocess *) 0, subproc); + procs.push_back(subproc); + } ++ subprocs_.ResetTokenAvailable(); + while (!subprocs_.running_.empty()) +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + for (size_t i = 0; i < procs.size(); ++i) { + ASSERT_EQ(ExitSuccess, procs[i]->Finish()); + ASSERT_NE("", procs[i]->GetOutput()); +@@ -254,9 +271,11 @@ TEST_F(SubprocessTest, SetWithLots) { + // that stdin is closed. + TEST_F(SubprocessTest, ReadStdin) { + Subprocess* subproc = subprocs_.Add("cat -"); ++ subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { +- subprocs_.DoWork(); ++ subprocs_.DoWork(NULL); + } ++ ASSERT_EQ(false, subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_EQ(1u, subprocs_.finished_.size()); + } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index a8f9b7139d..396bb7d874 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -33,6 +33,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Reserve(); + virtual void Release(); + virtual void Clear(); ++ virtual int GetMonitorFd(); + + bool Setup(); + +@@ -201,6 +202,10 @@ void GNUmakeTokenPool::Clear() { + Return(); + } + ++int GNUmakeTokenPool::GetMonitorFd() { ++ return(rfd_); ++} ++ + struct TokenPool *TokenPool::Get(void) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; + if (tokenpool->Setup()) +diff --git a/src/tokenpool.h b/src/tokenpool.h +index f560b1083b..301e1998ee 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -21,6 +21,12 @@ struct TokenPool { + virtual void Release() = 0; + virtual void Clear() = 0; + ++#ifdef _WIN32 ++ // @TODO ++#else ++ virtual int GetMonitorFd() = 0; ++#endif ++ + // returns NULL if token pool is not available + static struct TokenPool *Get(void); + }; + +From d09f3d77821b3b1fdf09fc0ef8e814907675eafb Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sun, 12 Nov 2017 16:58:55 +0200 +Subject: [PATCH 03/11] Ignore jobserver when -jN is forced on command line + +This emulates the behaviour of GNU make. + +- add parallelism_from_cmdline flag to build configuration +- set the flag when -jN is given on command line +- pass the flag to TokenPool::Get() +- GNUmakeTokenPool::Setup() + * prints a warning when the flag is true and jobserver was detected + * returns false, i.e. jobserver will be ignored +- ignore config.parallelism in CanRunMore() when we have a valid + TokenPool, because it gets always initialized to a default when not + given on the command line +--- + src/build.cc | 10 ++++++---- + src/build.h | 4 +++- + src/ninja.cc | 1 + + src/tokenpool-gnu-make.cc | 34 +++++++++++++++++++--------------- + src/tokenpool-none.cc | 4 ++-- + src/tokenpool.h | 4 ++-- + 6 files changed, 33 insertions(+), 24 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index a25c349050..406a84ec39 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -470,7 +470,7 @@ struct RealCommandRunner : public CommandRunner { + }; + + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { +- tokens_ = TokenPool::Get(); ++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); + } + + RealCommandRunner::~RealCommandRunner() { +@@ -492,9 +492,11 @@ void RealCommandRunner::Abort() { + } + + bool RealCommandRunner::CanRunMore() const { +- size_t subproc_number = +- subprocs_.running_.size() + subprocs_.finished_.size(); +- return (int)subproc_number < config_.parallelism ++ bool parallelism_limit_not_reached = ++ tokens_ || // ignore config_.parallelism ++ ((int) (subprocs_.running_.size() + ++ subprocs_.finished_.size()) < config_.parallelism); ++ return parallelism_limit_not_reached + && (subprocs_.running_.empty() || + (config_.max_load_average <= 0.0f || + GetLoadAverage() < config_.max_load_average)); +diff --git a/src/build.h b/src/build.h +index 35c7b97d12..dfde576573 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -159,7 +159,8 @@ struct CommandRunner { + + /// Options (e.g. verbosity, parallelism) passed to a build. + struct BuildConfig { +- BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), ++ BuildConfig() : verbosity(NORMAL), dry_run(false), ++ parallelism(1), parallelism_from_cmdline(false), + failures_allowed(1), max_load_average(-0.0f) {} + + enum Verbosity { +@@ -171,6 +172,7 @@ struct BuildConfig { + Verbosity verbosity; + bool dry_run; + int parallelism; ++ bool parallelism_from_cmdline; + int failures_allowed; + /// The maximum load average we must not exceed. A negative value + /// means that we do not have any limit. +diff --git a/src/ninja.cc b/src/ninja.cc +index df39ba92d1..d904c56c4e 100644 +--- a/src/ninja.cc ++++ b/src/ninja.cc +@@ -1447,6 +1447,7 @@ int ReadFlags(int* argc, char*** argv, + // We want to run N jobs in parallel. For N = 0, INT_MAX + // is close enough to infinite for most sane builds. + config->parallelism = value > 0 ? value : INT_MAX; ++ config->parallelism_from_cmdline = true; + deferGuessParallelism.needGuess = false; + break; + } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index 396bb7d874..af4be05a31 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(); ++ bool Setup(bool ignore); + + private: + int available_; +@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup() { ++bool GNUmakeTokenPool::Setup(bool ignore) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -109,16 +109,20 @@ bool GNUmakeTokenPool::Setup() { + if (!jobserver) + jobserver = strstr(value, "--jobserver-auth="); + if (jobserver) { +- int rfd = -1; +- int wfd = -1; +- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && +- CheckFd(rfd) && +- CheckFd(wfd) && +- SetAlarmHandler()) { +- printf("ninja: using GNU make jobserver.\n"); +- rfd_ = rfd; +- wfd_ = wfd; +- return true; ++ if (ignore) { ++ printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ } else { ++ int rfd = -1; ++ int wfd = -1; ++ if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && ++ CheckFd(rfd) && ++ CheckFd(wfd) && ++ SetAlarmHandler()) { ++ printf("ninja: using GNU make jobserver.\n"); ++ rfd_ = rfd; ++ wfd_ = wfd; ++ return true; ++ } + } + } + } +@@ -206,9 +210,9 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(void) { ++struct TokenPool *TokenPool::Get(bool ignore) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup()) ++ if (tokenpool->Setup(ignore)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 602b3316f5..199b22264b 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -22,6 +22,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(void) { ++struct TokenPool *TokenPool::Get(bool ignore) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 301e1998ee..878a0933c2 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -1,4 +1,4 @@ +-// Copyright 2016 Google Inc. All Rights Reserved. ++// Copyright 2016-2017 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -28,5 +28,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(void); ++ static struct TokenPool *Get(bool ignore); + }; + +From dfe4ca753caee65bf9041e2b4e883dfa172a5c6a Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sun, 12 Nov 2017 18:04:12 +0200 +Subject: [PATCH 04/11] Honor -lN from MAKEFLAGS + +This emulates the behaviour of GNU make. + +- build: make a copy of max_load_average and pass it to TokenPool. +- GNUmakeTokenPool: if we detect a jobserver and a valid -lN argument in + MAKEFLAGS then set max_load_average to N. +--- + src/build.cc | 10 +++++++--- + src/tokenpool-gnu-make.cc | 19 +++++++++++++++---- + src/tokenpool-none.cc | 2 +- + src/tokenpool.h | 2 +- + 4 files changed, 24 insertions(+), 9 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index 406a84ec39..9e6272d035 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -464,13 +464,17 @@ struct RealCommandRunner : public CommandRunner { + virtual void Abort(); + + const BuildConfig& config_; ++ // copy of config_.max_load_average; can be modified by TokenPool setup ++ double max_load_average_; + SubprocessSet subprocs_; + TokenPool *tokens_; + map subproc_to_edge_; + }; + + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { +- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline); ++ max_load_average_ = config.max_load_average; ++ tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, ++ max_load_average_); + } + + RealCommandRunner::~RealCommandRunner() { +@@ -498,8 +502,8 @@ bool RealCommandRunner::CanRunMore() const { + subprocs_.finished_.size()) < config_.parallelism); + return parallelism_limit_not_reached + && (subprocs_.running_.empty() || +- (config_.max_load_average <= 0.0f || +- GetLoadAverage() < config_.max_load_average)); ++ (max_load_average_ <= 0.0f || ++ GetLoadAverage() < max_load_average_)); + } + + bool RealCommandRunner::AcquireToken() { +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index af4be05a31..fb654c4d88 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -35,7 +35,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(bool ignore); ++ bool Setup(bool ignore, double& max_load_average); + + private: + int available_; +@@ -100,7 +100,7 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup(bool ignore) { ++bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -118,9 +118,20 @@ bool GNUmakeTokenPool::Setup(bool ignore) { + CheckFd(rfd) && + CheckFd(wfd) && + SetAlarmHandler()) { ++ const char *l_arg = strstr(value, " -l"); ++ int load_limit = -1; ++ + printf("ninja: using GNU make jobserver.\n"); + rfd_ = rfd; + wfd_ = wfd; ++ ++ // translate GNU make -lN to ninja -lN ++ if (l_arg && ++ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && ++ (load_limit > 0)) { ++ max_load_average = load_limit; ++ } ++ + return true; + } + } +@@ -210,9 +221,9 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(bool ignore) { ++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore)) ++ if (tokenpool->Setup(ignore, max_load_average)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 199b22264b..e8e25426c3 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -22,6 +22,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore) { ++struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 878a0933c2..f9e8cc2ee0 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -28,5 +28,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore); ++ static struct TokenPool *Get(bool ignore, double& max_load_average); + }; + +From 1c10047fc6a3269ba42839da19361e09cbc06ff0 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Wed, 6 Dec 2017 22:14:21 +0200 +Subject: [PATCH 05/11] Use LinePrinter for TokenPool messages + +- replace printf() with calls to LinePrinter +- print GNU make jobserver message only when verbose build is requested +--- + src/build.cc | 1 + + src/tokenpool-gnu-make.cc | 22 ++++++++++++++++------ + src/tokenpool-none.cc | 4 +++- + src/tokenpool.h | 4 +++- + 4 files changed, 23 insertions(+), 8 deletions(-) + +diff --git a/src/build.cc b/src/build.cc +index 9e6272d035..662e4bd7be 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -474,6 +474,7 @@ struct RealCommandRunner : public CommandRunner { + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { + max_load_average_ = config.max_load_average; + tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, ++ config_.verbosity == BuildConfig::VERBOSE, + max_load_average_); + } + +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index fb654c4d88..b0d3e6ebc4 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -23,6 +23,8 @@ + #include + #include + ++#include "line_printer.h" ++ + // TokenPool implementation for GNU make jobserver + // (http://make.mad-scientist.net/papers/jobserver-implementation/) + struct GNUmakeTokenPool : public TokenPool { +@@ -35,7 +37,7 @@ struct GNUmakeTokenPool : public TokenPool { + virtual void Clear(); + virtual int GetMonitorFd(); + +- bool Setup(bool ignore, double& max_load_average); ++ bool Setup(bool ignore, bool verbose, double& max_load_average); + + private: + int available_; +@@ -100,7 +102,9 @@ bool GNUmakeTokenPool::SetAlarmHandler() { + } + } + +-bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { ++bool GNUmakeTokenPool::Setup(bool ignore, ++ bool verbose, ++ double& max_load_average) { + const char *value = getenv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 +@@ -109,8 +113,10 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + if (!jobserver) + jobserver = strstr(value, "--jobserver-auth="); + if (jobserver) { ++ LinePrinter printer; ++ + if (ignore) { +- printf("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); + } else { + int rfd = -1; + int wfd = -1; +@@ -121,7 +127,9 @@ bool GNUmakeTokenPool::Setup(bool ignore, double& max_load_average) { + const char *l_arg = strstr(value, " -l"); + int load_limit = -1; + +- printf("ninja: using GNU make jobserver.\n"); ++ if (verbose) { ++ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); ++ } + rfd_ = rfd; + wfd_ = wfd; + +@@ -221,9 +229,11 @@ int GNUmakeTokenPool::GetMonitorFd() { + return(rfd_); + } + +-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { ++struct TokenPool *TokenPool::Get(bool ignore, ++ bool verbose, ++ double& max_load_average) { + GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore, max_load_average)) ++ if (tokenpool->Setup(ignore, verbose, max_load_average)) + return tokenpool; + else + delete tokenpool; +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index e8e25426c3..1c1c499c8d 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -22,6 +22,8 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore, double& max_load_average) { ++struct TokenPool *TokenPool::Get(bool ignore, ++ bool verbose, ++ double& max_load_average) { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index f9e8cc2ee0..4bf477f20c 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -28,5 +28,7 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore, double& max_load_average); ++ static struct TokenPool *Get(bool ignore, ++ bool verbose, ++ double& max_load_average); + }; + +From fdbf68416e3574add3bffd0b637d0694fbaba320 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sat, 7 Apr 2018 17:11:21 +0300 +Subject: [PATCH 06/11] Prepare PR for merging + +- fix Windows build error in no-op TokenPool implementation +- improve Acquire() to block for a maximum of 100ms +- address review comments +--- + src/build.h | 2 ++ + src/tokenpool-gnu-make.cc | 53 +++++++++++++++++++++++++++++++++------ + src/tokenpool-none.cc | 7 +----- + 3 files changed, 49 insertions(+), 13 deletions(-) + +diff --git a/src/build.h b/src/build.h +index dfde576573..66ddefb888 100644 +--- a/src/build.h ++++ b/src/build.h +@@ -151,6 +151,8 @@ struct CommandRunner { + bool success() const { return status == ExitSuccess; } + }; + /// Wait for a command to complete, or return false if interrupted. ++ /// If more_ready is true then the optional TokenPool is monitored too ++ /// and we return when a token becomes available. + virtual bool WaitForCommand(Result* result, bool more_ready) = 0; + + virtual std::vector GetActiveEdges() { return std::vector(); } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index b0d3e6ebc4..4132bb06d9 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. ++// Copyright 2016-2018 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -153,6 +154,15 @@ bool GNUmakeTokenPool::Acquire() { + if (available_ > 0) + return true; + ++ // Please read ++ // ++ // http://make.mad-scientist.net/papers/jobserver-implementation/ ++ // ++ // for the reasoning behind the following code. ++ // ++ // Try to read one character from the pipe. Returns true on success. ++ // ++ // First check if read() would succeed without blocking. + #ifdef USE_PPOLL + pollfd pollfds[] = {{rfd_, POLLIN, 0}}; + int ret = poll(pollfds, 1, 0); +@@ -164,33 +174,62 @@ bool GNUmakeTokenPool::Acquire() { + int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); + #endif + if (ret > 0) { ++ // Handle potential race condition: ++ // - the above check succeeded, i.e. read() should not block ++ // - the character disappears before we call read() ++ // ++ // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_ ++ // can safely be closed by signal handlers without affecting rfd_. + dup_rfd_ = dup(rfd_); + + if (dup_rfd_ != -1) { + struct sigaction act, old_act; + int ret = 0; + ++ // Temporarily replace SIGCHLD handler with our own + memset(&act, 0, sizeof(act)); + act.sa_handler = CloseDupRfd; + if (sigaction(SIGCHLD, &act, &old_act) == 0) { +- char buf; +- +- // block until token read, child exits or timeout +- alarm(1); +- ret = read(dup_rfd_, &buf, 1); +- alarm(0); ++ struct itimerval timeout; ++ ++ // install a 100ms timeout that generates SIGALARM on expiration ++ memset(&timeout, 0, sizeof(timeout)); ++ timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] ++ if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { ++ char buf; ++ ++ // Now try to read() from dup_rfd_. Return values from read(): ++ // ++ // 1. token read -> 1 ++ // 2. pipe closed -> 0 ++ // 3. alarm expires -> -1 (EINTR) ++ // 4. child exits -> -1 (EINTR) ++ // 5. alarm expired before entering read() -> -1 (EBADF) ++ // 6. child exited before entering read() -> -1 (EBADF) ++ // 7. child exited before handler is installed -> go to 1 - 3 ++ ret = read(dup_rfd_, &buf, 1); ++ ++ // disarm timer ++ memset(&timeout, 0, sizeof(timeout)); ++ setitimer(ITIMER_REAL, &timeout, NULL); ++ } + + sigaction(SIGCHLD, &old_act, NULL); + } + + CloseDupRfd(0); + ++ // Case 1 from above list + if (ret > 0) { + available_++; + return true; + } + } + } ++ ++ // read() would block, i.e. no token available, ++ // cases 2-6 from above list or ++ // select() / poll() / dup() / sigaction() / setitimer() failed + return false; + } + +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 1c1c499c8d..4c592875b4 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. ++// Copyright 2016-2018 Google Inc. All Rights Reserved. + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -14,11 +14,6 @@ + + #include "tokenpool.h" + +-#include +-#include +-#include +-#include +-#include + #include + + // No-op TokenPool implementation + +From ec6220a0baf7d3a6eaf1a2b75bf8960ddfe24c2f Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Fri, 25 May 2018 00:17:07 +0300 +Subject: [PATCH 07/11] Add tests for TokenPool + +- TokenPool setup +- GetMonitorFd() API +- implicit token and tokens in jobserver pipe +- Acquire() / Reserve() / Release() protocol +- Clear() API +--- + configure.py | 1 + + src/tokenpool_test.cc | 198 ++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 199 insertions(+) + create mode 100644 src/tokenpool_test.cc + +diff --git a/configure.py b/configure.py +index db3492c93c..dc8a0066b7 100755 +--- a/configure.py ++++ b/configure.py +@@ -590,6 +590,7 @@ def has_re2c(): + 'string_piece_util_test', + 'subprocess_test', + 'test', ++ 'tokenpool_test', + 'util_test']: + objs += cxx(name, variables=cxxvariables) + if platform.is_windows(): +diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc +new file mode 100644 +index 0000000000..6c89064ca4 +--- /dev/null ++++ b/src/tokenpool_test.cc +@@ -0,0 +1,198 @@ ++// Copyright 2018 Google Inc. All Rights Reserved. ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); ++// you may not use this file except in compliance with the License. ++// You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, ++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++// See the License for the specific language governing permissions and ++// limitations under the License. ++ ++#include "tokenpool.h" ++ ++#include "test.h" ++ ++#ifndef _WIN32 ++#include ++#include ++#include ++ ++#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") ++#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true); ++#endif ++ ++namespace { ++ ++const double kLoadAverageDefault = -1.23456789; ++ ++struct TokenPoolTest : public testing::Test { ++ double load_avg_; ++ TokenPool *tokens_; ++#ifndef _WIN32 ++ char buf_[1024]; ++ int fds_[2]; ++#endif ++ ++ virtual void SetUp() { ++ load_avg_ = kLoadAverageDefault; ++ tokens_ = NULL; ++#ifndef _WIN32 ++ ENVIRONMENT_CLEAR(); ++ if (pipe(fds_) < 0) ++ ASSERT_TRUE(false); ++#endif ++ } ++ ++ void CreatePool(const char *format, bool ignore_jobserver) { ++#ifndef _WIN32 ++ if (format) { ++ sprintf(buf_, format, fds_[0], fds_[1]); ++ ENVIRONMENT_INIT(buf_); ++ } ++#endif ++ tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_); ++ } ++ ++ void CreateDefaultPool() { ++ CreatePool("foo --jobserver-auth=%d,%d bar", false); ++ } ++ ++ virtual void TearDown() { ++ if (tokens_) ++ delete tokens_; ++#ifndef _WIN32 ++ close(fds_[0]); ++ close(fds_[1]); ++ ENVIRONMENT_CLEAR(); ++#endif ++ } ++}; ++ ++} // anonymous namespace ++ ++// verifies none implementation ++TEST_F(TokenPoolTest, NoTokenPool) { ++ CreatePool(NULL, false); ++ ++ EXPECT_EQ(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++#ifndef _WIN32 ++TEST_F(TokenPoolTest, SuccessfulOldSetup) { ++ // GNUmake <= 4.1 ++ CreatePool("foo --jobserver-fds=%d,%d bar", false); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, SuccessfulNewSetup) { ++ // GNUmake => 4.2 ++ CreateDefaultPool(); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, IgnoreWithJN) { ++ CreatePool("foo --jobserver-auth=%d,%d bar", true); ++ ++ EXPECT_EQ(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, HonorLN) { ++ CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false); ++ ++ EXPECT_NE(NULL, tokens_); ++ EXPECT_EQ(9.0, load_avg_); ++} ++ ++TEST_F(TokenPoolTest, MonitorFD) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); ++} ++ ++TEST_F(TokenPoolTest, ImplicitToken) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++ ++TEST_F(TokenPoolTest, TwoTokens) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // jobserver offers 2nd token ++ ASSERT_EQ(1u, write(fds_[1], "T", 1)); ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // release 2nd token ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // release implict token - must return 2nd token back to jobserver ++ tokens_->Release(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // there must be one token in the pipe ++ EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++ ++TEST_F(TokenPoolTest, Clear) { ++ CreateDefaultPool(); ++ ++ ASSERT_NE(NULL, tokens_); ++ EXPECT_EQ(kLoadAverageDefault, load_avg_); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ // jobserver offers 2nd & 3rd token ++ ASSERT_EQ(2u, write(fds_[1], "TT", 2)); ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ tokens_->Reserve(); ++ EXPECT_FALSE(tokens_->Acquire()); ++ ++ tokens_->Clear(); ++ EXPECT_TRUE(tokens_->Acquire()); ++ ++ // there must be two tokens in the pipe ++ EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); ++ ++ // implicit token ++ EXPECT_TRUE(tokens_->Acquire()); ++} ++#endif + +From e59d8858327126d1593fd0b8e607975a79072e92 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Thu, 24 May 2018 18:52:45 +0300 +Subject: [PATCH 08/11] Add tests for subprocess module + +- add TokenPoolTest stub to provide TokenPool::GetMonitorFd() +- add two tests + * both tests set up a dummy GNUmake jobserver pipe + * both tests call DoWork() with TokenPoolTest + * test 1: verify that DoWork() detects when a token is available + * test 2: verify that DoWork() works as before without a token +- the tests are not compiled in under Windows +--- + src/subprocess_test.cc | 76 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 76 insertions(+) + +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index 4bc8083e26..6264c8bf11 100644 +--- a/src/subprocess_test.cc ++++ b/src/subprocess_test.cc +@@ -13,6 +13,7 @@ + // limitations under the License. + + #include "subprocess.h" ++#include "tokenpool.h" + + #include "test.h" + +@@ -34,8 +35,23 @@ const char* kSimpleCommand = "cmd /c dir \\"; + const char* kSimpleCommand = "ls /"; + #endif + ++struct TokenPoolTest : public TokenPool { ++ bool Acquire() { return false; } ++ void Reserve() {} ++ void Release() {} ++ void Clear() {} ++ ++#ifdef _WIN32 ++ // @TODO ++#else ++ int _fd; ++ int GetMonitorFd() { return _fd; } ++#endif ++}; ++ + struct SubprocessTest : public testing::Test { + SubprocessSet subprocs_; ++ TokenPoolTest tokens_; + }; + + } // anonymous namespace +@@ -280,3 +296,63 @@ TEST_F(SubprocessTest, ReadStdin) { + ASSERT_EQ(1u, subprocs_.finished_.size()); + } + #endif // _WIN32 ++ ++// @TODO: remove once TokenPool implementation for Windows is available ++#ifndef _WIN32 ++TEST_F(SubprocessTest, TokenAvailable) { ++ Subprocess* subproc = subprocs_.Add(kSimpleCommand); ++ ASSERT_NE((Subprocess *) 0, subproc); ++ ++ // simulate GNUmake jobserver pipe with 1 token ++ int fds[2]; ++ ASSERT_EQ(0u, pipe(fds)); ++ tokens_._fd = fds[0]; ++ ASSERT_EQ(1u, write(fds[1], "T", 1)); ++ ++ subprocs_.ResetTokenAvailable(); ++ subprocs_.DoWork(&tokens_); ++ ++ EXPECT_TRUE(subprocs_.IsTokenAvailable()); ++ EXPECT_EQ(0u, subprocs_.finished_.size()); ++ ++ // remove token to let DoWork() wait for command again ++ char token; ++ ASSERT_EQ(1u, read(fds[0], &token, 1)); ++ ++ while (!subproc->Done()) { ++ subprocs_.DoWork(&tokens_); ++ } ++ ++ close(fds[1]); ++ close(fds[0]); ++ ++ EXPECT_EQ(ExitSuccess, subproc->Finish()); ++ EXPECT_NE("", subproc->GetOutput()); ++ ++ EXPECT_EQ(1u, subprocs_.finished_.size()); ++} ++ ++TEST_F(SubprocessTest, TokenNotAvailable) { ++ Subprocess* subproc = subprocs_.Add(kSimpleCommand); ++ ASSERT_NE((Subprocess *) 0, subproc); ++ ++ // simulate GNUmake jobserver pipe with 0 tokens ++ int fds[2]; ++ ASSERT_EQ(0u, pipe(fds)); ++ tokens_._fd = fds[0]; ++ ++ subprocs_.ResetTokenAvailable(); ++ while (!subproc->Done()) { ++ subprocs_.DoWork(&tokens_); ++ } ++ ++ close(fds[1]); ++ close(fds[0]); ++ ++ EXPECT_FALSE(subprocs_.IsTokenAvailable()); ++ EXPECT_EQ(ExitSuccess, subproc->Finish()); ++ EXPECT_NE("", subproc->GetOutput()); ++ ++ EXPECT_EQ(1u, subprocs_.finished_.size()); ++} ++#endif // _WIN32 + +From 0145e2d4db64ea6c21aeb371928e4071f65164eb Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Sat, 26 May 2018 23:17:51 +0300 +Subject: [PATCH 09/11] Add tests for build module + +Add tests that verify the token functionality of the builder main loop. +We replace the default fake command runner with a special version where +the tests can control each call to AcquireToken(), CanRunMore() and +WaitForCommand(). +--- + src/build_test.cc | 364 ++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 364 insertions(+) + +diff --git a/src/build_test.cc b/src/build_test.cc +index 7a5ff4015a..dd41dfbe1d 100644 +--- a/src/build_test.cc ++++ b/src/build_test.cc +@@ -15,6 +15,7 @@ + #include "build.h" + + #include ++#include + + #include "build_log.h" + #include "deps_log.h" +@@ -3990,3 +3991,366 @@ TEST_F(BuildTest, ValidationWithCircularDependency) { + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err); } + +/// The token tests are concerned with the main loop functionality when @@ -706,6 +2177,16 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + } +} + ++TEST_F(BuildTokenTest, CompleteNoWork) { ++ // plan should not execute anything ++ string err; ++ ++ EXPECT_TRUE(builder_.Build(&err)); ++ EXPECT_EQ("", err); ++ ++ EXPECT_EQ(0u, token_command_runner_.commands_ran_.size()); ++} ++ +TEST_F(BuildTokenTest, DoNotAquireToken) { + // plan should execute one command + string err; @@ -906,109 +2387,106 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + token_command_runner_.commands_ran_[1] == "cat in1 > out1")); + EXPECT_TRUE(token_command_runner_.commands_ran_[2] == "cat out1 out2 > out12"); +} ---- a/src/exit_status.h -+++ b/src/exit_status.h -@@ -18,7 +18,8 @@ - enum ExitStatus { - ExitSuccess, - ExitFailure, -- ExitInterrupted -+ ExitTokenAvailable, -+ ExitInterrupted, - }; + +From f016e5430c9123d34a73ea7ad28693b20ee59d6d Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Mon, 8 Oct 2018 17:47:50 +0300 +Subject: [PATCH 10/11] Add Win32 implementation for GNUmakeTokenPool + +GNU make uses a semaphore as jobserver protocol on Win32. See also + + https://www.gnu.org/software/make/manual/html_node/Windows-Jobserver.html + +Usage is pretty simple and straightforward, i.e. WaitForSingleObject() +to obtain a token and ReleaseSemaphore() to return it. + +Unfortunately subprocess-win32.cc uses an I/O completion port (IOCP). +IOCPs aren't waitable objects, i.e. we can't use WaitForMultipleObjects() +to wait on the IOCP and the token semaphore at the same time. + +Therefore GNUmakeTokenPoolWin32 creates a child thread that waits on the +token semaphore and posts a dummy I/O completion status on the IOCP when +it was able to obtain a token. That unblocks SubprocessSet::DoWork() and +it can then check if a token became available or not. + +- split existing GNUmakeTokenPool into common and platform bits +- add GNUmakeTokenPool interface +- move the Posix bits to GNUmakeTokenPoolPosix +- add the Win32 bits as GNUmakeTokenPoolWin32 +- move Setup() method up to TokenPool interface +- update Subprocess & TokenPool tests accordingly +--- + configure.py | 8 +- + src/build.cc | 11 +- + src/subprocess-win32.cc | 9 ++ + src/subprocess_test.cc | 34 ++++- + src/tokenpool-gnu-make-posix.cc | 203 +++++++++++++++++++++++++++ + src/tokenpool-gnu-make-win32.cc | 237 ++++++++++++++++++++++++++++++++ + src/tokenpool-gnu-make.cc | 203 ++------------------------- + src/tokenpool-gnu-make.h | 40 ++++++ + src/tokenpool-none.cc | 4 +- + src/tokenpool.h | 18 ++- + src/tokenpool_test.cc | 113 ++++++++++++--- + 11 files changed, 653 insertions(+), 227 deletions(-) + create mode 100644 src/tokenpool-gnu-make-posix.cc + create mode 100644 src/tokenpool-gnu-make-win32.cc + create mode 100644 src/tokenpool-gnu-make.h + +diff --git a/configure.py b/configure.py +index dc8a0066b7..a239b90eef 100755 +--- a/configure.py ++++ b/configure.py +@@ -517,12 +517,13 @@ def has_re2c(): + 'state', + 'status', + 'string_piece_util', ++ 'tokenpool-gnu-make', + 'util', + 'version']: + objs += cxx(name, variables=cxxvariables) + if platform.is_windows(): + for name in ['subprocess-win32', +- 'tokenpool-none', ++ 'tokenpool-gnu-make-win32', + 'includes_normalize-win32', + 'msvc_helper-win32', + 'msvc_helper_main-win32']: +@@ -531,8 +532,9 @@ def has_re2c(): + objs += cxx('minidump-win32', variables=cxxvariables) + objs += cc('getopt') + else: +- objs += cxx('subprocess-posix') +- objs += cxx('tokenpool-gnu-make') ++ for name in ['subprocess-posix', ++ 'tokenpool-gnu-make-posix']: ++ objs += cxx(name) + if platform.is_aix(): + objs += cc('getopt') + if platform.is_msvc(): +diff --git a/src/build.cc b/src/build.cc +index 662e4bd7be..20c3bdc2a0 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -473,9 +473,14 @@ struct RealCommandRunner : public CommandRunner { - #endif // NINJA_EXIT_STATUS_H_ ---- a/src/ninja.cc -+++ b/src/ninja.cc -@@ -1289,6 +1289,7 @@ int ReadFlags(int* argc, char*** argv, - // We want to run N jobs in parallel. For N = 0, INT_MAX - // is close enough to infinite for most sane builds. - config->parallelism = value > 0 ? value : INT_MAX; -+ config->parallelism_from_cmdline = true; - break; - } - case 'k': { ---- a/src/subprocess-posix.cc -+++ b/src/subprocess-posix.cc -@@ -13,6 +13,7 @@ - // limitations under the License. - - #include "subprocess.h" -+#include "tokenpool.h" - - #include - #include -@@ -249,7 +250,7 @@ Subprocess *SubprocessSet::Add(const str - } - - #ifdef USE_PPOLL --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - vector fds; - nfds_t nfds = 0; - -@@ -263,6 +264,12 @@ bool SubprocessSet::DoWork() { - ++nfds; - } - -+ if (tokens) { -+ pollfd pfd = { tokens->GetMonitorFd(), POLLIN | POLLPRI, 0 }; -+ fds.push_back(pfd); -+ ++nfds; -+ } -+ - interrupted_ = 0; - int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); - if (ret == -1) { -@@ -295,11 +302,20 @@ bool SubprocessSet::DoWork() { - ++i; - } - -+ if (tokens) { -+ pollfd *pfd = &fds[nfds - 1]; -+ if (pfd->fd >= 0) { -+ assert(pfd->fd == tokens->GetMonitorFd()); -+ if (pfd->revents != 0) -+ token_available_ = true; + RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) { + max_load_average_ = config.max_load_average; +- tokens_ = TokenPool::Get(config_.parallelism_from_cmdline, +- config_.verbosity == BuildConfig::VERBOSE, +- max_load_average_); ++ if ((tokens_ = TokenPool::Get()) != NULL) { ++ if (!tokens_->Setup(config_.parallelism_from_cmdline, ++ config_.verbosity == BuildConfig::VERBOSE, ++ max_load_average_)) { ++ delete tokens_; ++ tokens_ = NULL; + } + } -+ - return IsInterrupted(); } - #else // !defined(USE_PPOLL) --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - fd_set set; - int nfds = 0; - FD_ZERO(&set); -@@ -314,6 +330,13 @@ bool SubprocessSet::DoWork() { - } - } - -+ if (tokens) { -+ int fd = tokens->GetMonitorFd(); -+ FD_SET(fd, &set); -+ if (nfds < fd+1) -+ nfds = fd+1; -+ } -+ - interrupted_ = 0; - int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); - if (ret == -1) { -@@ -342,6 +365,12 @@ bool SubprocessSet::DoWork() { - ++i; - } - -+ if (tokens) { -+ int fd = tokens->GetMonitorFd(); -+ if ((fd >= 0) && FD_ISSET(fd, &set)) -+ token_available_ = true; -+ } -+ - return IsInterrupted(); - } - #endif // !defined(USE_PPOLL) + RealCommandRunner::~RealCommandRunner() { +diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc +index 66d2c2c430..ce3e2c20a4 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -13,6 +13,7 @@ @@ -1019,13 +2497,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 #include #include -@@ -251,11 +252,14 @@ Subprocess *SubprocessSet::Add(const str - return subprocess; - } - --bool SubprocessSet::DoWork() { -+bool SubprocessSet::DoWork(TokenPool* tokens) { - DWORD bytes_read; +@@ -256,6 +257,9 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { Subprocess* subproc; OVERLAPPED* overlapped; @@ -1035,7 +2507,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 if (!GetQueuedCompletionStatus(ioport_, &bytes_read, (PULONG_PTR)&subproc, &overlapped, INFINITE)) { if (GetLastError() != ERROR_BROKEN_PIPE) -@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork() { +@@ -266,6 +270,11 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { // delivered by NotifyInterrupted above. return true; @@ -1047,58 +2519,18 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 subproc->OnPipeReady(); if (subproc->Done()) { ---- a/src/subprocess.h -+++ b/src/subprocess.h -@@ -76,6 +76,8 @@ struct Subprocess { - friend struct SubprocessSet; - }; - -+struct TokenPool; -+ - /// SubprocessSet runs a ppoll/pselect() loop around a set of Subprocesses. - /// DoWork() waits for any state change in subprocesses; finished_ - /// is a queue of subprocesses as they finish. -@@ -84,13 +86,17 @@ struct SubprocessSet { - ~SubprocessSet(); - - Subprocess* Add(const std::string& command, bool use_console = false); -- bool DoWork(); -+ bool DoWork(struct TokenPool* tokens); - Subprocess* NextFinished(); - void Clear(); - - std::vector running_; - std::queue finished_; - -+ bool token_available_; -+ bool IsTokenAvailable() { return token_available_; } -+ void ResetTokenAvailable() { token_available_ = false; } -+ - #ifdef _WIN32 - static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType); - static HANDLE ioport_; +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index 6264c8bf11..f625963462 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc -@@ -13,6 +13,7 @@ - // limitations under the License. - - #include "subprocess.h" -+#include "tokenpool.h" - - #include "test.h" - -@@ -34,8 +35,30 @@ const char* kSimpleCommand = "cmd /c dir - const char* kSimpleCommand = "ls /"; - #endif - -+struct TestTokenPool : public TokenPool { -+ bool Acquire() { return false; } -+ void Reserve() {} -+ void Release() {} -+ void Clear() {} +@@ -40,9 +40,16 @@ struct TokenPoolTest : public TokenPool { + void Reserve() {} + void Release() {} + void Clear() {} + bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; } -+ -+#ifdef _WIN32 + + #ifdef _WIN32 +- // @TODO + bool _token_available; + void WaitForTokenAvailability(HANDLE ioport) { + if (_token_available) @@ -1106,197 +2538,31 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + PostQueuedCompletionStatus(ioport, 0, (ULONG_PTR) this, NULL); + } + bool TokenIsAvailable(ULONG_PTR key) { return key == (ULONG_PTR) this; } -+#else -+ int _fd; -+ int GetMonitorFd() { return _fd; } -+#endif -+}; -+ - struct SubprocessTest : public testing::Test { - SubprocessSet subprocs_; -+ TestTokenPool tokens_; - }; - - } // anonymous namespace -@@ -45,10 +68,12 @@ TEST_F(SubprocessTest, BadCommandStderr) - Subprocess* subproc = subprocs_.Add("cmd /c ninja_no_such_command"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { - // Pretend we discovered that stderr was ready for writing. -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -59,10 +84,12 @@ TEST_F(SubprocessTest, NoSuchCommand) { - Subprocess* subproc = subprocs_.Add("ninja_no_such_command"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { - // Pretend we discovered that stderr was ready for writing. -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitFailure, subproc->Finish()); - EXPECT_NE("", subproc->GetOutput()); -@@ -78,9 +105,11 @@ TEST_F(SubprocessTest, InterruptChild) { - Subprocess* subproc = subprocs_.Add("kill -INT $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); + #else + int _fd; + int GetMonitorFd() { return _fd; } +@@ -297,34 +304,48 @@ TEST_F(SubprocessTest, ReadStdin) { } -@@ -90,7 +119,7 @@ TEST_F(SubprocessTest, InterruptParent) - ASSERT_NE((Subprocess *) 0, subproc); + #endif // _WIN32 - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -102,9 +131,11 @@ TEST_F(SubprocessTest, InterruptChildWit - Subprocess* subproc = subprocs_.Add("kill -TERM $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -114,7 +145,7 @@ TEST_F(SubprocessTest, InterruptParentWi - ASSERT_NE((Subprocess *) 0, subproc); - - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -126,9 +157,11 @@ TEST_F(SubprocessTest, InterruptChildWit - Subprocess* subproc = subprocs_.Add("kill -HUP $$"); - ASSERT_NE((Subprocess *) 0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitInterrupted, subproc->Finish()); - } -@@ -138,7 +171,7 @@ TEST_F(SubprocessTest, InterruptParentWi - ASSERT_NE((Subprocess *) 0, subproc); - - while (!subproc->Done()) { -- bool interrupted = subprocs_.DoWork(); -+ bool interrupted = subprocs_.DoWork(NULL); - if (interrupted) - return; - } -@@ -153,9 +186,11 @@ TEST_F(SubprocessTest, Console) { - subprocs_.Add("test -t 0 -a -t 1 -a -t 2", /*use_console=*/true); - ASSERT_NE((Subprocess*)0, subproc); - -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - - EXPECT_EQ(ExitSuccess, subproc->Finish()); - } -@@ -167,9 +202,11 @@ TEST_F(SubprocessTest, SetWithSingle) { +-// @TODO: remove once TokenPool implementation for Windows is available +-#ifndef _WIN32 + TEST_F(SubprocessTest, TokenAvailable) { Subprocess* subproc = subprocs_.Add(kSimpleCommand); ASSERT_NE((Subprocess *) 0, subproc); -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_NE("", subproc->GetOutput()); - -@@ -200,12 +237,13 @@ TEST_F(SubprocessTest, SetWithMulti) { - ASSERT_EQ("", processes[i]->GetOutput()); - } - -+ subprocs_.ResetTokenAvailable(); - while (!processes[0]->Done() || !processes[1]->Done() || - !processes[2]->Done()) { - ASSERT_GT(subprocs_.running_.size(), 0u); -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -- -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(0u, subprocs_.running_.size()); - ASSERT_EQ(3u, subprocs_.finished_.size()); - -@@ -237,8 +275,10 @@ TEST_F(SubprocessTest, SetWithLots) { - ASSERT_NE((Subprocess *) 0, subproc); - procs.push_back(subproc); - } -+ subprocs_.ResetTokenAvailable(); - while (!subprocs_.running_.empty()) -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - for (size_t i = 0; i < procs.size(); ++i) { - ASSERT_EQ(ExitSuccess, procs[i]->Finish()); - ASSERT_NE("", procs[i]->GetOutput()); -@@ -254,10 +294,91 @@ TEST_F(SubprocessTest, SetWithLots) { - // that stdin is closed. - TEST_F(SubprocessTest, ReadStdin) { - Subprocess* subproc = subprocs_.Add("cat -"); -+ subprocs_.ResetTokenAvailable(); - while (!subproc->Done()) { -- subprocs_.DoWork(); -+ subprocs_.DoWork(NULL); - } -+ ASSERT_FALSE(subprocs_.IsTokenAvailable()); - ASSERT_EQ(ExitSuccess, subproc->Finish()); - ASSERT_EQ(1u, subprocs_.finished_.size()); - } - #endif // _WIN32 -+ -+TEST_F(SubprocessTest, TokenAvailable) { -+ Subprocess* subproc = subprocs_.Add(kSimpleCommand); -+ ASSERT_NE((Subprocess *) 0, subproc); -+ -+ // simulate GNUmake jobserver pipe with 1 token + // simulate GNUmake jobserver pipe with 1 token +#ifdef _WIN32 + tokens_._token_available = true; +#else -+ int fds[2]; -+ ASSERT_EQ(0u, pipe(fds)); -+ tokens_._fd = fds[0]; -+ ASSERT_EQ(1u, write(fds[1], "T", 1)); + int fds[2]; + ASSERT_EQ(0u, pipe(fds)); + tokens_._fd = fds[0]; + ASSERT_EQ(1u, write(fds[1], "T", 1)); +#endif -+ -+ subprocs_.ResetTokenAvailable(); -+ subprocs_.DoWork(&tokens_); + + subprocs_.ResetTokenAvailable(); + subprocs_.DoWork(&tokens_); +#ifdef _WIN32 + tokens_._token_available = false; + // we need to loop here as we have no conrol where the token @@ -1305,63 +2571,62 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + subprocs_.DoWork(&tokens_); + } +#endif -+ -+ EXPECT_TRUE(subprocs_.IsTokenAvailable()); -+ EXPECT_EQ(0u, subprocs_.finished_.size()); -+ -+ // remove token to let DoWork() wait for command again + + EXPECT_TRUE(subprocs_.IsTokenAvailable()); + EXPECT_EQ(0u, subprocs_.finished_.size()); + + // remove token to let DoWork() wait for command again +#ifndef _WIN32 -+ char token; -+ ASSERT_EQ(1u, read(fds[0], &token, 1)); + char token; + ASSERT_EQ(1u, read(fds[0], &token, 1)); +#endif -+ -+ while (!subproc->Done()) { -+ subprocs_.DoWork(&tokens_); -+ } -+ + + while (!subproc->Done()) { + subprocs_.DoWork(&tokens_); + } + +#ifndef _WIN32 -+ close(fds[1]); -+ close(fds[0]); + close(fds[1]); + close(fds[0]); +#endif -+ -+ EXPECT_EQ(ExitSuccess, subproc->Finish()); -+ EXPECT_NE("", subproc->GetOutput()); -+ -+ EXPECT_EQ(1u, subprocs_.finished_.size()); -+} -+ -+TEST_F(SubprocessTest, TokenNotAvailable) { -+ Subprocess* subproc = subprocs_.Add(kSimpleCommand); -+ ASSERT_NE((Subprocess *) 0, subproc); -+ -+ // simulate GNUmake jobserver pipe with 0 tokens + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -337,17 +358,23 @@ TEST_F(SubprocessTest, TokenNotAvailable) { + ASSERT_NE((Subprocess *) 0, subproc); + + // simulate GNUmake jobserver pipe with 0 tokens +#ifdef _WIN32 + tokens_._token_available = false; +#else -+ int fds[2]; -+ ASSERT_EQ(0u, pipe(fds)); -+ tokens_._fd = fds[0]; + int fds[2]; + ASSERT_EQ(0u, pipe(fds)); + tokens_._fd = fds[0]; +#endif -+ -+ subprocs_.ResetTokenAvailable(); -+ while (!subproc->Done()) { -+ subprocs_.DoWork(&tokens_); -+ } -+ + + subprocs_.ResetTokenAvailable(); + while (!subproc->Done()) { + subprocs_.DoWork(&tokens_); + } + +#ifndef _WIN32 -+ close(fds[1]); -+ close(fds[0]); + close(fds[1]); + close(fds[0]); +#endif -+ -+ EXPECT_FALSE(subprocs_.IsTokenAvailable()); -+ EXPECT_EQ(ExitSuccess, subproc->Finish()); -+ EXPECT_NE("", subproc->GetOutput()); -+ -+ EXPECT_EQ(1u, subprocs_.finished_.size()); -+} + + EXPECT_FALSE(subprocs_.IsTokenAvailable()); + EXPECT_EQ(ExitSuccess, subproc->Finish()); +@@ -355,4 +382,3 @@ TEST_F(SubprocessTest, TokenNotAvailable) { + + EXPECT_EQ(1u, subprocs_.finished_.size()); + } +-#endif // _WIN32 +diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc +new file mode 100644 +index 0000000000..70d84bfff7 --- /dev/null +++ b/src/tokenpool-gnu-make-posix.cc -@@ -0,0 +1,202 @@ +@@ -0,0 +1,203 @@ +// Copyright 2016-2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); @@ -1396,8 +2661,8 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + + virtual int GetMonitorFd(); + -+ virtual const char* GetEnv(const char* name) { return getenv(name); }; -+ virtual bool ParseAuth(const char* jobserver); ++ virtual const char *GetEnv(const char *name) { return getenv(name); }; ++ virtual bool ParseAuth(const char *jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + @@ -1446,13 +2711,14 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + act.sa_handler = CloseDupRfd; + if (sigaction(SIGALRM, &act, &old_act_) < 0) { + perror("sigaction:"); -+ return false; ++ return(false); ++ } else { ++ restore_ = true; ++ return(true); + } -+ restore_ = true; -+ return true; +} + -+bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) { ++bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) { + int rfd = -1; + int wfd = -1; + if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && @@ -1558,15 +2824,18 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +} + +int GNUmakeTokenPoolPosix::GetMonitorFd() { -+ return rfd_; ++ return(rfd_); +} + -+TokenPool* TokenPool::Get() { ++struct TokenPool *TokenPool::Get() { + return new GNUmakeTokenPoolPosix; +} +diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc +new file mode 100644 +index 0000000000..2719f2c1fc --- /dev/null +++ b/src/tokenpool-gnu-make-win32.cc -@@ -0,0 +1,239 @@ +@@ -0,0 +1,237 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); @@ -1583,8 +2852,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + +#include "tokenpool-gnu-make.h" + -+// Always include this first. -+// Otherwise the other system headers don't work correctly under Win32 ++// always include first to make sure other headers do the correct thing... +#include + +#include @@ -1602,8 +2870,8 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + virtual void WaitForTokenAvailability(HANDLE ioport); + virtual bool TokenIsAvailable(ULONG_PTR key); + -+ virtual const char* GetEnv(const char* name); -+ virtual bool ParseAuth(const char* jobserver); ++ virtual const char *GetEnv(const char *name); ++ virtual bool ParseAuth(const char *jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + @@ -1668,19 +2936,19 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + } +} + -+const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) { ++const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) { + // getenv() does not work correctly together with tokenpool_tests.cc + static char buffer[MAX_PATH + 1]; -+ if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0) ++ if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0) + return NULL; -+ return buffer; ++ return(buffer); +} + -+bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) { ++bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { + // match "--jobserver-auth=gmake_semaphore_..." -+ const char* start = strchr(jobserver, '='); ++ const char *start = strchr(jobserver, '='); + if (start) { -+ const char* end = start; ++ const char *end = start; + unsigned int len; + char c, *auth; + @@ -1689,15 +2957,14 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + break; + len = end - start; // includes string terminator in count + -+ if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) { ++ if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) { + strncpy(auth, start + 1, len - 1); + auth[len - 1] = '\0'; + -+ if ((semaphore_jobserver_ = -+ OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ -+ FALSE, /* Child processes DON'T inherit */ -+ auth /* Semaphore name */ -+ )) != NULL) { ++ if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ ++ FALSE, /* Child processes DON'T inherit */ ++ auth /* Semaphore name */ ++ )) != NULL) { + free(auth); + return true; + } @@ -1742,7 +3009,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +} + +DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) { -+ GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param; ++ GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param; + return This->SemaphoreThread(); +} + @@ -1787,7 +3054,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + +bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) { + // alert child thread to break wait on token semaphore -+ QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL); ++ QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL); + + // return true when GetQueuedCompletionStatus() returned our key + return key == (ULONG_PTR) this; @@ -1803,120 +3070,273 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + Win32Fatal("WaitForSingleObject"); +} + -+TokenPool* TokenPool::Get() { ++struct TokenPool *TokenPool::Get() { + return new GNUmakeTokenPoolWin32; +} ---- /dev/null +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index 4132bb06d9..92ff611721 100644 +--- a/src/tokenpool-gnu-make.cc +++ b/src/tokenpool-gnu-make.cc -@@ -0,0 +1,108 @@ -+// Copyright 2016-2018 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ +@@ -12,101 +12,26 @@ + // See the License for the specific language governing permissions and + // limitations under the License. + +-#include "tokenpool.h" +#include "tokenpool-gnu-make.h" -+ + +-#include +-#include +-#include +-#include +-#include +-#include +#include -+#include -+#include -+ -+#include "line_printer.h" -+ + #include + #include +-#include + + #include "line_printer.h" + +-// TokenPool implementation for GNU make jobserver +-// (http://make.mad-scientist.net/papers/jobserver-implementation/) +-struct GNUmakeTokenPool : public TokenPool { +- GNUmakeTokenPool(); +- virtual ~GNUmakeTokenPool(); +- +- virtual bool Acquire(); +- virtual void Reserve(); +- virtual void Release(); +- virtual void Clear(); +- virtual int GetMonitorFd(); +- +- bool Setup(bool ignore, bool verbose, double& max_load_average); +- +- private: +- int available_; +- int used_; +- +-#ifdef _WIN32 +- // @TODO +-#else +- int rfd_; +- int wfd_; +- +- struct sigaction old_act_; +- bool restore_; +- +- static int dup_rfd_; +- static void CloseDupRfd(int signum); +- +- bool CheckFd(int fd); +- bool SetAlarmHandler(); +-#endif +- +- void Return(); +-}; +- +// TokenPool implementation for GNU make jobserver - common bits -+// every instance owns an implicit token -> available_ == 1 + // every instance owns an implicit token -> available_ == 1 +-GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0), +- rfd_(-1), wfd_(-1), restore_(false) { +GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) { -+} -+ -+GNUmakeTokenPool::~GNUmakeTokenPool() { -+} -+ -+bool GNUmakeTokenPool::Setup(bool ignore, -+ bool verbose, -+ double& max_load_average) { -+ const char* value = GetEnv("MAKEFLAGS"); -+ if (!value) -+ return false; -+ -+ // GNU make <= 4.1 -+ const char* jobserver = strstr(value, "--jobserver-fds="); -+ if (!jobserver) -+ // GNU make => 4.2 -+ jobserver = strstr(value, "--jobserver-auth="); -+ if (jobserver) { -+ LinePrinter printer; -+ -+ if (ignore) { -+ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); -+ } else { -+ if (ParseAuth(jobserver)) { -+ const char* l_arg = strstr(value, " -l"); -+ int load_limit = -1; -+ -+ if (verbose) { -+ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); -+ } -+ -+ // translate GNU make -lN to ninja -lN -+ if (l_arg && -+ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && -+ (load_limit > 0)) { -+ max_load_average = load_limit; -+ } -+ -+ return true; -+ } -+ } -+ } -+ -+ return false; -+} -+ -+bool GNUmakeTokenPool::Acquire() { -+ if (available_ > 0) -+ return true; -+ + } + + GNUmakeTokenPool::~GNUmakeTokenPool() { +- Clear(); +- if (restore_) +- sigaction(SIGALRM, &old_act_, NULL); +-} +- +-bool GNUmakeTokenPool::CheckFd(int fd) { +- if (fd < 0) +- return false; +- int ret = fcntl(fd, F_GETFD); +- if (ret < 0) +- return false; +- return true; +-} +- +-int GNUmakeTokenPool::dup_rfd_ = -1; +- +-void GNUmakeTokenPool::CloseDupRfd(int signum) { +- close(dup_rfd_); +- dup_rfd_ = -1; +-} +- +-bool GNUmakeTokenPool::SetAlarmHandler() { +- struct sigaction act; +- memset(&act, 0, sizeof(act)); +- act.sa_handler = CloseDupRfd; +- if (sigaction(SIGALRM, &act, &old_act_) < 0) { +- perror("sigaction:"); +- return(false); +- } else { +- restore_ = true; +- return(true); +- } + } + + bool GNUmakeTokenPool::Setup(bool ignore, + bool verbose, + double& max_load_average) { +- const char *value = getenv("MAKEFLAGS"); ++ const char *value = GetEnv("MAKEFLAGS"); + if (value) { + // GNU make <= 4.1 + const char *jobserver = strstr(value, "--jobserver-fds="); +@@ -119,20 +44,13 @@ bool GNUmakeTokenPool::Setup(bool ignore, + if (ignore) { + printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); + } else { +- int rfd = -1; +- int wfd = -1; +- if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && +- CheckFd(rfd) && +- CheckFd(wfd) && +- SetAlarmHandler()) { ++ if (ParseAuth(jobserver)) { + const char *l_arg = strstr(value, " -l"); + int load_limit = -1; + + if (verbose) { + printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); + } +- rfd_ = rfd; +- wfd_ = wfd; + + // translate GNU make -lN to ninja -lN + if (l_arg && +@@ -154,83 +72,14 @@ bool GNUmakeTokenPool::Acquire() { + if (available_ > 0) + return true; + +- // Please read +- // +- // http://make.mad-scientist.net/papers/jobserver-implementation/ +- // +- // for the reasoning behind the following code. +- // +- // Try to read one character from the pipe. Returns true on success. +- // +- // First check if read() would succeed without blocking. +-#ifdef USE_PPOLL +- pollfd pollfds[] = {{rfd_, POLLIN, 0}}; +- int ret = poll(pollfds, 1, 0); +-#else +- fd_set set; +- struct timeval timeout = { 0, 0 }; +- FD_ZERO(&set); +- FD_SET(rfd_, &set); +- int ret = select(rfd_ + 1, &set, NULL, NULL, &timeout); +-#endif +- if (ret > 0) { +- // Handle potential race condition: +- // - the above check succeeded, i.e. read() should not block +- // - the character disappears before we call read() +- // +- // Create a duplicate of rfd_. The duplicate file descriptor dup_rfd_ +- // can safely be closed by signal handlers without affecting rfd_. +- dup_rfd_ = dup(rfd_); +- +- if (dup_rfd_ != -1) { +- struct sigaction act, old_act; +- int ret = 0; +- +- // Temporarily replace SIGCHLD handler with our own +- memset(&act, 0, sizeof(act)); +- act.sa_handler = CloseDupRfd; +- if (sigaction(SIGCHLD, &act, &old_act) == 0) { +- struct itimerval timeout; +- +- // install a 100ms timeout that generates SIGALARM on expiration +- memset(&timeout, 0, sizeof(timeout)); +- timeout.it_value.tv_usec = 100 * 1000; // [ms] -> [usec] +- if (setitimer(ITIMER_REAL, &timeout, NULL) == 0) { +- char buf; +- +- // Now try to read() from dup_rfd_. Return values from read(): +- // +- // 1. token read -> 1 +- // 2. pipe closed -> 0 +- // 3. alarm expires -> -1 (EINTR) +- // 4. child exits -> -1 (EINTR) +- // 5. alarm expired before entering read() -> -1 (EBADF) +- // 6. child exited before entering read() -> -1 (EBADF) +- // 7. child exited before handler is installed -> go to 1 - 3 +- ret = read(dup_rfd_, &buf, 1); +- +- // disarm timer +- memset(&timeout, 0, sizeof(timeout)); +- setitimer(ITIMER_REAL, &timeout, NULL); +- } +- +- sigaction(SIGCHLD, &old_act, NULL); +- } +- +- CloseDupRfd(0); +- +- // Case 1 from above list +- if (ret > 0) { +- available_++; +- return true; +- } +- } + if (AcquireToken()) { + // token acquired + available_++; + return true; -+ } -+ -+ // no token available -+ return false; -+} -+ -+void GNUmakeTokenPool::Reserve() { -+ available_--; -+ used_++; -+} -+ -+void GNUmakeTokenPool::Return() { ++ } else { ++ // no token available ++ return false; + } +- +- // read() would block, i.e. no token available, +- // cases 2-6 from above list or +- // select() / poll() / dup() / sigaction() / setitimer() failed +- return false; + } + + void GNUmakeTokenPool::Reserve() { +@@ -239,15 +88,8 @@ void GNUmakeTokenPool::Reserve() { + } + + void GNUmakeTokenPool::Return() { +- const char buf = '+'; +- while (1) { +- int ret = write(wfd_, &buf, 1); +- if (ret > 0) +- available_--; +- if ((ret != -1) || (errno != EINTR)) +- return; +- // write got interrupted - retry +- } + if (ReturnToken()) + available_--; -+} -+ -+void GNUmakeTokenPool::Release() { -+ available_++; -+ used_--; -+ if (available_ > 1) -+ Return(); -+} -+ -+void GNUmakeTokenPool::Clear() { -+ while (used_ > 0) -+ Release(); -+ while (available_ > 1) -+ Return(); -+} + } + + void GNUmakeTokenPool::Release() { +@@ -263,18 +105,3 @@ void GNUmakeTokenPool::Clear() { + while (available_ > 1) + Return(); + } +- +-int GNUmakeTokenPool::GetMonitorFd() { +- return(rfd_); +-} +- +-struct TokenPool *TokenPool::Get(bool ignore, +- bool verbose, +- double& max_load_average) { +- GNUmakeTokenPool *tokenpool = new GNUmakeTokenPool; +- if (tokenpool->Setup(ignore, verbose, max_load_average)) +- return tokenpool; +- else +- delete tokenpool; +- return NULL; +-} +diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h +new file mode 100644 +index 0000000000..d3852088e2 --- /dev/null +++ b/src/tokenpool-gnu-make.h @@ -0,0 +1,40 @@ @@ -1939,7 +3359,7 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +// interface to GNU make token pool +struct GNUmakeTokenPool : public TokenPool { + GNUmakeTokenPool(); -+ ~GNUmakeTokenPool(); ++ virtual ~GNUmakeTokenPool(); + + // token pool implementation + virtual bool Acquire(); @@ -1949,8 +3369,8 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + virtual bool Setup(bool ignore, bool verbose, double& max_load_average); + + // platform specific implementation -+ virtual const char* GetEnv(const char* name) = 0; -+ virtual bool ParseAuth(const char* jobserver) = 0; ++ virtual const char *GetEnv(const char *name) = 0; ++ virtual bool ParseAuth(const char *jobserver) = 0; + virtual bool AcquireToken() = 0; + virtual bool ReturnToken() = 0; + @@ -1960,81 +3380,83 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + + void Return(); +}; ---- /dev/null +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +index 4c592875b4..613d16882d 100644 +--- a/src/tokenpool-none.cc ++++ b/src/tokenpool-none.cc +@@ -17,8 +17,6 @@ + #include + + // No-op TokenPool implementation +-struct TokenPool *TokenPool::Get(bool ignore, +- bool verbose, +- double& max_load_average) { ++struct TokenPool *TokenPool::Get() { + return NULL; + } +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 4bf477f20c..1be8e1d5ce 100644 +--- a/src/tokenpool.h +++ b/src/tokenpool.h -@@ -0,0 +1,42 @@ +@@ -1,4 +1,4 @@ +-// Copyright 2016-2017 Google Inc. All Rights Reserved. +// Copyright 2016-2018 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. +@@ -12,6 +12,10 @@ + // See the License for the specific language governing permissions and + // limitations under the License. + +#ifdef _WIN32 +#include +#endif + -+// interface to token pool -+struct TokenPool { -+ virtual ~TokenPool() {} -+ -+ virtual bool Acquire() = 0; -+ virtual void Reserve() = 0; -+ virtual void Release() = 0; -+ virtual void Clear() = 0; -+ + // interface to token pool + struct TokenPool { + virtual ~TokenPool() {} +@@ -21,14 +25,18 @@ struct TokenPool { + virtual void Release() = 0; + virtual void Clear() = 0; + + // returns false if token pool setup failed + virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0; + -+#ifdef _WIN32 + #ifdef _WIN32 +- // @TODO + virtual void WaitForTokenAvailability(HANDLE ioport) = 0; + // returns true if a token has become available + // key is result from GetQueuedCompletionStatus() + virtual bool TokenIsAvailable(ULONG_PTR key) = 0; -+#else -+ virtual int GetMonitorFd() = 0; -+#endif -+ -+ // returns NULL if token pool is not available -+ static TokenPool* Get(); -+}; ---- /dev/null + #else + virtual int GetMonitorFd() = 0; + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(bool ignore, +- bool verbose, +- double& max_load_average); ++ static struct TokenPool *Get(); + }; +diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc +index 6c89064ca4..8d4fd7d33a 100644 +--- a/src/tokenpool_test.cc +++ b/src/tokenpool_test.cc -@@ -0,0 +1,269 @@ -+// Copyright 2018 Google Inc. All Rights Reserved. -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ -+#include "tokenpool.h" -+ -+#include "test.h" -+ +@@ -16,13 +16,25 @@ + + #include "test.h" + +-#ifndef _WIN32 +#ifdef _WIN32 +#include +#else +#include +#endif + -+#include -+#include -+ + #include + #include +-#include + +#ifdef _WIN32 +// should contain all valid characters +#define SEMAPHORE_NAME "abcdefghijklmnopqrstwxyz01234567890_" @@ -2043,40 +3465,45 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +#define ENVIRONMENT_INIT(v) SetEnvironmentVariable("MAKEFLAGS", v) +#else +#define AUTH_FORMAT(tmpl) "foo " tmpl "=%d,%d bar" -+#define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") + #define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS") +-#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true); +#define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true) -+#endif -+ -+namespace { -+ -+const double kLoadAverageDefault = -1.23456789; -+ -+struct TokenPoolTest : public testing::Test { -+ double load_avg_; -+ TokenPool* tokens_; -+ char buf_[1024]; + #endif + + namespace { +@@ -32,43 +44,60 @@ const double kLoadAverageDefault = -1.23456789; + struct TokenPoolTest : public testing::Test { + double load_avg_; + TokenPool *tokens_; +-#ifndef _WIN32 + char buf_[1024]; +#ifdef _WIN32 -+ const char* semaphore_name_; ++ const char *semaphore_name_; + HANDLE semaphore_; +#else -+ int fds_[2]; -+#endif -+ -+ virtual void SetUp() { -+ load_avg_ = kLoadAverageDefault; -+ tokens_ = NULL; -+ ENVIRONMENT_CLEAR(); + int fds_[2]; + #endif + + virtual void SetUp() { + load_avg_ = kLoadAverageDefault; + tokens_ = NULL; +-#ifndef _WIN32 + ENVIRONMENT_CLEAR(); +#ifdef _WIN32 + semaphore_name_ = SEMAPHORE_NAME; + if ((semaphore_ = CreateSemaphore(0, 0, 2, SEMAPHORE_NAME)) == NULL) +#else -+ if (pipe(fds_) < 0) -+#endif + if (pipe(fds_) < 0) +- ASSERT_TRUE(false); + #endif + ASSERT_TRUE(false); -+ } -+ -+ void CreatePool(const char* format, bool ignore_jobserver = false) { -+ if (format) { + } + +- void CreatePool(const char *format, bool ignore_jobserver) { +-#ifndef _WIN32 ++ void CreatePool(const char *format, bool ignore_jobserver = false) { + if (format) { +- sprintf(buf_, format, fds_[0], fds_[1]); + sprintf(buf_, format, +#ifdef _WIN32 + semaphore_name_ @@ -2084,73 +3511,69 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + fds_[0], fds_[1] +#endif + ); -+ ENVIRONMENT_INIT(buf_); -+ } + ENVIRONMENT_INIT(buf_); + } +-#endif +- tokens_ = TokenPool::Get(ignore_jobserver, false, load_avg_); + if ((tokens_ = TokenPool::Get()) != NULL) { + if (!tokens_->Setup(ignore_jobserver, false, load_avg_)) { + delete tokens_; + tokens_ = NULL; + } + } -+ } -+ -+ void CreateDefaultPool() { + } + + void CreateDefaultPool() { +- CreatePool("foo --jobserver-auth=%d,%d bar", false); + CreatePool(AUTH_FORMAT("--jobserver-auth")); -+ } -+ -+ virtual void TearDown() { -+ if (tokens_) -+ delete tokens_; + } + + virtual void TearDown() { + if (tokens_) + delete tokens_; +-#ifndef _WIN32 +#ifdef _WIN32 + CloseHandle(semaphore_); +#else -+ close(fds_[0]); -+ close(fds_[1]); -+#endif + close(fds_[0]); + close(fds_[1]); +- ENVIRONMENT_CLEAR(); + #endif + ENVIRONMENT_CLEAR(); -+ } -+}; -+ -+} // anonymous namespace -+ -+// verifies none implementation -+TEST_F(TokenPoolTest, NoTokenPool) { -+ CreatePool(NULL, false); -+ -+ EXPECT_EQ(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, SuccessfulOldSetup) { -+ // GNUmake <= 4.1 + } + }; + +@@ -82,10 +111,9 @@ TEST_F(TokenPoolTest, NoTokenPool) { + EXPECT_EQ(kLoadAverageDefault, load_avg_); + } + +-#ifndef _WIN32 + TEST_F(TokenPoolTest, SuccessfulOldSetup) { + // GNUmake <= 4.1 +- CreatePool("foo --jobserver-fds=%d,%d bar", false); + CreatePool(AUTH_FORMAT("--jobserver-fds")); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, SuccessfulNewSetup) { -+ // GNUmake => 4.2 -+ CreateDefaultPool(); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, IgnoreWithJN) { + + EXPECT_NE(NULL, tokens_); + EXPECT_EQ(kLoadAverageDefault, load_avg_); +@@ -100,19 +128,37 @@ TEST_F(TokenPoolTest, SuccessfulNewSetup) { + } + + TEST_F(TokenPoolTest, IgnoreWithJN) { +- CreatePool("foo --jobserver-auth=%d,%d bar", true); + CreatePool(AUTH_FORMAT("--jobserver-auth"), true); -+ -+ EXPECT_EQ(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+} -+ -+TEST_F(TokenPoolTest, HonorLN) { + + EXPECT_EQ(NULL, tokens_); + EXPECT_EQ(kLoadAverageDefault, load_avg_); + } + + TEST_F(TokenPoolTest, HonorLN) { +- CreatePool("foo -l9 --jobserver-auth=%d,%d bar", false); + CreatePool(AUTH_FORMAT("-l9 --jobserver-auth")); -+ -+ EXPECT_NE(NULL, tokens_); -+ EXPECT_EQ(9.0, load_avg_); -+} -+ + + EXPECT_NE(NULL, tokens_); + EXPECT_EQ(9.0, load_avg_); + } + +#ifdef _WIN32 +TEST_F(TokenPoolTest, SemaphoreNotFound) { + semaphore_name_ = SEMAPHORE_NAME "_foobar"; @@ -2169,101 +3592,66 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + EXPECT_TRUE(tokens_->TokenIsAvailable((ULONG_PTR)tokens_)); +} +#else -+TEST_F(TokenPoolTest, MonitorFD) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); -+} + TEST_F(TokenPoolTest, MonitorFD) { + CreateDefaultPool(); + +@@ -121,6 +167,7 @@ TEST_F(TokenPoolTest, MonitorFD) { + + EXPECT_EQ(fds_[0], tokens_->GetMonitorFd()); + } +#endif -+ -+TEST_F(TokenPoolTest, ImplicitToken) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+} -+ -+TEST_F(TokenPoolTest, TwoTokens) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // jobserver offers 2nd token + + TEST_F(TokenPoolTest, ImplicitToken) { + CreateDefaultPool(); +@@ -147,7 +194,13 @@ TEST_F(TokenPoolTest, TwoTokens) { + EXPECT_FALSE(tokens_->Acquire()); + + // jobserver offers 2nd token +#ifdef _WIN32 + LONG previous; + ASSERT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); + ASSERT_EQ(0, previous); +#else -+ ASSERT_EQ(1u, write(fds_[1], "T", 1)); + ASSERT_EQ(1u, write(fds_[1], "T", 1)); +#endif -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // release 2nd token -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ -+ // release implict token - must return 2nd token back to jobserver -+ tokens_->Release(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ + EXPECT_TRUE(tokens_->Acquire()); + tokens_->Reserve(); + EXPECT_FALSE(tokens_->Acquire()); +@@ -160,8 +213,14 @@ TEST_F(TokenPoolTest, TwoTokens) { + tokens_->Release(); + EXPECT_TRUE(tokens_->Acquire()); + +- // there must be one token in the pipe + // there must be one token available +#ifdef _WIN32 + EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); + EXPECT_TRUE(ReleaseSemaphore(semaphore_, 1, &previous)); + EXPECT_EQ(0, previous); +#else -+ EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); + EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_))); +#endif -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+} -+ -+TEST_F(TokenPoolTest, Clear) { -+ CreateDefaultPool(); -+ -+ ASSERT_NE(NULL, tokens_); -+ EXPECT_EQ(kLoadAverageDefault, load_avg_); -+ -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ // jobserver offers 2nd & 3rd token + + // implicit token + EXPECT_TRUE(tokens_->Acquire()); +@@ -179,7 +238,13 @@ TEST_F(TokenPoolTest, Clear) { + EXPECT_FALSE(tokens_->Acquire()); + + // jobserver offers 2nd & 3rd token +#ifdef _WIN32 + LONG previous; + ASSERT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); + ASSERT_EQ(0, previous); +#else -+ ASSERT_EQ(2u, write(fds_[1], "TT", 2)); + ASSERT_EQ(2u, write(fds_[1], "TT", 2)); +#endif -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ tokens_->Reserve(); -+ EXPECT_FALSE(tokens_->Acquire()); -+ -+ tokens_->Clear(); -+ EXPECT_TRUE(tokens_->Acquire()); -+ + EXPECT_TRUE(tokens_->Acquire()); + tokens_->Reserve(); + EXPECT_TRUE(tokens_->Acquire()); +@@ -189,10 +254,16 @@ TEST_F(TokenPoolTest, Clear) { + tokens_->Clear(); + EXPECT_TRUE(tokens_->Acquire()); + +- // there must be two tokens in the pipe + // there must be two tokens available +#ifdef _WIN32 + EXPECT_EQ(WAIT_OBJECT_0, WaitForSingleObject(semaphore_, 0)); @@ -2271,9 +3659,632 @@ https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 + EXPECT_TRUE(ReleaseSemaphore(semaphore_, 2, &previous)); + EXPECT_EQ(0, previous); +#else -+ EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); + EXPECT_EQ(2u, read(fds_[0], buf_, sizeof(buf_))); +#endif + + // implicit token + EXPECT_TRUE(tokens_->Acquire()); + } +-#endif + +From 2b9c81c0ec1226d8795e7725529f13be41eaa385 Mon Sep 17 00:00:00 2001 +From: Stefan Becker +Date: Fri, 14 Dec 2018 13:27:11 +0200 +Subject: [PATCH 11/11] Prepare PR for merging - part II + +- remove unnecessary "struct" from TokenPool +- add PAPCFUNC cast to QueryUserAPC() +- remove hard-coded MAKEFLAGS string from win32 +- remove useless build test CompleteNoWork +- rename TokenPoolTest to TestTokenPool +- add tokenpool modules to CMake build +- remove unused no-op TokenPool implementation +- fix errors flagged by codespell & clang-tidy +- address review comments from + +https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-195195803 +https://github.com/ninja-build/ninja/pull/1140#pullrequestreview-185089255 +https://github.com/ninja-build/ninja/pull/1140#issuecomment-473898963 +https://github.com/ninja-build/ninja/pull/1140#issuecomment-596624610 +--- + CMakeLists.txt | 8 ++++- + src/build.cc | 2 +- + src/build_test.cc | 12 +------ + src/subprocess-posix.cc | 4 +-- + src/subprocess-win32.cc | 2 +- + src/subprocess.h | 2 +- + src/subprocess_test.cc | 26 +++++++------- + src/tokenpool-gnu-make-posix.cc | 21 +++++------ + src/tokenpool-gnu-make-win32.cc | 36 ++++++++++--------- + src/tokenpool-gnu-make.cc | 63 +++++++++++++++++---------------- + src/tokenpool-gnu-make.h | 6 ++-- + src/tokenpool-none.cc | 22 ------------ + src/tokenpool.h | 2 +- + src/tokenpool_test.cc | 8 ++--- + 14 files changed, 94 insertions(+), 120 deletions(-) + delete mode 100644 src/tokenpool-none.cc + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 57ae548f5b..e2876fe413 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -112,6 +112,7 @@ add_library(libninja OBJECT + src/state.cc + src/status.cc + src/string_piece_util.cc ++ src/tokenpool-gnu-make.cc + src/util.cc + src/version.cc + ) +@@ -123,9 +124,13 @@ if(WIN32) + src/msvc_helper_main-win32.cc + src/getopt.c + src/minidump-win32.cc ++ src/tokenpool-gnu-make-win32.cc + ) + else() +- target_sources(libninja PRIVATE src/subprocess-posix.cc) ++ target_sources(libninja PRIVATE ++ src/subprocess-posix.cc ++ src/tokenpool-gnu-make-posix.cc ++ ) + if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") + target_sources(libninja PRIVATE src/getopt.c) + endif() +@@ -204,6 +209,7 @@ if(BUILD_TESTING) + src/string_piece_util_test.cc + src/subprocess_test.cc + src/test.cc ++ src/tokenpool_test.cc + src/util_test.cc + ) + if(WIN32) +diff --git a/src/build.cc b/src/build.cc +index 20c3bdc2a0..854df08c2a 100644 +--- a/src/build.cc ++++ b/src/build.cc +@@ -467,7 +467,7 @@ struct RealCommandRunner : public CommandRunner { + // copy of config_.max_load_average; can be modified by TokenPool setup + double max_load_average_; + SubprocessSet subprocs_; +- TokenPool *tokens_; ++ TokenPool* tokens_; + map subproc_to_edge_; + }; + +diff --git a/src/build_test.cc b/src/build_test.cc +index dd41dfbe1d..8901c9518f 100644 +--- a/src/build_test.cc ++++ b/src/build_test.cc +@@ -4098,7 +4098,7 @@ struct BuildTokenTest : public BuildTest { + void ExpectWaitForCommand(int count, ...); + + private: +- void EnqueueBooleans(vector& booleans, int count, va_list ao); ++ void EnqueueBooleans(vector& booleans, int count, va_list ap); + }; + + void BuildTokenTest::SetUp() { +@@ -4144,16 +4144,6 @@ void BuildTokenTest::EnqueueBooleans(vector& booleans, int count, va_list + } + } + +-TEST_F(BuildTokenTest, CompleteNoWork) { +- // plan should not execute anything +- string err; +- +- EXPECT_TRUE(builder_.Build(&err)); +- EXPECT_EQ("", err); +- +- EXPECT_EQ(0u, token_command_runner_.commands_ran_.size()); +-} +- + TEST_F(BuildTokenTest, DoNotAquireToken) { + // plan should execute one command + string err; +diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc +index 74451b0be2..31839276c4 100644 +--- a/src/subprocess-posix.cc ++++ b/src/subprocess-posix.cc +@@ -250,7 +250,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + } + + #ifdef USE_PPOLL +-bool SubprocessSet::DoWork(struct TokenPool* tokens) { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + vector fds; + nfds_t nfds = 0; + +@@ -315,7 +315,7 @@ bool SubprocessSet::DoWork(struct TokenPool* tokens) { + } + + #else // !defined(USE_PPOLL) +-bool SubprocessSet::DoWork(struct TokenPool* tokens) { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + fd_set set; + int nfds = 0; + FD_ZERO(&set); +diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc +index ce3e2c20a4..2926e9a221 100644 +--- a/src/subprocess-win32.cc ++++ b/src/subprocess-win32.cc +@@ -252,7 +252,7 @@ Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + return subprocess; + } + +-bool SubprocessSet::DoWork(struct TokenPool* tokens) { ++bool SubprocessSet::DoWork(TokenPool* tokens) { + DWORD bytes_read; + Subprocess* subproc; + OVERLAPPED* overlapped; +diff --git a/src/subprocess.h b/src/subprocess.h +index 9ea67ea477..1ec78171e8 100644 +--- a/src/subprocess.h ++++ b/src/subprocess.h +@@ -86,7 +86,7 @@ struct SubprocessSet { + ~SubprocessSet(); + + Subprocess* Add(const std::string& command, bool use_console = false); +- bool DoWork(struct TokenPool* tokens); ++ bool DoWork(TokenPool* tokens); + Subprocess* NextFinished(); + void Clear(); + +diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc +index f625963462..7b146f31be 100644 +--- a/src/subprocess_test.cc ++++ b/src/subprocess_test.cc +@@ -35,7 +35,7 @@ const char* kSimpleCommand = "cmd /c dir \\"; + const char* kSimpleCommand = "ls /"; + #endif + +-struct TokenPoolTest : public TokenPool { ++struct TestTokenPool : public TokenPool { + bool Acquire() { return false; } + void Reserve() {} + void Release() {} +@@ -58,7 +58,7 @@ struct TokenPoolTest : public TokenPool { + + struct SubprocessTest : public testing::Test { + SubprocessSet subprocs_; +- TokenPoolTest tokens_; ++ TestTokenPool tokens_; + }; + + } // anonymous namespace +@@ -73,7 +73,7 @@ TEST_F(SubprocessTest, BadCommandStderr) { + // Pretend we discovered that stderr was ready for writing. + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -89,7 +89,7 @@ TEST_F(SubprocessTest, NoSuchCommand) { + // Pretend we discovered that stderr was ready for writing. + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitFailure, subproc->Finish()); + EXPECT_NE("", subproc->GetOutput()); +@@ -109,7 +109,7 @@ TEST_F(SubprocessTest, InterruptChild) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -135,7 +135,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigTerm) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -161,7 +161,7 @@ TEST_F(SubprocessTest, InterruptChildWithSigHup) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); + } +@@ -190,7 +190,7 @@ TEST_F(SubprocessTest, Console) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } +@@ -206,7 +206,7 @@ TEST_F(SubprocessTest, SetWithSingle) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_NE("", subproc->GetOutput()); + +@@ -243,7 +243,7 @@ TEST_F(SubprocessTest, SetWithMulti) { + ASSERT_GT(subprocs_.running_.size(), 0u); + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(0u, subprocs_.running_.size()); + ASSERT_EQ(3u, subprocs_.finished_.size()); + +@@ -278,7 +278,7 @@ TEST_F(SubprocessTest, SetWithLots) { + subprocs_.ResetTokenAvailable(); + while (!subprocs_.running_.empty()) + subprocs_.DoWork(NULL); +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + for (size_t i = 0; i < procs.size(); ++i) { + ASSERT_EQ(ExitSuccess, procs[i]->Finish()); + ASSERT_NE("", procs[i]->GetOutput()); +@@ -298,7 +298,7 @@ TEST_F(SubprocessTest, ReadStdin) { + while (!subproc->Done()) { + subprocs_.DoWork(NULL); + } +- ASSERT_EQ(false, subprocs_.IsTokenAvailable()); ++ ASSERT_FALSE(subprocs_.IsTokenAvailable()); + ASSERT_EQ(ExitSuccess, subproc->Finish()); + ASSERT_EQ(1u, subprocs_.finished_.size()); + } +@@ -322,7 +322,7 @@ TEST_F(SubprocessTest, TokenAvailable) { + subprocs_.DoWork(&tokens_); + #ifdef _WIN32 + tokens_._token_available = false; +- // we need to loop here as we have no conrol where the token ++ // we need to loop here as we have no control where the token + // I/O completion post ends up in the queue + while (!subproc->Done() && !subprocs_.IsTokenAvailable()) { + subprocs_.DoWork(&tokens_); +diff --git a/src/tokenpool-gnu-make-posix.cc b/src/tokenpool-gnu-make-posix.cc +index 70d84bfff7..353bda226a 100644 +--- a/src/tokenpool-gnu-make-posix.cc ++++ b/src/tokenpool-gnu-make-posix.cc +@@ -32,8 +32,8 @@ struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool { + + virtual int GetMonitorFd(); + +- virtual const char *GetEnv(const char *name) { return getenv(name); }; +- virtual bool ParseAuth(const char *jobserver); ++ virtual const char* GetEnv(const char* name) { return getenv(name); }; ++ virtual bool ParseAuth(const char* jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + +@@ -64,9 +64,7 @@ bool GNUmakeTokenPoolPosix::CheckFd(int fd) { + if (fd < 0) + return false; + int ret = fcntl(fd, F_GETFD); +- if (ret < 0) +- return false; +- return true; ++ return ret >= 0; + } + + int GNUmakeTokenPoolPosix::dup_rfd_ = -1; +@@ -82,14 +80,13 @@ bool GNUmakeTokenPoolPosix::SetAlarmHandler() { + act.sa_handler = CloseDupRfd; + if (sigaction(SIGALRM, &act, &old_act_) < 0) { + perror("sigaction:"); +- return(false); +- } else { +- restore_ = true; +- return(true); ++ return false; + } ++ restore_ = true; ++ return true; + } + +-bool GNUmakeTokenPoolPosix::ParseAuth(const char *jobserver) { ++bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) { + int rfd = -1; + int wfd = -1; + if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) && +@@ -195,9 +192,9 @@ bool GNUmakeTokenPoolPosix::ReturnToken() { + } + + int GNUmakeTokenPoolPosix::GetMonitorFd() { +- return(rfd_); ++ return rfd_; + } + +-struct TokenPool *TokenPool::Get() { ++TokenPool* TokenPool::Get() { + return new GNUmakeTokenPoolPosix; + } +diff --git a/src/tokenpool-gnu-make-win32.cc b/src/tokenpool-gnu-make-win32.cc +index 2719f2c1fc..b2bb52fadb 100644 +--- a/src/tokenpool-gnu-make-win32.cc ++++ b/src/tokenpool-gnu-make-win32.cc +@@ -14,7 +14,8 @@ + + #include "tokenpool-gnu-make.h" + +-// always include first to make sure other headers do the correct thing... ++// Always include this first. ++// Otherwise the other system headers don't work correctly under Win32 + #include + + #include +@@ -32,8 +33,8 @@ struct GNUmakeTokenPoolWin32 : public GNUmakeTokenPool { + virtual void WaitForTokenAvailability(HANDLE ioport); + virtual bool TokenIsAvailable(ULONG_PTR key); + +- virtual const char *GetEnv(const char *name); +- virtual bool ParseAuth(const char *jobserver); ++ virtual const char* GetEnv(const char* name); ++ virtual bool ParseAuth(const char* jobserver); + virtual bool AcquireToken(); + virtual bool ReturnToken(); + +@@ -98,19 +99,19 @@ GNUmakeTokenPoolWin32::~GNUmakeTokenPoolWin32() { + } + } + +-const char *GNUmakeTokenPoolWin32::GetEnv(const char *name) { ++const char* GNUmakeTokenPoolWin32::GetEnv(const char* name) { + // getenv() does not work correctly together with tokenpool_tests.cc + static char buffer[MAX_PATH + 1]; +- if (GetEnvironmentVariable("MAKEFLAGS", buffer, sizeof(buffer)) == 0) ++ if (GetEnvironmentVariable(name, buffer, sizeof(buffer)) == 0) + return NULL; +- return(buffer); ++ return buffer; + } + +-bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { ++bool GNUmakeTokenPoolWin32::ParseAuth(const char* jobserver) { + // match "--jobserver-auth=gmake_semaphore_..." +- const char *start = strchr(jobserver, '='); ++ const char* start = strchr(jobserver, '='); + if (start) { +- const char *end = start; ++ const char* end = start; + unsigned int len; + char c, *auth; + +@@ -119,14 +120,15 @@ bool GNUmakeTokenPoolWin32::ParseAuth(const char *jobserver) { + break; + len = end - start; // includes string terminator in count + +- if ((len > 1) && ((auth = (char *)malloc(len)) != NULL)) { ++ if ((len > 1) && ((auth = (char*)malloc(len)) != NULL)) { + strncpy(auth, start + 1, len - 1); + auth[len - 1] = '\0'; + +- if ((semaphore_jobserver_ = OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ +- FALSE, /* Child processes DON'T inherit */ +- auth /* Semaphore name */ +- )) != NULL) { ++ if ((semaphore_jobserver_ = ++ OpenSemaphore(SEMAPHORE_ALL_ACCESS, /* Semaphore access setting */ ++ FALSE, /* Child processes DON'T inherit */ ++ auth /* Semaphore name */ ++ )) != NULL) { + free(auth); + return true; + } +@@ -171,7 +173,7 @@ DWORD GNUmakeTokenPoolWin32::SemaphoreThread() { + } + + DWORD WINAPI GNUmakeTokenPoolWin32::SemaphoreThreadWrapper(LPVOID param) { +- GNUmakeTokenPoolWin32 *This = (GNUmakeTokenPoolWin32 *) param; ++ GNUmakeTokenPoolWin32* This = (GNUmakeTokenPoolWin32*) param; + return This->SemaphoreThread(); + } + +@@ -216,7 +218,7 @@ void GNUmakeTokenPoolWin32::WaitForTokenAvailability(HANDLE ioport) { + + bool GNUmakeTokenPoolWin32::TokenIsAvailable(ULONG_PTR key) { + // alert child thread to break wait on token semaphore +- QueueUserAPC(&NoopAPCFunc, child_, (ULONG_PTR)NULL); ++ QueueUserAPC((PAPCFUNC)&NoopAPCFunc, child_, (ULONG_PTR)NULL); + + // return true when GetQueuedCompletionStatus() returned our key + return key == (ULONG_PTR) this; +@@ -232,6 +234,6 @@ void GNUmakeTokenPoolWin32::WaitForObject(HANDLE object) { + Win32Fatal("WaitForSingleObject"); + } + +-struct TokenPool *TokenPool::Get() { ++TokenPool* TokenPool::Get() { + return new GNUmakeTokenPoolWin32; + } +diff --git a/src/tokenpool-gnu-make.cc b/src/tokenpool-gnu-make.cc +index 92ff611721..60e0552924 100644 +--- a/src/tokenpool-gnu-make.cc ++++ b/src/tokenpool-gnu-make.cc +@@ -31,36 +31,37 @@ GNUmakeTokenPool::~GNUmakeTokenPool() { + bool GNUmakeTokenPool::Setup(bool ignore, + bool verbose, + double& max_load_average) { +- const char *value = GetEnv("MAKEFLAGS"); +- if (value) { +- // GNU make <= 4.1 +- const char *jobserver = strstr(value, "--jobserver-fds="); ++ const char* value = GetEnv("MAKEFLAGS"); ++ if (!value) ++ return false; + -+ // implicit token -+ EXPECT_TRUE(tokens_->Acquire()); -+} ++ // GNU make <= 4.1 ++ const char* jobserver = strstr(value, "--jobserver-fds="); ++ if (!jobserver) + // GNU make => 4.2 +- if (!jobserver) +- jobserver = strstr(value, "--jobserver-auth="); +- if (jobserver) { +- LinePrinter printer; +- +- if (ignore) { +- printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); +- } else { +- if (ParseAuth(jobserver)) { +- const char *l_arg = strstr(value, " -l"); +- int load_limit = -1; +- +- if (verbose) { +- printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); +- } +- +- // translate GNU make -lN to ninja -lN +- if (l_arg && +- (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && +- (load_limit > 0)) { +- max_load_average = load_limit; +- } +- +- return true; ++ jobserver = strstr(value, "--jobserver-auth="); ++ if (jobserver) { ++ LinePrinter printer; ++ ++ if (ignore) { ++ printer.PrintOnNewLine("ninja: warning: -jN forced on command line; ignoring GNU make jobserver.\n"); ++ } else { ++ if (ParseAuth(jobserver)) { ++ const char* l_arg = strstr(value, " -l"); ++ int load_limit = -1; ++ ++ if (verbose) { ++ printer.PrintOnNewLine("ninja: using GNU make jobserver.\n"); ++ } ++ ++ // translate GNU make -lN to ninja -lN ++ if (l_arg && ++ (sscanf(l_arg + 3, "%d ", &load_limit) == 1) && ++ (load_limit > 0)) { ++ max_load_average = load_limit; + } ++ ++ return true; + } + } + } +@@ -76,10 +77,10 @@ bool GNUmakeTokenPool::Acquire() { + // token acquired + available_++; + return true; +- } else { +- // no token available +- return false; + } ++ ++ // no token available ++ return false; + } + + void GNUmakeTokenPool::Reserve() { +diff --git a/src/tokenpool-gnu-make.h b/src/tokenpool-gnu-make.h +index d3852088e2..c94cca5e2d 100644 +--- a/src/tokenpool-gnu-make.h ++++ b/src/tokenpool-gnu-make.h +@@ -17,7 +17,7 @@ + // interface to GNU make token pool + struct GNUmakeTokenPool : public TokenPool { + GNUmakeTokenPool(); +- virtual ~GNUmakeTokenPool(); ++ ~GNUmakeTokenPool(); + + // token pool implementation + virtual bool Acquire(); +@@ -27,8 +27,8 @@ struct GNUmakeTokenPool : public TokenPool { + virtual bool Setup(bool ignore, bool verbose, double& max_load_average); + + // platform specific implementation +- virtual const char *GetEnv(const char *name) = 0; +- virtual bool ParseAuth(const char *jobserver) = 0; ++ virtual const char* GetEnv(const char* name) = 0; ++ virtual bool ParseAuth(const char* jobserver) = 0; + virtual bool AcquireToken() = 0; + virtual bool ReturnToken() = 0; + +diff --git a/src/tokenpool-none.cc b/src/tokenpool-none.cc +deleted file mode 100644 +index 613d16882d..0000000000 +--- a/src/tokenpool-none.cc ++++ /dev/null +@@ -1,22 +0,0 @@ +-// Copyright 2016-2018 Google Inc. All Rights Reserved. +-// +-// Licensed under the Apache License, Version 2.0 (the "License"); +-// you may not use this file except in compliance with the License. +-// You may obtain a copy of the License at +-// +-// http://www.apache.org/licenses/LICENSE-2.0 +-// +-// Unless required by applicable law or agreed to in writing, software +-// distributed under the License is distributed on an "AS IS" BASIS, +-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-// See the License for the specific language governing permissions and +-// limitations under the License. +- +-#include "tokenpool.h" +- +-#include +- +-// No-op TokenPool implementation +-struct TokenPool *TokenPool::Get() { +- return NULL; +-} +diff --git a/src/tokenpool.h b/src/tokenpool.h +index 1be8e1d5ce..931c22754d 100644 +--- a/src/tokenpool.h ++++ b/src/tokenpool.h +@@ -38,5 +38,5 @@ struct TokenPool { + #endif + + // returns NULL if token pool is not available +- static struct TokenPool *Get(); ++ static TokenPool* Get(); + }; +diff --git a/src/tokenpool_test.cc b/src/tokenpool_test.cc +index 8d4fd7d33a..8d3061cb30 100644 +--- a/src/tokenpool_test.cc ++++ b/src/tokenpool_test.cc +@@ -43,10 +43,10 @@ const double kLoadAverageDefault = -1.23456789; + + struct TokenPoolTest : public testing::Test { + double load_avg_; +- TokenPool *tokens_; ++ TokenPool* tokens_; + char buf_[1024]; + #ifdef _WIN32 +- const char *semaphore_name_; ++ const char* semaphore_name_; + HANDLE semaphore_; + #else + int fds_[2]; +@@ -65,7 +65,7 @@ struct TokenPoolTest : public testing::Test { + ASSERT_TRUE(false); + } + +- void CreatePool(const char *format, bool ignore_jobserver = false) { ++ void CreatePool(const char* format, bool ignore_jobserver = false) { + if (format) { + sprintf(buf_, format, + #ifdef _WIN32 +@@ -209,7 +209,7 @@ TEST_F(TokenPoolTest, TwoTokens) { + tokens_->Release(); + EXPECT_TRUE(tokens_->Acquire()); + +- // release implict token - must return 2nd token back to jobserver ++ // release implicit token - must return 2nd token back to jobserver + tokens_->Release(); + EXPECT_TRUE(tokens_->Acquire()); +