From ade56b8d9e81553315ba9feaef2e207ad55b06f5 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Tue, 12 Oct 2021 14:44:50 +0200 Subject: [PATCH 01/22] mt76: update to the latest version 83598c2e872f mt76: avoid possible infinite loop in mt76_tx_status_check Signed-off-by: Felix Fietkau --- package/kernel/mt76/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package/kernel/mt76/Makefile b/package/kernel/mt76/Makefile index 511960ce9b..97cb9e8d59 100644 --- a/package/kernel/mt76/Makefile +++ b/package/kernel/mt76/Makefile @@ -8,9 +8,9 @@ PKG_LICENSE_FILES:= PKG_SOURCE_URL:=https://github.com/openwrt/mt76 PKG_SOURCE_PROTO:=git -PKG_SOURCE_DATE:=2021-10-11 -PKG_SOURCE_VERSION:=1af0242d1241cba761415c492e5ed9673beaa13f -PKG_MIRROR_HASH:=39346f1ef24fbd62f447fc116e1ed411dc2b60787a3df9edfe048adc79150232 +PKG_SOURCE_DATE:=2021-10-12 +PKG_SOURCE_VERSION:=83598c2e872fd8af29976a5f11afe94e620d6f4f +PKG_MIRROR_HASH:=f179375318c44fb058403aed98056c4df8df7541e4c9d093ec5c22a2eda9475e PKG_MAINTAINER:=Felix Fietkau PKG_USE_NINJA:=0 From c1685d92e4452e4c9cc6ee5d7612b259396007a8 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 13 Oct 2021 00:33:44 +0100 Subject: [PATCH 02/22] netifd: update to git HEAD be8cd8f interface: don't fork() to start jail interface 7a048bd interface, ubus: rework netns up/down Signed-off-by: Daniel Golle --- package/network/config/netifd/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package/network/config/netifd/Makefile b/package/network/config/netifd/Makefile index 5e6ebbde12..727c3ac690 100644 --- a/package/network/config/netifd/Makefile +++ b/package/network/config/netifd/Makefile @@ -5,9 +5,9 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL=$(PROJECT_GIT)/project/netifd.git -PKG_SOURCE_DATE:=2021-09-29 -PKG_SOURCE_VERSION:=448ffc154fe78ee1c8e4b2cda1a479f0d8af8b37 -PKG_MIRROR_HASH:=eec346ae3e27b8e37b91600e1ee822504b8715bbaf360a6bd962e61b6a6ad13e +PKG_SOURCE_DATE:=2021-10-13 +PKG_SOURCE_VERSION:=7a048bd6871df952ff1324bec7175891c9b38f99 +PKG_MIRROR_HASH:=b4fdca3c45713710f4ea69f1ed0fb16d3b07b70002e6b39640ce6e56364817d0 PKG_MAINTAINER:=Felix Fietkau PKG_LICENSE:=GPL-2.0 From 213ce1d837f5eb98d76ae159e2f0b70f6e6c2537 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Wed, 13 Oct 2021 00:38:12 +0100 Subject: [PATCH 03/22] procd: update to git HEAD 97bcdcf uxc: fix segfault caused by use-after-free 6398e05 uxc: don't free the stack 324ebd0 jail: fs: add support for asymmetric mount bind c44ab7f jail: netifd: generate netifd uci config and mount it 82dd390 jail: make use of per-container netifd via ubus The new per-jail netifd is now configured by filtering the host network configuration. As libuci is used for that, procd-ujail now depends on libuci. Signed-off-by: Daniel Golle --- package/system/procd/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package/system/procd/Makefile b/package/system/procd/Makefile index 1e43c16330..6f506423f8 100644 --- a/package/system/procd/Makefile +++ b/package/system/procd/Makefile @@ -12,9 +12,9 @@ PKG_RELEASE:=$(AUTORELEASE) PKG_SOURCE_PROTO:=git PKG_SOURCE_URL=$(PROJECT_GIT)/project/procd.git -PKG_SOURCE_DATE:=2021-09-15 -PKG_SOURCE_VERSION:=1eb4371e2534296d04580cb8b9cb5baa5f07e27d -PKG_MIRROR_HASH:=d0e61032482eebb225ffbca8034b3d838eaa3db0dcd9b0166e106e729ce5320c +PKG_SOURCE_DATE:=2021-10-13 +PKG_SOURCE_VERSION:=82dd39024f63e53becd07c89831ed7d5fac7f339 +PKG_MIRROR_HASH:=a9449894b4ce0b2860680c1691270d5fe9c84e4eecd48008ed732ef2d19af7db CMAKE_INSTALL:=1 PKG_LICENSE:=GPL-2.0 @@ -68,7 +68,7 @@ define Package/procd-ujail SECTION:=base CATEGORY:=Base system DEPENDS:=@KERNEL_NAMESPACES +@KERNEL_UTS_NS +@KERNEL_IPC_NS +@KERNEL_PID_NS \ - +libubox +libubus +libblobmsg-json + +libubox +libubus +libuci +libblobmsg-json TITLE:=OpenWrt process jail helper endef From 4056a40160fbd2f42d0a83f0aed6c2687a6dfa1f Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Mon, 11 Oct 2021 10:29:40 -1000 Subject: [PATCH 04/22] armvirt: enable MMIO_CMDLINE_DEVICES for firecracker support This Kernel option allows to run OpenWrt witin a `firecracker` micro VM. Firecracker is a KVM-based tool for superfast booting VMs on x86_64 and aarch64. It makes rootfs available to the guest as a virtio-mmio device and passes its address via the kernel cmdline. A kernel without CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES will not recognize the rootfs virtio-mmio device. Suggested-by: Packet Please Signed-off-by: Paul Spooren --- target/linux/armvirt/config-5.10 | 1 + 1 file changed, 1 insertion(+) diff --git a/target/linux/armvirt/config-5.10 b/target/linux/armvirt/config-5.10 index 5715a74195..74b6957d94 100644 --- a/target/linux/armvirt/config-5.10 +++ b/target/linux/armvirt/config-5.10 @@ -156,6 +156,7 @@ CONFIG_VIRTIO_BALLOON=y CONFIG_VIRTIO_BLK=y CONFIG_VIRTIO_CONSOLE=y CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y CONFIG_VIRTIO_NET=y CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_PCI_LEGACY=y From e07cc4699127d882f619c978c215bf2a762b61fa Mon Sep 17 00:00:00 2001 From: Paul Spooren Date: Mon, 11 Oct 2021 10:33:26 -1000 Subject: [PATCH 05/22] x86/64: enable MMIO_CMDLINE_DEVICES for firecracker support This Kernel option allows to run OpenWrt witin a `firecracker` micro VM. Firecracker is a KVM-based tool for superfast booting VMs on x86_64 and aarch64. It makes rootfs available to the guest as a virtio-mmio device and passes its address via the kernel cmdline. A kernel without CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES will not recognize the rootfs virtio-mmio device. Suggested-by: Packet Please Signed-off-by: Paul Spooren --- target/linux/x86/64/config-5.10 | 1 + 1 file changed, 1 insertion(+) diff --git a/target/linux/x86/64/config-5.10 b/target/linux/x86/64/config-5.10 index 4e979136a5..a62e5a795a 100644 --- a/target/linux/x86/64/config-5.10 +++ b/target/linux/x86/64/config-5.10 @@ -416,6 +416,7 @@ CONFIG_VIRTIO_BLK=y CONFIG_VIRTIO_CONSOLE=y CONFIG_VIRTIO_DMA_SHARED_BUFFER=y CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y CONFIG_VIRTIO_NET=y CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_PCI_LEGACY=y From 48e209e5c5a76dd0ce4a2a8b014416dc988269a3 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Wed, 13 Oct 2021 16:36:16 +0200 Subject: [PATCH 06/22] tools/fakeroot: explicitly pass CPP variable For some reason, the generated configure script fails to properly set up the internal preprocessor command variable, causing the host OS check for Darwin to fail after the last update. Explicitly setting CPP fixes this issue Signed-off-by: Felix Fietkau --- tools/fakeroot/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/fakeroot/Makefile b/tools/fakeroot/Makefile index 5c5f25e226..f4333fbfb3 100644 --- a/tools/fakeroot/Makefile +++ b/tools/fakeroot/Makefile @@ -19,7 +19,8 @@ include $(INCLUDE_DIR)/host-build.mk HOST_CONFIGURE_VARS += \ ac_cv_header_sys_capability_h=no \ - ac_cv_func_capset=no + ac_cv_func_capset=no \ + CPP="$(HOSTCC) -E" HOST_CONFIGURE_ARGS += \ --with-ipc=tcp From a3de42e72ce349c14263737c5d6ec7b00397cf14 Mon Sep 17 00:00:00 2001 From: David Bauer Date: Wed, 21 Oct 2020 20:50:01 +0200 Subject: [PATCH 07/22] hostapd: ubus: add notification for BSS transition response To allow steering daemons to be aware of the STA-decided transition target, publish WNM transition responses to ubus. This way, steerings daemons can learn about STA-chosen targets and send a better selection of transition candidates. Signed-off-by: David Bauer --- .../hostapd/patches/600-ubus_support.patch | 30 +++++++++++++++++ .../services/hostapd/src/src/ap/ubus.c | 33 +++++++++++++++++++ .../services/hostapd/src/src/ap/ubus.h | 11 +++++++ 3 files changed, 74 insertions(+) diff --git a/package/network/services/hostapd/patches/600-ubus_support.patch b/package/network/services/hostapd/patches/600-ubus_support.patch index 005ed54e0c..d77d245243 100644 --- a/package/network/services/hostapd/patches/600-ubus_support.patch +++ b/package/network/services/hostapd/patches/600-ubus_support.patch @@ -514,3 +514,33 @@ struct os_reltime backlogged_until; #endif /* CONFIG_AIRTIME_POLICY */ +--- a/src/ap/wnm_ap.c ++++ b/src/ap/wnm_ap.c +@@ -463,7 +463,7 @@ static void ieee802_11_rx_bss_trans_mgmt + size_t len) + { + u8 dialog_token, status_code, bss_termination_delay; +- const u8 *pos, *end; ++ const u8 *pos, *end, *target_bssid; + int enabled = hapd->conf->bss_transition; + struct sta_info *sta; + +@@ -510,6 +510,7 @@ static void ieee802_11_rx_bss_trans_mgmt + wpa_printf(MSG_DEBUG, "WNM: not enough room for Target BSSID field"); + return; + } ++ target_bssid = pos; + sta->agreed_to_steer = 1; + eloop_cancel_timeout(ap_sta_reset_steer_flag_timer, hapd, sta); + eloop_register_timeout(2, 0, ap_sta_reset_steer_flag_timer, +@@ -529,6 +530,10 @@ static void ieee802_11_rx_bss_trans_mgmt + MAC2STR(addr), status_code, bss_termination_delay); + } + ++ hostapd_ubus_notify_bss_transition_response(hapd, sta->addr, dialog_token, ++ status_code, bss_termination_delay, ++ target_bssid, pos, end - pos); ++ + wpa_hexdump(MSG_DEBUG, "WNM: BSS Transition Candidate List Entries", + pos, end - pos); + } diff --git a/package/network/services/hostapd/src/src/ap/ubus.c b/package/network/services/hostapd/src/src/ap/ubus.c index 367e1b652b..30bb353953 100644 --- a/package/network/services/hostapd/src/src/ap/ubus.c +++ b/package/network/services/hostapd/src/src/ap/ubus.c @@ -1757,3 +1757,36 @@ void hostapd_ubus_notify_radar_detected(struct hostapd_iface *iface, int frequen ubus_notify(ctx, &hapd->ubus.obj, "radar-detected", b.head, -1); } } + +void hostapd_ubus_notify_bss_transition_response( + struct hostapd_data *hapd, const u8 *addr, u8 dialog_token, u8 status_code, + u8 bss_termination_delay, const u8 *target_bssid, + const u8 *candidate_list, u16 candidate_list_len) +{ +#ifdef CONFIG_WNM_AP + char *cl_str; + u16 i; + + if (!hapd->ubus.obj.has_subscribers) + return; + + if (!addr) + return; + + blob_buf_init(&b, 0); + blobmsg_add_macaddr(&b, "address", addr); + blobmsg_add_u8(&b, "dialog-token", dialog_token); + blobmsg_add_u8(&b, "status-code", status_code); + blobmsg_add_u8(&b, "bss-termination-delay", bss_termination_delay); + if (target_bssid) + blobmsg_add_macaddr(&b, "target-bssid", target_bssid); + if (candidate_list_len > 0) { + cl_str = blobmsg_alloc_string_buffer(&b, "candidate-list", candidate_list_len * 2 + 1); + for (i = 0; i < candidate_list_len; i++) + snprintf(&cl_str[i*2], 3, "%02X", candidate_list[i]); + blobmsg_add_string_buffer(&b); + } + + ubus_notify(ctx, &hapd->ubus.obj, "bss-transition-response", b.head, -1); +#endif +} diff --git a/package/network/services/hostapd/src/src/ap/ubus.h b/package/network/services/hostapd/src/src/ap/ubus.h index acdac74360..899b001dd5 100644 --- a/package/network/services/hostapd/src/src/ap/ubus.h +++ b/package/network/services/hostapd/src/src/ap/ubus.h @@ -55,6 +55,10 @@ void hostapd_ubus_notify_beacon_report(struct hostapd_data *hapd, void hostapd_ubus_notify_radar_detected(struct hostapd_iface *iface, int frequency, int chan_width, int cf1, int cf2); +void hostapd_ubus_notify_bss_transition_response( + struct hostapd_data *hapd, const u8 *addr, u8 dialog_token, u8 status_code, + u8 bss_termination_delay, const u8 *target_bssid, + const u8 *candidate_list, u16 candidate_list_len); void hostapd_ubus_add(struct hapd_interfaces *interfaces); void hostapd_ubus_free(struct hapd_interfaces *interfaces); @@ -107,6 +111,13 @@ static inline void hostapd_ubus_notify_radar_detected(struct hostapd_iface *ifac { } +static inline void hostapd_ubus_notify_bss_transition_response( + struct hostapd_data *hapd, const u8 *addr, u8 dialog_token, u8 status_code, + u8 bss_termination_delay, const u8 *target_bssid, + const u8 *candidate_list, u16 candidate_list_len) +{ +} + static inline void hostapd_ubus_add(struct hapd_interfaces *interfaces) { } From 0eed96ca5d86bdc4158c8ef6814893bc2a3dc467 Mon Sep 17 00:00:00 2001 From: David Bauer Date: Sat, 5 Dec 2020 20:17:35 +0100 Subject: [PATCH 08/22] hostapd: ubus: add BSS transtiton request method The existing wnm_disassoc_imminent ubus method only supports issuing a bss transition request with the disassoc imminent flag set. For use-cases, where the client is requested to roam to another BSS without a pending disassoc, this existing method is not suitable. Add a new bss_transition_request ubus method, which provides a more universal way to dispatch a transition request. It takes the following arguments: Required: addr: String - MAC-address of the STA to send the request to (colon-seperated) Optional: abridged - Bool - Indicates if the abridged flag is set disassociation_imminent: Bool - Whether or not the disassoc_imminent flag is set disassociation_timer: I32 - number of TBTTs after which the client will be disassociated validity_period: I32 - number of TBTTs after which the beacon candidate list (if included) will be invalid neighbors: blob-array - Array of strings containing neighbor reports as hex-string Signed-off-by: David Bauer --- .../services/hostapd/src/src/ap/ubus.c | 188 ++++++++++++------ 1 file changed, 130 insertions(+), 58 deletions(-) diff --git a/package/network/services/hostapd/src/src/ap/ubus.c b/package/network/services/hostapd/src/src/ap/ubus.c index 30bb353953..69778596a4 100644 --- a/package/network/services/hostapd/src/src/ap/ubus.c +++ b/package/network/services/hostapd/src/src/ap/ubus.c @@ -1275,6 +1275,129 @@ hostapd_rrm_beacon_req(struct ubus_context *ctx, struct ubus_object *obj, #ifdef CONFIG_WNM_AP + +static int +hostapd_bss_tr_send(struct hostapd_data *hapd, u8 *addr, bool disassoc_imminent, bool abridged, + u16 disassoc_timer, u8 validity_period, struct blob_attr *neighbors) +{ + struct blob_attr *cur; + struct sta_info *sta; + int nr_len = 0; + int rem; + u8 *nr = NULL; + u8 req_mode = 0; + + sta = ap_get_sta(hapd, addr); + if (!sta) + return UBUS_STATUS_NOT_FOUND; + + if (neighbors) { + u8 *nr_cur; + + if (blobmsg_check_array(neighbors, + BLOBMSG_TYPE_STRING) < 0) + return UBUS_STATUS_INVALID_ARGUMENT; + + blobmsg_for_each_attr(cur, neighbors, rem) { + int len = strlen(blobmsg_get_string(cur)); + + if (len % 2) + return UBUS_STATUS_INVALID_ARGUMENT; + + nr_len += (len / 2) + 2; + } + + if (nr_len) { + nr = os_zalloc(nr_len); + if (!nr) + return UBUS_STATUS_UNKNOWN_ERROR; + } + + nr_cur = nr; + blobmsg_for_each_attr(cur, neighbors, rem) { + int len = strlen(blobmsg_get_string(cur)) / 2; + + *nr_cur++ = WLAN_EID_NEIGHBOR_REPORT; + *nr_cur++ = (u8) len; + if (hexstr2bin(blobmsg_data(cur), nr_cur, len)) { + free(nr); + return UBUS_STATUS_INVALID_ARGUMENT; + } + + nr_cur += len; + } + } + + if (nr) + req_mode |= WNM_BSS_TM_REQ_PREF_CAND_LIST_INCLUDED; + + if (abridged) + req_mode |= WNM_BSS_TM_REQ_ABRIDGED; + + if (disassoc_imminent) + req_mode |= WNM_BSS_TM_REQ_DISASSOC_IMMINENT; + + if (wnm_send_bss_tm_req(hapd, sta, req_mode, disassoc_timer, validity_period, NULL, + NULL, nr, nr_len, NULL, 0)) + return UBUS_STATUS_UNKNOWN_ERROR; + + return 0; +} + +enum { + BSS_TR_ADDR, + BSS_TR_DA_IMMINENT, + BSS_TR_DA_TIMER, + BSS_TR_VALID_PERIOD, + BSS_TR_NEIGHBORS, + BSS_TR_ABRIDGED, + __BSS_TR_DISASSOC_MAX +}; + +static const struct blobmsg_policy bss_tr_policy[__BSS_TR_DISASSOC_MAX] = { + [BSS_TR_ADDR] = { "addr", BLOBMSG_TYPE_STRING }, + [BSS_TR_DA_IMMINENT] = { "disassociation_imminent", BLOBMSG_TYPE_BOOL }, + [BSS_TR_DA_TIMER] = { "disassociation_timer", BLOBMSG_TYPE_INT32 }, + [BSS_TR_VALID_PERIOD] = { "validity_period", BLOBMSG_TYPE_INT32 }, + [BSS_TR_NEIGHBORS] = { "neighbors", BLOBMSG_TYPE_ARRAY }, + [BSS_TR_ABRIDGED] = { "abridged", BLOBMSG_TYPE_BOOL }, +}; + +static int +hostapd_bss_transition_request(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *ureq, const char *method, + struct blob_attr *msg) +{ + struct hostapd_data *hapd = container_of(obj, struct hostapd_data, ubus.obj); + struct blob_attr *tb[__BSS_TR_DISASSOC_MAX]; + struct sta_info *sta; + u32 da_timer = 0; + u32 valid_period = 0; + u8 addr[ETH_ALEN]; + bool abridged; + bool da_imminent; + + blobmsg_parse(bss_tr_policy, __BSS_TR_DISASSOC_MAX, tb, blob_data(msg), blob_len(msg)); + + if (!tb[BSS_TR_ADDR]) + return UBUS_STATUS_INVALID_ARGUMENT; + + if (hwaddr_aton(blobmsg_data(tb[BSS_TR_ADDR]), addr)) + return UBUS_STATUS_INVALID_ARGUMENT; + + if (tb[BSS_TR_DA_TIMER]) + da_timer = blobmsg_get_u32(tb[BSS_TR_DA_TIMER]); + + if (tb[BSS_TR_VALID_PERIOD]) + valid_period = blobmsg_get_u32(tb[BSS_TR_VALID_PERIOD]); + + da_imminent = !!(tb[BSS_TR_DA_IMMINENT] && blobmsg_get_bool(tb[BSS_TR_DA_IMMINENT])); + abridged = !!(tb[BSS_TR_ABRIDGED] && blobmsg_get_bool(tb[BSS_TR_ABRIDGED])); + + return hostapd_bss_tr_send(hapd, addr, da_imminent, abridged, da_timer, valid_period, + tb[BSS_TR_NEIGHBORS]); +} + enum { WNM_DISASSOC_ADDR, WNM_DISASSOC_DURATION, @@ -1297,14 +1420,10 @@ hostapd_wnm_disassoc_imminent(struct ubus_context *ctx, struct ubus_object *obj, { struct hostapd_data *hapd = container_of(obj, struct hostapd_data, ubus.obj); struct blob_attr *tb[__WNM_DISASSOC_MAX]; - struct blob_attr *cur; struct sta_info *sta; int duration = 10; - int rem; - int nr_len = 0; - u8 *nr = NULL; - u8 req_mode = WNM_BSS_TM_REQ_DISASSOC_IMMINENT; u8 addr[ETH_ALEN]; + bool abridged; blobmsg_parse(wnm_disassoc_policy, __WNM_DISASSOC_MAX, tb, blob_data(msg), blob_len(msg)); @@ -1314,61 +1433,13 @@ hostapd_wnm_disassoc_imminent(struct ubus_context *ctx, struct ubus_object *obj, if (hwaddr_aton(blobmsg_data(tb[WNM_DISASSOC_ADDR]), addr)) return UBUS_STATUS_INVALID_ARGUMENT; - if ((cur = tb[WNM_DISASSOC_DURATION]) != NULL) - duration = blobmsg_get_u32(cur); + if (tb[WNM_DISASSOC_DURATION]) + duration = blobmsg_get_u32(tb[WNM_DISASSOC_DURATION]); - sta = ap_get_sta(hapd, addr); - if (!sta) - return UBUS_STATUS_NOT_FOUND; + abridged = !!(tb[WNM_DISASSOC_ABRIDGED] && blobmsg_get_bool(tb[WNM_DISASSOC_ABRIDGED])); - if (tb[WNM_DISASSOC_NEIGHBORS]) { - u8 *nr_cur; - - if (blobmsg_check_array(tb[WNM_DISASSOC_NEIGHBORS], - BLOBMSG_TYPE_STRING) < 0) - return UBUS_STATUS_INVALID_ARGUMENT; - - blobmsg_for_each_attr(cur, tb[WNM_DISASSOC_NEIGHBORS], rem) { - int len = strlen(blobmsg_get_string(cur)); - - if (len % 2) - return UBUS_STATUS_INVALID_ARGUMENT; - - nr_len += (len / 2) + 2; - } - - if (nr_len) { - nr = os_zalloc(nr_len); - if (!nr) - return UBUS_STATUS_UNKNOWN_ERROR; - } - - nr_cur = nr; - blobmsg_for_each_attr(cur, tb[WNM_DISASSOC_NEIGHBORS], rem) { - int len = strlen(blobmsg_get_string(cur)) / 2; - - *nr_cur++ = WLAN_EID_NEIGHBOR_REPORT; - *nr_cur++ = (u8) len; - if (hexstr2bin(blobmsg_data(cur), nr_cur, len)) { - free(nr); - return UBUS_STATUS_INVALID_ARGUMENT; - } - - nr_cur += len; - } - } - - if (nr) - req_mode |= WNM_BSS_TM_REQ_PREF_CAND_LIST_INCLUDED; - - if (tb[WNM_DISASSOC_ABRIDGED] && blobmsg_get_bool(tb[WNM_DISASSOC_ABRIDGED])) - req_mode |= WNM_BSS_TM_REQ_ABRIDGED; - - if (wnm_send_bss_tm_req(hapd, sta, req_mode, duration, duration, NULL, - NULL, nr, nr_len, NULL, 0)) - return UBUS_STATUS_UNKNOWN_ERROR; - - return 0; + return hostapd_bss_tr_send(hapd, addr, true, abridged, duration, duration, + tb[WNM_DISASSOC_NEIGHBORS]); } #endif @@ -1454,6 +1525,7 @@ static const struct ubus_method bss_methods[] = { UBUS_METHOD("rrm_beacon_req", hostapd_rrm_beacon_req, beacon_req_policy), #ifdef CONFIG_WNM_AP UBUS_METHOD("wnm_disassoc_imminent", hostapd_wnm_disassoc_imminent, wnm_disassoc_policy), + UBUS_METHOD("bss_transition_request", hostapd_bss_transition_request, bss_tr_policy), #endif }; From f2e1e156c0b64b7fae72940c0895b30266729b1e Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Wed, 13 Oct 2021 17:49:31 +0200 Subject: [PATCH 09/22] kernel: backport a rewrite of the mips eBPF JIT implementation This adds support for eBPF JIT for 32 bit targets and significantly improves correctness. Signed-off-by: Felix Fietkau --- ...-uasm-Enable-muhu-opcode-for-MIPS-R6.patch | 65 + ...rkaround-for-Loongson-2F-nop-CPU-err.patch | 31 + ...ips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch | 3078 +++++++++++++++++ ...bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch | 1005 ++++++ ...f-Add-JIT-workarounds-for-CPU-errata.patch | 120 + ...0-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch | 61 + ...f-Remove-old-BPF-JIT-implementations.patch | 387 +++ ...CPU_MIPS64-for-remaining-MIPS64-CPUs.patch | 2 +- ...-uasm-Enable-muhu-opcode-for-MIPS-R6.patch | 65 + ...rkaround-for-Loongson-2F-nop-CPU-err.patch | 31 + ...ips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch | 3078 +++++++++++++++++ ...bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch | 1005 ++++++ ...f-Add-JIT-workarounds-for-CPU-errata.patch | 120 + ...1-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch | 54 + ...f-Remove-old-BPF-JIT-implementations.patch | 387 +++ ...-net-WireGuard-secure-network-tunnel.patch | 2 +- .../301-mips_image_cmdline_hack.patch | 2 +- .../301-mips_image_cmdline_hack.patch | 2 +- .../300-mips_expose_boot_raw.patch | 4 +- ...CPU_MIPS64-for-remaining-MIPS64-CPUs.patch | 2 +- .../300-mips_expose_boot_raw.patch | 4 +- ...122-mips-ralink-enable-zboot-support.patch | 2 +- 22 files changed, 9497 insertions(+), 10 deletions(-) create mode 100644 target/linux/generic/backport-5.10/050-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch create mode 100644 target/linux/generic/backport-5.10/050-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch create mode 100644 target/linux/generic/backport-5.10/050-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch create mode 100644 target/linux/generic/backport-5.10/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch create mode 100644 target/linux/generic/backport-5.10/050-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch create mode 100644 target/linux/generic/backport-5.10/050-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch create mode 100644 target/linux/generic/backport-5.10/050-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch create mode 100644 target/linux/generic/backport-5.4/071-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch create mode 100644 target/linux/generic/backport-5.4/071-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch create mode 100644 target/linux/generic/backport-5.4/071-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch create mode 100644 target/linux/generic/backport-5.4/071-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch create mode 100644 target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch create mode 100644 target/linux/generic/backport-5.4/071-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch create mode 100644 target/linux/generic/backport-5.4/071-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch diff --git a/target/linux/generic/backport-5.10/050-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch b/target/linux/generic/backport-5.10/050-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch new file mode 100644 index 0000000000..82feb7421d --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch @@ -0,0 +1,65 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:02 +0200 +Subject: [PATCH] MIPS: uasm: Enable muhu opcode for MIPS R6 + +Enable the 'muhu' instruction, complementing the existing 'mulu', needed +to implement a MIPS32 BPF JIT. + +Also fix a typo in the existing definition of 'dmulu'. + +Signed-off-by: Tony Ambardar + +This patch is a dependency for my 32-bit MIPS eBPF JIT. + +Signed-off-by: Johan Almbladh +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -145,6 +145,7 @@ Ip_u1(_mtlo); + Ip_u3u1u2(_mul); + Ip_u1u2(_multu); + Ip_u3u1u2(_mulu); ++Ip_u3u1u2(_muhu); + Ip_u3u1u2(_nor); + Ip_u3u1u2(_or); + Ip_u2u1u3(_ori); +--- a/arch/mips/mm/uasm-mips.c ++++ b/arch/mips/mm/uasm-mips.c +@@ -90,7 +90,7 @@ static const struct insn insn_table[insn + RS | RT | RD}, + [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, + [insn_dmultu] = {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT}, +- [insn_dmulu] = {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op), ++ [insn_dmulu] = {M(spec_op, 0, 0, 0, dmultu_dmulu_op, dmultu_op), + RS | RT | RD}, + [insn_drotr] = {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE}, + [insn_drotr32] = {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE}, +@@ -150,6 +150,8 @@ static const struct insn insn_table[insn + [insn_mtlo] = {M(spec_op, 0, 0, 0, 0, mtlo_op), RS}, + [insn_mulu] = {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op), + RS | RT | RD}, ++ [insn_muhu] = {M(spec_op, 0, 0, 0, multu_muhu_op, multu_op), ++ RS | RT | RD}, + #ifndef CONFIG_CPU_MIPSR6 + [insn_mul] = {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD}, + #else +--- a/arch/mips/mm/uasm.c ++++ b/arch/mips/mm/uasm.c +@@ -59,7 +59,7 @@ enum opcode { + insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, + insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, + insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, +- insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor, ++ insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, + insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, + insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, + insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, +@@ -344,6 +344,7 @@ I_u1(_mtlo) + I_u3u1u2(_mul) + I_u1u2(_multu) + I_u3u1u2(_mulu) ++I_u3u1u2(_muhu) + I_u3u1u2(_nor) + I_u3u1u2(_or) + I_u2u1u3(_ori) diff --git a/target/linux/generic/backport-5.10/050-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch b/target/linux/generic/backport-5.10/050-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch new file mode 100644 index 0000000000..3a4d573f80 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch @@ -0,0 +1,31 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:03 +0200 +Subject: [PATCH] mips: uasm: Add workaround for Loongson-2F nop CPU errata + +This patch implements a workaround for the Loongson-2F nop in generated, +code, if the existing option CONFIG_CPU_NOP_WORKAROUND is set. Before, +the binutils option -mfix-loongson2f-nop was enabled, but no workaround +was done when emitting MIPS code. Now, the nop pseudo instruction is +emitted as "or ax,ax,zero" instead of the default "sll zero,zero,0". This +is consistent with the workaround implemented by binutils. + +Link: https://sourceware.org/legacy-ml/binutils/2009-11/msg00387.html + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -249,7 +249,11 @@ static inline void uasm_l##lb(struct uas + #define uasm_i_bnezl(buf, rs, off) uasm_i_bnel(buf, rs, 0, off) + #define uasm_i_ehb(buf) uasm_i_sll(buf, 0, 0, 3) + #define uasm_i_move(buf, a, b) UASM_i_ADDU(buf, a, 0, b) ++#ifdef CONFIG_CPU_NOP_WORKAROUNDS ++#define uasm_i_nop(buf) uasm_i_or(buf, 1, 1, 0) ++#else + #define uasm_i_nop(buf) uasm_i_sll(buf, 0, 0, 0) ++#endif + #define uasm_i_ssnop(buf) uasm_i_sll(buf, 0, 0, 1) + + static inline void uasm_i_drotr_safe(u32 **p, unsigned int a1, diff --git a/target/linux/generic/backport-5.10/050-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch b/target/linux/generic/backport-5.10/050-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch new file mode 100644 index 0000000000..7980659961 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch @@ -0,0 +1,3078 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:04 +0200 +Subject: [PATCH] mips: bpf: Add eBPF JIT for 32-bit MIPS + +This is an implementation of an eBPF JIT for 32-bit MIPS I-V and MIPS32. +The implementation supports all 32-bit and 64-bit ALU and JMP operations, +including the recently-added atomics. 64-bit div/mod and 64-bit atomics +are implemented using function calls to math64 and atomic64 functions, +respectively. All 32-bit operations are implemented natively by the JIT, +except if the CPU lacks ll/sc instructions. + +Register mapping +================ +All 64-bit eBPF registers are mapped to native 32-bit MIPS register pairs, +and does not use any stack scratch space for register swapping. This means +that all eBPF register data is kept in CPU registers all the time, and +this simplifies the register management a lot. It also reduces the JIT's +pressure on temporary registers since we do not have to move data around. + +Native register pairs are ordered according to CPU endiannes, following +the O32 calling convention for passing 64-bit arguments and return values. +The eBPF return value, arguments and callee-saved registers are mapped to +their native MIPS equivalents. + +Since the 32 highest bits in the eBPF FP (frame pointer) register are +always zero, only one general-purpose register is actually needed for the +mapping. The MIPS fp register is used for this purpose. The high bits are +mapped to MIPS register r0. This saves us one CPU register, which is much +needed for temporaries, while still allowing us to treat the R10 (FP) +register just like any other eBPF register in the JIT. + +The MIPS gp (global pointer) and at (assembler temporary) registers are +used as internal temporary registers for constant blinding. CPU registers +t6-t9 are used internally by the JIT when constructing more complex 64-bit +operations. This is precisely what is needed - two registers to store an +operand value, and two more as scratch registers when performing the +operation. + +The register mapping is shown below. + + R0 - $v1, $v0 return value + R1 - $a1, $a0 argument 1, passed in registers + R2 - $a3, $a2 argument 2, passed in registers + R3 - $t1, $t0 argument 3, passed on stack + R4 - $t3, $t2 argument 4, passed on stack + R5 - $t4, $t3 argument 5, passed on stack + R6 - $s1, $s0 callee-saved + R7 - $s3, $s2 callee-saved + R8 - $s5, $s4 callee-saved + R9 - $s7, $s6 callee-saved + FP - $r0, $fp 32-bit frame pointer + AX - $gp, $at constant-blinding + $t6 - $t9 unallocated, JIT temporaries + +Jump offsets +============ +The JIT tries to map all conditional JMP operations to MIPS conditional +PC-relative branches. The MIPS branch offset field is 18 bits, in bytes, +which is equivalent to the eBPF 16-bit instruction offset. However, since +the JIT may emit more than one CPU instruction per eBPF instruction, the +field width may overflow. If that happens, the JIT converts the long +conditional jump to a short PC-relative branch with the condition +inverted, jumping over a long unconditional absolute jmp (j). + +This conversion will change the instruction offset mapping used for jumps, +and may in turn result in more branch offset overflows. The JIT therefore +dry-runs the translation until no more branches are converted and the +offsets do not change anymore. There is an upper bound on this of course, +and if the JIT hits that limit, the last two iterations are run with all +branches being converted. + +Tail call count +=============== +The current tail call count is stored in the 16-byte area of the caller's +stack frame that is reserved for the callee in the o32 ABI. The value is +initialized in the prologue, and propagated to the tail-callee by skipping +the initialization instructions when emitting the tail call. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp.c + create mode 100644 arch/mips/net/bpf_jit_comp.h + create mode 100644 arch/mips/net/bpf_jit_comp32.c + +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,4 +2,9 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o +-obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ ++ifeq ($(CONFIG_32BIT),y) ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++else ++ obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++endif +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -0,0 +1,1020 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions common to 32-bit and 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++/* ++ * Code overview ++ * ============= ++ * ++ * - bpf_jit_comp.h ++ * Common definitions and utilities. ++ * ++ * - bpf_jit_comp.c ++ * Implementation of JIT top-level logic and exported JIT API functions. ++ * Implementation of internal operations shared by 32-bit and 64-bit code. ++ * JMP and ALU JIT control code, register control code, shared ALU and ++ * JMP/JMP32 JIT operations. ++ * ++ * - bpf_jit_comp32.c ++ * Implementation of functions to JIT prologue, epilogue and a single eBPF ++ * instruction for 32-bit MIPS CPUs. The functions use shared operations ++ * where possible, and implement the rest for 32-bit MIPS such as ALU64 ++ * operations. ++ * ++ * - bpf_jit_comp64.c ++ * Ditto, for 64-bit MIPS CPUs. ++ * ++ * Zero and sign extension ++ * ======================== ++ * 32-bit MIPS instructions on 64-bit MIPS registers use sign extension, ++ * but the eBPF instruction set mandates zero extension. We let the verifier ++ * insert explicit zero-extensions after 32-bit ALU operations, both for ++ * 32-bit and 64-bit MIPS JITs. Conditional JMP32 operations on 64-bit MIPs ++ * are JITed with sign extensions inserted when so expected. ++ * ++ * ALU operations ++ * ============== ++ * ALU operations on 32/64-bit MIPS and ALU64 operations on 64-bit MIPS are ++ * JITed in the following steps. ALU64 operations on 32-bit MIPS are more ++ * complicated and therefore only processed by special implementations in ++ * step (3). ++ * ++ * 1) valid_alu_i: ++ * Determine if an immediate operation can be emitted as such, or if ++ * we must fall back to the register version. ++ * ++ * 2) rewrite_alu_i: ++ * Convert BPF operation and immediate value to a canonical form for ++ * JITing. In some degenerate cases this form may be a no-op. ++ * ++ * 3) emit_alu_{i,i64,r,64}: ++ * Emit instructions for an ALU or ALU64 immediate or register operation. ++ * ++ * JMP operations ++ * ============== ++ * JMP and JMP32 operations require an JIT instruction offset table for ++ * translating the jump offset. This table is computed by dry-running the ++ * JIT without actually emitting anything. However, the computed PC-relative ++ * offset may overflow the 18-bit offset field width of the native MIPS ++ * branch instruction. In such cases, the long jump is converted into the ++ * following sequence. ++ * ++ * ! +2 Inverted PC-relative branch ++ * nop Delay slot ++ * j Unconditional absolute long jump ++ * nop Delay slot ++ * ++ * Since this converted sequence alters the offset table, all offsets must ++ * be re-calculated. This may in turn trigger new branch conversions, so ++ * the process is repeated until no further changes are made. Normally it ++ * completes in 1-2 iterations. If JIT_MAX_ITERATIONS should reached, we ++ * fall back to converting every remaining jump operation. The branch ++ * conversion is independent of how the JMP or JMP32 condition is JITed. ++ * ++ * JMP32 and JMP operations are JITed as follows. ++ * ++ * 1) setup_jmp_{i,r}: ++ * Convert jump conditional and offset into a form that can be JITed. ++ * This form may be a no-op, a canonical form, or an inverted PC-relative ++ * jump if branch conversion is necessary. ++ * ++ * 2) valid_jmp_i: ++ * Determine if an immediate operations can be emitted as such, or if ++ * we must fall back to the register version. Applies to JMP32 for 32-bit ++ * MIPS, and both JMP and JMP32 for 64-bit MIPS. ++ * ++ * 3) emit_jmp_{i,i64,r,r64}: ++ * Emit instructions for an JMP or JMP32 immediate or register operation. ++ * ++ * 4) finish_jmp_{i,r}: ++ * Emit any instructions needed to finish the jump. This includes a nop ++ * for the delay slot if a branch was emitted, and a long absolute jump ++ * if the branch was converted. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* Convenience macros for descriptor access */ ++#define CONVERTED(desc) ((desc) & JIT_DESC_CONVERT) ++#define INDEX(desc) ((desc) & ~JIT_DESC_CONVERT) ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, sw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, sd, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ ctx->stack_used = max((int)ctx->stack_used, depth); ++ return depth; ++} ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, lw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, ld, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ return depth; ++} ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc) ++{ ++ u32 index = INDEX(ctx->descriptors[loc]); ++ unsigned long pc = (unsigned long)&ctx->target[ctx->jit_index]; ++ unsigned long addr = (unsigned long)&ctx->target[index]; ++ ++ if (!ctx->target) ++ return 0; ++ ++ if ((addr ^ pc) & ~MIPS_JMP_MASK) ++ return -1; ++ ++ return addr & MIPS_JMP_MASK; ++} ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off) ++{ ++ return (INDEX(ctx->descriptors[ctx->bpf_index + off]) - ++ ctx->jit_index - 1) * sizeof(u32); ++} ++ ++/* dst = imm (register width) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm) ++{ ++ if (imm >= -0x8000 && imm <= 0x7fff) { ++ emit(ctx, addiu, dst, MIPS_R_ZERO, imm); ++ } else { ++ emit(ctx, lui, dst, (s16)((u32)imm >> 16)); ++ emit(ctx, ori, dst, dst, (u16)(imm & 0xffff)); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* dst = src (register width) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, ori, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate ALU immediate range */ ++bool valid_alu_i(u8 op, s32 imm) ++{ ++ switch (BPF_OP(op)) { ++ case BPF_NEG: ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ /* All legal eBPF values are valid */ ++ return true; ++ case BPF_ADD: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_SUB: ++ /* -imm must be 16 bits */ ++ return imm >= -0x7fff && imm <= 0x8000; ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_MUL: ++ /* imm must be zero or a positive power of two */ ++ return imm == 0 || (imm > 0 && is_power_of_2(imm)); ++ case BPF_DIV: ++ case BPF_MOD: ++ /* imm must be an 17-bit power of two */ ++ return (u32)imm <= 0x10000 && is_power_of_2((u32)imm); ++ } ++ return false; ++} ++ ++/* Rewrite ALU immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val) ++{ ++ bool act = true; ++ ++ switch (BPF_OP(op)) { ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ case BPF_ADD: ++ case BPF_SUB: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm == 0 is a no-op */ ++ act = imm != 0; ++ break; ++ case BPF_MUL: ++ if (imm == 1) { ++ /* dst * 1 is a no-op */ ++ act = false; ++ } else if (imm == 0) { ++ /* dst * 0 is dst & 0 */ ++ op = BPF_AND; ++ } else { ++ /* dst * (1 << n) is dst << n */ ++ op = BPF_LSH; ++ imm = ilog2(abs(imm)); ++ } ++ break; ++ case BPF_DIV: ++ if (imm == 1) { ++ /* dst / 1 is a no-op */ ++ act = false; ++ } else { ++ /* dst / (1 << n) is dst >> n */ ++ op = BPF_RSH; ++ imm = ilog2(imm); ++ } ++ break; ++ case BPF_MOD: ++ /* dst % (1 << n) is dst & ((1 << n) - 1) */ ++ op = BPF_AND; ++ imm--; ++ break; ++ } ++ ++ *alu = op; ++ *val = imm; ++ return act; ++} ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, subu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, andi, dst, dst, (u16)imm); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, sll, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, srl, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, sra, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, addiu, dst, dst, -imm); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, dst, dst, src); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, dst, dst, src); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, dst, dst, src); ++ break; ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, sllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, srlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, srav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, addu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, subu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, dst, dst, src); ++ } else { ++ emit(ctx, multu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips32r6) { ++ emit(ctx, divu_r6, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips32r6) { ++ emit(ctx, modu, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, addu, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ } ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ emit(ctx, bne, MIPS_R_T9, res, 12); ++ emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ ++ clobber_reg(ctx, res); ++} ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ u8 msk = MIPS_R_T9; ++ ++ switch (width) { ++ /* Swap bytes in a word */ ++ case 32: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, rotr, dst, dst, 16); ++ } else { ++ emit(ctx, sll, tmp, dst, 16); /* tmp = dst << 16 */ ++ emit(ctx, srl, dst, dst, 16); /* dst = dst >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++ ++ emit(ctx, lui, msk, 0xff); /* msk = 0x00ff0000 */ ++ emit(ctx, ori, msk, msk, 0xff); /* msk = msk | 0xff */ ++ ++ emit(ctx, and, tmp, dst, msk); /* tmp = dst & msk */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, dst, 8); /* dst = dst >> 8 */ ++ emit(ctx, and, dst, dst, msk); /* dst = dst & msk */ ++ emit(ctx, or, dst, dst, tmp); /* reg = dst | tmp */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ case 16: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, andi, dst, dst, 0xffff); ++ } else { ++ emit(ctx, andi, tmp, dst, 0xff00); /* t = d & 0xff00 */ ++ emit(ctx, srl, tmp, tmp, 8); /* t = t >> 8 */ ++ emit(ctx, andi, dst, dst, 0x00ff); /* d = d & 0x00ff */ ++ emit(ctx, sll, dst, dst, 8); /* d = d << 8 */ ++ emit(ctx, or, dst, dst, tmp); /* d = d | t */ ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate jump immediate range */ ++bool valid_jmp_i(u8 op, s32 imm) ++{ ++ switch (op) { ++ case JIT_JNOP: ++ /* Immediate value not used */ ++ return true; ++ case BPF_JEQ: ++ case BPF_JNE: ++ /* No immediate operation */ ++ return false; ++ case BPF_JSET: ++ case JIT_JNSET: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_JGE: ++ case BPF_JLT: ++ case BPF_JSGE: ++ case BPF_JSLT: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_JGT: ++ case BPF_JLE: ++ case BPF_JSGT: ++ case BPF_JSLE: ++ /* imm + 1 must be 16 bits */ ++ return imm >= -0x8001 && imm <= 0x7ffe; ++ } ++ return false; ++} ++ ++/* Invert a conditional jump operation */ ++static u8 invert_jmp(u8 op) ++{ ++ switch (op) { ++ case BPF_JA: return JIT_JNOP; ++ case BPF_JEQ: return BPF_JNE; ++ case BPF_JNE: return BPF_JEQ; ++ case BPF_JSET: return JIT_JNSET; ++ case BPF_JGT: return BPF_JLE; ++ case BPF_JGE: return BPF_JLT; ++ case BPF_JLT: return BPF_JGE; ++ case BPF_JLE: return BPF_JGT; ++ case BPF_JSGT: return BPF_JSLE; ++ case BPF_JSGE: return BPF_JSLT; ++ case BPF_JSLT: return BPF_JSGE; ++ case BPF_JSLE: return BPF_JSGT; ++ } ++ return 0; ++} ++ ++/* Prepare a PC-relative jump operation */ ++static void setup_jmp(struct jit_context *ctx, u8 bpf_op, ++ s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ u32 *descp = &ctx->descriptors[ctx->bpf_index]; ++ int op = bpf_op; ++ int offset = 0; ++ ++ /* Do not compute offsets on the first pass */ ++ if (INDEX(*descp) == 0) ++ goto done; ++ ++ /* Skip jumps never taken */ ++ if (bpf_op == JIT_JNOP) ++ goto done; ++ ++ /* Convert jumps always taken */ ++ if (bpf_op == BPF_JA) ++ *descp |= JIT_DESC_CONVERT; ++ ++ /* ++ * Current ctx->jit_index points to the start of the branch preamble. ++ * Since the preamble differs among different branch conditionals, ++ * the current index cannot be used to compute the branch offset. ++ * Instead, we use the offset table value for the next instruction, ++ * which gives the index immediately after the branch delay slot. ++ */ ++ if (!CONVERTED(*descp)) { ++ int target = ctx->bpf_index + bpf_off + 1; ++ int origin = ctx->bpf_index + 1; ++ ++ offset = (INDEX(ctx->descriptors[target]) - ++ INDEX(ctx->descriptors[origin]) + 1) * sizeof(u32); ++ } ++ ++ /* ++ * The PC-relative branch offset field on MIPS is 18 bits signed, ++ * so if the computed offset is larger than this we generate a an ++ * absolute jump that we skip with an inverted conditional branch. ++ */ ++ if (CONVERTED(*descp) || offset < -0x20000 || offset > 0x1ffff) { ++ offset = 3 * sizeof(u32); ++ op = invert_jmp(bpf_op); ++ ctx->changes += !CONVERTED(*descp); ++ *descp |= JIT_DESC_CONVERT; ++ } ++ ++done: ++ *jit_off = offset; ++ *jit_op = op; ++} ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ bool always = false; ++ bool never = false; ++ ++ switch (bpf_op) { ++ case BPF_JEQ: ++ case BPF_JNE: ++ break; ++ case BPF_JSET: ++ case BPF_JLT: ++ never = imm == 0; ++ break; ++ case BPF_JGE: ++ always = imm == 0; ++ break; ++ case BPF_JGT: ++ never = (u32)imm == U32_MAX; ++ break; ++ case BPF_JLE: ++ always = (u32)imm == U32_MAX; ++ break; ++ case BPF_JSGT: ++ never = imm == S32_MAX && width == 32; ++ break; ++ case BPF_JSGE: ++ always = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLT: ++ never = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLE: ++ always = imm == S32_MAX && width == 32; ++ break; ++ } ++ ++ if (never) ++ bpf_op = JIT_JNOP; ++ if (always) ++ bpf_op = BPF_JA; ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ switch (bpf_op) { ++ case BPF_JSET: ++ break; ++ case BPF_JEQ: ++ case BPF_JGE: ++ case BPF_JLE: ++ case BPF_JSGE: ++ case BPF_JSLE: ++ if (same_reg) ++ bpf_op = BPF_JA; ++ break; ++ case BPF_JNE: ++ case BPF_JLT: ++ case BPF_JGT: ++ case BPF_JSGT: ++ case BPF_JSLT: ++ if (same_reg) ++ bpf_op = JIT_JNOP; ++ break; ++ } ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off) ++{ ++ /* Emit conditional branch delay slot */ ++ if (jit_op != JIT_JNOP) ++ emit(ctx, nop); ++ /* ++ * Emit an absolute long jump with delay slot, ++ * if the PC-relative branch was converted. ++ */ ++ if (CONVERTED(ctx->descriptors[ctx->bpf_index])) { ++ int target = get_target(ctx, ctx->bpf_index + bpf_off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ } ++ return 0; ++} ++ ++/* Jump immediate (32-bit) */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst & imm */ ++ case BPF_JSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump register (32-bit) */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ case BPF_JEQ: ++ emit(ctx, beq, dst, src, off); ++ break; ++ /* PC += off if dst != src */ ++ case BPF_JNE: ++ emit(ctx, bne, dst, src, off); ++ break; ++ /* PC += off if dst & src */ ++ case BPF_JSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off) ++{ ++ int target = get_target(ctx, ctx->bpf_index + off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx) ++{ ++ int target = get_target(ctx, ctx->program->len); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Build the program body from eBPF bytecode */ ++static int build_body(struct jit_context *ctx) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ unsigned int i; ++ ++ ctx->stack_used = 0; ++ for (i = 0; i < prog->len; i++) { ++ const struct bpf_insn *insn = &prog->insnsi[i]; ++ u32 *descp = &ctx->descriptors[i]; ++ int ret; ++ ++ access_reg(ctx, insn->src_reg); ++ access_reg(ctx, insn->dst_reg); ++ ++ ctx->bpf_index = i; ++ if (ctx->target == NULL) { ++ ctx->changes += INDEX(*descp) != ctx->jit_index; ++ *descp &= JIT_DESC_CONVERT; ++ *descp |= ctx->jit_index; ++ } ++ ++ ret = build_insn(insn, ctx); ++ if (ret < 0) ++ return ret; ++ ++ if (ret > 0) { ++ i++; ++ if (ctx->target == NULL) ++ descp[1] = ctx->jit_index; ++ } ++ } ++ ++ /* Store the end offset, where the epilogue begins */ ++ ctx->descriptors[prog->len] = ctx->jit_index; ++ return 0; ++} ++ ++/* Set the branch conversion flag on all instructions */ ++static void set_convert_flag(struct jit_context *ctx, bool enable) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ u32 flag = enable ? JIT_DESC_CONVERT : 0; ++ unsigned int i; ++ ++ for (i = 0; i <= prog->len; i++) ++ ctx->descriptors[i] = INDEX(ctx->descriptors[i]) | flag; ++} ++ ++static void jit_fill_hole(void *area, unsigned int size) ++{ ++ u32 *p; ++ ++ /* We are guaranteed to have aligned memory. */ ++ for (p = area; size >= sizeof(u32); size -= sizeof(u32)) ++ uasm_i_break(&p, BRK_BUG); /* Increments p */ ++} ++ ++bool bpf_jit_needs_zext(void) ++{ ++ return true; ++} ++ ++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ++{ ++ struct bpf_prog *tmp, *orig_prog = prog; ++ struct bpf_binary_header *header = NULL; ++ struct jit_context ctx; ++ bool tmp_blinded = false; ++ unsigned int tmp_idx; ++ unsigned int image_size; ++ u8 *image_ptr; ++ int tries; ++ ++ /* ++ * If BPF JIT was not enabled then we must fall back to ++ * the interpreter. ++ */ ++ if (!prog->jit_requested) ++ return orig_prog; ++ /* ++ * If constant blinding was enabled and we failed during blinding ++ * then we must fall back to the interpreter. Otherwise, we save ++ * the new JITed code. ++ */ ++ tmp = bpf_jit_blind_constants(prog); ++ if (IS_ERR(tmp)) ++ return orig_prog; ++ if (tmp != prog) { ++ tmp_blinded = true; ++ prog = tmp; ++ } ++ ++ memset(&ctx, 0, sizeof(ctx)); ++ ctx.program = prog; ++ ++ /* ++ * Not able to allocate memory for descriptors[], then ++ * we must fall back to the interpreter ++ */ ++ ctx.descriptors = kcalloc(prog->len + 1, sizeof(*ctx.descriptors), ++ GFP_KERNEL); ++ if (ctx.descriptors == NULL) ++ goto out_err; ++ ++ /* First pass discovers used resources */ ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ /* ++ * Second pass computes instruction offsets. ++ * If any PC-relative branches are out of range, a sequence of ++ * a PC-relative branch + a jump is generated, and we have to ++ * try again from the beginning to generate the new offsets. ++ * This is done until no additional conversions are necessary. ++ * The last two iterations are done with all branches being ++ * converted, to guarantee offset table convergence within a ++ * fixed number of iterations. ++ */ ++ ctx.jit_index = 0; ++ build_prologue(&ctx); ++ tmp_idx = ctx.jit_index; ++ ++ tries = JIT_MAX_ITERATIONS; ++ do { ++ ctx.jit_index = tmp_idx; ++ ctx.changes = 0; ++ if (tries == 2) ++ set_convert_flag(&ctx, true); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ } while (ctx.changes > 0 && --tries > 0); ++ ++ if (WARN_ONCE(ctx.changes > 0, "JIT offsets failed to converge")) ++ goto out_err; ++ ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Now we know the size of the structure to make */ ++ image_size = sizeof(u32) * ctx.jit_index; ++ header = bpf_jit_binary_alloc(image_size, &image_ptr, ++ sizeof(u32), jit_fill_hole); ++ /* ++ * Not able to allocate memory for the structure then ++ * we must fall back to the interpretation ++ */ ++ if (header == NULL) ++ goto out_err; ++ ++ /* Actual pass to generate final JIT code */ ++ ctx.target = (u32 *)image_ptr; ++ ctx.jit_index = 0; ++ ++ /* ++ * If building the JITed code fails somehow, ++ * we fall back to the interpretation. ++ */ ++ build_prologue(&ctx); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Populate line info meta data */ ++ set_convert_flag(&ctx, false); ++ bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]); ++ ++ /* Set as read-only exec and flush instruction cache */ ++ bpf_jit_binary_lock_ro(header); ++ flush_icache_range((unsigned long)header, ++ (unsigned long)&ctx.target[ctx.jit_index]); ++ ++ if (bpf_jit_enable > 1) ++ bpf_jit_dump(prog->len, image_size, 2, ctx.target); ++ ++ prog->bpf_func = (void *)ctx.target; ++ prog->jited = 1; ++ prog->jited_len = image_size; ++ ++out: ++ if (tmp_blinded) ++ bpf_jit_prog_release_other(prog, prog == orig_prog ? ++ tmp : orig_prog); ++ kfree(ctx.descriptors); ++ return prog; ++ ++out_err: ++ prog = orig_prog; ++ if (header) ++ bpf_jit_binary_free(header); ++ goto out; ++} +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -0,0 +1,211 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#ifndef _BPF_JIT_COMP_H ++#define _BPF_JIT_COMP_H ++ ++/* MIPS registers */ ++#define MIPS_R_ZERO 0 /* Const zero */ ++#define MIPS_R_AT 1 /* Asm temp */ ++#define MIPS_R_V0 2 /* Result */ ++#define MIPS_R_V1 3 /* Result */ ++#define MIPS_R_A0 4 /* Argument */ ++#define MIPS_R_A1 5 /* Argument */ ++#define MIPS_R_A2 6 /* Argument */ ++#define MIPS_R_A3 7 /* Argument */ ++#define MIPS_R_A4 8 /* Arg (n64) */ ++#define MIPS_R_A5 9 /* Arg (n64) */ ++#define MIPS_R_A6 10 /* Arg (n64) */ ++#define MIPS_R_A7 11 /* Arg (n64) */ ++#define MIPS_R_T0 8 /* Temp (o32) */ ++#define MIPS_R_T1 9 /* Temp (o32) */ ++#define MIPS_R_T2 10 /* Temp (o32) */ ++#define MIPS_R_T3 11 /* Temp (o32) */ ++#define MIPS_R_T4 12 /* Temporary */ ++#define MIPS_R_T5 13 /* Temporary */ ++#define MIPS_R_T6 14 /* Temporary */ ++#define MIPS_R_T7 15 /* Temporary */ ++#define MIPS_R_S0 16 /* Saved */ ++#define MIPS_R_S1 17 /* Saved */ ++#define MIPS_R_S2 18 /* Saved */ ++#define MIPS_R_S3 19 /* Saved */ ++#define MIPS_R_S4 20 /* Saved */ ++#define MIPS_R_S5 21 /* Saved */ ++#define MIPS_R_S6 22 /* Saved */ ++#define MIPS_R_S7 23 /* Saved */ ++#define MIPS_R_T8 24 /* Temporary */ ++#define MIPS_R_T9 25 /* Temporary */ ++/* MIPS_R_K0 26 Reserved */ ++/* MIPS_R_K1 27 Reserved */ ++#define MIPS_R_GP 28 /* Global ptr */ ++#define MIPS_R_SP 29 /* Stack ptr */ ++#define MIPS_R_FP 30 /* Frame ptr */ ++#define MIPS_R_RA 31 /* Return */ ++ ++/* ++ * Jump address mask for immediate jumps. The four most significant bits ++ * must be equal to PC. ++ */ ++#define MIPS_JMP_MASK 0x0fffffffUL ++ ++/* Maximum number of iterations in offset table computation */ ++#define JIT_MAX_ITERATIONS 8 ++ ++/* ++ * Jump pseudo-instructions used internally ++ * for branch conversion and branch optimization. ++ */ ++#define JIT_JNSET 0xe0 ++#define JIT_JNOP 0xf0 ++ ++/* Descriptor flag for PC-relative branch conversion */ ++#define JIT_DESC_CONVERT BIT(31) ++ ++/* JIT context for an eBPF program */ ++struct jit_context { ++ struct bpf_prog *program; /* The eBPF program being JITed */ ++ u32 *descriptors; /* eBPF to JITed CPU insn descriptors */ ++ u32 *target; /* JITed code buffer */ ++ u32 bpf_index; /* Index of current BPF program insn */ ++ u32 jit_index; /* Index of current JIT target insn */ ++ u32 changes; /* Number of PC-relative branch conv */ ++ u32 accessed; /* Bit mask of read eBPF registers */ ++ u32 clobbered; /* Bit mask of modified CPU registers */ ++ u32 stack_size; /* Total allocated stack size in bytes */ ++ u32 saved_size; /* Size of callee-saved registers */ ++ u32 stack_used; /* Stack size used for function calls */ ++}; ++ ++/* Emit the instruction if the JIT memory space has been allocated */ ++#define emit(ctx, func, ...) \ ++do { \ ++ if ((ctx)->target != NULL) { \ ++ u32 *p = &(ctx)->target[ctx->jit_index]; \ ++ uasm_i_##func(&p, ##__VA_ARGS__); \ ++ } \ ++ (ctx)->jit_index++; \ ++} while (0) ++ ++/* ++ * Mark a BPF register as accessed, it needs to be ++ * initialized by the program if expected, e.g. FP. ++ */ ++static inline void access_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->accessed |= BIT(reg); ++} ++ ++/* ++ * Mark a CPU register as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static inline void clobber_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->clobbered |= BIT(reg); ++} ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc); ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off); ++ ++/* dst = imm (32-bit) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); ++ ++/* dst = src (32-bit) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); ++ ++/* Validate ALU/ALU64 immediate range */ ++bool valid_alu_i(u8 op, s32 imm); ++ ++/* Rewrite ALU/ALU64 immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off); ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width); ++ ++/* Validate JMP/JMP32 immediate range */ ++bool valid_jmp_i(u8 op, s32 imm); ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off); ++ ++/* Conditional JMP/JMP32 immediate */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); ++ ++/* Conditional JMP/JMP32 register */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off); ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx); ++ ++/* ++ * Build program prologue to set up the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_prologue(struct jit_context *ctx); ++ ++/* ++ * Build the program epilogue to restore the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_epilogue(struct jit_context *ctx, int dest_reg); ++ ++/* ++ * Convert an eBPF instruction to native instruction, i.e ++ * JITs an eBPF instruction. ++ * Returns : ++ * 0 - Successfully JITed an 8-byte eBPF instruction ++ * >0 - Successfully JITed a 16-byte eBPF instruction ++ * <0 - Failed to JIT. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx); ++ ++#endif /* _BPF_JIT_COMP_H */ +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp32.c +@@ -0,0 +1,1741 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 32-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS a4-a7 are not available in the o32 ABI */ ++#undef MIPS_R_A4 ++#undef MIPS_R_A5 ++#undef MIPS_R_A6 ++#undef MIPS_R_A7 ++ ++/* Stack is 8-byte aligned in o32 ABI */ ++#define MIPS_STACK_ALIGNMENT 8 ++ ++/* ++ * The top 16 bytes of a stack frame is reserved for the callee in O32 ABI. ++ * This corresponds to stack space for register arguments a0-a3. ++ */ ++#define JIT_RESERVED_STACK 16 ++ ++/* Temporary 64-bit register used by JIT */ ++#define JIT_REG_TMP MAX_BPF_JIT_REG ++ ++/* ++ * Number of prologue bytes to skip when doing a tail call. ++ * Tail call count (TCC) initialization (8 bytes) always, plus ++ * R0-to-v0 assignment (4 bytes) if big endian. ++ */ ++#ifdef __BIG_ENDIAN ++#define JIT_TCALL_SKIP 12 ++#else ++#define JIT_TCALL_SKIP 8 ++#endif ++ ++/* CPU registers holding the callee return value */ ++#define JIT_RETURN_REGS \ ++ (BIT(MIPS_R_V0) | \ ++ BIT(MIPS_R_V1)) ++ ++/* CPU registers arguments passed to callee directly */ ++#define JIT_ARG_REGS \ ++ (BIT(MIPS_R_A0) | \ ++ BIT(MIPS_R_A1) | \ ++ BIT(MIPS_R_A2) | \ ++ BIT(MIPS_R_A3)) ++ ++/* CPU register arguments passed to callee on stack */ ++#define JIT_STACK_REGS \ ++ (BIT(MIPS_R_T0) | \ ++ BIT(MIPS_R_T1) | \ ++ BIT(MIPS_R_T2) | \ ++ BIT(MIPS_R_T3) | \ ++ BIT(MIPS_R_T4) | \ ++ BIT(MIPS_R_T5)) ++ ++/* Caller-saved CPU registers */ ++#define JIT_CALLER_REGS \ ++ (JIT_RETURN_REGS | \ ++ JIT_ARG_REGS | \ ++ JIT_STACK_REGS) ++ ++/* Callee-saved CPU registers */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* ++ * Mapping of 64-bit eBPF registers to 32-bit native MIPS registers. ++ * ++ * 1) Native register pairs are ordered according to CPU endiannes, following ++ * the MIPS convention for passing 64-bit arguments and return values. ++ * 2) The eBPF return value, arguments and callee-saved registers are mapped ++ * to their native MIPS equivalents. ++ * 3) Since the 32 highest bits in the eBPF FP register are always zero, ++ * only one general-purpose register is actually needed for the mapping. ++ * We use the fp register for this purpose, and map the highest bits to ++ * the MIPS register r0 (zero). ++ * 4) We use the MIPS gp and at registers as internal temporary registers ++ * for constant blinding. The gp register is callee-saved. ++ * 5) One 64-bit temporary register is mapped for use when sign-extending ++ * immediate operands. MIPS registers t6-t9 are available to the JIT ++ * for as temporaries when implementing complex 64-bit operations. ++ * ++ * With this scheme all eBPF registers are being mapped to native MIPS ++ * registers without having to use any stack scratch space. The direct ++ * register mapping (2) simplifies the handling of function calls. ++ */ ++static const u8 bpf2mips32[][2] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = {MIPS_R_V1, MIPS_R_V0}, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = {MIPS_R_A1, MIPS_R_A0}, ++ [BPF_REG_2] = {MIPS_R_A3, MIPS_R_A2}, ++ /* Remaining arguments, to be passed on the stack per O32 ABI */ ++ [BPF_REG_3] = {MIPS_R_T1, MIPS_R_T0}, ++ [BPF_REG_4] = {MIPS_R_T3, MIPS_R_T2}, ++ [BPF_REG_5] = {MIPS_R_T5, MIPS_R_T4}, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = {MIPS_R_S1, MIPS_R_S0}, ++ [BPF_REG_7] = {MIPS_R_S3, MIPS_R_S2}, ++ [BPF_REG_8] = {MIPS_R_S5, MIPS_R_S4}, ++ [BPF_REG_9] = {MIPS_R_S7, MIPS_R_S6}, ++ /* Read-only frame pointer to access the eBPF stack */ ++#ifdef __BIG_ENDIAN ++ [BPF_REG_FP] = {MIPS_R_FP, MIPS_R_ZERO}, ++#else ++ [BPF_REG_FP] = {MIPS_R_ZERO, MIPS_R_FP}, ++#endif ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = {MIPS_R_GP, MIPS_R_AT}, ++ /* Temporary register for internal JIT use */ ++ [JIT_REG_TMP] = {MIPS_R_T7, MIPS_R_T6}, ++}; ++ ++/* Get low CPU register for a 64-bit eBPF register mapping */ ++static inline u8 lo(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[0]; ++#else ++ return reg[1]; ++#endif ++} ++ ++/* Get high CPU register for a 64-bit eBPF register mapping */ ++static inline u8 hi(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[1]; ++#else ++ return reg[0]; ++#endif ++} ++ ++/* ++ * Mark a 64-bit CPU register pair as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static void clobber_reg64(struct jit_context *ctx, const u8 reg[]) ++{ ++ clobber_reg(ctx, reg[0]); ++ clobber_reg(ctx, reg[1]); ++} ++ ++/* dst = imm (sign-extended) */ ++static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ emit_mov_i(ctx, lo(dst), imm); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ else ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) ++{ ++ if (!ctx->program->aux->verifier_zext) { ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ } ++} ++ ++/* Load delay slot, if ISA mandates it */ ++static void emit_load_delay(struct jit_context *ctx) ++{ ++ if (!cpu_has_mips_2_3_4_5_r) ++ emit(ctx, nop); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, ++ const u8 dst[], s32 imm, u8 op) ++{ ++ u8 src = MIPS_R_T6; ++ ++ /* ++ * ADD/SUB with all but the max negative imm can be handled by ++ * inverting the operation and the imm value, saving one insn. ++ */ ++ if (imm > S32_MIN && imm < 0) ++ switch (op) { ++ case BPF_ADD: ++ op = BPF_SUB; ++ imm = -imm; ++ break; ++ case BPF_SUB: ++ op = BPF_ADD; ++ imm = -imm; ++ break; ++ } ++ ++ /* Move immediate to temporary register */ ++ emit_mov_i(ctx, src, imm); ++ ++ switch (op) { ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addu, lo(dst), lo(dst), src); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, subu, lo(dst), lo(dst), src); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), 1); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), src); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), src); ++ if (imm >= 0) ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), src); ++ if (imm < 0) { ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst + src */ ++ case BPF_ADD: ++ if (src == dst) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 31); ++ emit(ctx, addu, lo(dst), lo(dst), lo(dst)); ++ } else { ++ emit(ctx, addu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ } ++ emit(ctx, addu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ emit(ctx, subu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, subu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), lo(src)); ++ emit(ctx, or, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), lo(src)); ++ emit(ctx, and, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), lo(src)); ++ emit(ctx, xor, hi(dst), hi(dst), hi(src)); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU invert (64-bit) */ ++static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) ++{ ++ emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift immediate (64-bit) */ ++static void emit_shift_i64(struct jit_context *ctx, ++ const u8 dst[], u32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ if (imm < 32) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); ++ emit(ctx, sll, lo(dst), lo(dst), imm); ++ emit(ctx, sll, hi(dst), hi(dst), imm); ++ emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sll, hi(dst), lo(dst), imm - 32); ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, srl, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, srl, lo(dst), hi(dst), imm - 32); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, sra, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sra, lo(dst), hi(dst), imm - 32); ++ emit(ctx, sra, hi(dst), hi(dst), 31); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift register (64-bit) */ ++static void emit_shift_r64(struct jit_context *ctx, ++ const u8 dst[], u8 src, u8 op) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ ++ emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */ ++ emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */ ++ emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */ ++ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ ++ emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */ ++ emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ ++ emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ ++ emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ ++ emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ ++ emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ ++ emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ } ++ ++ /* +20: Done */ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU mul immediate (64x32-bit) */ ++static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ u8 src = MIPS_R_T6; ++ u8 tmp = MIPS_R_T9; ++ ++ switch (imm) { ++ /* dst = dst * 1 is a no-op */ ++ case 1: ++ break; ++ /* dst = dst * -1 */ ++ case -1: ++ emit_neg_i64(ctx, dst); ++ break; ++ case 0: ++ emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); ++ emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Full 64x32 multiply */ ++ default: ++ /* hi(dst) = hi(dst) * src(imm) */ ++ emit_mov_i(ctx, src, imm); ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, hi(dst), hi(dst), src); ++ } else { ++ emit(ctx, multu, hi(dst), src); ++ emit(ctx, mflo, hi(dst)); ++ } ++ ++ /* hi(dst) = hi(dst) - lo(dst) */ ++ if (imm < 0) ++ emit(ctx, subu, hi(dst), hi(dst), lo(dst)); ++ ++ /* tmp = lo(dst) * src(imm) >> 32 */ ++ /* lo(dst) = lo(dst) * src(imm) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), src); ++ emit(ctx, mulu, lo(dst), lo(dst), src); ++ } else { ++ emit(ctx, multu, lo(dst), src); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) += tmp */ ++ emit(ctx, addu, hi(dst), hi(dst), tmp); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* ALU mul register (64x64-bit) */ ++static void emit_mul_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[]) ++{ ++ u8 acc = MIPS_R_T8; ++ u8 tmp = MIPS_R_T9; ++ ++ /* acc = hi(dst) * lo(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, acc, hi(dst), lo(src)); ++ } else { ++ emit(ctx, multu, hi(dst), lo(src)); ++ emit(ctx, mflo, acc); ++ } ++ ++ /* tmp = lo(dst) * hi(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, tmp, lo(dst), hi(src)); ++ } else { ++ emit(ctx, multu, lo(dst), hi(src)); ++ emit(ctx, mflo, tmp); ++ } ++ ++ /* acc += tmp */ ++ emit(ctx, addu, acc, acc, tmp); ++ ++ /* tmp = lo(dst) * lo(src) >> 32 */ ++ /* lo(dst) = lo(dst) * lo(src) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), lo(src)); ++ emit(ctx, mulu, lo(dst), lo(dst), lo(src)); ++ } else { ++ emit(ctx, multu, lo(dst), lo(src)); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) = acc + tmp */ ++ emit(ctx, addu, hi(dst), acc, tmp); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Helper function for 64-bit modulo */ ++static u64 jit_mod64(u64 a, u64 b) ++{ ++ u64 rem; ++ ++ div64_u64_rem(a, b, &rem); ++ return rem; ++} ++ ++/* ALU div/mod register (64-bit) */ ++static void emit_divmod_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ ++ int exclude, k; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ ++ /* Put 64-bit arguments 1 and 2 in registers a0-a3 */ ++ for (k = 0; k < 2; k++) { ++ emit(ctx, move, MIPS_R_T9, src[k]); ++ emit(ctx, move, r1[k], dst[k]); ++ emit(ctx, move, r2[k], MIPS_R_T9); ++ } ++ ++ /* Emit function call */ ++ switch (BPF_OP(op)) { ++ /* dst = dst / src */ ++ case BPF_DIV: ++ addr = (u32)&div64_u64; ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ addr = (u32)&jit_mod64; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Store the 64-bit result in dst */ ++ emit(ctx, move, dst[0], r0[0]); ++ emit(ctx, move, dst[1], r0[1]); ++ ++ /* Restore caller-saved registers, excluding the computed result */ ++ exclude = BIT(lo(dst)) | BIT(hi(dst)); ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ ++ clobber_reg64(ctx, dst); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Swap bytes in a register word */ ++static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap half words in a register word */ ++static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */ ++ emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, rotr, tmp, hi(dst), 16); ++ emit(ctx, rotr, hi(dst), lo(dst), 16); ++ emit(ctx, wsbh, lo(dst), tmp); ++ emit(ctx, wsbh, hi(dst), hi(dst)); ++ } else { ++ emit_swap16_r(ctx, tmp, lo(dst)); ++ emit_swap16_r(ctx, lo(dst), hi(dst)); ++ emit(ctx, move, hi(dst), tmp); ++ ++ emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */ ++ emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */ ++ emit_swap8_r(ctx, lo(dst), lo(dst), tmp); ++ emit_swap8_r(ctx, hi(dst), hi(dst), tmp); ++ } ++ break; ++ /* Swap bytes in a word */ ++ /* Swap bytes in a half word */ ++ case 32: ++ case 16: ++ emit_bswap_r(ctx, lo(dst), width); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ emit(ctx, andi, lo(dst), lo(dst), 0xffff); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, ++ const u8 dst[], u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lw, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ if (dst[1] == src) { ++ emit(ctx, lw, dst[0], off + 4, src); ++ emit(ctx, lw, dst[1], off, src); ++ } else { ++ emit(ctx, lw, dst[1], off, src); ++ emit(ctx, lw, dst[0], off + 4, src); ++ } ++ emit_load_delay(ctx); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, ++ const u8 dst, const u8 src[], s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, lo(src), off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, lo(src), off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, lo(src), off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sw, src[1], off, dst); ++ emit(ctx, sw, src[0], off + 4, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write (32-bit, non-ll/sc fallback) */ ++static void emit_atomic_r32(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: dst+off if xchg, otherwise src, passed in register a0 ++ * Argument 2: src if xchg, othersize dst+off, passed in register a1 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, MIPS_R_A0, src); ++ emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Atomic read-modify-write (64-bit) */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, const u8 src[], s16 off, u8 code) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: 64-bit src, passed in registers a0-a1 ++ * Argument 2: 32-bit dst+off, passed in register a2 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, r1[0], src[0]); ++ emit(ctx, move, r1[1], src[1]); ++ emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic64_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic64_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic64_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic64_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic64_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* ++ * Conditional movz or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* ++ * Conditional movn or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* Emulation of 64-bit sltiu rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_sltiu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ if (imm < 0) { ++ emit_mov_i(ctx, rd, imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++ } else { /* imm >= 0 */ ++ if (imm > 0x7fff) { ++ emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ } else { ++ emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */ ++ } ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */ ++ } ++} ++ ++/* Emulation of 64-bit sltu rd, rs, rt */ ++static void emit_sltu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */ ++ emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */ ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */ ++ emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++} ++ ++/* Emulation of 64-bit slti rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_slti_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ u8 cmp; ++ ++ /* ++ * if ((rs < 0) ^ (imm < 0)) t1 = imm >u rsl ++ * else t1 = rsl > 31 */ ++ if (imm < 0) ++ emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */ ++ else ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ /* ++ * if ((imm < 0 && rsh != 0xffffffff) || ++ * (imm >= 0 && rsh != 0)) ++ * t1 = 0 ++ */ ++ if (imm < 0) { ++ emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */ ++ cmp = rd; ++ } else { /* imm >= 0 */ ++ cmp = hi(rs); ++ } ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */ ++ ++ /* ++ * if (imm < 0) rd = rsh < -1 ++ * else rd = rsh != 0 ++ * rd = rd | t1 ++ */ ++ emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */ ++ emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ ++} ++ ++/* Emulation of 64-bit(slt rd, rs, rt) */ ++static void emit_slt_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 t1 = MIPS_R_T7; ++ u8 t2 = MIPS_R_T8; ++ u8 t3 = MIPS_R_T9; ++ ++ /* ++ * if ((rs < 0) ^ (rt < 0)) t1 = rtl > 31 */ ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */ ++ ++ /* rd = (rsh < rth) | t1 */ ++ emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh = -0x7fff && imm <= 0x8000) { ++ emit(ctx, addiu, tmp, lo(dst), -imm); ++ } else if ((u32)imm <= 0xffff) { ++ emit(ctx, xori, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, xor, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) { /* Compare sign extension */ ++ emit(ctx, addu, MIPS_R_T9, hi(dst), 1); ++ emit(ctx, or, tmp, tmp, MIPS_R_T9); ++ } else { /* Compare zero extension */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ } ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, tmp, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst & imm */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ if ((u32)imm <= 0xffff) { ++ emit(ctx, andi, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, and, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) /* Sign-extension pulls in high word */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, tmp, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ } ++} ++ ++/* Jump register (64-bit) */ ++static void emit_jmp_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], s32 off, u8 op) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ case BPF_JEQ: ++ case BPF_JNE: ++ emit(ctx, subu, t1, lo(dst), lo(src)); ++ emit(ctx, subu, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, t1, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst & src */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ emit(ctx, and, t1, lo(dst), lo(src)); ++ emit(ctx, and, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, t1, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ } ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push stack arguments */ ++ push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK); ++ ++ /* Emit function call */ ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = lo(bpf2mips32[BPF_REG_2]); ++ u8 ind = lo(bpf2mips32[BPF_REG_3]); ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ * stack[sz] - remaining tail call count, initialized in prologue ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */ ++ emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */ ++ /* (next insn delay slot) */ ++ /* if (TCC-- <= 0) goto out */ ++ emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 < 0 */ ++ emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */ ++ emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */ ++ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */ ++ emit(ctx, addu, t1, t1, ary); /* t1 += ary */ ++ emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, t1); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Caller's stack frame : ++ * :----------------------------: ++ * : 64-bit eBPF args r3-r5 : ++ * :----------------------------: ++ * : Reserved / tail call count : ++ * +============================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +----------------------------+ <--- eBPF FP (MIPS zero,fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +----------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * +----------------------------+ ++ * | Reserved for 64-bit eBPF | ++ * | args r3-r5 & args passed | ++ * | on stack in kernel calls | ++ * Lower address +============================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; ++ const u8 *fp = bpf2mips32[BPF_REG_FP]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first two instructions initialize TCC in the reserved (for us) ++ * 16-byte area in the parent's stack frame. On a tail call, the ++ * calling function jumps into the prologue after these instructions. ++ */ ++ emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, ++ min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); ++ ++ /* ++ * Register eBPF R1 contains the 32-bit context pointer argument. ++ * A 32-bit argument is always passed in MIPS register a0, regardless ++ * of CPU endianness. Initialize R1 accordingly and zero-extend. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, lo(r1), MIPS_R_A0); ++#endif ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* Zero-extend the 32-bit argument */ ++ emit(ctx, move, hi(r1), MIPS_R_ZERO); ++ ++ /* If the eBPF frame pointer was accessed it must be saved */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg64(ctx, fp); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers and function arguments passed on the stack. ++ * The required space is computed automatically during resource ++ * usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved); ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ /* ++ * A 32-bit return value is always passed in MIPS register v0, ++ * but on big-endian targets the low part of R0 is mapped to v1. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, MIPS_R_V0, MIPS_R_V1); ++#endif ++ ++ /* Jump to the return address and adjust the stack pointer */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ const u8 *dst = bpf2mips32[insn->dst_reg]; ++ const u8 *src = bpf2mips32[insn->src_reg]; ++ const u8 *tmp = bpf2mips32[JIT_REG_TMP]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_mov_i(ctx, hi(dst), 0); ++ } else { ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_alu_i(ctx, lo(dst), 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, lo(dst), val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_se_i64(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_mov_r(ctx, hi(dst), hi(src)); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_neg_i64(ctx, dst); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ if (imm) ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ if (imm) ++ emit_shift_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst * imm (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ emit_mul_i64(ctx, dst, imm); ++ break; ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ /* ++ * Sign-extend the immediate value into a temporary register, ++ * and then do the operation on this register. ++ */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); ++ break; ++ /* dst = dst * src (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ emit_mul_r64(ctx, dst, src); ++ break; ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_divmod_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_mov_i(ctx, hi(dst), insn[1].imm); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ switch (BPF_SIZE(code)) { ++ case BPF_DW: ++ /* Sign-extend immediate value into temporary reg */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ break; ++ case BPF_W: ++ case BPF_H: ++ case BPF_B: ++ emit_mov_i(ctx, lo(tmp), imm); ++ break; ++ } ++ emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (cpu_has_llsc) ++ emit_atomic_r(ctx, lo(dst), lo(src), off, imm); ++ else /* Non-ll/sc fallback */ ++ emit_atomic_r32(ctx, lo(dst), lo(src), ++ off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* Atomics (64-bit) */ ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ emit_atomic_r64(ctx, lo(dst), src, off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, lo(dst), imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r64(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_i64(ctx, dst, imm, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.10/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch b/target/linux/generic/backport-5.10/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch new file mode 100644 index 0000000000..38b46c0b76 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch @@ -0,0 +1,1005 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:05 +0200 +Subject: [PATCH] mips: bpf: Add new eBPF JIT for 64-bit MIPS + +This is an implementation on of an eBPF JIT for 64-bit MIPS III-V and +MIPS64r1-r6. It uses the same framework introduced by the 32-bit JIT. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp64.c + +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -0,0 +1,991 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS t0-t3 are not available in the n64 ABI */ ++#undef MIPS_R_T0 ++#undef MIPS_R_T1 ++#undef MIPS_R_T2 ++#undef MIPS_R_T3 ++ ++/* Stack is 16-byte aligned in n64 ABI */ ++#define MIPS_STACK_ALIGNMENT 16 ++ ++/* Extra 64-bit eBPF registers used by JIT */ ++#define JIT_REG_TC (MAX_BPF_JIT_REG + 0) ++#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1) ++ ++/* Number of prologue bytes to skip when doing a tail call */ ++#define JIT_TCALL_SKIP 4 ++ ++/* Callee-saved CPU registers that the JIT must preserve */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* Caller-saved CPU registers available for JIT use */ ++#define JIT_CALLER_REGS \ ++ (BIT(MIPS_R_A5) | \ ++ BIT(MIPS_R_A6) | \ ++ BIT(MIPS_R_A7)) ++/* ++ * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers. ++ * MIPS registers t4 - t7 may be used by the JIT as temporary registers. ++ * MIPS registers t8 - t9 are reserved for single-register common functions. ++ */ ++static const u8 bpf2mips64[] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = MIPS_R_V0, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = MIPS_R_A0, ++ [BPF_REG_2] = MIPS_R_A1, ++ [BPF_REG_3] = MIPS_R_A2, ++ [BPF_REG_4] = MIPS_R_A3, ++ [BPF_REG_5] = MIPS_R_A4, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = MIPS_R_S0, ++ [BPF_REG_7] = MIPS_R_S1, ++ [BPF_REG_8] = MIPS_R_S2, ++ [BPF_REG_9] = MIPS_R_S3, ++ /* Read-only frame pointer to access the eBPF stack */ ++ [BPF_REG_FP] = MIPS_R_FP, ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = MIPS_R_AT, ++ /* Tail call count register, caller-saved */ ++ [JIT_REG_TC] = MIPS_R_A5, ++ /* Constant for register zero-extension */ ++ [JIT_REG_ZX] = MIPS_R_V1, ++}; ++ ++/* ++ * MIPS 32-bit operations on 64-bit registers generate a sign-extended ++ * result. However, the eBPF ISA mandates zero-extension, so we rely on the ++ * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic ++ * operations, right shift and byte swap require properly sign-extended ++ * operands or the result is unpredictable. We emit explicit sign-extensions ++ * in those cases. ++ */ ++ ++/* Sign extension */ ++static void emit_sext(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, sll, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension */ ++static void emit_zext(struct jit_context *ctx, u8 dst) ++{ ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); ++ } else { ++ emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]); ++ access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */ ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, u8 dst) ++{ ++ if (!ctx->program->aux->verifier_zext) ++ emit_zext(ctx, dst); ++} ++ ++/* dst = imm (64-bit) */ ++static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) ++{ ++ if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) { ++ emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64); ++ } else if (imm64 >= 0xffffffff80000000ULL || ++ (imm64 < 0x80000000 && imm64 > 0xffff)) { ++ emit(ctx, lui, dst, (s16)(imm64 >> 16)); ++ emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff); ++ } else { ++ u8 acc = MIPS_R_ZERO; ++ int k; ++ ++ for (k = 0; k < 4; k++) { ++ u16 half = imm64 >> (48 - 16 * k); ++ ++ if (acc == dst) ++ emit(ctx, dsll, dst, dst, 16); ++ ++ if (half) { ++ emit(ctx, ori, dst, acc, half); ++ acc = dst; ++ } ++ } ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, dsubu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, dsll_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, dsrl_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsra_safe, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, daddiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, daddiu, dst, dst, -imm); ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_i(ctx, dst, imm, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, dsllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, dsrlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsrav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, daddu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, dsubu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmulu, dst, dst, src); ++ } else { ++ emit(ctx, dmultu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips64r6) { ++ emit(ctx, ddivu_r6, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmodu, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_r(ctx, dst, src, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Swap sub words in a register double word */ ++static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */ ++ emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */ ++ emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & mask */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dsbh, dst, dst); ++ emit(ctx, dshd, dst, dst); ++ } else { ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */ ++ emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */ ++ emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */ ++ ++ emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff); ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */ ++ ++ emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */ ++ emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */ ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ /* Swap bytes in a word */ ++ case 32: ++ case 16: ++ emit_sext(ctx, dst, dst); ++ emit_bswap_r(ctx, dst, width); ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) ++ emit_zext(ctx, dst); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit_zext(ctx, dst); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, andi, dst, dst, 0xffff); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, dst, off, src); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, dst, off, src); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lwu, dst, off, src); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ emit(ctx, ld, dst, off, src); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, src, off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, src, off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, src, off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sd, src, off, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, lld, t1, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, daddu, t2, t1, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, t2, t1, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, t2, t1, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, t2, t1, src); ++ break; ++ } ++ emit(ctx, scd, t2, off, dst); ++ emit(ctx, beqz, t2, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ u8 tmp = MIPS_R_T6; ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Emit function call */ ++ emit_mov_i64(ctx, tmp, addr); ++ emit(ctx, jalr, MIPS_R_RA, tmp); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Re-initialize the JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = bpf2mips64[BPF_REG_2]; ++ u8 ind = bpf2mips64[BPF_REG_3]; ++ u8 tcc = bpf2mips64[JIT_REG_TC]; ++ u8 tmp = MIPS_R_T6; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/ ++ emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ ++ /* if (--TCC < 0) goto out */ ++ emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */ ++ emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */ ++ /* (next insn delay slot) */ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */ ++ emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */ ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, tmp); ++ access_reg(ctx, JIT_REG_TC); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Previous stack frame : ++ * +===========================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +---------------------------+ <--- eBPF FP (MIPS fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +---------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * Lower address +===========================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ u8 fp = bpf2mips64[BPF_REG_FP]; ++ u8 tc = bpf2mips64[JIT_REG_TC]; ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first instruction initializes the tail call count register. ++ * On a tail call, the calling function jumps into the prologue ++ * after this instruction. ++ */ ++ emit(ctx, addiu, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* ++ * If the eBPF frame pointer and tail call count registers were ++ * accessed they must be preserved. Mark them as clobbered here ++ * to save and restore them on the stack as needed. ++ */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg(ctx, fp); ++ if (ctx->accessed & BIT(JIT_REG_TC)) ++ clobber_reg(ctx, tc); ++ if (ctx->accessed & BIT(JIT_REG_ZX)) ++ clobber_reg(ctx, zx); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers needed by the JIT. The required space is ++ * computed automatically during resource usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ if (stack) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved); ++ ++ /* Initialize the ePF JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ ++ /* Release the stack frame */ ++ if (ctx->stack_size) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++ ++ /* Jump to return address and sign-extend the 32-bit return value */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */ ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ u8 dst = bpf2mips64[insn->dst_reg]; ++ u8 src = bpf2mips64[insn->src_reg]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_zext(ctx, dst); ++ } else { ++ emit_mov_r(ctx, dst, src); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_sext(ctx, dst, dst); ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ emit_alu_r(ctx, dst, src, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_sext(ctx, dst, dst); ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, dst, src); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_alu_i64(ctx, dst, 0, BPF_NEG); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm ((64-bit, arithmetic) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ /* dst = dst * imm (64-bit) */ ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i64(ctx, dst, val, alu); ++ } ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ /* dst = dst * src (64-bit) */ ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32)); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (BPF_SIZE(code) == BPF_DW) { ++ emit_atomic_r64(ctx, dst, src, off, imm); ++ } else { /* 32-bit, no fetch */ ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm); ++ } ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */ ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register, sign-extended */ ++ emit_mov_i(ctx, MIPS_R_T5, imm); ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, dst, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.10/050-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch b/target/linux/generic/backport-5.10/050-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch new file mode 100644 index 0000000000..63553ebe58 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch @@ -0,0 +1,120 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:06 +0200 +Subject: [PATCH] mips: bpf: Add JIT workarounds for CPU errata + +This patch adds workarounds for the following CPU errata to the MIPS +eBPF JIT, if enabled in the kernel configuration. + + - R10000 ll/sc weak ordering + - Loongson-3 ll/sc weak ordering + - Loongson-2F jump hang + +The Loongson-2F nop errata is implemented in uasm, which the JIT uses, +so no additional mitigations are needed for that. + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/net/bpf_jit_comp.c ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, + /* Atomic read-modify-write (32-bit) */ + void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + switch (code) { + case BPF_ADD: +@@ -420,18 +421,19 @@ void emit_atomic_r(struct jit_context *c + break; + } + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + + /* Atomic compare-and-exchange (32-bit) */ + void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + emit(ctx, bne, MIPS_R_T9, res, 12); + emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset); + emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ + clobber_reg(ctx, res); + } +--- a/arch/mips/net/bpf_jit_comp.h ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -87,7 +87,7 @@ struct jit_context { + }; + + /* Emit the instruction if the JIT memory space has been allocated */ +-#define emit(ctx, func, ...) \ ++#define __emit(ctx, func, ...) \ + do { \ + if ((ctx)->target != NULL) { \ + u32 *p = &(ctx)->target[ctx->jit_index]; \ +@@ -95,6 +95,30 @@ do { \ + } \ + (ctx)->jit_index++; \ + } while (0) ++#define emit(...) __emit(__VA_ARGS__) ++ ++/* Workaround for R10000 ll/sc errata */ ++#ifdef CONFIG_WAR_R10000 ++#define LLSC_beqz beqzl ++#else ++#define LLSC_beqz beqz ++#endif ++ ++/* Workaround for Loongson-3 ll/sc errata */ ++#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS ++#define LLSC_sync(ctx) emit(ctx, sync, 0) ++#define LLSC_offset 4 ++#else ++#define LLSC_sync(ctx) ++#define LLSC_offset 0 ++#endif ++ ++/* Workaround for Loongson-2F jump errata */ ++#ifdef CONFIG_CPU_JUMP_WORKAROUNDS ++#define JALR_MASK 0xffffffffcfffffffULL ++#else ++#define JALR_MASK (~0ULL) ++#endif + + /* + * Mark a BPF register as accessed, it needs to be +--- a/arch/mips/net/bpf_jit_comp64.c ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_c + u8 t1 = MIPS_R_T6; + u8 t2 = MIPS_R_T7; + ++ LLSC_sync(ctx); + emit(ctx, lld, t1, off, dst); + switch (code) { + case BPF_ADD: +@@ -391,7 +392,7 @@ static void emit_atomic_r64(struct jit_c + break; + } + emit(ctx, scd, t2, off, dst); +- emit(ctx, beqz, t2, -16); ++ emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + +@@ -414,7 +415,7 @@ static int emit_call(struct jit_context + push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); + + /* Emit function call */ +- emit_mov_i64(ctx, tmp, addr); ++ emit_mov_i64(ctx, tmp, addr & JALR_MASK); + emit(ctx, jalr, MIPS_R_RA, tmp); + emit(ctx, nop); /* Delay slot */ + diff --git a/target/linux/generic/backport-5.10/050-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch b/target/linux/generic/backport-5.10/050-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch new file mode 100644 index 0000000000..10685c5f3c --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch @@ -0,0 +1,61 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:07 +0200 +Subject: [PATCH] mips: bpf: Enable eBPF JITs + +This patch enables the new eBPF JITs for 32-bit and 64-bit MIPS. It also +disables the old cBPF JIT to so cBPF programs are converted to use the +new JIT. + +Workarounds for R4000 CPU errata are not implemented by the JIT, so the +JIT is disabled if any of those workarounds are configured. + +Signed-off-by: Johan Almbladh +--- + +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -3294,6 +3294,7 @@ S: Supported + F: arch/arm64/net/ + + BPF JIT for MIPS (32-BIT AND 64-BIT) ++M: Johan Almbladh + M: Paul Burton + L: netdev@vger.kernel.org + L: bpf@vger.kernel.org +--- a/arch/mips/Kconfig ++++ b/arch/mips/Kconfig +@@ -49,7 +49,6 @@ config MIPS + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES + select HAVE_ASM_MODVERSIONS +- select HAVE_CBPF_JIT if !64BIT && !CPU_MICROMIPS + select HAVE_CONTEXT_TRACKING + select HAVE_TIF_NOHZ + select HAVE_C_RECORDMCOUNT +@@ -57,7 +56,10 @@ config MIPS + select HAVE_DEBUG_STACKOVERFLOW + select HAVE_DMA_CONTIGUOUS + select HAVE_DYNAMIC_FTRACE +- select HAVE_EBPF_JIT if 64BIT && !CPU_MICROMIPS && TARGET_ISA_REV >= 2 ++ select HAVE_EBPF_JIT if !CPU_MICROMIPS && \ ++ !CPU_DADDI_WORKAROUNDS && \ ++ !CPU_R4000_WORKAROUNDS && \ ++ !CPU_R4400_WORKAROUNDS + select HAVE_EXIT_THREAD + select HAVE_FAST_GUP + select HAVE_FTRACE_MCOUNT_RECORD +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,9 +2,10 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o ++obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o + + ifeq ($(CONFIG_32BIT),y) +- obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp32.o + else +- obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp64.o + endif diff --git a/target/linux/generic/backport-5.10/050-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch b/target/linux/generic/backport-5.10/050-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch new file mode 100644 index 0000000000..e25c336831 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch @@ -0,0 +1,387 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:08 +0200 +Subject: [PATCH] mips: bpf: Remove old BPF JIT implementations + +This patch removes the old 32-bit cBPF and 64-bit eBPF JIT implementations. +They are replaced by a new eBPF implementation that supports both 32-bit +and 64-bit MIPS CPUs. + +Signed-off-by: Johan Almbladh +--- + delete mode 100644 arch/mips/net/bpf_jit.c + delete mode 100644 arch/mips/net/bpf_jit.h + delete mode 100644 arch/mips/net/bpf_jit_asm.S + delete mode 100644 arch/mips/net/ebpf_jit.c + +--- a/arch/mips/net/bpf_jit.h ++++ /dev/null +@@ -1,81 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0-only */ +-/* +- * Just-In-Time compiler for BPF filters on MIPS +- * +- * Copyright (c) 2014 Imagination Technologies Ltd. +- * Author: Markos Chandras +- */ +- +-#ifndef BPF_JIT_MIPS_OP_H +-#define BPF_JIT_MIPS_OP_H +- +-/* Registers used by JIT */ +-#define MIPS_R_ZERO 0 +-#define MIPS_R_V0 2 +-#define MIPS_R_A0 4 +-#define MIPS_R_A1 5 +-#define MIPS_R_T4 12 +-#define MIPS_R_T5 13 +-#define MIPS_R_T6 14 +-#define MIPS_R_T7 15 +-#define MIPS_R_S0 16 +-#define MIPS_R_S1 17 +-#define MIPS_R_S2 18 +-#define MIPS_R_S3 19 +-#define MIPS_R_S4 20 +-#define MIPS_R_S5 21 +-#define MIPS_R_S6 22 +-#define MIPS_R_S7 23 +-#define MIPS_R_SP 29 +-#define MIPS_R_RA 31 +- +-/* Conditional codes */ +-#define MIPS_COND_EQ 0x1 +-#define MIPS_COND_GE (0x1 << 1) +-#define MIPS_COND_GT (0x1 << 2) +-#define MIPS_COND_NE (0x1 << 3) +-#define MIPS_COND_ALL (0x1 << 4) +-/* Conditionals on X register or K immediate */ +-#define MIPS_COND_X (0x1 << 5) +-#define MIPS_COND_K (0x1 << 6) +- +-#define r_ret MIPS_R_V0 +- +-/* +- * Use 2 scratch registers to avoid pipeline interlocks. +- * There is no overhead during epilogue and prologue since +- * any of the $s0-$s6 registers will only be preserved if +- * they are going to actually be used. +- */ +-#define r_skb_hl MIPS_R_S0 /* skb header length */ +-#define r_skb_data MIPS_R_S1 /* skb actual data */ +-#define r_off MIPS_R_S2 +-#define r_A MIPS_R_S3 +-#define r_X MIPS_R_S4 +-#define r_skb MIPS_R_S5 +-#define r_M MIPS_R_S6 +-#define r_skb_len MIPS_R_S7 +-#define r_s0 MIPS_R_T4 /* scratch reg 1 */ +-#define r_s1 MIPS_R_T5 /* scratch reg 2 */ +-#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */ +-#define r_tmp MIPS_R_T7 /* No need to preserve this */ +-#define r_zero MIPS_R_ZERO +-#define r_sp MIPS_R_SP +-#define r_ra MIPS_R_RA +- +-#ifndef __ASSEMBLY__ +- +-/* Declare ASM helpers */ +- +-#define DECLARE_LOAD_FUNC(func) \ +- extern u8 func(unsigned long *skb, int offset); \ +- extern u8 func##_negative(unsigned long *skb, int offset); \ +- extern u8 func##_positive(unsigned long *skb, int offset) +- +-DECLARE_LOAD_FUNC(sk_load_word); +-DECLARE_LOAD_FUNC(sk_load_half); +-DECLARE_LOAD_FUNC(sk_load_byte); +- +-#endif +- +-#endif /* BPF_JIT_MIPS_OP_H */ +--- a/arch/mips/net/bpf_jit_asm.S ++++ /dev/null +@@ -1,285 +0,0 @@ +-/* +- * bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF +- * compiler. +- * +- * Copyright (C) 2015 Imagination Technologies Ltd. +- * Author: Markos Chandras +- * +- * This program is free software; you can redistribute it and/or modify it +- * under the terms of the GNU General Public License as published by the +- * Free Software Foundation; version 2 of the License. +- */ +- +-#include +-#include +-#include +-#include "bpf_jit.h" +- +-/* ABI +- * +- * r_skb_hl skb header length +- * r_skb_data skb data +- * r_off(a1) offset register +- * r_A BPF register A +- * r_X PF register X +- * r_skb(a0) *skb +- * r_M *scratch memory +- * r_skb_le skb length +- * r_s0 Scratch register 0 +- * r_s1 Scratch register 1 +- * +- * On entry: +- * a0: *skb +- * a1: offset (imm or imm + X) +- * +- * All non-BPF-ABI registers are free for use. On return, we only +- * care about r_ret. The BPF-ABI registers are assumed to remain +- * unmodified during the entire filter operation. +- */ +- +-#define skb a0 +-#define offset a1 +-#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */ +- +- /* We know better :) so prevent assembler reordering etc */ +- .set noreorder +- +-#define is_offset_negative(TYPE) \ +- /* If offset is negative we have more work to do */ \ +- slti t0, offset, 0; \ +- bgtz t0, bpf_slow_path_##TYPE##_neg; \ +- /* Be careful what follows in DS. */ +- +-#define is_offset_in_header(SIZE, TYPE) \ +- /* Reading from header? */ \ +- addiu $r_s0, $r_skb_hl, -SIZE; \ +- slt t0, $r_s0, offset; \ +- bgtz t0, bpf_slow_path_##TYPE; \ +- +-LEAF(sk_load_word) +- is_offset_negative(word) +-FEXPORT(sk_load_word_positive) +- is_offset_in_header(4, word) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- .set reorder +- lw $r_A, 0(t1) +- .set noreorder +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_A +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_A, 24 +- srl t1, $r_A, 24 +- srl t2, $r_A, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_A, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_word) +- +-LEAF(sk_load_half) +- is_offset_negative(half) +-FEXPORT(sk_load_half_positive) +- is_offset_in_header(2, half) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lhu $r_A, 0(t1) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh $r_A, $r_A +-# else +- sll t0, $r_A, 8 +- srl t1, $r_A, 8 +- andi t0, t0, 0xff00 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_half) +- +-LEAF(sk_load_byte) +- is_offset_negative(byte) +-FEXPORT(sk_load_byte_positive) +- is_offset_in_header(1, byte) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lbu $r_A, 0(t1) +- jr $r_ra +- move $r_ret, zero +- END(sk_load_byte) +- +-/* +- * call skb_copy_bits: +- * (prototype in linux/skbuff.h) +- * +- * int skb_copy_bits(sk_buff *skb, int offset, void *to, int len) +- * +- * o32 mandates we leave 4 spaces for argument registers in case +- * the callee needs to use them. Even though we don't care about +- * the argument registers ourselves, we need to allocate that space +- * to remain ABI compliant since the callee may want to use that space. +- * We also allocate 2 more spaces for $r_ra and our return register (*to). +- * +- * n64 is a bit different. The *caller* will allocate the space to preserve +- * the arguments. So in 64-bit kernels, we allocate the 4-arg space for no +- * good reason but it does not matter that much really. +- * +- * (void *to) is returned in r_s0 +- * +- */ +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-#define DS_OFFSET(SIZE) (4 * SZREG) +-#else +-#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE)) +-#endif +-#define bpf_slow_path_common(SIZE) \ +- /* Quick check. Are we within reasonable boundaries? */ \ +- LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \ +- sltu $r_s0, offset, $r_s1; \ +- beqz $r_s0, fault; \ +- /* Load 4th argument in DS */ \ +- LONG_ADDIU a3, zero, SIZE; \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, skb_copy_bits; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- /* Assign low slot to a2 */ \ +- PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \ +- jalr t0; \ +- /* Reset our destination slot (DS but it's ok) */ \ +- INT_S zero, (4 * SZREG)($r_sp); \ +- /* \ +- * skb_copy_bits returns 0 on success and -EFAULT \ +- * on error. Our data live in a2. Do not bother with \ +- * our data if an error has been returned. \ +- */ \ +- /* Restore our frame */ \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- INT_L $r_s0, (4 * SZREG)($r_sp); \ +- bltz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- move $r_ret, zero; \ +- +-NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp) +- bpf_slow_path_common(4) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_s0 +- jr $r_ra +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_s0, 24 +- srl t1, $r_s0, 24 +- srl t2, $r_s0, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_s0, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_word) +- +-NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp) +- bpf_slow_path_common(2) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- jr $r_ra +- wsbh $r_A, $r_s0 +-# else +- sll t0, $r_s0, 8 +- andi t1, $r_s0, 0xff00 +- andi t0, t0, 0xff00 +- srl t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_half) +- +-NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp) +- bpf_slow_path_common(1) +- jr $r_ra +- move $r_A, $r_s0 +- +- END(bpf_slow_path_byte) +- +-/* +- * Negative entry points +- */ +- .macro bpf_is_end_of_data +- li t0, SKF_LL_OFF +- /* Reading link layer data? */ +- slt t1, offset, t0 +- bgtz t1, fault +- /* Be careful what follows in DS. */ +- .endm +-/* +- * call skb_copy_bits: +- * (prototype in linux/filter.h) +- * +- * void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, +- * int k, unsigned int size) +- * +- * see above (bpf_slow_path_common) for ABI restrictions +- */ +-#define bpf_negative_common(SIZE) \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, bpf_internal_load_pointer_neg_helper; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- jalr t0; \ +- li a2, SIZE; \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- /* Check return pointer */ \ +- beqz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- /* Preserve our pointer */ \ +- move $r_s0, v0; \ +- /* Set return value */ \ +- move $r_ret, zero; \ +- +-bpf_slow_path_word_neg: +- bpf_is_end_of_data +-NESTED(sk_load_word_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(4) +- jr $r_ra +- lw $r_A, 0($r_s0) +- END(sk_load_word_negative) +- +-bpf_slow_path_half_neg: +- bpf_is_end_of_data +-NESTED(sk_load_half_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(2) +- jr $r_ra +- lhu $r_A, 0($r_s0) +- END(sk_load_half_negative) +- +-bpf_slow_path_byte_neg: +- bpf_is_end_of_data +-NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(1) +- jr $r_ra +- lbu $r_A, 0($r_s0) +- END(sk_load_byte_negative) +- +-fault: +- jr $r_ra +- addiu $r_ret, zero, 1 diff --git a/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch b/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch index 83bfd733d6..7e34bae8b4 100644 --- a/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch +++ b/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch @@ -25,7 +25,7 @@ Signed-off-by: Jason A. Donenfeld --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -2078,7 +2078,7 @@ config CPU_MIPS32 +@@ -2080,7 +2080,7 @@ config CPU_MIPS32 config CPU_MIPS64 bool default y if CPU_MIPS64_R1 || CPU_MIPS64_R2 || CPU_MIPS64_R5 || \ diff --git a/target/linux/generic/backport-5.4/071-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch b/target/linux/generic/backport-5.4/071-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch new file mode 100644 index 0000000000..82feb7421d --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch @@ -0,0 +1,65 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:02 +0200 +Subject: [PATCH] MIPS: uasm: Enable muhu opcode for MIPS R6 + +Enable the 'muhu' instruction, complementing the existing 'mulu', needed +to implement a MIPS32 BPF JIT. + +Also fix a typo in the existing definition of 'dmulu'. + +Signed-off-by: Tony Ambardar + +This patch is a dependency for my 32-bit MIPS eBPF JIT. + +Signed-off-by: Johan Almbladh +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -145,6 +145,7 @@ Ip_u1(_mtlo); + Ip_u3u1u2(_mul); + Ip_u1u2(_multu); + Ip_u3u1u2(_mulu); ++Ip_u3u1u2(_muhu); + Ip_u3u1u2(_nor); + Ip_u3u1u2(_or); + Ip_u2u1u3(_ori); +--- a/arch/mips/mm/uasm-mips.c ++++ b/arch/mips/mm/uasm-mips.c +@@ -90,7 +90,7 @@ static const struct insn insn_table[insn + RS | RT | RD}, + [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, + [insn_dmultu] = {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT}, +- [insn_dmulu] = {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op), ++ [insn_dmulu] = {M(spec_op, 0, 0, 0, dmultu_dmulu_op, dmultu_op), + RS | RT | RD}, + [insn_drotr] = {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE}, + [insn_drotr32] = {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE}, +@@ -150,6 +150,8 @@ static const struct insn insn_table[insn + [insn_mtlo] = {M(spec_op, 0, 0, 0, 0, mtlo_op), RS}, + [insn_mulu] = {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op), + RS | RT | RD}, ++ [insn_muhu] = {M(spec_op, 0, 0, 0, multu_muhu_op, multu_op), ++ RS | RT | RD}, + #ifndef CONFIG_CPU_MIPSR6 + [insn_mul] = {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD}, + #else +--- a/arch/mips/mm/uasm.c ++++ b/arch/mips/mm/uasm.c +@@ -59,7 +59,7 @@ enum opcode { + insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, + insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, + insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, +- insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor, ++ insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, + insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, + insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, + insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, +@@ -344,6 +344,7 @@ I_u1(_mtlo) + I_u3u1u2(_mul) + I_u1u2(_multu) + I_u3u1u2(_mulu) ++I_u3u1u2(_muhu) + I_u3u1u2(_nor) + I_u3u1u2(_or) + I_u2u1u3(_ori) diff --git a/target/linux/generic/backport-5.4/071-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch b/target/linux/generic/backport-5.4/071-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch new file mode 100644 index 0000000000..3a4d573f80 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch @@ -0,0 +1,31 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:03 +0200 +Subject: [PATCH] mips: uasm: Add workaround for Loongson-2F nop CPU errata + +This patch implements a workaround for the Loongson-2F nop in generated, +code, if the existing option CONFIG_CPU_NOP_WORKAROUND is set. Before, +the binutils option -mfix-loongson2f-nop was enabled, but no workaround +was done when emitting MIPS code. Now, the nop pseudo instruction is +emitted as "or ax,ax,zero" instead of the default "sll zero,zero,0". This +is consistent with the workaround implemented by binutils. + +Link: https://sourceware.org/legacy-ml/binutils/2009-11/msg00387.html + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -249,7 +249,11 @@ static inline void uasm_l##lb(struct uas + #define uasm_i_bnezl(buf, rs, off) uasm_i_bnel(buf, rs, 0, off) + #define uasm_i_ehb(buf) uasm_i_sll(buf, 0, 0, 3) + #define uasm_i_move(buf, a, b) UASM_i_ADDU(buf, a, 0, b) ++#ifdef CONFIG_CPU_NOP_WORKAROUNDS ++#define uasm_i_nop(buf) uasm_i_or(buf, 1, 1, 0) ++#else + #define uasm_i_nop(buf) uasm_i_sll(buf, 0, 0, 0) ++#endif + #define uasm_i_ssnop(buf) uasm_i_sll(buf, 0, 0, 1) + + static inline void uasm_i_drotr_safe(u32 **p, unsigned int a1, diff --git a/target/linux/generic/backport-5.4/071-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch b/target/linux/generic/backport-5.4/071-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch new file mode 100644 index 0000000000..7980659961 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch @@ -0,0 +1,3078 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:04 +0200 +Subject: [PATCH] mips: bpf: Add eBPF JIT for 32-bit MIPS + +This is an implementation of an eBPF JIT for 32-bit MIPS I-V and MIPS32. +The implementation supports all 32-bit and 64-bit ALU and JMP operations, +including the recently-added atomics. 64-bit div/mod and 64-bit atomics +are implemented using function calls to math64 and atomic64 functions, +respectively. All 32-bit operations are implemented natively by the JIT, +except if the CPU lacks ll/sc instructions. + +Register mapping +================ +All 64-bit eBPF registers are mapped to native 32-bit MIPS register pairs, +and does not use any stack scratch space for register swapping. This means +that all eBPF register data is kept in CPU registers all the time, and +this simplifies the register management a lot. It also reduces the JIT's +pressure on temporary registers since we do not have to move data around. + +Native register pairs are ordered according to CPU endiannes, following +the O32 calling convention for passing 64-bit arguments and return values. +The eBPF return value, arguments and callee-saved registers are mapped to +their native MIPS equivalents. + +Since the 32 highest bits in the eBPF FP (frame pointer) register are +always zero, only one general-purpose register is actually needed for the +mapping. The MIPS fp register is used for this purpose. The high bits are +mapped to MIPS register r0. This saves us one CPU register, which is much +needed for temporaries, while still allowing us to treat the R10 (FP) +register just like any other eBPF register in the JIT. + +The MIPS gp (global pointer) and at (assembler temporary) registers are +used as internal temporary registers for constant blinding. CPU registers +t6-t9 are used internally by the JIT when constructing more complex 64-bit +operations. This is precisely what is needed - two registers to store an +operand value, and two more as scratch registers when performing the +operation. + +The register mapping is shown below. + + R0 - $v1, $v0 return value + R1 - $a1, $a0 argument 1, passed in registers + R2 - $a3, $a2 argument 2, passed in registers + R3 - $t1, $t0 argument 3, passed on stack + R4 - $t3, $t2 argument 4, passed on stack + R5 - $t4, $t3 argument 5, passed on stack + R6 - $s1, $s0 callee-saved + R7 - $s3, $s2 callee-saved + R8 - $s5, $s4 callee-saved + R9 - $s7, $s6 callee-saved + FP - $r0, $fp 32-bit frame pointer + AX - $gp, $at constant-blinding + $t6 - $t9 unallocated, JIT temporaries + +Jump offsets +============ +The JIT tries to map all conditional JMP operations to MIPS conditional +PC-relative branches. The MIPS branch offset field is 18 bits, in bytes, +which is equivalent to the eBPF 16-bit instruction offset. However, since +the JIT may emit more than one CPU instruction per eBPF instruction, the +field width may overflow. If that happens, the JIT converts the long +conditional jump to a short PC-relative branch with the condition +inverted, jumping over a long unconditional absolute jmp (j). + +This conversion will change the instruction offset mapping used for jumps, +and may in turn result in more branch offset overflows. The JIT therefore +dry-runs the translation until no more branches are converted and the +offsets do not change anymore. There is an upper bound on this of course, +and if the JIT hits that limit, the last two iterations are run with all +branches being converted. + +Tail call count +=============== +The current tail call count is stored in the 16-byte area of the caller's +stack frame that is reserved for the callee in the o32 ABI. The value is +initialized in the prologue, and propagated to the tail-callee by skipping +the initialization instructions when emitting the tail call. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp.c + create mode 100644 arch/mips/net/bpf_jit_comp.h + create mode 100644 arch/mips/net/bpf_jit_comp32.c + +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,4 +2,9 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o +-obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ ++ifeq ($(CONFIG_32BIT),y) ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++else ++ obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++endif +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -0,0 +1,1020 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions common to 32-bit and 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++/* ++ * Code overview ++ * ============= ++ * ++ * - bpf_jit_comp.h ++ * Common definitions and utilities. ++ * ++ * - bpf_jit_comp.c ++ * Implementation of JIT top-level logic and exported JIT API functions. ++ * Implementation of internal operations shared by 32-bit and 64-bit code. ++ * JMP and ALU JIT control code, register control code, shared ALU and ++ * JMP/JMP32 JIT operations. ++ * ++ * - bpf_jit_comp32.c ++ * Implementation of functions to JIT prologue, epilogue and a single eBPF ++ * instruction for 32-bit MIPS CPUs. The functions use shared operations ++ * where possible, and implement the rest for 32-bit MIPS such as ALU64 ++ * operations. ++ * ++ * - bpf_jit_comp64.c ++ * Ditto, for 64-bit MIPS CPUs. ++ * ++ * Zero and sign extension ++ * ======================== ++ * 32-bit MIPS instructions on 64-bit MIPS registers use sign extension, ++ * but the eBPF instruction set mandates zero extension. We let the verifier ++ * insert explicit zero-extensions after 32-bit ALU operations, both for ++ * 32-bit and 64-bit MIPS JITs. Conditional JMP32 operations on 64-bit MIPs ++ * are JITed with sign extensions inserted when so expected. ++ * ++ * ALU operations ++ * ============== ++ * ALU operations on 32/64-bit MIPS and ALU64 operations on 64-bit MIPS are ++ * JITed in the following steps. ALU64 operations on 32-bit MIPS are more ++ * complicated and therefore only processed by special implementations in ++ * step (3). ++ * ++ * 1) valid_alu_i: ++ * Determine if an immediate operation can be emitted as such, or if ++ * we must fall back to the register version. ++ * ++ * 2) rewrite_alu_i: ++ * Convert BPF operation and immediate value to a canonical form for ++ * JITing. In some degenerate cases this form may be a no-op. ++ * ++ * 3) emit_alu_{i,i64,r,64}: ++ * Emit instructions for an ALU or ALU64 immediate or register operation. ++ * ++ * JMP operations ++ * ============== ++ * JMP and JMP32 operations require an JIT instruction offset table for ++ * translating the jump offset. This table is computed by dry-running the ++ * JIT without actually emitting anything. However, the computed PC-relative ++ * offset may overflow the 18-bit offset field width of the native MIPS ++ * branch instruction. In such cases, the long jump is converted into the ++ * following sequence. ++ * ++ * ! +2 Inverted PC-relative branch ++ * nop Delay slot ++ * j Unconditional absolute long jump ++ * nop Delay slot ++ * ++ * Since this converted sequence alters the offset table, all offsets must ++ * be re-calculated. This may in turn trigger new branch conversions, so ++ * the process is repeated until no further changes are made. Normally it ++ * completes in 1-2 iterations. If JIT_MAX_ITERATIONS should reached, we ++ * fall back to converting every remaining jump operation. The branch ++ * conversion is independent of how the JMP or JMP32 condition is JITed. ++ * ++ * JMP32 and JMP operations are JITed as follows. ++ * ++ * 1) setup_jmp_{i,r}: ++ * Convert jump conditional and offset into a form that can be JITed. ++ * This form may be a no-op, a canonical form, or an inverted PC-relative ++ * jump if branch conversion is necessary. ++ * ++ * 2) valid_jmp_i: ++ * Determine if an immediate operations can be emitted as such, or if ++ * we must fall back to the register version. Applies to JMP32 for 32-bit ++ * MIPS, and both JMP and JMP32 for 64-bit MIPS. ++ * ++ * 3) emit_jmp_{i,i64,r,r64}: ++ * Emit instructions for an JMP or JMP32 immediate or register operation. ++ * ++ * 4) finish_jmp_{i,r}: ++ * Emit any instructions needed to finish the jump. This includes a nop ++ * for the delay slot if a branch was emitted, and a long absolute jump ++ * if the branch was converted. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* Convenience macros for descriptor access */ ++#define CONVERTED(desc) ((desc) & JIT_DESC_CONVERT) ++#define INDEX(desc) ((desc) & ~JIT_DESC_CONVERT) ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, sw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, sd, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ ctx->stack_used = max((int)ctx->stack_used, depth); ++ return depth; ++} ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, lw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, ld, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ return depth; ++} ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc) ++{ ++ u32 index = INDEX(ctx->descriptors[loc]); ++ unsigned long pc = (unsigned long)&ctx->target[ctx->jit_index]; ++ unsigned long addr = (unsigned long)&ctx->target[index]; ++ ++ if (!ctx->target) ++ return 0; ++ ++ if ((addr ^ pc) & ~MIPS_JMP_MASK) ++ return -1; ++ ++ return addr & MIPS_JMP_MASK; ++} ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off) ++{ ++ return (INDEX(ctx->descriptors[ctx->bpf_index + off]) - ++ ctx->jit_index - 1) * sizeof(u32); ++} ++ ++/* dst = imm (register width) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm) ++{ ++ if (imm >= -0x8000 && imm <= 0x7fff) { ++ emit(ctx, addiu, dst, MIPS_R_ZERO, imm); ++ } else { ++ emit(ctx, lui, dst, (s16)((u32)imm >> 16)); ++ emit(ctx, ori, dst, dst, (u16)(imm & 0xffff)); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* dst = src (register width) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, ori, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate ALU immediate range */ ++bool valid_alu_i(u8 op, s32 imm) ++{ ++ switch (BPF_OP(op)) { ++ case BPF_NEG: ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ /* All legal eBPF values are valid */ ++ return true; ++ case BPF_ADD: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_SUB: ++ /* -imm must be 16 bits */ ++ return imm >= -0x7fff && imm <= 0x8000; ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_MUL: ++ /* imm must be zero or a positive power of two */ ++ return imm == 0 || (imm > 0 && is_power_of_2(imm)); ++ case BPF_DIV: ++ case BPF_MOD: ++ /* imm must be an 17-bit power of two */ ++ return (u32)imm <= 0x10000 && is_power_of_2((u32)imm); ++ } ++ return false; ++} ++ ++/* Rewrite ALU immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val) ++{ ++ bool act = true; ++ ++ switch (BPF_OP(op)) { ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ case BPF_ADD: ++ case BPF_SUB: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm == 0 is a no-op */ ++ act = imm != 0; ++ break; ++ case BPF_MUL: ++ if (imm == 1) { ++ /* dst * 1 is a no-op */ ++ act = false; ++ } else if (imm == 0) { ++ /* dst * 0 is dst & 0 */ ++ op = BPF_AND; ++ } else { ++ /* dst * (1 << n) is dst << n */ ++ op = BPF_LSH; ++ imm = ilog2(abs(imm)); ++ } ++ break; ++ case BPF_DIV: ++ if (imm == 1) { ++ /* dst / 1 is a no-op */ ++ act = false; ++ } else { ++ /* dst / (1 << n) is dst >> n */ ++ op = BPF_RSH; ++ imm = ilog2(imm); ++ } ++ break; ++ case BPF_MOD: ++ /* dst % (1 << n) is dst & ((1 << n) - 1) */ ++ op = BPF_AND; ++ imm--; ++ break; ++ } ++ ++ *alu = op; ++ *val = imm; ++ return act; ++} ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, subu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, andi, dst, dst, (u16)imm); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, sll, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, srl, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, sra, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, addiu, dst, dst, -imm); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, dst, dst, src); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, dst, dst, src); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, dst, dst, src); ++ break; ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, sllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, srlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, srav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, addu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, subu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, dst, dst, src); ++ } else { ++ emit(ctx, multu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips32r6) { ++ emit(ctx, divu_r6, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips32r6) { ++ emit(ctx, modu, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, addu, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ } ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ emit(ctx, bne, MIPS_R_T9, res, 12); ++ emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ ++ clobber_reg(ctx, res); ++} ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ u8 msk = MIPS_R_T9; ++ ++ switch (width) { ++ /* Swap bytes in a word */ ++ case 32: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, rotr, dst, dst, 16); ++ } else { ++ emit(ctx, sll, tmp, dst, 16); /* tmp = dst << 16 */ ++ emit(ctx, srl, dst, dst, 16); /* dst = dst >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++ ++ emit(ctx, lui, msk, 0xff); /* msk = 0x00ff0000 */ ++ emit(ctx, ori, msk, msk, 0xff); /* msk = msk | 0xff */ ++ ++ emit(ctx, and, tmp, dst, msk); /* tmp = dst & msk */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, dst, 8); /* dst = dst >> 8 */ ++ emit(ctx, and, dst, dst, msk); /* dst = dst & msk */ ++ emit(ctx, or, dst, dst, tmp); /* reg = dst | tmp */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ case 16: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, andi, dst, dst, 0xffff); ++ } else { ++ emit(ctx, andi, tmp, dst, 0xff00); /* t = d & 0xff00 */ ++ emit(ctx, srl, tmp, tmp, 8); /* t = t >> 8 */ ++ emit(ctx, andi, dst, dst, 0x00ff); /* d = d & 0x00ff */ ++ emit(ctx, sll, dst, dst, 8); /* d = d << 8 */ ++ emit(ctx, or, dst, dst, tmp); /* d = d | t */ ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate jump immediate range */ ++bool valid_jmp_i(u8 op, s32 imm) ++{ ++ switch (op) { ++ case JIT_JNOP: ++ /* Immediate value not used */ ++ return true; ++ case BPF_JEQ: ++ case BPF_JNE: ++ /* No immediate operation */ ++ return false; ++ case BPF_JSET: ++ case JIT_JNSET: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_JGE: ++ case BPF_JLT: ++ case BPF_JSGE: ++ case BPF_JSLT: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_JGT: ++ case BPF_JLE: ++ case BPF_JSGT: ++ case BPF_JSLE: ++ /* imm + 1 must be 16 bits */ ++ return imm >= -0x8001 && imm <= 0x7ffe; ++ } ++ return false; ++} ++ ++/* Invert a conditional jump operation */ ++static u8 invert_jmp(u8 op) ++{ ++ switch (op) { ++ case BPF_JA: return JIT_JNOP; ++ case BPF_JEQ: return BPF_JNE; ++ case BPF_JNE: return BPF_JEQ; ++ case BPF_JSET: return JIT_JNSET; ++ case BPF_JGT: return BPF_JLE; ++ case BPF_JGE: return BPF_JLT; ++ case BPF_JLT: return BPF_JGE; ++ case BPF_JLE: return BPF_JGT; ++ case BPF_JSGT: return BPF_JSLE; ++ case BPF_JSGE: return BPF_JSLT; ++ case BPF_JSLT: return BPF_JSGE; ++ case BPF_JSLE: return BPF_JSGT; ++ } ++ return 0; ++} ++ ++/* Prepare a PC-relative jump operation */ ++static void setup_jmp(struct jit_context *ctx, u8 bpf_op, ++ s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ u32 *descp = &ctx->descriptors[ctx->bpf_index]; ++ int op = bpf_op; ++ int offset = 0; ++ ++ /* Do not compute offsets on the first pass */ ++ if (INDEX(*descp) == 0) ++ goto done; ++ ++ /* Skip jumps never taken */ ++ if (bpf_op == JIT_JNOP) ++ goto done; ++ ++ /* Convert jumps always taken */ ++ if (bpf_op == BPF_JA) ++ *descp |= JIT_DESC_CONVERT; ++ ++ /* ++ * Current ctx->jit_index points to the start of the branch preamble. ++ * Since the preamble differs among different branch conditionals, ++ * the current index cannot be used to compute the branch offset. ++ * Instead, we use the offset table value for the next instruction, ++ * which gives the index immediately after the branch delay slot. ++ */ ++ if (!CONVERTED(*descp)) { ++ int target = ctx->bpf_index + bpf_off + 1; ++ int origin = ctx->bpf_index + 1; ++ ++ offset = (INDEX(ctx->descriptors[target]) - ++ INDEX(ctx->descriptors[origin]) + 1) * sizeof(u32); ++ } ++ ++ /* ++ * The PC-relative branch offset field on MIPS is 18 bits signed, ++ * so if the computed offset is larger than this we generate a an ++ * absolute jump that we skip with an inverted conditional branch. ++ */ ++ if (CONVERTED(*descp) || offset < -0x20000 || offset > 0x1ffff) { ++ offset = 3 * sizeof(u32); ++ op = invert_jmp(bpf_op); ++ ctx->changes += !CONVERTED(*descp); ++ *descp |= JIT_DESC_CONVERT; ++ } ++ ++done: ++ *jit_off = offset; ++ *jit_op = op; ++} ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ bool always = false; ++ bool never = false; ++ ++ switch (bpf_op) { ++ case BPF_JEQ: ++ case BPF_JNE: ++ break; ++ case BPF_JSET: ++ case BPF_JLT: ++ never = imm == 0; ++ break; ++ case BPF_JGE: ++ always = imm == 0; ++ break; ++ case BPF_JGT: ++ never = (u32)imm == U32_MAX; ++ break; ++ case BPF_JLE: ++ always = (u32)imm == U32_MAX; ++ break; ++ case BPF_JSGT: ++ never = imm == S32_MAX && width == 32; ++ break; ++ case BPF_JSGE: ++ always = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLT: ++ never = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLE: ++ always = imm == S32_MAX && width == 32; ++ break; ++ } ++ ++ if (never) ++ bpf_op = JIT_JNOP; ++ if (always) ++ bpf_op = BPF_JA; ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ switch (bpf_op) { ++ case BPF_JSET: ++ break; ++ case BPF_JEQ: ++ case BPF_JGE: ++ case BPF_JLE: ++ case BPF_JSGE: ++ case BPF_JSLE: ++ if (same_reg) ++ bpf_op = BPF_JA; ++ break; ++ case BPF_JNE: ++ case BPF_JLT: ++ case BPF_JGT: ++ case BPF_JSGT: ++ case BPF_JSLT: ++ if (same_reg) ++ bpf_op = JIT_JNOP; ++ break; ++ } ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off) ++{ ++ /* Emit conditional branch delay slot */ ++ if (jit_op != JIT_JNOP) ++ emit(ctx, nop); ++ /* ++ * Emit an absolute long jump with delay slot, ++ * if the PC-relative branch was converted. ++ */ ++ if (CONVERTED(ctx->descriptors[ctx->bpf_index])) { ++ int target = get_target(ctx, ctx->bpf_index + bpf_off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ } ++ return 0; ++} ++ ++/* Jump immediate (32-bit) */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst & imm */ ++ case BPF_JSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump register (32-bit) */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ case BPF_JEQ: ++ emit(ctx, beq, dst, src, off); ++ break; ++ /* PC += off if dst != src */ ++ case BPF_JNE: ++ emit(ctx, bne, dst, src, off); ++ break; ++ /* PC += off if dst & src */ ++ case BPF_JSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off) ++{ ++ int target = get_target(ctx, ctx->bpf_index + off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx) ++{ ++ int target = get_target(ctx, ctx->program->len); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Build the program body from eBPF bytecode */ ++static int build_body(struct jit_context *ctx) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ unsigned int i; ++ ++ ctx->stack_used = 0; ++ for (i = 0; i < prog->len; i++) { ++ const struct bpf_insn *insn = &prog->insnsi[i]; ++ u32 *descp = &ctx->descriptors[i]; ++ int ret; ++ ++ access_reg(ctx, insn->src_reg); ++ access_reg(ctx, insn->dst_reg); ++ ++ ctx->bpf_index = i; ++ if (ctx->target == NULL) { ++ ctx->changes += INDEX(*descp) != ctx->jit_index; ++ *descp &= JIT_DESC_CONVERT; ++ *descp |= ctx->jit_index; ++ } ++ ++ ret = build_insn(insn, ctx); ++ if (ret < 0) ++ return ret; ++ ++ if (ret > 0) { ++ i++; ++ if (ctx->target == NULL) ++ descp[1] = ctx->jit_index; ++ } ++ } ++ ++ /* Store the end offset, where the epilogue begins */ ++ ctx->descriptors[prog->len] = ctx->jit_index; ++ return 0; ++} ++ ++/* Set the branch conversion flag on all instructions */ ++static void set_convert_flag(struct jit_context *ctx, bool enable) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ u32 flag = enable ? JIT_DESC_CONVERT : 0; ++ unsigned int i; ++ ++ for (i = 0; i <= prog->len; i++) ++ ctx->descriptors[i] = INDEX(ctx->descriptors[i]) | flag; ++} ++ ++static void jit_fill_hole(void *area, unsigned int size) ++{ ++ u32 *p; ++ ++ /* We are guaranteed to have aligned memory. */ ++ for (p = area; size >= sizeof(u32); size -= sizeof(u32)) ++ uasm_i_break(&p, BRK_BUG); /* Increments p */ ++} ++ ++bool bpf_jit_needs_zext(void) ++{ ++ return true; ++} ++ ++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ++{ ++ struct bpf_prog *tmp, *orig_prog = prog; ++ struct bpf_binary_header *header = NULL; ++ struct jit_context ctx; ++ bool tmp_blinded = false; ++ unsigned int tmp_idx; ++ unsigned int image_size; ++ u8 *image_ptr; ++ int tries; ++ ++ /* ++ * If BPF JIT was not enabled then we must fall back to ++ * the interpreter. ++ */ ++ if (!prog->jit_requested) ++ return orig_prog; ++ /* ++ * If constant blinding was enabled and we failed during blinding ++ * then we must fall back to the interpreter. Otherwise, we save ++ * the new JITed code. ++ */ ++ tmp = bpf_jit_blind_constants(prog); ++ if (IS_ERR(tmp)) ++ return orig_prog; ++ if (tmp != prog) { ++ tmp_blinded = true; ++ prog = tmp; ++ } ++ ++ memset(&ctx, 0, sizeof(ctx)); ++ ctx.program = prog; ++ ++ /* ++ * Not able to allocate memory for descriptors[], then ++ * we must fall back to the interpreter ++ */ ++ ctx.descriptors = kcalloc(prog->len + 1, sizeof(*ctx.descriptors), ++ GFP_KERNEL); ++ if (ctx.descriptors == NULL) ++ goto out_err; ++ ++ /* First pass discovers used resources */ ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ /* ++ * Second pass computes instruction offsets. ++ * If any PC-relative branches are out of range, a sequence of ++ * a PC-relative branch + a jump is generated, and we have to ++ * try again from the beginning to generate the new offsets. ++ * This is done until no additional conversions are necessary. ++ * The last two iterations are done with all branches being ++ * converted, to guarantee offset table convergence within a ++ * fixed number of iterations. ++ */ ++ ctx.jit_index = 0; ++ build_prologue(&ctx); ++ tmp_idx = ctx.jit_index; ++ ++ tries = JIT_MAX_ITERATIONS; ++ do { ++ ctx.jit_index = tmp_idx; ++ ctx.changes = 0; ++ if (tries == 2) ++ set_convert_flag(&ctx, true); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ } while (ctx.changes > 0 && --tries > 0); ++ ++ if (WARN_ONCE(ctx.changes > 0, "JIT offsets failed to converge")) ++ goto out_err; ++ ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Now we know the size of the structure to make */ ++ image_size = sizeof(u32) * ctx.jit_index; ++ header = bpf_jit_binary_alloc(image_size, &image_ptr, ++ sizeof(u32), jit_fill_hole); ++ /* ++ * Not able to allocate memory for the structure then ++ * we must fall back to the interpretation ++ */ ++ if (header == NULL) ++ goto out_err; ++ ++ /* Actual pass to generate final JIT code */ ++ ctx.target = (u32 *)image_ptr; ++ ctx.jit_index = 0; ++ ++ /* ++ * If building the JITed code fails somehow, ++ * we fall back to the interpretation. ++ */ ++ build_prologue(&ctx); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Populate line info meta data */ ++ set_convert_flag(&ctx, false); ++ bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]); ++ ++ /* Set as read-only exec and flush instruction cache */ ++ bpf_jit_binary_lock_ro(header); ++ flush_icache_range((unsigned long)header, ++ (unsigned long)&ctx.target[ctx.jit_index]); ++ ++ if (bpf_jit_enable > 1) ++ bpf_jit_dump(prog->len, image_size, 2, ctx.target); ++ ++ prog->bpf_func = (void *)ctx.target; ++ prog->jited = 1; ++ prog->jited_len = image_size; ++ ++out: ++ if (tmp_blinded) ++ bpf_jit_prog_release_other(prog, prog == orig_prog ? ++ tmp : orig_prog); ++ kfree(ctx.descriptors); ++ return prog; ++ ++out_err: ++ prog = orig_prog; ++ if (header) ++ bpf_jit_binary_free(header); ++ goto out; ++} +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -0,0 +1,211 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#ifndef _BPF_JIT_COMP_H ++#define _BPF_JIT_COMP_H ++ ++/* MIPS registers */ ++#define MIPS_R_ZERO 0 /* Const zero */ ++#define MIPS_R_AT 1 /* Asm temp */ ++#define MIPS_R_V0 2 /* Result */ ++#define MIPS_R_V1 3 /* Result */ ++#define MIPS_R_A0 4 /* Argument */ ++#define MIPS_R_A1 5 /* Argument */ ++#define MIPS_R_A2 6 /* Argument */ ++#define MIPS_R_A3 7 /* Argument */ ++#define MIPS_R_A4 8 /* Arg (n64) */ ++#define MIPS_R_A5 9 /* Arg (n64) */ ++#define MIPS_R_A6 10 /* Arg (n64) */ ++#define MIPS_R_A7 11 /* Arg (n64) */ ++#define MIPS_R_T0 8 /* Temp (o32) */ ++#define MIPS_R_T1 9 /* Temp (o32) */ ++#define MIPS_R_T2 10 /* Temp (o32) */ ++#define MIPS_R_T3 11 /* Temp (o32) */ ++#define MIPS_R_T4 12 /* Temporary */ ++#define MIPS_R_T5 13 /* Temporary */ ++#define MIPS_R_T6 14 /* Temporary */ ++#define MIPS_R_T7 15 /* Temporary */ ++#define MIPS_R_S0 16 /* Saved */ ++#define MIPS_R_S1 17 /* Saved */ ++#define MIPS_R_S2 18 /* Saved */ ++#define MIPS_R_S3 19 /* Saved */ ++#define MIPS_R_S4 20 /* Saved */ ++#define MIPS_R_S5 21 /* Saved */ ++#define MIPS_R_S6 22 /* Saved */ ++#define MIPS_R_S7 23 /* Saved */ ++#define MIPS_R_T8 24 /* Temporary */ ++#define MIPS_R_T9 25 /* Temporary */ ++/* MIPS_R_K0 26 Reserved */ ++/* MIPS_R_K1 27 Reserved */ ++#define MIPS_R_GP 28 /* Global ptr */ ++#define MIPS_R_SP 29 /* Stack ptr */ ++#define MIPS_R_FP 30 /* Frame ptr */ ++#define MIPS_R_RA 31 /* Return */ ++ ++/* ++ * Jump address mask for immediate jumps. The four most significant bits ++ * must be equal to PC. ++ */ ++#define MIPS_JMP_MASK 0x0fffffffUL ++ ++/* Maximum number of iterations in offset table computation */ ++#define JIT_MAX_ITERATIONS 8 ++ ++/* ++ * Jump pseudo-instructions used internally ++ * for branch conversion and branch optimization. ++ */ ++#define JIT_JNSET 0xe0 ++#define JIT_JNOP 0xf0 ++ ++/* Descriptor flag for PC-relative branch conversion */ ++#define JIT_DESC_CONVERT BIT(31) ++ ++/* JIT context for an eBPF program */ ++struct jit_context { ++ struct bpf_prog *program; /* The eBPF program being JITed */ ++ u32 *descriptors; /* eBPF to JITed CPU insn descriptors */ ++ u32 *target; /* JITed code buffer */ ++ u32 bpf_index; /* Index of current BPF program insn */ ++ u32 jit_index; /* Index of current JIT target insn */ ++ u32 changes; /* Number of PC-relative branch conv */ ++ u32 accessed; /* Bit mask of read eBPF registers */ ++ u32 clobbered; /* Bit mask of modified CPU registers */ ++ u32 stack_size; /* Total allocated stack size in bytes */ ++ u32 saved_size; /* Size of callee-saved registers */ ++ u32 stack_used; /* Stack size used for function calls */ ++}; ++ ++/* Emit the instruction if the JIT memory space has been allocated */ ++#define emit(ctx, func, ...) \ ++do { \ ++ if ((ctx)->target != NULL) { \ ++ u32 *p = &(ctx)->target[ctx->jit_index]; \ ++ uasm_i_##func(&p, ##__VA_ARGS__); \ ++ } \ ++ (ctx)->jit_index++; \ ++} while (0) ++ ++/* ++ * Mark a BPF register as accessed, it needs to be ++ * initialized by the program if expected, e.g. FP. ++ */ ++static inline void access_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->accessed |= BIT(reg); ++} ++ ++/* ++ * Mark a CPU register as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static inline void clobber_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->clobbered |= BIT(reg); ++} ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc); ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off); ++ ++/* dst = imm (32-bit) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); ++ ++/* dst = src (32-bit) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); ++ ++/* Validate ALU/ALU64 immediate range */ ++bool valid_alu_i(u8 op, s32 imm); ++ ++/* Rewrite ALU/ALU64 immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off); ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width); ++ ++/* Validate JMP/JMP32 immediate range */ ++bool valid_jmp_i(u8 op, s32 imm); ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off); ++ ++/* Conditional JMP/JMP32 immediate */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); ++ ++/* Conditional JMP/JMP32 register */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off); ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx); ++ ++/* ++ * Build program prologue to set up the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_prologue(struct jit_context *ctx); ++ ++/* ++ * Build the program epilogue to restore the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_epilogue(struct jit_context *ctx, int dest_reg); ++ ++/* ++ * Convert an eBPF instruction to native instruction, i.e ++ * JITs an eBPF instruction. ++ * Returns : ++ * 0 - Successfully JITed an 8-byte eBPF instruction ++ * >0 - Successfully JITed a 16-byte eBPF instruction ++ * <0 - Failed to JIT. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx); ++ ++#endif /* _BPF_JIT_COMP_H */ +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp32.c +@@ -0,0 +1,1741 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 32-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS a4-a7 are not available in the o32 ABI */ ++#undef MIPS_R_A4 ++#undef MIPS_R_A5 ++#undef MIPS_R_A6 ++#undef MIPS_R_A7 ++ ++/* Stack is 8-byte aligned in o32 ABI */ ++#define MIPS_STACK_ALIGNMENT 8 ++ ++/* ++ * The top 16 bytes of a stack frame is reserved for the callee in O32 ABI. ++ * This corresponds to stack space for register arguments a0-a3. ++ */ ++#define JIT_RESERVED_STACK 16 ++ ++/* Temporary 64-bit register used by JIT */ ++#define JIT_REG_TMP MAX_BPF_JIT_REG ++ ++/* ++ * Number of prologue bytes to skip when doing a tail call. ++ * Tail call count (TCC) initialization (8 bytes) always, plus ++ * R0-to-v0 assignment (4 bytes) if big endian. ++ */ ++#ifdef __BIG_ENDIAN ++#define JIT_TCALL_SKIP 12 ++#else ++#define JIT_TCALL_SKIP 8 ++#endif ++ ++/* CPU registers holding the callee return value */ ++#define JIT_RETURN_REGS \ ++ (BIT(MIPS_R_V0) | \ ++ BIT(MIPS_R_V1)) ++ ++/* CPU registers arguments passed to callee directly */ ++#define JIT_ARG_REGS \ ++ (BIT(MIPS_R_A0) | \ ++ BIT(MIPS_R_A1) | \ ++ BIT(MIPS_R_A2) | \ ++ BIT(MIPS_R_A3)) ++ ++/* CPU register arguments passed to callee on stack */ ++#define JIT_STACK_REGS \ ++ (BIT(MIPS_R_T0) | \ ++ BIT(MIPS_R_T1) | \ ++ BIT(MIPS_R_T2) | \ ++ BIT(MIPS_R_T3) | \ ++ BIT(MIPS_R_T4) | \ ++ BIT(MIPS_R_T5)) ++ ++/* Caller-saved CPU registers */ ++#define JIT_CALLER_REGS \ ++ (JIT_RETURN_REGS | \ ++ JIT_ARG_REGS | \ ++ JIT_STACK_REGS) ++ ++/* Callee-saved CPU registers */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* ++ * Mapping of 64-bit eBPF registers to 32-bit native MIPS registers. ++ * ++ * 1) Native register pairs are ordered according to CPU endiannes, following ++ * the MIPS convention for passing 64-bit arguments and return values. ++ * 2) The eBPF return value, arguments and callee-saved registers are mapped ++ * to their native MIPS equivalents. ++ * 3) Since the 32 highest bits in the eBPF FP register are always zero, ++ * only one general-purpose register is actually needed for the mapping. ++ * We use the fp register for this purpose, and map the highest bits to ++ * the MIPS register r0 (zero). ++ * 4) We use the MIPS gp and at registers as internal temporary registers ++ * for constant blinding. The gp register is callee-saved. ++ * 5) One 64-bit temporary register is mapped for use when sign-extending ++ * immediate operands. MIPS registers t6-t9 are available to the JIT ++ * for as temporaries when implementing complex 64-bit operations. ++ * ++ * With this scheme all eBPF registers are being mapped to native MIPS ++ * registers without having to use any stack scratch space. The direct ++ * register mapping (2) simplifies the handling of function calls. ++ */ ++static const u8 bpf2mips32[][2] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = {MIPS_R_V1, MIPS_R_V0}, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = {MIPS_R_A1, MIPS_R_A0}, ++ [BPF_REG_2] = {MIPS_R_A3, MIPS_R_A2}, ++ /* Remaining arguments, to be passed on the stack per O32 ABI */ ++ [BPF_REG_3] = {MIPS_R_T1, MIPS_R_T0}, ++ [BPF_REG_4] = {MIPS_R_T3, MIPS_R_T2}, ++ [BPF_REG_5] = {MIPS_R_T5, MIPS_R_T4}, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = {MIPS_R_S1, MIPS_R_S0}, ++ [BPF_REG_7] = {MIPS_R_S3, MIPS_R_S2}, ++ [BPF_REG_8] = {MIPS_R_S5, MIPS_R_S4}, ++ [BPF_REG_9] = {MIPS_R_S7, MIPS_R_S6}, ++ /* Read-only frame pointer to access the eBPF stack */ ++#ifdef __BIG_ENDIAN ++ [BPF_REG_FP] = {MIPS_R_FP, MIPS_R_ZERO}, ++#else ++ [BPF_REG_FP] = {MIPS_R_ZERO, MIPS_R_FP}, ++#endif ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = {MIPS_R_GP, MIPS_R_AT}, ++ /* Temporary register for internal JIT use */ ++ [JIT_REG_TMP] = {MIPS_R_T7, MIPS_R_T6}, ++}; ++ ++/* Get low CPU register for a 64-bit eBPF register mapping */ ++static inline u8 lo(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[0]; ++#else ++ return reg[1]; ++#endif ++} ++ ++/* Get high CPU register for a 64-bit eBPF register mapping */ ++static inline u8 hi(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[1]; ++#else ++ return reg[0]; ++#endif ++} ++ ++/* ++ * Mark a 64-bit CPU register pair as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static void clobber_reg64(struct jit_context *ctx, const u8 reg[]) ++{ ++ clobber_reg(ctx, reg[0]); ++ clobber_reg(ctx, reg[1]); ++} ++ ++/* dst = imm (sign-extended) */ ++static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ emit_mov_i(ctx, lo(dst), imm); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ else ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) ++{ ++ if (!ctx->program->aux->verifier_zext) { ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ } ++} ++ ++/* Load delay slot, if ISA mandates it */ ++static void emit_load_delay(struct jit_context *ctx) ++{ ++ if (!cpu_has_mips_2_3_4_5_r) ++ emit(ctx, nop); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, ++ const u8 dst[], s32 imm, u8 op) ++{ ++ u8 src = MIPS_R_T6; ++ ++ /* ++ * ADD/SUB with all but the max negative imm can be handled by ++ * inverting the operation and the imm value, saving one insn. ++ */ ++ if (imm > S32_MIN && imm < 0) ++ switch (op) { ++ case BPF_ADD: ++ op = BPF_SUB; ++ imm = -imm; ++ break; ++ case BPF_SUB: ++ op = BPF_ADD; ++ imm = -imm; ++ break; ++ } ++ ++ /* Move immediate to temporary register */ ++ emit_mov_i(ctx, src, imm); ++ ++ switch (op) { ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addu, lo(dst), lo(dst), src); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, subu, lo(dst), lo(dst), src); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), 1); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), src); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), src); ++ if (imm >= 0) ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), src); ++ if (imm < 0) { ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst + src */ ++ case BPF_ADD: ++ if (src == dst) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 31); ++ emit(ctx, addu, lo(dst), lo(dst), lo(dst)); ++ } else { ++ emit(ctx, addu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ } ++ emit(ctx, addu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ emit(ctx, subu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, subu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), lo(src)); ++ emit(ctx, or, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), lo(src)); ++ emit(ctx, and, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), lo(src)); ++ emit(ctx, xor, hi(dst), hi(dst), hi(src)); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU invert (64-bit) */ ++static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) ++{ ++ emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift immediate (64-bit) */ ++static void emit_shift_i64(struct jit_context *ctx, ++ const u8 dst[], u32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ if (imm < 32) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); ++ emit(ctx, sll, lo(dst), lo(dst), imm); ++ emit(ctx, sll, hi(dst), hi(dst), imm); ++ emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sll, hi(dst), lo(dst), imm - 32); ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, srl, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, srl, lo(dst), hi(dst), imm - 32); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, sra, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sra, lo(dst), hi(dst), imm - 32); ++ emit(ctx, sra, hi(dst), hi(dst), 31); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift register (64-bit) */ ++static void emit_shift_r64(struct jit_context *ctx, ++ const u8 dst[], u8 src, u8 op) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ ++ emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */ ++ emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */ ++ emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */ ++ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ ++ emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */ ++ emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ ++ emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ ++ emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ ++ emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ ++ emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ ++ emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ } ++ ++ /* +20: Done */ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU mul immediate (64x32-bit) */ ++static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ u8 src = MIPS_R_T6; ++ u8 tmp = MIPS_R_T9; ++ ++ switch (imm) { ++ /* dst = dst * 1 is a no-op */ ++ case 1: ++ break; ++ /* dst = dst * -1 */ ++ case -1: ++ emit_neg_i64(ctx, dst); ++ break; ++ case 0: ++ emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); ++ emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Full 64x32 multiply */ ++ default: ++ /* hi(dst) = hi(dst) * src(imm) */ ++ emit_mov_i(ctx, src, imm); ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, hi(dst), hi(dst), src); ++ } else { ++ emit(ctx, multu, hi(dst), src); ++ emit(ctx, mflo, hi(dst)); ++ } ++ ++ /* hi(dst) = hi(dst) - lo(dst) */ ++ if (imm < 0) ++ emit(ctx, subu, hi(dst), hi(dst), lo(dst)); ++ ++ /* tmp = lo(dst) * src(imm) >> 32 */ ++ /* lo(dst) = lo(dst) * src(imm) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), src); ++ emit(ctx, mulu, lo(dst), lo(dst), src); ++ } else { ++ emit(ctx, multu, lo(dst), src); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) += tmp */ ++ emit(ctx, addu, hi(dst), hi(dst), tmp); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* ALU mul register (64x64-bit) */ ++static void emit_mul_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[]) ++{ ++ u8 acc = MIPS_R_T8; ++ u8 tmp = MIPS_R_T9; ++ ++ /* acc = hi(dst) * lo(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, acc, hi(dst), lo(src)); ++ } else { ++ emit(ctx, multu, hi(dst), lo(src)); ++ emit(ctx, mflo, acc); ++ } ++ ++ /* tmp = lo(dst) * hi(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, tmp, lo(dst), hi(src)); ++ } else { ++ emit(ctx, multu, lo(dst), hi(src)); ++ emit(ctx, mflo, tmp); ++ } ++ ++ /* acc += tmp */ ++ emit(ctx, addu, acc, acc, tmp); ++ ++ /* tmp = lo(dst) * lo(src) >> 32 */ ++ /* lo(dst) = lo(dst) * lo(src) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), lo(src)); ++ emit(ctx, mulu, lo(dst), lo(dst), lo(src)); ++ } else { ++ emit(ctx, multu, lo(dst), lo(src)); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) = acc + tmp */ ++ emit(ctx, addu, hi(dst), acc, tmp); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Helper function for 64-bit modulo */ ++static u64 jit_mod64(u64 a, u64 b) ++{ ++ u64 rem; ++ ++ div64_u64_rem(a, b, &rem); ++ return rem; ++} ++ ++/* ALU div/mod register (64-bit) */ ++static void emit_divmod_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ ++ int exclude, k; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ ++ /* Put 64-bit arguments 1 and 2 in registers a0-a3 */ ++ for (k = 0; k < 2; k++) { ++ emit(ctx, move, MIPS_R_T9, src[k]); ++ emit(ctx, move, r1[k], dst[k]); ++ emit(ctx, move, r2[k], MIPS_R_T9); ++ } ++ ++ /* Emit function call */ ++ switch (BPF_OP(op)) { ++ /* dst = dst / src */ ++ case BPF_DIV: ++ addr = (u32)&div64_u64; ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ addr = (u32)&jit_mod64; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Store the 64-bit result in dst */ ++ emit(ctx, move, dst[0], r0[0]); ++ emit(ctx, move, dst[1], r0[1]); ++ ++ /* Restore caller-saved registers, excluding the computed result */ ++ exclude = BIT(lo(dst)) | BIT(hi(dst)); ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ ++ clobber_reg64(ctx, dst); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Swap bytes in a register word */ ++static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap half words in a register word */ ++static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */ ++ emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, rotr, tmp, hi(dst), 16); ++ emit(ctx, rotr, hi(dst), lo(dst), 16); ++ emit(ctx, wsbh, lo(dst), tmp); ++ emit(ctx, wsbh, hi(dst), hi(dst)); ++ } else { ++ emit_swap16_r(ctx, tmp, lo(dst)); ++ emit_swap16_r(ctx, lo(dst), hi(dst)); ++ emit(ctx, move, hi(dst), tmp); ++ ++ emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */ ++ emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */ ++ emit_swap8_r(ctx, lo(dst), lo(dst), tmp); ++ emit_swap8_r(ctx, hi(dst), hi(dst), tmp); ++ } ++ break; ++ /* Swap bytes in a word */ ++ /* Swap bytes in a half word */ ++ case 32: ++ case 16: ++ emit_bswap_r(ctx, lo(dst), width); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ emit(ctx, andi, lo(dst), lo(dst), 0xffff); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, ++ const u8 dst[], u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lw, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ if (dst[1] == src) { ++ emit(ctx, lw, dst[0], off + 4, src); ++ emit(ctx, lw, dst[1], off, src); ++ } else { ++ emit(ctx, lw, dst[1], off, src); ++ emit(ctx, lw, dst[0], off + 4, src); ++ } ++ emit_load_delay(ctx); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, ++ const u8 dst, const u8 src[], s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, lo(src), off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, lo(src), off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, lo(src), off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sw, src[1], off, dst); ++ emit(ctx, sw, src[0], off + 4, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write (32-bit, non-ll/sc fallback) */ ++static void emit_atomic_r32(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: dst+off if xchg, otherwise src, passed in register a0 ++ * Argument 2: src if xchg, othersize dst+off, passed in register a1 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, MIPS_R_A0, src); ++ emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Atomic read-modify-write (64-bit) */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, const u8 src[], s16 off, u8 code) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: 64-bit src, passed in registers a0-a1 ++ * Argument 2: 32-bit dst+off, passed in register a2 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, r1[0], src[0]); ++ emit(ctx, move, r1[1], src[1]); ++ emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic64_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic64_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic64_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic64_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic64_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* ++ * Conditional movz or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* ++ * Conditional movn or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* Emulation of 64-bit sltiu rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_sltiu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ if (imm < 0) { ++ emit_mov_i(ctx, rd, imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++ } else { /* imm >= 0 */ ++ if (imm > 0x7fff) { ++ emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ } else { ++ emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */ ++ } ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */ ++ } ++} ++ ++/* Emulation of 64-bit sltu rd, rs, rt */ ++static void emit_sltu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */ ++ emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */ ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */ ++ emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++} ++ ++/* Emulation of 64-bit slti rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_slti_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ u8 cmp; ++ ++ /* ++ * if ((rs < 0) ^ (imm < 0)) t1 = imm >u rsl ++ * else t1 = rsl > 31 */ ++ if (imm < 0) ++ emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */ ++ else ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ /* ++ * if ((imm < 0 && rsh != 0xffffffff) || ++ * (imm >= 0 && rsh != 0)) ++ * t1 = 0 ++ */ ++ if (imm < 0) { ++ emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */ ++ cmp = rd; ++ } else { /* imm >= 0 */ ++ cmp = hi(rs); ++ } ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */ ++ ++ /* ++ * if (imm < 0) rd = rsh < -1 ++ * else rd = rsh != 0 ++ * rd = rd | t1 ++ */ ++ emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */ ++ emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ ++} ++ ++/* Emulation of 64-bit(slt rd, rs, rt) */ ++static void emit_slt_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 t1 = MIPS_R_T7; ++ u8 t2 = MIPS_R_T8; ++ u8 t3 = MIPS_R_T9; ++ ++ /* ++ * if ((rs < 0) ^ (rt < 0)) t1 = rtl > 31 */ ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */ ++ ++ /* rd = (rsh < rth) | t1 */ ++ emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh = -0x7fff && imm <= 0x8000) { ++ emit(ctx, addiu, tmp, lo(dst), -imm); ++ } else if ((u32)imm <= 0xffff) { ++ emit(ctx, xori, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, xor, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) { /* Compare sign extension */ ++ emit(ctx, addu, MIPS_R_T9, hi(dst), 1); ++ emit(ctx, or, tmp, tmp, MIPS_R_T9); ++ } else { /* Compare zero extension */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ } ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, tmp, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst & imm */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ if ((u32)imm <= 0xffff) { ++ emit(ctx, andi, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, and, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) /* Sign-extension pulls in high word */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, tmp, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ } ++} ++ ++/* Jump register (64-bit) */ ++static void emit_jmp_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], s32 off, u8 op) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ case BPF_JEQ: ++ case BPF_JNE: ++ emit(ctx, subu, t1, lo(dst), lo(src)); ++ emit(ctx, subu, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, t1, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst & src */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ emit(ctx, and, t1, lo(dst), lo(src)); ++ emit(ctx, and, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, t1, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ } ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push stack arguments */ ++ push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK); ++ ++ /* Emit function call */ ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = lo(bpf2mips32[BPF_REG_2]); ++ u8 ind = lo(bpf2mips32[BPF_REG_3]); ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ * stack[sz] - remaining tail call count, initialized in prologue ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */ ++ emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */ ++ /* (next insn delay slot) */ ++ /* if (TCC-- <= 0) goto out */ ++ emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 < 0 */ ++ emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */ ++ emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */ ++ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */ ++ emit(ctx, addu, t1, t1, ary); /* t1 += ary */ ++ emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, t1); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Caller's stack frame : ++ * :----------------------------: ++ * : 64-bit eBPF args r3-r5 : ++ * :----------------------------: ++ * : Reserved / tail call count : ++ * +============================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +----------------------------+ <--- eBPF FP (MIPS zero,fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +----------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * +----------------------------+ ++ * | Reserved for 64-bit eBPF | ++ * | args r3-r5 & args passed | ++ * | on stack in kernel calls | ++ * Lower address +============================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; ++ const u8 *fp = bpf2mips32[BPF_REG_FP]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first two instructions initialize TCC in the reserved (for us) ++ * 16-byte area in the parent's stack frame. On a tail call, the ++ * calling function jumps into the prologue after these instructions. ++ */ ++ emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, ++ min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); ++ ++ /* ++ * Register eBPF R1 contains the 32-bit context pointer argument. ++ * A 32-bit argument is always passed in MIPS register a0, regardless ++ * of CPU endianness. Initialize R1 accordingly and zero-extend. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, lo(r1), MIPS_R_A0); ++#endif ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* Zero-extend the 32-bit argument */ ++ emit(ctx, move, hi(r1), MIPS_R_ZERO); ++ ++ /* If the eBPF frame pointer was accessed it must be saved */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg64(ctx, fp); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers and function arguments passed on the stack. ++ * The required space is computed automatically during resource ++ * usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved); ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ /* ++ * A 32-bit return value is always passed in MIPS register v0, ++ * but on big-endian targets the low part of R0 is mapped to v1. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, MIPS_R_V0, MIPS_R_V1); ++#endif ++ ++ /* Jump to the return address and adjust the stack pointer */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ const u8 *dst = bpf2mips32[insn->dst_reg]; ++ const u8 *src = bpf2mips32[insn->src_reg]; ++ const u8 *tmp = bpf2mips32[JIT_REG_TMP]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_mov_i(ctx, hi(dst), 0); ++ } else { ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_alu_i(ctx, lo(dst), 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, lo(dst), val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_se_i64(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_mov_r(ctx, hi(dst), hi(src)); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_neg_i64(ctx, dst); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ if (imm) ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ if (imm) ++ emit_shift_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst * imm (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ emit_mul_i64(ctx, dst, imm); ++ break; ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ /* ++ * Sign-extend the immediate value into a temporary register, ++ * and then do the operation on this register. ++ */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); ++ break; ++ /* dst = dst * src (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ emit_mul_r64(ctx, dst, src); ++ break; ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_divmod_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_mov_i(ctx, hi(dst), insn[1].imm); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ switch (BPF_SIZE(code)) { ++ case BPF_DW: ++ /* Sign-extend immediate value into temporary reg */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ break; ++ case BPF_W: ++ case BPF_H: ++ case BPF_B: ++ emit_mov_i(ctx, lo(tmp), imm); ++ break; ++ } ++ emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (cpu_has_llsc) ++ emit_atomic_r(ctx, lo(dst), lo(src), off, imm); ++ else /* Non-ll/sc fallback */ ++ emit_atomic_r32(ctx, lo(dst), lo(src), ++ off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* Atomics (64-bit) */ ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ emit_atomic_r64(ctx, lo(dst), src, off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, lo(dst), imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r64(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_i64(ctx, dst, imm, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.4/071-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch b/target/linux/generic/backport-5.4/071-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch new file mode 100644 index 0000000000..38b46c0b76 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch @@ -0,0 +1,1005 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:05 +0200 +Subject: [PATCH] mips: bpf: Add new eBPF JIT for 64-bit MIPS + +This is an implementation on of an eBPF JIT for 64-bit MIPS III-V and +MIPS64r1-r6. It uses the same framework introduced by the 32-bit JIT. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp64.c + +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -0,0 +1,991 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS t0-t3 are not available in the n64 ABI */ ++#undef MIPS_R_T0 ++#undef MIPS_R_T1 ++#undef MIPS_R_T2 ++#undef MIPS_R_T3 ++ ++/* Stack is 16-byte aligned in n64 ABI */ ++#define MIPS_STACK_ALIGNMENT 16 ++ ++/* Extra 64-bit eBPF registers used by JIT */ ++#define JIT_REG_TC (MAX_BPF_JIT_REG + 0) ++#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1) ++ ++/* Number of prologue bytes to skip when doing a tail call */ ++#define JIT_TCALL_SKIP 4 ++ ++/* Callee-saved CPU registers that the JIT must preserve */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* Caller-saved CPU registers available for JIT use */ ++#define JIT_CALLER_REGS \ ++ (BIT(MIPS_R_A5) | \ ++ BIT(MIPS_R_A6) | \ ++ BIT(MIPS_R_A7)) ++/* ++ * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers. ++ * MIPS registers t4 - t7 may be used by the JIT as temporary registers. ++ * MIPS registers t8 - t9 are reserved for single-register common functions. ++ */ ++static const u8 bpf2mips64[] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = MIPS_R_V0, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = MIPS_R_A0, ++ [BPF_REG_2] = MIPS_R_A1, ++ [BPF_REG_3] = MIPS_R_A2, ++ [BPF_REG_4] = MIPS_R_A3, ++ [BPF_REG_5] = MIPS_R_A4, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = MIPS_R_S0, ++ [BPF_REG_7] = MIPS_R_S1, ++ [BPF_REG_8] = MIPS_R_S2, ++ [BPF_REG_9] = MIPS_R_S3, ++ /* Read-only frame pointer to access the eBPF stack */ ++ [BPF_REG_FP] = MIPS_R_FP, ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = MIPS_R_AT, ++ /* Tail call count register, caller-saved */ ++ [JIT_REG_TC] = MIPS_R_A5, ++ /* Constant for register zero-extension */ ++ [JIT_REG_ZX] = MIPS_R_V1, ++}; ++ ++/* ++ * MIPS 32-bit operations on 64-bit registers generate a sign-extended ++ * result. However, the eBPF ISA mandates zero-extension, so we rely on the ++ * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic ++ * operations, right shift and byte swap require properly sign-extended ++ * operands or the result is unpredictable. We emit explicit sign-extensions ++ * in those cases. ++ */ ++ ++/* Sign extension */ ++static void emit_sext(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, sll, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension */ ++static void emit_zext(struct jit_context *ctx, u8 dst) ++{ ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); ++ } else { ++ emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]); ++ access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */ ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, u8 dst) ++{ ++ if (!ctx->program->aux->verifier_zext) ++ emit_zext(ctx, dst); ++} ++ ++/* dst = imm (64-bit) */ ++static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) ++{ ++ if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) { ++ emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64); ++ } else if (imm64 >= 0xffffffff80000000ULL || ++ (imm64 < 0x80000000 && imm64 > 0xffff)) { ++ emit(ctx, lui, dst, (s16)(imm64 >> 16)); ++ emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff); ++ } else { ++ u8 acc = MIPS_R_ZERO; ++ int k; ++ ++ for (k = 0; k < 4; k++) { ++ u16 half = imm64 >> (48 - 16 * k); ++ ++ if (acc == dst) ++ emit(ctx, dsll, dst, dst, 16); ++ ++ if (half) { ++ emit(ctx, ori, dst, acc, half); ++ acc = dst; ++ } ++ } ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, dsubu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, dsll_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, dsrl_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsra_safe, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, daddiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, daddiu, dst, dst, -imm); ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_i(ctx, dst, imm, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, dsllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, dsrlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsrav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, daddu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, dsubu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmulu, dst, dst, src); ++ } else { ++ emit(ctx, dmultu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips64r6) { ++ emit(ctx, ddivu_r6, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmodu, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_r(ctx, dst, src, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Swap sub words in a register double word */ ++static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */ ++ emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */ ++ emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & mask */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dsbh, dst, dst); ++ emit(ctx, dshd, dst, dst); ++ } else { ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */ ++ emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */ ++ emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */ ++ ++ emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff); ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */ ++ ++ emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */ ++ emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */ ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ /* Swap bytes in a word */ ++ case 32: ++ case 16: ++ emit_sext(ctx, dst, dst); ++ emit_bswap_r(ctx, dst, width); ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) ++ emit_zext(ctx, dst); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit_zext(ctx, dst); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, andi, dst, dst, 0xffff); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, dst, off, src); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, dst, off, src); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lwu, dst, off, src); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ emit(ctx, ld, dst, off, src); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, src, off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, src, off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, src, off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sd, src, off, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, lld, t1, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, daddu, t2, t1, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, t2, t1, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, t2, t1, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, t2, t1, src); ++ break; ++ } ++ emit(ctx, scd, t2, off, dst); ++ emit(ctx, beqz, t2, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ u8 tmp = MIPS_R_T6; ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Emit function call */ ++ emit_mov_i64(ctx, tmp, addr); ++ emit(ctx, jalr, MIPS_R_RA, tmp); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Re-initialize the JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = bpf2mips64[BPF_REG_2]; ++ u8 ind = bpf2mips64[BPF_REG_3]; ++ u8 tcc = bpf2mips64[JIT_REG_TC]; ++ u8 tmp = MIPS_R_T6; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/ ++ emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ ++ /* if (--TCC < 0) goto out */ ++ emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */ ++ emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */ ++ /* (next insn delay slot) */ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */ ++ emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */ ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, tmp); ++ access_reg(ctx, JIT_REG_TC); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Previous stack frame : ++ * +===========================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +---------------------------+ <--- eBPF FP (MIPS fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +---------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * Lower address +===========================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ u8 fp = bpf2mips64[BPF_REG_FP]; ++ u8 tc = bpf2mips64[JIT_REG_TC]; ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first instruction initializes the tail call count register. ++ * On a tail call, the calling function jumps into the prologue ++ * after this instruction. ++ */ ++ emit(ctx, addiu, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* ++ * If the eBPF frame pointer and tail call count registers were ++ * accessed they must be preserved. Mark them as clobbered here ++ * to save and restore them on the stack as needed. ++ */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg(ctx, fp); ++ if (ctx->accessed & BIT(JIT_REG_TC)) ++ clobber_reg(ctx, tc); ++ if (ctx->accessed & BIT(JIT_REG_ZX)) ++ clobber_reg(ctx, zx); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers needed by the JIT. The required space is ++ * computed automatically during resource usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ if (stack) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved); ++ ++ /* Initialize the ePF JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ ++ /* Release the stack frame */ ++ if (ctx->stack_size) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++ ++ /* Jump to return address and sign-extend the 32-bit return value */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */ ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ u8 dst = bpf2mips64[insn->dst_reg]; ++ u8 src = bpf2mips64[insn->src_reg]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_zext(ctx, dst); ++ } else { ++ emit_mov_r(ctx, dst, src); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_sext(ctx, dst, dst); ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ emit_alu_r(ctx, dst, src, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_sext(ctx, dst, dst); ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, dst, src); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_alu_i64(ctx, dst, 0, BPF_NEG); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm ((64-bit, arithmetic) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ /* dst = dst * imm (64-bit) */ ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i64(ctx, dst, val, alu); ++ } ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ /* dst = dst * src (64-bit) */ ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32)); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (BPF_SIZE(code) == BPF_DW) { ++ emit_atomic_r64(ctx, dst, src, off, imm); ++ } else { /* 32-bit, no fetch */ ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm); ++ } ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */ ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register, sign-extended */ ++ emit_mov_i(ctx, MIPS_R_T5, imm); ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, dst, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch b/target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch new file mode 100644 index 0000000000..63553ebe58 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch @@ -0,0 +1,120 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:06 +0200 +Subject: [PATCH] mips: bpf: Add JIT workarounds for CPU errata + +This patch adds workarounds for the following CPU errata to the MIPS +eBPF JIT, if enabled in the kernel configuration. + + - R10000 ll/sc weak ordering + - Loongson-3 ll/sc weak ordering + - Loongson-2F jump hang + +The Loongson-2F nop errata is implemented in uasm, which the JIT uses, +so no additional mitigations are needed for that. + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/net/bpf_jit_comp.c ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, + /* Atomic read-modify-write (32-bit) */ + void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + switch (code) { + case BPF_ADD: +@@ -420,18 +421,19 @@ void emit_atomic_r(struct jit_context *c + break; + } + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + + /* Atomic compare-and-exchange (32-bit) */ + void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + emit(ctx, bne, MIPS_R_T9, res, 12); + emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset); + emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ + clobber_reg(ctx, res); + } +--- a/arch/mips/net/bpf_jit_comp.h ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -87,7 +87,7 @@ struct jit_context { + }; + + /* Emit the instruction if the JIT memory space has been allocated */ +-#define emit(ctx, func, ...) \ ++#define __emit(ctx, func, ...) \ + do { \ + if ((ctx)->target != NULL) { \ + u32 *p = &(ctx)->target[ctx->jit_index]; \ +@@ -95,6 +95,30 @@ do { \ + } \ + (ctx)->jit_index++; \ + } while (0) ++#define emit(...) __emit(__VA_ARGS__) ++ ++/* Workaround for R10000 ll/sc errata */ ++#ifdef CONFIG_WAR_R10000 ++#define LLSC_beqz beqzl ++#else ++#define LLSC_beqz beqz ++#endif ++ ++/* Workaround for Loongson-3 ll/sc errata */ ++#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS ++#define LLSC_sync(ctx) emit(ctx, sync, 0) ++#define LLSC_offset 4 ++#else ++#define LLSC_sync(ctx) ++#define LLSC_offset 0 ++#endif ++ ++/* Workaround for Loongson-2F jump errata */ ++#ifdef CONFIG_CPU_JUMP_WORKAROUNDS ++#define JALR_MASK 0xffffffffcfffffffULL ++#else ++#define JALR_MASK (~0ULL) ++#endif + + /* + * Mark a BPF register as accessed, it needs to be +--- a/arch/mips/net/bpf_jit_comp64.c ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_c + u8 t1 = MIPS_R_T6; + u8 t2 = MIPS_R_T7; + ++ LLSC_sync(ctx); + emit(ctx, lld, t1, off, dst); + switch (code) { + case BPF_ADD: +@@ -391,7 +392,7 @@ static void emit_atomic_r64(struct jit_c + break; + } + emit(ctx, scd, t2, off, dst); +- emit(ctx, beqz, t2, -16); ++ emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + +@@ -414,7 +415,7 @@ static int emit_call(struct jit_context + push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); + + /* Emit function call */ +- emit_mov_i64(ctx, tmp, addr); ++ emit_mov_i64(ctx, tmp, addr & JALR_MASK); + emit(ctx, jalr, MIPS_R_RA, tmp); + emit(ctx, nop); /* Delay slot */ + diff --git a/target/linux/generic/backport-5.4/071-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch b/target/linux/generic/backport-5.4/071-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch new file mode 100644 index 0000000000..c8ce0becf9 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch @@ -0,0 +1,54 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:07 +0200 +Subject: [PATCH] mips: bpf: Enable eBPF JITs + +This patch enables the new eBPF JITs for 32-bit and 64-bit MIPS. It also +disables the old cBPF JIT to so cBPF programs are converted to use the +new JIT. + +Workarounds for R4000 CPU errata are not implemented by the JIT, so the +JIT is disabled if any of those workarounds are configured. + +Signed-off-by: Johan Almbladh +--- + +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -3099,6 +3099,7 @@ S: Supported + F: arch/arm64/net/ + + BPF JIT for MIPS (32-BIT AND 64-BIT) ++M: Johan Almbladh + M: Paul Burton + L: netdev@vger.kernel.org + L: bpf@vger.kernel.org +--- a/arch/mips/Kconfig ++++ b/arch/mips/Kconfig +@@ -46,8 +46,10 @@ config MIPS + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES + select HAVE_ASM_MODVERSIONS +- select HAVE_CBPF_JIT if !64BIT && !CPU_MICROMIPS +- select HAVE_EBPF_JIT if 64BIT && !CPU_MICROMIPS && TARGET_ISA_REV >= 2 ++ select HAVE_EBPF_JIT if !CPU_MICROMIPS && \ ++ !CPU_DADDI_WORKAROUNDS && \ ++ !CPU_R4000_WORKAROUNDS && \ ++ !CPU_R4400_WORKAROUNDS + select HAVE_CONTEXT_TRACKING + select HAVE_COPY_THREAD_TLS + select HAVE_C_RECORDMCOUNT +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,9 +2,10 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o ++obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o + + ifeq ($(CONFIG_32BIT),y) +- obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp32.o + else +- obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp64.o + endif diff --git a/target/linux/generic/backport-5.4/071-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch b/target/linux/generic/backport-5.4/071-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch new file mode 100644 index 0000000000..e25c336831 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch @@ -0,0 +1,387 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:08 +0200 +Subject: [PATCH] mips: bpf: Remove old BPF JIT implementations + +This patch removes the old 32-bit cBPF and 64-bit eBPF JIT implementations. +They are replaced by a new eBPF implementation that supports both 32-bit +and 64-bit MIPS CPUs. + +Signed-off-by: Johan Almbladh +--- + delete mode 100644 arch/mips/net/bpf_jit.c + delete mode 100644 arch/mips/net/bpf_jit.h + delete mode 100644 arch/mips/net/bpf_jit_asm.S + delete mode 100644 arch/mips/net/ebpf_jit.c + +--- a/arch/mips/net/bpf_jit.h ++++ /dev/null +@@ -1,81 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0-only */ +-/* +- * Just-In-Time compiler for BPF filters on MIPS +- * +- * Copyright (c) 2014 Imagination Technologies Ltd. +- * Author: Markos Chandras +- */ +- +-#ifndef BPF_JIT_MIPS_OP_H +-#define BPF_JIT_MIPS_OP_H +- +-/* Registers used by JIT */ +-#define MIPS_R_ZERO 0 +-#define MIPS_R_V0 2 +-#define MIPS_R_A0 4 +-#define MIPS_R_A1 5 +-#define MIPS_R_T4 12 +-#define MIPS_R_T5 13 +-#define MIPS_R_T6 14 +-#define MIPS_R_T7 15 +-#define MIPS_R_S0 16 +-#define MIPS_R_S1 17 +-#define MIPS_R_S2 18 +-#define MIPS_R_S3 19 +-#define MIPS_R_S4 20 +-#define MIPS_R_S5 21 +-#define MIPS_R_S6 22 +-#define MIPS_R_S7 23 +-#define MIPS_R_SP 29 +-#define MIPS_R_RA 31 +- +-/* Conditional codes */ +-#define MIPS_COND_EQ 0x1 +-#define MIPS_COND_GE (0x1 << 1) +-#define MIPS_COND_GT (0x1 << 2) +-#define MIPS_COND_NE (0x1 << 3) +-#define MIPS_COND_ALL (0x1 << 4) +-/* Conditionals on X register or K immediate */ +-#define MIPS_COND_X (0x1 << 5) +-#define MIPS_COND_K (0x1 << 6) +- +-#define r_ret MIPS_R_V0 +- +-/* +- * Use 2 scratch registers to avoid pipeline interlocks. +- * There is no overhead during epilogue and prologue since +- * any of the $s0-$s6 registers will only be preserved if +- * they are going to actually be used. +- */ +-#define r_skb_hl MIPS_R_S0 /* skb header length */ +-#define r_skb_data MIPS_R_S1 /* skb actual data */ +-#define r_off MIPS_R_S2 +-#define r_A MIPS_R_S3 +-#define r_X MIPS_R_S4 +-#define r_skb MIPS_R_S5 +-#define r_M MIPS_R_S6 +-#define r_skb_len MIPS_R_S7 +-#define r_s0 MIPS_R_T4 /* scratch reg 1 */ +-#define r_s1 MIPS_R_T5 /* scratch reg 2 */ +-#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */ +-#define r_tmp MIPS_R_T7 /* No need to preserve this */ +-#define r_zero MIPS_R_ZERO +-#define r_sp MIPS_R_SP +-#define r_ra MIPS_R_RA +- +-#ifndef __ASSEMBLY__ +- +-/* Declare ASM helpers */ +- +-#define DECLARE_LOAD_FUNC(func) \ +- extern u8 func(unsigned long *skb, int offset); \ +- extern u8 func##_negative(unsigned long *skb, int offset); \ +- extern u8 func##_positive(unsigned long *skb, int offset) +- +-DECLARE_LOAD_FUNC(sk_load_word); +-DECLARE_LOAD_FUNC(sk_load_half); +-DECLARE_LOAD_FUNC(sk_load_byte); +- +-#endif +- +-#endif /* BPF_JIT_MIPS_OP_H */ +--- a/arch/mips/net/bpf_jit_asm.S ++++ /dev/null +@@ -1,285 +0,0 @@ +-/* +- * bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF +- * compiler. +- * +- * Copyright (C) 2015 Imagination Technologies Ltd. +- * Author: Markos Chandras +- * +- * This program is free software; you can redistribute it and/or modify it +- * under the terms of the GNU General Public License as published by the +- * Free Software Foundation; version 2 of the License. +- */ +- +-#include +-#include +-#include +-#include "bpf_jit.h" +- +-/* ABI +- * +- * r_skb_hl skb header length +- * r_skb_data skb data +- * r_off(a1) offset register +- * r_A BPF register A +- * r_X PF register X +- * r_skb(a0) *skb +- * r_M *scratch memory +- * r_skb_le skb length +- * r_s0 Scratch register 0 +- * r_s1 Scratch register 1 +- * +- * On entry: +- * a0: *skb +- * a1: offset (imm or imm + X) +- * +- * All non-BPF-ABI registers are free for use. On return, we only +- * care about r_ret. The BPF-ABI registers are assumed to remain +- * unmodified during the entire filter operation. +- */ +- +-#define skb a0 +-#define offset a1 +-#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */ +- +- /* We know better :) so prevent assembler reordering etc */ +- .set noreorder +- +-#define is_offset_negative(TYPE) \ +- /* If offset is negative we have more work to do */ \ +- slti t0, offset, 0; \ +- bgtz t0, bpf_slow_path_##TYPE##_neg; \ +- /* Be careful what follows in DS. */ +- +-#define is_offset_in_header(SIZE, TYPE) \ +- /* Reading from header? */ \ +- addiu $r_s0, $r_skb_hl, -SIZE; \ +- slt t0, $r_s0, offset; \ +- bgtz t0, bpf_slow_path_##TYPE; \ +- +-LEAF(sk_load_word) +- is_offset_negative(word) +-FEXPORT(sk_load_word_positive) +- is_offset_in_header(4, word) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- .set reorder +- lw $r_A, 0(t1) +- .set noreorder +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_A +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_A, 24 +- srl t1, $r_A, 24 +- srl t2, $r_A, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_A, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_word) +- +-LEAF(sk_load_half) +- is_offset_negative(half) +-FEXPORT(sk_load_half_positive) +- is_offset_in_header(2, half) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lhu $r_A, 0(t1) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh $r_A, $r_A +-# else +- sll t0, $r_A, 8 +- srl t1, $r_A, 8 +- andi t0, t0, 0xff00 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_half) +- +-LEAF(sk_load_byte) +- is_offset_negative(byte) +-FEXPORT(sk_load_byte_positive) +- is_offset_in_header(1, byte) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lbu $r_A, 0(t1) +- jr $r_ra +- move $r_ret, zero +- END(sk_load_byte) +- +-/* +- * call skb_copy_bits: +- * (prototype in linux/skbuff.h) +- * +- * int skb_copy_bits(sk_buff *skb, int offset, void *to, int len) +- * +- * o32 mandates we leave 4 spaces for argument registers in case +- * the callee needs to use them. Even though we don't care about +- * the argument registers ourselves, we need to allocate that space +- * to remain ABI compliant since the callee may want to use that space. +- * We also allocate 2 more spaces for $r_ra and our return register (*to). +- * +- * n64 is a bit different. The *caller* will allocate the space to preserve +- * the arguments. So in 64-bit kernels, we allocate the 4-arg space for no +- * good reason but it does not matter that much really. +- * +- * (void *to) is returned in r_s0 +- * +- */ +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-#define DS_OFFSET(SIZE) (4 * SZREG) +-#else +-#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE)) +-#endif +-#define bpf_slow_path_common(SIZE) \ +- /* Quick check. Are we within reasonable boundaries? */ \ +- LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \ +- sltu $r_s0, offset, $r_s1; \ +- beqz $r_s0, fault; \ +- /* Load 4th argument in DS */ \ +- LONG_ADDIU a3, zero, SIZE; \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, skb_copy_bits; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- /* Assign low slot to a2 */ \ +- PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \ +- jalr t0; \ +- /* Reset our destination slot (DS but it's ok) */ \ +- INT_S zero, (4 * SZREG)($r_sp); \ +- /* \ +- * skb_copy_bits returns 0 on success and -EFAULT \ +- * on error. Our data live in a2. Do not bother with \ +- * our data if an error has been returned. \ +- */ \ +- /* Restore our frame */ \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- INT_L $r_s0, (4 * SZREG)($r_sp); \ +- bltz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- move $r_ret, zero; \ +- +-NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp) +- bpf_slow_path_common(4) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_s0 +- jr $r_ra +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_s0, 24 +- srl t1, $r_s0, 24 +- srl t2, $r_s0, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_s0, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_word) +- +-NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp) +- bpf_slow_path_common(2) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- jr $r_ra +- wsbh $r_A, $r_s0 +-# else +- sll t0, $r_s0, 8 +- andi t1, $r_s0, 0xff00 +- andi t0, t0, 0xff00 +- srl t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_half) +- +-NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp) +- bpf_slow_path_common(1) +- jr $r_ra +- move $r_A, $r_s0 +- +- END(bpf_slow_path_byte) +- +-/* +- * Negative entry points +- */ +- .macro bpf_is_end_of_data +- li t0, SKF_LL_OFF +- /* Reading link layer data? */ +- slt t1, offset, t0 +- bgtz t1, fault +- /* Be careful what follows in DS. */ +- .endm +-/* +- * call skb_copy_bits: +- * (prototype in linux/filter.h) +- * +- * void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, +- * int k, unsigned int size) +- * +- * see above (bpf_slow_path_common) for ABI restrictions +- */ +-#define bpf_negative_common(SIZE) \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, bpf_internal_load_pointer_neg_helper; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- jalr t0; \ +- li a2, SIZE; \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- /* Check return pointer */ \ +- beqz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- /* Preserve our pointer */ \ +- move $r_s0, v0; \ +- /* Set return value */ \ +- move $r_ret, zero; \ +- +-bpf_slow_path_word_neg: +- bpf_is_end_of_data +-NESTED(sk_load_word_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(4) +- jr $r_ra +- lw $r_A, 0($r_s0) +- END(sk_load_word_negative) +- +-bpf_slow_path_half_neg: +- bpf_is_end_of_data +-NESTED(sk_load_half_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(2) +- jr $r_ra +- lhu $r_A, 0($r_s0) +- END(sk_load_half_negative) +- +-bpf_slow_path_byte_neg: +- bpf_is_end_of_data +-NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(1) +- jr $r_ra +- lbu $r_A, 0($r_s0) +- END(sk_load_byte_negative) +- +-fault: +- jr $r_ra +- addiu $r_ret, zero, 1 diff --git a/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch b/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch index 9e37bbb60c..a29da1e7b2 100644 --- a/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch +++ b/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch @@ -192,7 +192,7 @@ Signed-off-by: Jason A. Donenfeld --- a/MAINTAINERS +++ b/MAINTAINERS -@@ -17584,6 +17584,14 @@ L: linux-gpio@vger.kernel.org +@@ -17585,6 +17585,14 @@ L: linux-gpio@vger.kernel.org S: Maintained F: drivers/gpio/gpio-ws16c48.c diff --git a/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch b/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch index ecf86b66a6..19b81f0301 100644 --- a/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch +++ b/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch @@ -10,7 +10,7 @@ Signed-off-by: Gabor Juhos --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1165,6 +1165,10 @@ config MIPS_MSC +@@ -1167,6 +1167,10 @@ config MIPS_MSC config SYNC_R4K bool diff --git a/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch b/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch index ada65cd2a0..af6b51aa4f 100644 --- a/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch +++ b/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch @@ -10,7 +10,7 @@ Signed-off-by: Gabor Juhos --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1159,6 +1159,10 @@ config SYNC_R4K +@@ -1161,6 +1161,10 @@ config SYNC_R4K config MIPS_MACHINE def_bool n diff --git a/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch b/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch index a04864ab4d..59f399d3cf 100644 --- a/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch +++ b/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch @@ -9,7 +9,7 @@ Acked-by: Rob Landley --- --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1080,9 +1080,6 @@ config FW_ARC +@@ -1082,9 +1082,6 @@ config FW_ARC config ARCH_MAY_HAVE_PC_FDC bool @@ -19,7 +19,7 @@ Acked-by: Rob Landley config CEVT_BCM1480 bool -@@ -3172,6 +3169,18 @@ choice +@@ -3174,6 +3171,18 @@ choice bool "Extend builtin kernel arguments with bootloader arguments" endchoice diff --git a/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch b/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch index 4e7a532156..b5ec03f286 100644 --- a/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch +++ b/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch @@ -25,7 +25,7 @@ Signed-off-by: Jason A. Donenfeld --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -2037,7 +2037,8 @@ config CPU_MIPS32 +@@ -2039,7 +2039,8 @@ config CPU_MIPS32 config CPU_MIPS64 bool diff --git a/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch b/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch index 7b9ae65c60..c08d6a61f3 100644 --- a/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch +++ b/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch @@ -9,7 +9,7 @@ Acked-by: Rob Landley --- --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1069,9 +1069,6 @@ config FW_ARC +@@ -1071,9 +1071,6 @@ config FW_ARC config ARCH_MAY_HAVE_PC_FDC bool @@ -19,7 +19,7 @@ Acked-by: Rob Landley config CEVT_BCM1480 bool -@@ -3044,6 +3041,18 @@ choice +@@ -3046,6 +3043,18 @@ choice bool "Extend builtin kernel arguments with bootloader arguments" endchoice diff --git a/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch b/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch index 34617898c8..4ffbb2bebb 100644 --- a/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch +++ b/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch @@ -16,7 +16,7 @@ Signed-off-by: Thomas Bogendoerfer --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -626,6 +626,7 @@ config RALINK +@@ -628,6 +628,7 @@ config RALINK select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_LITTLE_ENDIAN select SYS_SUPPORTS_MIPS16 From 3a93704a63cf018187411108d241c35ec83992c0 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 20 Aug 2021 18:31:48 +0100 Subject: [PATCH 10/22] mediatek: add EEPROM data for BPi-R64 2.4GHz wmac EEPROM data extracted from vendor image found at http://forum.banana-pi.org/t/bpi-r64-mt7622-mac80211-wifi-driver/10246/77 http://forum.banana-pi.org/uploads/short-url/jworbyBYpvrw9VQ2sx92B9z6DWS.bin MAC address in the EEPROM has been zero'd which results in random address on boot. Signed-off-by: Daniel Golle --- .../910-dts-mt7622-bpi-r64-wifi-eeprom.patch | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 target/linux/mediatek/patches-5.10/910-dts-mt7622-bpi-r64-wifi-eeprom.patch diff --git a/target/linux/mediatek/patches-5.10/910-dts-mt7622-bpi-r64-wifi-eeprom.patch b/target/linux/mediatek/patches-5.10/910-dts-mt7622-bpi-r64-wifi-eeprom.patch new file mode 100644 index 0000000000..190096de2f --- /dev/null +++ b/target/linux/mediatek/patches-5.10/910-dts-mt7622-bpi-r64-wifi-eeprom.patch @@ -0,0 +1,31 @@ +--- a/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts ++++ b/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts +@@ -636,5 +636,28 @@ + }; + + &wmac { ++ mediatek,eeprom-data = <0x22760500 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x44000020 0x0 0x10002000 ++ 0x4400 0x4000000 0x0 0x0 ++ 0x200000b3 0x40b6c3c3 0x26000000 0x41c42600 ++ 0x41c4 0x26000000 0xc0c52600 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0xc6c6 ++ 0xc3c3c2c1 0xc300c3 0x818181 0x83c1c182 ++ 0x83838382 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x84002e00 0x90000087 0x8a000000 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0xb000009 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x7707>; ++ + status = "okay"; + }; From b118efa0d25f5b60226a9d316eb838dd6be22f78 Mon Sep 17 00:00:00 2001 From: Florian Eckert Date: Fri, 15 Oct 2021 09:19:12 +0200 Subject: [PATCH 11/22] buildsystem: add CONFIG_SECCOMP Until now, this feature was switched on via the kernel configuration option KERNEL_SECCOMP. The follwing change a7f794cd2aa104fdbd4c6e38f9b76373bf9b96e1 now requires that the package procd-seccomp must also enabled for buildinmg. However, this is not the case we have no dependency and the imagebuilder cannot build the image, because of the implicit package selection. This change adds a new configuration option CONFIG_SECCOMP. The new option has the same behaviour as the configuration option CONFIG_SELINUX. If the CONFIG_SECCOMP is selected then the package procd-seccomp and KERNEL_SECCOMP is enabled for this build. Signed-off-by: Florian Eckert Signed-off-by: Daniel Golle --- config/Config-build.in | 12 ++++++++++++ include/target.mk | 2 +- package/system/procd/Makefile | 3 +-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/config/Config-build.in b/config/Config-build.in index f0e1aaa695..ca6f513450 100644 --- a/config/Config-build.in +++ b/config/Config-build.in @@ -386,4 +386,16 @@ menu "Global build settings" endchoice + config SECCOMP + bool "Enable SECCOMP" + select KERNEL_SECCOMP + select PACKAGE_procd-seccomp + depends on (aarch64 || arm || armeb || mips || mipsel || i386 || powerpc || x86_64) + depends on !TARGET_uml + default y + help + This option enables seccomp kernel features to safely + execute untrusted bytecode and selects the seccomp-variants + of procd + endmenu diff --git a/include/target.mk b/include/target.mk index 03192d3ebe..60760bf602 100644 --- a/include/target.mk +++ b/include/target.mk @@ -39,7 +39,7 @@ DEFAULT_PACKAGES+=procd-ujail endif # include seccomp ld-preload hooks if kernel supports it -ifneq ($(CONFIG_KERNEL_SECCOMP),) +ifneq ($(CONFIG_SECCOMP),) DEFAULT_PACKAGES+=procd-seccomp endif diff --git a/package/system/procd/Makefile b/package/system/procd/Makefile index 6f506423f8..4c76045062 100644 --- a/package/system/procd/Makefile +++ b/package/system/procd/Makefile @@ -82,8 +82,7 @@ endef define Package/procd-seccomp SECTION:=base CATEGORY:=Base system - DEPENDS:=@(aarch64||arm||armeb||mips||mipsel||i386||powerpc||x86_64) @!TARGET_uml \ - @KERNEL_SECCOMP +libubox +libblobmsg-json + DEPENDS:=@SECCOMP +libubox +libblobmsg-json TITLE:=OpenWrt process seccomp helper + utrace endef From b898e869e2204ee2dba63cacbef2c1f469999b1e Mon Sep 17 00:00:00 2001 From: Luis Araneda Date: Sat, 9 Oct 2021 23:55:02 -0300 Subject: [PATCH 12/22] zynq: kernel: refresh config using "make kernel_oldconfig" Several configs are now part of generic Signed-off-by: Luis Araneda --- target/linux/zynq/config-5.4 | 96 +----------------------------------- 1 file changed, 2 insertions(+), 94 deletions(-) diff --git a/target/linux/zynq/config-5.4 b/target/linux/zynq/config-5.4 index 498d057e3a..98b2bd0f93 100644 --- a/target/linux/zynq/config-5.4 +++ b/target/linux/zynq/config-5.4 @@ -3,23 +3,6 @@ CONFIG_ALIGNMENT_TRAP=y # CONFIG_ALTERA_PR_IP_CORE is not set CONFIG_ARCH_32BIT_OFF_T=y CONFIG_ARCH_CLOCKSOURCE_DATA=y -CONFIG_ARCH_HAS_BINFMT_FLAT=y -CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y -CONFIG_ARCH_HAS_ELF_RANDOMIZE=y -CONFIG_ARCH_HAS_FORTIFY_SOURCE=y -CONFIG_ARCH_HAS_GCOV_PROFILE_ALL=y -CONFIG_ARCH_HAS_KCOV=y -CONFIG_ARCH_HAS_KEEPINITRD=y -CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y -CONFIG_ARCH_HAS_PHYS_TO_DMA=y -CONFIG_ARCH_HAS_RESET_CONTROLLER=y -CONFIG_ARCH_HAS_SETUP_DMA_OPS=y -CONFIG_ARCH_HAS_SET_MEMORY=y -CONFIG_ARCH_HAS_STRICT_KERNEL_RWX=y -CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y -CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS=y -CONFIG_ARCH_HAS_TICK_BROADCAST=y -CONFIG_ARCH_HAVE_CUSTOM_GPIO_H=y CONFIG_ARCH_HIBERNATION_POSSIBLE=y CONFIG_ARCH_KEEP_MEMBLOCK=y CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y @@ -29,25 +12,13 @@ CONFIG_ARCH_MULTI_V7=y CONFIG_ARCH_NR_GPIO=1024 CONFIG_ARCH_OPTIONAL_KERNEL_RWX=y CONFIG_ARCH_OPTIONAL_KERNEL_RWX_DEFAULT=y -CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=y -CONFIG_ARCH_SUPPORTS_BIG_ENDIAN=y -CONFIG_ARCH_SUPPORTS_UPROBES=y CONFIG_ARCH_SUSPEND_POSSIBLE=y -CONFIG_ARCH_USE_BUILTIN_BSWAP=y -CONFIG_ARCH_USE_CMPXCHG_LOCKREF=y CONFIG_ARCH_VEXPRESS=y CONFIG_ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA=y # CONFIG_ARCH_VEXPRESS_SPC is not set -CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT=y -CONFIG_ARCH_WANT_GENERAL_HUGETLB=y -CONFIG_ARCH_WANT_IPC_PARSE_VERSION=y CONFIG_ARCH_ZYNQ=y CONFIG_ARM=y CONFIG_ARM_AMBA=y -CONFIG_ARM_CCI=y -CONFIG_ARM_CCI400_COMMON=y -CONFIG_ARM_CCI400_PMU=y -CONFIG_ARM_CCI_PMU=y CONFIG_ARM_CPU_SUSPEND=y CONFIG_ARM_ERRATA_643719=y CONFIG_ARM_ERRATA_720789=y @@ -61,15 +32,11 @@ CONFIG_ARM_HAS_SG_CHAIN=y CONFIG_ARM_HEAVY_MB=y CONFIG_ARM_L1_CACHE_SHIFT=6 CONFIG_ARM_L1_CACHE_SHIFT_6=y -# CONFIG_ARM_LPAE is not set CONFIG_ARM_PATCH_IDIV=y CONFIG_ARM_PATCH_PHYS_VIRT=y # CONFIG_ARM_PL172_MPMC is not set -CONFIG_ARM_PMU=y # CONFIG_ARM_SMMU is not set -# CONFIG_ARM_SP805_WATCHDOG is not set CONFIG_ARM_THUMB=y -# CONFIG_ARM_THUMBEE is not set CONFIG_ARM_TIMER_SP804=y CONFIG_ARM_UNWIND=y CONFIG_ARM_VIRT_EXT=y @@ -88,7 +55,6 @@ CONFIG_BOUNCE=y CONFIG_CACHE_L2X0=y CONFIG_CADENCE_TTC_TIMER=y CONFIG_CADENCE_WATCHDOG=y -CONFIG_CC_HAS_KASAN_GENERIC=y CONFIG_CLKDEV_LOOKUP=y CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK=y CONFIG_CLKSRC_MMIO=y @@ -119,8 +85,6 @@ CONFIG_COREDUMP=y CONFIG_CPU_32v6K=y CONFIG_CPU_32v7=y CONFIG_CPU_ABRT_EV7=y -# CONFIG_CPU_BIG_ENDIAN is not set -# CONFIG_CPU_BPREDICT_DISABLE is not set CONFIG_CPU_CACHE_V7=y CONFIG_CPU_CACHE_VIPT=y CONFIG_CPU_COPY_V6=y @@ -138,13 +102,9 @@ CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_CPU_FREQ_STAT=y CONFIG_CPU_HAS_ASID=y -# CONFIG_CPU_HOTPLUG_STATE_CONTROL is not set -# CONFIG_CPU_ICACHE_DISABLE is not set -# CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND is not set CONFIG_CPU_IDLE=y CONFIG_CPU_IDLE_GOV_LADDER=y CONFIG_CPU_IDLE_GOV_MENU=y -# CONFIG_CPU_IDLE_GOV_TEO is not set CONFIG_CPU_PABRT_V7=y CONFIG_CPU_PM=y CONFIG_CPU_RMAP=y @@ -170,7 +130,6 @@ CONFIG_CRYPTO_NULL2=y CONFIG_CRYPTO_RNG2=y CONFIG_DCACHE_WORD_ACCESS=y CONFIG_DEBUG_LL_INCLUDE="mach/debug-macro.S" -# CONFIG_DEBUG_USER is not set CONFIG_DMADEVICES=y CONFIG_DMA_CMA=y CONFIG_DMA_ENGINE=y @@ -183,7 +142,6 @@ CONFIG_DRM_FBDEV_EMULATION=y CONFIG_DRM_FBDEV_OVERALLOC=100 CONFIG_DRM_KMS_FB_HELPER=y CONFIG_DRM_KMS_HELPER=y -# CONFIG_DRM_MCDE is not set CONFIG_DRM_PANEL=y CONFIG_DRM_PANEL_BRIDGE=y CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y @@ -231,8 +189,6 @@ CONFIG_FREEZER=y CONFIG_FS_IOMAP=y CONFIG_FS_MBCACHE=y CONFIG_FW_LOADER_PAGED_BUF=y -# CONFIG_FXOS8700_I2C is not set -# CONFIG_FXOS8700_SPI is not set CONFIG_GENERIC_ALLOCATOR=y CONFIG_GENERIC_ARCH_TOPOLOGY=y CONFIG_GENERIC_BUG=y @@ -254,6 +210,7 @@ CONFIG_GENERIC_SCHED_CLOCK=y CONFIG_GENERIC_SMP_IDLE_THREAD=y CONFIG_GENERIC_STRNCPY_FROM_USER=y CONFIG_GENERIC_STRNLEN_USER=y +CONFIG_GENERIC_TIME_VSYSCALL=y CONFIG_GLOB=y CONFIG_GPIOLIB=y CONFIG_GPIOLIB_IRQCHIP=y @@ -265,51 +222,7 @@ CONFIG_HARDEN_BRANCH_PREDICTOR=y CONFIG_HARDIRQS_SW_RESEND=y CONFIG_HAS_DMA=y CONFIG_HAS_IOMEM=y -CONFIG_HAVE_ARCH_AUDITSYSCALL=y -CONFIG_HAVE_ARCH_BITREVERSE=y -CONFIG_HAVE_ARCH_JUMP_LABEL=y -CONFIG_HAVE_ARCH_KGDB=y -CONFIG_HAVE_ARCH_PFN_VALID=y -CONFIG_HAVE_ARCH_SECCOMP_FILTER=y -CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST=y -CONFIG_HAVE_ARCH_TRACEHOOK=y -CONFIG_HAVE_ARM_SCU=y -CONFIG_HAVE_ARM_SMCCC=y -CONFIG_HAVE_ARM_TWD=y -CONFIG_HAVE_CLK=y -CONFIG_HAVE_CLK_PREPARE=y -CONFIG_HAVE_CONTEXT_TRACKING=y -CONFIG_HAVE_COPY_THREAD_TLS=y -CONFIG_HAVE_C_RECORDMCOUNT=y -CONFIG_HAVE_DEBUG_KMEMLEAK=y -CONFIG_HAVE_DMA_CONTIGUOUS=y -CONFIG_HAVE_DYNAMIC_FTRACE=y -CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y -CONFIG_HAVE_EBPF_JIT=y -CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y -CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y -CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y -CONFIG_HAVE_FUNCTION_TRACER=y -CONFIG_HAVE_HW_BREAKPOINT=y -CONFIG_HAVE_IDE=y -CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y -CONFIG_HAVE_LD_DEAD_CODE_DATA_ELIMINATION=y -CONFIG_HAVE_MOD_ARCH_SPECIFIC=y -CONFIG_HAVE_NET_DSA=y -CONFIG_HAVE_OPROFILE=y -CONFIG_HAVE_OPTPROBES=y -CONFIG_HAVE_PATA_PLATFORM=y -CONFIG_HAVE_PCI=y -CONFIG_HAVE_PERF_EVENTS=y -CONFIG_HAVE_PERF_REGS=y -CONFIG_HAVE_PERF_USER_STACK_DUMP=y -CONFIG_HAVE_PROC_CPU=y -CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y -CONFIG_HAVE_RSEQ=y CONFIG_HAVE_SMP=y -CONFIG_HAVE_SYSCALL_TRACEPOINTS=y -CONFIG_HAVE_UID16=y -CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y CONFIG_HDMI=y CONFIG_HID=y CONFIG_HID_GENERIC=y @@ -405,7 +318,6 @@ CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_OF_ARASAN=y # CONFIG_MMC_SDHCI_PCI is not set CONFIG_MMC_SDHCI_PLTFM=y -CONFIG_MODULES_TREE_LOOKUP=y CONFIG_MODULES_USE_ELF_REL=y CONFIG_MODULE_FORCE_UNLOAD=y # CONFIG_MODULE_STRIPPED is not set @@ -499,7 +411,6 @@ CONFIG_PLAT_VERSATILE=y CONFIG_PM=y CONFIG_PMBUS=y CONFIG_PM_CLK=y -# CONFIG_PM_DEBUG is not set CONFIG_PM_SLEEP=y CONFIG_PM_SLEEP_SMP=y CONFIG_POWER_RESET=y @@ -526,6 +437,7 @@ CONFIG_RPS=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_PCF8563=y CONFIG_RTC_I2C_AND_SPI=y +CONFIG_RTC_MC146818_LIB=y CONFIG_RWSEM_SPIN_ON_OWNER=y CONFIG_SCHED_MC=y CONFIG_SCHED_SMT=y @@ -533,7 +445,6 @@ CONFIG_SENSORS_PMBUS=y CONFIG_SENSORS_UCD9000=y CONFIG_SENSORS_UCD9200=y # CONFIG_SERIAL_8250 is not set -# CONFIG_SERIAL_AMBA_PL011 is not set CONFIG_SERIAL_XILINX_PS_UART=y CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y CONFIG_SERIO=y @@ -568,7 +479,6 @@ CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0 CONFIG_THERMAL_GOV_STEP_WISE=y CONFIG_THERMAL_HWMON=y CONFIG_THERMAL_OF=y -# CONFIG_THUMB2_KERNEL is not set CONFIG_TICK_CPU_ACCOUNTING=y CONFIG_TIMER_OF=y CONFIG_TIMER_PROBE=y @@ -586,14 +496,12 @@ CONFIG_UIO_PDRV_GENIRQ=y # CONFIG_UIO_SERCOS3 is not set CONFIG_UNCOMPRESS_INCLUDE="debug/uncompress.h" CONFIG_UNWINDER_ARM=y -# CONFIG_UNWINDER_FRAME_POINTER is not set CONFIG_USB=y CONFIG_USB_CHIPIDEA=y CONFIG_USB_CHIPIDEA_HOST=y CONFIG_USB_CHIPIDEA_OF=y CONFIG_USB_CHIPIDEA_UDC=y CONFIG_USB_COMMON=y -# CONFIG_USB_EHCI_FSL is not set CONFIG_USB_EHCI_HCD=y # CONFIG_USB_EHCI_HCD_PLATFORM is not set # CONFIG_USB_EHCI_TT_NEWSCHED is not set From 97a427412a2cb015360df7762efd8ed7573111c8 Mon Sep 17 00:00:00 2001 From: Luis Araneda Date: Sat, 9 Oct 2021 23:55:03 -0300 Subject: [PATCH 13/22] zynq: kernel: remove wireless extensions symbols This fixes compilation of several wireless drivers that require support for the old wireless extension to work. One example is kmod-hermes. The symbols are set to "y" on generic configuration. But they were wrongly disabled on the target-specific configuration. Signed-off-by: Luis Araneda --- target/linux/zynq/config-5.4 | 3 --- 1 file changed, 3 deletions(-) diff --git a/target/linux/zynq/config-5.4 b/target/linux/zynq/config-5.4 index 98b2bd0f93..38538789d8 100644 --- a/target/linux/zynq/config-5.4 +++ b/target/linux/zynq/config-5.4 @@ -531,9 +531,6 @@ CONFIG_VT_CONSOLE=y CONFIG_VT_CONSOLE_SLEEP=y # CONFIG_VT_HW_CONSOLE_BINDING is not set CONFIG_WATCHDOG_CORE=y -# CONFIG_WEXT_PRIV is not set -# CONFIG_WEXT_SPY is not set -# CONFIG_WIRELESS_EXT is not set # CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set CONFIG_XILINX_EMACLITE=y # CONFIG_XILINX_PR_DECOUPLER is not set From 861ff4d0808d05bd66e74f52688c6c76c4be2f9f Mon Sep 17 00:00:00 2001 From: Luis Araneda Date: Sat, 9 Oct 2021 23:55:04 -0300 Subject: [PATCH 14/22] zynq: kernel: copy config from 5.4 to 5.10 Signed-off-by: Luis Araneda --- target/linux/zynq/config-5.10 | 550 ++++++++++++++++++++++++++++++++++ 1 file changed, 550 insertions(+) create mode 100644 target/linux/zynq/config-5.10 diff --git a/target/linux/zynq/config-5.10 b/target/linux/zynq/config-5.10 new file mode 100644 index 0000000000..38538789d8 --- /dev/null +++ b/target/linux/zynq/config-5.10 @@ -0,0 +1,550 @@ +CONFIG_ALIGNMENT_TRAP=y +# CONFIG_ALTERA_FREEZE_BRIDGE is not set +# CONFIG_ALTERA_PR_IP_CORE is not set +CONFIG_ARCH_32BIT_OFF_T=y +CONFIG_ARCH_CLOCKSOURCE_DATA=y +CONFIG_ARCH_HIBERNATION_POSSIBLE=y +CONFIG_ARCH_KEEP_MEMBLOCK=y +CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y +CONFIG_ARCH_MULTIPLATFORM=y +CONFIG_ARCH_MULTI_V6_V7=y +CONFIG_ARCH_MULTI_V7=y +CONFIG_ARCH_NR_GPIO=1024 +CONFIG_ARCH_OPTIONAL_KERNEL_RWX=y +CONFIG_ARCH_OPTIONAL_KERNEL_RWX_DEFAULT=y +CONFIG_ARCH_SUSPEND_POSSIBLE=y +CONFIG_ARCH_VEXPRESS=y +CONFIG_ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA=y +# CONFIG_ARCH_VEXPRESS_SPC is not set +CONFIG_ARCH_ZYNQ=y +CONFIG_ARM=y +CONFIG_ARM_AMBA=y +CONFIG_ARM_CPU_SUSPEND=y +CONFIG_ARM_ERRATA_643719=y +CONFIG_ARM_ERRATA_720789=y +CONFIG_ARM_ERRATA_754322=y +CONFIG_ARM_ERRATA_754327=y +CONFIG_ARM_ERRATA_764369=y +CONFIG_ARM_ERRATA_775420=y +CONFIG_ARM_GIC=y +CONFIG_ARM_GLOBAL_TIMER=y +CONFIG_ARM_HAS_SG_CHAIN=y +CONFIG_ARM_HEAVY_MB=y +CONFIG_ARM_L1_CACHE_SHIFT=6 +CONFIG_ARM_L1_CACHE_SHIFT_6=y +CONFIG_ARM_PATCH_IDIV=y +CONFIG_ARM_PATCH_PHYS_VIRT=y +# CONFIG_ARM_PL172_MPMC is not set +# CONFIG_ARM_SMMU is not set +CONFIG_ARM_THUMB=y +CONFIG_ARM_TIMER_SP804=y +CONFIG_ARM_UNWIND=y +CONFIG_ARM_VIRT_EXT=y +CONFIG_ARM_ZYNQ_CPUIDLE=y +CONFIG_ATAGS=y +CONFIG_AUTO_ZRELADDR=y +# CONFIG_AXI_DMAC is not set +CONFIG_BINFMT_FLAT_ARGVP_ENVP_ON_STACK=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_COUNT=16 +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_BLK_MQ_PCI=y +CONFIG_BLK_PM=y +CONFIG_BOUNCE=y +CONFIG_CACHE_L2X0=y +CONFIG_CADENCE_TTC_TIMER=y +CONFIG_CADENCE_WATCHDOG=y +CONFIG_CLKDEV_LOOKUP=y +CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK=y +CONFIG_CLKSRC_MMIO=y +CONFIG_CLKSRC_VERSATILE=y +CONFIG_CLK_SP810=y +CONFIG_CLK_VEXPRESS_OSC=y +CONFIG_CLONE_BACKWARDS=y +CONFIG_CMA=y +CONFIG_CMA_ALIGNMENT=8 +CONFIG_CMA_AREAS=7 +# CONFIG_CMA_DEBUG is not set +# CONFIG_CMA_DEBUGFS is not set +CONFIG_CMA_SIZE_MBYTES=16 +# CONFIG_CMA_SIZE_SEL_MAX is not set +CONFIG_CMA_SIZE_SEL_MBYTES=y +# CONFIG_CMA_SIZE_SEL_MIN is not set +# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set +CONFIG_COMMON_CLK=y +# CONFIG_COMMON_CLK_AXI_CLKGEN is not set +CONFIG_COMMON_CLK_SI570=y +CONFIG_COMMON_CLK_VERSATILE=y +CONFIG_COMPAT_32BIT_TIME=y +CONFIG_CONNECTOR=y +CONFIG_CONSOLE_TRANSLATIONS=y +CONFIG_CONTIG_ALLOC=y +CONFIG_COREDUMP=y +# CONFIG_CPUFREQ_DT is not set +CONFIG_CPU_32v6K=y +CONFIG_CPU_32v7=y +CONFIG_CPU_ABRT_EV7=y +CONFIG_CPU_CACHE_V7=y +CONFIG_CPU_CACHE_VIPT=y +CONFIG_CPU_COPY_V6=y +CONFIG_CPU_CP15=y +CONFIG_CPU_CP15_MMU=y +CONFIG_CPU_FREQ=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE is not set +CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ATTR_SET=y +CONFIG_CPU_FREQ_GOV_COMMON=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_HAS_ASID=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_CPU_IDLE_GOV_MENU=y +CONFIG_CPU_PABRT_V7=y +CONFIG_CPU_PM=y +CONFIG_CPU_RMAP=y +CONFIG_CPU_SPECTRE=y +CONFIG_CPU_THERMAL=y +CONFIG_CPU_THUMB_CAPABLE=y +CONFIG_CPU_TLB_V7=y +CONFIG_CPU_V7=y +CONFIG_CRC16=y +# CONFIG_CRC32_SARWATE is not set +CONFIG_CRC32_SLICEBY8=y +CONFIG_CROSS_MEMORY_ATTACH=y +CONFIG_CRYPTO_AEAD=y +CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_CRC32=y +CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_HASH=y +CONFIG_CRYPTO_HASH2=y +CONFIG_CRYPTO_HW=y +CONFIG_CRYPTO_MANAGER=y +CONFIG_CRYPTO_MANAGER2=y +CONFIG_CRYPTO_NULL2=y +CONFIG_CRYPTO_RNG2=y +CONFIG_DCACHE_WORD_ACCESS=y +CONFIG_DEBUG_LL_INCLUDE="mach/debug-macro.S" +CONFIG_DMADEVICES=y +CONFIG_DMA_CMA=y +CONFIG_DMA_ENGINE=y +CONFIG_DMA_OF=y +CONFIG_DMA_REMAP=y +CONFIG_DMA_SHARED_BUFFER=y +CONFIG_DRM=y +CONFIG_DRM_BRIDGE=y +CONFIG_DRM_FBDEV_EMULATION=y +CONFIG_DRM_FBDEV_OVERALLOC=100 +CONFIG_DRM_KMS_FB_HELPER=y +CONFIG_DRM_KMS_HELPER=y +CONFIG_DRM_PANEL=y +CONFIG_DRM_PANEL_BRIDGE=y +CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y +CONFIG_DTC=y +CONFIG_DUMMY_CONSOLE=y +CONFIG_E1000E=y +CONFIG_EDAC=y +CONFIG_EDAC_ATOMIC_SCRUB=y +# CONFIG_EDAC_DEBUG is not set +CONFIG_EDAC_LEGACY_SYSFS=y +CONFIG_EDAC_SUPPORT=y +# CONFIG_EDAC_SYNOPSYS is not set +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_AT25=y +CONFIG_ELF_CORE=y +CONFIG_ENABLE_MUST_CHECK=y +CONFIG_EXT4_FS=y +CONFIG_EXTCON=y +CONFIG_F2FS_FS=y +CONFIG_FB=y +CONFIG_FB_CFB_COPYAREA=y +CONFIG_FB_CFB_FILLRECT=y +CONFIG_FB_CFB_IMAGEBLIT=y +CONFIG_FB_CMDLINE=y +CONFIG_FB_DEFERRED_IO=y +CONFIG_FB_SYS_COPYAREA=y +CONFIG_FB_SYS_FILLRECT=y +CONFIG_FB_SYS_FOPS=y +CONFIG_FB_SYS_IMAGEBLIT=y +# CONFIG_FB_XILINX is not set +CONFIG_FHANDLE=y +CONFIG_FIXED_PHY=y +CONFIG_FIX_EARLYCON_MEM=y +CONFIG_FPGA=y +CONFIG_FPGA_BRIDGE=y +# CONFIG_FPGA_DFL is not set +# CONFIG_FPGA_MGR_ALTERA_CVP is not set +# CONFIG_FPGA_MGR_ALTERA_PS_SPI is not set +# CONFIG_FPGA_MGR_ICE40_SPI is not set +# CONFIG_FPGA_MGR_MACHXO2_SPI is not set +# CONFIG_FPGA_MGR_XILINX_SPI is not set +CONFIG_FPGA_MGR_ZYNQ_FPGA=y +CONFIG_FPGA_REGION=y +CONFIG_FREEZER=y +CONFIG_FS_IOMAP=y +CONFIG_FS_MBCACHE=y +CONFIG_FW_LOADER_PAGED_BUF=y +CONFIG_GENERIC_ALLOCATOR=y +CONFIG_GENERIC_ARCH_TOPOLOGY=y +CONFIG_GENERIC_BUG=y +CONFIG_GENERIC_CLOCKEVENTS=y +CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y +CONFIG_GENERIC_CPU_AUTOPROBE=y +CONFIG_GENERIC_EARLY_IOREMAP=y +CONFIG_GENERIC_IDLE_POLL_SETUP=y +CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK=y +CONFIG_GENERIC_IRQ_MIGRATION=y +CONFIG_GENERIC_IRQ_MULTI_HANDLER=y +CONFIG_GENERIC_IRQ_SHOW=y +CONFIG_GENERIC_IRQ_SHOW_LEVEL=y +CONFIG_GENERIC_MSI_IRQ=y +CONFIG_GENERIC_MSI_IRQ_DOMAIN=y +CONFIG_GENERIC_PCI_IOMAP=y +CONFIG_GENERIC_PINCONF=y +CONFIG_GENERIC_SCHED_CLOCK=y +CONFIG_GENERIC_SMP_IDLE_THREAD=y +CONFIG_GENERIC_STRNCPY_FROM_USER=y +CONFIG_GENERIC_STRNLEN_USER=y +CONFIG_GENERIC_TIME_VSYSCALL=y +CONFIG_GLOB=y +CONFIG_GPIOLIB=y +CONFIG_GPIOLIB_IRQCHIP=y +CONFIG_GPIO_GENERIC=y +CONFIG_GPIO_GENERIC_PLATFORM=y +CONFIG_GPIO_ZYNQ=y +CONFIG_HANDLE_DOMAIN_IRQ=y +CONFIG_HARDEN_BRANCH_PREDICTOR=y +CONFIG_HARDIRQS_SW_RESEND=y +CONFIG_HAS_DMA=y +CONFIG_HAS_IOMEM=y +CONFIG_HAVE_SMP=y +CONFIG_HDMI=y +CONFIG_HID=y +CONFIG_HID_GENERIC=y +CONFIG_HID_MICROSOFT=y +CONFIG_HIGHMEM=y +# CONFIG_HIGHPTE is not set +CONFIG_HOTPLUG_CPU=y +CONFIG_HWMON=y +CONFIG_HW_CONSOLE=y +CONFIG_HZ_FIXED=0 +CONFIG_I2C=y +CONFIG_I2C_ALGOBIT=y +CONFIG_I2C_BOARDINFO=y +CONFIG_I2C_CADENCE=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_COMPAT=y +CONFIG_I2C_HELPER_AUTO=y +CONFIG_I2C_MUX=y +CONFIG_I2C_MUX_PCA954x=y +CONFIG_ICST=y +CONFIG_IIO=y +CONFIG_IIO_BUFFER=y +CONFIG_IIO_KFIFO_BUF=y +CONFIG_IIO_TRIGGER=y +CONFIG_IIO_TRIGGERED_BUFFER=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_INITRAMFS_SOURCE="" +CONFIG_INPUT=y +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_KEYBOARD=y +# CONFIG_INPUT_MISC is not set +CONFIG_INPUT_MOUSE=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_MOUSEDEV_PSAUX=y +CONFIG_INPUT_MOUSEDEV_SCREEN_X=1024 +CONFIG_INPUT_MOUSEDEV_SCREEN_Y=768 +CONFIG_INPUT_POLLDEV=y +CONFIG_INPUT_SPARSEKMAP=y +# CONFIG_IOMMU_DEBUGFS is not set +# CONFIG_IOMMU_IO_PGTABLE_ARMV7S is not set +# CONFIG_IOMMU_IO_PGTABLE_LPAE is not set +CONFIG_IOMMU_SUPPORT=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IRQCHIP=y +CONFIG_IRQ_DOMAIN=y +CONFIG_IRQ_DOMAIN_HIERARCHY=y +CONFIG_IRQ_FORCED_THREADING=y +CONFIG_IRQ_WORK=y +# CONFIG_ISDN is not set +CONFIG_JBD2=y +# CONFIG_JFFS2_COMPRESSION_OPTIONS is not set +CONFIG_JFFS2_ZLIB=y +CONFIG_KERNEL_GZIP=y +# CONFIG_KERNEL_XZ is not set +CONFIG_KEYBOARD_ATKBD=y +CONFIG_KEYBOARD_GPIO=y +CONFIG_KEYBOARD_GPIO_POLLED=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +CONFIG_LEDS_TRIGGER_CAMERA=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_GPIO=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_TRANSIENT=y +CONFIG_LIBFDT=y +CONFIG_LOCK_DEBUGGING_SUPPORT=y +CONFIG_LOCK_SPIN_ON_OWNER=y +CONFIG_MACB=y +# CONFIG_MACB_PCI is not set +CONFIG_MACB_USE_HWSTAMP=y +CONFIG_MARVELL_PHY=y +CONFIG_MDIO_BITBANG=y +CONFIG_MDIO_BUS=y +CONFIG_MDIO_DEVICE=y +# CONFIG_MDIO_GPIO is not set +CONFIG_MEMFD_CREATE=y +CONFIG_MEMORY=y +CONFIG_MEMORY_ISOLATION=y +CONFIG_MFD_CORE=y +CONFIG_MFD_SYSCON=y +CONFIG_MFD_VEXPRESS_SYSREG=y +CONFIG_MIGHT_HAVE_CACHE_L2X0=y +CONFIG_MIGRATION=y +CONFIG_MMC=y +CONFIG_MMC_BLOCK=y +CONFIG_MMC_CQHCI=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_OF_ARASAN=y +# CONFIG_MMC_SDHCI_PCI is not set +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MODULES_USE_ELF_REL=y +CONFIG_MODULE_FORCE_UNLOAD=y +# CONFIG_MODULE_STRIPPED is not set +# CONFIG_MOUSE_BCM5974 is not set +# CONFIG_MOUSE_CYAPA is not set +CONFIG_MOUSE_PS2=y +CONFIG_MOUSE_PS2_ALPS=y +CONFIG_MOUSE_PS2_BYD=y +CONFIG_MOUSE_PS2_CYPRESS=y +# CONFIG_MOUSE_PS2_ELANTECH is not set +CONFIG_MOUSE_PS2_FOCALTECH=y +CONFIG_MOUSE_PS2_LOGIPS2PP=y +CONFIG_MOUSE_PS2_SMBUS=y +CONFIG_MOUSE_PS2_SYNAPTICS=y +CONFIG_MOUSE_PS2_SYNAPTICS_SMBUS=y +# CONFIG_MOUSE_PS2_TOUCHKIT is not set +CONFIG_MOUSE_PS2_TRACKPOINT=y +# CONFIG_MOUSE_SERIAL is not set +# CONFIG_MOUSE_VSXXXAA is not set +# CONFIG_MTD_CFI_INTELEXT is not set +CONFIG_MTD_CMDLINE_PARTS=y +# CONFIG_MTD_COMPLEX_MAPPINGS is not set +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_SPI_NOR=y +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y +CONFIG_MTD_SPLIT_FIRMWARE=y +# CONFIG_MTD_SPLIT_SQUASHFS_ROOT is not set +CONFIG_MUTEX_SPIN_ON_OWNER=y +CONFIG_NEED_DMA_MAP_STATE=y +CONFIG_NEON=y +CONFIG_NET_FLOW_LIMIT=y +CONFIG_NET_PTP_CLASSIFY=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_FARADAY is not set +# CONFIG_NET_VENDOR_MARVELL is not set +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROCHIP is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_VIA is not set +CONFIG_NLS=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NOP_USB_XCEIV=y +CONFIG_NO_HZ=y +CONFIG_NO_HZ_COMMON=y +CONFIG_NO_HZ_IDLE=y +CONFIG_NO_IOPORT_MAP=y +CONFIG_NR_CPUS=4 +CONFIG_NVMEM=y +CONFIG_NVMEM_SYSFS=y +CONFIG_OF=y +CONFIG_OF_ADDRESS=y +CONFIG_OF_EARLY_FLATTREE=y +CONFIG_OF_FLATTREE=y +# CONFIG_OF_FPGA_REGION is not set +CONFIG_OF_GPIO=y +CONFIG_OF_IRQ=y +CONFIG_OF_KOBJ=y +CONFIG_OF_MDIO=y +CONFIG_OF_NET=y +CONFIG_OLD_SIGACTION=y +CONFIG_OLD_SIGSUSPEND3=y +CONFIG_OUTER_CACHE=y +CONFIG_OUTER_CACHE_SYNC=y +CONFIG_PADATA=y +CONFIG_PAGE_OFFSET=0xC0000000 +# CONFIG_PARTITION_ADVANCED is not set +CONFIG_PCI=y +CONFIG_PCIE_XILINX=y +CONFIG_PCI_DOMAINS=y +CONFIG_PCI_DOMAINS_GENERIC=y +CONFIG_PCI_MSI=y +CONFIG_PCI_MSI_IRQ_DOMAIN=y +CONFIG_PERF_USE_VMALLOC=y +CONFIG_PGTABLE_LEVELS=2 +CONFIG_PHYLIB=y +CONFIG_PINCTRL=y +# CONFIG_PINCTRL_SINGLE is not set +CONFIG_PINCTRL_ZYNQ=y +CONFIG_PL310_ERRATA_588369=y +CONFIG_PL310_ERRATA_727915=y +CONFIG_PL310_ERRATA_753970=y +CONFIG_PL310_ERRATA_769419=y +CONFIG_PL330_DMA=y +# CONFIG_PL353_SMC is not set +CONFIG_PLAT_VERSATILE=y +CONFIG_PM=y +CONFIG_PMBUS=y +CONFIG_PM_CLK=y +CONFIG_PM_SLEEP=y +CONFIG_PM_SLEEP_SMP=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_VEXPRESS=y +CONFIG_POWER_SUPPLY=y +CONFIG_PPS=y +CONFIG_PROC_EVENTS=y +CONFIG_PTP_1588_CLOCK=y +CONFIG_R8169=y +CONFIG_RAS=y +CONFIG_RATIONAL=y +CONFIG_REALTEK_PHY=y +CONFIG_REFCOUNT_FULL=y +CONFIG_REGMAP=y +CONFIG_REGMAP_I2C=y +CONFIG_REGMAP_MMIO=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +# CONFIG_REGULATOR_VEXPRESS is not set +CONFIG_RESET_CONTROLLER=y +CONFIG_RESET_ZYNQ=y +CONFIG_RFS_ACCEL=y +CONFIG_RPS=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_PCF8563=y +CONFIG_RTC_I2C_AND_SPI=y +CONFIG_RTC_MC146818_LIB=y +CONFIG_RWSEM_SPIN_ON_OWNER=y +CONFIG_SCHED_MC=y +CONFIG_SCHED_SMT=y +CONFIG_SENSORS_PMBUS=y +CONFIG_SENSORS_UCD9000=y +CONFIG_SENSORS_UCD9200=y +# CONFIG_SERIAL_8250 is not set +CONFIG_SERIAL_XILINX_PS_UART=y +CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y +CONFIG_SERIO=y +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIO_SERPORT=y +CONFIG_SMP=y +CONFIG_SMP_ON_UP=y +CONFIG_SOC_BUS=y +CONFIG_SPARSE_IRQ=y +CONFIG_SPI=y +CONFIG_SPI_BITBANG=y +CONFIG_SPI_CADENCE=y +CONFIG_SPI_MASTER=y +CONFIG_SPI_MEM=y +CONFIG_SPI_XILINX=y +CONFIG_SPI_ZYNQ_QSPI=y +CONFIG_SRAM=y +CONFIG_SRAM_EXEC=y +CONFIG_SRCU=y +# CONFIG_STRIP_ASM_SYMS is not set +CONFIG_SUSPEND=y +CONFIG_SUSPEND_FREEZER=y +CONFIG_SWPHY=y +CONFIG_SWP_EMULATE=y +CONFIG_SYNC_FILE=y +CONFIG_SYSFS_SYSCALL=y +CONFIG_SYS_SUPPORTS_APM_EMULATION=y +# CONFIG_TEXTSEARCH is not set +CONFIG_THERMAL=y +CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y +CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0 +CONFIG_THERMAL_GOV_STEP_WISE=y +CONFIG_THERMAL_HWMON=y +CONFIG_THERMAL_OF=y +CONFIG_TICK_CPU_ACCOUNTING=y +CONFIG_TIMER_OF=y +CONFIG_TIMER_PROBE=y +CONFIG_TREE_RCU=y +CONFIG_TREE_SRCU=y +CONFIG_UIO=y +# CONFIG_UIO_AEC is not set +# CONFIG_UIO_CIF is not set +# CONFIG_UIO_DMEM_GENIRQ is not set +# CONFIG_UIO_MF624 is not set +# CONFIG_UIO_NETX is not set +# CONFIG_UIO_PCI_GENERIC is not set +CONFIG_UIO_PDRV_GENIRQ=y +# CONFIG_UIO_PRUSS is not set +# CONFIG_UIO_SERCOS3 is not set +CONFIG_UNCOMPRESS_INCLUDE="debug/uncompress.h" +CONFIG_UNWINDER_ARM=y +CONFIG_USB=y +CONFIG_USB_CHIPIDEA=y +CONFIG_USB_CHIPIDEA_HOST=y +CONFIG_USB_CHIPIDEA_OF=y +CONFIG_USB_CHIPIDEA_UDC=y +CONFIG_USB_COMMON=y +CONFIG_USB_EHCI_HCD=y +# CONFIG_USB_EHCI_HCD_PLATFORM is not set +# CONFIG_USB_EHCI_TT_NEWSCHED is not set +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_XILINX=y +CONFIG_USB_HID=y +CONFIG_USB_NET_DRIVERS=y +CONFIG_USB_OTG=y +CONFIG_USB_OTG_FSM=y +CONFIG_USB_PHY=y +CONFIG_USB_ROLE_SWITCH=y +CONFIG_USB_SUPPORT=y +CONFIG_USB_ULPI=y +CONFIG_USB_ULPI_BUS=y +CONFIG_USB_ULPI_VIEWPORT=y +CONFIG_USE_OF=y +CONFIG_VEXPRESS_CONFIG=y +CONFIG_VEXPRESS_SYSCFG=y +CONFIG_VFP=y +CONFIG_VFPv3=y +CONFIG_VGA_ARB=y +CONFIG_VGA_ARB_MAX_GPUS=16 +CONFIG_VITESSE_PHY=y +CONFIG_VM_EVENT_COUNTERS=y +CONFIG_VT=y +CONFIG_VT_CONSOLE=y +CONFIG_VT_CONSOLE_SLEEP=y +# CONFIG_VT_HW_CONSOLE_BINDING is not set +CONFIG_WATCHDOG_CORE=y +# CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set +CONFIG_XILINX_EMACLITE=y +# CONFIG_XILINX_PR_DECOUPLER is not set +CONFIG_XILINX_WATCHDOG=y +CONFIG_XILINX_XADC=y +CONFIG_XPS=y +CONFIG_XZ_DEC_ARM=y +CONFIG_XZ_DEC_ARMTHUMB=y +CONFIG_XZ_DEC_BCJ=y +CONFIG_XZ_DEC_IA64=y +CONFIG_XZ_DEC_POWERPC=y +CONFIG_XZ_DEC_SPARC=y +CONFIG_XZ_DEC_X86=y +CONFIG_ZBOOT_ROM_BSS=0x0 +CONFIG_ZBOOT_ROM_TEXT=0x0 +CONFIG_ZLIB_DEFLATE=y +CONFIG_ZLIB_INFLATE=y From 16ba7ca8beb8ef6b9b59a1d0ce423cdfaeea2314 Mon Sep 17 00:00:00 2001 From: Luis Araneda Date: Sat, 9 Oct 2021 23:55:05 -0300 Subject: [PATCH 15/22] zynq: kernel: update config for 5.10 Update config with make kernel_oldconfig Signed-off-by: Luis Araneda --- target/linux/zynq/config-5.10 | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/target/linux/zynq/config-5.10 b/target/linux/zynq/config-5.10 index 38538789d8..d18bb39d8d 100644 --- a/target/linux/zynq/config-5.10 +++ b/target/linux/zynq/config-5.10 @@ -2,7 +2,6 @@ CONFIG_ALIGNMENT_TRAP=y # CONFIG_ALTERA_FREEZE_BRIDGE is not set # CONFIG_ALTERA_PR_IP_CORE is not set CONFIG_ARCH_32BIT_OFF_T=y -CONFIG_ARCH_CLOCKSOURCE_DATA=y CONFIG_ARCH_HIBERNATION_POSSIBLE=y CONFIG_ARCH_KEEP_MEMBLOCK=y CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y @@ -12,6 +11,8 @@ CONFIG_ARCH_MULTI_V7=y CONFIG_ARCH_NR_GPIO=1024 CONFIG_ARCH_OPTIONAL_KERNEL_RWX=y CONFIG_ARCH_OPTIONAL_KERNEL_RWX_DEFAULT=y +CONFIG_ARCH_SELECT_MEMORY_MODEL=y +CONFIG_ARCH_SPARSEMEM_ENABLE=y CONFIG_ARCH_SUSPEND_POSSIBLE=y CONFIG_ARCH_VEXPRESS=y CONFIG_ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA=y @@ -75,7 +76,6 @@ CONFIG_CMA_SIZE_SEL_MBYTES=y CONFIG_COMMON_CLK=y # CONFIG_COMMON_CLK_AXI_CLKGEN is not set CONFIG_COMMON_CLK_SI570=y -CONFIG_COMMON_CLK_VERSATILE=y CONFIG_COMPAT_32BIT_TIME=y CONFIG_CONNECTOR=y CONFIG_CONSOLE_TRANSLATIONS=y @@ -117,15 +117,10 @@ CONFIG_CRC16=y # CONFIG_CRC32_SARWATE is not set CONFIG_CRC32_SLICEBY8=y CONFIG_CROSS_MEMORY_ATTACH=y -CONFIG_CRYPTO_AEAD=y -CONFIG_CRYPTO_AEAD2=y CONFIG_CRYPTO_CRC32=y CONFIG_CRYPTO_CRC32C=y -CONFIG_CRYPTO_HASH=y -CONFIG_CRYPTO_HASH2=y +CONFIG_CRYPTO_GF128MUL=y CONFIG_CRYPTO_HW=y -CONFIG_CRYPTO_MANAGER=y -CONFIG_CRYPTO_MANAGER2=y CONFIG_CRYPTO_NULL2=y CONFIG_CRYPTO_RNG2=y CONFIG_DCACHE_WORD_ACCESS=y @@ -134,6 +129,7 @@ CONFIG_DMADEVICES=y CONFIG_DMA_CMA=y CONFIG_DMA_ENGINE=y CONFIG_DMA_OF=y +CONFIG_DMA_OPS=y CONFIG_DMA_REMAP=y CONFIG_DMA_SHARED_BUFFER=y CONFIG_DRM=y @@ -188,6 +184,7 @@ CONFIG_FPGA_REGION=y CONFIG_FREEZER=y CONFIG_FS_IOMAP=y CONFIG_FS_MBCACHE=y +CONFIG_FW_CACHE=y CONFIG_FW_LOADER_PAGED_BUF=y CONFIG_GENERIC_ALLOCATOR=y CONFIG_GENERIC_ARCH_TOPOLOGY=y @@ -196,6 +193,7 @@ CONFIG_GENERIC_CLOCKEVENTS=y CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y CONFIG_GENERIC_CPU_AUTOPROBE=y CONFIG_GENERIC_EARLY_IOREMAP=y +CONFIG_GENERIC_GETTIMEOFDAY=y CONFIG_GENERIC_IDLE_POLL_SETUP=y CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK=y CONFIG_GENERIC_IRQ_MIGRATION=y @@ -211,6 +209,7 @@ CONFIG_GENERIC_SMP_IDLE_THREAD=y CONFIG_GENERIC_STRNCPY_FROM_USER=y CONFIG_GENERIC_STRNLEN_USER=y CONFIG_GENERIC_TIME_VSYSCALL=y +CONFIG_GENERIC_VDSO_32=y CONFIG_GLOB=y CONFIG_GPIOLIB=y CONFIG_GPIOLIB_IRQCHIP=y @@ -280,6 +279,7 @@ CONFIG_IRQ_WORK=y CONFIG_JBD2=y # CONFIG_JFFS2_COMPRESSION_OPTIONS is not set CONFIG_JFFS2_ZLIB=y +CONFIG_KCMP=y CONFIG_KERNEL_GZIP=y # CONFIG_KERNEL_XZ is not set CONFIG_KEYBOARD_ATKBD=y @@ -293,6 +293,7 @@ CONFIG_LEDS_TRIGGER_GPIO=y CONFIG_LEDS_TRIGGER_ONESHOT=y CONFIG_LEDS_TRIGGER_TRANSIENT=y CONFIG_LIBFDT=y +CONFIG_LLD_VERSION=0 CONFIG_LOCK_DEBUGGING_SUPPORT=y CONFIG_LOCK_SPIN_ON_OWNER=y CONFIG_MACB=y @@ -302,6 +303,7 @@ CONFIG_MARVELL_PHY=y CONFIG_MDIO_BITBANG=y CONFIG_MDIO_BUS=y CONFIG_MDIO_DEVICE=y +CONFIG_MDIO_DEVRES=y # CONFIG_MDIO_GPIO is not set CONFIG_MEMFD_CREATE=y CONFIG_MEMORY=y @@ -394,10 +396,12 @@ CONFIG_PCIE_XILINX=y CONFIG_PCI_DOMAINS=y CONFIG_PCI_DOMAINS_GENERIC=y CONFIG_PCI_MSI=y +CONFIG_PCI_MSI_ARCH_FALLBACKS=y CONFIG_PCI_MSI_IRQ_DOMAIN=y CONFIG_PERF_USE_VMALLOC=y CONFIG_PGTABLE_LEVELS=2 CONFIG_PHYLIB=y +CONFIG_PHYLINK=y CONFIG_PINCTRL=y # CONFIG_PINCTRL_SINGLE is not set CONFIG_PINCTRL_ZYNQ=y @@ -423,7 +427,6 @@ CONFIG_R8169=y CONFIG_RAS=y CONFIG_RATIONAL=y CONFIG_REALTEK_PHY=y -CONFIG_REFCOUNT_FULL=y CONFIG_REGMAP=y CONFIG_REGMAP_I2C=y CONFIG_REGMAP_MMIO=y @@ -499,12 +502,12 @@ CONFIG_UNWINDER_ARM=y CONFIG_USB=y CONFIG_USB_CHIPIDEA=y CONFIG_USB_CHIPIDEA_HOST=y -CONFIG_USB_CHIPIDEA_OF=y CONFIG_USB_CHIPIDEA_UDC=y CONFIG_USB_COMMON=y CONFIG_USB_EHCI_HCD=y # CONFIG_USB_EHCI_HCD_PLATFORM is not set # CONFIG_USB_EHCI_TT_NEWSCHED is not set +# CONFIG_USB_ETH is not set CONFIG_USB_GADGET=y CONFIG_USB_GADGET_XILINX=y CONFIG_USB_HID=y @@ -519,7 +522,6 @@ CONFIG_USB_ULPI_BUS=y CONFIG_USB_ULPI_VIEWPORT=y CONFIG_USE_OF=y CONFIG_VEXPRESS_CONFIG=y -CONFIG_VEXPRESS_SYSCFG=y CONFIG_VFP=y CONFIG_VFPv3=y CONFIG_VGA_ARB=y From 88537e76ec06012826d72e175b77c452cb41d256 Mon Sep 17 00:00:00 2001 From: Luis Araneda Date: Sat, 9 Oct 2021 23:55:06 -0300 Subject: [PATCH 16/22] zynq: switch to kernel 5.10 Use kernel 5.10 by default compile-tested: all devices from target (wth ALL_KMODS) run-tested: Digilent Zybo Z7-20 Signed-off-by: Luis Araneda --- target/linux/zynq/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/linux/zynq/Makefile b/target/linux/zynq/Makefile index b5988c9e05..83807913e4 100644 --- a/target/linux/zynq/Makefile +++ b/target/linux/zynq/Makefile @@ -17,7 +17,7 @@ define Target/Description Build firmware image for Zynq 7000 SoC devices. endef -KERNEL_PATCHVER:=5.4 +KERNEL_PATCHVER:=5.10 include $(INCLUDE_DIR)/target.mk From 5b3d62247c97b620424e137d02ee5b4d72b584aa Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Thu, 14 Oct 2021 19:15:49 -0700 Subject: [PATCH 17/22] ramips: fix dtc warnings for telco-electronics_x1 In all other dts files, the entire block is not edited like this. They're edited separately. Signed-off-by: Rosen Penev --- .../dts/mt7621_telco-electronics_x1.dts | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts b/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts index 51c1d868cb..ce700e0c2b 100644 --- a/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts +++ b/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts @@ -166,25 +166,26 @@ &pcie { status = "okay"; +}; - pcie0 { - wifi@0,0 { - compatible = "pci14c3,7603"; - reg = <0x0000 0 0 0 0>; - mediatek,mtd-eeprom = <&factory 0x0000>; - ieee80211-freq-limit = <2400000 2500000>; - }; +&pcie0 { + wifi@0,0 { + compatible = "pci14c3,7603"; + reg = <0x0000 0 0 0 0>; + mediatek,mtd-eeprom = <&factory 0x0000>; + ieee80211-freq-limit = <2400000 2500000>; }; +}; - pcie1 { - wifi@0,0 { - compatible = "pci14c3,7662"; - reg = <0x0000 0 0 0 0>; - mediatek,mtd-eeprom = <&factory 0x8000>; - ieee80211-freq-limit = <5000000 6000000>; - led { - led-sources = <2>; - }; +&pcie1 { + wifi@0,0 { + compatible = "pci14c3,7662"; + reg = <0x0000 0 0 0 0>; + mediatek,mtd-eeprom = <&factory 0x8000>; + ieee80211-freq-limit = <5000000 6000000>; + + led { + led-sources = <2>; }; }; }; From a1ac8728f80314c574201013e7fea58536c2b3ee Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Sun, 17 Oct 2021 16:05:34 +0200 Subject: [PATCH 18/22] ramips: remove kmod-mt7663-firmware-sta from device packages This firmware should only be used for mobile devices (e.g. laptops), where AP mode functionality is typically not used. This firmware supports a lot of power saving offload functionality at the expense of AP mode support. Signed-off-by: Felix Fietkau --- target/linux/ramips/image/mt7621.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target/linux/ramips/image/mt7621.mk b/target/linux/ramips/image/mt7621.mk index ecb102467b..c81e79c900 100644 --- a/target/linux/ramips/image/mt7621.mk +++ b/target/linux/ramips/image/mt7621.mk @@ -825,7 +825,7 @@ define Device/linksys_e5600 DEVICE_VENDOR := Linksys DEVICE_MODEL := E5600 DEVICE_PACKAGES := kmod-mt7603 kmod-mt7615e kmod-mt7663-firmware-ap \ - kmod-mt7663-firmware-sta uboot-envtools + uboot-envtools UBINIZE_OPTS := -E 5 IMAGES += factory.bin IMAGE/sysupgrade.bin := sysupgrade-tar | check-size | append-metadata @@ -1316,7 +1316,7 @@ define Device/tplink_archer-a6-v3 DEVICE_MODEL := Archer A6 DEVICE_VARIANT := V3 DEVICE_PACKAGES := kmod-mt7603 kmod-mt7615e \ - kmod-mt7663-firmware-ap kmod-mt7663-firmware-sta + kmod-mt7663-firmware-ap TPLINK_BOARD_ID := ARCHER-A6-V3 KERNEL := $(KERNEL_DTB) | uImage lzma IMAGE_SIZE := 15744k @@ -1329,7 +1329,7 @@ define Device/tplink_archer-c6-v3 DEVICE_MODEL := Archer C6 DEVICE_VARIANT := V3 DEVICE_PACKAGES := kmod-mt7603 kmod-mt7615e \ - kmod-mt7663-firmware-ap kmod-mt7663-firmware-sta + kmod-mt7663-firmware-ap TPLINK_BOARD_ID := ARCHER-C6-V3 KERNEL := $(KERNEL_DTB) | uImage lzma IMAGE_SIZE := 15744k From 28d8e6a8711ba78f1684a205e11b0dbd4ff2b2f3 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Wed, 6 Oct 2021 10:54:48 +0200 Subject: [PATCH 19/22] wolfssl: build with WOLFSSL_ALT_CERT_CHAINS "Alternate certification chains, as oppossed to requiring full chain validataion. Certificate validation behavior is relaxed, similar to openssl and browsers. Only the peer certificate must validate to a trusted certificate. Without this, all certificates sent by a peer must be used in the trust chain or the connection will be rejected." This fixes e.g. uclient-fetch and curl connecting to servers using a Let's Encrypt certificate which are cross-signed by the now expired DST Root CA X3, see [0]. This is the recommended solution from upstream [1]. The binary size increases by ~12.3kb: 1236160 staging_dir/target-mipsel_24kc_musl/usr/lib/libwolfssl.so.4.8.1.39c36f2f 1248704 staging_dir/target-mipsel_24kc_musl/usr/lib/libwolfssl.so.4.8.1.39c36f2f [0] https://github.com/openwrt/packages/issues/16674 [1] https://github.com/wolfSSL/wolfssl/issues/4443#issuecomment-934926793 Signed-off-by: Andre Heider [bump PKG_RELEASE] Signed-off-by: David Bauer --- package/libs/wolfssl/Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/package/libs/wolfssl/Makefile b/package/libs/wolfssl/Makefile index 6ef80e88a9..e52cff30b1 100644 --- a/package/libs/wolfssl/Makefile +++ b/package/libs/wolfssl/Makefile @@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=wolfssl PKG_VERSION:=4.8.1-stable -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://github.com/wolfSSL/wolfssl/archive/v$(PKG_VERSION) @@ -59,7 +59,13 @@ define Package/libwolfssl/config source "$(SOURCE)/Config.in" endef -TARGET_CFLAGS += $(FPIC) -DFP_MAX_BITS=8192 -fomit-frame-pointer -flto +TARGET_CFLAGS += \ + $(FPIC) \ + -fomit-frame-pointer \ + -flto \ + -DFP_MAX_BITS=8192 \ + -DWOLFSSL_ALT_CERT_CHAINS + TARGET_LDFLAGS += -flto # --enable-stunnel needed for OpenSSL API compatibility bits From c76300707e8d705d9efc7ed4bb1b7449d0a5fe00 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Sat, 2 Oct 2021 10:50:49 +0200 Subject: [PATCH 20/22] wolfssl: always build with --enable-reproducible-build This gates out anything that might introduce semantically frivolous jitter, maximizing chance of identical object files. The binary size shrinks by 8kb: 1244352 staging_dir/target-mipsel_24kc_musl/usr/lib/libwolfssl.so.4.8.1.39c36f2f 1236160 staging_dir/target-mipsel_24kc_musl/usr/lib/libwolfssl.so.4.8.1.39c36f2f Signed-off-by: Andre Heider --- package/libs/wolfssl/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/package/libs/wolfssl/Makefile b/package/libs/wolfssl/Makefile index e52cff30b1..edbe95b654 100644 --- a/package/libs/wolfssl/Makefile +++ b/package/libs/wolfssl/Makefile @@ -70,6 +70,7 @@ TARGET_LDFLAGS += -flto # --enable-stunnel needed for OpenSSL API compatibility bits CONFIGURE_ARGS += \ + --enable-reproducible-build \ --enable-lighty \ --enable-opensslall \ --enable-opensslextra \ From 7cb5af30f48d6788cd471138820a772610a7f8e0 Mon Sep 17 00:00:00 2001 From: Andre Heider Date: Sat, 2 Oct 2021 10:40:00 +0200 Subject: [PATCH 21/22] wolfssl: remove --enable-sha512 configure switch It's the default anyway and this just looks confusing, as if it wasn't. Switch to AUTORELEASE while at it. The binary size is unchanged. Signed-off-by: Andre Heider --- package/libs/wolfssl/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/libs/wolfssl/Makefile b/package/libs/wolfssl/Makefile index edbe95b654..93f2cc3ac3 100644 --- a/package/libs/wolfssl/Makefile +++ b/package/libs/wolfssl/Makefile @@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=wolfssl PKG_VERSION:=4.8.1-stable -PKG_RELEASE:=2 +PKG_RELEASE:=$(AUTORELEASE) PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://github.com/wolfSSL/wolfssl/archive/v$(PKG_VERSION) @@ -104,7 +104,7 @@ endif ifeq ($(CONFIG_WOLFSSL_HAS_WPAS),y) CONFIGURE_ARGS += \ - --enable-wpas --enable-sha512 --enable-fortress --enable-fastmath + --enable-wpas --enable-fortress --enable-fastmath endif define Build/InstallDev From 333f93333ee52309ce664dcf78a84b79d50f9f48 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Sun, 17 Oct 2021 21:21:49 +0100 Subject: [PATCH 22/22] procd: update to git HEAD 9b1e035 jail: netifd: code cosmetics d2a2ecc jail: netifd: fix error handling issue reported by coverity e1d7cee jail: netifd: check target netns fd before using it 59f7699 uxc: add missing 'break' statement Signed-off-by: Daniel Golle --- package/system/procd/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package/system/procd/Makefile b/package/system/procd/Makefile index 4c76045062..e5f4c2a50a 100644 --- a/package/system/procd/Makefile +++ b/package/system/procd/Makefile @@ -12,9 +12,9 @@ PKG_RELEASE:=$(AUTORELEASE) PKG_SOURCE_PROTO:=git PKG_SOURCE_URL=$(PROJECT_GIT)/project/procd.git -PKG_SOURCE_DATE:=2021-10-13 -PKG_SOURCE_VERSION:=82dd39024f63e53becd07c89831ed7d5fac7f339 -PKG_MIRROR_HASH:=a9449894b4ce0b2860680c1691270d5fe9c84e4eecd48008ed732ef2d19af7db +PKG_SOURCE_DATE:=2021-10-17 +PKG_SOURCE_VERSION:=59f769920276229a3ee2dcbe75ea54095cf14ffe +PKG_MIRROR_HASH:=2514d914bf317c5965ef90440e2c6bd8c409805db02348af4ed4aa58a0496fcf CMAKE_INSTALL:=1 PKG_LICENSE:=GPL-2.0