diff --git a/config/Config-build.in b/config/Config-build.in index f0e1aaa695..ca6f513450 100644 --- a/config/Config-build.in +++ b/config/Config-build.in @@ -386,4 +386,16 @@ menu "Global build settings" endchoice + config SECCOMP + bool "Enable SECCOMP" + select KERNEL_SECCOMP + select PACKAGE_procd-seccomp + depends on (aarch64 || arm || armeb || mips || mipsel || i386 || powerpc || x86_64) + depends on !TARGET_uml + default y + help + This option enables seccomp kernel features to safely + execute untrusted bytecode and selects the seccomp-variants + of procd + endmenu diff --git a/include/target.mk b/include/target.mk index 7e41ae3f1b..8fc51913a0 100644 --- a/include/target.mk +++ b/include/target.mk @@ -39,7 +39,7 @@ DEFAULT_PACKAGES+=procd-ujail endif # include seccomp ld-preload hooks if kernel supports it -ifneq ($(CONFIG_KERNEL_SECCOMP),) +ifneq ($(CONFIG_SECCOMP),) DEFAULT_PACKAGES+=procd-seccomp endif diff --git a/package/kernel/mt76/Makefile b/package/kernel/mt76/Makefile index 511960ce9b..97cb9e8d59 100644 --- a/package/kernel/mt76/Makefile +++ b/package/kernel/mt76/Makefile @@ -8,9 +8,9 @@ PKG_LICENSE_FILES:= PKG_SOURCE_URL:=https://github.com/openwrt/mt76 PKG_SOURCE_PROTO:=git -PKG_SOURCE_DATE:=2021-10-11 -PKG_SOURCE_VERSION:=1af0242d1241cba761415c492e5ed9673beaa13f -PKG_MIRROR_HASH:=39346f1ef24fbd62f447fc116e1ed411dc2b60787a3df9edfe048adc79150232 +PKG_SOURCE_DATE:=2021-10-12 +PKG_SOURCE_VERSION:=83598c2e872fd8af29976a5f11afe94e620d6f4f +PKG_MIRROR_HASH:=f179375318c44fb058403aed98056c4df8df7541e4c9d093ec5c22a2eda9475e PKG_MAINTAINER:=Felix Fietkau PKG_USE_NINJA:=0 diff --git a/package/libs/wolfssl/Makefile b/package/libs/wolfssl/Makefile index 6ef80e88a9..93f2cc3ac3 100644 --- a/package/libs/wolfssl/Makefile +++ b/package/libs/wolfssl/Makefile @@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=wolfssl PKG_VERSION:=4.8.1-stable -PKG_RELEASE:=1 +PKG_RELEASE:=$(AUTORELEASE) PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://github.com/wolfSSL/wolfssl/archive/v$(PKG_VERSION) @@ -59,11 +59,18 @@ define Package/libwolfssl/config source "$(SOURCE)/Config.in" endef -TARGET_CFLAGS += $(FPIC) -DFP_MAX_BITS=8192 -fomit-frame-pointer -flto +TARGET_CFLAGS += \ + $(FPIC) \ + -fomit-frame-pointer \ + -flto \ + -DFP_MAX_BITS=8192 \ + -DWOLFSSL_ALT_CERT_CHAINS + TARGET_LDFLAGS += -flto # --enable-stunnel needed for OpenSSL API compatibility bits CONFIGURE_ARGS += \ + --enable-reproducible-build \ --enable-lighty \ --enable-opensslall \ --enable-opensslextra \ @@ -97,7 +104,7 @@ endif ifeq ($(CONFIG_WOLFSSL_HAS_WPAS),y) CONFIGURE_ARGS += \ - --enable-wpas --enable-sha512 --enable-fortress --enable-fastmath + --enable-wpas --enable-fortress --enable-fastmath endif define Build/InstallDev diff --git a/package/network/config/netifd/Makefile b/package/network/config/netifd/Makefile index 5e6ebbde12..727c3ac690 100644 --- a/package/network/config/netifd/Makefile +++ b/package/network/config/netifd/Makefile @@ -5,9 +5,9 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL=$(PROJECT_GIT)/project/netifd.git -PKG_SOURCE_DATE:=2021-09-29 -PKG_SOURCE_VERSION:=448ffc154fe78ee1c8e4b2cda1a479f0d8af8b37 -PKG_MIRROR_HASH:=eec346ae3e27b8e37b91600e1ee822504b8715bbaf360a6bd962e61b6a6ad13e +PKG_SOURCE_DATE:=2021-10-13 +PKG_SOURCE_VERSION:=7a048bd6871df952ff1324bec7175891c9b38f99 +PKG_MIRROR_HASH:=b4fdca3c45713710f4ea69f1ed0fb16d3b07b70002e6b39640ce6e56364817d0 PKG_MAINTAINER:=Felix Fietkau PKG_LICENSE:=GPL-2.0 diff --git a/package/network/services/hostapd/patches/600-ubus_support.patch b/package/network/services/hostapd/patches/600-ubus_support.patch index 005ed54e0c..d77d245243 100644 --- a/package/network/services/hostapd/patches/600-ubus_support.patch +++ b/package/network/services/hostapd/patches/600-ubus_support.patch @@ -514,3 +514,33 @@ struct os_reltime backlogged_until; #endif /* CONFIG_AIRTIME_POLICY */ +--- a/src/ap/wnm_ap.c ++++ b/src/ap/wnm_ap.c +@@ -463,7 +463,7 @@ static void ieee802_11_rx_bss_trans_mgmt + size_t len) + { + u8 dialog_token, status_code, bss_termination_delay; +- const u8 *pos, *end; ++ const u8 *pos, *end, *target_bssid; + int enabled = hapd->conf->bss_transition; + struct sta_info *sta; + +@@ -510,6 +510,7 @@ static void ieee802_11_rx_bss_trans_mgmt + wpa_printf(MSG_DEBUG, "WNM: not enough room for Target BSSID field"); + return; + } ++ target_bssid = pos; + sta->agreed_to_steer = 1; + eloop_cancel_timeout(ap_sta_reset_steer_flag_timer, hapd, sta); + eloop_register_timeout(2, 0, ap_sta_reset_steer_flag_timer, +@@ -529,6 +530,10 @@ static void ieee802_11_rx_bss_trans_mgmt + MAC2STR(addr), status_code, bss_termination_delay); + } + ++ hostapd_ubus_notify_bss_transition_response(hapd, sta->addr, dialog_token, ++ status_code, bss_termination_delay, ++ target_bssid, pos, end - pos); ++ + wpa_hexdump(MSG_DEBUG, "WNM: BSS Transition Candidate List Entries", + pos, end - pos); + } diff --git a/package/network/services/hostapd/src/src/ap/ubus.c b/package/network/services/hostapd/src/src/ap/ubus.c index 367e1b652b..69778596a4 100644 --- a/package/network/services/hostapd/src/src/ap/ubus.c +++ b/package/network/services/hostapd/src/src/ap/ubus.c @@ -1275,6 +1275,129 @@ hostapd_rrm_beacon_req(struct ubus_context *ctx, struct ubus_object *obj, #ifdef CONFIG_WNM_AP + +static int +hostapd_bss_tr_send(struct hostapd_data *hapd, u8 *addr, bool disassoc_imminent, bool abridged, + u16 disassoc_timer, u8 validity_period, struct blob_attr *neighbors) +{ + struct blob_attr *cur; + struct sta_info *sta; + int nr_len = 0; + int rem; + u8 *nr = NULL; + u8 req_mode = 0; + + sta = ap_get_sta(hapd, addr); + if (!sta) + return UBUS_STATUS_NOT_FOUND; + + if (neighbors) { + u8 *nr_cur; + + if (blobmsg_check_array(neighbors, + BLOBMSG_TYPE_STRING) < 0) + return UBUS_STATUS_INVALID_ARGUMENT; + + blobmsg_for_each_attr(cur, neighbors, rem) { + int len = strlen(blobmsg_get_string(cur)); + + if (len % 2) + return UBUS_STATUS_INVALID_ARGUMENT; + + nr_len += (len / 2) + 2; + } + + if (nr_len) { + nr = os_zalloc(nr_len); + if (!nr) + return UBUS_STATUS_UNKNOWN_ERROR; + } + + nr_cur = nr; + blobmsg_for_each_attr(cur, neighbors, rem) { + int len = strlen(blobmsg_get_string(cur)) / 2; + + *nr_cur++ = WLAN_EID_NEIGHBOR_REPORT; + *nr_cur++ = (u8) len; + if (hexstr2bin(blobmsg_data(cur), nr_cur, len)) { + free(nr); + return UBUS_STATUS_INVALID_ARGUMENT; + } + + nr_cur += len; + } + } + + if (nr) + req_mode |= WNM_BSS_TM_REQ_PREF_CAND_LIST_INCLUDED; + + if (abridged) + req_mode |= WNM_BSS_TM_REQ_ABRIDGED; + + if (disassoc_imminent) + req_mode |= WNM_BSS_TM_REQ_DISASSOC_IMMINENT; + + if (wnm_send_bss_tm_req(hapd, sta, req_mode, disassoc_timer, validity_period, NULL, + NULL, nr, nr_len, NULL, 0)) + return UBUS_STATUS_UNKNOWN_ERROR; + + return 0; +} + +enum { + BSS_TR_ADDR, + BSS_TR_DA_IMMINENT, + BSS_TR_DA_TIMER, + BSS_TR_VALID_PERIOD, + BSS_TR_NEIGHBORS, + BSS_TR_ABRIDGED, + __BSS_TR_DISASSOC_MAX +}; + +static const struct blobmsg_policy bss_tr_policy[__BSS_TR_DISASSOC_MAX] = { + [BSS_TR_ADDR] = { "addr", BLOBMSG_TYPE_STRING }, + [BSS_TR_DA_IMMINENT] = { "disassociation_imminent", BLOBMSG_TYPE_BOOL }, + [BSS_TR_DA_TIMER] = { "disassociation_timer", BLOBMSG_TYPE_INT32 }, + [BSS_TR_VALID_PERIOD] = { "validity_period", BLOBMSG_TYPE_INT32 }, + [BSS_TR_NEIGHBORS] = { "neighbors", BLOBMSG_TYPE_ARRAY }, + [BSS_TR_ABRIDGED] = { "abridged", BLOBMSG_TYPE_BOOL }, +}; + +static int +hostapd_bss_transition_request(struct ubus_context *ctx, struct ubus_object *obj, + struct ubus_request_data *ureq, const char *method, + struct blob_attr *msg) +{ + struct hostapd_data *hapd = container_of(obj, struct hostapd_data, ubus.obj); + struct blob_attr *tb[__BSS_TR_DISASSOC_MAX]; + struct sta_info *sta; + u32 da_timer = 0; + u32 valid_period = 0; + u8 addr[ETH_ALEN]; + bool abridged; + bool da_imminent; + + blobmsg_parse(bss_tr_policy, __BSS_TR_DISASSOC_MAX, tb, blob_data(msg), blob_len(msg)); + + if (!tb[BSS_TR_ADDR]) + return UBUS_STATUS_INVALID_ARGUMENT; + + if (hwaddr_aton(blobmsg_data(tb[BSS_TR_ADDR]), addr)) + return UBUS_STATUS_INVALID_ARGUMENT; + + if (tb[BSS_TR_DA_TIMER]) + da_timer = blobmsg_get_u32(tb[BSS_TR_DA_TIMER]); + + if (tb[BSS_TR_VALID_PERIOD]) + valid_period = blobmsg_get_u32(tb[BSS_TR_VALID_PERIOD]); + + da_imminent = !!(tb[BSS_TR_DA_IMMINENT] && blobmsg_get_bool(tb[BSS_TR_DA_IMMINENT])); + abridged = !!(tb[BSS_TR_ABRIDGED] && blobmsg_get_bool(tb[BSS_TR_ABRIDGED])); + + return hostapd_bss_tr_send(hapd, addr, da_imminent, abridged, da_timer, valid_period, + tb[BSS_TR_NEIGHBORS]); +} + enum { WNM_DISASSOC_ADDR, WNM_DISASSOC_DURATION, @@ -1297,14 +1420,10 @@ hostapd_wnm_disassoc_imminent(struct ubus_context *ctx, struct ubus_object *obj, { struct hostapd_data *hapd = container_of(obj, struct hostapd_data, ubus.obj); struct blob_attr *tb[__WNM_DISASSOC_MAX]; - struct blob_attr *cur; struct sta_info *sta; int duration = 10; - int rem; - int nr_len = 0; - u8 *nr = NULL; - u8 req_mode = WNM_BSS_TM_REQ_DISASSOC_IMMINENT; u8 addr[ETH_ALEN]; + bool abridged; blobmsg_parse(wnm_disassoc_policy, __WNM_DISASSOC_MAX, tb, blob_data(msg), blob_len(msg)); @@ -1314,61 +1433,13 @@ hostapd_wnm_disassoc_imminent(struct ubus_context *ctx, struct ubus_object *obj, if (hwaddr_aton(blobmsg_data(tb[WNM_DISASSOC_ADDR]), addr)) return UBUS_STATUS_INVALID_ARGUMENT; - if ((cur = tb[WNM_DISASSOC_DURATION]) != NULL) - duration = blobmsg_get_u32(cur); + if (tb[WNM_DISASSOC_DURATION]) + duration = blobmsg_get_u32(tb[WNM_DISASSOC_DURATION]); - sta = ap_get_sta(hapd, addr); - if (!sta) - return UBUS_STATUS_NOT_FOUND; + abridged = !!(tb[WNM_DISASSOC_ABRIDGED] && blobmsg_get_bool(tb[WNM_DISASSOC_ABRIDGED])); - if (tb[WNM_DISASSOC_NEIGHBORS]) { - u8 *nr_cur; - - if (blobmsg_check_array(tb[WNM_DISASSOC_NEIGHBORS], - BLOBMSG_TYPE_STRING) < 0) - return UBUS_STATUS_INVALID_ARGUMENT; - - blobmsg_for_each_attr(cur, tb[WNM_DISASSOC_NEIGHBORS], rem) { - int len = strlen(blobmsg_get_string(cur)); - - if (len % 2) - return UBUS_STATUS_INVALID_ARGUMENT; - - nr_len += (len / 2) + 2; - } - - if (nr_len) { - nr = os_zalloc(nr_len); - if (!nr) - return UBUS_STATUS_UNKNOWN_ERROR; - } - - nr_cur = nr; - blobmsg_for_each_attr(cur, tb[WNM_DISASSOC_NEIGHBORS], rem) { - int len = strlen(blobmsg_get_string(cur)) / 2; - - *nr_cur++ = WLAN_EID_NEIGHBOR_REPORT; - *nr_cur++ = (u8) len; - if (hexstr2bin(blobmsg_data(cur), nr_cur, len)) { - free(nr); - return UBUS_STATUS_INVALID_ARGUMENT; - } - - nr_cur += len; - } - } - - if (nr) - req_mode |= WNM_BSS_TM_REQ_PREF_CAND_LIST_INCLUDED; - - if (tb[WNM_DISASSOC_ABRIDGED] && blobmsg_get_bool(tb[WNM_DISASSOC_ABRIDGED])) - req_mode |= WNM_BSS_TM_REQ_ABRIDGED; - - if (wnm_send_bss_tm_req(hapd, sta, req_mode, duration, duration, NULL, - NULL, nr, nr_len, NULL, 0)) - return UBUS_STATUS_UNKNOWN_ERROR; - - return 0; + return hostapd_bss_tr_send(hapd, addr, true, abridged, duration, duration, + tb[WNM_DISASSOC_NEIGHBORS]); } #endif @@ -1454,6 +1525,7 @@ static const struct ubus_method bss_methods[] = { UBUS_METHOD("rrm_beacon_req", hostapd_rrm_beacon_req, beacon_req_policy), #ifdef CONFIG_WNM_AP UBUS_METHOD("wnm_disassoc_imminent", hostapd_wnm_disassoc_imminent, wnm_disassoc_policy), + UBUS_METHOD("bss_transition_request", hostapd_bss_transition_request, bss_tr_policy), #endif }; @@ -1757,3 +1829,36 @@ void hostapd_ubus_notify_radar_detected(struct hostapd_iface *iface, int frequen ubus_notify(ctx, &hapd->ubus.obj, "radar-detected", b.head, -1); } } + +void hostapd_ubus_notify_bss_transition_response( + struct hostapd_data *hapd, const u8 *addr, u8 dialog_token, u8 status_code, + u8 bss_termination_delay, const u8 *target_bssid, + const u8 *candidate_list, u16 candidate_list_len) +{ +#ifdef CONFIG_WNM_AP + char *cl_str; + u16 i; + + if (!hapd->ubus.obj.has_subscribers) + return; + + if (!addr) + return; + + blob_buf_init(&b, 0); + blobmsg_add_macaddr(&b, "address", addr); + blobmsg_add_u8(&b, "dialog-token", dialog_token); + blobmsg_add_u8(&b, "status-code", status_code); + blobmsg_add_u8(&b, "bss-termination-delay", bss_termination_delay); + if (target_bssid) + blobmsg_add_macaddr(&b, "target-bssid", target_bssid); + if (candidate_list_len > 0) { + cl_str = blobmsg_alloc_string_buffer(&b, "candidate-list", candidate_list_len * 2 + 1); + for (i = 0; i < candidate_list_len; i++) + snprintf(&cl_str[i*2], 3, "%02X", candidate_list[i]); + blobmsg_add_string_buffer(&b); + } + + ubus_notify(ctx, &hapd->ubus.obj, "bss-transition-response", b.head, -1); +#endif +} diff --git a/package/network/services/hostapd/src/src/ap/ubus.h b/package/network/services/hostapd/src/src/ap/ubus.h index acdac74360..899b001dd5 100644 --- a/package/network/services/hostapd/src/src/ap/ubus.h +++ b/package/network/services/hostapd/src/src/ap/ubus.h @@ -55,6 +55,10 @@ void hostapd_ubus_notify_beacon_report(struct hostapd_data *hapd, void hostapd_ubus_notify_radar_detected(struct hostapd_iface *iface, int frequency, int chan_width, int cf1, int cf2); +void hostapd_ubus_notify_bss_transition_response( + struct hostapd_data *hapd, const u8 *addr, u8 dialog_token, u8 status_code, + u8 bss_termination_delay, const u8 *target_bssid, + const u8 *candidate_list, u16 candidate_list_len); void hostapd_ubus_add(struct hapd_interfaces *interfaces); void hostapd_ubus_free(struct hapd_interfaces *interfaces); @@ -107,6 +111,13 @@ static inline void hostapd_ubus_notify_radar_detected(struct hostapd_iface *ifac { } +static inline void hostapd_ubus_notify_bss_transition_response( + struct hostapd_data *hapd, const u8 *addr, u8 dialog_token, u8 status_code, + u8 bss_termination_delay, const u8 *target_bssid, + const u8 *candidate_list, u16 candidate_list_len) +{ +} + static inline void hostapd_ubus_add(struct hapd_interfaces *interfaces) { } diff --git a/package/system/procd/Makefile b/package/system/procd/Makefile index 1e43c16330..e5f4c2a50a 100644 --- a/package/system/procd/Makefile +++ b/package/system/procd/Makefile @@ -12,9 +12,9 @@ PKG_RELEASE:=$(AUTORELEASE) PKG_SOURCE_PROTO:=git PKG_SOURCE_URL=$(PROJECT_GIT)/project/procd.git -PKG_SOURCE_DATE:=2021-09-15 -PKG_SOURCE_VERSION:=1eb4371e2534296d04580cb8b9cb5baa5f07e27d -PKG_MIRROR_HASH:=d0e61032482eebb225ffbca8034b3d838eaa3db0dcd9b0166e106e729ce5320c +PKG_SOURCE_DATE:=2021-10-17 +PKG_SOURCE_VERSION:=59f769920276229a3ee2dcbe75ea54095cf14ffe +PKG_MIRROR_HASH:=2514d914bf317c5965ef90440e2c6bd8c409805db02348af4ed4aa58a0496fcf CMAKE_INSTALL:=1 PKG_LICENSE:=GPL-2.0 @@ -68,7 +68,7 @@ define Package/procd-ujail SECTION:=base CATEGORY:=Base system DEPENDS:=@KERNEL_NAMESPACES +@KERNEL_UTS_NS +@KERNEL_IPC_NS +@KERNEL_PID_NS \ - +libubox +libubus +libblobmsg-json + +libubox +libubus +libuci +libblobmsg-json TITLE:=OpenWrt process jail helper endef @@ -82,8 +82,7 @@ endef define Package/procd-seccomp SECTION:=base CATEGORY:=Base system - DEPENDS:=@(aarch64||arm||armeb||mips||mipsel||i386||powerpc||x86_64) @!TARGET_uml \ - @KERNEL_SECCOMP +libubox +libblobmsg-json + DEPENDS:=@SECCOMP +libubox +libblobmsg-json TITLE:=OpenWrt process seccomp helper + utrace endef diff --git a/target/linux/armvirt/config-5.10 b/target/linux/armvirt/config-5.10 index 5715a74195..74b6957d94 100644 --- a/target/linux/armvirt/config-5.10 +++ b/target/linux/armvirt/config-5.10 @@ -156,6 +156,7 @@ CONFIG_VIRTIO_BALLOON=y CONFIG_VIRTIO_BLK=y CONFIG_VIRTIO_CONSOLE=y CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y CONFIG_VIRTIO_NET=y CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_PCI_LEGACY=y diff --git a/target/linux/generic/backport-5.10/050-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch b/target/linux/generic/backport-5.10/050-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch new file mode 100644 index 0000000000..82feb7421d --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch @@ -0,0 +1,65 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:02 +0200 +Subject: [PATCH] MIPS: uasm: Enable muhu opcode for MIPS R6 + +Enable the 'muhu' instruction, complementing the existing 'mulu', needed +to implement a MIPS32 BPF JIT. + +Also fix a typo in the existing definition of 'dmulu'. + +Signed-off-by: Tony Ambardar + +This patch is a dependency for my 32-bit MIPS eBPF JIT. + +Signed-off-by: Johan Almbladh +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -145,6 +145,7 @@ Ip_u1(_mtlo); + Ip_u3u1u2(_mul); + Ip_u1u2(_multu); + Ip_u3u1u2(_mulu); ++Ip_u3u1u2(_muhu); + Ip_u3u1u2(_nor); + Ip_u3u1u2(_or); + Ip_u2u1u3(_ori); +--- a/arch/mips/mm/uasm-mips.c ++++ b/arch/mips/mm/uasm-mips.c +@@ -90,7 +90,7 @@ static const struct insn insn_table[insn + RS | RT | RD}, + [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, + [insn_dmultu] = {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT}, +- [insn_dmulu] = {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op), ++ [insn_dmulu] = {M(spec_op, 0, 0, 0, dmultu_dmulu_op, dmultu_op), + RS | RT | RD}, + [insn_drotr] = {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE}, + [insn_drotr32] = {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE}, +@@ -150,6 +150,8 @@ static const struct insn insn_table[insn + [insn_mtlo] = {M(spec_op, 0, 0, 0, 0, mtlo_op), RS}, + [insn_mulu] = {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op), + RS | RT | RD}, ++ [insn_muhu] = {M(spec_op, 0, 0, 0, multu_muhu_op, multu_op), ++ RS | RT | RD}, + #ifndef CONFIG_CPU_MIPSR6 + [insn_mul] = {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD}, + #else +--- a/arch/mips/mm/uasm.c ++++ b/arch/mips/mm/uasm.c +@@ -59,7 +59,7 @@ enum opcode { + insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, + insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, + insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, +- insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor, ++ insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, + insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, + insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, + insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, +@@ -344,6 +344,7 @@ I_u1(_mtlo) + I_u3u1u2(_mul) + I_u1u2(_multu) + I_u3u1u2(_mulu) ++I_u3u1u2(_muhu) + I_u3u1u2(_nor) + I_u3u1u2(_or) + I_u2u1u3(_ori) diff --git a/target/linux/generic/backport-5.10/050-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch b/target/linux/generic/backport-5.10/050-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch new file mode 100644 index 0000000000..3a4d573f80 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch @@ -0,0 +1,31 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:03 +0200 +Subject: [PATCH] mips: uasm: Add workaround for Loongson-2F nop CPU errata + +This patch implements a workaround for the Loongson-2F nop in generated, +code, if the existing option CONFIG_CPU_NOP_WORKAROUND is set. Before, +the binutils option -mfix-loongson2f-nop was enabled, but no workaround +was done when emitting MIPS code. Now, the nop pseudo instruction is +emitted as "or ax,ax,zero" instead of the default "sll zero,zero,0". This +is consistent with the workaround implemented by binutils. + +Link: https://sourceware.org/legacy-ml/binutils/2009-11/msg00387.html + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -249,7 +249,11 @@ static inline void uasm_l##lb(struct uas + #define uasm_i_bnezl(buf, rs, off) uasm_i_bnel(buf, rs, 0, off) + #define uasm_i_ehb(buf) uasm_i_sll(buf, 0, 0, 3) + #define uasm_i_move(buf, a, b) UASM_i_ADDU(buf, a, 0, b) ++#ifdef CONFIG_CPU_NOP_WORKAROUNDS ++#define uasm_i_nop(buf) uasm_i_or(buf, 1, 1, 0) ++#else + #define uasm_i_nop(buf) uasm_i_sll(buf, 0, 0, 0) ++#endif + #define uasm_i_ssnop(buf) uasm_i_sll(buf, 0, 0, 1) + + static inline void uasm_i_drotr_safe(u32 **p, unsigned int a1, diff --git a/target/linux/generic/backport-5.10/050-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch b/target/linux/generic/backport-5.10/050-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch new file mode 100644 index 0000000000..7980659961 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch @@ -0,0 +1,3078 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:04 +0200 +Subject: [PATCH] mips: bpf: Add eBPF JIT for 32-bit MIPS + +This is an implementation of an eBPF JIT for 32-bit MIPS I-V and MIPS32. +The implementation supports all 32-bit and 64-bit ALU and JMP operations, +including the recently-added atomics. 64-bit div/mod and 64-bit atomics +are implemented using function calls to math64 and atomic64 functions, +respectively. All 32-bit operations are implemented natively by the JIT, +except if the CPU lacks ll/sc instructions. + +Register mapping +================ +All 64-bit eBPF registers are mapped to native 32-bit MIPS register pairs, +and does not use any stack scratch space for register swapping. This means +that all eBPF register data is kept in CPU registers all the time, and +this simplifies the register management a lot. It also reduces the JIT's +pressure on temporary registers since we do not have to move data around. + +Native register pairs are ordered according to CPU endiannes, following +the O32 calling convention for passing 64-bit arguments and return values. +The eBPF return value, arguments and callee-saved registers are mapped to +their native MIPS equivalents. + +Since the 32 highest bits in the eBPF FP (frame pointer) register are +always zero, only one general-purpose register is actually needed for the +mapping. The MIPS fp register is used for this purpose. The high bits are +mapped to MIPS register r0. This saves us one CPU register, which is much +needed for temporaries, while still allowing us to treat the R10 (FP) +register just like any other eBPF register in the JIT. + +The MIPS gp (global pointer) and at (assembler temporary) registers are +used as internal temporary registers for constant blinding. CPU registers +t6-t9 are used internally by the JIT when constructing more complex 64-bit +operations. This is precisely what is needed - two registers to store an +operand value, and two more as scratch registers when performing the +operation. + +The register mapping is shown below. + + R0 - $v1, $v0 return value + R1 - $a1, $a0 argument 1, passed in registers + R2 - $a3, $a2 argument 2, passed in registers + R3 - $t1, $t0 argument 3, passed on stack + R4 - $t3, $t2 argument 4, passed on stack + R5 - $t4, $t3 argument 5, passed on stack + R6 - $s1, $s0 callee-saved + R7 - $s3, $s2 callee-saved + R8 - $s5, $s4 callee-saved + R9 - $s7, $s6 callee-saved + FP - $r0, $fp 32-bit frame pointer + AX - $gp, $at constant-blinding + $t6 - $t9 unallocated, JIT temporaries + +Jump offsets +============ +The JIT tries to map all conditional JMP operations to MIPS conditional +PC-relative branches. The MIPS branch offset field is 18 bits, in bytes, +which is equivalent to the eBPF 16-bit instruction offset. However, since +the JIT may emit more than one CPU instruction per eBPF instruction, the +field width may overflow. If that happens, the JIT converts the long +conditional jump to a short PC-relative branch with the condition +inverted, jumping over a long unconditional absolute jmp (j). + +This conversion will change the instruction offset mapping used for jumps, +and may in turn result in more branch offset overflows. The JIT therefore +dry-runs the translation until no more branches are converted and the +offsets do not change anymore. There is an upper bound on this of course, +and if the JIT hits that limit, the last two iterations are run with all +branches being converted. + +Tail call count +=============== +The current tail call count is stored in the 16-byte area of the caller's +stack frame that is reserved for the callee in the o32 ABI. The value is +initialized in the prologue, and propagated to the tail-callee by skipping +the initialization instructions when emitting the tail call. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp.c + create mode 100644 arch/mips/net/bpf_jit_comp.h + create mode 100644 arch/mips/net/bpf_jit_comp32.c + +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,4 +2,9 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o +-obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ ++ifeq ($(CONFIG_32BIT),y) ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++else ++ obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++endif +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -0,0 +1,1020 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions common to 32-bit and 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++/* ++ * Code overview ++ * ============= ++ * ++ * - bpf_jit_comp.h ++ * Common definitions and utilities. ++ * ++ * - bpf_jit_comp.c ++ * Implementation of JIT top-level logic and exported JIT API functions. ++ * Implementation of internal operations shared by 32-bit and 64-bit code. ++ * JMP and ALU JIT control code, register control code, shared ALU and ++ * JMP/JMP32 JIT operations. ++ * ++ * - bpf_jit_comp32.c ++ * Implementation of functions to JIT prologue, epilogue and a single eBPF ++ * instruction for 32-bit MIPS CPUs. The functions use shared operations ++ * where possible, and implement the rest for 32-bit MIPS such as ALU64 ++ * operations. ++ * ++ * - bpf_jit_comp64.c ++ * Ditto, for 64-bit MIPS CPUs. ++ * ++ * Zero and sign extension ++ * ======================== ++ * 32-bit MIPS instructions on 64-bit MIPS registers use sign extension, ++ * but the eBPF instruction set mandates zero extension. We let the verifier ++ * insert explicit zero-extensions after 32-bit ALU operations, both for ++ * 32-bit and 64-bit MIPS JITs. Conditional JMP32 operations on 64-bit MIPs ++ * are JITed with sign extensions inserted when so expected. ++ * ++ * ALU operations ++ * ============== ++ * ALU operations on 32/64-bit MIPS and ALU64 operations on 64-bit MIPS are ++ * JITed in the following steps. ALU64 operations on 32-bit MIPS are more ++ * complicated and therefore only processed by special implementations in ++ * step (3). ++ * ++ * 1) valid_alu_i: ++ * Determine if an immediate operation can be emitted as such, or if ++ * we must fall back to the register version. ++ * ++ * 2) rewrite_alu_i: ++ * Convert BPF operation and immediate value to a canonical form for ++ * JITing. In some degenerate cases this form may be a no-op. ++ * ++ * 3) emit_alu_{i,i64,r,64}: ++ * Emit instructions for an ALU or ALU64 immediate or register operation. ++ * ++ * JMP operations ++ * ============== ++ * JMP and JMP32 operations require an JIT instruction offset table for ++ * translating the jump offset. This table is computed by dry-running the ++ * JIT without actually emitting anything. However, the computed PC-relative ++ * offset may overflow the 18-bit offset field width of the native MIPS ++ * branch instruction. In such cases, the long jump is converted into the ++ * following sequence. ++ * ++ * ! +2 Inverted PC-relative branch ++ * nop Delay slot ++ * j Unconditional absolute long jump ++ * nop Delay slot ++ * ++ * Since this converted sequence alters the offset table, all offsets must ++ * be re-calculated. This may in turn trigger new branch conversions, so ++ * the process is repeated until no further changes are made. Normally it ++ * completes in 1-2 iterations. If JIT_MAX_ITERATIONS should reached, we ++ * fall back to converting every remaining jump operation. The branch ++ * conversion is independent of how the JMP or JMP32 condition is JITed. ++ * ++ * JMP32 and JMP operations are JITed as follows. ++ * ++ * 1) setup_jmp_{i,r}: ++ * Convert jump conditional and offset into a form that can be JITed. ++ * This form may be a no-op, a canonical form, or an inverted PC-relative ++ * jump if branch conversion is necessary. ++ * ++ * 2) valid_jmp_i: ++ * Determine if an immediate operations can be emitted as such, or if ++ * we must fall back to the register version. Applies to JMP32 for 32-bit ++ * MIPS, and both JMP and JMP32 for 64-bit MIPS. ++ * ++ * 3) emit_jmp_{i,i64,r,r64}: ++ * Emit instructions for an JMP or JMP32 immediate or register operation. ++ * ++ * 4) finish_jmp_{i,r}: ++ * Emit any instructions needed to finish the jump. This includes a nop ++ * for the delay slot if a branch was emitted, and a long absolute jump ++ * if the branch was converted. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* Convenience macros for descriptor access */ ++#define CONVERTED(desc) ((desc) & JIT_DESC_CONVERT) ++#define INDEX(desc) ((desc) & ~JIT_DESC_CONVERT) ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, sw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, sd, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ ctx->stack_used = max((int)ctx->stack_used, depth); ++ return depth; ++} ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, lw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, ld, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ return depth; ++} ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc) ++{ ++ u32 index = INDEX(ctx->descriptors[loc]); ++ unsigned long pc = (unsigned long)&ctx->target[ctx->jit_index]; ++ unsigned long addr = (unsigned long)&ctx->target[index]; ++ ++ if (!ctx->target) ++ return 0; ++ ++ if ((addr ^ pc) & ~MIPS_JMP_MASK) ++ return -1; ++ ++ return addr & MIPS_JMP_MASK; ++} ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off) ++{ ++ return (INDEX(ctx->descriptors[ctx->bpf_index + off]) - ++ ctx->jit_index - 1) * sizeof(u32); ++} ++ ++/* dst = imm (register width) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm) ++{ ++ if (imm >= -0x8000 && imm <= 0x7fff) { ++ emit(ctx, addiu, dst, MIPS_R_ZERO, imm); ++ } else { ++ emit(ctx, lui, dst, (s16)((u32)imm >> 16)); ++ emit(ctx, ori, dst, dst, (u16)(imm & 0xffff)); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* dst = src (register width) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, ori, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate ALU immediate range */ ++bool valid_alu_i(u8 op, s32 imm) ++{ ++ switch (BPF_OP(op)) { ++ case BPF_NEG: ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ /* All legal eBPF values are valid */ ++ return true; ++ case BPF_ADD: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_SUB: ++ /* -imm must be 16 bits */ ++ return imm >= -0x7fff && imm <= 0x8000; ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_MUL: ++ /* imm must be zero or a positive power of two */ ++ return imm == 0 || (imm > 0 && is_power_of_2(imm)); ++ case BPF_DIV: ++ case BPF_MOD: ++ /* imm must be an 17-bit power of two */ ++ return (u32)imm <= 0x10000 && is_power_of_2((u32)imm); ++ } ++ return false; ++} ++ ++/* Rewrite ALU immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val) ++{ ++ bool act = true; ++ ++ switch (BPF_OP(op)) { ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ case BPF_ADD: ++ case BPF_SUB: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm == 0 is a no-op */ ++ act = imm != 0; ++ break; ++ case BPF_MUL: ++ if (imm == 1) { ++ /* dst * 1 is a no-op */ ++ act = false; ++ } else if (imm == 0) { ++ /* dst * 0 is dst & 0 */ ++ op = BPF_AND; ++ } else { ++ /* dst * (1 << n) is dst << n */ ++ op = BPF_LSH; ++ imm = ilog2(abs(imm)); ++ } ++ break; ++ case BPF_DIV: ++ if (imm == 1) { ++ /* dst / 1 is a no-op */ ++ act = false; ++ } else { ++ /* dst / (1 << n) is dst >> n */ ++ op = BPF_RSH; ++ imm = ilog2(imm); ++ } ++ break; ++ case BPF_MOD: ++ /* dst % (1 << n) is dst & ((1 << n) - 1) */ ++ op = BPF_AND; ++ imm--; ++ break; ++ } ++ ++ *alu = op; ++ *val = imm; ++ return act; ++} ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, subu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, andi, dst, dst, (u16)imm); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, sll, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, srl, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, sra, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, addiu, dst, dst, -imm); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, dst, dst, src); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, dst, dst, src); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, dst, dst, src); ++ break; ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, sllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, srlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, srav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, addu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, subu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, dst, dst, src); ++ } else { ++ emit(ctx, multu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips32r6) { ++ emit(ctx, divu_r6, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips32r6) { ++ emit(ctx, modu, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, addu, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ } ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ emit(ctx, bne, MIPS_R_T9, res, 12); ++ emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ ++ clobber_reg(ctx, res); ++} ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ u8 msk = MIPS_R_T9; ++ ++ switch (width) { ++ /* Swap bytes in a word */ ++ case 32: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, rotr, dst, dst, 16); ++ } else { ++ emit(ctx, sll, tmp, dst, 16); /* tmp = dst << 16 */ ++ emit(ctx, srl, dst, dst, 16); /* dst = dst >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++ ++ emit(ctx, lui, msk, 0xff); /* msk = 0x00ff0000 */ ++ emit(ctx, ori, msk, msk, 0xff); /* msk = msk | 0xff */ ++ ++ emit(ctx, and, tmp, dst, msk); /* tmp = dst & msk */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, dst, 8); /* dst = dst >> 8 */ ++ emit(ctx, and, dst, dst, msk); /* dst = dst & msk */ ++ emit(ctx, or, dst, dst, tmp); /* reg = dst | tmp */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ case 16: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, andi, dst, dst, 0xffff); ++ } else { ++ emit(ctx, andi, tmp, dst, 0xff00); /* t = d & 0xff00 */ ++ emit(ctx, srl, tmp, tmp, 8); /* t = t >> 8 */ ++ emit(ctx, andi, dst, dst, 0x00ff); /* d = d & 0x00ff */ ++ emit(ctx, sll, dst, dst, 8); /* d = d << 8 */ ++ emit(ctx, or, dst, dst, tmp); /* d = d | t */ ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate jump immediate range */ ++bool valid_jmp_i(u8 op, s32 imm) ++{ ++ switch (op) { ++ case JIT_JNOP: ++ /* Immediate value not used */ ++ return true; ++ case BPF_JEQ: ++ case BPF_JNE: ++ /* No immediate operation */ ++ return false; ++ case BPF_JSET: ++ case JIT_JNSET: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_JGE: ++ case BPF_JLT: ++ case BPF_JSGE: ++ case BPF_JSLT: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_JGT: ++ case BPF_JLE: ++ case BPF_JSGT: ++ case BPF_JSLE: ++ /* imm + 1 must be 16 bits */ ++ return imm >= -0x8001 && imm <= 0x7ffe; ++ } ++ return false; ++} ++ ++/* Invert a conditional jump operation */ ++static u8 invert_jmp(u8 op) ++{ ++ switch (op) { ++ case BPF_JA: return JIT_JNOP; ++ case BPF_JEQ: return BPF_JNE; ++ case BPF_JNE: return BPF_JEQ; ++ case BPF_JSET: return JIT_JNSET; ++ case BPF_JGT: return BPF_JLE; ++ case BPF_JGE: return BPF_JLT; ++ case BPF_JLT: return BPF_JGE; ++ case BPF_JLE: return BPF_JGT; ++ case BPF_JSGT: return BPF_JSLE; ++ case BPF_JSGE: return BPF_JSLT; ++ case BPF_JSLT: return BPF_JSGE; ++ case BPF_JSLE: return BPF_JSGT; ++ } ++ return 0; ++} ++ ++/* Prepare a PC-relative jump operation */ ++static void setup_jmp(struct jit_context *ctx, u8 bpf_op, ++ s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ u32 *descp = &ctx->descriptors[ctx->bpf_index]; ++ int op = bpf_op; ++ int offset = 0; ++ ++ /* Do not compute offsets on the first pass */ ++ if (INDEX(*descp) == 0) ++ goto done; ++ ++ /* Skip jumps never taken */ ++ if (bpf_op == JIT_JNOP) ++ goto done; ++ ++ /* Convert jumps always taken */ ++ if (bpf_op == BPF_JA) ++ *descp |= JIT_DESC_CONVERT; ++ ++ /* ++ * Current ctx->jit_index points to the start of the branch preamble. ++ * Since the preamble differs among different branch conditionals, ++ * the current index cannot be used to compute the branch offset. ++ * Instead, we use the offset table value for the next instruction, ++ * which gives the index immediately after the branch delay slot. ++ */ ++ if (!CONVERTED(*descp)) { ++ int target = ctx->bpf_index + bpf_off + 1; ++ int origin = ctx->bpf_index + 1; ++ ++ offset = (INDEX(ctx->descriptors[target]) - ++ INDEX(ctx->descriptors[origin]) + 1) * sizeof(u32); ++ } ++ ++ /* ++ * The PC-relative branch offset field on MIPS is 18 bits signed, ++ * so if the computed offset is larger than this we generate a an ++ * absolute jump that we skip with an inverted conditional branch. ++ */ ++ if (CONVERTED(*descp) || offset < -0x20000 || offset > 0x1ffff) { ++ offset = 3 * sizeof(u32); ++ op = invert_jmp(bpf_op); ++ ctx->changes += !CONVERTED(*descp); ++ *descp |= JIT_DESC_CONVERT; ++ } ++ ++done: ++ *jit_off = offset; ++ *jit_op = op; ++} ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ bool always = false; ++ bool never = false; ++ ++ switch (bpf_op) { ++ case BPF_JEQ: ++ case BPF_JNE: ++ break; ++ case BPF_JSET: ++ case BPF_JLT: ++ never = imm == 0; ++ break; ++ case BPF_JGE: ++ always = imm == 0; ++ break; ++ case BPF_JGT: ++ never = (u32)imm == U32_MAX; ++ break; ++ case BPF_JLE: ++ always = (u32)imm == U32_MAX; ++ break; ++ case BPF_JSGT: ++ never = imm == S32_MAX && width == 32; ++ break; ++ case BPF_JSGE: ++ always = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLT: ++ never = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLE: ++ always = imm == S32_MAX && width == 32; ++ break; ++ } ++ ++ if (never) ++ bpf_op = JIT_JNOP; ++ if (always) ++ bpf_op = BPF_JA; ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ switch (bpf_op) { ++ case BPF_JSET: ++ break; ++ case BPF_JEQ: ++ case BPF_JGE: ++ case BPF_JLE: ++ case BPF_JSGE: ++ case BPF_JSLE: ++ if (same_reg) ++ bpf_op = BPF_JA; ++ break; ++ case BPF_JNE: ++ case BPF_JLT: ++ case BPF_JGT: ++ case BPF_JSGT: ++ case BPF_JSLT: ++ if (same_reg) ++ bpf_op = JIT_JNOP; ++ break; ++ } ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off) ++{ ++ /* Emit conditional branch delay slot */ ++ if (jit_op != JIT_JNOP) ++ emit(ctx, nop); ++ /* ++ * Emit an absolute long jump with delay slot, ++ * if the PC-relative branch was converted. ++ */ ++ if (CONVERTED(ctx->descriptors[ctx->bpf_index])) { ++ int target = get_target(ctx, ctx->bpf_index + bpf_off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ } ++ return 0; ++} ++ ++/* Jump immediate (32-bit) */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst & imm */ ++ case BPF_JSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump register (32-bit) */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ case BPF_JEQ: ++ emit(ctx, beq, dst, src, off); ++ break; ++ /* PC += off if dst != src */ ++ case BPF_JNE: ++ emit(ctx, bne, dst, src, off); ++ break; ++ /* PC += off if dst & src */ ++ case BPF_JSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off) ++{ ++ int target = get_target(ctx, ctx->bpf_index + off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx) ++{ ++ int target = get_target(ctx, ctx->program->len); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Build the program body from eBPF bytecode */ ++static int build_body(struct jit_context *ctx) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ unsigned int i; ++ ++ ctx->stack_used = 0; ++ for (i = 0; i < prog->len; i++) { ++ const struct bpf_insn *insn = &prog->insnsi[i]; ++ u32 *descp = &ctx->descriptors[i]; ++ int ret; ++ ++ access_reg(ctx, insn->src_reg); ++ access_reg(ctx, insn->dst_reg); ++ ++ ctx->bpf_index = i; ++ if (ctx->target == NULL) { ++ ctx->changes += INDEX(*descp) != ctx->jit_index; ++ *descp &= JIT_DESC_CONVERT; ++ *descp |= ctx->jit_index; ++ } ++ ++ ret = build_insn(insn, ctx); ++ if (ret < 0) ++ return ret; ++ ++ if (ret > 0) { ++ i++; ++ if (ctx->target == NULL) ++ descp[1] = ctx->jit_index; ++ } ++ } ++ ++ /* Store the end offset, where the epilogue begins */ ++ ctx->descriptors[prog->len] = ctx->jit_index; ++ return 0; ++} ++ ++/* Set the branch conversion flag on all instructions */ ++static void set_convert_flag(struct jit_context *ctx, bool enable) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ u32 flag = enable ? JIT_DESC_CONVERT : 0; ++ unsigned int i; ++ ++ for (i = 0; i <= prog->len; i++) ++ ctx->descriptors[i] = INDEX(ctx->descriptors[i]) | flag; ++} ++ ++static void jit_fill_hole(void *area, unsigned int size) ++{ ++ u32 *p; ++ ++ /* We are guaranteed to have aligned memory. */ ++ for (p = area; size >= sizeof(u32); size -= sizeof(u32)) ++ uasm_i_break(&p, BRK_BUG); /* Increments p */ ++} ++ ++bool bpf_jit_needs_zext(void) ++{ ++ return true; ++} ++ ++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ++{ ++ struct bpf_prog *tmp, *orig_prog = prog; ++ struct bpf_binary_header *header = NULL; ++ struct jit_context ctx; ++ bool tmp_blinded = false; ++ unsigned int tmp_idx; ++ unsigned int image_size; ++ u8 *image_ptr; ++ int tries; ++ ++ /* ++ * If BPF JIT was not enabled then we must fall back to ++ * the interpreter. ++ */ ++ if (!prog->jit_requested) ++ return orig_prog; ++ /* ++ * If constant blinding was enabled and we failed during blinding ++ * then we must fall back to the interpreter. Otherwise, we save ++ * the new JITed code. ++ */ ++ tmp = bpf_jit_blind_constants(prog); ++ if (IS_ERR(tmp)) ++ return orig_prog; ++ if (tmp != prog) { ++ tmp_blinded = true; ++ prog = tmp; ++ } ++ ++ memset(&ctx, 0, sizeof(ctx)); ++ ctx.program = prog; ++ ++ /* ++ * Not able to allocate memory for descriptors[], then ++ * we must fall back to the interpreter ++ */ ++ ctx.descriptors = kcalloc(prog->len + 1, sizeof(*ctx.descriptors), ++ GFP_KERNEL); ++ if (ctx.descriptors == NULL) ++ goto out_err; ++ ++ /* First pass discovers used resources */ ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ /* ++ * Second pass computes instruction offsets. ++ * If any PC-relative branches are out of range, a sequence of ++ * a PC-relative branch + a jump is generated, and we have to ++ * try again from the beginning to generate the new offsets. ++ * This is done until no additional conversions are necessary. ++ * The last two iterations are done with all branches being ++ * converted, to guarantee offset table convergence within a ++ * fixed number of iterations. ++ */ ++ ctx.jit_index = 0; ++ build_prologue(&ctx); ++ tmp_idx = ctx.jit_index; ++ ++ tries = JIT_MAX_ITERATIONS; ++ do { ++ ctx.jit_index = tmp_idx; ++ ctx.changes = 0; ++ if (tries == 2) ++ set_convert_flag(&ctx, true); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ } while (ctx.changes > 0 && --tries > 0); ++ ++ if (WARN_ONCE(ctx.changes > 0, "JIT offsets failed to converge")) ++ goto out_err; ++ ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Now we know the size of the structure to make */ ++ image_size = sizeof(u32) * ctx.jit_index; ++ header = bpf_jit_binary_alloc(image_size, &image_ptr, ++ sizeof(u32), jit_fill_hole); ++ /* ++ * Not able to allocate memory for the structure then ++ * we must fall back to the interpretation ++ */ ++ if (header == NULL) ++ goto out_err; ++ ++ /* Actual pass to generate final JIT code */ ++ ctx.target = (u32 *)image_ptr; ++ ctx.jit_index = 0; ++ ++ /* ++ * If building the JITed code fails somehow, ++ * we fall back to the interpretation. ++ */ ++ build_prologue(&ctx); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Populate line info meta data */ ++ set_convert_flag(&ctx, false); ++ bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]); ++ ++ /* Set as read-only exec and flush instruction cache */ ++ bpf_jit_binary_lock_ro(header); ++ flush_icache_range((unsigned long)header, ++ (unsigned long)&ctx.target[ctx.jit_index]); ++ ++ if (bpf_jit_enable > 1) ++ bpf_jit_dump(prog->len, image_size, 2, ctx.target); ++ ++ prog->bpf_func = (void *)ctx.target; ++ prog->jited = 1; ++ prog->jited_len = image_size; ++ ++out: ++ if (tmp_blinded) ++ bpf_jit_prog_release_other(prog, prog == orig_prog ? ++ tmp : orig_prog); ++ kfree(ctx.descriptors); ++ return prog; ++ ++out_err: ++ prog = orig_prog; ++ if (header) ++ bpf_jit_binary_free(header); ++ goto out; ++} +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -0,0 +1,211 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#ifndef _BPF_JIT_COMP_H ++#define _BPF_JIT_COMP_H ++ ++/* MIPS registers */ ++#define MIPS_R_ZERO 0 /* Const zero */ ++#define MIPS_R_AT 1 /* Asm temp */ ++#define MIPS_R_V0 2 /* Result */ ++#define MIPS_R_V1 3 /* Result */ ++#define MIPS_R_A0 4 /* Argument */ ++#define MIPS_R_A1 5 /* Argument */ ++#define MIPS_R_A2 6 /* Argument */ ++#define MIPS_R_A3 7 /* Argument */ ++#define MIPS_R_A4 8 /* Arg (n64) */ ++#define MIPS_R_A5 9 /* Arg (n64) */ ++#define MIPS_R_A6 10 /* Arg (n64) */ ++#define MIPS_R_A7 11 /* Arg (n64) */ ++#define MIPS_R_T0 8 /* Temp (o32) */ ++#define MIPS_R_T1 9 /* Temp (o32) */ ++#define MIPS_R_T2 10 /* Temp (o32) */ ++#define MIPS_R_T3 11 /* Temp (o32) */ ++#define MIPS_R_T4 12 /* Temporary */ ++#define MIPS_R_T5 13 /* Temporary */ ++#define MIPS_R_T6 14 /* Temporary */ ++#define MIPS_R_T7 15 /* Temporary */ ++#define MIPS_R_S0 16 /* Saved */ ++#define MIPS_R_S1 17 /* Saved */ ++#define MIPS_R_S2 18 /* Saved */ ++#define MIPS_R_S3 19 /* Saved */ ++#define MIPS_R_S4 20 /* Saved */ ++#define MIPS_R_S5 21 /* Saved */ ++#define MIPS_R_S6 22 /* Saved */ ++#define MIPS_R_S7 23 /* Saved */ ++#define MIPS_R_T8 24 /* Temporary */ ++#define MIPS_R_T9 25 /* Temporary */ ++/* MIPS_R_K0 26 Reserved */ ++/* MIPS_R_K1 27 Reserved */ ++#define MIPS_R_GP 28 /* Global ptr */ ++#define MIPS_R_SP 29 /* Stack ptr */ ++#define MIPS_R_FP 30 /* Frame ptr */ ++#define MIPS_R_RA 31 /* Return */ ++ ++/* ++ * Jump address mask for immediate jumps. The four most significant bits ++ * must be equal to PC. ++ */ ++#define MIPS_JMP_MASK 0x0fffffffUL ++ ++/* Maximum number of iterations in offset table computation */ ++#define JIT_MAX_ITERATIONS 8 ++ ++/* ++ * Jump pseudo-instructions used internally ++ * for branch conversion and branch optimization. ++ */ ++#define JIT_JNSET 0xe0 ++#define JIT_JNOP 0xf0 ++ ++/* Descriptor flag for PC-relative branch conversion */ ++#define JIT_DESC_CONVERT BIT(31) ++ ++/* JIT context for an eBPF program */ ++struct jit_context { ++ struct bpf_prog *program; /* The eBPF program being JITed */ ++ u32 *descriptors; /* eBPF to JITed CPU insn descriptors */ ++ u32 *target; /* JITed code buffer */ ++ u32 bpf_index; /* Index of current BPF program insn */ ++ u32 jit_index; /* Index of current JIT target insn */ ++ u32 changes; /* Number of PC-relative branch conv */ ++ u32 accessed; /* Bit mask of read eBPF registers */ ++ u32 clobbered; /* Bit mask of modified CPU registers */ ++ u32 stack_size; /* Total allocated stack size in bytes */ ++ u32 saved_size; /* Size of callee-saved registers */ ++ u32 stack_used; /* Stack size used for function calls */ ++}; ++ ++/* Emit the instruction if the JIT memory space has been allocated */ ++#define emit(ctx, func, ...) \ ++do { \ ++ if ((ctx)->target != NULL) { \ ++ u32 *p = &(ctx)->target[ctx->jit_index]; \ ++ uasm_i_##func(&p, ##__VA_ARGS__); \ ++ } \ ++ (ctx)->jit_index++; \ ++} while (0) ++ ++/* ++ * Mark a BPF register as accessed, it needs to be ++ * initialized by the program if expected, e.g. FP. ++ */ ++static inline void access_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->accessed |= BIT(reg); ++} ++ ++/* ++ * Mark a CPU register as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static inline void clobber_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->clobbered |= BIT(reg); ++} ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc); ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off); ++ ++/* dst = imm (32-bit) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); ++ ++/* dst = src (32-bit) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); ++ ++/* Validate ALU/ALU64 immediate range */ ++bool valid_alu_i(u8 op, s32 imm); ++ ++/* Rewrite ALU/ALU64 immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off); ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width); ++ ++/* Validate JMP/JMP32 immediate range */ ++bool valid_jmp_i(u8 op, s32 imm); ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off); ++ ++/* Conditional JMP/JMP32 immediate */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); ++ ++/* Conditional JMP/JMP32 register */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off); ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx); ++ ++/* ++ * Build program prologue to set up the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_prologue(struct jit_context *ctx); ++ ++/* ++ * Build the program epilogue to restore the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_epilogue(struct jit_context *ctx, int dest_reg); ++ ++/* ++ * Convert an eBPF instruction to native instruction, i.e ++ * JITs an eBPF instruction. ++ * Returns : ++ * 0 - Successfully JITed an 8-byte eBPF instruction ++ * >0 - Successfully JITed a 16-byte eBPF instruction ++ * <0 - Failed to JIT. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx); ++ ++#endif /* _BPF_JIT_COMP_H */ +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp32.c +@@ -0,0 +1,1741 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 32-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS a4-a7 are not available in the o32 ABI */ ++#undef MIPS_R_A4 ++#undef MIPS_R_A5 ++#undef MIPS_R_A6 ++#undef MIPS_R_A7 ++ ++/* Stack is 8-byte aligned in o32 ABI */ ++#define MIPS_STACK_ALIGNMENT 8 ++ ++/* ++ * The top 16 bytes of a stack frame is reserved for the callee in O32 ABI. ++ * This corresponds to stack space for register arguments a0-a3. ++ */ ++#define JIT_RESERVED_STACK 16 ++ ++/* Temporary 64-bit register used by JIT */ ++#define JIT_REG_TMP MAX_BPF_JIT_REG ++ ++/* ++ * Number of prologue bytes to skip when doing a tail call. ++ * Tail call count (TCC) initialization (8 bytes) always, plus ++ * R0-to-v0 assignment (4 bytes) if big endian. ++ */ ++#ifdef __BIG_ENDIAN ++#define JIT_TCALL_SKIP 12 ++#else ++#define JIT_TCALL_SKIP 8 ++#endif ++ ++/* CPU registers holding the callee return value */ ++#define JIT_RETURN_REGS \ ++ (BIT(MIPS_R_V0) | \ ++ BIT(MIPS_R_V1)) ++ ++/* CPU registers arguments passed to callee directly */ ++#define JIT_ARG_REGS \ ++ (BIT(MIPS_R_A0) | \ ++ BIT(MIPS_R_A1) | \ ++ BIT(MIPS_R_A2) | \ ++ BIT(MIPS_R_A3)) ++ ++/* CPU register arguments passed to callee on stack */ ++#define JIT_STACK_REGS \ ++ (BIT(MIPS_R_T0) | \ ++ BIT(MIPS_R_T1) | \ ++ BIT(MIPS_R_T2) | \ ++ BIT(MIPS_R_T3) | \ ++ BIT(MIPS_R_T4) | \ ++ BIT(MIPS_R_T5)) ++ ++/* Caller-saved CPU registers */ ++#define JIT_CALLER_REGS \ ++ (JIT_RETURN_REGS | \ ++ JIT_ARG_REGS | \ ++ JIT_STACK_REGS) ++ ++/* Callee-saved CPU registers */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* ++ * Mapping of 64-bit eBPF registers to 32-bit native MIPS registers. ++ * ++ * 1) Native register pairs are ordered according to CPU endiannes, following ++ * the MIPS convention for passing 64-bit arguments and return values. ++ * 2) The eBPF return value, arguments and callee-saved registers are mapped ++ * to their native MIPS equivalents. ++ * 3) Since the 32 highest bits in the eBPF FP register are always zero, ++ * only one general-purpose register is actually needed for the mapping. ++ * We use the fp register for this purpose, and map the highest bits to ++ * the MIPS register r0 (zero). ++ * 4) We use the MIPS gp and at registers as internal temporary registers ++ * for constant blinding. The gp register is callee-saved. ++ * 5) One 64-bit temporary register is mapped for use when sign-extending ++ * immediate operands. MIPS registers t6-t9 are available to the JIT ++ * for as temporaries when implementing complex 64-bit operations. ++ * ++ * With this scheme all eBPF registers are being mapped to native MIPS ++ * registers without having to use any stack scratch space. The direct ++ * register mapping (2) simplifies the handling of function calls. ++ */ ++static const u8 bpf2mips32[][2] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = {MIPS_R_V1, MIPS_R_V0}, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = {MIPS_R_A1, MIPS_R_A0}, ++ [BPF_REG_2] = {MIPS_R_A3, MIPS_R_A2}, ++ /* Remaining arguments, to be passed on the stack per O32 ABI */ ++ [BPF_REG_3] = {MIPS_R_T1, MIPS_R_T0}, ++ [BPF_REG_4] = {MIPS_R_T3, MIPS_R_T2}, ++ [BPF_REG_5] = {MIPS_R_T5, MIPS_R_T4}, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = {MIPS_R_S1, MIPS_R_S0}, ++ [BPF_REG_7] = {MIPS_R_S3, MIPS_R_S2}, ++ [BPF_REG_8] = {MIPS_R_S5, MIPS_R_S4}, ++ [BPF_REG_9] = {MIPS_R_S7, MIPS_R_S6}, ++ /* Read-only frame pointer to access the eBPF stack */ ++#ifdef __BIG_ENDIAN ++ [BPF_REG_FP] = {MIPS_R_FP, MIPS_R_ZERO}, ++#else ++ [BPF_REG_FP] = {MIPS_R_ZERO, MIPS_R_FP}, ++#endif ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = {MIPS_R_GP, MIPS_R_AT}, ++ /* Temporary register for internal JIT use */ ++ [JIT_REG_TMP] = {MIPS_R_T7, MIPS_R_T6}, ++}; ++ ++/* Get low CPU register for a 64-bit eBPF register mapping */ ++static inline u8 lo(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[0]; ++#else ++ return reg[1]; ++#endif ++} ++ ++/* Get high CPU register for a 64-bit eBPF register mapping */ ++static inline u8 hi(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[1]; ++#else ++ return reg[0]; ++#endif ++} ++ ++/* ++ * Mark a 64-bit CPU register pair as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static void clobber_reg64(struct jit_context *ctx, const u8 reg[]) ++{ ++ clobber_reg(ctx, reg[0]); ++ clobber_reg(ctx, reg[1]); ++} ++ ++/* dst = imm (sign-extended) */ ++static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ emit_mov_i(ctx, lo(dst), imm); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ else ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) ++{ ++ if (!ctx->program->aux->verifier_zext) { ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ } ++} ++ ++/* Load delay slot, if ISA mandates it */ ++static void emit_load_delay(struct jit_context *ctx) ++{ ++ if (!cpu_has_mips_2_3_4_5_r) ++ emit(ctx, nop); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, ++ const u8 dst[], s32 imm, u8 op) ++{ ++ u8 src = MIPS_R_T6; ++ ++ /* ++ * ADD/SUB with all but the max negative imm can be handled by ++ * inverting the operation and the imm value, saving one insn. ++ */ ++ if (imm > S32_MIN && imm < 0) ++ switch (op) { ++ case BPF_ADD: ++ op = BPF_SUB; ++ imm = -imm; ++ break; ++ case BPF_SUB: ++ op = BPF_ADD; ++ imm = -imm; ++ break; ++ } ++ ++ /* Move immediate to temporary register */ ++ emit_mov_i(ctx, src, imm); ++ ++ switch (op) { ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addu, lo(dst), lo(dst), src); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, subu, lo(dst), lo(dst), src); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), 1); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), src); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), src); ++ if (imm >= 0) ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), src); ++ if (imm < 0) { ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst + src */ ++ case BPF_ADD: ++ if (src == dst) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 31); ++ emit(ctx, addu, lo(dst), lo(dst), lo(dst)); ++ } else { ++ emit(ctx, addu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ } ++ emit(ctx, addu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ emit(ctx, subu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, subu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), lo(src)); ++ emit(ctx, or, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), lo(src)); ++ emit(ctx, and, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), lo(src)); ++ emit(ctx, xor, hi(dst), hi(dst), hi(src)); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU invert (64-bit) */ ++static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) ++{ ++ emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift immediate (64-bit) */ ++static void emit_shift_i64(struct jit_context *ctx, ++ const u8 dst[], u32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ if (imm < 32) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); ++ emit(ctx, sll, lo(dst), lo(dst), imm); ++ emit(ctx, sll, hi(dst), hi(dst), imm); ++ emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sll, hi(dst), lo(dst), imm - 32); ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, srl, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, srl, lo(dst), hi(dst), imm - 32); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, sra, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sra, lo(dst), hi(dst), imm - 32); ++ emit(ctx, sra, hi(dst), hi(dst), 31); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift register (64-bit) */ ++static void emit_shift_r64(struct jit_context *ctx, ++ const u8 dst[], u8 src, u8 op) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ ++ emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */ ++ emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */ ++ emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */ ++ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ ++ emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */ ++ emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ ++ emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ ++ emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ ++ emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ ++ emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ ++ emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ } ++ ++ /* +20: Done */ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU mul immediate (64x32-bit) */ ++static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ u8 src = MIPS_R_T6; ++ u8 tmp = MIPS_R_T9; ++ ++ switch (imm) { ++ /* dst = dst * 1 is a no-op */ ++ case 1: ++ break; ++ /* dst = dst * -1 */ ++ case -1: ++ emit_neg_i64(ctx, dst); ++ break; ++ case 0: ++ emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); ++ emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Full 64x32 multiply */ ++ default: ++ /* hi(dst) = hi(dst) * src(imm) */ ++ emit_mov_i(ctx, src, imm); ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, hi(dst), hi(dst), src); ++ } else { ++ emit(ctx, multu, hi(dst), src); ++ emit(ctx, mflo, hi(dst)); ++ } ++ ++ /* hi(dst) = hi(dst) - lo(dst) */ ++ if (imm < 0) ++ emit(ctx, subu, hi(dst), hi(dst), lo(dst)); ++ ++ /* tmp = lo(dst) * src(imm) >> 32 */ ++ /* lo(dst) = lo(dst) * src(imm) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), src); ++ emit(ctx, mulu, lo(dst), lo(dst), src); ++ } else { ++ emit(ctx, multu, lo(dst), src); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) += tmp */ ++ emit(ctx, addu, hi(dst), hi(dst), tmp); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* ALU mul register (64x64-bit) */ ++static void emit_mul_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[]) ++{ ++ u8 acc = MIPS_R_T8; ++ u8 tmp = MIPS_R_T9; ++ ++ /* acc = hi(dst) * lo(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, acc, hi(dst), lo(src)); ++ } else { ++ emit(ctx, multu, hi(dst), lo(src)); ++ emit(ctx, mflo, acc); ++ } ++ ++ /* tmp = lo(dst) * hi(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, tmp, lo(dst), hi(src)); ++ } else { ++ emit(ctx, multu, lo(dst), hi(src)); ++ emit(ctx, mflo, tmp); ++ } ++ ++ /* acc += tmp */ ++ emit(ctx, addu, acc, acc, tmp); ++ ++ /* tmp = lo(dst) * lo(src) >> 32 */ ++ /* lo(dst) = lo(dst) * lo(src) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), lo(src)); ++ emit(ctx, mulu, lo(dst), lo(dst), lo(src)); ++ } else { ++ emit(ctx, multu, lo(dst), lo(src)); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) = acc + tmp */ ++ emit(ctx, addu, hi(dst), acc, tmp); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Helper function for 64-bit modulo */ ++static u64 jit_mod64(u64 a, u64 b) ++{ ++ u64 rem; ++ ++ div64_u64_rem(a, b, &rem); ++ return rem; ++} ++ ++/* ALU div/mod register (64-bit) */ ++static void emit_divmod_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ ++ int exclude, k; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ ++ /* Put 64-bit arguments 1 and 2 in registers a0-a3 */ ++ for (k = 0; k < 2; k++) { ++ emit(ctx, move, MIPS_R_T9, src[k]); ++ emit(ctx, move, r1[k], dst[k]); ++ emit(ctx, move, r2[k], MIPS_R_T9); ++ } ++ ++ /* Emit function call */ ++ switch (BPF_OP(op)) { ++ /* dst = dst / src */ ++ case BPF_DIV: ++ addr = (u32)&div64_u64; ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ addr = (u32)&jit_mod64; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Store the 64-bit result in dst */ ++ emit(ctx, move, dst[0], r0[0]); ++ emit(ctx, move, dst[1], r0[1]); ++ ++ /* Restore caller-saved registers, excluding the computed result */ ++ exclude = BIT(lo(dst)) | BIT(hi(dst)); ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ ++ clobber_reg64(ctx, dst); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Swap bytes in a register word */ ++static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap half words in a register word */ ++static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */ ++ emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, rotr, tmp, hi(dst), 16); ++ emit(ctx, rotr, hi(dst), lo(dst), 16); ++ emit(ctx, wsbh, lo(dst), tmp); ++ emit(ctx, wsbh, hi(dst), hi(dst)); ++ } else { ++ emit_swap16_r(ctx, tmp, lo(dst)); ++ emit_swap16_r(ctx, lo(dst), hi(dst)); ++ emit(ctx, move, hi(dst), tmp); ++ ++ emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */ ++ emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */ ++ emit_swap8_r(ctx, lo(dst), lo(dst), tmp); ++ emit_swap8_r(ctx, hi(dst), hi(dst), tmp); ++ } ++ break; ++ /* Swap bytes in a word */ ++ /* Swap bytes in a half word */ ++ case 32: ++ case 16: ++ emit_bswap_r(ctx, lo(dst), width); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ emit(ctx, andi, lo(dst), lo(dst), 0xffff); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, ++ const u8 dst[], u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lw, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ if (dst[1] == src) { ++ emit(ctx, lw, dst[0], off + 4, src); ++ emit(ctx, lw, dst[1], off, src); ++ } else { ++ emit(ctx, lw, dst[1], off, src); ++ emit(ctx, lw, dst[0], off + 4, src); ++ } ++ emit_load_delay(ctx); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, ++ const u8 dst, const u8 src[], s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, lo(src), off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, lo(src), off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, lo(src), off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sw, src[1], off, dst); ++ emit(ctx, sw, src[0], off + 4, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write (32-bit, non-ll/sc fallback) */ ++static void emit_atomic_r32(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: dst+off if xchg, otherwise src, passed in register a0 ++ * Argument 2: src if xchg, othersize dst+off, passed in register a1 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, MIPS_R_A0, src); ++ emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Atomic read-modify-write (64-bit) */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, const u8 src[], s16 off, u8 code) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: 64-bit src, passed in registers a0-a1 ++ * Argument 2: 32-bit dst+off, passed in register a2 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, r1[0], src[0]); ++ emit(ctx, move, r1[1], src[1]); ++ emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic64_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic64_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic64_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic64_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic64_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* ++ * Conditional movz or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* ++ * Conditional movn or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* Emulation of 64-bit sltiu rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_sltiu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ if (imm < 0) { ++ emit_mov_i(ctx, rd, imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++ } else { /* imm >= 0 */ ++ if (imm > 0x7fff) { ++ emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ } else { ++ emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */ ++ } ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */ ++ } ++} ++ ++/* Emulation of 64-bit sltu rd, rs, rt */ ++static void emit_sltu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */ ++ emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */ ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */ ++ emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++} ++ ++/* Emulation of 64-bit slti rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_slti_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ u8 cmp; ++ ++ /* ++ * if ((rs < 0) ^ (imm < 0)) t1 = imm >u rsl ++ * else t1 = rsl > 31 */ ++ if (imm < 0) ++ emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */ ++ else ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ /* ++ * if ((imm < 0 && rsh != 0xffffffff) || ++ * (imm >= 0 && rsh != 0)) ++ * t1 = 0 ++ */ ++ if (imm < 0) { ++ emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */ ++ cmp = rd; ++ } else { /* imm >= 0 */ ++ cmp = hi(rs); ++ } ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */ ++ ++ /* ++ * if (imm < 0) rd = rsh < -1 ++ * else rd = rsh != 0 ++ * rd = rd | t1 ++ */ ++ emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */ ++ emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ ++} ++ ++/* Emulation of 64-bit(slt rd, rs, rt) */ ++static void emit_slt_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 t1 = MIPS_R_T7; ++ u8 t2 = MIPS_R_T8; ++ u8 t3 = MIPS_R_T9; ++ ++ /* ++ * if ((rs < 0) ^ (rt < 0)) t1 = rtl > 31 */ ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */ ++ ++ /* rd = (rsh < rth) | t1 */ ++ emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh = -0x7fff && imm <= 0x8000) { ++ emit(ctx, addiu, tmp, lo(dst), -imm); ++ } else if ((u32)imm <= 0xffff) { ++ emit(ctx, xori, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, xor, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) { /* Compare sign extension */ ++ emit(ctx, addu, MIPS_R_T9, hi(dst), 1); ++ emit(ctx, or, tmp, tmp, MIPS_R_T9); ++ } else { /* Compare zero extension */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ } ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, tmp, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst & imm */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ if ((u32)imm <= 0xffff) { ++ emit(ctx, andi, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, and, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) /* Sign-extension pulls in high word */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, tmp, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ } ++} ++ ++/* Jump register (64-bit) */ ++static void emit_jmp_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], s32 off, u8 op) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ case BPF_JEQ: ++ case BPF_JNE: ++ emit(ctx, subu, t1, lo(dst), lo(src)); ++ emit(ctx, subu, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, t1, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst & src */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ emit(ctx, and, t1, lo(dst), lo(src)); ++ emit(ctx, and, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, t1, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ } ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push stack arguments */ ++ push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK); ++ ++ /* Emit function call */ ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = lo(bpf2mips32[BPF_REG_2]); ++ u8 ind = lo(bpf2mips32[BPF_REG_3]); ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ * stack[sz] - remaining tail call count, initialized in prologue ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */ ++ emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */ ++ /* (next insn delay slot) */ ++ /* if (TCC-- <= 0) goto out */ ++ emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 < 0 */ ++ emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */ ++ emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */ ++ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */ ++ emit(ctx, addu, t1, t1, ary); /* t1 += ary */ ++ emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, t1); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Caller's stack frame : ++ * :----------------------------: ++ * : 64-bit eBPF args r3-r5 : ++ * :----------------------------: ++ * : Reserved / tail call count : ++ * +============================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +----------------------------+ <--- eBPF FP (MIPS zero,fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +----------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * +----------------------------+ ++ * | Reserved for 64-bit eBPF | ++ * | args r3-r5 & args passed | ++ * | on stack in kernel calls | ++ * Lower address +============================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; ++ const u8 *fp = bpf2mips32[BPF_REG_FP]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first two instructions initialize TCC in the reserved (for us) ++ * 16-byte area in the parent's stack frame. On a tail call, the ++ * calling function jumps into the prologue after these instructions. ++ */ ++ emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, ++ min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); ++ ++ /* ++ * Register eBPF R1 contains the 32-bit context pointer argument. ++ * A 32-bit argument is always passed in MIPS register a0, regardless ++ * of CPU endianness. Initialize R1 accordingly and zero-extend. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, lo(r1), MIPS_R_A0); ++#endif ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* Zero-extend the 32-bit argument */ ++ emit(ctx, move, hi(r1), MIPS_R_ZERO); ++ ++ /* If the eBPF frame pointer was accessed it must be saved */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg64(ctx, fp); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers and function arguments passed on the stack. ++ * The required space is computed automatically during resource ++ * usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved); ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ /* ++ * A 32-bit return value is always passed in MIPS register v0, ++ * but on big-endian targets the low part of R0 is mapped to v1. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, MIPS_R_V0, MIPS_R_V1); ++#endif ++ ++ /* Jump to the return address and adjust the stack pointer */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ const u8 *dst = bpf2mips32[insn->dst_reg]; ++ const u8 *src = bpf2mips32[insn->src_reg]; ++ const u8 *tmp = bpf2mips32[JIT_REG_TMP]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_mov_i(ctx, hi(dst), 0); ++ } else { ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_alu_i(ctx, lo(dst), 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, lo(dst), val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_se_i64(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_mov_r(ctx, hi(dst), hi(src)); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_neg_i64(ctx, dst); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ if (imm) ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ if (imm) ++ emit_shift_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst * imm (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ emit_mul_i64(ctx, dst, imm); ++ break; ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ /* ++ * Sign-extend the immediate value into a temporary register, ++ * and then do the operation on this register. ++ */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); ++ break; ++ /* dst = dst * src (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ emit_mul_r64(ctx, dst, src); ++ break; ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_divmod_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_mov_i(ctx, hi(dst), insn[1].imm); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ switch (BPF_SIZE(code)) { ++ case BPF_DW: ++ /* Sign-extend immediate value into temporary reg */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ break; ++ case BPF_W: ++ case BPF_H: ++ case BPF_B: ++ emit_mov_i(ctx, lo(tmp), imm); ++ break; ++ } ++ emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (cpu_has_llsc) ++ emit_atomic_r(ctx, lo(dst), lo(src), off, imm); ++ else /* Non-ll/sc fallback */ ++ emit_atomic_r32(ctx, lo(dst), lo(src), ++ off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* Atomics (64-bit) */ ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ emit_atomic_r64(ctx, lo(dst), src, off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, lo(dst), imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r64(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_i64(ctx, dst, imm, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.10/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch b/target/linux/generic/backport-5.10/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch new file mode 100644 index 0000000000..38b46c0b76 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch @@ -0,0 +1,1005 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:05 +0200 +Subject: [PATCH] mips: bpf: Add new eBPF JIT for 64-bit MIPS + +This is an implementation on of an eBPF JIT for 64-bit MIPS III-V and +MIPS64r1-r6. It uses the same framework introduced by the 32-bit JIT. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp64.c + +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -0,0 +1,991 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS t0-t3 are not available in the n64 ABI */ ++#undef MIPS_R_T0 ++#undef MIPS_R_T1 ++#undef MIPS_R_T2 ++#undef MIPS_R_T3 ++ ++/* Stack is 16-byte aligned in n64 ABI */ ++#define MIPS_STACK_ALIGNMENT 16 ++ ++/* Extra 64-bit eBPF registers used by JIT */ ++#define JIT_REG_TC (MAX_BPF_JIT_REG + 0) ++#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1) ++ ++/* Number of prologue bytes to skip when doing a tail call */ ++#define JIT_TCALL_SKIP 4 ++ ++/* Callee-saved CPU registers that the JIT must preserve */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* Caller-saved CPU registers available for JIT use */ ++#define JIT_CALLER_REGS \ ++ (BIT(MIPS_R_A5) | \ ++ BIT(MIPS_R_A6) | \ ++ BIT(MIPS_R_A7)) ++/* ++ * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers. ++ * MIPS registers t4 - t7 may be used by the JIT as temporary registers. ++ * MIPS registers t8 - t9 are reserved for single-register common functions. ++ */ ++static const u8 bpf2mips64[] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = MIPS_R_V0, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = MIPS_R_A0, ++ [BPF_REG_2] = MIPS_R_A1, ++ [BPF_REG_3] = MIPS_R_A2, ++ [BPF_REG_4] = MIPS_R_A3, ++ [BPF_REG_5] = MIPS_R_A4, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = MIPS_R_S0, ++ [BPF_REG_7] = MIPS_R_S1, ++ [BPF_REG_8] = MIPS_R_S2, ++ [BPF_REG_9] = MIPS_R_S3, ++ /* Read-only frame pointer to access the eBPF stack */ ++ [BPF_REG_FP] = MIPS_R_FP, ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = MIPS_R_AT, ++ /* Tail call count register, caller-saved */ ++ [JIT_REG_TC] = MIPS_R_A5, ++ /* Constant for register zero-extension */ ++ [JIT_REG_ZX] = MIPS_R_V1, ++}; ++ ++/* ++ * MIPS 32-bit operations on 64-bit registers generate a sign-extended ++ * result. However, the eBPF ISA mandates zero-extension, so we rely on the ++ * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic ++ * operations, right shift and byte swap require properly sign-extended ++ * operands or the result is unpredictable. We emit explicit sign-extensions ++ * in those cases. ++ */ ++ ++/* Sign extension */ ++static void emit_sext(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, sll, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension */ ++static void emit_zext(struct jit_context *ctx, u8 dst) ++{ ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); ++ } else { ++ emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]); ++ access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */ ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, u8 dst) ++{ ++ if (!ctx->program->aux->verifier_zext) ++ emit_zext(ctx, dst); ++} ++ ++/* dst = imm (64-bit) */ ++static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) ++{ ++ if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) { ++ emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64); ++ } else if (imm64 >= 0xffffffff80000000ULL || ++ (imm64 < 0x80000000 && imm64 > 0xffff)) { ++ emit(ctx, lui, dst, (s16)(imm64 >> 16)); ++ emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff); ++ } else { ++ u8 acc = MIPS_R_ZERO; ++ int k; ++ ++ for (k = 0; k < 4; k++) { ++ u16 half = imm64 >> (48 - 16 * k); ++ ++ if (acc == dst) ++ emit(ctx, dsll, dst, dst, 16); ++ ++ if (half) { ++ emit(ctx, ori, dst, acc, half); ++ acc = dst; ++ } ++ } ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, dsubu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, dsll_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, dsrl_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsra_safe, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, daddiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, daddiu, dst, dst, -imm); ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_i(ctx, dst, imm, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, dsllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, dsrlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsrav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, daddu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, dsubu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmulu, dst, dst, src); ++ } else { ++ emit(ctx, dmultu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips64r6) { ++ emit(ctx, ddivu_r6, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmodu, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_r(ctx, dst, src, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Swap sub words in a register double word */ ++static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */ ++ emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */ ++ emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & mask */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dsbh, dst, dst); ++ emit(ctx, dshd, dst, dst); ++ } else { ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */ ++ emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */ ++ emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */ ++ ++ emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff); ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */ ++ ++ emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */ ++ emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */ ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ /* Swap bytes in a word */ ++ case 32: ++ case 16: ++ emit_sext(ctx, dst, dst); ++ emit_bswap_r(ctx, dst, width); ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) ++ emit_zext(ctx, dst); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit_zext(ctx, dst); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, andi, dst, dst, 0xffff); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, dst, off, src); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, dst, off, src); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lwu, dst, off, src); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ emit(ctx, ld, dst, off, src); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, src, off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, src, off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, src, off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sd, src, off, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, lld, t1, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, daddu, t2, t1, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, t2, t1, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, t2, t1, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, t2, t1, src); ++ break; ++ } ++ emit(ctx, scd, t2, off, dst); ++ emit(ctx, beqz, t2, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ u8 tmp = MIPS_R_T6; ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Emit function call */ ++ emit_mov_i64(ctx, tmp, addr); ++ emit(ctx, jalr, MIPS_R_RA, tmp); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Re-initialize the JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = bpf2mips64[BPF_REG_2]; ++ u8 ind = bpf2mips64[BPF_REG_3]; ++ u8 tcc = bpf2mips64[JIT_REG_TC]; ++ u8 tmp = MIPS_R_T6; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/ ++ emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ ++ /* if (--TCC < 0) goto out */ ++ emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */ ++ emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */ ++ /* (next insn delay slot) */ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */ ++ emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */ ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, tmp); ++ access_reg(ctx, JIT_REG_TC); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Previous stack frame : ++ * +===========================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +---------------------------+ <--- eBPF FP (MIPS fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +---------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * Lower address +===========================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ u8 fp = bpf2mips64[BPF_REG_FP]; ++ u8 tc = bpf2mips64[JIT_REG_TC]; ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first instruction initializes the tail call count register. ++ * On a tail call, the calling function jumps into the prologue ++ * after this instruction. ++ */ ++ emit(ctx, addiu, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* ++ * If the eBPF frame pointer and tail call count registers were ++ * accessed they must be preserved. Mark them as clobbered here ++ * to save and restore them on the stack as needed. ++ */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg(ctx, fp); ++ if (ctx->accessed & BIT(JIT_REG_TC)) ++ clobber_reg(ctx, tc); ++ if (ctx->accessed & BIT(JIT_REG_ZX)) ++ clobber_reg(ctx, zx); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers needed by the JIT. The required space is ++ * computed automatically during resource usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ if (stack) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved); ++ ++ /* Initialize the ePF JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ ++ /* Release the stack frame */ ++ if (ctx->stack_size) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++ ++ /* Jump to return address and sign-extend the 32-bit return value */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */ ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ u8 dst = bpf2mips64[insn->dst_reg]; ++ u8 src = bpf2mips64[insn->src_reg]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_zext(ctx, dst); ++ } else { ++ emit_mov_r(ctx, dst, src); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_sext(ctx, dst, dst); ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ emit_alu_r(ctx, dst, src, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_sext(ctx, dst, dst); ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, dst, src); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_alu_i64(ctx, dst, 0, BPF_NEG); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm ((64-bit, arithmetic) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ /* dst = dst * imm (64-bit) */ ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i64(ctx, dst, val, alu); ++ } ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ /* dst = dst * src (64-bit) */ ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32)); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (BPF_SIZE(code) == BPF_DW) { ++ emit_atomic_r64(ctx, dst, src, off, imm); ++ } else { /* 32-bit, no fetch */ ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm); ++ } ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */ ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register, sign-extended */ ++ emit_mov_i(ctx, MIPS_R_T5, imm); ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, dst, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.10/050-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch b/target/linux/generic/backport-5.10/050-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch new file mode 100644 index 0000000000..63553ebe58 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch @@ -0,0 +1,120 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:06 +0200 +Subject: [PATCH] mips: bpf: Add JIT workarounds for CPU errata + +This patch adds workarounds for the following CPU errata to the MIPS +eBPF JIT, if enabled in the kernel configuration. + + - R10000 ll/sc weak ordering + - Loongson-3 ll/sc weak ordering + - Loongson-2F jump hang + +The Loongson-2F nop errata is implemented in uasm, which the JIT uses, +so no additional mitigations are needed for that. + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/net/bpf_jit_comp.c ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, + /* Atomic read-modify-write (32-bit) */ + void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + switch (code) { + case BPF_ADD: +@@ -420,18 +421,19 @@ void emit_atomic_r(struct jit_context *c + break; + } + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + + /* Atomic compare-and-exchange (32-bit) */ + void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + emit(ctx, bne, MIPS_R_T9, res, 12); + emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset); + emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ + clobber_reg(ctx, res); + } +--- a/arch/mips/net/bpf_jit_comp.h ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -87,7 +87,7 @@ struct jit_context { + }; + + /* Emit the instruction if the JIT memory space has been allocated */ +-#define emit(ctx, func, ...) \ ++#define __emit(ctx, func, ...) \ + do { \ + if ((ctx)->target != NULL) { \ + u32 *p = &(ctx)->target[ctx->jit_index]; \ +@@ -95,6 +95,30 @@ do { \ + } \ + (ctx)->jit_index++; \ + } while (0) ++#define emit(...) __emit(__VA_ARGS__) ++ ++/* Workaround for R10000 ll/sc errata */ ++#ifdef CONFIG_WAR_R10000 ++#define LLSC_beqz beqzl ++#else ++#define LLSC_beqz beqz ++#endif ++ ++/* Workaround for Loongson-3 ll/sc errata */ ++#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS ++#define LLSC_sync(ctx) emit(ctx, sync, 0) ++#define LLSC_offset 4 ++#else ++#define LLSC_sync(ctx) ++#define LLSC_offset 0 ++#endif ++ ++/* Workaround for Loongson-2F jump errata */ ++#ifdef CONFIG_CPU_JUMP_WORKAROUNDS ++#define JALR_MASK 0xffffffffcfffffffULL ++#else ++#define JALR_MASK (~0ULL) ++#endif + + /* + * Mark a BPF register as accessed, it needs to be +--- a/arch/mips/net/bpf_jit_comp64.c ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_c + u8 t1 = MIPS_R_T6; + u8 t2 = MIPS_R_T7; + ++ LLSC_sync(ctx); + emit(ctx, lld, t1, off, dst); + switch (code) { + case BPF_ADD: +@@ -391,7 +392,7 @@ static void emit_atomic_r64(struct jit_c + break; + } + emit(ctx, scd, t2, off, dst); +- emit(ctx, beqz, t2, -16); ++ emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + +@@ -414,7 +415,7 @@ static int emit_call(struct jit_context + push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); + + /* Emit function call */ +- emit_mov_i64(ctx, tmp, addr); ++ emit_mov_i64(ctx, tmp, addr & JALR_MASK); + emit(ctx, jalr, MIPS_R_RA, tmp); + emit(ctx, nop); /* Delay slot */ + diff --git a/target/linux/generic/backport-5.10/050-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch b/target/linux/generic/backport-5.10/050-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch new file mode 100644 index 0000000000..10685c5f3c --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch @@ -0,0 +1,61 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:07 +0200 +Subject: [PATCH] mips: bpf: Enable eBPF JITs + +This patch enables the new eBPF JITs for 32-bit and 64-bit MIPS. It also +disables the old cBPF JIT to so cBPF programs are converted to use the +new JIT. + +Workarounds for R4000 CPU errata are not implemented by the JIT, so the +JIT is disabled if any of those workarounds are configured. + +Signed-off-by: Johan Almbladh +--- + +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -3294,6 +3294,7 @@ S: Supported + F: arch/arm64/net/ + + BPF JIT for MIPS (32-BIT AND 64-BIT) ++M: Johan Almbladh + M: Paul Burton + L: netdev@vger.kernel.org + L: bpf@vger.kernel.org +--- a/arch/mips/Kconfig ++++ b/arch/mips/Kconfig +@@ -49,7 +49,6 @@ config MIPS + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES + select HAVE_ASM_MODVERSIONS +- select HAVE_CBPF_JIT if !64BIT && !CPU_MICROMIPS + select HAVE_CONTEXT_TRACKING + select HAVE_TIF_NOHZ + select HAVE_C_RECORDMCOUNT +@@ -57,7 +56,10 @@ config MIPS + select HAVE_DEBUG_STACKOVERFLOW + select HAVE_DMA_CONTIGUOUS + select HAVE_DYNAMIC_FTRACE +- select HAVE_EBPF_JIT if 64BIT && !CPU_MICROMIPS && TARGET_ISA_REV >= 2 ++ select HAVE_EBPF_JIT if !CPU_MICROMIPS && \ ++ !CPU_DADDI_WORKAROUNDS && \ ++ !CPU_R4000_WORKAROUNDS && \ ++ !CPU_R4400_WORKAROUNDS + select HAVE_EXIT_THREAD + select HAVE_FAST_GUP + select HAVE_FTRACE_MCOUNT_RECORD +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,9 +2,10 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o ++obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o + + ifeq ($(CONFIG_32BIT),y) +- obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp32.o + else +- obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp64.o + endif diff --git a/target/linux/generic/backport-5.10/050-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch b/target/linux/generic/backport-5.10/050-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch new file mode 100644 index 0000000000..e25c336831 --- /dev/null +++ b/target/linux/generic/backport-5.10/050-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch @@ -0,0 +1,387 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:08 +0200 +Subject: [PATCH] mips: bpf: Remove old BPF JIT implementations + +This patch removes the old 32-bit cBPF and 64-bit eBPF JIT implementations. +They are replaced by a new eBPF implementation that supports both 32-bit +and 64-bit MIPS CPUs. + +Signed-off-by: Johan Almbladh +--- + delete mode 100644 arch/mips/net/bpf_jit.c + delete mode 100644 arch/mips/net/bpf_jit.h + delete mode 100644 arch/mips/net/bpf_jit_asm.S + delete mode 100644 arch/mips/net/ebpf_jit.c + +--- a/arch/mips/net/bpf_jit.h ++++ /dev/null +@@ -1,81 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0-only */ +-/* +- * Just-In-Time compiler for BPF filters on MIPS +- * +- * Copyright (c) 2014 Imagination Technologies Ltd. +- * Author: Markos Chandras +- */ +- +-#ifndef BPF_JIT_MIPS_OP_H +-#define BPF_JIT_MIPS_OP_H +- +-/* Registers used by JIT */ +-#define MIPS_R_ZERO 0 +-#define MIPS_R_V0 2 +-#define MIPS_R_A0 4 +-#define MIPS_R_A1 5 +-#define MIPS_R_T4 12 +-#define MIPS_R_T5 13 +-#define MIPS_R_T6 14 +-#define MIPS_R_T7 15 +-#define MIPS_R_S0 16 +-#define MIPS_R_S1 17 +-#define MIPS_R_S2 18 +-#define MIPS_R_S3 19 +-#define MIPS_R_S4 20 +-#define MIPS_R_S5 21 +-#define MIPS_R_S6 22 +-#define MIPS_R_S7 23 +-#define MIPS_R_SP 29 +-#define MIPS_R_RA 31 +- +-/* Conditional codes */ +-#define MIPS_COND_EQ 0x1 +-#define MIPS_COND_GE (0x1 << 1) +-#define MIPS_COND_GT (0x1 << 2) +-#define MIPS_COND_NE (0x1 << 3) +-#define MIPS_COND_ALL (0x1 << 4) +-/* Conditionals on X register or K immediate */ +-#define MIPS_COND_X (0x1 << 5) +-#define MIPS_COND_K (0x1 << 6) +- +-#define r_ret MIPS_R_V0 +- +-/* +- * Use 2 scratch registers to avoid pipeline interlocks. +- * There is no overhead during epilogue and prologue since +- * any of the $s0-$s6 registers will only be preserved if +- * they are going to actually be used. +- */ +-#define r_skb_hl MIPS_R_S0 /* skb header length */ +-#define r_skb_data MIPS_R_S1 /* skb actual data */ +-#define r_off MIPS_R_S2 +-#define r_A MIPS_R_S3 +-#define r_X MIPS_R_S4 +-#define r_skb MIPS_R_S5 +-#define r_M MIPS_R_S6 +-#define r_skb_len MIPS_R_S7 +-#define r_s0 MIPS_R_T4 /* scratch reg 1 */ +-#define r_s1 MIPS_R_T5 /* scratch reg 2 */ +-#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */ +-#define r_tmp MIPS_R_T7 /* No need to preserve this */ +-#define r_zero MIPS_R_ZERO +-#define r_sp MIPS_R_SP +-#define r_ra MIPS_R_RA +- +-#ifndef __ASSEMBLY__ +- +-/* Declare ASM helpers */ +- +-#define DECLARE_LOAD_FUNC(func) \ +- extern u8 func(unsigned long *skb, int offset); \ +- extern u8 func##_negative(unsigned long *skb, int offset); \ +- extern u8 func##_positive(unsigned long *skb, int offset) +- +-DECLARE_LOAD_FUNC(sk_load_word); +-DECLARE_LOAD_FUNC(sk_load_half); +-DECLARE_LOAD_FUNC(sk_load_byte); +- +-#endif +- +-#endif /* BPF_JIT_MIPS_OP_H */ +--- a/arch/mips/net/bpf_jit_asm.S ++++ /dev/null +@@ -1,285 +0,0 @@ +-/* +- * bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF +- * compiler. +- * +- * Copyright (C) 2015 Imagination Technologies Ltd. +- * Author: Markos Chandras +- * +- * This program is free software; you can redistribute it and/or modify it +- * under the terms of the GNU General Public License as published by the +- * Free Software Foundation; version 2 of the License. +- */ +- +-#include +-#include +-#include +-#include "bpf_jit.h" +- +-/* ABI +- * +- * r_skb_hl skb header length +- * r_skb_data skb data +- * r_off(a1) offset register +- * r_A BPF register A +- * r_X PF register X +- * r_skb(a0) *skb +- * r_M *scratch memory +- * r_skb_le skb length +- * r_s0 Scratch register 0 +- * r_s1 Scratch register 1 +- * +- * On entry: +- * a0: *skb +- * a1: offset (imm or imm + X) +- * +- * All non-BPF-ABI registers are free for use. On return, we only +- * care about r_ret. The BPF-ABI registers are assumed to remain +- * unmodified during the entire filter operation. +- */ +- +-#define skb a0 +-#define offset a1 +-#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */ +- +- /* We know better :) so prevent assembler reordering etc */ +- .set noreorder +- +-#define is_offset_negative(TYPE) \ +- /* If offset is negative we have more work to do */ \ +- slti t0, offset, 0; \ +- bgtz t0, bpf_slow_path_##TYPE##_neg; \ +- /* Be careful what follows in DS. */ +- +-#define is_offset_in_header(SIZE, TYPE) \ +- /* Reading from header? */ \ +- addiu $r_s0, $r_skb_hl, -SIZE; \ +- slt t0, $r_s0, offset; \ +- bgtz t0, bpf_slow_path_##TYPE; \ +- +-LEAF(sk_load_word) +- is_offset_negative(word) +-FEXPORT(sk_load_word_positive) +- is_offset_in_header(4, word) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- .set reorder +- lw $r_A, 0(t1) +- .set noreorder +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_A +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_A, 24 +- srl t1, $r_A, 24 +- srl t2, $r_A, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_A, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_word) +- +-LEAF(sk_load_half) +- is_offset_negative(half) +-FEXPORT(sk_load_half_positive) +- is_offset_in_header(2, half) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lhu $r_A, 0(t1) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh $r_A, $r_A +-# else +- sll t0, $r_A, 8 +- srl t1, $r_A, 8 +- andi t0, t0, 0xff00 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_half) +- +-LEAF(sk_load_byte) +- is_offset_negative(byte) +-FEXPORT(sk_load_byte_positive) +- is_offset_in_header(1, byte) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lbu $r_A, 0(t1) +- jr $r_ra +- move $r_ret, zero +- END(sk_load_byte) +- +-/* +- * call skb_copy_bits: +- * (prototype in linux/skbuff.h) +- * +- * int skb_copy_bits(sk_buff *skb, int offset, void *to, int len) +- * +- * o32 mandates we leave 4 spaces for argument registers in case +- * the callee needs to use them. Even though we don't care about +- * the argument registers ourselves, we need to allocate that space +- * to remain ABI compliant since the callee may want to use that space. +- * We also allocate 2 more spaces for $r_ra and our return register (*to). +- * +- * n64 is a bit different. The *caller* will allocate the space to preserve +- * the arguments. So in 64-bit kernels, we allocate the 4-arg space for no +- * good reason but it does not matter that much really. +- * +- * (void *to) is returned in r_s0 +- * +- */ +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-#define DS_OFFSET(SIZE) (4 * SZREG) +-#else +-#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE)) +-#endif +-#define bpf_slow_path_common(SIZE) \ +- /* Quick check. Are we within reasonable boundaries? */ \ +- LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \ +- sltu $r_s0, offset, $r_s1; \ +- beqz $r_s0, fault; \ +- /* Load 4th argument in DS */ \ +- LONG_ADDIU a3, zero, SIZE; \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, skb_copy_bits; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- /* Assign low slot to a2 */ \ +- PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \ +- jalr t0; \ +- /* Reset our destination slot (DS but it's ok) */ \ +- INT_S zero, (4 * SZREG)($r_sp); \ +- /* \ +- * skb_copy_bits returns 0 on success and -EFAULT \ +- * on error. Our data live in a2. Do not bother with \ +- * our data if an error has been returned. \ +- */ \ +- /* Restore our frame */ \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- INT_L $r_s0, (4 * SZREG)($r_sp); \ +- bltz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- move $r_ret, zero; \ +- +-NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp) +- bpf_slow_path_common(4) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_s0 +- jr $r_ra +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_s0, 24 +- srl t1, $r_s0, 24 +- srl t2, $r_s0, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_s0, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_word) +- +-NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp) +- bpf_slow_path_common(2) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- jr $r_ra +- wsbh $r_A, $r_s0 +-# else +- sll t0, $r_s0, 8 +- andi t1, $r_s0, 0xff00 +- andi t0, t0, 0xff00 +- srl t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_half) +- +-NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp) +- bpf_slow_path_common(1) +- jr $r_ra +- move $r_A, $r_s0 +- +- END(bpf_slow_path_byte) +- +-/* +- * Negative entry points +- */ +- .macro bpf_is_end_of_data +- li t0, SKF_LL_OFF +- /* Reading link layer data? */ +- slt t1, offset, t0 +- bgtz t1, fault +- /* Be careful what follows in DS. */ +- .endm +-/* +- * call skb_copy_bits: +- * (prototype in linux/filter.h) +- * +- * void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, +- * int k, unsigned int size) +- * +- * see above (bpf_slow_path_common) for ABI restrictions +- */ +-#define bpf_negative_common(SIZE) \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, bpf_internal_load_pointer_neg_helper; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- jalr t0; \ +- li a2, SIZE; \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- /* Check return pointer */ \ +- beqz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- /* Preserve our pointer */ \ +- move $r_s0, v0; \ +- /* Set return value */ \ +- move $r_ret, zero; \ +- +-bpf_slow_path_word_neg: +- bpf_is_end_of_data +-NESTED(sk_load_word_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(4) +- jr $r_ra +- lw $r_A, 0($r_s0) +- END(sk_load_word_negative) +- +-bpf_slow_path_half_neg: +- bpf_is_end_of_data +-NESTED(sk_load_half_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(2) +- jr $r_ra +- lhu $r_A, 0($r_s0) +- END(sk_load_half_negative) +- +-bpf_slow_path_byte_neg: +- bpf_is_end_of_data +-NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(1) +- jr $r_ra +- lbu $r_A, 0($r_s0) +- END(sk_load_byte_negative) +- +-fault: +- jr $r_ra +- addiu $r_ret, zero, 1 diff --git a/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch b/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch index 83bfd733d6..7e34bae8b4 100644 --- a/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch +++ b/target/linux/generic/backport-5.10/103-v5.13-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch @@ -25,7 +25,7 @@ Signed-off-by: Jason A. Donenfeld --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -2078,7 +2078,7 @@ config CPU_MIPS32 +@@ -2080,7 +2080,7 @@ config CPU_MIPS32 config CPU_MIPS64 bool default y if CPU_MIPS64_R1 || CPU_MIPS64_R2 || CPU_MIPS64_R5 || \ diff --git a/target/linux/generic/backport-5.4/071-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch b/target/linux/generic/backport-5.4/071-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch new file mode 100644 index 0000000000..82feb7421d --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-00-MIPS-uasm-Enable-muhu-opcode-for-MIPS-R6.patch @@ -0,0 +1,65 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:02 +0200 +Subject: [PATCH] MIPS: uasm: Enable muhu opcode for MIPS R6 + +Enable the 'muhu' instruction, complementing the existing 'mulu', needed +to implement a MIPS32 BPF JIT. + +Also fix a typo in the existing definition of 'dmulu'. + +Signed-off-by: Tony Ambardar + +This patch is a dependency for my 32-bit MIPS eBPF JIT. + +Signed-off-by: Johan Almbladh +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -145,6 +145,7 @@ Ip_u1(_mtlo); + Ip_u3u1u2(_mul); + Ip_u1u2(_multu); + Ip_u3u1u2(_mulu); ++Ip_u3u1u2(_muhu); + Ip_u3u1u2(_nor); + Ip_u3u1u2(_or); + Ip_u2u1u3(_ori); +--- a/arch/mips/mm/uasm-mips.c ++++ b/arch/mips/mm/uasm-mips.c +@@ -90,7 +90,7 @@ static const struct insn insn_table[insn + RS | RT | RD}, + [insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET}, + [insn_dmultu] = {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT}, +- [insn_dmulu] = {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op), ++ [insn_dmulu] = {M(spec_op, 0, 0, 0, dmultu_dmulu_op, dmultu_op), + RS | RT | RD}, + [insn_drotr] = {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE}, + [insn_drotr32] = {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE}, +@@ -150,6 +150,8 @@ static const struct insn insn_table[insn + [insn_mtlo] = {M(spec_op, 0, 0, 0, 0, mtlo_op), RS}, + [insn_mulu] = {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op), + RS | RT | RD}, ++ [insn_muhu] = {M(spec_op, 0, 0, 0, multu_muhu_op, multu_op), ++ RS | RT | RD}, + #ifndef CONFIG_CPU_MIPSR6 + [insn_mul] = {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD}, + #else +--- a/arch/mips/mm/uasm.c ++++ b/arch/mips/mm/uasm.c +@@ -59,7 +59,7 @@ enum opcode { + insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld, + insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi, + insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0, +- insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor, ++ insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor, + insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, + insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll, + insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, +@@ -344,6 +344,7 @@ I_u1(_mtlo) + I_u3u1u2(_mul) + I_u1u2(_multu) + I_u3u1u2(_mulu) ++I_u3u1u2(_muhu) + I_u3u1u2(_nor) + I_u3u1u2(_or) + I_u2u1u3(_ori) diff --git a/target/linux/generic/backport-5.4/071-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch b/target/linux/generic/backport-5.4/071-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch new file mode 100644 index 0000000000..3a4d573f80 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-01-mips-uasm-Add-workaround-for-Loongson-2F-nop-CPU-err.patch @@ -0,0 +1,31 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:03 +0200 +Subject: [PATCH] mips: uasm: Add workaround for Loongson-2F nop CPU errata + +This patch implements a workaround for the Loongson-2F nop in generated, +code, if the existing option CONFIG_CPU_NOP_WORKAROUND is set. Before, +the binutils option -mfix-loongson2f-nop was enabled, but no workaround +was done when emitting MIPS code. Now, the nop pseudo instruction is +emitted as "or ax,ax,zero" instead of the default "sll zero,zero,0". This +is consistent with the workaround implemented by binutils. + +Link: https://sourceware.org/legacy-ml/binutils/2009-11/msg00387.html + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/include/asm/uasm.h ++++ b/arch/mips/include/asm/uasm.h +@@ -249,7 +249,11 @@ static inline void uasm_l##lb(struct uas + #define uasm_i_bnezl(buf, rs, off) uasm_i_bnel(buf, rs, 0, off) + #define uasm_i_ehb(buf) uasm_i_sll(buf, 0, 0, 3) + #define uasm_i_move(buf, a, b) UASM_i_ADDU(buf, a, 0, b) ++#ifdef CONFIG_CPU_NOP_WORKAROUNDS ++#define uasm_i_nop(buf) uasm_i_or(buf, 1, 1, 0) ++#else + #define uasm_i_nop(buf) uasm_i_sll(buf, 0, 0, 0) ++#endif + #define uasm_i_ssnop(buf) uasm_i_sll(buf, 0, 0, 1) + + static inline void uasm_i_drotr_safe(u32 **p, unsigned int a1, diff --git a/target/linux/generic/backport-5.4/071-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch b/target/linux/generic/backport-5.4/071-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch new file mode 100644 index 0000000000..7980659961 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-02-mips-bpf-Add-eBPF-JIT-for-32-bit-MIPS.patch @@ -0,0 +1,3078 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:04 +0200 +Subject: [PATCH] mips: bpf: Add eBPF JIT for 32-bit MIPS + +This is an implementation of an eBPF JIT for 32-bit MIPS I-V and MIPS32. +The implementation supports all 32-bit and 64-bit ALU and JMP operations, +including the recently-added atomics. 64-bit div/mod and 64-bit atomics +are implemented using function calls to math64 and atomic64 functions, +respectively. All 32-bit operations are implemented natively by the JIT, +except if the CPU lacks ll/sc instructions. + +Register mapping +================ +All 64-bit eBPF registers are mapped to native 32-bit MIPS register pairs, +and does not use any stack scratch space for register swapping. This means +that all eBPF register data is kept in CPU registers all the time, and +this simplifies the register management a lot. It also reduces the JIT's +pressure on temporary registers since we do not have to move data around. + +Native register pairs are ordered according to CPU endiannes, following +the O32 calling convention for passing 64-bit arguments and return values. +The eBPF return value, arguments and callee-saved registers are mapped to +their native MIPS equivalents. + +Since the 32 highest bits in the eBPF FP (frame pointer) register are +always zero, only one general-purpose register is actually needed for the +mapping. The MIPS fp register is used for this purpose. The high bits are +mapped to MIPS register r0. This saves us one CPU register, which is much +needed for temporaries, while still allowing us to treat the R10 (FP) +register just like any other eBPF register in the JIT. + +The MIPS gp (global pointer) and at (assembler temporary) registers are +used as internal temporary registers for constant blinding. CPU registers +t6-t9 are used internally by the JIT when constructing more complex 64-bit +operations. This is precisely what is needed - two registers to store an +operand value, and two more as scratch registers when performing the +operation. + +The register mapping is shown below. + + R0 - $v1, $v0 return value + R1 - $a1, $a0 argument 1, passed in registers + R2 - $a3, $a2 argument 2, passed in registers + R3 - $t1, $t0 argument 3, passed on stack + R4 - $t3, $t2 argument 4, passed on stack + R5 - $t4, $t3 argument 5, passed on stack + R6 - $s1, $s0 callee-saved + R7 - $s3, $s2 callee-saved + R8 - $s5, $s4 callee-saved + R9 - $s7, $s6 callee-saved + FP - $r0, $fp 32-bit frame pointer + AX - $gp, $at constant-blinding + $t6 - $t9 unallocated, JIT temporaries + +Jump offsets +============ +The JIT tries to map all conditional JMP operations to MIPS conditional +PC-relative branches. The MIPS branch offset field is 18 bits, in bytes, +which is equivalent to the eBPF 16-bit instruction offset. However, since +the JIT may emit more than one CPU instruction per eBPF instruction, the +field width may overflow. If that happens, the JIT converts the long +conditional jump to a short PC-relative branch with the condition +inverted, jumping over a long unconditional absolute jmp (j). + +This conversion will change the instruction offset mapping used for jumps, +and may in turn result in more branch offset overflows. The JIT therefore +dry-runs the translation until no more branches are converted and the +offsets do not change anymore. There is an upper bound on this of course, +and if the JIT hits that limit, the last two iterations are run with all +branches being converted. + +Tail call count +=============== +The current tail call count is stored in the 16-byte area of the caller's +stack frame that is reserved for the callee in the o32 ABI. The value is +initialized in the prologue, and propagated to the tail-callee by skipping +the initialization instructions when emitting the tail call. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp.c + create mode 100644 arch/mips/net/bpf_jit_comp.h + create mode 100644 arch/mips/net/bpf_jit_comp32.c + +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,4 +2,9 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o +-obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ ++ifeq ($(CONFIG_32BIT),y) ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++else ++ obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++endif +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -0,0 +1,1020 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions common to 32-bit and 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++/* ++ * Code overview ++ * ============= ++ * ++ * - bpf_jit_comp.h ++ * Common definitions and utilities. ++ * ++ * - bpf_jit_comp.c ++ * Implementation of JIT top-level logic and exported JIT API functions. ++ * Implementation of internal operations shared by 32-bit and 64-bit code. ++ * JMP and ALU JIT control code, register control code, shared ALU and ++ * JMP/JMP32 JIT operations. ++ * ++ * - bpf_jit_comp32.c ++ * Implementation of functions to JIT prologue, epilogue and a single eBPF ++ * instruction for 32-bit MIPS CPUs. The functions use shared operations ++ * where possible, and implement the rest for 32-bit MIPS such as ALU64 ++ * operations. ++ * ++ * - bpf_jit_comp64.c ++ * Ditto, for 64-bit MIPS CPUs. ++ * ++ * Zero and sign extension ++ * ======================== ++ * 32-bit MIPS instructions on 64-bit MIPS registers use sign extension, ++ * but the eBPF instruction set mandates zero extension. We let the verifier ++ * insert explicit zero-extensions after 32-bit ALU operations, both for ++ * 32-bit and 64-bit MIPS JITs. Conditional JMP32 operations on 64-bit MIPs ++ * are JITed with sign extensions inserted when so expected. ++ * ++ * ALU operations ++ * ============== ++ * ALU operations on 32/64-bit MIPS and ALU64 operations on 64-bit MIPS are ++ * JITed in the following steps. ALU64 operations on 32-bit MIPS are more ++ * complicated and therefore only processed by special implementations in ++ * step (3). ++ * ++ * 1) valid_alu_i: ++ * Determine if an immediate operation can be emitted as such, or if ++ * we must fall back to the register version. ++ * ++ * 2) rewrite_alu_i: ++ * Convert BPF operation and immediate value to a canonical form for ++ * JITing. In some degenerate cases this form may be a no-op. ++ * ++ * 3) emit_alu_{i,i64,r,64}: ++ * Emit instructions for an ALU or ALU64 immediate or register operation. ++ * ++ * JMP operations ++ * ============== ++ * JMP and JMP32 operations require an JIT instruction offset table for ++ * translating the jump offset. This table is computed by dry-running the ++ * JIT without actually emitting anything. However, the computed PC-relative ++ * offset may overflow the 18-bit offset field width of the native MIPS ++ * branch instruction. In such cases, the long jump is converted into the ++ * following sequence. ++ * ++ * ! +2 Inverted PC-relative branch ++ * nop Delay slot ++ * j Unconditional absolute long jump ++ * nop Delay slot ++ * ++ * Since this converted sequence alters the offset table, all offsets must ++ * be re-calculated. This may in turn trigger new branch conversions, so ++ * the process is repeated until no further changes are made. Normally it ++ * completes in 1-2 iterations. If JIT_MAX_ITERATIONS should reached, we ++ * fall back to converting every remaining jump operation. The branch ++ * conversion is independent of how the JMP or JMP32 condition is JITed. ++ * ++ * JMP32 and JMP operations are JITed as follows. ++ * ++ * 1) setup_jmp_{i,r}: ++ * Convert jump conditional and offset into a form that can be JITed. ++ * This form may be a no-op, a canonical form, or an inverted PC-relative ++ * jump if branch conversion is necessary. ++ * ++ * 2) valid_jmp_i: ++ * Determine if an immediate operations can be emitted as such, or if ++ * we must fall back to the register version. Applies to JMP32 for 32-bit ++ * MIPS, and both JMP and JMP32 for 64-bit MIPS. ++ * ++ * 3) emit_jmp_{i,i64,r,r64}: ++ * Emit instructions for an JMP or JMP32 immediate or register operation. ++ * ++ * 4) finish_jmp_{i,r}: ++ * Emit any instructions needed to finish the jump. This includes a nop ++ * for the delay slot if a branch was emitted, and a long absolute jump ++ * if the branch was converted. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* Convenience macros for descriptor access */ ++#define CONVERTED(desc) ((desc) & JIT_DESC_CONVERT) ++#define INDEX(desc) ((desc) & ~JIT_DESC_CONVERT) ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, sw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, sd, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ ctx->stack_used = max((int)ctx->stack_used, depth); ++ return depth; ++} ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) ++{ ++ int reg; ++ ++ for (reg = 0; reg < BITS_PER_BYTE * sizeof(mask); reg++) ++ if (mask & BIT(reg)) { ++ if ((excl & BIT(reg)) == 0) { ++ if (sizeof(long) == 4) ++ emit(ctx, lw, reg, depth, MIPS_R_SP); ++ else /* sizeof(long) == 8 */ ++ emit(ctx, ld, reg, depth, MIPS_R_SP); ++ } ++ depth += sizeof(long); ++ } ++ ++ return depth; ++} ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc) ++{ ++ u32 index = INDEX(ctx->descriptors[loc]); ++ unsigned long pc = (unsigned long)&ctx->target[ctx->jit_index]; ++ unsigned long addr = (unsigned long)&ctx->target[index]; ++ ++ if (!ctx->target) ++ return 0; ++ ++ if ((addr ^ pc) & ~MIPS_JMP_MASK) ++ return -1; ++ ++ return addr & MIPS_JMP_MASK; ++} ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off) ++{ ++ return (INDEX(ctx->descriptors[ctx->bpf_index + off]) - ++ ctx->jit_index - 1) * sizeof(u32); ++} ++ ++/* dst = imm (register width) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm) ++{ ++ if (imm >= -0x8000 && imm <= 0x7fff) { ++ emit(ctx, addiu, dst, MIPS_R_ZERO, imm); ++ } else { ++ emit(ctx, lui, dst, (s16)((u32)imm >> 16)); ++ emit(ctx, ori, dst, dst, (u16)(imm & 0xffff)); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* dst = src (register width) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, ori, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate ALU immediate range */ ++bool valid_alu_i(u8 op, s32 imm) ++{ ++ switch (BPF_OP(op)) { ++ case BPF_NEG: ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ /* All legal eBPF values are valid */ ++ return true; ++ case BPF_ADD: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_SUB: ++ /* -imm must be 16 bits */ ++ return imm >= -0x7fff && imm <= 0x8000; ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_MUL: ++ /* imm must be zero or a positive power of two */ ++ return imm == 0 || (imm > 0 && is_power_of_2(imm)); ++ case BPF_DIV: ++ case BPF_MOD: ++ /* imm must be an 17-bit power of two */ ++ return (u32)imm <= 0x10000 && is_power_of_2((u32)imm); ++ } ++ return false; ++} ++ ++/* Rewrite ALU immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val) ++{ ++ bool act = true; ++ ++ switch (BPF_OP(op)) { ++ case BPF_LSH: ++ case BPF_RSH: ++ case BPF_ARSH: ++ case BPF_ADD: ++ case BPF_SUB: ++ case BPF_OR: ++ case BPF_XOR: ++ /* imm == 0 is a no-op */ ++ act = imm != 0; ++ break; ++ case BPF_MUL: ++ if (imm == 1) { ++ /* dst * 1 is a no-op */ ++ act = false; ++ } else if (imm == 0) { ++ /* dst * 0 is dst & 0 */ ++ op = BPF_AND; ++ } else { ++ /* dst * (1 << n) is dst << n */ ++ op = BPF_LSH; ++ imm = ilog2(abs(imm)); ++ } ++ break; ++ case BPF_DIV: ++ if (imm == 1) { ++ /* dst / 1 is a no-op */ ++ act = false; ++ } else { ++ /* dst / (1 << n) is dst >> n */ ++ op = BPF_RSH; ++ imm = ilog2(imm); ++ } ++ break; ++ case BPF_MOD: ++ /* dst % (1 << n) is dst & ((1 << n) - 1) */ ++ op = BPF_AND; ++ imm--; ++ break; ++ } ++ ++ *alu = op; ++ *val = imm; ++ return act; ++} ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, subu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, andi, dst, dst, (u16)imm); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, sll, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, srl, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, sra, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, addiu, dst, dst, -imm); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, dst, dst, src); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, dst, dst, src); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, dst, dst, src); ++ break; ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, sllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, srlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, srav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, addu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, subu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, dst, dst, src); ++ } else { ++ emit(ctx, multu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips32r6) { ++ emit(ctx, divu_r6, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips32r6) { ++ emit(ctx, modu, dst, dst, src); ++ } else { ++ emit(ctx, divu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, addu, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, MIPS_R_T8, MIPS_R_T9, src); ++ break; ++ } ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) ++{ ++ emit(ctx, ll, MIPS_R_T9, off, dst); ++ emit(ctx, bne, MIPS_R_T9, res, 12); ++ emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ ++ emit(ctx, sc, MIPS_R_T8, off, dst); ++ emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ ++ clobber_reg(ctx, res); ++} ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ u8 msk = MIPS_R_T9; ++ ++ switch (width) { ++ /* Swap bytes in a word */ ++ case 32: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, rotr, dst, dst, 16); ++ } else { ++ emit(ctx, sll, tmp, dst, 16); /* tmp = dst << 16 */ ++ emit(ctx, srl, dst, dst, 16); /* dst = dst >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++ ++ emit(ctx, lui, msk, 0xff); /* msk = 0x00ff0000 */ ++ emit(ctx, ori, msk, msk, 0xff); /* msk = msk | 0xff */ ++ ++ emit(ctx, and, tmp, dst, msk); /* tmp = dst & msk */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, dst, 8); /* dst = dst >> 8 */ ++ emit(ctx, and, dst, dst, msk); /* dst = dst & msk */ ++ emit(ctx, or, dst, dst, tmp); /* reg = dst | tmp */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ case 16: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, wsbh, dst, dst); ++ emit(ctx, andi, dst, dst, 0xffff); ++ } else { ++ emit(ctx, andi, tmp, dst, 0xff00); /* t = d & 0xff00 */ ++ emit(ctx, srl, tmp, tmp, 8); /* t = t >> 8 */ ++ emit(ctx, andi, dst, dst, 0x00ff); /* d = d & 0x00ff */ ++ emit(ctx, sll, dst, dst, 8); /* d = d << 8 */ ++ emit(ctx, or, dst, dst, tmp); /* d = d | t */ ++ } ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Validate jump immediate range */ ++bool valid_jmp_i(u8 op, s32 imm) ++{ ++ switch (op) { ++ case JIT_JNOP: ++ /* Immediate value not used */ ++ return true; ++ case BPF_JEQ: ++ case BPF_JNE: ++ /* No immediate operation */ ++ return false; ++ case BPF_JSET: ++ case JIT_JNSET: ++ /* imm must be 16 bits unsigned */ ++ return imm >= 0 && imm <= 0xffff; ++ case BPF_JGE: ++ case BPF_JLT: ++ case BPF_JSGE: ++ case BPF_JSLT: ++ /* imm must be 16 bits */ ++ return imm >= -0x8000 && imm <= 0x7fff; ++ case BPF_JGT: ++ case BPF_JLE: ++ case BPF_JSGT: ++ case BPF_JSLE: ++ /* imm + 1 must be 16 bits */ ++ return imm >= -0x8001 && imm <= 0x7ffe; ++ } ++ return false; ++} ++ ++/* Invert a conditional jump operation */ ++static u8 invert_jmp(u8 op) ++{ ++ switch (op) { ++ case BPF_JA: return JIT_JNOP; ++ case BPF_JEQ: return BPF_JNE; ++ case BPF_JNE: return BPF_JEQ; ++ case BPF_JSET: return JIT_JNSET; ++ case BPF_JGT: return BPF_JLE; ++ case BPF_JGE: return BPF_JLT; ++ case BPF_JLT: return BPF_JGE; ++ case BPF_JLE: return BPF_JGT; ++ case BPF_JSGT: return BPF_JSLE; ++ case BPF_JSGE: return BPF_JSLT; ++ case BPF_JSLT: return BPF_JSGE; ++ case BPF_JSLE: return BPF_JSGT; ++ } ++ return 0; ++} ++ ++/* Prepare a PC-relative jump operation */ ++static void setup_jmp(struct jit_context *ctx, u8 bpf_op, ++ s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ u32 *descp = &ctx->descriptors[ctx->bpf_index]; ++ int op = bpf_op; ++ int offset = 0; ++ ++ /* Do not compute offsets on the first pass */ ++ if (INDEX(*descp) == 0) ++ goto done; ++ ++ /* Skip jumps never taken */ ++ if (bpf_op == JIT_JNOP) ++ goto done; ++ ++ /* Convert jumps always taken */ ++ if (bpf_op == BPF_JA) ++ *descp |= JIT_DESC_CONVERT; ++ ++ /* ++ * Current ctx->jit_index points to the start of the branch preamble. ++ * Since the preamble differs among different branch conditionals, ++ * the current index cannot be used to compute the branch offset. ++ * Instead, we use the offset table value for the next instruction, ++ * which gives the index immediately after the branch delay slot. ++ */ ++ if (!CONVERTED(*descp)) { ++ int target = ctx->bpf_index + bpf_off + 1; ++ int origin = ctx->bpf_index + 1; ++ ++ offset = (INDEX(ctx->descriptors[target]) - ++ INDEX(ctx->descriptors[origin]) + 1) * sizeof(u32); ++ } ++ ++ /* ++ * The PC-relative branch offset field on MIPS is 18 bits signed, ++ * so if the computed offset is larger than this we generate a an ++ * absolute jump that we skip with an inverted conditional branch. ++ */ ++ if (CONVERTED(*descp) || offset < -0x20000 || offset > 0x1ffff) { ++ offset = 3 * sizeof(u32); ++ op = invert_jmp(bpf_op); ++ ctx->changes += !CONVERTED(*descp); ++ *descp |= JIT_DESC_CONVERT; ++ } ++ ++done: ++ *jit_off = offset; ++ *jit_op = op; ++} ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ bool always = false; ++ bool never = false; ++ ++ switch (bpf_op) { ++ case BPF_JEQ: ++ case BPF_JNE: ++ break; ++ case BPF_JSET: ++ case BPF_JLT: ++ never = imm == 0; ++ break; ++ case BPF_JGE: ++ always = imm == 0; ++ break; ++ case BPF_JGT: ++ never = (u32)imm == U32_MAX; ++ break; ++ case BPF_JLE: ++ always = (u32)imm == U32_MAX; ++ break; ++ case BPF_JSGT: ++ never = imm == S32_MAX && width == 32; ++ break; ++ case BPF_JSGE: ++ always = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLT: ++ never = imm == S32_MIN && width == 32; ++ break; ++ case BPF_JSLE: ++ always = imm == S32_MAX && width == 32; ++ break; ++ } ++ ++ if (never) ++ bpf_op = JIT_JNOP; ++ if (always) ++ bpf_op = BPF_JA; ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off) ++{ ++ switch (bpf_op) { ++ case BPF_JSET: ++ break; ++ case BPF_JEQ: ++ case BPF_JGE: ++ case BPF_JLE: ++ case BPF_JSGE: ++ case BPF_JSLE: ++ if (same_reg) ++ bpf_op = BPF_JA; ++ break; ++ case BPF_JNE: ++ case BPF_JLT: ++ case BPF_JGT: ++ case BPF_JSGT: ++ case BPF_JSLT: ++ if (same_reg) ++ bpf_op = JIT_JNOP; ++ break; ++ } ++ setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); ++} ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off) ++{ ++ /* Emit conditional branch delay slot */ ++ if (jit_op != JIT_JNOP) ++ emit(ctx, nop); ++ /* ++ * Emit an absolute long jump with delay slot, ++ * if the PC-relative branch was converted. ++ */ ++ if (CONVERTED(ctx->descriptors[ctx->bpf_index])) { ++ int target = get_target(ctx, ctx->bpf_index + bpf_off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ } ++ return 0; ++} ++ ++/* Jump immediate (32-bit) */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst & imm */ ++ case BPF_JSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slti, MIPS_R_T9, dst, imm); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slti, MIPS_R_T9, dst, imm + 1); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump register (32-bit) */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) ++{ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ case BPF_JEQ: ++ emit(ctx, beq, dst, src, off); ++ break; ++ /* PC += off if dst != src */ ++ case BPF_JNE: ++ emit(ctx, bne, dst, src, off); ++ break; ++ /* PC += off if dst & src */ ++ case BPF_JSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case JIT_JNSET: ++ emit(ctx, and, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit(ctx, sltu, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit(ctx, sltu, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit(ctx, slt, MIPS_R_T9, dst, src); ++ emit(ctx, bnez, MIPS_R_T9, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit(ctx, slt, MIPS_R_T9, src, dst); ++ emit(ctx, beqz, MIPS_R_T9, off); ++ break; ++ } ++} ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off) ++{ ++ int target = get_target(ctx, ctx->bpf_index + off + 1); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx) ++{ ++ int target = get_target(ctx, ctx->program->len); ++ ++ if (target < 0) ++ return -1; ++ emit(ctx, j, target); ++ emit(ctx, nop); ++ return 0; ++} ++ ++/* Build the program body from eBPF bytecode */ ++static int build_body(struct jit_context *ctx) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ unsigned int i; ++ ++ ctx->stack_used = 0; ++ for (i = 0; i < prog->len; i++) { ++ const struct bpf_insn *insn = &prog->insnsi[i]; ++ u32 *descp = &ctx->descriptors[i]; ++ int ret; ++ ++ access_reg(ctx, insn->src_reg); ++ access_reg(ctx, insn->dst_reg); ++ ++ ctx->bpf_index = i; ++ if (ctx->target == NULL) { ++ ctx->changes += INDEX(*descp) != ctx->jit_index; ++ *descp &= JIT_DESC_CONVERT; ++ *descp |= ctx->jit_index; ++ } ++ ++ ret = build_insn(insn, ctx); ++ if (ret < 0) ++ return ret; ++ ++ if (ret > 0) { ++ i++; ++ if (ctx->target == NULL) ++ descp[1] = ctx->jit_index; ++ } ++ } ++ ++ /* Store the end offset, where the epilogue begins */ ++ ctx->descriptors[prog->len] = ctx->jit_index; ++ return 0; ++} ++ ++/* Set the branch conversion flag on all instructions */ ++static void set_convert_flag(struct jit_context *ctx, bool enable) ++{ ++ const struct bpf_prog *prog = ctx->program; ++ u32 flag = enable ? JIT_DESC_CONVERT : 0; ++ unsigned int i; ++ ++ for (i = 0; i <= prog->len; i++) ++ ctx->descriptors[i] = INDEX(ctx->descriptors[i]) | flag; ++} ++ ++static void jit_fill_hole(void *area, unsigned int size) ++{ ++ u32 *p; ++ ++ /* We are guaranteed to have aligned memory. */ ++ for (p = area; size >= sizeof(u32); size -= sizeof(u32)) ++ uasm_i_break(&p, BRK_BUG); /* Increments p */ ++} ++ ++bool bpf_jit_needs_zext(void) ++{ ++ return true; ++} ++ ++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ++{ ++ struct bpf_prog *tmp, *orig_prog = prog; ++ struct bpf_binary_header *header = NULL; ++ struct jit_context ctx; ++ bool tmp_blinded = false; ++ unsigned int tmp_idx; ++ unsigned int image_size; ++ u8 *image_ptr; ++ int tries; ++ ++ /* ++ * If BPF JIT was not enabled then we must fall back to ++ * the interpreter. ++ */ ++ if (!prog->jit_requested) ++ return orig_prog; ++ /* ++ * If constant blinding was enabled and we failed during blinding ++ * then we must fall back to the interpreter. Otherwise, we save ++ * the new JITed code. ++ */ ++ tmp = bpf_jit_blind_constants(prog); ++ if (IS_ERR(tmp)) ++ return orig_prog; ++ if (tmp != prog) { ++ tmp_blinded = true; ++ prog = tmp; ++ } ++ ++ memset(&ctx, 0, sizeof(ctx)); ++ ctx.program = prog; ++ ++ /* ++ * Not able to allocate memory for descriptors[], then ++ * we must fall back to the interpreter ++ */ ++ ctx.descriptors = kcalloc(prog->len + 1, sizeof(*ctx.descriptors), ++ GFP_KERNEL); ++ if (ctx.descriptors == NULL) ++ goto out_err; ++ ++ /* First pass discovers used resources */ ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ /* ++ * Second pass computes instruction offsets. ++ * If any PC-relative branches are out of range, a sequence of ++ * a PC-relative branch + a jump is generated, and we have to ++ * try again from the beginning to generate the new offsets. ++ * This is done until no additional conversions are necessary. ++ * The last two iterations are done with all branches being ++ * converted, to guarantee offset table convergence within a ++ * fixed number of iterations. ++ */ ++ ctx.jit_index = 0; ++ build_prologue(&ctx); ++ tmp_idx = ctx.jit_index; ++ ++ tries = JIT_MAX_ITERATIONS; ++ do { ++ ctx.jit_index = tmp_idx; ++ ctx.changes = 0; ++ if (tries == 2) ++ set_convert_flag(&ctx, true); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ } while (ctx.changes > 0 && --tries > 0); ++ ++ if (WARN_ONCE(ctx.changes > 0, "JIT offsets failed to converge")) ++ goto out_err; ++ ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Now we know the size of the structure to make */ ++ image_size = sizeof(u32) * ctx.jit_index; ++ header = bpf_jit_binary_alloc(image_size, &image_ptr, ++ sizeof(u32), jit_fill_hole); ++ /* ++ * Not able to allocate memory for the structure then ++ * we must fall back to the interpretation ++ */ ++ if (header == NULL) ++ goto out_err; ++ ++ /* Actual pass to generate final JIT code */ ++ ctx.target = (u32 *)image_ptr; ++ ctx.jit_index = 0; ++ ++ /* ++ * If building the JITed code fails somehow, ++ * we fall back to the interpretation. ++ */ ++ build_prologue(&ctx); ++ if (build_body(&ctx) < 0) ++ goto out_err; ++ build_epilogue(&ctx, MIPS_R_RA); ++ ++ /* Populate line info meta data */ ++ set_convert_flag(&ctx, false); ++ bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]); ++ ++ /* Set as read-only exec and flush instruction cache */ ++ bpf_jit_binary_lock_ro(header); ++ flush_icache_range((unsigned long)header, ++ (unsigned long)&ctx.target[ctx.jit_index]); ++ ++ if (bpf_jit_enable > 1) ++ bpf_jit_dump(prog->len, image_size, 2, ctx.target); ++ ++ prog->bpf_func = (void *)ctx.target; ++ prog->jited = 1; ++ prog->jited_len = image_size; ++ ++out: ++ if (tmp_blinded) ++ bpf_jit_prog_release_other(prog, prog == orig_prog ? ++ tmp : orig_prog); ++ kfree(ctx.descriptors); ++ return prog; ++ ++out_err: ++ prog = orig_prog; ++ if (header) ++ bpf_jit_binary_free(header); ++ goto out; ++} +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -0,0 +1,211 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#ifndef _BPF_JIT_COMP_H ++#define _BPF_JIT_COMP_H ++ ++/* MIPS registers */ ++#define MIPS_R_ZERO 0 /* Const zero */ ++#define MIPS_R_AT 1 /* Asm temp */ ++#define MIPS_R_V0 2 /* Result */ ++#define MIPS_R_V1 3 /* Result */ ++#define MIPS_R_A0 4 /* Argument */ ++#define MIPS_R_A1 5 /* Argument */ ++#define MIPS_R_A2 6 /* Argument */ ++#define MIPS_R_A3 7 /* Argument */ ++#define MIPS_R_A4 8 /* Arg (n64) */ ++#define MIPS_R_A5 9 /* Arg (n64) */ ++#define MIPS_R_A6 10 /* Arg (n64) */ ++#define MIPS_R_A7 11 /* Arg (n64) */ ++#define MIPS_R_T0 8 /* Temp (o32) */ ++#define MIPS_R_T1 9 /* Temp (o32) */ ++#define MIPS_R_T2 10 /* Temp (o32) */ ++#define MIPS_R_T3 11 /* Temp (o32) */ ++#define MIPS_R_T4 12 /* Temporary */ ++#define MIPS_R_T5 13 /* Temporary */ ++#define MIPS_R_T6 14 /* Temporary */ ++#define MIPS_R_T7 15 /* Temporary */ ++#define MIPS_R_S0 16 /* Saved */ ++#define MIPS_R_S1 17 /* Saved */ ++#define MIPS_R_S2 18 /* Saved */ ++#define MIPS_R_S3 19 /* Saved */ ++#define MIPS_R_S4 20 /* Saved */ ++#define MIPS_R_S5 21 /* Saved */ ++#define MIPS_R_S6 22 /* Saved */ ++#define MIPS_R_S7 23 /* Saved */ ++#define MIPS_R_T8 24 /* Temporary */ ++#define MIPS_R_T9 25 /* Temporary */ ++/* MIPS_R_K0 26 Reserved */ ++/* MIPS_R_K1 27 Reserved */ ++#define MIPS_R_GP 28 /* Global ptr */ ++#define MIPS_R_SP 29 /* Stack ptr */ ++#define MIPS_R_FP 30 /* Frame ptr */ ++#define MIPS_R_RA 31 /* Return */ ++ ++/* ++ * Jump address mask for immediate jumps. The four most significant bits ++ * must be equal to PC. ++ */ ++#define MIPS_JMP_MASK 0x0fffffffUL ++ ++/* Maximum number of iterations in offset table computation */ ++#define JIT_MAX_ITERATIONS 8 ++ ++/* ++ * Jump pseudo-instructions used internally ++ * for branch conversion and branch optimization. ++ */ ++#define JIT_JNSET 0xe0 ++#define JIT_JNOP 0xf0 ++ ++/* Descriptor flag for PC-relative branch conversion */ ++#define JIT_DESC_CONVERT BIT(31) ++ ++/* JIT context for an eBPF program */ ++struct jit_context { ++ struct bpf_prog *program; /* The eBPF program being JITed */ ++ u32 *descriptors; /* eBPF to JITed CPU insn descriptors */ ++ u32 *target; /* JITed code buffer */ ++ u32 bpf_index; /* Index of current BPF program insn */ ++ u32 jit_index; /* Index of current JIT target insn */ ++ u32 changes; /* Number of PC-relative branch conv */ ++ u32 accessed; /* Bit mask of read eBPF registers */ ++ u32 clobbered; /* Bit mask of modified CPU registers */ ++ u32 stack_size; /* Total allocated stack size in bytes */ ++ u32 saved_size; /* Size of callee-saved registers */ ++ u32 stack_used; /* Stack size used for function calls */ ++}; ++ ++/* Emit the instruction if the JIT memory space has been allocated */ ++#define emit(ctx, func, ...) \ ++do { \ ++ if ((ctx)->target != NULL) { \ ++ u32 *p = &(ctx)->target[ctx->jit_index]; \ ++ uasm_i_##func(&p, ##__VA_ARGS__); \ ++ } \ ++ (ctx)->jit_index++; \ ++} while (0) ++ ++/* ++ * Mark a BPF register as accessed, it needs to be ++ * initialized by the program if expected, e.g. FP. ++ */ ++static inline void access_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->accessed |= BIT(reg); ++} ++ ++/* ++ * Mark a CPU register as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static inline void clobber_reg(struct jit_context *ctx, u8 reg) ++{ ++ ctx->clobbered |= BIT(reg); ++} ++ ++/* ++ * Push registers on the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be written is returned. ++ */ ++int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* ++ * Pop registers from the stack, starting at a given depth from the stack ++ * pointer and increasing. The next depth to be read is returned. ++ */ ++int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); ++ ++/* Compute the 28-bit jump target address from a BPF program location */ ++int get_target(struct jit_context *ctx, u32 loc); ++ ++/* Compute the PC-relative offset to relative BPF program offset */ ++int get_offset(const struct jit_context *ctx, int off); ++ ++/* dst = imm (32-bit) */ ++void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); ++ ++/* dst = src (32-bit) */ ++void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); ++ ++/* Validate ALU/ALU64 immediate range */ ++bool valid_alu_i(u8 op, s32 imm); ++ ++/* Rewrite ALU/ALU64 immediate operation */ ++bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); ++ ++/* ALU immediate operation (32-bit) */ ++void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); ++ ++/* ALU register operation (32-bit) */ ++void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); ++ ++/* Atomic read-modify-write (32-bit) */ ++void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); ++ ++/* Atomic compare-and-exchange (32-bit) */ ++void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off); ++ ++/* Swap bytes and truncate a register word or half word */ ++void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width); ++ ++/* Validate JMP/JMP32 immediate range */ ++bool valid_jmp_i(u8 op, s32 imm); ++ ++/* Prepare a PC-relative jump operation with immediate conditional */ ++void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Prepare a PC-relative jump operation with register conditional */ ++void setup_jmp_r(struct jit_context *ctx, bool same_reg, ++ u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); ++ ++/* Finish a PC-relative jump operation */ ++int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off); ++ ++/* Conditional JMP/JMP32 immediate */ ++void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); ++ ++/* Conditional JMP/JMP32 register */ ++void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); ++ ++/* Jump always */ ++int emit_ja(struct jit_context *ctx, s16 off); ++ ++/* Jump to epilogue */ ++int emit_exit(struct jit_context *ctx); ++ ++/* ++ * Build program prologue to set up the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_prologue(struct jit_context *ctx); ++ ++/* ++ * Build the program epilogue to restore the stack and registers. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++void build_epilogue(struct jit_context *ctx, int dest_reg); ++ ++/* ++ * Convert an eBPF instruction to native instruction, i.e ++ * JITs an eBPF instruction. ++ * Returns : ++ * 0 - Successfully JITed an 8-byte eBPF instruction ++ * >0 - Successfully JITed a 16-byte eBPF instruction ++ * <0 - Failed to JIT. ++ * This function is implemented separately for 32-bit and 64-bit JITs. ++ */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx); ++ ++#endif /* _BPF_JIT_COMP_H */ +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp32.c +@@ -0,0 +1,1741 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 32-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS a4-a7 are not available in the o32 ABI */ ++#undef MIPS_R_A4 ++#undef MIPS_R_A5 ++#undef MIPS_R_A6 ++#undef MIPS_R_A7 ++ ++/* Stack is 8-byte aligned in o32 ABI */ ++#define MIPS_STACK_ALIGNMENT 8 ++ ++/* ++ * The top 16 bytes of a stack frame is reserved for the callee in O32 ABI. ++ * This corresponds to stack space for register arguments a0-a3. ++ */ ++#define JIT_RESERVED_STACK 16 ++ ++/* Temporary 64-bit register used by JIT */ ++#define JIT_REG_TMP MAX_BPF_JIT_REG ++ ++/* ++ * Number of prologue bytes to skip when doing a tail call. ++ * Tail call count (TCC) initialization (8 bytes) always, plus ++ * R0-to-v0 assignment (4 bytes) if big endian. ++ */ ++#ifdef __BIG_ENDIAN ++#define JIT_TCALL_SKIP 12 ++#else ++#define JIT_TCALL_SKIP 8 ++#endif ++ ++/* CPU registers holding the callee return value */ ++#define JIT_RETURN_REGS \ ++ (BIT(MIPS_R_V0) | \ ++ BIT(MIPS_R_V1)) ++ ++/* CPU registers arguments passed to callee directly */ ++#define JIT_ARG_REGS \ ++ (BIT(MIPS_R_A0) | \ ++ BIT(MIPS_R_A1) | \ ++ BIT(MIPS_R_A2) | \ ++ BIT(MIPS_R_A3)) ++ ++/* CPU register arguments passed to callee on stack */ ++#define JIT_STACK_REGS \ ++ (BIT(MIPS_R_T0) | \ ++ BIT(MIPS_R_T1) | \ ++ BIT(MIPS_R_T2) | \ ++ BIT(MIPS_R_T3) | \ ++ BIT(MIPS_R_T4) | \ ++ BIT(MIPS_R_T5)) ++ ++/* Caller-saved CPU registers */ ++#define JIT_CALLER_REGS \ ++ (JIT_RETURN_REGS | \ ++ JIT_ARG_REGS | \ ++ JIT_STACK_REGS) ++ ++/* Callee-saved CPU registers */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* ++ * Mapping of 64-bit eBPF registers to 32-bit native MIPS registers. ++ * ++ * 1) Native register pairs are ordered according to CPU endiannes, following ++ * the MIPS convention for passing 64-bit arguments and return values. ++ * 2) The eBPF return value, arguments and callee-saved registers are mapped ++ * to their native MIPS equivalents. ++ * 3) Since the 32 highest bits in the eBPF FP register are always zero, ++ * only one general-purpose register is actually needed for the mapping. ++ * We use the fp register for this purpose, and map the highest bits to ++ * the MIPS register r0 (zero). ++ * 4) We use the MIPS gp and at registers as internal temporary registers ++ * for constant blinding. The gp register is callee-saved. ++ * 5) One 64-bit temporary register is mapped for use when sign-extending ++ * immediate operands. MIPS registers t6-t9 are available to the JIT ++ * for as temporaries when implementing complex 64-bit operations. ++ * ++ * With this scheme all eBPF registers are being mapped to native MIPS ++ * registers without having to use any stack scratch space. The direct ++ * register mapping (2) simplifies the handling of function calls. ++ */ ++static const u8 bpf2mips32[][2] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = {MIPS_R_V1, MIPS_R_V0}, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = {MIPS_R_A1, MIPS_R_A0}, ++ [BPF_REG_2] = {MIPS_R_A3, MIPS_R_A2}, ++ /* Remaining arguments, to be passed on the stack per O32 ABI */ ++ [BPF_REG_3] = {MIPS_R_T1, MIPS_R_T0}, ++ [BPF_REG_4] = {MIPS_R_T3, MIPS_R_T2}, ++ [BPF_REG_5] = {MIPS_R_T5, MIPS_R_T4}, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = {MIPS_R_S1, MIPS_R_S0}, ++ [BPF_REG_7] = {MIPS_R_S3, MIPS_R_S2}, ++ [BPF_REG_8] = {MIPS_R_S5, MIPS_R_S4}, ++ [BPF_REG_9] = {MIPS_R_S7, MIPS_R_S6}, ++ /* Read-only frame pointer to access the eBPF stack */ ++#ifdef __BIG_ENDIAN ++ [BPF_REG_FP] = {MIPS_R_FP, MIPS_R_ZERO}, ++#else ++ [BPF_REG_FP] = {MIPS_R_ZERO, MIPS_R_FP}, ++#endif ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = {MIPS_R_GP, MIPS_R_AT}, ++ /* Temporary register for internal JIT use */ ++ [JIT_REG_TMP] = {MIPS_R_T7, MIPS_R_T6}, ++}; ++ ++/* Get low CPU register for a 64-bit eBPF register mapping */ ++static inline u8 lo(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[0]; ++#else ++ return reg[1]; ++#endif ++} ++ ++/* Get high CPU register for a 64-bit eBPF register mapping */ ++static inline u8 hi(const u8 reg[]) ++{ ++#ifdef __BIG_ENDIAN ++ return reg[1]; ++#else ++ return reg[0]; ++#endif ++} ++ ++/* ++ * Mark a 64-bit CPU register pair as clobbered, it needs to be ++ * saved/restored by the program if callee-saved. ++ */ ++static void clobber_reg64(struct jit_context *ctx, const u8 reg[]) ++{ ++ clobber_reg(ctx, reg[0]); ++ clobber_reg(ctx, reg[1]); ++} ++ ++/* dst = imm (sign-extended) */ ++static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ emit_mov_i(ctx, lo(dst), imm); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ else ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) ++{ ++ if (!ctx->program->aux->verifier_zext) { ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ } ++} ++ ++/* Load delay slot, if ISA mandates it */ ++static void emit_load_delay(struct jit_context *ctx) ++{ ++ if (!cpu_has_mips_2_3_4_5_r) ++ emit(ctx, nop); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, ++ const u8 dst[], s32 imm, u8 op) ++{ ++ u8 src = MIPS_R_T6; ++ ++ /* ++ * ADD/SUB with all but the max negative imm can be handled by ++ * inverting the operation and the imm value, saving one insn. ++ */ ++ if (imm > S32_MIN && imm < 0) ++ switch (op) { ++ case BPF_ADD: ++ op = BPF_SUB; ++ imm = -imm; ++ break; ++ case BPF_SUB: ++ op = BPF_ADD; ++ imm = -imm; ++ break; ++ } ++ ++ /* Move immediate to temporary register */ ++ emit_mov_i(ctx, src, imm); ++ ++ switch (op) { ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, addu, lo(dst), lo(dst), src); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), src); ++ emit(ctx, subu, lo(dst), lo(dst), src); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), hi(dst), 1); ++ break; ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), src); ++ if (imm < 0) ++ emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); ++ break; ++ /* dst = dst & imm */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), src); ++ if (imm >= 0) ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), src); ++ if (imm < 0) { ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, addiu, hi(dst), hi(dst), -1); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst + src */ ++ case BPF_ADD: ++ if (src == dst) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 31); ++ emit(ctx, addu, lo(dst), lo(dst), lo(dst)); ++ } else { ++ emit(ctx, addu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ } ++ emit(ctx, addu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); ++ emit(ctx, subu, lo(dst), lo(dst), lo(src)); ++ emit(ctx, subu, hi(dst), hi(dst), hi(src)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ctx, or, lo(dst), lo(dst), lo(src)); ++ emit(ctx, or, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ctx, and, lo(dst), lo(dst), lo(src)); ++ emit(ctx, and, hi(dst), hi(dst), hi(src)); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ctx, xor, lo(dst), lo(dst), lo(src)); ++ emit(ctx, xor, hi(dst), hi(dst), hi(src)); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU invert (64-bit) */ ++static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) ++{ ++ emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); ++ emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); ++ emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); ++ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift immediate (64-bit) */ ++static void emit_shift_i64(struct jit_context *ctx, ++ const u8 dst[], u32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ if (imm < 32) { ++ emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); ++ emit(ctx, sll, lo(dst), lo(dst), imm); ++ emit(ctx, sll, hi(dst), hi(dst), imm); ++ emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sll, hi(dst), lo(dst), imm - 32); ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, srl, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, srl, lo(dst), hi(dst), imm - 32); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ } ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ if (imm < 32) { ++ emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); ++ emit(ctx, srl, lo(dst), lo(dst), imm); ++ emit(ctx, sra, hi(dst), hi(dst), imm); ++ emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); ++ } else { ++ emit(ctx, sra, lo(dst), hi(dst), imm - 32); ++ emit(ctx, sra, hi(dst), hi(dst), 31); ++ } ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU shift register (64-bit) */ ++static void emit_shift_r64(struct jit_context *ctx, ++ const u8 dst[], u8 src, u8 op) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ ++ emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */ ++ emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */ ++ emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */ ++ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ ++ emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ ++ emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */ ++ emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ ++ emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ ++ emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ ++ emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ /* Next: shift >= 32 */ ++ emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ ++ emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ ++ emit(ctx, b, 20); /* PC += 20 */ ++ /* +16: shift < 32 */ ++ emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ ++ emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ ++ emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ ++ emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ ++ emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ ++ break; ++ } ++ ++ /* +20: Done */ ++ clobber_reg64(ctx, dst); ++} ++ ++/* ALU mul immediate (64x32-bit) */ ++static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) ++{ ++ u8 src = MIPS_R_T6; ++ u8 tmp = MIPS_R_T9; ++ ++ switch (imm) { ++ /* dst = dst * 1 is a no-op */ ++ case 1: ++ break; ++ /* dst = dst * -1 */ ++ case -1: ++ emit_neg_i64(ctx, dst); ++ break; ++ case 0: ++ emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); ++ emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Full 64x32 multiply */ ++ default: ++ /* hi(dst) = hi(dst) * src(imm) */ ++ emit_mov_i(ctx, src, imm); ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, hi(dst), hi(dst), src); ++ } else { ++ emit(ctx, multu, hi(dst), src); ++ emit(ctx, mflo, hi(dst)); ++ } ++ ++ /* hi(dst) = hi(dst) - lo(dst) */ ++ if (imm < 0) ++ emit(ctx, subu, hi(dst), hi(dst), lo(dst)); ++ ++ /* tmp = lo(dst) * src(imm) >> 32 */ ++ /* lo(dst) = lo(dst) * src(imm) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), src); ++ emit(ctx, mulu, lo(dst), lo(dst), src); ++ } else { ++ emit(ctx, multu, lo(dst), src); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) += tmp */ ++ emit(ctx, addu, hi(dst), hi(dst), tmp); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* ALU mul register (64x64-bit) */ ++static void emit_mul_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[]) ++{ ++ u8 acc = MIPS_R_T8; ++ u8 tmp = MIPS_R_T9; ++ ++ /* acc = hi(dst) * lo(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, acc, hi(dst), lo(src)); ++ } else { ++ emit(ctx, multu, hi(dst), lo(src)); ++ emit(ctx, mflo, acc); ++ } ++ ++ /* tmp = lo(dst) * hi(src) */ ++ if (cpu_has_mips32r1 || cpu_has_mips32r6) { ++ emit(ctx, mul, tmp, lo(dst), hi(src)); ++ } else { ++ emit(ctx, multu, lo(dst), hi(src)); ++ emit(ctx, mflo, tmp); ++ } ++ ++ /* acc += tmp */ ++ emit(ctx, addu, acc, acc, tmp); ++ ++ /* tmp = lo(dst) * lo(src) >> 32 */ ++ /* lo(dst) = lo(dst) * lo(src) */ ++ if (cpu_has_mips32r6) { ++ emit(ctx, muhu, tmp, lo(dst), lo(src)); ++ emit(ctx, mulu, lo(dst), lo(dst), lo(src)); ++ } else { ++ emit(ctx, multu, lo(dst), lo(src)); ++ emit(ctx, mflo, lo(dst)); ++ emit(ctx, mfhi, tmp); ++ } ++ ++ /* hi(dst) = acc + tmp */ ++ emit(ctx, addu, hi(dst), acc, tmp); ++ clobber_reg64(ctx, dst); ++} ++ ++/* Helper function for 64-bit modulo */ ++static u64 jit_mod64(u64 a, u64 b) ++{ ++ u64 rem; ++ ++ div64_u64_rem(a, b, &rem); ++ return rem; ++} ++ ++/* ALU div/mod register (64-bit) */ ++static void emit_divmod_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], u8 op) ++{ ++ const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ ++ int exclude, k; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ ++ /* Put 64-bit arguments 1 and 2 in registers a0-a3 */ ++ for (k = 0; k < 2; k++) { ++ emit(ctx, move, MIPS_R_T9, src[k]); ++ emit(ctx, move, r1[k], dst[k]); ++ emit(ctx, move, r2[k], MIPS_R_T9); ++ } ++ ++ /* Emit function call */ ++ switch (BPF_OP(op)) { ++ /* dst = dst / src */ ++ case BPF_DIV: ++ addr = (u32)&div64_u64; ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ addr = (u32)&jit_mod64; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Store the 64-bit result in dst */ ++ emit(ctx, move, dst[0], r0[0]); ++ emit(ctx, move, dst[1], r0[1]); ++ ++ /* Restore caller-saved registers, excluding the computed result */ ++ exclude = BIT(lo(dst)) | BIT(hi(dst)); ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ ++ clobber_reg64(ctx, dst); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Swap bytes in a register word */ ++static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */ ++ emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ ++ emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap half words in a register word */ ++static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */ ++ emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ u8 tmp = MIPS_R_T8; ++ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips32r2 || cpu_has_mips32r6) { ++ emit(ctx, rotr, tmp, hi(dst), 16); ++ emit(ctx, rotr, hi(dst), lo(dst), 16); ++ emit(ctx, wsbh, lo(dst), tmp); ++ emit(ctx, wsbh, hi(dst), hi(dst)); ++ } else { ++ emit_swap16_r(ctx, tmp, lo(dst)); ++ emit_swap16_r(ctx, lo(dst), hi(dst)); ++ emit(ctx, move, hi(dst), tmp); ++ ++ emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */ ++ emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */ ++ emit_swap8_r(ctx, lo(dst), lo(dst), tmp); ++ emit_swap8_r(ctx, hi(dst), hi(dst), tmp); ++ } ++ break; ++ /* Swap bytes in a word */ ++ /* Swap bytes in a half word */ ++ case 32: ++ case 16: ++ emit_bswap_r(ctx, lo(dst), width); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ clobber_reg(ctx, hi(dst)); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ emit(ctx, andi, lo(dst), lo(dst), 0xffff); ++ clobber_reg64(ctx, dst); ++ break; ++ } ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, ++ const u8 dst[], u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lw, lo(dst), off, src); ++ emit(ctx, move, hi(dst), MIPS_R_ZERO); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ if (dst[1] == src) { ++ emit(ctx, lw, dst[0], off + 4, src); ++ emit(ctx, lw, dst[1], off, src); ++ } else { ++ emit(ctx, lw, dst[1], off, src); ++ emit(ctx, lw, dst[0], off + 4, src); ++ } ++ emit_load_delay(ctx); ++ break; ++ } ++ clobber_reg64(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, ++ const u8 dst, const u8 src[], s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, lo(src), off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, lo(src), off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, lo(src), off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sw, src[1], off, dst); ++ emit(ctx, sw, src[0], off + 4, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write (32-bit, non-ll/sc fallback) */ ++static void emit_atomic_r32(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: dst+off if xchg, otherwise src, passed in register a0 ++ * Argument 2: src if xchg, othersize dst+off, passed in register a1 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, MIPS_R_A0, src); ++ emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* Atomic read-modify-write (64-bit) */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, const u8 src[], s16 off, u8 code) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ ++ u32 exclude = 0; ++ u32 addr = 0; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ 0, JIT_RESERVED_STACK); ++ /* ++ * Argument 1: 64-bit src, passed in registers a0-a1 ++ * Argument 2: 32-bit dst+off, passed in register a2 ++ */ ++ emit(ctx, move, MIPS_R_T9, dst); ++ emit(ctx, move, r1[0], src[0]); ++ emit(ctx, move, r1[1], src[1]); ++ emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off); ++ ++ /* Emit function call */ ++ switch (code) { ++ case BPF_ADD: ++ addr = (u32)&atomic64_add; ++ break; ++ case BPF_SUB: ++ addr = (u32)&atomic64_sub; ++ break; ++ case BPF_OR: ++ addr = (u32)&atomic64_or; ++ break; ++ case BPF_AND: ++ addr = (u32)&atomic64_and; ++ break; ++ case BPF_XOR: ++ addr = (u32)&atomic64_xor; ++ break; ++ } ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers, except any fetched value */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, ++ exclude, JIT_RESERVED_STACK); ++ emit_load_delay(ctx); ++ clobber_reg(ctx, MIPS_R_RA); ++} ++ ++/* ++ * Conditional movz or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* ++ * Conditional movn or an emulated equivalent. ++ * Note that the rs register may be modified. ++ */ ++static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) ++{ ++ if (cpu_has_mips_2) { ++ emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */ ++ } else if (cpu_has_mips32r6) { ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */ ++ emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */ ++ if (rs != MIPS_R_ZERO) ++ emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ ++ } else { ++ emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */ ++ emit(ctx, nop); /* +0: delay slot */ ++ emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ ++ } ++ clobber_reg(ctx, rd); ++ clobber_reg(ctx, rs); ++} ++ ++/* Emulation of 64-bit sltiu rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_sltiu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ if (imm < 0) { ++ emit_mov_i(ctx, rd, imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++ } else { /* imm >= 0 */ ++ if (imm > 0x7fff) { ++ emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */ ++ emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ ++ } else { ++ emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */ ++ } ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */ ++ } ++} ++ ++/* Emulation of 64-bit sltu rd, rs, rt */ ++static void emit_sltu_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */ ++ emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */ ++ emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */ ++ emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */ ++ emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ ++} ++ ++/* Emulation of 64-bit slti rd, rs, imm, where imm may be S32_MAX + 1 */ ++static void emit_slti_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], s64 imm) ++{ ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ u8 cmp; ++ ++ /* ++ * if ((rs < 0) ^ (imm < 0)) t1 = imm >u rsl ++ * else t1 = rsl > 31 */ ++ if (imm < 0) ++ emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */ ++ else ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ /* ++ * if ((imm < 0 && rsh != 0xffffffff) || ++ * (imm >= 0 && rsh != 0)) ++ * t1 = 0 ++ */ ++ if (imm < 0) { ++ emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */ ++ cmp = rd; ++ } else { /* imm >= 0 */ ++ cmp = hi(rs); ++ } ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */ ++ ++ /* ++ * if (imm < 0) rd = rsh < -1 ++ * else rd = rsh != 0 ++ * rd = rd | t1 ++ */ ++ emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */ ++ emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ ++} ++ ++/* Emulation of 64-bit(slt rd, rs, rt) */ ++static void emit_slt_r64(struct jit_context *ctx, u8 rd, ++ const u8 rs[], const u8 rt[]) ++{ ++ u8 t1 = MIPS_R_T7; ++ u8 t2 = MIPS_R_T8; ++ u8 t3 = MIPS_R_T9; ++ ++ /* ++ * if ((rs < 0) ^ (rt < 0)) t1 = rtl > 31 */ ++ emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ ++ emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */ ++ ++ /* rd = (rsh < rth) | t1 */ ++ emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh = -0x7fff && imm <= 0x8000) { ++ emit(ctx, addiu, tmp, lo(dst), -imm); ++ } else if ((u32)imm <= 0xffff) { ++ emit(ctx, xori, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, xor, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) { /* Compare sign extension */ ++ emit(ctx, addu, MIPS_R_T9, hi(dst), 1); ++ emit(ctx, or, tmp, tmp, MIPS_R_T9); ++ } else { /* Compare zero extension */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ } ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, tmp, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst & imm */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ if ((u32)imm <= 0xffff) { ++ emit(ctx, andi, tmp, lo(dst), imm); ++ } else { /* Register fallback */ ++ emit_mov_i(ctx, tmp, imm); ++ emit(ctx, and, tmp, lo(dst), tmp); ++ } ++ if (imm < 0) /* Sign-extension pulls in high word */ ++ emit(ctx, or, tmp, tmp, hi(dst)); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, tmp, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst > imm */ ++ case BPF_JGT: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm */ ++ case BPF_JGE: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm */ ++ case BPF_JLT: ++ emit_sltiu_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm */ ++ case BPF_JLE: ++ emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst > imm (signed) */ ++ case BPF_JSGT: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst >= imm (signed) */ ++ case BPF_JSGE: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, beqz, tmp, off); ++ break; ++ /* PC += off if dst < imm (signed) */ ++ case BPF_JSLT: ++ emit_slti_r64(ctx, tmp, dst, imm); ++ emit(ctx, bnez, tmp, off); ++ break; ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JSLE: ++ emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); ++ emit(ctx, bnez, tmp, off); ++ break; ++ } ++} ++ ++/* Jump register (64-bit) */ ++static void emit_jmp_r64(struct jit_context *ctx, ++ const u8 dst[], const u8 src[], s32 off, u8 op) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ switch (op) { ++ /* No-op, used internally for branch optimization */ ++ case JIT_JNOP: ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ case BPF_JEQ: ++ case BPF_JNE: ++ emit(ctx, subu, t1, lo(dst), lo(src)); ++ emit(ctx, subu, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JEQ) ++ emit(ctx, beqz, t1, off); ++ else /* BPF_JNE */ ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst & src */ ++ /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ ++ case BPF_JSET: ++ case JIT_JNSET: ++ emit(ctx, and, t1, lo(dst), lo(src)); ++ emit(ctx, and, t2, hi(dst), hi(src)); ++ emit(ctx, or, t1, t1, t2); ++ if (op == BPF_JSET) ++ emit(ctx, bnez, t1, off); ++ else /* JIT_JNSET */ ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src */ ++ case BPF_JGT: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src */ ++ case BPF_JGE: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src */ ++ case BPF_JLT: ++ emit_sltu_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src */ ++ case BPF_JLE: ++ emit_sltu_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst > src (signed) */ ++ case BPF_JSGT: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst >= src (signed) */ ++ case BPF_JSGE: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, beqz, t1, off); ++ break; ++ /* PC += off if dst < src (signed) */ ++ case BPF_JSLT: ++ emit_slt_r64(ctx, t1, dst, src); ++ emit(ctx, bnez, t1, off); ++ break; ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JSLE: ++ emit_slt_r64(ctx, t1, src, dst); ++ emit(ctx, beqz, t1, off); ++ break; ++ } ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push stack arguments */ ++ push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK); ++ ++ /* Emit function call */ ++ emit_mov_i(ctx, MIPS_R_T9, addr); ++ emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); ++ emit(ctx, nop); /* Delay slot */ ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = lo(bpf2mips32[BPF_REG_2]); ++ u8 ind = lo(bpf2mips32[BPF_REG_3]); ++ u8 t1 = MIPS_R_T8; ++ u8 t2 = MIPS_R_T9; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ * stack[sz] - remaining tail call count, initialized in prologue ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */ ++ emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */ ++ /* (next insn delay slot) */ ++ /* if (TCC-- <= 0) goto out */ ++ emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 < 0 */ ++ emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */ ++ emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */ ++ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */ ++ emit(ctx, addu, t1, t1, ary); /* t1 += ary */ ++ emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */ ++ emit_load_delay(ctx); /* Load delay slot */ ++ emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, t1); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Caller's stack frame : ++ * :----------------------------: ++ * : 64-bit eBPF args r3-r5 : ++ * :----------------------------: ++ * : Reserved / tail call count : ++ * +============================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +----------------------------+ <--- eBPF FP (MIPS zero,fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +----------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * +----------------------------+ ++ * | Reserved for 64-bit eBPF | ++ * | args r3-r5 & args passed | ++ * | on stack in kernel calls | ++ * Lower address +============================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ const u8 *r1 = bpf2mips32[BPF_REG_1]; ++ const u8 *fp = bpf2mips32[BPF_REG_FP]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first two instructions initialize TCC in the reserved (for us) ++ * 16-byte area in the parent's stack frame. On a tail call, the ++ * calling function jumps into the prologue after these instructions. ++ */ ++ emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, ++ min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); ++ ++ /* ++ * Register eBPF R1 contains the 32-bit context pointer argument. ++ * A 32-bit argument is always passed in MIPS register a0, regardless ++ * of CPU endianness. Initialize R1 accordingly and zero-extend. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, lo(r1), MIPS_R_A0); ++#endif ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* Zero-extend the 32-bit argument */ ++ emit(ctx, move, hi(r1), MIPS_R_ZERO); ++ ++ /* If the eBPF frame pointer was accessed it must be saved */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg64(ctx, fp); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers and function arguments passed on the stack. ++ * The required space is computed automatically during resource ++ * usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved); ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ /* ++ * A 32-bit return value is always passed in MIPS register v0, ++ * but on big-endian targets the low part of R0 is mapped to v1. ++ */ ++#ifdef __BIG_ENDIAN ++ emit(ctx, move, MIPS_R_V0, MIPS_R_V1); ++#endif ++ ++ /* Jump to the return address and adjust the stack pointer */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ const u8 *dst = bpf2mips32[insn->dst_reg]; ++ const u8 *src = bpf2mips32[insn->src_reg]; ++ const u8 *tmp = bpf2mips32[JIT_REG_TMP]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_mov_i(ctx, hi(dst), 0); ++ } else { ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_alu_i(ctx, lo(dst), 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, lo(dst), val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_se_i64(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, lo(dst), lo(src)); ++ emit_mov_r(ctx, hi(dst), hi(src)); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_neg_i64(ctx, dst); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ if (imm) ++ emit_alu_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ if (imm) ++ emit_shift_i64(ctx, dst, imm, BPF_OP(code)); ++ break; ++ /* dst = dst * imm (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ emit_mul_i64(ctx, dst, imm); ++ break; ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ /* ++ * Sign-extend the immediate value into a temporary register, ++ * and then do the operation on this register. ++ */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); ++ break; ++ /* dst = dst * src (64-bit) */ ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ emit_mul_r64(ctx, dst, src); ++ break; ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_divmod_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i(ctx, lo(dst), imm); ++ emit_mov_i(ctx, hi(dst), insn[1].imm); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ switch (BPF_SIZE(code)) { ++ case BPF_DW: ++ /* Sign-extend immediate value into temporary reg */ ++ emit_mov_se_i64(ctx, tmp, imm); ++ break; ++ case BPF_W: ++ case BPF_H: ++ case BPF_B: ++ emit_mov_i(ctx, lo(tmp), imm); ++ break; ++ } ++ emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (cpu_has_llsc) ++ emit_atomic_r(ctx, lo(dst), lo(src), off, imm); ++ else /* Non-ll/sc fallback */ ++ emit_atomic_r32(ctx, lo(dst), lo(src), ++ off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* Atomics (64-bit) */ ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ emit_atomic_r64(ctx, lo(dst), src, off, imm); ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, lo(dst), imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T6, imm); ++ emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r64(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_i64(ctx, dst, imm, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.4/071-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch b/target/linux/generic/backport-5.4/071-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch new file mode 100644 index 0000000000..38b46c0b76 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch @@ -0,0 +1,1005 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:05 +0200 +Subject: [PATCH] mips: bpf: Add new eBPF JIT for 64-bit MIPS + +This is an implementation on of an eBPF JIT for 64-bit MIPS III-V and +MIPS64r1-r6. It uses the same framework introduced by the 32-bit JIT. + +Signed-off-by: Johan Almbladh +--- + create mode 100644 arch/mips/net/bpf_jit_comp64.c + +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -0,0 +1,991 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal ++ * Copyright (c) 2011 Mircea Gherzan ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS t0-t3 are not available in the n64 ABI */ ++#undef MIPS_R_T0 ++#undef MIPS_R_T1 ++#undef MIPS_R_T2 ++#undef MIPS_R_T3 ++ ++/* Stack is 16-byte aligned in n64 ABI */ ++#define MIPS_STACK_ALIGNMENT 16 ++ ++/* Extra 64-bit eBPF registers used by JIT */ ++#define JIT_REG_TC (MAX_BPF_JIT_REG + 0) ++#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1) ++ ++/* Number of prologue bytes to skip when doing a tail call */ ++#define JIT_TCALL_SKIP 4 ++ ++/* Callee-saved CPU registers that the JIT must preserve */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* Caller-saved CPU registers available for JIT use */ ++#define JIT_CALLER_REGS \ ++ (BIT(MIPS_R_A5) | \ ++ BIT(MIPS_R_A6) | \ ++ BIT(MIPS_R_A7)) ++/* ++ * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers. ++ * MIPS registers t4 - t7 may be used by the JIT as temporary registers. ++ * MIPS registers t8 - t9 are reserved for single-register common functions. ++ */ ++static const u8 bpf2mips64[] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = MIPS_R_V0, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = MIPS_R_A0, ++ [BPF_REG_2] = MIPS_R_A1, ++ [BPF_REG_3] = MIPS_R_A2, ++ [BPF_REG_4] = MIPS_R_A3, ++ [BPF_REG_5] = MIPS_R_A4, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = MIPS_R_S0, ++ [BPF_REG_7] = MIPS_R_S1, ++ [BPF_REG_8] = MIPS_R_S2, ++ [BPF_REG_9] = MIPS_R_S3, ++ /* Read-only frame pointer to access the eBPF stack */ ++ [BPF_REG_FP] = MIPS_R_FP, ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = MIPS_R_AT, ++ /* Tail call count register, caller-saved */ ++ [JIT_REG_TC] = MIPS_R_A5, ++ /* Constant for register zero-extension */ ++ [JIT_REG_ZX] = MIPS_R_V1, ++}; ++ ++/* ++ * MIPS 32-bit operations on 64-bit registers generate a sign-extended ++ * result. However, the eBPF ISA mandates zero-extension, so we rely on the ++ * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic ++ * operations, right shift and byte swap require properly sign-extended ++ * operands or the result is unpredictable. We emit explicit sign-extensions ++ * in those cases. ++ */ ++ ++/* Sign extension */ ++static void emit_sext(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, sll, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension */ ++static void emit_zext(struct jit_context *ctx, u8 dst) ++{ ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); ++ } else { ++ emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]); ++ access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */ ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, u8 dst) ++{ ++ if (!ctx->program->aux->verifier_zext) ++ emit_zext(ctx, dst); ++} ++ ++/* dst = imm (64-bit) */ ++static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) ++{ ++ if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) { ++ emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64); ++ } else if (imm64 >= 0xffffffff80000000ULL || ++ (imm64 < 0x80000000 && imm64 > 0xffff)) { ++ emit(ctx, lui, dst, (s16)(imm64 >> 16)); ++ emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff); ++ } else { ++ u8 acc = MIPS_R_ZERO; ++ int k; ++ ++ for (k = 0; k < 4; k++) { ++ u16 half = imm64 >> (48 - 16 * k); ++ ++ if (acc == dst) ++ emit(ctx, dsll, dst, dst, 16); ++ ++ if (half) { ++ emit(ctx, ori, dst, acc, half); ++ acc = dst; ++ } ++ } ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, dsubu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, dsll_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, dsrl_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsra_safe, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, daddiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, daddiu, dst, dst, -imm); ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_i(ctx, dst, imm, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, dsllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, dsrlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsrav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, daddu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, dsubu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmulu, dst, dst, src); ++ } else { ++ emit(ctx, dmultu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips64r6) { ++ emit(ctx, ddivu_r6, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmodu, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_r(ctx, dst, src, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Swap sub words in a register double word */ ++static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */ ++ emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */ ++ emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & mask */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dsbh, dst, dst); ++ emit(ctx, dshd, dst, dst); ++ } else { ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */ ++ emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */ ++ emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */ ++ ++ emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff); ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */ ++ ++ emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */ ++ emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */ ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ /* Swap bytes in a word */ ++ case 32: ++ case 16: ++ emit_sext(ctx, dst, dst); ++ emit_bswap_r(ctx, dst, width); ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) ++ emit_zext(ctx, dst); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit_zext(ctx, dst); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, andi, dst, dst, 0xffff); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, dst, off, src); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, dst, off, src); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lwu, dst, off, src); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ emit(ctx, ld, dst, off, src); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, src, off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, src, off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, src, off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sd, src, off, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, lld, t1, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, daddu, t2, t1, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, t2, t1, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, t2, t1, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, t2, t1, src); ++ break; ++ } ++ emit(ctx, scd, t2, off, dst); ++ emit(ctx, beqz, t2, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ u8 tmp = MIPS_R_T6; ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Emit function call */ ++ emit_mov_i64(ctx, tmp, addr); ++ emit(ctx, jalr, MIPS_R_RA, tmp); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Re-initialize the JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = bpf2mips64[BPF_REG_2]; ++ u8 ind = bpf2mips64[BPF_REG_3]; ++ u8 tcc = bpf2mips64[JIT_REG_TC]; ++ u8 tmp = MIPS_R_T6; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/ ++ emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ ++ /* if (--TCC < 0) goto out */ ++ emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */ ++ emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */ ++ /* (next insn delay slot) */ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */ ++ emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */ ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, tmp); ++ access_reg(ctx, JIT_REG_TC); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Previous stack frame : ++ * +===========================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +---------------------------+ <--- eBPF FP (MIPS fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +---------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * Lower address +===========================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ u8 fp = bpf2mips64[BPF_REG_FP]; ++ u8 tc = bpf2mips64[JIT_REG_TC]; ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first instruction initializes the tail call count register. ++ * On a tail call, the calling function jumps into the prologue ++ * after this instruction. ++ */ ++ emit(ctx, addiu, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* ++ * If the eBPF frame pointer and tail call count registers were ++ * accessed they must be preserved. Mark them as clobbered here ++ * to save and restore them on the stack as needed. ++ */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg(ctx, fp); ++ if (ctx->accessed & BIT(JIT_REG_TC)) ++ clobber_reg(ctx, tc); ++ if (ctx->accessed & BIT(JIT_REG_ZX)) ++ clobber_reg(ctx, zx); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers needed by the JIT. The required space is ++ * computed automatically during resource usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ if (stack) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved); ++ ++ /* Initialize the ePF JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ ++ /* Release the stack frame */ ++ if (ctx->stack_size) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++ ++ /* Jump to return address and sign-extend the 32-bit return value */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */ ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ u8 dst = bpf2mips64[insn->dst_reg]; ++ u8 src = bpf2mips64[insn->src_reg]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_zext(ctx, dst); ++ } else { ++ emit_mov_r(ctx, dst, src); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_sext(ctx, dst, dst); ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ emit_alu_r(ctx, dst, src, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_sext(ctx, dst, dst); ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, dst, src); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_alu_i64(ctx, dst, 0, BPF_NEG); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm ((64-bit, arithmetic) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ /* dst = dst * imm (64-bit) */ ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i64(ctx, dst, val, alu); ++ } ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ /* dst = dst * src (64-bit) */ ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32)); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (BPF_SIZE(code) == BPF_DW) { ++ emit_atomic_r64(ctx, dst, src, off, imm); ++ } else { /* 32-bit, no fetch */ ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm); ++ } ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */ ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register, sign-extended */ ++ emit_mov_i(ctx, MIPS_R_T5, imm); ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, dst, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} diff --git a/target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch b/target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch new file mode 100644 index 0000000000..63553ebe58 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-04-mips-bpf-Add-JIT-workarounds-for-CPU-errata.patch @@ -0,0 +1,120 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:06 +0200 +Subject: [PATCH] mips: bpf: Add JIT workarounds for CPU errata + +This patch adds workarounds for the following CPU errata to the MIPS +eBPF JIT, if enabled in the kernel configuration. + + - R10000 ll/sc weak ordering + - Loongson-3 ll/sc weak ordering + - Loongson-2F jump hang + +The Loongson-2F nop errata is implemented in uasm, which the JIT uses, +so no additional mitigations are needed for that. + +Signed-off-by: Johan Almbladh +Reviewed-by: Jiaxun Yang +--- + +--- a/arch/mips/net/bpf_jit_comp.c ++++ b/arch/mips/net/bpf_jit_comp.c +@@ -404,6 +404,7 @@ void emit_alu_r(struct jit_context *ctx, + /* Atomic read-modify-write (32-bit) */ + void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + switch (code) { + case BPF_ADD: +@@ -420,18 +421,19 @@ void emit_atomic_r(struct jit_context *c + break; + } + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -16); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + + /* Atomic compare-and-exchange (32-bit) */ + void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) + { ++ LLSC_sync(ctx); + emit(ctx, ll, MIPS_R_T9, off, dst); + emit(ctx, bne, MIPS_R_T9, res, 12); + emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ + emit(ctx, sc, MIPS_R_T8, off, dst); +- emit(ctx, beqz, MIPS_R_T8, -20); ++ emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset); + emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ + clobber_reg(ctx, res); + } +--- a/arch/mips/net/bpf_jit_comp.h ++++ b/arch/mips/net/bpf_jit_comp.h +@@ -87,7 +87,7 @@ struct jit_context { + }; + + /* Emit the instruction if the JIT memory space has been allocated */ +-#define emit(ctx, func, ...) \ ++#define __emit(ctx, func, ...) \ + do { \ + if ((ctx)->target != NULL) { \ + u32 *p = &(ctx)->target[ctx->jit_index]; \ +@@ -95,6 +95,30 @@ do { \ + } \ + (ctx)->jit_index++; \ + } while (0) ++#define emit(...) __emit(__VA_ARGS__) ++ ++/* Workaround for R10000 ll/sc errata */ ++#ifdef CONFIG_WAR_R10000 ++#define LLSC_beqz beqzl ++#else ++#define LLSC_beqz beqz ++#endif ++ ++/* Workaround for Loongson-3 ll/sc errata */ ++#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS ++#define LLSC_sync(ctx) emit(ctx, sync, 0) ++#define LLSC_offset 4 ++#else ++#define LLSC_sync(ctx) ++#define LLSC_offset 0 ++#endif ++ ++/* Workaround for Loongson-2F jump errata */ ++#ifdef CONFIG_CPU_JUMP_WORKAROUNDS ++#define JALR_MASK 0xffffffffcfffffffULL ++#else ++#define JALR_MASK (~0ULL) ++#endif + + /* + * Mark a BPF register as accessed, it needs to be +--- a/arch/mips/net/bpf_jit_comp64.c ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -375,6 +375,7 @@ static void emit_atomic_r64(struct jit_c + u8 t1 = MIPS_R_T6; + u8 t2 = MIPS_R_T7; + ++ LLSC_sync(ctx); + emit(ctx, lld, t1, off, dst); + switch (code) { + case BPF_ADD: +@@ -391,7 +392,7 @@ static void emit_atomic_r64(struct jit_c + break; + } + emit(ctx, scd, t2, off, dst); +- emit(ctx, beqz, t2, -16); ++ emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset); + emit(ctx, nop); /* Delay slot */ + } + +@@ -414,7 +415,7 @@ static int emit_call(struct jit_context + push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); + + /* Emit function call */ +- emit_mov_i64(ctx, tmp, addr); ++ emit_mov_i64(ctx, tmp, addr & JALR_MASK); + emit(ctx, jalr, MIPS_R_RA, tmp); + emit(ctx, nop); /* Delay slot */ + diff --git a/target/linux/generic/backport-5.4/071-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch b/target/linux/generic/backport-5.4/071-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch new file mode 100644 index 0000000000..c8ce0becf9 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-05-mips-bpf-Enable-eBPF-JITs.patch @@ -0,0 +1,54 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:07 +0200 +Subject: [PATCH] mips: bpf: Enable eBPF JITs + +This patch enables the new eBPF JITs for 32-bit and 64-bit MIPS. It also +disables the old cBPF JIT to so cBPF programs are converted to use the +new JIT. + +Workarounds for R4000 CPU errata are not implemented by the JIT, so the +JIT is disabled if any of those workarounds are configured. + +Signed-off-by: Johan Almbladh +--- + +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -3099,6 +3099,7 @@ S: Supported + F: arch/arm64/net/ + + BPF JIT for MIPS (32-BIT AND 64-BIT) ++M: Johan Almbladh + M: Paul Burton + L: netdev@vger.kernel.org + L: bpf@vger.kernel.org +--- a/arch/mips/Kconfig ++++ b/arch/mips/Kconfig +@@ -46,8 +46,10 @@ config MIPS + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES + select HAVE_ASM_MODVERSIONS +- select HAVE_CBPF_JIT if !64BIT && !CPU_MICROMIPS +- select HAVE_EBPF_JIT if 64BIT && !CPU_MICROMIPS && TARGET_ISA_REV >= 2 ++ select HAVE_EBPF_JIT if !CPU_MICROMIPS && \ ++ !CPU_DADDI_WORKAROUNDS && \ ++ !CPU_R4000_WORKAROUNDS && \ ++ !CPU_R4400_WORKAROUNDS + select HAVE_CONTEXT_TRACKING + select HAVE_COPY_THREAD_TLS + select HAVE_C_RECORDMCOUNT +--- a/arch/mips/net/Makefile ++++ b/arch/mips/net/Makefile +@@ -2,9 +2,10 @@ + # MIPS networking code + + obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o ++obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o + + ifeq ($(CONFIG_32BIT),y) +- obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o bpf_jit_comp32.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp32.o + else +- obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o ++ obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp64.o + endif diff --git a/target/linux/generic/backport-5.4/071-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch b/target/linux/generic/backport-5.4/071-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch new file mode 100644 index 0000000000..e25c336831 --- /dev/null +++ b/target/linux/generic/backport-5.4/071-v5.16-06-mips-bpf-Remove-old-BPF-JIT-implementations.patch @@ -0,0 +1,387 @@ +From: Johan Almbladh +Date: Tue, 5 Oct 2021 18:54:08 +0200 +Subject: [PATCH] mips: bpf: Remove old BPF JIT implementations + +This patch removes the old 32-bit cBPF and 64-bit eBPF JIT implementations. +They are replaced by a new eBPF implementation that supports both 32-bit +and 64-bit MIPS CPUs. + +Signed-off-by: Johan Almbladh +--- + delete mode 100644 arch/mips/net/bpf_jit.c + delete mode 100644 arch/mips/net/bpf_jit.h + delete mode 100644 arch/mips/net/bpf_jit_asm.S + delete mode 100644 arch/mips/net/ebpf_jit.c + +--- a/arch/mips/net/bpf_jit.h ++++ /dev/null +@@ -1,81 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0-only */ +-/* +- * Just-In-Time compiler for BPF filters on MIPS +- * +- * Copyright (c) 2014 Imagination Technologies Ltd. +- * Author: Markos Chandras +- */ +- +-#ifndef BPF_JIT_MIPS_OP_H +-#define BPF_JIT_MIPS_OP_H +- +-/* Registers used by JIT */ +-#define MIPS_R_ZERO 0 +-#define MIPS_R_V0 2 +-#define MIPS_R_A0 4 +-#define MIPS_R_A1 5 +-#define MIPS_R_T4 12 +-#define MIPS_R_T5 13 +-#define MIPS_R_T6 14 +-#define MIPS_R_T7 15 +-#define MIPS_R_S0 16 +-#define MIPS_R_S1 17 +-#define MIPS_R_S2 18 +-#define MIPS_R_S3 19 +-#define MIPS_R_S4 20 +-#define MIPS_R_S5 21 +-#define MIPS_R_S6 22 +-#define MIPS_R_S7 23 +-#define MIPS_R_SP 29 +-#define MIPS_R_RA 31 +- +-/* Conditional codes */ +-#define MIPS_COND_EQ 0x1 +-#define MIPS_COND_GE (0x1 << 1) +-#define MIPS_COND_GT (0x1 << 2) +-#define MIPS_COND_NE (0x1 << 3) +-#define MIPS_COND_ALL (0x1 << 4) +-/* Conditionals on X register or K immediate */ +-#define MIPS_COND_X (0x1 << 5) +-#define MIPS_COND_K (0x1 << 6) +- +-#define r_ret MIPS_R_V0 +- +-/* +- * Use 2 scratch registers to avoid pipeline interlocks. +- * There is no overhead during epilogue and prologue since +- * any of the $s0-$s6 registers will only be preserved if +- * they are going to actually be used. +- */ +-#define r_skb_hl MIPS_R_S0 /* skb header length */ +-#define r_skb_data MIPS_R_S1 /* skb actual data */ +-#define r_off MIPS_R_S2 +-#define r_A MIPS_R_S3 +-#define r_X MIPS_R_S4 +-#define r_skb MIPS_R_S5 +-#define r_M MIPS_R_S6 +-#define r_skb_len MIPS_R_S7 +-#define r_s0 MIPS_R_T4 /* scratch reg 1 */ +-#define r_s1 MIPS_R_T5 /* scratch reg 2 */ +-#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */ +-#define r_tmp MIPS_R_T7 /* No need to preserve this */ +-#define r_zero MIPS_R_ZERO +-#define r_sp MIPS_R_SP +-#define r_ra MIPS_R_RA +- +-#ifndef __ASSEMBLY__ +- +-/* Declare ASM helpers */ +- +-#define DECLARE_LOAD_FUNC(func) \ +- extern u8 func(unsigned long *skb, int offset); \ +- extern u8 func##_negative(unsigned long *skb, int offset); \ +- extern u8 func##_positive(unsigned long *skb, int offset) +- +-DECLARE_LOAD_FUNC(sk_load_word); +-DECLARE_LOAD_FUNC(sk_load_half); +-DECLARE_LOAD_FUNC(sk_load_byte); +- +-#endif +- +-#endif /* BPF_JIT_MIPS_OP_H */ +--- a/arch/mips/net/bpf_jit_asm.S ++++ /dev/null +@@ -1,285 +0,0 @@ +-/* +- * bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF +- * compiler. +- * +- * Copyright (C) 2015 Imagination Technologies Ltd. +- * Author: Markos Chandras +- * +- * This program is free software; you can redistribute it and/or modify it +- * under the terms of the GNU General Public License as published by the +- * Free Software Foundation; version 2 of the License. +- */ +- +-#include +-#include +-#include +-#include "bpf_jit.h" +- +-/* ABI +- * +- * r_skb_hl skb header length +- * r_skb_data skb data +- * r_off(a1) offset register +- * r_A BPF register A +- * r_X PF register X +- * r_skb(a0) *skb +- * r_M *scratch memory +- * r_skb_le skb length +- * r_s0 Scratch register 0 +- * r_s1 Scratch register 1 +- * +- * On entry: +- * a0: *skb +- * a1: offset (imm or imm + X) +- * +- * All non-BPF-ABI registers are free for use. On return, we only +- * care about r_ret. The BPF-ABI registers are assumed to remain +- * unmodified during the entire filter operation. +- */ +- +-#define skb a0 +-#define offset a1 +-#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */ +- +- /* We know better :) so prevent assembler reordering etc */ +- .set noreorder +- +-#define is_offset_negative(TYPE) \ +- /* If offset is negative we have more work to do */ \ +- slti t0, offset, 0; \ +- bgtz t0, bpf_slow_path_##TYPE##_neg; \ +- /* Be careful what follows in DS. */ +- +-#define is_offset_in_header(SIZE, TYPE) \ +- /* Reading from header? */ \ +- addiu $r_s0, $r_skb_hl, -SIZE; \ +- slt t0, $r_s0, offset; \ +- bgtz t0, bpf_slow_path_##TYPE; \ +- +-LEAF(sk_load_word) +- is_offset_negative(word) +-FEXPORT(sk_load_word_positive) +- is_offset_in_header(4, word) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- .set reorder +- lw $r_A, 0(t1) +- .set noreorder +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_A +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_A, 24 +- srl t1, $r_A, 24 +- srl t2, $r_A, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_A, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_word) +- +-LEAF(sk_load_half) +- is_offset_negative(half) +-FEXPORT(sk_load_half_positive) +- is_offset_in_header(2, half) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lhu $r_A, 0(t1) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh $r_A, $r_A +-# else +- sll t0, $r_A, 8 +- srl t1, $r_A, 8 +- andi t0, t0, 0xff00 +- or $r_A, t0, t1 +-# endif +-#endif +- jr $r_ra +- move $r_ret, zero +- END(sk_load_half) +- +-LEAF(sk_load_byte) +- is_offset_negative(byte) +-FEXPORT(sk_load_byte_positive) +- is_offset_in_header(1, byte) +- /* Offset within header boundaries */ +- PTR_ADDU t1, $r_skb_data, offset +- lbu $r_A, 0(t1) +- jr $r_ra +- move $r_ret, zero +- END(sk_load_byte) +- +-/* +- * call skb_copy_bits: +- * (prototype in linux/skbuff.h) +- * +- * int skb_copy_bits(sk_buff *skb, int offset, void *to, int len) +- * +- * o32 mandates we leave 4 spaces for argument registers in case +- * the callee needs to use them. Even though we don't care about +- * the argument registers ourselves, we need to allocate that space +- * to remain ABI compliant since the callee may want to use that space. +- * We also allocate 2 more spaces for $r_ra and our return register (*to). +- * +- * n64 is a bit different. The *caller* will allocate the space to preserve +- * the arguments. So in 64-bit kernels, we allocate the 4-arg space for no +- * good reason but it does not matter that much really. +- * +- * (void *to) is returned in r_s0 +- * +- */ +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-#define DS_OFFSET(SIZE) (4 * SZREG) +-#else +-#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE)) +-#endif +-#define bpf_slow_path_common(SIZE) \ +- /* Quick check. Are we within reasonable boundaries? */ \ +- LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \ +- sltu $r_s0, offset, $r_s1; \ +- beqz $r_s0, fault; \ +- /* Load 4th argument in DS */ \ +- LONG_ADDIU a3, zero, SIZE; \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, skb_copy_bits; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- /* Assign low slot to a2 */ \ +- PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \ +- jalr t0; \ +- /* Reset our destination slot (DS but it's ok) */ \ +- INT_S zero, (4 * SZREG)($r_sp); \ +- /* \ +- * skb_copy_bits returns 0 on success and -EFAULT \ +- * on error. Our data live in a2. Do not bother with \ +- * our data if an error has been returned. \ +- */ \ +- /* Restore our frame */ \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- INT_L $r_s0, (4 * SZREG)($r_sp); \ +- bltz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- move $r_ret, zero; \ +- +-NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp) +- bpf_slow_path_common(4) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- wsbh t0, $r_s0 +- jr $r_ra +- rotr $r_A, t0, 16 +-# else +- sll t0, $r_s0, 24 +- srl t1, $r_s0, 24 +- srl t2, $r_s0, 8 +- or t0, t0, t1 +- andi t2, t2, 0xff00 +- andi t1, $r_s0, 0xff00 +- or t0, t0, t2 +- sll t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_word) +- +-NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp) +- bpf_slow_path_common(2) +-#ifdef CONFIG_CPU_LITTLE_ENDIAN +-# if MIPS_ISA_REV >= 2 +- jr $r_ra +- wsbh $r_A, $r_s0 +-# else +- sll t0, $r_s0, 8 +- andi t1, $r_s0, 0xff00 +- andi t0, t0, 0xff00 +- srl t1, t1, 8 +- jr $r_ra +- or $r_A, t0, t1 +-# endif +-#else +- jr $r_ra +- move $r_A, $r_s0 +-#endif +- +- END(bpf_slow_path_half) +- +-NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp) +- bpf_slow_path_common(1) +- jr $r_ra +- move $r_A, $r_s0 +- +- END(bpf_slow_path_byte) +- +-/* +- * Negative entry points +- */ +- .macro bpf_is_end_of_data +- li t0, SKF_LL_OFF +- /* Reading link layer data? */ +- slt t1, offset, t0 +- bgtz t1, fault +- /* Be careful what follows in DS. */ +- .endm +-/* +- * call skb_copy_bits: +- * (prototype in linux/filter.h) +- * +- * void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, +- * int k, unsigned int size) +- * +- * see above (bpf_slow_path_common) for ABI restrictions +- */ +-#define bpf_negative_common(SIZE) \ +- PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \ +- PTR_LA t0, bpf_internal_load_pointer_neg_helper; \ +- PTR_S $r_ra, (5 * SZREG)($r_sp); \ +- jalr t0; \ +- li a2, SIZE; \ +- PTR_L $r_ra, (5 * SZREG)($r_sp); \ +- /* Check return pointer */ \ +- beqz v0, fault; \ +- PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \ +- /* Preserve our pointer */ \ +- move $r_s0, v0; \ +- /* Set return value */ \ +- move $r_ret, zero; \ +- +-bpf_slow_path_word_neg: +- bpf_is_end_of_data +-NESTED(sk_load_word_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(4) +- jr $r_ra +- lw $r_A, 0($r_s0) +- END(sk_load_word_negative) +- +-bpf_slow_path_half_neg: +- bpf_is_end_of_data +-NESTED(sk_load_half_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(2) +- jr $r_ra +- lhu $r_A, 0($r_s0) +- END(sk_load_half_negative) +- +-bpf_slow_path_byte_neg: +- bpf_is_end_of_data +-NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp) +- bpf_negative_common(1) +- jr $r_ra +- lbu $r_A, 0($r_s0) +- END(sk_load_byte_negative) +- +-fault: +- jr $r_ra +- addiu $r_ret, zero, 1 diff --git a/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch b/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch index 9e37bbb60c..a29da1e7b2 100644 --- a/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch +++ b/target/linux/generic/backport-5.4/080-wireguard-0072-net-WireGuard-secure-network-tunnel.patch @@ -192,7 +192,7 @@ Signed-off-by: Jason A. Donenfeld --- a/MAINTAINERS +++ b/MAINTAINERS -@@ -17584,6 +17584,14 @@ L: linux-gpio@vger.kernel.org +@@ -17585,6 +17585,14 @@ L: linux-gpio@vger.kernel.org S: Maintained F: drivers/gpio/gpio-ws16c48.c diff --git a/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch b/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch index ecf86b66a6..19b81f0301 100644 --- a/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch +++ b/target/linux/generic/hack-5.10/301-mips_image_cmdline_hack.patch @@ -10,7 +10,7 @@ Signed-off-by: Gabor Juhos --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1165,6 +1165,10 @@ config MIPS_MSC +@@ -1167,6 +1167,10 @@ config MIPS_MSC config SYNC_R4K bool diff --git a/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch b/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch index ada65cd2a0..af6b51aa4f 100644 --- a/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch +++ b/target/linux/generic/hack-5.4/301-mips_image_cmdline_hack.patch @@ -10,7 +10,7 @@ Signed-off-by: Gabor Juhos --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1159,6 +1159,10 @@ config SYNC_R4K +@@ -1161,6 +1161,10 @@ config SYNC_R4K config MIPS_MACHINE def_bool n diff --git a/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch b/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch index a04864ab4d..59f399d3cf 100644 --- a/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch +++ b/target/linux/generic/pending-5.10/300-mips_expose_boot_raw.patch @@ -9,7 +9,7 @@ Acked-by: Rob Landley --- --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1080,9 +1080,6 @@ config FW_ARC +@@ -1082,9 +1082,6 @@ config FW_ARC config ARCH_MAY_HAVE_PC_FDC bool @@ -19,7 +19,7 @@ Acked-by: Rob Landley config CEVT_BCM1480 bool -@@ -3172,6 +3169,18 @@ choice +@@ -3174,6 +3171,18 @@ choice bool "Extend builtin kernel arguments with bootloader arguments" endchoice diff --git a/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch b/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch index 4e7a532156..b5ec03f286 100644 --- a/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch +++ b/target/linux/generic/pending-5.4/103-MIPS-select-CPU_MIPS64-for-remaining-MIPS64-CPUs.patch @@ -25,7 +25,7 @@ Signed-off-by: Jason A. Donenfeld --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -2037,7 +2037,8 @@ config CPU_MIPS32 +@@ -2039,7 +2039,8 @@ config CPU_MIPS32 config CPU_MIPS64 bool diff --git a/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch b/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch index 7b9ae65c60..c08d6a61f3 100644 --- a/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch +++ b/target/linux/generic/pending-5.4/300-mips_expose_boot_raw.patch @@ -9,7 +9,7 @@ Acked-by: Rob Landley --- --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -1069,9 +1069,6 @@ config FW_ARC +@@ -1071,9 +1071,6 @@ config FW_ARC config ARCH_MAY_HAVE_PC_FDC bool @@ -19,7 +19,7 @@ Acked-by: Rob Landley config CEVT_BCM1480 bool -@@ -3044,6 +3041,18 @@ choice +@@ -3046,6 +3043,18 @@ choice bool "Extend builtin kernel arguments with bootloader arguments" endchoice diff --git a/target/linux/mediatek/patches-5.10/910-dts-mt7622-bpi-r64-wifi-eeprom.patch b/target/linux/mediatek/patches-5.10/910-dts-mt7622-bpi-r64-wifi-eeprom.patch new file mode 100644 index 0000000000..190096de2f --- /dev/null +++ b/target/linux/mediatek/patches-5.10/910-dts-mt7622-bpi-r64-wifi-eeprom.patch @@ -0,0 +1,31 @@ +--- a/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts ++++ b/arch/arm64/boot/dts/mediatek/mt7622-bananapi-bpi-r64.dts +@@ -636,5 +636,28 @@ + }; + + &wmac { ++ mediatek,eeprom-data = <0x22760500 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x44000020 0x0 0x10002000 ++ 0x4400 0x4000000 0x0 0x0 ++ 0x200000b3 0x40b6c3c3 0x26000000 0x41c42600 ++ 0x41c4 0x26000000 0xc0c52600 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0xc6c6 ++ 0xc3c3c2c1 0xc300c3 0x818181 0x83c1c182 ++ 0x83838382 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x84002e00 0x90000087 0x8a000000 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0xb000009 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x0 ++ 0x0 0x0 0x0 0x7707>; ++ + status = "okay"; + }; diff --git a/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts b/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts index 51c1d868cb..ce700e0c2b 100644 --- a/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts +++ b/target/linux/ramips/dts/mt7621_telco-electronics_x1.dts @@ -166,25 +166,26 @@ &pcie { status = "okay"; +}; - pcie0 { - wifi@0,0 { - compatible = "pci14c3,7603"; - reg = <0x0000 0 0 0 0>; - mediatek,mtd-eeprom = <&factory 0x0000>; - ieee80211-freq-limit = <2400000 2500000>; - }; +&pcie0 { + wifi@0,0 { + compatible = "pci14c3,7603"; + reg = <0x0000 0 0 0 0>; + mediatek,mtd-eeprom = <&factory 0x0000>; + ieee80211-freq-limit = <2400000 2500000>; }; +}; - pcie1 { - wifi@0,0 { - compatible = "pci14c3,7662"; - reg = <0x0000 0 0 0 0>; - mediatek,mtd-eeprom = <&factory 0x8000>; - ieee80211-freq-limit = <5000000 6000000>; - led { - led-sources = <2>; - }; +&pcie1 { + wifi@0,0 { + compatible = "pci14c3,7662"; + reg = <0x0000 0 0 0 0>; + mediatek,mtd-eeprom = <&factory 0x8000>; + ieee80211-freq-limit = <5000000 6000000>; + + led { + led-sources = <2>; }; }; }; diff --git a/target/linux/ramips/image/mt7621.mk b/target/linux/ramips/image/mt7621.mk index 1cea7d37f8..29e3123ec2 100644 --- a/target/linux/ramips/image/mt7621.mk +++ b/target/linux/ramips/image/mt7621.mk @@ -826,7 +826,7 @@ define Device/linksys_e5600 DEVICE_VENDOR := Linksys DEVICE_MODEL := E5600 DEVICE_PACKAGES := kmod-mt7603 kmod-mt7615e kmod-mt7663-firmware-ap \ - kmod-mt7663-firmware-sta uboot-envtools + uboot-envtools UBINIZE_OPTS := -E 5 IMAGES += factory.bin IMAGE/sysupgrade.bin := sysupgrade-tar | check-size | append-metadata @@ -1317,7 +1317,7 @@ define Device/tplink_archer-a6-v3 DEVICE_MODEL := Archer A6 DEVICE_VARIANT := V3 DEVICE_PACKAGES := kmod-mt7603 kmod-mt7615e \ - kmod-mt7663-firmware-ap kmod-mt7663-firmware-sta + kmod-mt7663-firmware-ap TPLINK_BOARD_ID := ARCHER-A6-V3 KERNEL := $(KERNEL_DTB) | uImage lzma IMAGE_SIZE := 15744k @@ -1330,7 +1330,7 @@ define Device/tplink_archer-c6-v3 DEVICE_MODEL := Archer C6 DEVICE_VARIANT := V3 DEVICE_PACKAGES := kmod-mt7603 kmod-mt7615e \ - kmod-mt7663-firmware-ap kmod-mt7663-firmware-sta + kmod-mt7663-firmware-ap TPLINK_BOARD_ID := ARCHER-C6-V3 KERNEL := $(KERNEL_DTB) | uImage lzma IMAGE_SIZE := 15744k diff --git a/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch b/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch index 34617898c8..4ffbb2bebb 100644 --- a/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch +++ b/target/linux/ramips/patches-5.4/0122-mips-ralink-enable-zboot-support.patch @@ -16,7 +16,7 @@ Signed-off-by: Thomas Bogendoerfer --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig -@@ -626,6 +626,7 @@ config RALINK +@@ -628,6 +628,7 @@ config RALINK select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_LITTLE_ENDIAN select SYS_SUPPORTS_MIPS16 diff --git a/target/linux/x86/64/config-5.10 b/target/linux/x86/64/config-5.10 index 4e979136a5..a62e5a795a 100644 --- a/target/linux/x86/64/config-5.10 +++ b/target/linux/x86/64/config-5.10 @@ -416,6 +416,7 @@ CONFIG_VIRTIO_BLK=y CONFIG_VIRTIO_CONSOLE=y CONFIG_VIRTIO_DMA_SHARED_BUFFER=y CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y CONFIG_VIRTIO_NET=y CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_PCI_LEGACY=y diff --git a/target/linux/zynq/Makefile b/target/linux/zynq/Makefile index b5988c9e05..83807913e4 100644 --- a/target/linux/zynq/Makefile +++ b/target/linux/zynq/Makefile @@ -17,7 +17,7 @@ define Target/Description Build firmware image for Zynq 7000 SoC devices. endef -KERNEL_PATCHVER:=5.4 +KERNEL_PATCHVER:=5.10 include $(INCLUDE_DIR)/target.mk diff --git a/target/linux/zynq/config-5.10 b/target/linux/zynq/config-5.10 new file mode 100644 index 0000000000..d18bb39d8d --- /dev/null +++ b/target/linux/zynq/config-5.10 @@ -0,0 +1,552 @@ +CONFIG_ALIGNMENT_TRAP=y +# CONFIG_ALTERA_FREEZE_BRIDGE is not set +# CONFIG_ALTERA_PR_IP_CORE is not set +CONFIG_ARCH_32BIT_OFF_T=y +CONFIG_ARCH_HIBERNATION_POSSIBLE=y +CONFIG_ARCH_KEEP_MEMBLOCK=y +CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y +CONFIG_ARCH_MULTIPLATFORM=y +CONFIG_ARCH_MULTI_V6_V7=y +CONFIG_ARCH_MULTI_V7=y +CONFIG_ARCH_NR_GPIO=1024 +CONFIG_ARCH_OPTIONAL_KERNEL_RWX=y +CONFIG_ARCH_OPTIONAL_KERNEL_RWX_DEFAULT=y +CONFIG_ARCH_SELECT_MEMORY_MODEL=y +CONFIG_ARCH_SPARSEMEM_ENABLE=y +CONFIG_ARCH_SUSPEND_POSSIBLE=y +CONFIG_ARCH_VEXPRESS=y +CONFIG_ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA=y +# CONFIG_ARCH_VEXPRESS_SPC is not set +CONFIG_ARCH_ZYNQ=y +CONFIG_ARM=y +CONFIG_ARM_AMBA=y +CONFIG_ARM_CPU_SUSPEND=y +CONFIG_ARM_ERRATA_643719=y +CONFIG_ARM_ERRATA_720789=y +CONFIG_ARM_ERRATA_754322=y +CONFIG_ARM_ERRATA_754327=y +CONFIG_ARM_ERRATA_764369=y +CONFIG_ARM_ERRATA_775420=y +CONFIG_ARM_GIC=y +CONFIG_ARM_GLOBAL_TIMER=y +CONFIG_ARM_HAS_SG_CHAIN=y +CONFIG_ARM_HEAVY_MB=y +CONFIG_ARM_L1_CACHE_SHIFT=6 +CONFIG_ARM_L1_CACHE_SHIFT_6=y +CONFIG_ARM_PATCH_IDIV=y +CONFIG_ARM_PATCH_PHYS_VIRT=y +# CONFIG_ARM_PL172_MPMC is not set +# CONFIG_ARM_SMMU is not set +CONFIG_ARM_THUMB=y +CONFIG_ARM_TIMER_SP804=y +CONFIG_ARM_UNWIND=y +CONFIG_ARM_VIRT_EXT=y +CONFIG_ARM_ZYNQ_CPUIDLE=y +CONFIG_ATAGS=y +CONFIG_AUTO_ZRELADDR=y +# CONFIG_AXI_DMAC is not set +CONFIG_BINFMT_FLAT_ARGVP_ENVP_ON_STACK=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_COUNT=16 +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_BLK_MQ_PCI=y +CONFIG_BLK_PM=y +CONFIG_BOUNCE=y +CONFIG_CACHE_L2X0=y +CONFIG_CADENCE_TTC_TIMER=y +CONFIG_CADENCE_WATCHDOG=y +CONFIG_CLKDEV_LOOKUP=y +CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK=y +CONFIG_CLKSRC_MMIO=y +CONFIG_CLKSRC_VERSATILE=y +CONFIG_CLK_SP810=y +CONFIG_CLK_VEXPRESS_OSC=y +CONFIG_CLONE_BACKWARDS=y +CONFIG_CMA=y +CONFIG_CMA_ALIGNMENT=8 +CONFIG_CMA_AREAS=7 +# CONFIG_CMA_DEBUG is not set +# CONFIG_CMA_DEBUGFS is not set +CONFIG_CMA_SIZE_MBYTES=16 +# CONFIG_CMA_SIZE_SEL_MAX is not set +CONFIG_CMA_SIZE_SEL_MBYTES=y +# CONFIG_CMA_SIZE_SEL_MIN is not set +# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set +CONFIG_COMMON_CLK=y +# CONFIG_COMMON_CLK_AXI_CLKGEN is not set +CONFIG_COMMON_CLK_SI570=y +CONFIG_COMPAT_32BIT_TIME=y +CONFIG_CONNECTOR=y +CONFIG_CONSOLE_TRANSLATIONS=y +CONFIG_CONTIG_ALLOC=y +CONFIG_COREDUMP=y +# CONFIG_CPUFREQ_DT is not set +CONFIG_CPU_32v6K=y +CONFIG_CPU_32v7=y +CONFIG_CPU_ABRT_EV7=y +CONFIG_CPU_CACHE_V7=y +CONFIG_CPU_CACHE_VIPT=y +CONFIG_CPU_COPY_V6=y +CONFIG_CPU_CP15=y +CONFIG_CPU_CP15_MMU=y +CONFIG_CPU_FREQ=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE is not set +CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ATTR_SET=y +CONFIG_CPU_FREQ_GOV_COMMON=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_HAS_ASID=y +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_CPU_IDLE_GOV_MENU=y +CONFIG_CPU_PABRT_V7=y +CONFIG_CPU_PM=y +CONFIG_CPU_RMAP=y +CONFIG_CPU_SPECTRE=y +CONFIG_CPU_THERMAL=y +CONFIG_CPU_THUMB_CAPABLE=y +CONFIG_CPU_TLB_V7=y +CONFIG_CPU_V7=y +CONFIG_CRC16=y +# CONFIG_CRC32_SARWATE is not set +CONFIG_CRC32_SLICEBY8=y +CONFIG_CROSS_MEMORY_ATTACH=y +CONFIG_CRYPTO_CRC32=y +CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_GF128MUL=y +CONFIG_CRYPTO_HW=y +CONFIG_CRYPTO_NULL2=y +CONFIG_CRYPTO_RNG2=y +CONFIG_DCACHE_WORD_ACCESS=y +CONFIG_DEBUG_LL_INCLUDE="mach/debug-macro.S" +CONFIG_DMADEVICES=y +CONFIG_DMA_CMA=y +CONFIG_DMA_ENGINE=y +CONFIG_DMA_OF=y +CONFIG_DMA_OPS=y +CONFIG_DMA_REMAP=y +CONFIG_DMA_SHARED_BUFFER=y +CONFIG_DRM=y +CONFIG_DRM_BRIDGE=y +CONFIG_DRM_FBDEV_EMULATION=y +CONFIG_DRM_FBDEV_OVERALLOC=100 +CONFIG_DRM_KMS_FB_HELPER=y +CONFIG_DRM_KMS_HELPER=y +CONFIG_DRM_PANEL=y +CONFIG_DRM_PANEL_BRIDGE=y +CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y +CONFIG_DTC=y +CONFIG_DUMMY_CONSOLE=y +CONFIG_E1000E=y +CONFIG_EDAC=y +CONFIG_EDAC_ATOMIC_SCRUB=y +# CONFIG_EDAC_DEBUG is not set +CONFIG_EDAC_LEGACY_SYSFS=y +CONFIG_EDAC_SUPPORT=y +# CONFIG_EDAC_SYNOPSYS is not set +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_AT25=y +CONFIG_ELF_CORE=y +CONFIG_ENABLE_MUST_CHECK=y +CONFIG_EXT4_FS=y +CONFIG_EXTCON=y +CONFIG_F2FS_FS=y +CONFIG_FB=y +CONFIG_FB_CFB_COPYAREA=y +CONFIG_FB_CFB_FILLRECT=y +CONFIG_FB_CFB_IMAGEBLIT=y +CONFIG_FB_CMDLINE=y +CONFIG_FB_DEFERRED_IO=y +CONFIG_FB_SYS_COPYAREA=y +CONFIG_FB_SYS_FILLRECT=y +CONFIG_FB_SYS_FOPS=y +CONFIG_FB_SYS_IMAGEBLIT=y +# CONFIG_FB_XILINX is not set +CONFIG_FHANDLE=y +CONFIG_FIXED_PHY=y +CONFIG_FIX_EARLYCON_MEM=y +CONFIG_FPGA=y +CONFIG_FPGA_BRIDGE=y +# CONFIG_FPGA_DFL is not set +# CONFIG_FPGA_MGR_ALTERA_CVP is not set +# CONFIG_FPGA_MGR_ALTERA_PS_SPI is not set +# CONFIG_FPGA_MGR_ICE40_SPI is not set +# CONFIG_FPGA_MGR_MACHXO2_SPI is not set +# CONFIG_FPGA_MGR_XILINX_SPI is not set +CONFIG_FPGA_MGR_ZYNQ_FPGA=y +CONFIG_FPGA_REGION=y +CONFIG_FREEZER=y +CONFIG_FS_IOMAP=y +CONFIG_FS_MBCACHE=y +CONFIG_FW_CACHE=y +CONFIG_FW_LOADER_PAGED_BUF=y +CONFIG_GENERIC_ALLOCATOR=y +CONFIG_GENERIC_ARCH_TOPOLOGY=y +CONFIG_GENERIC_BUG=y +CONFIG_GENERIC_CLOCKEVENTS=y +CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y +CONFIG_GENERIC_CPU_AUTOPROBE=y +CONFIG_GENERIC_EARLY_IOREMAP=y +CONFIG_GENERIC_GETTIMEOFDAY=y +CONFIG_GENERIC_IDLE_POLL_SETUP=y +CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK=y +CONFIG_GENERIC_IRQ_MIGRATION=y +CONFIG_GENERIC_IRQ_MULTI_HANDLER=y +CONFIG_GENERIC_IRQ_SHOW=y +CONFIG_GENERIC_IRQ_SHOW_LEVEL=y +CONFIG_GENERIC_MSI_IRQ=y +CONFIG_GENERIC_MSI_IRQ_DOMAIN=y +CONFIG_GENERIC_PCI_IOMAP=y +CONFIG_GENERIC_PINCONF=y +CONFIG_GENERIC_SCHED_CLOCK=y +CONFIG_GENERIC_SMP_IDLE_THREAD=y +CONFIG_GENERIC_STRNCPY_FROM_USER=y +CONFIG_GENERIC_STRNLEN_USER=y +CONFIG_GENERIC_TIME_VSYSCALL=y +CONFIG_GENERIC_VDSO_32=y +CONFIG_GLOB=y +CONFIG_GPIOLIB=y +CONFIG_GPIOLIB_IRQCHIP=y +CONFIG_GPIO_GENERIC=y +CONFIG_GPIO_GENERIC_PLATFORM=y +CONFIG_GPIO_ZYNQ=y +CONFIG_HANDLE_DOMAIN_IRQ=y +CONFIG_HARDEN_BRANCH_PREDICTOR=y +CONFIG_HARDIRQS_SW_RESEND=y +CONFIG_HAS_DMA=y +CONFIG_HAS_IOMEM=y +CONFIG_HAVE_SMP=y +CONFIG_HDMI=y +CONFIG_HID=y +CONFIG_HID_GENERIC=y +CONFIG_HID_MICROSOFT=y +CONFIG_HIGHMEM=y +# CONFIG_HIGHPTE is not set +CONFIG_HOTPLUG_CPU=y +CONFIG_HWMON=y +CONFIG_HW_CONSOLE=y +CONFIG_HZ_FIXED=0 +CONFIG_I2C=y +CONFIG_I2C_ALGOBIT=y +CONFIG_I2C_BOARDINFO=y +CONFIG_I2C_CADENCE=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_COMPAT=y +CONFIG_I2C_HELPER_AUTO=y +CONFIG_I2C_MUX=y +CONFIG_I2C_MUX_PCA954x=y +CONFIG_ICST=y +CONFIG_IIO=y +CONFIG_IIO_BUFFER=y +CONFIG_IIO_KFIFO_BUF=y +CONFIG_IIO_TRIGGER=y +CONFIG_IIO_TRIGGERED_BUFFER=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_INITRAMFS_SOURCE="" +CONFIG_INPUT=y +CONFIG_INPUT_EVDEV=y +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_KEYBOARD=y +# CONFIG_INPUT_MISC is not set +CONFIG_INPUT_MOUSE=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_MOUSEDEV_PSAUX=y +CONFIG_INPUT_MOUSEDEV_SCREEN_X=1024 +CONFIG_INPUT_MOUSEDEV_SCREEN_Y=768 +CONFIG_INPUT_POLLDEV=y +CONFIG_INPUT_SPARSEKMAP=y +# CONFIG_IOMMU_DEBUGFS is not set +# CONFIG_IOMMU_IO_PGTABLE_ARMV7S is not set +# CONFIG_IOMMU_IO_PGTABLE_LPAE is not set +CONFIG_IOMMU_SUPPORT=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IRQCHIP=y +CONFIG_IRQ_DOMAIN=y +CONFIG_IRQ_DOMAIN_HIERARCHY=y +CONFIG_IRQ_FORCED_THREADING=y +CONFIG_IRQ_WORK=y +# CONFIG_ISDN is not set +CONFIG_JBD2=y +# CONFIG_JFFS2_COMPRESSION_OPTIONS is not set +CONFIG_JFFS2_ZLIB=y +CONFIG_KCMP=y +CONFIG_KERNEL_GZIP=y +# CONFIG_KERNEL_XZ is not set +CONFIG_KEYBOARD_ATKBD=y +CONFIG_KEYBOARD_GPIO=y +CONFIG_KEYBOARD_GPIO_POLLED=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +CONFIG_LEDS_TRIGGER_CAMERA=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_GPIO=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_TRANSIENT=y +CONFIG_LIBFDT=y +CONFIG_LLD_VERSION=0 +CONFIG_LOCK_DEBUGGING_SUPPORT=y +CONFIG_LOCK_SPIN_ON_OWNER=y +CONFIG_MACB=y +# CONFIG_MACB_PCI is not set +CONFIG_MACB_USE_HWSTAMP=y +CONFIG_MARVELL_PHY=y +CONFIG_MDIO_BITBANG=y +CONFIG_MDIO_BUS=y +CONFIG_MDIO_DEVICE=y +CONFIG_MDIO_DEVRES=y +# CONFIG_MDIO_GPIO is not set +CONFIG_MEMFD_CREATE=y +CONFIG_MEMORY=y +CONFIG_MEMORY_ISOLATION=y +CONFIG_MFD_CORE=y +CONFIG_MFD_SYSCON=y +CONFIG_MFD_VEXPRESS_SYSREG=y +CONFIG_MIGHT_HAVE_CACHE_L2X0=y +CONFIG_MIGRATION=y +CONFIG_MMC=y +CONFIG_MMC_BLOCK=y +CONFIG_MMC_CQHCI=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_OF_ARASAN=y +# CONFIG_MMC_SDHCI_PCI is not set +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MODULES_USE_ELF_REL=y +CONFIG_MODULE_FORCE_UNLOAD=y +# CONFIG_MODULE_STRIPPED is not set +# CONFIG_MOUSE_BCM5974 is not set +# CONFIG_MOUSE_CYAPA is not set +CONFIG_MOUSE_PS2=y +CONFIG_MOUSE_PS2_ALPS=y +CONFIG_MOUSE_PS2_BYD=y +CONFIG_MOUSE_PS2_CYPRESS=y +# CONFIG_MOUSE_PS2_ELANTECH is not set +CONFIG_MOUSE_PS2_FOCALTECH=y +CONFIG_MOUSE_PS2_LOGIPS2PP=y +CONFIG_MOUSE_PS2_SMBUS=y +CONFIG_MOUSE_PS2_SYNAPTICS=y +CONFIG_MOUSE_PS2_SYNAPTICS_SMBUS=y +# CONFIG_MOUSE_PS2_TOUCHKIT is not set +CONFIG_MOUSE_PS2_TRACKPOINT=y +# CONFIG_MOUSE_SERIAL is not set +# CONFIG_MOUSE_VSXXXAA is not set +# CONFIG_MTD_CFI_INTELEXT is not set +CONFIG_MTD_CMDLINE_PARTS=y +# CONFIG_MTD_COMPLEX_MAPPINGS is not set +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_SPI_NOR=y +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y +CONFIG_MTD_SPLIT_FIRMWARE=y +# CONFIG_MTD_SPLIT_SQUASHFS_ROOT is not set +CONFIG_MUTEX_SPIN_ON_OWNER=y +CONFIG_NEED_DMA_MAP_STATE=y +CONFIG_NEON=y +CONFIG_NET_FLOW_LIMIT=y +CONFIG_NET_PTP_CLASSIFY=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_FARADAY is not set +# CONFIG_NET_VENDOR_MARVELL is not set +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_MICROCHIP is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_VIA is not set +CONFIG_NLS=y +CONFIG_NLS_ASCII=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NOP_USB_XCEIV=y +CONFIG_NO_HZ=y +CONFIG_NO_HZ_COMMON=y +CONFIG_NO_HZ_IDLE=y +CONFIG_NO_IOPORT_MAP=y +CONFIG_NR_CPUS=4 +CONFIG_NVMEM=y +CONFIG_NVMEM_SYSFS=y +CONFIG_OF=y +CONFIG_OF_ADDRESS=y +CONFIG_OF_EARLY_FLATTREE=y +CONFIG_OF_FLATTREE=y +# CONFIG_OF_FPGA_REGION is not set +CONFIG_OF_GPIO=y +CONFIG_OF_IRQ=y +CONFIG_OF_KOBJ=y +CONFIG_OF_MDIO=y +CONFIG_OF_NET=y +CONFIG_OLD_SIGACTION=y +CONFIG_OLD_SIGSUSPEND3=y +CONFIG_OUTER_CACHE=y +CONFIG_OUTER_CACHE_SYNC=y +CONFIG_PADATA=y +CONFIG_PAGE_OFFSET=0xC0000000 +# CONFIG_PARTITION_ADVANCED is not set +CONFIG_PCI=y +CONFIG_PCIE_XILINX=y +CONFIG_PCI_DOMAINS=y +CONFIG_PCI_DOMAINS_GENERIC=y +CONFIG_PCI_MSI=y +CONFIG_PCI_MSI_ARCH_FALLBACKS=y +CONFIG_PCI_MSI_IRQ_DOMAIN=y +CONFIG_PERF_USE_VMALLOC=y +CONFIG_PGTABLE_LEVELS=2 +CONFIG_PHYLIB=y +CONFIG_PHYLINK=y +CONFIG_PINCTRL=y +# CONFIG_PINCTRL_SINGLE is not set +CONFIG_PINCTRL_ZYNQ=y +CONFIG_PL310_ERRATA_588369=y +CONFIG_PL310_ERRATA_727915=y +CONFIG_PL310_ERRATA_753970=y +CONFIG_PL310_ERRATA_769419=y +CONFIG_PL330_DMA=y +# CONFIG_PL353_SMC is not set +CONFIG_PLAT_VERSATILE=y +CONFIG_PM=y +CONFIG_PMBUS=y +CONFIG_PM_CLK=y +CONFIG_PM_SLEEP=y +CONFIG_PM_SLEEP_SMP=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_VEXPRESS=y +CONFIG_POWER_SUPPLY=y +CONFIG_PPS=y +CONFIG_PROC_EVENTS=y +CONFIG_PTP_1588_CLOCK=y +CONFIG_R8169=y +CONFIG_RAS=y +CONFIG_RATIONAL=y +CONFIG_REALTEK_PHY=y +CONFIG_REGMAP=y +CONFIG_REGMAP_I2C=y +CONFIG_REGMAP_MMIO=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +# CONFIG_REGULATOR_VEXPRESS is not set +CONFIG_RESET_CONTROLLER=y +CONFIG_RESET_ZYNQ=y +CONFIG_RFS_ACCEL=y +CONFIG_RPS=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_PCF8563=y +CONFIG_RTC_I2C_AND_SPI=y +CONFIG_RTC_MC146818_LIB=y +CONFIG_RWSEM_SPIN_ON_OWNER=y +CONFIG_SCHED_MC=y +CONFIG_SCHED_SMT=y +CONFIG_SENSORS_PMBUS=y +CONFIG_SENSORS_UCD9000=y +CONFIG_SENSORS_UCD9200=y +# CONFIG_SERIAL_8250 is not set +CONFIG_SERIAL_XILINX_PS_UART=y +CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y +CONFIG_SERIO=y +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIO_SERPORT=y +CONFIG_SMP=y +CONFIG_SMP_ON_UP=y +CONFIG_SOC_BUS=y +CONFIG_SPARSE_IRQ=y +CONFIG_SPI=y +CONFIG_SPI_BITBANG=y +CONFIG_SPI_CADENCE=y +CONFIG_SPI_MASTER=y +CONFIG_SPI_MEM=y +CONFIG_SPI_XILINX=y +CONFIG_SPI_ZYNQ_QSPI=y +CONFIG_SRAM=y +CONFIG_SRAM_EXEC=y +CONFIG_SRCU=y +# CONFIG_STRIP_ASM_SYMS is not set +CONFIG_SUSPEND=y +CONFIG_SUSPEND_FREEZER=y +CONFIG_SWPHY=y +CONFIG_SWP_EMULATE=y +CONFIG_SYNC_FILE=y +CONFIG_SYSFS_SYSCALL=y +CONFIG_SYS_SUPPORTS_APM_EMULATION=y +# CONFIG_TEXTSEARCH is not set +CONFIG_THERMAL=y +CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y +CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0 +CONFIG_THERMAL_GOV_STEP_WISE=y +CONFIG_THERMAL_HWMON=y +CONFIG_THERMAL_OF=y +CONFIG_TICK_CPU_ACCOUNTING=y +CONFIG_TIMER_OF=y +CONFIG_TIMER_PROBE=y +CONFIG_TREE_RCU=y +CONFIG_TREE_SRCU=y +CONFIG_UIO=y +# CONFIG_UIO_AEC is not set +# CONFIG_UIO_CIF is not set +# CONFIG_UIO_DMEM_GENIRQ is not set +# CONFIG_UIO_MF624 is not set +# CONFIG_UIO_NETX is not set +# CONFIG_UIO_PCI_GENERIC is not set +CONFIG_UIO_PDRV_GENIRQ=y +# CONFIG_UIO_PRUSS is not set +# CONFIG_UIO_SERCOS3 is not set +CONFIG_UNCOMPRESS_INCLUDE="debug/uncompress.h" +CONFIG_UNWINDER_ARM=y +CONFIG_USB=y +CONFIG_USB_CHIPIDEA=y +CONFIG_USB_CHIPIDEA_HOST=y +CONFIG_USB_CHIPIDEA_UDC=y +CONFIG_USB_COMMON=y +CONFIG_USB_EHCI_HCD=y +# CONFIG_USB_EHCI_HCD_PLATFORM is not set +# CONFIG_USB_EHCI_TT_NEWSCHED is not set +# CONFIG_USB_ETH is not set +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_XILINX=y +CONFIG_USB_HID=y +CONFIG_USB_NET_DRIVERS=y +CONFIG_USB_OTG=y +CONFIG_USB_OTG_FSM=y +CONFIG_USB_PHY=y +CONFIG_USB_ROLE_SWITCH=y +CONFIG_USB_SUPPORT=y +CONFIG_USB_ULPI=y +CONFIG_USB_ULPI_BUS=y +CONFIG_USB_ULPI_VIEWPORT=y +CONFIG_USE_OF=y +CONFIG_VEXPRESS_CONFIG=y +CONFIG_VFP=y +CONFIG_VFPv3=y +CONFIG_VGA_ARB=y +CONFIG_VGA_ARB_MAX_GPUS=16 +CONFIG_VITESSE_PHY=y +CONFIG_VM_EVENT_COUNTERS=y +CONFIG_VT=y +CONFIG_VT_CONSOLE=y +CONFIG_VT_CONSOLE_SLEEP=y +# CONFIG_VT_HW_CONSOLE_BINDING is not set +CONFIG_WATCHDOG_CORE=y +# CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set +CONFIG_XILINX_EMACLITE=y +# CONFIG_XILINX_PR_DECOUPLER is not set +CONFIG_XILINX_WATCHDOG=y +CONFIG_XILINX_XADC=y +CONFIG_XPS=y +CONFIG_XZ_DEC_ARM=y +CONFIG_XZ_DEC_ARMTHUMB=y +CONFIG_XZ_DEC_BCJ=y +CONFIG_XZ_DEC_IA64=y +CONFIG_XZ_DEC_POWERPC=y +CONFIG_XZ_DEC_SPARC=y +CONFIG_XZ_DEC_X86=y +CONFIG_ZBOOT_ROM_BSS=0x0 +CONFIG_ZBOOT_ROM_TEXT=0x0 +CONFIG_ZLIB_DEFLATE=y +CONFIG_ZLIB_INFLATE=y diff --git a/target/linux/zynq/config-5.4 b/target/linux/zynq/config-5.4 index 498d057e3a..38538789d8 100644 --- a/target/linux/zynq/config-5.4 +++ b/target/linux/zynq/config-5.4 @@ -3,23 +3,6 @@ CONFIG_ALIGNMENT_TRAP=y # CONFIG_ALTERA_PR_IP_CORE is not set CONFIG_ARCH_32BIT_OFF_T=y CONFIG_ARCH_CLOCKSOURCE_DATA=y -CONFIG_ARCH_HAS_BINFMT_FLAT=y -CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y -CONFIG_ARCH_HAS_ELF_RANDOMIZE=y -CONFIG_ARCH_HAS_FORTIFY_SOURCE=y -CONFIG_ARCH_HAS_GCOV_PROFILE_ALL=y -CONFIG_ARCH_HAS_KCOV=y -CONFIG_ARCH_HAS_KEEPINITRD=y -CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y -CONFIG_ARCH_HAS_PHYS_TO_DMA=y -CONFIG_ARCH_HAS_RESET_CONTROLLER=y -CONFIG_ARCH_HAS_SETUP_DMA_OPS=y -CONFIG_ARCH_HAS_SET_MEMORY=y -CONFIG_ARCH_HAS_STRICT_KERNEL_RWX=y -CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y -CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS=y -CONFIG_ARCH_HAS_TICK_BROADCAST=y -CONFIG_ARCH_HAVE_CUSTOM_GPIO_H=y CONFIG_ARCH_HIBERNATION_POSSIBLE=y CONFIG_ARCH_KEEP_MEMBLOCK=y CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y @@ -29,25 +12,13 @@ CONFIG_ARCH_MULTI_V7=y CONFIG_ARCH_NR_GPIO=1024 CONFIG_ARCH_OPTIONAL_KERNEL_RWX=y CONFIG_ARCH_OPTIONAL_KERNEL_RWX_DEFAULT=y -CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=y -CONFIG_ARCH_SUPPORTS_BIG_ENDIAN=y -CONFIG_ARCH_SUPPORTS_UPROBES=y CONFIG_ARCH_SUSPEND_POSSIBLE=y -CONFIG_ARCH_USE_BUILTIN_BSWAP=y -CONFIG_ARCH_USE_CMPXCHG_LOCKREF=y CONFIG_ARCH_VEXPRESS=y CONFIG_ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA=y # CONFIG_ARCH_VEXPRESS_SPC is not set -CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT=y -CONFIG_ARCH_WANT_GENERAL_HUGETLB=y -CONFIG_ARCH_WANT_IPC_PARSE_VERSION=y CONFIG_ARCH_ZYNQ=y CONFIG_ARM=y CONFIG_ARM_AMBA=y -CONFIG_ARM_CCI=y -CONFIG_ARM_CCI400_COMMON=y -CONFIG_ARM_CCI400_PMU=y -CONFIG_ARM_CCI_PMU=y CONFIG_ARM_CPU_SUSPEND=y CONFIG_ARM_ERRATA_643719=y CONFIG_ARM_ERRATA_720789=y @@ -61,15 +32,11 @@ CONFIG_ARM_HAS_SG_CHAIN=y CONFIG_ARM_HEAVY_MB=y CONFIG_ARM_L1_CACHE_SHIFT=6 CONFIG_ARM_L1_CACHE_SHIFT_6=y -# CONFIG_ARM_LPAE is not set CONFIG_ARM_PATCH_IDIV=y CONFIG_ARM_PATCH_PHYS_VIRT=y # CONFIG_ARM_PL172_MPMC is not set -CONFIG_ARM_PMU=y # CONFIG_ARM_SMMU is not set -# CONFIG_ARM_SP805_WATCHDOG is not set CONFIG_ARM_THUMB=y -# CONFIG_ARM_THUMBEE is not set CONFIG_ARM_TIMER_SP804=y CONFIG_ARM_UNWIND=y CONFIG_ARM_VIRT_EXT=y @@ -88,7 +55,6 @@ CONFIG_BOUNCE=y CONFIG_CACHE_L2X0=y CONFIG_CADENCE_TTC_TIMER=y CONFIG_CADENCE_WATCHDOG=y -CONFIG_CC_HAS_KASAN_GENERIC=y CONFIG_CLKDEV_LOOKUP=y CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK=y CONFIG_CLKSRC_MMIO=y @@ -119,8 +85,6 @@ CONFIG_COREDUMP=y CONFIG_CPU_32v6K=y CONFIG_CPU_32v7=y CONFIG_CPU_ABRT_EV7=y -# CONFIG_CPU_BIG_ENDIAN is not set -# CONFIG_CPU_BPREDICT_DISABLE is not set CONFIG_CPU_CACHE_V7=y CONFIG_CPU_CACHE_VIPT=y CONFIG_CPU_COPY_V6=y @@ -138,13 +102,9 @@ CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_CPU_FREQ_STAT=y CONFIG_CPU_HAS_ASID=y -# CONFIG_CPU_HOTPLUG_STATE_CONTROL is not set -# CONFIG_CPU_ICACHE_DISABLE is not set -# CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND is not set CONFIG_CPU_IDLE=y CONFIG_CPU_IDLE_GOV_LADDER=y CONFIG_CPU_IDLE_GOV_MENU=y -# CONFIG_CPU_IDLE_GOV_TEO is not set CONFIG_CPU_PABRT_V7=y CONFIG_CPU_PM=y CONFIG_CPU_RMAP=y @@ -170,7 +130,6 @@ CONFIG_CRYPTO_NULL2=y CONFIG_CRYPTO_RNG2=y CONFIG_DCACHE_WORD_ACCESS=y CONFIG_DEBUG_LL_INCLUDE="mach/debug-macro.S" -# CONFIG_DEBUG_USER is not set CONFIG_DMADEVICES=y CONFIG_DMA_CMA=y CONFIG_DMA_ENGINE=y @@ -183,7 +142,6 @@ CONFIG_DRM_FBDEV_EMULATION=y CONFIG_DRM_FBDEV_OVERALLOC=100 CONFIG_DRM_KMS_FB_HELPER=y CONFIG_DRM_KMS_HELPER=y -# CONFIG_DRM_MCDE is not set CONFIG_DRM_PANEL=y CONFIG_DRM_PANEL_BRIDGE=y CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=y @@ -231,8 +189,6 @@ CONFIG_FREEZER=y CONFIG_FS_IOMAP=y CONFIG_FS_MBCACHE=y CONFIG_FW_LOADER_PAGED_BUF=y -# CONFIG_FXOS8700_I2C is not set -# CONFIG_FXOS8700_SPI is not set CONFIG_GENERIC_ALLOCATOR=y CONFIG_GENERIC_ARCH_TOPOLOGY=y CONFIG_GENERIC_BUG=y @@ -254,6 +210,7 @@ CONFIG_GENERIC_SCHED_CLOCK=y CONFIG_GENERIC_SMP_IDLE_THREAD=y CONFIG_GENERIC_STRNCPY_FROM_USER=y CONFIG_GENERIC_STRNLEN_USER=y +CONFIG_GENERIC_TIME_VSYSCALL=y CONFIG_GLOB=y CONFIG_GPIOLIB=y CONFIG_GPIOLIB_IRQCHIP=y @@ -265,51 +222,7 @@ CONFIG_HARDEN_BRANCH_PREDICTOR=y CONFIG_HARDIRQS_SW_RESEND=y CONFIG_HAS_DMA=y CONFIG_HAS_IOMEM=y -CONFIG_HAVE_ARCH_AUDITSYSCALL=y -CONFIG_HAVE_ARCH_BITREVERSE=y -CONFIG_HAVE_ARCH_JUMP_LABEL=y -CONFIG_HAVE_ARCH_KGDB=y -CONFIG_HAVE_ARCH_PFN_VALID=y -CONFIG_HAVE_ARCH_SECCOMP_FILTER=y -CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST=y -CONFIG_HAVE_ARCH_TRACEHOOK=y -CONFIG_HAVE_ARM_SCU=y -CONFIG_HAVE_ARM_SMCCC=y -CONFIG_HAVE_ARM_TWD=y -CONFIG_HAVE_CLK=y -CONFIG_HAVE_CLK_PREPARE=y -CONFIG_HAVE_CONTEXT_TRACKING=y -CONFIG_HAVE_COPY_THREAD_TLS=y -CONFIG_HAVE_C_RECORDMCOUNT=y -CONFIG_HAVE_DEBUG_KMEMLEAK=y -CONFIG_HAVE_DMA_CONTIGUOUS=y -CONFIG_HAVE_DYNAMIC_FTRACE=y -CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y -CONFIG_HAVE_EBPF_JIT=y -CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y -CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y -CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y -CONFIG_HAVE_FUNCTION_TRACER=y -CONFIG_HAVE_HW_BREAKPOINT=y -CONFIG_HAVE_IDE=y -CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y -CONFIG_HAVE_LD_DEAD_CODE_DATA_ELIMINATION=y -CONFIG_HAVE_MOD_ARCH_SPECIFIC=y -CONFIG_HAVE_NET_DSA=y -CONFIG_HAVE_OPROFILE=y -CONFIG_HAVE_OPTPROBES=y -CONFIG_HAVE_PATA_PLATFORM=y -CONFIG_HAVE_PCI=y -CONFIG_HAVE_PERF_EVENTS=y -CONFIG_HAVE_PERF_REGS=y -CONFIG_HAVE_PERF_USER_STACK_DUMP=y -CONFIG_HAVE_PROC_CPU=y -CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y -CONFIG_HAVE_RSEQ=y CONFIG_HAVE_SMP=y -CONFIG_HAVE_SYSCALL_TRACEPOINTS=y -CONFIG_HAVE_UID16=y -CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y CONFIG_HDMI=y CONFIG_HID=y CONFIG_HID_GENERIC=y @@ -405,7 +318,6 @@ CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_OF_ARASAN=y # CONFIG_MMC_SDHCI_PCI is not set CONFIG_MMC_SDHCI_PLTFM=y -CONFIG_MODULES_TREE_LOOKUP=y CONFIG_MODULES_USE_ELF_REL=y CONFIG_MODULE_FORCE_UNLOAD=y # CONFIG_MODULE_STRIPPED is not set @@ -499,7 +411,6 @@ CONFIG_PLAT_VERSATILE=y CONFIG_PM=y CONFIG_PMBUS=y CONFIG_PM_CLK=y -# CONFIG_PM_DEBUG is not set CONFIG_PM_SLEEP=y CONFIG_PM_SLEEP_SMP=y CONFIG_POWER_RESET=y @@ -526,6 +437,7 @@ CONFIG_RPS=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_PCF8563=y CONFIG_RTC_I2C_AND_SPI=y +CONFIG_RTC_MC146818_LIB=y CONFIG_RWSEM_SPIN_ON_OWNER=y CONFIG_SCHED_MC=y CONFIG_SCHED_SMT=y @@ -533,7 +445,6 @@ CONFIG_SENSORS_PMBUS=y CONFIG_SENSORS_UCD9000=y CONFIG_SENSORS_UCD9200=y # CONFIG_SERIAL_8250 is not set -# CONFIG_SERIAL_AMBA_PL011 is not set CONFIG_SERIAL_XILINX_PS_UART=y CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y CONFIG_SERIO=y @@ -568,7 +479,6 @@ CONFIG_THERMAL_EMERGENCY_POWEROFF_DELAY_MS=0 CONFIG_THERMAL_GOV_STEP_WISE=y CONFIG_THERMAL_HWMON=y CONFIG_THERMAL_OF=y -# CONFIG_THUMB2_KERNEL is not set CONFIG_TICK_CPU_ACCOUNTING=y CONFIG_TIMER_OF=y CONFIG_TIMER_PROBE=y @@ -586,14 +496,12 @@ CONFIG_UIO_PDRV_GENIRQ=y # CONFIG_UIO_SERCOS3 is not set CONFIG_UNCOMPRESS_INCLUDE="debug/uncompress.h" CONFIG_UNWINDER_ARM=y -# CONFIG_UNWINDER_FRAME_POINTER is not set CONFIG_USB=y CONFIG_USB_CHIPIDEA=y CONFIG_USB_CHIPIDEA_HOST=y CONFIG_USB_CHIPIDEA_OF=y CONFIG_USB_CHIPIDEA_UDC=y CONFIG_USB_COMMON=y -# CONFIG_USB_EHCI_FSL is not set CONFIG_USB_EHCI_HCD=y # CONFIG_USB_EHCI_HCD_PLATFORM is not set # CONFIG_USB_EHCI_TT_NEWSCHED is not set @@ -623,9 +531,6 @@ CONFIG_VT_CONSOLE=y CONFIG_VT_CONSOLE_SLEEP=y # CONFIG_VT_HW_CONSOLE_BINDING is not set CONFIG_WATCHDOG_CORE=y -# CONFIG_WEXT_PRIV is not set -# CONFIG_WEXT_SPY is not set -# CONFIG_WIRELESS_EXT is not set # CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set CONFIG_XILINX_EMACLITE=y # CONFIG_XILINX_PR_DECOUPLER is not set diff --git a/tools/fakeroot/Makefile b/tools/fakeroot/Makefile index 5c5f25e226..f4333fbfb3 100644 --- a/tools/fakeroot/Makefile +++ b/tools/fakeroot/Makefile @@ -19,7 +19,8 @@ include $(INCLUDE_DIR)/host-build.mk HOST_CONFIGURE_VARS += \ ac_cv_header_sys_capability_h=no \ - ac_cv_func_capset=no + ac_cv_func_capset=no \ + CPP="$(HOSTCC) -E" HOST_CONFIGURE_ARGS += \ --with-ipc=tcp